HDDS-12454. Create new module for multitenancy with Ranger

hadoop-ozone/dist/pom.xml

@@ -103,6 +103,11 @@
       <artifactId>ozone-manager</artifactId>
       <scope>runtime</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-multitenancy-ranger</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>ozone-recon</artifactId>

hadoop-ozone/dist/src/main/license/jar-report.txt

@@ -231,6 +231,7 @@ share/ozone/lib/ozone-insight.jar
 share/ozone/lib/ozone-interface-client.jar
 share/ozone/lib/ozone-interface-storage.jar
 share/ozone/lib/ozone-manager.jar
+share/ozone/lib/ozone-multitenancy-ranger.jar
 share/ozone/lib/ozone-reconcodegen.jar
 share/ozone/lib/ozone-recon.jar
 share/ozone/lib/ozone-s3-secret-store.jar

hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh

@@ -2799,6 +2799,33 @@ function ozone_validate_classpath_util
   fi
 }
 
+## @description Add items from .classpath file to the classpath
+## @audience private
+## @stability evolving
+## @replaceable no
+function ozone_add_classpath_from_file() {
+  local classpath_file="$1"
+
+  if [[ ! -e "$classpath_file" ]]; then
+    echo "Skip non-existent classpath file: $classpath_file" >&2
+    return
+  fi
+
+  local classpath
+  # shellcheck disable=SC1090,SC2086
+  source "$classpath_file"
+  local original_ifs=$IFS
+  IFS=':'
+
+  local jar
+  # shellcheck disable=SC2154
+  for jar in $classpath; do
+    ozone_add_classpath "$jar"
+  done
+
+  IFS=$original_ifs
+}
+
 ## @description Add all the required jar files to the classpath
 ## @audience private
 ## @stability evolving
@@ -2818,15 +2845,7 @@ function ozone_assemble_classpath() {
     echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing"
     exit 255
   fi
-  # shellcheck disable=SC1090,SC2086
-  source "$CLASSPATH_FILE"
-  OIFS=$IFS
-  IFS=':'
-
-  # shellcheck disable=SC2154
-  for jar in $classpath; do
-    ozone_add_classpath "$jar"
-  done
+  ozone_add_classpath_from_file "$CLASSPATH_FILE"
   ozone_add_classpath "${OZONE_HOME}/share/ozone/web"
 
   #Add optional jars to the classpath
@@ -2835,9 +2854,6 @@ function ozone_assemble_classpath() {
   if [[ -d "$OPTIONAL_CLASSPATH_DIR" ]]; then
     ozone_add_classpath "$OPTIONAL_CLASSPATH_DIR/*"
   fi
-
-  # TODO can be moved earlier? (after 'for jar in $classpath' loop)
-  IFS=$OIFS
 }
 
 ## @description  Fallback to value of `oldvar` if `newvar` is undefined

hadoop-ozone/dist/src/shell/shellprofile.d/hadoop-ozone-manager.sh

@@ -19,10 +19,14 @@ if [[ "${OZONE_SHELL_EXECNAME}" == ozone ]]; then
 fi
 
 _ozone_manager_hadoop_finalize() {
-  if [[ "${OZONE_CLASSNAME}" == "org.apache.hadoop.ozone.om.OzoneManagerStarter" ]] &&
-     [[ -n ${OZONE_MANAGER_CLASSPATH} ]];
-  then
-    echo "Ozone Manager classpath extended by ${OZONE_MANAGER_CLASSPATH}"
-    ozone_add_to_classpath_userpath "${OZONE_MANAGER_CLASSPATH}"
+  if [[ "${OZONE_CLASSNAME}" == "org.apache.hadoop.ozone.om.OzoneManagerStarter" ]]; then
+    if [[ -n ${OZONE_MANAGER_CLASSPATH} ]]; then
+      echo "Ozone Manager classpath extended by ${OZONE_MANAGER_CLASSPATH}"
+      ozone_add_to_classpath_userpath "${OZONE_MANAGER_CLASSPATH}"
+    fi
+
+    if [[ ! "$OZONE_CLASSPATH" =~ "ozone-multitenancy" ]]; then
+      ozone_add_classpath_from_file "${OZONE_HOME}/share/ozone/classpath/ozone-multitenancy-ranger.classpath"
+    fi
   fi
 }

hadoop-ozone/integration-test-recon/pom.xml

@@ -149,24 +149,12 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>ozone-manager</artifactId>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>ozone-manager</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>

hadoop-ozone/integration-test/pom.xml

@@ -435,6 +435,11 @@
       <artifactId>ozone-mini-cluster</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-multitenancy-ranger</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>ozone-recon</artifactId>

hadoop-ozone/multitenancy-ranger/dev-support/findbugsExcludeFile.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<FindBugsFilter>
+</FindBugsFilter>

hadoop-ozone/multitenancy-ranger/pom.xml

@@ -0,0 +1,190 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.ozone</groupId>
+    <artifactId>ozone</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>ozone-multitenancy-ranger</artifactId>
+  <version>2.1.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+  <name>Apache Ozone Multitenancy with Ranger</name>
+  <description>Implementation of multitenancy for Apache Ozone Manager Server using Apache Ranger</description>
+
+  <properties>
+    <classpath.skip>false</classpath.skip>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ranger</groupId>
+      <artifactId>ranger-intg</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ranger</groupId>
+      <artifactId>ranger-plugins-common</artifactId>
+      <!-- Workaround to prevent slf4j binding conflicts until ranger-intg is
+       fixed to not introduce this to the classpath -->
+      <exclusions>
+        <exclusion>
+          <groupId>ch.qos.logback</groupId>
+          <artifactId>logback-classic</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-bundle</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.cloud.bigdataoss</groupId>
+          <artifactId>gcs-connector</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.nimbusds</groupId>
+          <artifactId>nimbus-jose-jwt</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-bundle</artifactId>
+        </exclusion>
+        <exclusion>
+          <!-- depend on jcl-over-slf4j instead -->
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>net.minidev</groupId>
+          <artifactId>json-smart</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-storage-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.kafka</groupId>
+          <artifactId>kafka-clients</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.lucene</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.solr</groupId>
+          <artifactId>solr-solrj</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.elasticsearch</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.elasticsearch.client</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.elasticsearch.plugin</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.opensearch.client</groupId>
+          <artifactId>opensearch-rest-client</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-config</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-manager</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Test dependencies -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-test-utils</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-manager</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <proc>none</proc>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
+        <configuration>
+          <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java → hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java

@@ -25,7 +25,6 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_SERVICE;
 import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 
-import com.google.common.base.Preconditions;
 import com.sun.jersey.api.client.ClientResponse;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -34,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.ozone.OmUtils;
@@ -81,9 +81,9 @@ public RangerClientMultiTenantAccessController(ConfigurationSource conf)
     // OMMultiTenantManager#checkAndEnableMultiTenancy at this point.
 
     String rangerHttpsAddress = conf.get(OZONE_RANGER_HTTPS_ADDRESS_KEY);
-    Preconditions.checkNotNull(rangerHttpsAddress);
+    Objects.requireNonNull(rangerHttpsAddress);
     rangerServiceName = conf.get(OZONE_RANGER_SERVICE);
-    Preconditions.checkNotNull(rangerServiceName);
+    Objects.requireNonNull(rangerServiceName);
 
     // Determine auth type (KERBEROS or SIMPLE)
     final String authType;
@@ -109,14 +109,14 @@ public RangerClientMultiTenantAccessController(ConfigurationSource conf)
       authType = AuthenticationMethod.KERBEROS.name();
 
       String configuredOmPrincipal = conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY);
-      Preconditions.checkNotNull(configuredOmPrincipal);
+      Objects.requireNonNull(configuredOmPrincipal);
 
       // Replace _HOST pattern with host name in the Kerberos principal.
       // Ranger client currently does not do this automatically.
       omPrincipal = SecurityUtil.getServerPrincipal(
           configuredOmPrincipal, OmUtils.getOmAddress(conf).getHostName());
       final String keytabPath = conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY);
-      Preconditions.checkNotNull(keytabPath);
+      Objects.requireNonNull(keytabPath);
 
       // Convert to short name to be used in some Ranger requests
       shortName = UserGroupInformation.createRemoteUser(omPrincipal)