diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index ed082deb4bb4..d01c13218424 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -103,6 +103,11 @@ ozone-manager runtime + + org.apache.ozone + ozone-multitenancy-ranger + runtime + org.apache.ozone ozone-recon diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index ef45ef12566d..9aeb1285178a 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -231,6 +231,7 @@ share/ozone/lib/ozone-insight.jar share/ozone/lib/ozone-interface-client.jar share/ozone/lib/ozone-interface-storage.jar share/ozone/lib/ozone-manager.jar +share/ozone/lib/ozone-multitenancy-ranger.jar share/ozone/lib/ozone-reconcodegen.jar share/ozone/lib/ozone-recon.jar share/ozone/lib/ozone-s3-secret-store.jar diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index 5eda5777c353..bee4225070cc 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@ -2799,6 +2799,33 @@ function ozone_validate_classpath_util fi } +## @description Add items from .classpath file to the classpath +## @audience private +## @stability evolving +## @replaceable no +function ozone_add_classpath_from_file() { + local classpath_file="$1" + + if [[ ! -e "$classpath_file" ]]; then + echo "Skip non-existent classpath file: $classpath_file" >&2 + return + fi + + local classpath + # shellcheck disable=SC1090,SC2086 + source "$classpath_file" + local original_ifs=$IFS + IFS=':' + + local jar + # shellcheck disable=SC2154 + for jar in $classpath; do + ozone_add_classpath "$jar" + done + + IFS=$original_ifs +} + ## @description Add all the required jar files to the classpath ## @audience private ## @stability evolving @@ -2818,15 +2845,7 @@ function ozone_assemble_classpath() { echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing" exit 255 fi - # shellcheck disable=SC1090,SC2086 - source "$CLASSPATH_FILE" - OIFS=$IFS - IFS=':' - - # shellcheck disable=SC2154 - for jar in $classpath; do - ozone_add_classpath "$jar" - done + ozone_add_classpath_from_file "$CLASSPATH_FILE" ozone_add_classpath "${OZONE_HOME}/share/ozone/web" #Add optional jars to the classpath @@ -2835,9 +2854,6 @@ function ozone_assemble_classpath() { if [[ -d "$OPTIONAL_CLASSPATH_DIR" ]]; then ozone_add_classpath "$OPTIONAL_CLASSPATH_DIR/*" fi - - # TODO can be moved earlier? (after 'for jar in $classpath' loop) - IFS=$OIFS } ## @description Fallback to value of `oldvar` if `newvar` is undefined diff --git a/hadoop-ozone/dist/src/shell/shellprofile.d/hadoop-ozone-manager.sh b/hadoop-ozone/dist/src/shell/shellprofile.d/hadoop-ozone-manager.sh index af5d842efdb7..668b2f6d26a8 100644 --- a/hadoop-ozone/dist/src/shell/shellprofile.d/hadoop-ozone-manager.sh +++ b/hadoop-ozone/dist/src/shell/shellprofile.d/hadoop-ozone-manager.sh @@ -19,10 +19,14 @@ if [[ "${OZONE_SHELL_EXECNAME}" == ozone ]]; then fi _ozone_manager_hadoop_finalize() { - if [[ "${OZONE_CLASSNAME}" == "org.apache.hadoop.ozone.om.OzoneManagerStarter" ]] && - [[ -n ${OZONE_MANAGER_CLASSPATH} ]]; - then - echo "Ozone Manager classpath extended by ${OZONE_MANAGER_CLASSPATH}" - ozone_add_to_classpath_userpath "${OZONE_MANAGER_CLASSPATH}" + if [[ "${OZONE_CLASSNAME}" == "org.apache.hadoop.ozone.om.OzoneManagerStarter" ]]; then + if [[ -n ${OZONE_MANAGER_CLASSPATH} ]]; then + echo "Ozone Manager classpath extended by ${OZONE_MANAGER_CLASSPATH}" + ozone_add_to_classpath_userpath "${OZONE_MANAGER_CLASSPATH}" + fi + + if [[ ! "$OZONE_CLASSPATH" =~ "ozone-multitenancy" ]]; then + ozone_add_classpath_from_file "${OZONE_HOME}/share/ozone/classpath/ozone-multitenancy-ranger.classpath" + fi fi } diff --git a/hadoop-ozone/integration-test-recon/pom.xml b/hadoop-ozone/integration-test-recon/pom.xml index 47db1fc0c426..45d8a3ee2486 100644 --- a/hadoop-ozone/integration-test-recon/pom.xml +++ b/hadoop-ozone/integration-test-recon/pom.xml @@ -149,24 +149,12 @@ org.apache.ozone ozone-manager test - - - com.sun.jersey - * - - org.apache.ozone ozone-manager test-jar test - - - com.sun.jersey - * - - org.apache.ozone diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index 22906c611add..d70cea608970 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -435,6 +435,11 @@ ozone-mini-cluster test + + org.apache.ozone + ozone-multitenancy-ranger + test + org.apache.ozone ozone-recon diff --git a/hadoop-ozone/multitenancy-ranger/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/multitenancy-ranger/dev-support/findbugsExcludeFile.xml new file mode 100644 index 000000000000..55abc2630178 --- /dev/null +++ b/hadoop-ozone/multitenancy-ranger/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,19 @@ + + + + diff --git a/hadoop-ozone/multitenancy-ranger/pom.xml b/hadoop-ozone/multitenancy-ranger/pom.xml new file mode 100644 index 000000000000..ad025f9c6e59 --- /dev/null +++ b/hadoop-ozone/multitenancy-ranger/pom.xml @@ -0,0 +1,190 @@ + + + + 4.0.0 + + org.apache.ozone + ozone + 2.1.0-SNAPSHOT + + ozone-multitenancy-ranger + 2.1.0-SNAPSHOT + jar + Apache Ozone Multitenancy with Ranger + Implementation of multitenancy for Apache Ozone Manager Server using Apache Ranger + + + false + + + + + com.sun.jersey + jersey-client + + + org.apache.ranger + ranger-intg + + + org.apache.ranger + ranger-plugins-common + + + + ch.qos.logback + logback-classic + + + com.amazonaws + aws-java-sdk-bundle + + + com.google.cloud.bigdataoss + gcs-connector + + + com.nimbusds + nimbus-jose-jwt + + + com.sun.jersey + jersey-bundle + + + + commons-logging + commons-logging + + + net.minidev + json-smart + + + org.apache.hive + hive-storage-api + + + org.apache.kafka + kafka-clients + + + org.apache.lucene + * + + + org.apache.solr + solr-solrj + + + org.elasticsearch + * + + + org.elasticsearch.client + * + + + org.elasticsearch.plugin + * + + + org.opensearch.client + opensearch-rest-client + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.ozone + hdds-common + provided + + + org.apache.ozone + hdds-config + provided + + + org.apache.ozone + ozone-common + provided + + + org.apache.ozone + ozone-manager + provided + + + org.slf4j + slf4j-api + provided + + + + + org.apache.hadoop + hadoop-auth + test + + + org.apache.hadoop + hadoop-common + test-jar + test + + + org.apache.ozone + hdds-common + test-jar + test + + + org.apache.ozone + hdds-test-utils + test-jar + test + + + org.apache.ozone + ozone-manager + test-jar + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + none + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java b/hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java similarity index 98% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java rename to hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java index b776d7981eb8..936259a2b948 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java +++ b/hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerClientMultiTenantAccessController.java @@ -25,7 +25,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_SERVICE; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import com.google.common.base.Preconditions; import com.sun.jersey.api.client.ClientResponse; import java.io.IOException; import java.util.ArrayList; @@ -34,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.OmUtils; @@ -81,9 +81,9 @@ public RangerClientMultiTenantAccessController(ConfigurationSource conf) // OMMultiTenantManager#checkAndEnableMultiTenancy at this point. String rangerHttpsAddress = conf.get(OZONE_RANGER_HTTPS_ADDRESS_KEY); - Preconditions.checkNotNull(rangerHttpsAddress); + Objects.requireNonNull(rangerHttpsAddress); rangerServiceName = conf.get(OZONE_RANGER_SERVICE); - Preconditions.checkNotNull(rangerServiceName); + Objects.requireNonNull(rangerServiceName); // Determine auth type (KERBEROS or SIMPLE) final String authType; @@ -109,14 +109,14 @@ public RangerClientMultiTenantAccessController(ConfigurationSource conf) authType = AuthenticationMethod.KERBEROS.name(); String configuredOmPrincipal = conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY); - Preconditions.checkNotNull(configuredOmPrincipal); + Objects.requireNonNull(configuredOmPrincipal); // Replace _HOST pattern with host name in the Kerberos principal. // Ranger client currently does not do this automatically. omPrincipal = SecurityUtil.getServerPrincipal( configuredOmPrincipal, OmUtils.getOmAddress(conf).getHostName()); final String keytabPath = conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY); - Preconditions.checkNotNull(keytabPath); + Objects.requireNonNull(keytabPath); // Convert to short name to be used in some Ranger requests shortName = UserGroupInformation.createRemoteUser(omPrincipal) diff --git a/hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/package-info.java b/hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/package-info.java new file mode 100644 index 000000000000..a96d8254e801 --- /dev/null +++ b/hadoop-ozone/multitenancy-ranger/src/main/java/org/apache/hadoop/ozone/om/multitenant/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Ozone Multi tenancy implementation with Apache Ranger. + */ +package org.apache.hadoop.ozone.om.multitenant; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerClientMultiTenantAccessController.java b/hadoop-ozone/multitenancy-ranger/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerClientMultiTenantAccessController.java similarity index 100% rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerClientMultiTenantAccessController.java rename to hadoop-ozone/multitenancy-ranger/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerClientMultiTenantAccessController.java diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 551eca63de27..c9c2f991b5f0 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -46,10 +46,6 @@ com.google.protobuf protobuf-java - - com.sun.jersey - jersey-client - commons-codec commons-codec @@ -181,79 +177,6 @@ org.apache.ozone rocksdb-checkpoint-differ - - org.apache.ranger - ranger-intg - - - org.apache.ranger - ranger-plugins-common - - - - ch.qos.logback - logback-classic - - - com.amazonaws - aws-java-sdk-bundle - - - com.google.cloud.bigdataoss - gcs-connector - - - com.nimbusds - nimbus-jose-jwt - - - com.sun.jersey - jersey-bundle - - - - commons-logging - commons-logging - - - net.minidev - json-smart - - - org.apache.hive - hive-storage-api - - - org.apache.kafka - kafka-clients - - - org.apache.lucene - * - - - org.apache.solr - solr-solrj - - - org.elasticsearch - * - - - org.elasticsearch.client - * - - - org.elasticsearch.plugin - * - - - org.opensearch.client - opensearch-rest-client - - - org.apache.ratis ratis-common diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 33ab6235f291..b7e4e8428611 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -42,6 +42,7 @@ interface-client interface-storage mini-cluster + multitenancy-ranger ozone-manager ozonefs ozonefs-common diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 079022c2f282..2beaa42898d3 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -184,12 +184,6 @@ org.apache.ozone ozone-manager - - - com.sun.jersey - * - - org.apache.ozone diff --git a/pom.xml b/pom.xml index 464edd49f8d1..292d52a257b9 100644 --- a/pom.xml +++ b/pom.xml @@ -1234,6 +1234,11 @@ ozone-mini-cluster ${ozone.version} + + org.apache.ozone + ozone-multitenancy-ranger + ${ozone.version} + org.apache.ozone ozone-recon