From 0c36871436fda6e14d335236f76759fa1db37a95 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 9 Mar 2025 20:16:11 -0700 Subject: [PATCH 1/6] HDDS-11454. Ozone Ranger Integration in docker --- hadoop-ozone/dist/src/main/compose/ozone/.env | 1 + .../compose/ozone/docker-compose.ranger.yaml | 92 ++++ .../main/compose/ozone/docker-compose.yaml | 12 + .../main/compose/ozone/ranger-plugin-setup.sh | 51 ++ .../ranger-plugin/enable-ozone-plugin.sh | 497 ++++++++++++++++++ .../ozone/ranger-plugin/install.properties | 83 +++ .../compose/ozone/ranger-plugin/ranger-config | 19 + .../ranger-plugin/ranger-plugin-install.sh | 35 ++ .../src/main/compose/ozone/test-ranger.sh | 29 + 9 files changed, 819 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env index 6507664fad7f..1d7b75baaeee 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/.env +++ b/hadoop-ozone/dist/src/main/compose/ozone/.env @@ -19,3 +19,4 @@ HADOOP_IMAGE=apache/hadoop OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_OPTS= +RANGER_VERSION=2.6.0 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml new file mode 100644 index 000000000000..f2ab1d324182 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + om: + volumes: + - ../..:/opt/hadoop + - ./../../share/ranger-${RANGER_VERSION}-ozone-plugin:/opt/hadoop/ranger-ozone-plugin + env_file: + - docker-config + - ranger-plugin/ranger-config + environment: + - ENSURE_OM_INITIALIZED=/data/metadata/om/current/VERSION + - OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1} + - OZONE_MANAGER_CLASSPATH=/opt/hadoop/ranger-ozone-plugin/lib/libext/*:/opt/hadoop/ranger-ozone-plugin/conf + networks: + - ozone + command: bash -c "/opt/hadoop/ranger-ozone-plugin/ranger-plugin-install.sh && /opt/hadoop/bin/ozone om" + ranger: + image: apache/ranger:${RANGER_VERSION} + container_name: ranger + hostname: ranger.example.com + stdin_open: true + tty: true + networks: + - ozone + ports: + - "6080:6080" + depends_on: + ranger-zk: + condition: service_started + ranger-db: + condition: service_healthy + ranger-solr: + condition: service_started + environment: + - RANGER_VERSION=${RANGER_VERSION} + - RANGER_DB_TYPE=postgres + - DEBUG_ADMIN=${DEBUG_ADMIN:-false} + command: + - /home/ranger/scripts/ranger.sh + + ranger-zk: + image: apache/ranger-zk:${RANGER_VERSION} + container_name: ranger-zk + hostname: ranger-zk.example.com + networks: + - ozone + ports: + - "2181:2181" + + ranger-solr: + image: apache/ranger-solr:${RANGER_VERSION} + container_name: ranger-solr + hostname: ranger-solr.example.com + networks: + - ozone + ports: + - "8983:8983" + command: + - solr-precreate + - ranger_audits + - /opt/solr/server/solr/configsets/ranger_audits/ + + ranger-db: + image: apache/ranger-db:${RANGER_VERSION} + container_name: ranger-postgres + hostname: ranger-db.example.com + networks: + - ozone + healthcheck: + test: 'su -c "pg_isready -q" postgres' + interval: 10s + timeout: 2s + retries: 30 + +networks: + ozone: + name: ozone_nw diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index dee24e570021..f3bd2da7fa30 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -36,6 +36,8 @@ services: environment: <<: *replication OZONE_OPTS: + networks: + - ozone command: ["ozone","datanode"] om: <<: *common-config @@ -43,6 +45,8 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: <<: *replication + networks: + - ozone ports: - 9874:9874 - 9862:9862 @@ -57,6 +61,8 @@ services: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} OZONE_OPTS: <<: *replication + networks: + - ozone command: ["ozone","scm"] httpfs: <<: *common-config @@ -65,6 +71,8 @@ services: <<: *replication ports: - 14000:14000 + networks: + - ozone command: [ "ozone","httpfs" ] s3g: <<: *common-config @@ -74,6 +82,8 @@ services: ports: - 9878:9878 - 19878:19878 + networks: + - ozone command: ["ozone","s3g"] recon: <<: *common-config @@ -83,4 +93,6 @@ services: environment: OZONE_OPTS: <<: *replication + networks: + - ozone command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh new file mode 100644 index 000000000000..536c5cc0b2b5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source .env + +downloadIfNotPresent() { + local fileName=$1 + local urlBase=$2 + + if [ ! -f "./../../share/${fileName}" ] + then + echo "downloading ${urlBase}/${fileName}.." + + curl -L ${urlBase}/${fileName} --output ./../../share/${fileName} + else + echo "file already in cache: ${fileName}" + fi +} + +downloadIfNotPresent ranger-${RANGER_VERSION}-ozone-plugin.tar.gz https://dist.apache.org/repos/dist/release/ranger/${RANGER_VERSION}/plugins/ozone + +if [ ! -d ./../../share/ranger-${RANGER_VERSION}-ozone-plugin ] +then + mkdir -p ./../../share/ranger-${RANGER_VERSION}-ozone-plugin + tar xvfz ./../../share/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz -C ./../../share/ranger-${RANGER_VERSION}-ozone-plugin --strip-components 1 +else + echo "ranger-${RANGER_VERSION}-ozone-plugin directory exists already!" +fi + +cp -f ranger-plugin/install.properties ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ +cp -f ranger-plugin/ranger-plugin-install.sh ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ +cp -f ranger-plugin/enable-ozone-plugin.sh ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ + +chmod +x ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ranger-plugin-install.sh +chmod +x ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/enable-ozone-plugin.sh + +echo "copied files successfully!" diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh new file mode 100644 index 000000000000..ac78458a340a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh @@ -0,0 +1,497 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function getInstallProperty() { + local propertyName=$1 + local propertyValue="" + + for file in "${COMPONENT_INSTALL_ARGS}" "${INSTALL_ARGS}" + do + if [ -f "${file}" ] + then + propertyValue=`grep "^${propertyName}[ \t]*=" ${file} | awk -F= '{ sub("^[ \t]*", "", $2); sub("[ \t]*$", "", $2); print $2 }'` + if [ "${propertyValue}" != "" ] + then + break + fi + fi + done + + echo ${propertyValue} +} + +# +# Base env variable for Ranger related files/directories +# +PROJ_NAME=ranger + +# +# The script should be run by "root" user +# + +if [ ! -w /etc/passwd ] +then + echo "ERROR: $0 script should be run as root." + exit 1 +fi + +#Check for JAVA_HOME +if [ "${JAVA_HOME}" == "" ] +then + echo "ERROR: JAVA_HOME environment property not defined, aborting installation." + exit 1 +fi + +# +# Identify the component, action from the script file +# + +basedir=`dirname $0` +if [ "${basedir}" = "." ] +then + basedir=`pwd` +elif [ "${basedir}" = ".." ] +then + basedir=`(cd .. ;pwd)` +fi + +# +# As this script is common to all component, find the component name based on the script-name +# + +COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^disable-::' | sed -e 's:^enable-::'` + +echo "${COMPONENT_NAME}" | grep 'plugin' > /dev/null 2>&1 + +if [ $? -ne 0 ] +then + echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." + exit 0 +fi + +HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` + +CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" + +if [ "${HCOMPONENT_NAME}" = "hdfs" ] +then + HCOMPONENT_NAME="hadoop" +fi + +# +# Based on script name, identify if the action is enabled or disabled +# + +basename $0 | cut -d. -f1 | grep '^enable-' > /dev/null 2>&1 + +if [ $? -eq 0 ] +then + action=enable +else + action=disable +fi + + +# +# environment variables for enable|disable scripts +# + +PROJ_INSTALL_DIR=`(cd ${basedir} ; pwd)` +SET_ENV_SCRIPT_NAME=set-${COMPONENT_NAME}-env.sh +SET_ENV_SCRIPT_TEMPLATE=${PROJ_INSTALL_DIR}/install/conf.templates/enable/${SET_ENV_SCRIPT_NAME} +DEFAULT_XML_CONFIG=${PROJ_INSTALL_DIR}/install/conf.templates/default/configuration.xml +PROJ_LIB_DIR=${PROJ_INSTALL_DIR}/lib +PROJ_INSTALL_LIB_DIR="${PROJ_INSTALL_DIR}/install/lib" +INSTALL_ARGS="${PROJ_INSTALL_DIR}/install.properties" +COMPONENT_INSTALL_ARGS="${PROJ_INSTALL_DIR}/${COMPONENT_NAME}-install.properties" +JAVA=$JAVA_HOME/bin/java + +PLUGIN_DEPENDENT_LIB_DIR=lib/"${PROJ_NAME}-${COMPONENT_NAME}-impl" +PROJ_LIB_PLUGIN_DIR=${PROJ_INSTALL_DIR}/${PLUGIN_DEPENDENT_LIB_DIR} + +HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'COMPONENT_INSTALL_DIR_NAME') + +CUSTOM_USER=$(getInstallProperty 'CUSTOM_USER') +CUSTOM_USER=${CUSTOM_USER// } + +CUSTOM_GROUP=$(getInstallProperty 'CUSTOM_GROUP') +CUSTOM_GROUP=${CUSTOM_GROUP// } + +CUSTOM_GROUP_STATUS=${CUSTOM_GROUP}; +CUSTOM_USER_STATUS=${CUSTOM_USER}; +egrep "^$CUSTOM_GROUP" /etc/group >& /dev/null +if [ $? -ne 0 ] +then + CUSTOM_GROUP_STATUS="" +fi +id -u ${CUSTOM_USER} > /dev/null 2>&1 +if [ $? -ne 0 ] +then + CUSTOM_USER_STATUS="" +fi + +if [ ! -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] +then + echo "Custom user and group is available, using custom user and group." + CFG_OWNER_INF="${CUSTOM_USER}:${CUSTOM_GROUP}" +elif [ ! -z "${CUSTOM_USER_STATUS}" ] && [ -z "${CUSTOM_GROUP_STATUS}" ] +then + echo "Custom user is available, using custom user and default group." + CFG_OWNER_INF="${CUSTOM_USER}:${HCOMPONENT_NAME}" +elif [ -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] +then + echo "Custom group is available, using default user and custom group." + CFG_OWNER_INF="${HCOMPONENT_NAME}:${CUSTOM_GROUP}" +else + echo "Custom user and group are not available, using default user and group." + CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" +fi + +if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] +then + HCOMPONENT_INSTALL_DIR_NAME=${HCOMPONENT_NAME} +fi + +firstletter=${HCOMPONENT_INSTALL_DIR_NAME:0:1} +if [ "$firstletter" = "/" ]; then + hdir=${HCOMPONENT_INSTALL_DIR_NAME} +else + hdir=${PROJ_INSTALL_DIR}/../${HCOMPONENT_INSTALL_DIR_NAME} +fi + +# +# TEST - START +# +if [ ! -d ${hdir} ] +then + mkdir -p ${hdir} +fi +# +# TEST - END +# +HCOMPONENT_INSTALL_DIR=`(cd ${hdir} ; pwd)` +HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/lib +if [ "${HCOMPONENT_NAME}" = "hadoop" ] || + [ "${HCOMPONENT_NAME}" = "yarn" ]; then + HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib +fi + +HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf +HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive +SET_ENV_SCRIPT=${HCOMPONENT_CONF_DIR}/${SET_ENV_SCRIPT_NAME} + + +if [ ! -d "${HCOMPONENT_INSTALL_DIR}" ] +then + echo "ERROR: Unable to find the install directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_INSTALL_DIR}] not found." + echo "Exiting installation." + exit 1 +fi + +if [ ! -d "${HCOMPONENT_CONF_DIR}" ] +then + echo "ERROR: Unable to find the conf directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_CONF_DIR}] not found." + echo "Exiting installation." + exit 1 +fi + +if [ ! -d "${HCOMPONENT_LIB_DIR}" ] +then + mkdir -p "${HCOMPONENT_LIB_DIR}" + if [ ! -d "${HCOMPONENT_LIB_DIR}" ] + then + echo "ERROR: Unable to find the lib directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_LIB_DIR}] not found." + echo "Exiting installation." + exit 1 + fi +fi + +# +# Common functions used by all enable/disable scripts +# + +log() { + echo "+ `date` : $*" +} + + +create_jceks() { + + alias=$1 + pass=$2 + jceksFile=$3 + + if [ -f "${jceksFile}" ] + then + jcebdir=`dirname ${jceksFile}` + jcebname=`basename ${jceksFile}` + archive_jce=${jcebdir}/.${jcebname}.`date '+%Y%m%d%H%M%S'` + log "Saving current JCE file: ${jceksFile} to ${archive_jce} ..." + cp ${jceksFile} ${archive_jce} + fi + + tempFile=/tmp/jce.$$.out + + $JAVA_HOME/bin/java -cp ":${PROJ_INSTALL_LIB_DIR}/*:" org.apache.ranger.credentialapi.buildks create "${alias}" -value "${pass}" -provider "jceks://file${jceksFile}" > ${tempFile} 2>&1 + + if [ $? -ne 0 ] + then + echo "Unable to store password in non-plain text format. Error: [`cat ${tempFile}`]" + echo "Exiting plugin installation" + rm -f ${tempFile} + exit 0 + fi + + rm -f ${tempFile} +} + +log "${HCOMPONENT_NAME}: lib folder=$HCOMPONENT_LIB_DIR conf folder=$HCOMPONENT_CONF_DIR" + +# +# If there is a set-ranger-${COMPONENT}-env.sh, install it +# +dt=`date '+%Y%m%d-%H%M%S'` + +if [ -f "${SET_ENV_SCRIPT_TEMPLATE}" ] +then + # + # If the setenv script already exists, move it to the archive folder + # + if [ -f "${SET_ENV_SCRIPT}" ] + then + if [ ! -d "${HCOMPONENT_ARCHIVE_CONF_DIR}" ] + then + mkdir -p ${HCOMPONENT_ARCHIVE_CONF_DIR} + fi + log "Saving current ${SET_ENV_SCRIPT_NAME} to ${HCOMPONENT_ARCHIVE_CONF_DIR} ..." + mv ${SET_ENV_SCRIPT} ${HCOMPONENT_ARCHIVE_CONF_DIR}/${SET_ENV_SCRIPT_NAME}.${dt} + fi + + if [ "${action}" = "enable" ] + then + + cp ${SET_ENV_SCRIPT_TEMPLATE} ${SET_ENV_SCRIPT} + + DEST_SCRIPT_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/${HCOMPONENT_NAME}-config.sh + + DEST_SCRIPT_ARCHIVE_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/.${HCOMPONENT_NAME}-config.sh.${dt} + + if [ -f "${DEST_SCRIPT_FILE}" ] + then + + log "Saving current ${DEST_SCRIPT_FILE} to ${DEST_SCRIPT_ARCHIVE_FILE} ..." + + cp ${DEST_SCRIPT_FILE} ${DEST_SCRIPT_ARCHIVE_FILE} + + grep 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > /dev/null 2>&1 + if [ $? -eq 0 ] + then + ts=`date '+%Y%m%d%H%M%S'` + grep -v 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > ${DEST_SCRIPT_FILE}.${ts} + if [ $? -eq 0 ] + then + log "Removing old reference to xasecure setenv source ..." + cat ${DEST_SCRIPT_FILE}.${ts} > ${DEST_SCRIPT_FILE} + rm -f ${DEST_SCRIPT_FILE}.${ts} + fi + fi + + grep "[ \t]*.[ \t]*${SET_ENV_SCRIPT}" ${DEST_SCRIPT_FILE} > /dev/null + if [ $? -ne 0 ] + then + log "Appending sourcing script, ${SET_ENV_SCRIPT_NAME} in the file: ${DEST_SCRIPT_FILE} " + cat >> ${DEST_SCRIPT_FILE} <\n`date`\n" > ${HCOMPONENT_CONF_DIR}/ranger-security.xml + chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/ranger-security.xml + chmod a+r ${HCOMPONENT_CONF_DIR}/ranger-security.xml + for cf in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.xml + do + cfb=`basename ${cf}` + if [ -f "${HCOMPONENT_CONF_DIR}/${cfb}" ] + then + log "Saving ${HCOMPONENT_CONF_DIR}/${cfb} to ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} ..." + cp ${HCOMPONENT_CONF_DIR}/${cfb} ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} + fi + cp ${cf} ${HCOMPONENT_CONF_DIR}/ + chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/${cfb} + chmod a+r ${HCOMPONENT_CONF_DIR}/${cfb} + done + else + if [ -f ${HCOMPONENT_CONF_DIR}/ranger-security.xml ] + then + mv ${HCOMPONENT_CONF_DIR}/ranger-security.xml ${HCOMPONENT_CONF_DIR}/.ranger-security.xml.`date '+%Y%m%d%H%M%S'` + fi + fi + + # + # Ensure that POLICY_CACHE_FILE_PATH is accessible + # + REPO_NAME=$(getInstallProperty 'REPOSITORY_NAME') + export POLICY_CACHE_FILE_PATH=/etc/${PROJ_NAME}/${REPO_NAME}/policycache + export CREDENTIAL_PROVIDER_FILE=/etc/${PROJ_NAME}/${REPO_NAME}/cred.jceks + if [ ! -d ${POLICY_CACHE_FILE_PATH} ] + then + mkdir -p ${POLICY_CACHE_FILE_PATH} + fi + chmod a+rx /etc/${PROJ_NAME} + chmod a+rx /etc/${PROJ_NAME}/${REPO_NAME} + chmod a+rx ${POLICY_CACHE_FILE_PATH} + chown -R ${CFG_OWNER_INF} /etc/${PROJ_NAME}/${REPO_NAME} + + for f in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.cfg + do + if [ -f "${f}" ] + then + fn=`basename $f` + orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` + fullpathorgfn="${HCOMPONENT_CONF_DIR}/${orgfn}" + if [ ! -f ${fullpathorgfn} ] + then + if [ -f ${DEFAULT_XML_CONFIG} ] + then + log "Creating default file from [${DEFAULT_XML_CONFIG}] for [${fullpathorgfn}] .." + cp ${DEFAULT_XML_CONFIG} ${fullpathorgfn} + chown ${CFG_OWNER_INF} ${fullpathorgfn} + chmod a+r ${fullpathorgfn} + else + echo "ERROR: Unable to find ${fullpathorgfn}" + exit 1 + fi + fi + archivefn="${HCOMPONENT_CONF_DIR}/.${orgfn}.${dt}" + newfn="${HCOMPONENT_CONF_DIR}/.${orgfn}-new.${dt}" + log "Saving current config file: ${fullpathorgfn} to ${archivefn} ..." + cp ${fullpathorgfn} ${archivefn} + if [ $? -eq 0 ] + then + ${JAVA} -cp "${INSTALL_CP}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS} + if [ $? -eq 0 ] + then + diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + cat ${newfn} > ${fullpathorgfn} + fi + + else + echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" + echo "exiting ...." + exit 1 + fi + else + echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" + echo "exiting ...." + exit 1 + fi + fi + done +fi + +# +# Create library link +# +if [ "${action}" = "enable" ] +then + dt=`date '+%Y%m%d%H%M%S'` + # + # Encrypt the password and keep it secure in Credential Provider API + # + CredFile=${CREDENTIAL_PROVIDER_FILE} + if ! [ `echo ${CredFile} | grep '^/.*'` ] + then + echo "ERROR:Please enter the Credential File Store with proper file path" + exit 1 + fi + + pardir=`dirname ${CredFile}` + + if [ ! -d "${pardir}" ] + then + mkdir -p "${pardir}" + if [ $? -ne 0 ] + then + echo "ERROR: Unable to create credential store file path" + exit 1 + fi + chmod a+rx "${pardir}" + fi +fi + + +#Check Properties whether in File, return code 1 if not exist +#$1 -> propertyName; $2 -> fileName +checkPropertyInFile(){ + validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f1-) # for validation + if test -z "$validate" ; then return 1; fi +} + +#Add Properties to File +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName +addPropertyToFile(){ + echo "$1=$2">>$3 + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + if test -z "$validate" ; then log "[E] Failed to add properties '$1' to $3 file!"; exit 1; fi + echo "Property $1 added successfully with : '$2'" +} + +#Update Properties to File +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName +updatePropertyToFile(){ + sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + if test -z "$validate" ; then log "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi + echo "Property $1 updated successfully with : '$2'" +} + +#Add or Update Properties to File +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName +addOrUpdatePropertyToFile(){ + checkPropertyInFile $1 $3 + if [ $? -eq 1 ] + then + addPropertyToFile $1 $2 $3 + else + updatePropertyToFile $1 $2 $3 + fi +} + + +# Set notice to restart the ${HCOMPONENT_NAME} +echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." + +exit 0 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties new file mode 100644 index 000000000000..a744a1ac5b25 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +POLICY_MGR_URL=http://ranger:6080 +REPOSITORY_NAME=dev_ozone +COMPONENT_INSTALL_DIR_NAME=ranger-ozone-plugin + +CUSTOM_USER=hadoop +CUSTOM_GROUP=hadoop + +XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits + +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/audit/archive +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +XAAUDIT.SUMMARY.ENABLE=true + +XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/audit/solr/spool + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.HDFS.ENABLE=false +XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/audit/hdfs/spool + +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config new file mode 100644 index 000000000000..86bd016039c5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To enable Ranger as the Authorizer in Ozone +OZONE-SITE.XML_ozone.acl.enabled=true +OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh new file mode 100644 index 000000000000..afc0f819bd4b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE_HOME=/opt/hadoop +cd "${OZONE_HOME}"/ranger-ozone-plugin || exit + +if [[ ! -f "${OZONE_HOME}"/.setupDone ]]; +then + if [ ! -d conf ]; then + mkdir -p conf + echo "conf directory created!" + else + echo "conf directory exists already!" + fi + echo "export JAVA_HOME=${JAVA_HOME}" >> conf/ozone-env.sh + sudo JAVA_HOME=/usr/lib/jvm/jre/ ./enable-ozone-plugin.sh + touch "${OZONE_HOME}"/.setupDone +else + echo "Ranger Ozone Plugin Installation is already complete!" +fi diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh new file mode 100644 index 000000000000..dfdff11f05cb --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:misc + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +export SECURITY_ENABLED=true +export COMPOSE_FILE=docker-compose.yaml:docker-compose.ranger.yaml + +start_docker_env From 99dca44d39b7e37b103e838eaa0d93b4f94ed132 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 9 Mar 2025 23:51:20 -0700 Subject: [PATCH 2/6] fix acceptance tests in compose/ozone --- hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml | 4 ++++ hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index f3bd2da7fa30..992d9e03cf7a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -96,3 +96,7 @@ services: networks: - ozone command: ["ozone","recon"] + +networks: + ozone: + name: ozone_nw diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh index dfdff11f05cb..fb061bf5e5cc 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh @@ -23,7 +23,7 @@ export COMPOSE_DIR # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -export SECURITY_ENABLED=true +export SECURITY_ENABLED=false export COMPOSE_FILE=docker-compose.yaml:docker-compose.ranger.yaml start_docker_env From a20fe39bc263bab4627c4b1fb5e6179e4e6bec85 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Mar 2025 12:20:27 -0700 Subject: [PATCH 3/6] address review comments --- .../dist/src/main/compose/ozone/ranger-plugin-setup.sh | 4 ++-- hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh index 536c5cc0b2b5..758658690bd9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh @@ -25,13 +25,13 @@ downloadIfNotPresent() { then echo "downloading ${urlBase}/${fileName}.." - curl -L ${urlBase}/${fileName} --output ./../../share/${fileName} + curl -L "${urlBase}/${fileName}" --output ./../../share/${fileName} else echo "file already in cache: ${fileName}" fi } -downloadIfNotPresent ranger-${RANGER_VERSION}-ozone-plugin.tar.gz https://dist.apache.org/repos/dist/release/ranger/${RANGER_VERSION}/plugins/ozone +downloadIfNotPresent ranger-${RANGER_VERSION}-ozone-plugin.tar.gz "https://www.apache.org/dyn/closer.lua?action=download&filename=ranger/${RANGER_VERSION}/plugins/ozone" if [ ! -d ./../../share/ranger-${RANGER_VERSION}-ozone-plugin ] then diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh index fb061bf5e5cc..8a7440ba1518 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:misc +#suite:failing COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR From c74be19a2144bfe0ef240edafc8c0d48e60c9e60 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Mar 2025 12:27:03 -0700 Subject: [PATCH 4/6] update network bridge to exclude _ character --- .../dist/src/main/compose/ozone/docker-compose.ranger.yaml | 2 +- hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml index f2ab1d324182..86a19210b17c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml @@ -89,4 +89,4 @@ services: networks: ozone: - name: ozone_nw + name: ozonenw diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 992d9e03cf7a..b8cae2a4cb03 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -99,4 +99,4 @@ services: networks: ozone: - name: ozone_nw + name: ozonenw From 37165681131cd0f10d263487b616fa3b512cc3b2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Mar 2025 14:46:48 -0700 Subject: [PATCH 5/6] remove custom network name and use default network --- .../compose/ozone/docker-compose.ranger.yaml | 13 ------------- .../src/main/compose/ozone/docker-compose.yaml | 16 ---------------- 2 files changed, 29 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml index 86a19210b17c..9eb095f3d482 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml @@ -26,8 +26,6 @@ services: - ENSURE_OM_INITIALIZED=/data/metadata/om/current/VERSION - OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1} - OZONE_MANAGER_CLASSPATH=/opt/hadoop/ranger-ozone-plugin/lib/libext/*:/opt/hadoop/ranger-ozone-plugin/conf - networks: - - ozone command: bash -c "/opt/hadoop/ranger-ozone-plugin/ranger-plugin-install.sh && /opt/hadoop/bin/ozone om" ranger: image: apache/ranger:${RANGER_VERSION} @@ -35,8 +33,6 @@ services: hostname: ranger.example.com stdin_open: true tty: true - networks: - - ozone ports: - "6080:6080" depends_on: @@ -57,8 +53,6 @@ services: image: apache/ranger-zk:${RANGER_VERSION} container_name: ranger-zk hostname: ranger-zk.example.com - networks: - - ozone ports: - "2181:2181" @@ -66,8 +60,6 @@ services: image: apache/ranger-solr:${RANGER_VERSION} container_name: ranger-solr hostname: ranger-solr.example.com - networks: - - ozone ports: - "8983:8983" command: @@ -79,14 +71,9 @@ services: image: apache/ranger-db:${RANGER_VERSION} container_name: ranger-postgres hostname: ranger-db.example.com - networks: - - ozone healthcheck: test: 'su -c "pg_isready -q" postgres' interval: 10s timeout: 2s retries: 30 -networks: - ozone: - name: ozonenw diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index b8cae2a4cb03..dee24e570021 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -36,8 +36,6 @@ services: environment: <<: *replication OZONE_OPTS: - networks: - - ozone command: ["ozone","datanode"] om: <<: *common-config @@ -45,8 +43,6 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: <<: *replication - networks: - - ozone ports: - 9874:9874 - 9862:9862 @@ -61,8 +57,6 @@ services: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} OZONE_OPTS: <<: *replication - networks: - - ozone command: ["ozone","scm"] httpfs: <<: *common-config @@ -71,8 +65,6 @@ services: <<: *replication ports: - 14000:14000 - networks: - - ozone command: [ "ozone","httpfs" ] s3g: <<: *common-config @@ -82,8 +74,6 @@ services: ports: - 9878:9878 - 19878:19878 - networks: - - ozone command: ["ozone","s3g"] recon: <<: *common-config @@ -93,10 +83,4 @@ services: environment: OZONE_OPTS: <<: *replication - networks: - - ozone command: ["ozone","recon"] - -networks: - ozone: - name: ozonenw From b63227a9fcc946708110d69685ec4db490d5a06b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 7 Jun 2025 16:57:22 -0700 Subject: [PATCH 6/6] improvise using PR #8575 --- hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +- .../compose/ozone/docker-compose.ranger.yaml | 39 +- .../main/compose/ozone/ranger-plugin-setup.sh | 51 -- .../ranger-plugin/enable-ozone-plugin.sh | 497 ------------------ .../ozone/ranger-plugin/install.properties | 83 --- .../compose/ozone/ranger-plugin/ranger-config | 19 - .../ranger-plugin/ranger-plugin-install.sh | 35 -- .../src/main/compose/ozone/test-ranger.sh | 49 ++ 8 files changed, 74 insertions(+), 701 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env index 1d7b75baaeee..dcc2240cb5e6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/.env +++ b/hadoop-ozone/dist/src/main/compose/ozone/.env @@ -19,4 +19,4 @@ HADOOP_IMAGE=apache/hadoop OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_OPTS= -RANGER_VERSION=2.6.0 +RANGER_VERSION=${ranger.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml index 9eb095f3d482..189139f09510 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.ranger.yaml @@ -14,23 +14,32 @@ # See the License for the specific language governing permissions and # limitations under the License. +x-om-ranger-config: + &om-ranger-config + environment: + OZONE_MANAGER_CLASSPATH: "/opt/ranger/ozone-plugin/lib/libext/*:/opt/ozone/conf" + OZONE-SITE.XML_ozone.acl.enabled: "true" + OZONE-SITE.XML_ozone.acl.authorizer.class: "org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer" + OZONE-SITE.XML_ozone.om.ranger.https-address: "http://ranger:6080" + OZONE-SITE.XML_ozone.om.ranger.https.admin.api.user: "admin" + OZONE-SITE.XML_ozone.om.ranger.https.admin.api.passwd: "rangerR0cks!" + OZONE-SITE.XML_ozone.om.ranger.service: "dev_ozone" + OZONE-SITE.XML_ozone.om.tenant.dev.skip.ranger: "false" + volumes: + - ../..:/opt/hadoop + - ${RANGER_OZONE_PLUGIN_DIR}:/opt/ranger/ozone-plugin + tmpfs: + - /opt/ozone/conf + command: bash -c "sudo --preserve-env /opt/ranger/ozone-plugin/enable-ozone-plugin.sh && /opt/hadoop/bin/ozone om" + services: om: - volumes: - - ../..:/opt/hadoop - - ./../../share/ranger-${RANGER_VERSION}-ozone-plugin:/opt/hadoop/ranger-ozone-plugin - env_file: - - docker-config - - ranger-plugin/ranger-config - environment: - - ENSURE_OM_INITIALIZED=/data/metadata/om/current/VERSION - - OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1} - - OZONE_MANAGER_CLASSPATH=/opt/hadoop/ranger-ozone-plugin/lib/libext/*:/opt/hadoop/ranger-ozone-plugin/conf - command: bash -c "/opt/hadoop/ranger-ozone-plugin/ranger-plugin-install.sh && /opt/hadoop/bin/ozone om" + <<: *om-ranger-config + ranger: image: apache/ranger:${RANGER_VERSION} container_name: ranger - hostname: ranger.example.com + hostname: ranger stdin_open: true tty: true ports: @@ -52,14 +61,14 @@ services: ranger-zk: image: apache/ranger-zk:${RANGER_VERSION} container_name: ranger-zk - hostname: ranger-zk.example.com + hostname: ranger-zk ports: - "2181:2181" ranger-solr: image: apache/ranger-solr:${RANGER_VERSION} container_name: ranger-solr - hostname: ranger-solr.example.com + hostname: ranger-solr ports: - "8983:8983" command: @@ -70,7 +79,7 @@ services: ranger-db: image: apache/ranger-db:${RANGER_VERSION} container_name: ranger-postgres - hostname: ranger-db.example.com + hostname: ranger-db healthcheck: test: 'su -c "pg_isready -q" postgres' interval: 10s diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh deleted file mode 100644 index 758658690bd9..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin-setup.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source .env - -downloadIfNotPresent() { - local fileName=$1 - local urlBase=$2 - - if [ ! -f "./../../share/${fileName}" ] - then - echo "downloading ${urlBase}/${fileName}.." - - curl -L "${urlBase}/${fileName}" --output ./../../share/${fileName} - else - echo "file already in cache: ${fileName}" - fi -} - -downloadIfNotPresent ranger-${RANGER_VERSION}-ozone-plugin.tar.gz "https://www.apache.org/dyn/closer.lua?action=download&filename=ranger/${RANGER_VERSION}/plugins/ozone" - -if [ ! -d ./../../share/ranger-${RANGER_VERSION}-ozone-plugin ] -then - mkdir -p ./../../share/ranger-${RANGER_VERSION}-ozone-plugin - tar xvfz ./../../share/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz -C ./../../share/ranger-${RANGER_VERSION}-ozone-plugin --strip-components 1 -else - echo "ranger-${RANGER_VERSION}-ozone-plugin directory exists already!" -fi - -cp -f ranger-plugin/install.properties ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ -cp -f ranger-plugin/ranger-plugin-install.sh ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ -cp -f ranger-plugin/enable-ozone-plugin.sh ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ - -chmod +x ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/ranger-plugin-install.sh -chmod +x ./../../share/ranger-${RANGER_VERSION}-ozone-plugin/enable-ozone-plugin.sh - -echo "copied files successfully!" diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh deleted file mode 100644 index ac78458a340a..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/enable-ozone-plugin.sh +++ /dev/null @@ -1,497 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function getInstallProperty() { - local propertyName=$1 - local propertyValue="" - - for file in "${COMPONENT_INSTALL_ARGS}" "${INSTALL_ARGS}" - do - if [ -f "${file}" ] - then - propertyValue=`grep "^${propertyName}[ \t]*=" ${file} | awk -F= '{ sub("^[ \t]*", "", $2); sub("[ \t]*$", "", $2); print $2 }'` - if [ "${propertyValue}" != "" ] - then - break - fi - fi - done - - echo ${propertyValue} -} - -# -# Base env variable for Ranger related files/directories -# -PROJ_NAME=ranger - -# -# The script should be run by "root" user -# - -if [ ! -w /etc/passwd ] -then - echo "ERROR: $0 script should be run as root." - exit 1 -fi - -#Check for JAVA_HOME -if [ "${JAVA_HOME}" == "" ] -then - echo "ERROR: JAVA_HOME environment property not defined, aborting installation." - exit 1 -fi - -# -# Identify the component, action from the script file -# - -basedir=`dirname $0` -if [ "${basedir}" = "." ] -then - basedir=`pwd` -elif [ "${basedir}" = ".." ] -then - basedir=`(cd .. ;pwd)` -fi - -# -# As this script is common to all component, find the component name based on the script-name -# - -COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^disable-::' | sed -e 's:^enable-::'` - -echo "${COMPONENT_NAME}" | grep 'plugin' > /dev/null 2>&1 - -if [ $? -ne 0 ] -then - echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." - exit 0 -fi - -HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` - -CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" - -if [ "${HCOMPONENT_NAME}" = "hdfs" ] -then - HCOMPONENT_NAME="hadoop" -fi - -# -# Based on script name, identify if the action is enabled or disabled -# - -basename $0 | cut -d. -f1 | grep '^enable-' > /dev/null 2>&1 - -if [ $? -eq 0 ] -then - action=enable -else - action=disable -fi - - -# -# environment variables for enable|disable scripts -# - -PROJ_INSTALL_DIR=`(cd ${basedir} ; pwd)` -SET_ENV_SCRIPT_NAME=set-${COMPONENT_NAME}-env.sh -SET_ENV_SCRIPT_TEMPLATE=${PROJ_INSTALL_DIR}/install/conf.templates/enable/${SET_ENV_SCRIPT_NAME} -DEFAULT_XML_CONFIG=${PROJ_INSTALL_DIR}/install/conf.templates/default/configuration.xml -PROJ_LIB_DIR=${PROJ_INSTALL_DIR}/lib -PROJ_INSTALL_LIB_DIR="${PROJ_INSTALL_DIR}/install/lib" -INSTALL_ARGS="${PROJ_INSTALL_DIR}/install.properties" -COMPONENT_INSTALL_ARGS="${PROJ_INSTALL_DIR}/${COMPONENT_NAME}-install.properties" -JAVA=$JAVA_HOME/bin/java - -PLUGIN_DEPENDENT_LIB_DIR=lib/"${PROJ_NAME}-${COMPONENT_NAME}-impl" -PROJ_LIB_PLUGIN_DIR=${PROJ_INSTALL_DIR}/${PLUGIN_DEPENDENT_LIB_DIR} - -HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'COMPONENT_INSTALL_DIR_NAME') - -CUSTOM_USER=$(getInstallProperty 'CUSTOM_USER') -CUSTOM_USER=${CUSTOM_USER// } - -CUSTOM_GROUP=$(getInstallProperty 'CUSTOM_GROUP') -CUSTOM_GROUP=${CUSTOM_GROUP// } - -CUSTOM_GROUP_STATUS=${CUSTOM_GROUP}; -CUSTOM_USER_STATUS=${CUSTOM_USER}; -egrep "^$CUSTOM_GROUP" /etc/group >& /dev/null -if [ $? -ne 0 ] -then - CUSTOM_GROUP_STATUS="" -fi -id -u ${CUSTOM_USER} > /dev/null 2>&1 -if [ $? -ne 0 ] -then - CUSTOM_USER_STATUS="" -fi - -if [ ! -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user and group is available, using custom user and group." - CFG_OWNER_INF="${CUSTOM_USER}:${CUSTOM_GROUP}" -elif [ ! -z "${CUSTOM_USER_STATUS}" ] && [ -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user is available, using custom user and default group." - CFG_OWNER_INF="${CUSTOM_USER}:${HCOMPONENT_NAME}" -elif [ -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom group is available, using default user and custom group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${CUSTOM_GROUP}" -else - echo "Custom user and group are not available, using default user and group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" -fi - -if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] -then - HCOMPONENT_INSTALL_DIR_NAME=${HCOMPONENT_NAME} -fi - -firstletter=${HCOMPONENT_INSTALL_DIR_NAME:0:1} -if [ "$firstletter" = "/" ]; then - hdir=${HCOMPONENT_INSTALL_DIR_NAME} -else - hdir=${PROJ_INSTALL_DIR}/../${HCOMPONENT_INSTALL_DIR_NAME} -fi - -# -# TEST - START -# -if [ ! -d ${hdir} ] -then - mkdir -p ${hdir} -fi -# -# TEST - END -# -HCOMPONENT_INSTALL_DIR=`(cd ${hdir} ; pwd)` -HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/lib -if [ "${HCOMPONENT_NAME}" = "hadoop" ] || - [ "${HCOMPONENT_NAME}" = "yarn" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib -fi - -HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf -HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive -SET_ENV_SCRIPT=${HCOMPONENT_CONF_DIR}/${SET_ENV_SCRIPT_NAME} - - -if [ ! -d "${HCOMPONENT_INSTALL_DIR}" ] -then - echo "ERROR: Unable to find the install directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_INSTALL_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_CONF_DIR}" ] -then - echo "ERROR: Unable to find the conf directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_CONF_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_LIB_DIR}" ] -then - mkdir -p "${HCOMPONENT_LIB_DIR}" - if [ ! -d "${HCOMPONENT_LIB_DIR}" ] - then - echo "ERROR: Unable to find the lib directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_LIB_DIR}] not found." - echo "Exiting installation." - exit 1 - fi -fi - -# -# Common functions used by all enable/disable scripts -# - -log() { - echo "+ `date` : $*" -} - - -create_jceks() { - - alias=$1 - pass=$2 - jceksFile=$3 - - if [ -f "${jceksFile}" ] - then - jcebdir=`dirname ${jceksFile}` - jcebname=`basename ${jceksFile}` - archive_jce=${jcebdir}/.${jcebname}.`date '+%Y%m%d%H%M%S'` - log "Saving current JCE file: ${jceksFile} to ${archive_jce} ..." - cp ${jceksFile} ${archive_jce} - fi - - tempFile=/tmp/jce.$$.out - - $JAVA_HOME/bin/java -cp ":${PROJ_INSTALL_LIB_DIR}/*:" org.apache.ranger.credentialapi.buildks create "${alias}" -value "${pass}" -provider "jceks://file${jceksFile}" > ${tempFile} 2>&1 - - if [ $? -ne 0 ] - then - echo "Unable to store password in non-plain text format. Error: [`cat ${tempFile}`]" - echo "Exiting plugin installation" - rm -f ${tempFile} - exit 0 - fi - - rm -f ${tempFile} -} - -log "${HCOMPONENT_NAME}: lib folder=$HCOMPONENT_LIB_DIR conf folder=$HCOMPONENT_CONF_DIR" - -# -# If there is a set-ranger-${COMPONENT}-env.sh, install it -# -dt=`date '+%Y%m%d-%H%M%S'` - -if [ -f "${SET_ENV_SCRIPT_TEMPLATE}" ] -then - # - # If the setenv script already exists, move it to the archive folder - # - if [ -f "${SET_ENV_SCRIPT}" ] - then - if [ ! -d "${HCOMPONENT_ARCHIVE_CONF_DIR}" ] - then - mkdir -p ${HCOMPONENT_ARCHIVE_CONF_DIR} - fi - log "Saving current ${SET_ENV_SCRIPT_NAME} to ${HCOMPONENT_ARCHIVE_CONF_DIR} ..." - mv ${SET_ENV_SCRIPT} ${HCOMPONENT_ARCHIVE_CONF_DIR}/${SET_ENV_SCRIPT_NAME}.${dt} - fi - - if [ "${action}" = "enable" ] - then - - cp ${SET_ENV_SCRIPT_TEMPLATE} ${SET_ENV_SCRIPT} - - DEST_SCRIPT_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/${HCOMPONENT_NAME}-config.sh - - DEST_SCRIPT_ARCHIVE_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/.${HCOMPONENT_NAME}-config.sh.${dt} - - if [ -f "${DEST_SCRIPT_FILE}" ] - then - - log "Saving current ${DEST_SCRIPT_FILE} to ${DEST_SCRIPT_ARCHIVE_FILE} ..." - - cp ${DEST_SCRIPT_FILE} ${DEST_SCRIPT_ARCHIVE_FILE} - - grep 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > /dev/null 2>&1 - if [ $? -eq 0 ] - then - ts=`date '+%Y%m%d%H%M%S'` - grep -v 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > ${DEST_SCRIPT_FILE}.${ts} - if [ $? -eq 0 ] - then - log "Removing old reference to xasecure setenv source ..." - cat ${DEST_SCRIPT_FILE}.${ts} > ${DEST_SCRIPT_FILE} - rm -f ${DEST_SCRIPT_FILE}.${ts} - fi - fi - - grep "[ \t]*.[ \t]*${SET_ENV_SCRIPT}" ${DEST_SCRIPT_FILE} > /dev/null - if [ $? -ne 0 ] - then - log "Appending sourcing script, ${SET_ENV_SCRIPT_NAME} in the file: ${DEST_SCRIPT_FILE} " - cat >> ${DEST_SCRIPT_FILE} <\n`date`\n" > ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chmod a+r ${HCOMPONENT_CONF_DIR}/ranger-security.xml - for cf in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.xml - do - cfb=`basename ${cf}` - if [ -f "${HCOMPONENT_CONF_DIR}/${cfb}" ] - then - log "Saving ${HCOMPONENT_CONF_DIR}/${cfb} to ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} ..." - cp ${HCOMPONENT_CONF_DIR}/${cfb} ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} - fi - cp ${cf} ${HCOMPONENT_CONF_DIR}/ - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/${cfb} - chmod a+r ${HCOMPONENT_CONF_DIR}/${cfb} - done - else - if [ -f ${HCOMPONENT_CONF_DIR}/ranger-security.xml ] - then - mv ${HCOMPONENT_CONF_DIR}/ranger-security.xml ${HCOMPONENT_CONF_DIR}/.ranger-security.xml.`date '+%Y%m%d%H%M%S'` - fi - fi - - # - # Ensure that POLICY_CACHE_FILE_PATH is accessible - # - REPO_NAME=$(getInstallProperty 'REPOSITORY_NAME') - export POLICY_CACHE_FILE_PATH=/etc/${PROJ_NAME}/${REPO_NAME}/policycache - export CREDENTIAL_PROVIDER_FILE=/etc/${PROJ_NAME}/${REPO_NAME}/cred.jceks - if [ ! -d ${POLICY_CACHE_FILE_PATH} ] - then - mkdir -p ${POLICY_CACHE_FILE_PATH} - fi - chmod a+rx /etc/${PROJ_NAME} - chmod a+rx /etc/${PROJ_NAME}/${REPO_NAME} - chmod a+rx ${POLICY_CACHE_FILE_PATH} - chown -R ${CFG_OWNER_INF} /etc/${PROJ_NAME}/${REPO_NAME} - - for f in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.cfg - do - if [ -f "${f}" ] - then - fn=`basename $f` - orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` - fullpathorgfn="${HCOMPONENT_CONF_DIR}/${orgfn}" - if [ ! -f ${fullpathorgfn} ] - then - if [ -f ${DEFAULT_XML_CONFIG} ] - then - log "Creating default file from [${DEFAULT_XML_CONFIG}] for [${fullpathorgfn}] .." - cp ${DEFAULT_XML_CONFIG} ${fullpathorgfn} - chown ${CFG_OWNER_INF} ${fullpathorgfn} - chmod a+r ${fullpathorgfn} - else - echo "ERROR: Unable to find ${fullpathorgfn}" - exit 1 - fi - fi - archivefn="${HCOMPONENT_CONF_DIR}/.${orgfn}.${dt}" - newfn="${HCOMPONENT_CONF_DIR}/.${orgfn}-new.${dt}" - log "Saving current config file: ${fullpathorgfn} to ${archivefn} ..." - cp ${fullpathorgfn} ${archivefn} - if [ $? -eq 0 ] - then - ${JAVA} -cp "${INSTALL_CP}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS} - if [ $? -eq 0 ] - then - diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 - if [ $? -ne 0 ] - then - cat ${newfn} > ${fullpathorgfn} - fi - - else - echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" - echo "exiting ...." - exit 1 - fi - else - echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" - echo "exiting ...." - exit 1 - fi - fi - done -fi - -# -# Create library link -# -if [ "${action}" = "enable" ] -then - dt=`date '+%Y%m%d%H%M%S'` - # - # Encrypt the password and keep it secure in Credential Provider API - # - CredFile=${CREDENTIAL_PROVIDER_FILE} - if ! [ `echo ${CredFile} | grep '^/.*'` ] - then - echo "ERROR:Please enter the Credential File Store with proper file path" - exit 1 - fi - - pardir=`dirname ${CredFile}` - - if [ ! -d "${pardir}" ] - then - mkdir -p "${pardir}" - if [ $? -ne 0 ] - then - echo "ERROR: Unable to create credential store file path" - exit 1 - fi - chmod a+rx "${pardir}" - fi -fi - - -#Check Properties whether in File, return code 1 if not exist -#$1 -> propertyName; $2 -> fileName -checkPropertyInFile(){ - validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f1-) # for validation - if test -z "$validate" ; then return 1; fi -} - -#Add Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addPropertyToFile(){ - echo "$1=$2">>$3 - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] Failed to add properties '$1' to $3 file!"; exit 1; fi - echo "Property $1 added successfully with : '$2'" -} - -#Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -updatePropertyToFile(){ - sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi - echo "Property $1 updated successfully with : '$2'" -} - -#Add or Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addOrUpdatePropertyToFile(){ - checkPropertyInFile $1 $3 - if [ $? -eq 1 ] - then - addPropertyToFile $1 $2 $3 - else - updatePropertyToFile $1 $2 $3 - fi -} - - -# Set notice to restart the ${HCOMPONENT_NAME} -echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." - -exit 0 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties deleted file mode 100644 index a744a1ac5b25..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/install.properties +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -POLICY_MGR_URL=http://ranger:6080 -REPOSITORY_NAME=dev_ozone -COMPONENT_INSTALL_DIR_NAME=ranger-ozone-plugin - -CUSTOM_USER=hadoop -CUSTOM_GROUP=hadoop - -XAAUDIT.SOLR.IS_ENABLED=true -XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 -XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 -XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits - -XAAUDIT.HDFS.IS_ENABLED=false -XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit -XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/audit -XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/audit/archive -XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log -XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 -XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 -XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 -XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log -XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 -XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 -XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 - -XAAUDIT.SUMMARY.ENABLE=true - -XAAUDIT.SOLR.ENABLE=true -XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits -XAAUDIT.SOLR.USER=NONE -XAAUDIT.SOLR.PASSWORD=NONE -XAAUDIT.SOLR.ZOOKEEPER=NONE -XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/audit/solr/spool - -XAAUDIT.ELASTICSEARCH.ENABLE=false -XAAUDIT.ELASTICSEARCH.URL=NONE -XAAUDIT.ELASTICSEARCH.USER=NONE -XAAUDIT.ELASTICSEARCH.PASSWORD=NONE -XAAUDIT.ELASTICSEARCH.INDEX=NONE -XAAUDIT.ELASTICSEARCH.PORT=NONE -XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE - -XAAUDIT.HDFS.ENABLE=false -XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit -XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/audit/hdfs/spool - -XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME -XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY -XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER -XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER - -XAAUDIT.LOG4J.ENABLE=true -XAAUDIT.LOG4J.IS_ASYNC=false -XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 -XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 -XAAUDIT.LOG4J.DESTINATION.LOG4J=true -XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit - -XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false -XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE -XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE -XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE -XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE - -SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks -SSL_KEYSTORE_PASSWORD=myKeyFilePassword -SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks -SSL_TRUSTSTORE_PASSWORD=changeit diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config deleted file mode 100644 index 86bd016039c5..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-config +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# To enable Ranger as the Authorizer in Ozone -OZONE-SITE.XML_ozone.acl.enabled=true -OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh b/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh deleted file mode 100644 index afc0f819bd4b..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/ranger-plugin/ranger-plugin-install.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE_HOME=/opt/hadoop -cd "${OZONE_HOME}"/ranger-ozone-plugin || exit - -if [[ ! -f "${OZONE_HOME}"/.setupDone ]]; -then - if [ ! -d conf ]; then - mkdir -p conf - echo "conf directory created!" - else - echo "conf directory exists already!" - fi - echo "export JAVA_HOME=${JAVA_HOME}" >> conf/ozone-env.sh - sudo JAVA_HOME=/usr/lib/jvm/jre/ ./enable-ozone-plugin.sh - touch "${OZONE_HOME}"/.setupDone -else - echo "Ranger Ozone Plugin Installation is already complete!" -fi diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh index 8a7440ba1518..5a9f58531bac 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test-ranger.sh @@ -20,10 +20,59 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR +: "${RANGER_VERSION:=2.6.0}" +: "${DOWNLOAD_DIR:=${TEMP_DIR:-/tmp}}" + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" export SECURITY_ENABLED=false export COMPOSE_FILE=docker-compose.yaml:docker-compose.ranger.yaml +curl -LO https://downloads.apache.org/ranger/KEYS +gpg --import KEYS + +download_if_not_exists() { + local url="$1" + local f="$2" + + if [[ -e "${f}" ]]; then + echo "${f} already downloaded" + else + echo "Downloading ${f} from ${url}" + curl --fail --location --output "${f}" --show-error --silent "${url}" || rm -fv "${f}" + fi +} + +download_and_verify_apache_release() { + local remote_path="$1" + + local f="$(basename "${remote_path}")" + local base_url="${APACHE_MIRROR_URL:-https://www.apache.org/dyn/closer.lua?action=download&filename=}" + local checksum_base_url="${APACHE_OFFICIAL_URL:-https://downloads.apache.org/}" + local download_dir="${DOWNLOAD_DIR:-/tmp}" + + download_if_not_exists "${base_url}${remote_path}" "${download_dir}/${f}" + download_if_not_exists "${checksum_base_url}${remote_path}.asc" "${download_dir}/${f}.asc" + gpg --verify "${download_dir}/${f}.asc" "${download_dir}/${f}" || exit 1 +} + +download_and_verify_apache_release "ranger/${RANGER_VERSION}/apache-ranger-${RANGER_VERSION}.tar.gz" +tar -C "${DOWNLOAD_DIR}" -x -z -f "${DOWNLOAD_DIR}/apache-ranger-${RANGER_VERSION}.tar.gz" +export RANGER_SOURCE_DIR="${DOWNLOAD_DIR}/apache-ranger-${RANGER_VERSION}" +chmod -R go+rX "${RANGER_SOURCE_DIR}" + +download_and_verify_apache_release "ranger/${RANGER_VERSION}/plugins/ozone/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz" +tar -C "${DOWNLOAD_DIR}" -x -z -f "${DOWNLOAD_DIR}/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz" +export RANGER_OZONE_PLUGIN_DIR="${DOWNLOAD_DIR}/ranger-${RANGER_VERSION}-ozone-plugin" +chmod -R go+rX "${RANGER_OZONE_PLUGIN_DIR}" + +# customizations before install +sed -i \ + -e 's@^POLICY_MGR_URL=.*@POLICY_MGR_URL=http://ranger:6080@' \ + -e 's@^REPOSITORY_NAME=.*@REPOSITORY_NAME=dev_ozone@' \ + -e 's@^CUSTOM_USER=ozone@CUSTOM_USER=hadoop@' \ + "${RANGER_OZONE_PLUGIN_DIR}/install.properties" + start_docker_env +wait_for_port ranger 6080 120