diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 1de7adf83d24..99fce8c2d44e 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -81,19 +81,6 @@ function ozonecmd_case # Corresponding Ratis issue https://issues.apache.org/jira/browse/RATIS-534. RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false ${RATIS_OPTS}" - # Get the version string - JAVA_VERSION_STRING=$(java -version 2>&1) - - # Extract the major version number - JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | grep -oE '[0-9]+(\.[0-9]+)*' | head -n 1 | awk -F. '{print ($1 == 1 ? $2 : $1)}') - - # Add JVM parameter for Java 9+ - # (org.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true) to allow Netty unsafe memory allocation. - # Corresponding issue https://issues.apache.org/jira/browse/HDDS-10382. - if [[ "${JAVA_MAJOR_VERSION}" -ge "9" ]]; then - NETTY_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${NETTY_OPTS}" - fi - case ${subcmd} in auditparser) OZONE_CLASSNAME=org.apache.hadoop.ozone.audit.parser.AuditParser @@ -118,7 +105,7 @@ function ozonecmd_case datanode) OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" ozone_deprecate_envvar HDDS_DN_OPTS OZONE_DATANODE_OPTS - OZONE_DATANODE_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_DATANODE_OPTS}" + OZONE_DATANODE_OPTS="${RATIS_OPTS} ${OZONE_DATANODE_OPTS}" OZONE_DATANODE_OPTS="-Dlog4j.configurationFile=${OZONE_CONF_DIR}/dn-audit-log4j2.properties,${OZONE_CONF_DIR}/dn-container-log4j2.properties ${OZONE_DATANODE_OPTS}" OZONE_DATANODE_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_DATANODE_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService @@ -138,7 +125,7 @@ function ozonecmd_case ;; freon) OZONE_CLASSNAME=org.apache.hadoop.ozone.freon.Freon - OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; getconf) @@ -149,7 +136,7 @@ function ozonecmd_case OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" OZONE_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter ozone_deprecate_envvar HDFS_OM_OPTS OZONE_OM_OPTS - OZONE_OM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_OM_OPTS}" + OZONE_OM_OPTS="${RATIS_OPTS} ${OZONE_OM_OPTS}" OZONE_OM_OPTS="${OZONE_OM_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/om-audit-log4j2.properties" OZONE_OM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_OM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-manager" @@ -157,7 +144,7 @@ function ozonecmd_case sh | shell) OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneShell ozone_deprecate_envvar HDFS_OM_SH_OPTS OZONE_SH_OPTS - OZONE_SH_OPTS="${OZONE_SH_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_SH_OPTS="${OZONE_SH_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; s3) @@ -168,7 +155,7 @@ function ozonecmd_case OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" OZONE_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManagerStarter' ozone_deprecate_envvar HDFS_STORAGECONTAINERMANAGER_OPTS OZONE_SCM_OPTS - OZONE_SCM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_SCM_OPTS}" + OZONE_SCM_OPTS="${RATIS_OPTS} ${OZONE_SCM_OPTS}" OZONE_SCM_OPTS="${OZONE_SCM_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/scm-audit-log4j2.properties" OZONE_SCM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_SCM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="hdds-server-scm" @@ -176,12 +163,12 @@ function ozonecmd_case s3g) OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" OZONE_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway' - OZONE_S3G_OPTS="${OZONE_S3G_OPTS} ${NETTY_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_S3G_OPTS="${OZONE_S3G_OPTS} ${RATIS_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-s3gateway" ;; httpfs) OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" - OZONE_OPTS="${OZONE_OPTS} ${NETTY_OPTS} -Dhttpfs.home.dir=${OZONE_HOME} -Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log -Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_OPTS="${OZONE_OPTS} ${RATIS_OPTS} -Dhttpfs.home.dir=${OZONE_HOME} -Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log -Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}" OZONE_CLASSNAME='org.apache.ozone.fs.http.server.HttpFSServerWebServer' OZONE_RUN_ARTIFACT_NAME="ozone-httpfsgateway" ;; @@ -197,12 +184,12 @@ function ozonecmd_case recon) OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" OZONE_CLASSNAME='org.apache.hadoop.ozone.recon.ReconServer' - OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-recon" ;; fs) OZONE_CLASSNAME=org.apache.hadoop.fs.ozone.OzoneFsShell - OZONE_FS_OPTS="${OZONE_FS_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_FS_OPTS="${OZONE_FS_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; daemonlog) @@ -227,17 +214,17 @@ function ozonecmd_case ;; admin) OZONE_CLASSNAME=org.apache.hadoop.hdds.cli.OzoneAdmin - OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; debug) OZONE_CLASSNAME=org.apache.hadoop.ozone.debug.OzoneDebug - OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; repair) OZONE_CLASSNAME=org.apache.hadoop.ozone.repair.OzoneRepair - OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" + OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; checknative) diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index b21453fa0d6b..84e2b73836e5 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@ -1412,6 +1412,14 @@ function ozone_java_setup # Extract the major version number JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | sed -E -n 's/.* version "([^.-]*).*"/\1/p' | cut -d' ' -f1) + # Add JVM parameter (org.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true) + # to allow netty unsafe memory allocation in Java 9+. + RATIS_OPTS="${RATIS_OPTS:-}" + + if [[ "${JAVA_MAJOR_VERSION}" -ge 9 ]]; then + RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${RATIS_OPTS}" + fi + ozone_set_module_access_args }