Skip to content

HDDS-11236. Move Java version-specific NETTY_OPTS to ozone-functions.sh #7001

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 3 commits into from
Jul 30, 2024
Merged

HDDS-11236. Move Java version-specific NETTY_OPTS to ozone-functions.sh #7001

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
41a6c44
Move Java version-specific NETTY_OPTS to ozone-functions.sh
sarvekshayr Jul 29, 2024
6874000
Initialise RATIS_OPTS
sarvekshayr Jul 29, 2024
2015219
Added RATIS_OPTS
sarvekshayr Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 12 additions & 25 deletions hadoop-ozone/dist/src/shell/ozone/ozone
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,6 @@ function ozonecmd_case
# Corresponding Ratis issue https://issues.apache.org/jira/browse/RATIS-534.
RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false ${RATIS_OPTS}"

# Get the version string
JAVA_VERSION_STRING=$(java -version 2>&1)

# Extract the major version number
JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | grep -oE '[0-9]+(\.[0-9]+)*' | head -n 1 | awk -F. '{print ($1 == 1 ? $2 : $1)}')

# Add JVM parameter for Java 9+
# (org.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true) to allow Netty unsafe memory allocation.
# Corresponding issue https://issues.apache.org/jira/browse/HDDS-10382.
if [[ "${JAVA_MAJOR_VERSION}" -ge "9" ]]; then
NETTY_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${NETTY_OPTS}"
fi

case ${subcmd} in
auditparser)
OZONE_CLASSNAME=org.apache.hadoop.ozone.audit.parser.AuditParser
Expand All @@ -118,7 +105,7 @@ function ozonecmd_case
datanode)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
ozone_deprecate_envvar HDDS_DN_OPTS OZONE_DATANODE_OPTS
OZONE_DATANODE_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_DATANODE_OPTS}"
OZONE_DATANODE_OPTS="${RATIS_OPTS} ${OZONE_DATANODE_OPTS}"
OZONE_DATANODE_OPTS="-Dlog4j.configurationFile=${OZONE_CONF_DIR}/dn-audit-log4j2.properties,${OZONE_CONF_DIR}/dn-container-log4j2.properties ${OZONE_DATANODE_OPTS}"
OZONE_DATANODE_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_DATANODE_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
Expand All @@ -138,7 +125,7 @@ function ozonecmd_case
;;
freon)
OZONE_CLASSNAME=org.apache.hadoop.ozone.freon.Freon
OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
getconf)
Expand All @@ -149,15 +136,15 @@ function ozonecmd_case
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter
ozone_deprecate_envvar HDFS_OM_OPTS OZONE_OM_OPTS
OZONE_OM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_OM_OPTS}"
OZONE_OM_OPTS="${RATIS_OPTS} ${OZONE_OM_OPTS}"
OZONE_OM_OPTS="${OZONE_OM_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/om-audit-log4j2.properties"
OZONE_OM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_OM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-manager"
;;
sh | shell)
OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneShell
ozone_deprecate_envvar HDFS_OM_SH_OPTS OZONE_SH_OPTS
OZONE_SH_OPTS="${OZONE_SH_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_SH_OPTS="${OZONE_SH_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
s3)
Expand All @@ -168,20 +155,20 @@ function ozonecmd_case
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManagerStarter'
ozone_deprecate_envvar HDFS_STORAGECONTAINERMANAGER_OPTS OZONE_SCM_OPTS
OZONE_SCM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_SCM_OPTS}"
OZONE_SCM_OPTS="${RATIS_OPTS} ${OZONE_SCM_OPTS}"
OZONE_SCM_OPTS="${OZONE_SCM_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/scm-audit-log4j2.properties"
OZONE_SCM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector ${OZONE_SCM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="hdds-server-scm"
;;
s3g)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway'
OZONE_S3G_OPTS="${OZONE_S3G_OPTS} ${NETTY_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_S3G_OPTS="${OZONE_S3G_OPTS} ${RATIS_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-s3gateway"
;;
httpfs)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_OPTS="${OZONE_OPTS} ${NETTY_OPTS} -Dhttpfs.home.dir=${OZONE_HOME} -Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log -Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_OPTS="${OZONE_OPTS} ${RATIS_OPTS} -Dhttpfs.home.dir=${OZONE_HOME} -Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log -Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_CLASSNAME='org.apache.ozone.fs.http.server.HttpFSServerWebServer'
OZONE_RUN_ARTIFACT_NAME="ozone-httpfsgateway"
;;
Expand All @@ -197,12 +184,12 @@ function ozonecmd_case
recon)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.recon.ReconServer'
OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-recon"
;;
fs)
OZONE_CLASSNAME=org.apache.hadoop.fs.ozone.OzoneFsShell
OZONE_FS_OPTS="${OZONE_FS_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_FS_OPTS="${OZONE_FS_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
daemonlog)
Expand All @@ -227,17 +214,17 @@ function ozonecmd_case
;;
admin)
OZONE_CLASSNAME=org.apache.hadoop.hdds.cli.OzoneAdmin
OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
debug)
OZONE_CLASSNAME=org.apache.hadoop.ozone.debug.OzoneDebug
OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
repair)
OZONE_CLASSNAME=org.apache.hadoop.ozone.repair.OzoneRepair
OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
checknative)
Expand Down
8 changes: 8 additions & 0 deletions hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1412,6 +1412,14 @@ function ozone_java_setup
# Extract the major version number
JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | sed -E -n 's/.* version "([^.-]*).*"/\1/p' | cut -d' ' -f1)

# Add JVM parameter (org.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true)
# to allow netty unsafe memory allocation in Java 9+.
RATIS_OPTS="${RATIS_OPTS:-}"

if [[ "${JAVA_MAJOR_VERSION}" -ge 9 ]]; then
RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${RATIS_OPTS}"
fi

ozone_set_module_access_args
}

Expand Down