diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json index 3025e610758..3cbda98976a 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json @@ -1,6 +1,32 @@ { "BIGTOP": { "stack-select": { + "ALLUXIO": { + "ALLUXIO_MASTER": { + "STACK-SELECT-PACKAGE": "alluxio", + "INSTALL": [ + "alluxio" + ], + "PATCH": [ + "alluxio" + ], + "STANDARD": [ + "alluxio" + ] + }, + "ALLUXIO_WORKER": { + "STACK-SELECT-PACKAGE": "alluxio", + "INSTALL": [ + "alluxio" + ], + "PATCH": [ + "alluxio" + ], + "STANDARD": [ + "alluxio" + ] + } + }, "HBASE": { "HBASE_CLIENT": { "STACK-SELECT-PACKAGE": "hbase-client", diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/alerts.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/alerts.json new file mode 100755 index 00000000000..b82dec8c804 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/alerts.json @@ -0,0 +1,59 @@ +{ + "ALLUXIO": { + "service": [], + "ALLUXIO_MASTER": [ + { + "name": "ALLUXIO_MASTER_PROCESS", + "label": "alluxio master", + "description": "This host-level alert is triggered if the alluxio msater cannot be determined to be up.", + "interval": 1, + "scope": "HOST", + "source": { + "type": "PORT", + "uri": "{{alluxio-log4j-properties/alluxio.master.rpc.port}}", + "default_port": 19998, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5 + } + } + } + } + ], + "ALLUXIO_WORKER": [ + { + "name": "ALLUXIO_WORKER_PROCESS", + "label": "alluxio worker", + "description": "This host-level alert is triggered if the alluxio worker cannot be determined to be up.", + "interval": 1, + "scope": "HOST", + "source": { + "type": "PORT", + "uri": "{{alluxio-log4j-properties/alluxio.worker.rpc.port}}", + "default_port": 29999, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5 + } + } + } + } + ] + } +} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-env.xml new file mode 100755 index 00000000000..72917b6ed0c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-env.xml @@ -0,0 +1,232 @@ + + + + + + alluxio_user + Alluxio User + alluxio + USER + + user + false + + + cluster-env + user_group + + + alluxio-env + alluxio_group + + + + + + + alluxio_group + Alluxio Group + hadoop + GROUP + alluxio group + + user + + + + + alluxio_log_dir + Alluxio Log directory + /var/log/alluxio + Alluxio Log Dir + + directory + + + + + alluxio_pid_dir + Alluxio PID directory + /var/run/alluxio + + directory + + + + + + alluxio_service_keytab + Alluxio keytab path + + + + alluxio_service_principal + Alluxio principal + KERBEROS_PRINCIPAL + + + + + alluxio_keytab + Alluxio keytab path + + + + alluxio_principal + Alluxio principal + KERBEROS_PRINCIPAL + + + + + + content + This is the jinja template for alluxio-env.sh file + + +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# Copy this file as alluxio-env.sh and edit it to configure Alluxio for your +# site. This file is sourced to launch Alluxio servers or use Alluxio shell +# commands. +# +# This file is an optional approach to configure Alluxio options by setting the +# following listed environment variables. Note that, setting this file will not +# affect jobs (e.g., Spark job or MapReduce job) that are using Alluxio client +# as a library. Alternatively, it is recommended to create alluxio-site.properties file, +# which supports all the configuration options provided by Alluxio +# (https://docs.alluxio.io/os/user/stable/en/reference/Properties-List.html), +# and is respected by both external jobs and Alluxio servers (or shell). + +# This file also allows you to configure Alluxio remote debugging as outlined here: +# https://docs.alluxio.io/os/user/stable/en/operation/Troubleshooting.html#alluxio-remote-debug + +# JVM is required to run Alluxio. +# If its path is not set in Shell, please specify either of following environment variables +# JAVA_HOME +# JAVA + +JAVA_HOME={{java_home}} + +# The directory where log files are stored. (Default: ${ALLUXIO_HOME}/logs). +# ALLUXIO_LOGS_DIR + +# The hostname for log server. If set, remote logging is enabled. +# ALLUXIO_LOGSERVER_HOSTNAME +# If remote logging is enabled, the port for log server. (Default: 45600) +# ALLUXIO_LOGSERVER_PORT +# If remote logging is enabled, the directory where log server log files are stored. (Default: ${ALLUXIO_HOME}/logs) +# ALLUXIO_LOGSERVER_LOGS_DIR + +# Config properties set for Alluxio master, worker and shell. (Default: "") +# E.g. "-Dalluxio.master.rpc.port=39999" +# ALLUXIO_JAVA_OPTS + +ALLUXIO_JAVA_OPTS+=" -Djava.library.path=/usr/hdp/current/hadoop-client/lib/native/ " + +# Config properties set for Alluxio master daemon. (Default: "") +# E.g. "-Dalluxio.master.rpc.port=39999" +# ALLUXIO_MASTER_JAVA_OPTS + +# Config properties set for Alluxio job master daemon. (Default: "") +# E.g. " -javaagent:./jmx_prometheus_javaagent-0.16.0.jar=8080:./config.yaml " +# ALLUXIO_JOB_MASTER_JAVA_OPTS + +# Config properties set for Alluxio worker daemon. (Default: "") +# E.g. "-Dalluxio.worker.rpc.port=49999" to set worker port, "-Xms2048M -Xmx2048M" to limit the heap size of worker. +# ALLUXIO_WORKER_JAVA_OPTS + +# Config properties set for Alluxio job worker daemon. (Default: "") +# E.g. " -javaagent:./jmx_prometheus_javaagent-0.16.0.jar=18080:./config.yaml " +# ALLUXIO_JOB_WORKER_JAVA_OPTS + +# Config properties set for Alluxio proxy daemon. (Default: "") +# E.g. "-Xms2048M -Xmx2048M" to limit the heap size of proxy. +# ALLUXIO_PROXY_JAVA_OPTS + +# Config properties set for Alluxio log server daemon. (Default: "") +# E.g. "-Xms2048M -Xmx2048M" to limit the heap size of log server. +# ALLUXIO_LOGSERVER_JAVA_OPTS + +# Config properties set for Alluxio shell. (Default: "") +# E.g. "-Dalluxio.user.file.writetype.default=CACHE_THROUGH" +# ALLUXIO_USER_JAVA_OPTS + +# Additional classpath entries for Alluxio processes. (Default: "") +# E.g. "/path/to/library1/:/path/to/library2/" +# ALLUXIO_CLASSPATH + +# Configuring remote debugging for Alluxio master process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60001" +# ALLUXIO_MASTER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio secondary master process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60010" +# ALLUXIO_SECONDARY_MASTER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio job master process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60002" +# ALLUXIO_JOB_MASTER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio worker process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60003" +# ALLUXIO_WORKER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio job worker process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60004" +# ALLUXIO_JOB_WORKER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio proxy process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60005" +# ALLUXIO_PROXY_ATTACH_OPTS + +# Configuring remote debugging for Alluxio log server process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60006" +# ALLUXIO_LOGSERVER_ATTACH_OPTS + +# Configuring remote debugging for Alluxio fuse process. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60009" +# ALLUXIO_FUSE_ATTACH_OPTS + +# Configuring remote debugging for Alluxio shell. (Default: "") +# E.g. "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=60000" +# ALLUXIO_USER_ATTACH_OPTS + +# Config LANG to support different characters in filenames +# export LANG=en_US.UTF-8 + + + + content + + + + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-log4j-properties.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-log4j-properties.xml new file mode 100755 index 00000000000..a6ff8df89ec --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-log4j-properties.xml @@ -0,0 +1,244 @@ + + + + + + content + alluxio-log4j2-properties + +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# May get overridden by System Property + +log4j.rootLogger=INFO, ${alluxio.logger.type}, ${alluxio.remote.logger.type} + +log4j.category.alluxio.logserver=INFO, ${alluxio.logserver.logger.type} +log4j.additivity.alluxio.logserver=false + +log4j.logger.AUDIT_LOG=INFO, ${alluxio.master.audit.logger.type} +log4j.logger.JOB_MASTER_AUDIT_LOG=INFO, ${alluxio.job.master.audit.logger.type} +log4j.logger.PROXY_AUDIT_LOG=INFO, ${alluxio.proxy.audit.logger.type} +log4j.additivity.AUDIT_LOG=false +log4j.additivity.JOB_MASTER_AUDIT_LOG=false +log4j.additivity.PROXY_AUDIT_LOG=false + +# Configures an appender whose name is "" (empty string) to be NullAppender. +# By default, if a Java class does not specify a particular appender, log4j will +# use "" as the appender name, then it will use Null appender. +log4j.appender.=org.apache.log4j.varia.NullAppender + +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.Target=System.out +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# The ParquetWriter logs for every row group which is not noisy for large row group size, +# but very noisy for small row group size. +log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordWriter=WARN +log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordReader=WARN + +# Appender for Job Master +log4j.appender.JOB_MASTER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.JOB_MASTER_LOGGER.File=${alluxio.logs.dir}/job_master.log +log4j.appender.JOB_MASTER_LOGGER.MaxFileSize=10MB +log4j.appender.JOB_MASTER_LOGGER.MaxBackupIndex=100 +log4j.appender.JOB_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.JOB_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Job Workers +log4j.appender.JOB_WORKER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.JOB_WORKER_LOGGER.File=${alluxio.logs.dir}/job_worker.log +log4j.appender.JOB_WORKER_LOGGER.MaxFileSize=10MB +log4j.appender.JOB_WORKER_LOGGER.MaxBackupIndex=100 +log4j.appender.JOB_WORKER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.JOB_WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Master +log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log +log4j.appender.MASTER_LOGGER.MaxFileSize=10MB +log4j.appender.MASTER_LOGGER.MaxBackupIndex=100 +log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Master +log4j.appender.SECONDARY_MASTER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.SECONDARY_MASTER_LOGGER.File=${alluxio.logs.dir}/secondary_master.log +log4j.appender.SECONDARY_MASTER_LOGGER.MaxFileSize=10MB +log4j.appender.SECONDARY_MASTER_LOGGER.MaxBackupIndex=100 +log4j.appender.SECONDARY_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.SECONDARY_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Master audit +log4j.appender.MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/master_audit.log +log4j.appender.MASTER_AUDIT_LOGGER.MaxFileSize=10MB +log4j.appender.MASTER_AUDIT_LOGGER.MaxBackupIndex=100 +log4j.appender.MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Job Master audit +log4j.appender.JOB_MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.JOB_MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/job_master_audit.log +log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxFileSize=10MB +log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxBackupIndex=100 +log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Proxy +log4j.appender.PROXY_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.PROXY_LOGGER.File=${alluxio.logs.dir}/proxy.log +log4j.appender.PROXY_LOGGER.MaxFileSize=10MB +log4j.appender.PROXY_LOGGER.MaxBackupIndex=100 +log4j.appender.PROXY_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.PROXY_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Proxy audit +log4j.appender.PROXY_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.PROXY_AUDIT_LOGGER.File=${alluxio.logs.dir}/proxy_audit.log +log4j.appender.PROXY_AUDIT_LOGGER.MaxFileSize=10MB +log4j.appender.PROXY_AUDIT_LOGGER.MaxBackupIndex=100 +log4j.appender.PROXY_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.PROXY_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}[%t](%F:%M:%L) - %m%n + +# Appender for Workers +log4j.appender.WORKER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.WORKER_LOGGER.File=${alluxio.logs.dir}/worker.log +log4j.appender.WORKER_LOGGER.MaxFileSize=10MB +log4j.appender.WORKER_LOGGER.MaxBackupIndex=100 +log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Remote appender for Job Master +log4j.appender.REMOTE_JOB_MASTER_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_JOB_MASTER_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_JOB_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_JOB_MASTER_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_JOB_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_JOB_MASTER_LOGGER.filter.ID.ProcessType=JOB_MASTER +log4j.appender.REMOTE_JOB_MASTER_LOGGER.Threshold=WARN + +# Remote appender for Job Workers +log4j.appender.REMOTE_JOB_WORKER_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_JOB_WORKER_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_JOB_WORKER_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_JOB_WORKER_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_JOB_WORKER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_JOB_WORKER_LOGGER.filter.ID.ProcessType=JOB_WORKER +log4j.appender.REMOTE_JOB_WORKER_LOGGER.Threshold=WARN + +# Remote appender for Master +log4j.appender.REMOTE_MASTER_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_MASTER_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_MASTER_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_MASTER_LOGGER.filter.ID.ProcessType=MASTER +log4j.appender.REMOTE_MASTER_LOGGER.Threshold=WARN + +# Remote appender for Secondary Master +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.filter.ID.ProcessType=SECONDARY_MASTER +log4j.appender.REMOTE_SECONDARY_MASTER_LOGGER.Threshold=WARN + +# Remote appender for Proxy +log4j.appender.REMOTE_PROXY_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_PROXY_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_PROXY_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_PROXY_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_PROXY_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_PROXY_LOGGER.filter.ID.ProcessType=PROXY +log4j.appender.REMOTE_PROXY_LOGGER.Threshold=WARN + +# Remote appender for Workers +log4j.appender.REMOTE_WORKER_LOGGER=org.apache.log4j.net.SocketAppender +log4j.appender.REMOTE_WORKER_LOGGER.Port=${alluxio.logserver.port} +log4j.appender.REMOTE_WORKER_LOGGER.RemoteHost=${alluxio.logserver.hostname} +log4j.appender.REMOTE_WORKER_LOGGER.ReconnectionDelay=10000 +log4j.appender.REMOTE_WORKER_LOGGER.filter.ID=alluxio.AlluxioRemoteLogFilter +log4j.appender.REMOTE_WORKER_LOGGER.filter.ID.ProcessType=WORKER +log4j.appender.REMOTE_WORKER_LOGGER.Threshold=WARN + +# (Local) appender for log server itself +log4j.appender.LOGSERVER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.LOGSERVER_LOGGER.File=${alluxio.logs.dir}/logserver.log +log4j.appender.LOGSERVER_LOGGER.MaxFileSize=10MB +log4j.appender.LOGSERVER_LOGGER.MaxBackupIndex=100 +log4j.appender.LOGSERVER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.LOGSERVER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# (Local) appender for log server to log on behalf of log clients +# No need to configure file path because log server will dynamically +# figure out for each appender. +log4j.appender.LOGSERVER_CLIENT_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.LOGSERVER_CLIENT_LOGGER.MaxFileSize=10MB +log4j.appender.LOGSERVER_CLIENT_LOGGER.MaxBackupIndex=100 +log4j.appender.LOGSERVER_CLIENT_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.LOGSERVER_CLIENT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for User +log4j.appender.USER_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.USER_LOGGER.File=${alluxio.user.logs.dir}/user_${user.name}.log +log4j.appender.USER_LOGGER.MaxFileSize=10MB +log4j.appender.USER_LOGGER.MaxBackupIndex=10 +log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Appender for Fuse +log4j.appender.FUSE_LOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.FUSE_LOGGER.File=${alluxio.logs.dir}/fuse.log +log4j.appender.FUSE_LOGGER.MaxFileSize=100MB +log4j.appender.FUSE_LOGGER.MaxBackupIndex=10 +log4j.appender.FUSE_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.FUSE_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p [%t](%F:%L) - %m%n + +# Disable noisy DEBUG logs +log4j.logger.com.amazonaws.util.EC2MetadataUtils=OFF +log4j.logger.io.grpc.netty.NettyServerHandler=OFF + +# Disable noisy INFO logs from ratis +log4j.logger.org.apache.ratis.grpc.server.GrpcLogAppender=ERROR +log4j.logger.org.apache.ratis.grpc.server.GrpcServerProtocolService=WARN +log4j.logger.org.apache.ratis.server.impl.FollowerInfo=WARN +log4j.logger.org.apache.ratis.server.leader.FollowerInfo=WARN +log4j.logger.org.apache.ratis.server.impl.RaftServerImpl=WARN +log4j.logger.org.apache.ratis.server.RaftServer$Division=WARN + + + + content + + + + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-metrics-properties.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-metrics-properties.xml new file mode 100755 index 00000000000..f76330a82a1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-metrics-properties.xml @@ -0,0 +1,132 @@ + + + + + + content + metrics.properties + +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# syntax: sink.[name].[options]=[value] + +# This file configures Alluxio's internal metrics system. +# The metrics can be configured to report its metrics to one or more sinks. The master, worker +# and client can have different configurations. +# +# A "sink" specifies where metrics are delivered to. One or more sinks can be assigned. +# +# The [name] field specifies the name of a sink. +# +# The [options] field is the specific property of this source or sink. The +# source or sink is responsible for parsing this property. +# +# Notes: +# 1. To add a new sink, set the "class" option to a fully qualified class +# name (see examples below). +# 2. Some sinks involve a polling period. The minimum allowed polling period +# is 1 second. +# 3. A metrics specific configuration +# "alluxio.metrics.conf.file=${ALLUXIO_HOME}/conf/metrics.properties" should be +# added to Java properties using -Dalluxio.metrics.conf.file=xxx if you want to +# customize metrics system. You can also put the file in ${ALLUXIO_HOME}/conf +# and it will be loaded automatically. +# + +## List of available sinks and their properties. + +# alluxio.metrics.sink.ConsoleSink +# Name: Default: Description: +# period 10 Poll period +# unit seconds Units of poll period + +# alluxio.metrics.sink.CsvSink +# Name: Default: Description: +# period 10 Poll period +# unit seconds Units of poll period +# directory /tmp Where to store CSV files + +# alluxio.metrics.sink.JmxSink + +# alluxio.metrics.sink.MetricsServlet +# Name: Default: Description: +# path /metrics/json Path prefix from the web server root + +# alluxio.metrics.sink.PrometheusMetricsServlet +# Name: Default: Description: +# path /metrics/prometheus Path prefix from the web server root + +# alluxio.metrics.sink.GraphiteSink +# Name: Default: Description: +# host NONE Hostname of Graphite server +# port NONE Port of Graphite server +# period 10 Poll period +# unit seconds Units of poll period +# prefix EMPTY STRING Prefix to prepend to metric name + +# alluxio.metrics.sink.Slf4jSink +# Name: Default: Description: +# period 10 Poll period +# unit seconds Units of poll period +# filter-class null Contains all metrics +# filter-regex null Contains all metrics + +## Examples +# Enable JmxSink by class name +# sink.jmx.class=alluxio.metrics.sink.JmxSink +# Jmx domain +# sink.jmx.domain=org.alluxio + +# Enable ConsoleSink by class name +# sink.console.class=alluxio.metrics.sink.ConsoleSink + +# Polling period for ConsoleSink +# sink.console.period=10 + +# sink.console.unit=seconds + +# Enable CsvSink +# sink.csv.class=alluxio.metrics.sink.CsvSink + +# Polling period for CsvSink +# sink.csv.period=1 + +# sink.csv.unit=minutes + +# Polling directory for CsvSink +# sink.csv.directory=/tmp/ + + + + content + + + + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-site-properties.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-site-properties.xml new file mode 100755 index 00000000000..e61c58e779f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/configuration/alluxio-site-properties.xml @@ -0,0 +1,153 @@ + + + + + alluxio.master.rpc.port + alluxio.master.rpc.port + 19998 + + + + + alluxio.master.web.port + alluxio.master.web.port + 19999 + + + + + alluxio.worker.rpc.port + alluxio.worker.rpc.port + 29999 + + + + + alluxio.worker.web.port + alluxio.worker.web.port + 30000 + + + + + alluxio.worker.memory + Alluxio Worker Memory(GB) + 1GB + + + + + alluxio.underfs.hdfs.address + Alluxio UnderFS hdfs Address + /alluxio/underFSStorage + + + + + alluxio.master.metastore.dir + Alluxio UnderFS Address + /usr/bigtop/current/alluxio/metastore + + + + + content + alluxio-log4j-properties + +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +# Site specific configuration properties for Alluxio +# Details about all configuration properties https://docs.alluxio.io/os/user/stable/en/reference/Properties-List.html + +# Common properties +# alluxio.master.hostname=localhost +# alluxio.master.mount.table.root.ufs=${alluxio.work.dir}/underFSStorage + +alluxio.master.hostname={{alluxio_master_host}} +#alluxio.master.mount.table.root.ufs={{namenode_address}}/alluxio/root/ +alluxio.underfs.address={{underfs_hdfs_addr}} +alluxio.underfs.hdfs.configuration=/etc/hadoop/conf/core-site.xml:/etc/hadoop/conf/hdfs-site.xml + +alluxio.master.metastore.dir={{alluxio_master_metastore_dir}} + +#ha +#alluxio.master.embedded.journal.addresses +{{master_embedded_journal_addresses_config}} + +# Security properties +# alluxio.security.authorization.permission.enabled=true +# alluxio.security.authentication.type=SIMPLE + +alluxio.hadoop.security.authentication={{alluxio_authentication}} +alluxio.master.mount.table.root.option.alluxio.security.underfs.hdfs.kerberos.client.keytab.file={{alluxio_kerberos_keytab}} +alluxio.master.mount.table.root.option.alluxio.security.underfs.hdfs.kerberos.client.principal={{alluxio_kerberos_principal}} +alluxio.master.mount.table.root.option.alluxio.security.underfs.hdfs.impersonation.enabled=true +alluxio.master.security.impersonation.yarn.users=* + +# +# alluxio.worker.tieredstore.levels=1 +# alluxio.worker.tieredstore.level0.alias=MEM +# alluxio.worker.tieredstore.level0.dirs.path=/mnt/ramdisk + +# Worker properties +alluxio.worker.ramdisk.size=1GB +alluxio.worker.tieredstore.levels=1 +alluxio.worker.tieredstore.level0.alias=MEM +alluxio.worker.tieredstore.level0.dirs.path=/mnt/ramdisk +alluxio.worker.tieredstore.level0.dirs.quota={{worker_mem}} +alluxio.worker.tieredstore.level0.watermark.high.ratio=0.9 +alluxio.worker.tieredstore.level0.watermark.low.ratio=0.7 + +#alluxio.worker.tieredstore.level1.alias=SSD +#alluxio.worker.tieredstore.level1.dirs.path=/data/alluxio +#alluxio.worker.tieredstore.level1.dirs.quota=25GB +#alluxio.worker.tieredstore.level1.watermark.high.ratio=0.9 +#alluxio.worker.tieredstore.level1.watermark.low.ratio=0.7 + +alluxio.worker.tieredstore.level1.alias=HDD +alluxio.worker.tieredstore.level1.dirs.path={{hdd_dirs}} +alluxio.worker.tieredstore.level1.dirs.quota={{hdd_quota}} +alluxio.worker.tieredstore.level1.watermark.high.ratio=0.9 +alluxio.worker.tieredstore.level1.watermark.low.ratio=0.7 + +# User properties +# alluxio.user.file.readtype.default=CACHE +# alluxio.user.file.writetype.default=ASYNC_THROUGH + +#port +alluxio.master.rpc.port={{alluxio_master_rpc_port}} +alluxio.master.web.port={{alluxio_master_web_port}} + +alluxio.worker.rpc.port={{alluxio_worker_rpc_port}} +alluxio.worker.web.port={{alluxio_worker_web_port}} + + + + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/kerberos.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/kerberos.json new file mode 100755 index 00000000000..fbf1a608384 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/kerberos.json @@ -0,0 +1,70 @@ +{ + "services": [ + { + "name": "ALLUXIO", + "identities": [ + { + "name": "alluxio_smokeuser", + "reference": "/smokeuser" + }, + { + "name": "alluxio_user", + "principal": { + "value": "${alluxio-env/alluxio_user}${principal_suffix}@${realm}", + "type" : "user", + "local_username" : "${alluxio-env/alluxio_user}", + "configuration": "alluxio-env/alluxio_principal" + }, + "keytab": { + "file": "${keytab_dir}/alluxio.headless.keytab", + "owner": { + "name": "${alluxio-env/alluxio_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "alluxio-env/alluxio_keytab" + } + }, + { + "name": "alluxio_service_keytab", + "principal": { + "value": "alluxio/_HOST@${realm}", + "type" : "service", + "configuration": "alluxio-env/alluxio_service_principal", + "local_username" : "${alluxio-env/alluxio_user}" + }, + "keytab": { + "file": "${keytab_dir}/alluxio.service.keytab", + "owner": { + "name": "${alluxio-env/alluxio_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "alluxio-env/alluxio_service_keytab" + } + } + ], + "configurations": [ + { + "alluxio-env": { + "alluxio.authentication": "KERBEROS" + } + } + ], + "components": [ + { + "name": "ALLUXIO_MASTER" + }, + { + "name": "ALLUXIO_WORKER" + } + ] + } + ] +} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/metainfo.xml new file mode 100755 index 00000000000..7b3994d3468 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/metainfo.xml @@ -0,0 +1,132 @@ + + + + 2.0 + + + ALLUXIO + Alluxio + Alluxio is world’s first open source data orchestration technology for analytics and AI for the cloud. This service is <b>Technical Preview</b>. + 2.9.3 + + + ALLUXIO_MASTER + Alluxio Master + MASTER + 1+ + true + + + HDFS/HDFS_CLIENT + host + + true + + + + + + PYTHON + 600 + + + + alluxio_master + true + + + + + + ALLUXIO_WORKER + Alluxio Worker + SLAVE + 1+ + true + + + PYTHON + + + + HDFS/HDFS_CLIENT + host + + true + + + + + + alluxio_worker + true + + + + + + + + alluxio-env + alluxio-site-properties + alluxio-log4j-properties + alluxio-metrics-properties + + + + + PYTHON + 300 + + + + HDFS + + + + + quicklinks.json + true + + + + true + + + + redhat9,redhat8,redhat7,amazonlinux2,redhat6,suse11,suse12 + + + alluxio_${stack_version} + + + + + debian7,debian9,ubuntu12,ubuntu14,ubuntu16,ubuntu18 + + + alluxio_${stack_version} + + + + + + + + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/master.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/master.py new file mode 100755 index 00000000000..92d3c63e366 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/master.py @@ -0,0 +1,184 @@ +#!/usr/bin/python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management import * +from resource_management.core import sudo +import time + + +class AlluxioMaster(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + + env.set_params(params) + + Directory( + [ + params.alluxio_pid_dir, + params.alluxio_master_metastore_dir, + params.alluxio_journal_dir, + ], + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o775, + create_parents=True, + ) + + Directory( + [params.alluxio_log_dir, os.path.join(params.alluxio_log_dir, "user")], + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o777, + create_parents=True, + ) + + # create alluxio-site.properties in alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "alluxio-site.properties"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_site_properties), + mode=0o644, + ) + + # create alluxio-env.sh in alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "alluxio-env.sh"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_env_sh), + mode=0o775, + ) + + # create log4j2.properties alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "log4j.properties"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_log4j2_properties), + mode=0o644, + ) + + # masters + File( + format("{alluxio_conf_dir}/masters"), + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o644, + content=Template("masters.j2", conf_dir=params.alluxio_conf_dir), + ) + + # workers + File( + format("{alluxio_conf_dir}/workers"), + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o644, + content=Template("workers.j2", conf_dir=params.alluxio_conf_dir), + ) + + # hdfs dir + params.HdfsResource( + params.alluxio_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.alluxio_user, + mode=0o775, + ) + params.HdfsResource( + params.underfs_hdfs_addr, + type="directory", + action="create_on_execute", + owner=params.alluxio_user, + mode=0o775, + ) + + def start(self, env, upgrade_type=None): + import params + + env.set_params(params) + + self.configure(env) + if not params.alluxio_master_metastore_formatted: + Execute( + params.alluxio_master_format, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + # start + Execute( + params.alluxio_master_start_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + # generate pid,multiple judyment + tryTimes = 5 + while tryTimes > 0: + Execute( + params.alluxio_master_pid_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + if sudo.read_file(params.alluxio_master_pid_file) != "": + break + else: + Logger.info("waiting from fe start...") + time.sleep(60) + tryTimes = tryTimes - 1 + if tryTimes == 0: + Logger.error("start error,pls check logs.") + + def stop(self, env, upgrade_type=None): + import params + + env.set_params(params) + self.configure(env) + + Execute( + params.alluxio_master_stop_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + def status(self, env): + import params + + env.set_params(params) + check_process_status(params.alluxio_master_pid_file) + + def get_user(self): + import params + + return params.alluxio_user + + def get_pid_files(self): + import params + + return [params.alluxio_master_pid_file] + + +if __name__ == "__main__": + AlluxioMaster().execute() diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/params.py new file mode 100755 index 00000000000..2a6f7ee8304 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/params.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import socket +import os +from urllib.parse import urlparse + +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management import * +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.version import ( + format_stack_version, + get_major_version, +) +from resource_management.libraries.functions.copy_tarball import ( + get_sysprep_skip_copy_tarballs_hdfs, +) +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import ( + get_not_managed_resources, +) +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.copy_tarball import get_current_version +from resource_management.libraries.functions.stack_features import ( + check_stack_feature, + get_stack_feature_version, +) +from resource_management.libraries.functions import StackFeature + + +config = Script.get_config() + +java_home = config["ambariLevelParams"]["java_home"] +stack_root = Script.get_stack_root() + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() +sudo = AMBARI_SUDO_BINARY +fqdn = socket.getfqdn().lower() + +retryAble = default("/commandParams/command_retry_enabled", False) + +cluster_name = config["clusterName"] +stack_name = default("/clusterLevelParams/stack_name", None) +stack_root = Script.get_stack_root() +# 3.2 +stack_version_unformatted = config["clusterLevelParams"]["stack_version"] +# 3.2.0.0 +stack_version_formatted = format_stack_version(stack_version_unformatted) +# 3.2 +major_stack_version = get_major_version(stack_version_formatted) + +# 3.2.1.0-001 +effective_version = get_current_version(service="ALLUXIO") + +sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() + +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade +version = default("/commandParams/version", None) + +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") + +if stack_version_formatted and check_stack_feature( + StackFeature.ROLLING_UPGRADE, stack_version_formatted +): + hadoop_home = stack_select.get_hadoop_dir("home") + +hdfs_user = config["configurations"]["hadoop-env"]["hdfs_user"] +hdfs_principal_name = config["configurations"]["hadoop-env"]["hdfs_principal_name"] +hdfs_user_keytab = config["configurations"]["hadoop-env"]["hdfs_user_keytab"] +user_group = config["configurations"]["cluster-env"]["user_group"] + + +component_directory = "alluxio" +alluxio_home = format("{stack_root}/current/{component_directory}") +alluxio_conf_dir = format("{stack_root}/current/{component_directory}/conf") + +alluxio_user = config["configurations"]["alluxio-env"]["alluxio_user"] +alluxio_group = config["configurations"]["alluxio-env"]["alluxio_group"] +alluxio_pid_dir = config["configurations"]["alluxio-env"]["alluxio_pid_dir"] +alluxio_log_dir = config["configurations"]["alluxio-env"]["alluxio_log_dir"] +alluxio_work_dir = format("{alluxio_pid_dir}/work") + +alluxio_journal_dir = format("{stack_root}/current/{component_directory}/journal") + +host_name = config["agentLevelParams"]["hostname"] + +# alluxio masters address +alluxio_masters = config["clusterHostInfo"]["alluxio_master_hosts"] +alluxio_masters_str = "\n".join(alluxio_masters) +alluxio_master_host = host_name + +masters_journal_port = "19200" +master_embedded_journal_addresses = "" +master_embedded_journal_addresses_config = "" +# get comma separated lists of masters_journal_host hosts from alluxio_masters +if len(alluxio_masters) > 1: + alluxio_master_host = host_name + index = 0 + for host in alluxio_masters: + masters_journal_host = host + if masters_journal_port is not None: + masters_journal_host = host + ":" + str(masters_journal_port) + + master_embedded_journal_addresses += masters_journal_host + index += 1 + if index < len(alluxio_masters): + master_embedded_journal_addresses += "," + master_embedded_journal_addresses_config = ( + "alluxio.master.embedded.journal.addresses=" + master_embedded_journal_addresses + ) +elif len(alluxio_masters) == 1: + alluxio_master_host = alluxio_masters[0] +print(alluxio_master_host) +print(master_embedded_journal_addresses_config) + + +# alluxio.underfs.address +alluxio_master_metastore_dir = config["configurations"]["alluxio-site-properties"][ + "alluxio.master.metastore.dir" +] +if alluxio_master_metastore_dir is None: + alluxio_master_metastore_dir = "/usr/hdp/current/alluxio/metastore" +alluxio_master_metastore_formatted = False +if ( + os.path.exists(alluxio_master_metastore_dir) + and os.path.isdir(alluxio_master_metastore_dir) + and len(os.listdir(alluxio_master_metastore_dir)) > 0 +): + alluxio_master_metastore_formatted = True + + +alluxio_master_rpc_port = config["configurations"]["alluxio-site-properties"][ + "alluxio.master.rpc.port" +] +alluxio_master_web_port = config["configurations"]["alluxio-site-properties"][ + "alluxio.master.web.port" +] + +alluxio_worker_rpc_port = config["configurations"]["alluxio-site-properties"][ + "alluxio.worker.rpc.port" +] +alluxio_worker_web_port = config["configurations"]["alluxio-site-properties"][ + "alluxio.worker.web.port" +] + +# alluxio workers address +alluxio_workers = config["clusterHostInfo"]["alluxio_worker_hosts"] +alluxio_workers_str = "\n".join(alluxio_workers) + +# alluxio worker memory alotment +worker_mem = config["configurations"]["alluxio-site-properties"][ + "alluxio.worker.memory" +] + +# Find current stack and version to push agent files to +stack_name = default("/hostLevelParams/stack_name", None) +stack_version = config["hostLevelParams"]["stack_version"] + +# hadoop params +namenode_address = None +if "dfs.namenode.rpc-address" in config["configurations"]["hdfs-site"]: + namenode_rpcaddress = config["configurations"]["hdfs-site"][ + "dfs.namenode.rpc-address" + ] + namenode_address = format("hdfs://{namenode_rpcaddress}") +else: + namenode_address = config["configurations"]["core-site"]["fs.defaultFS"] + +# alluxio underfs address +underfs_hdfs_addr = ( + namenode_address + + config["configurations"]["alluxio-site-properties"]["alluxio.underfs.hdfs.address"] +) + + +# alluxio hdd dirs + + +alluxio_site_properties = config["configurations"]["alluxio-site-properties"]["content"] +alluxio_env_sh = config["configurations"]["alluxio-env"]["content"] + +alluxio_log4j2_properties = config["configurations"]["alluxio-log4j-properties"][ + "content" +] + +alluxio_hdfs_user_dir = format("/user/{alluxio_user}") + +smoke_user = config["configurations"]["cluster-env"]["smokeuser"] + +alluxio_authentication = "SIMPLE" + +# security_enabled +security_enabled = default("/configurations/cluster-env/security_enabled", None) +kinit_path_local = get_kinit_path( + default("/configurations/kerberos-env/executable_search_paths", None) +) + +if security_enabled: + alluxio_authentication = "KERBEROS" + alluxio_kerberos_keytab = config["configurations"]["alluxio-env"]["alluxio_keytab"] + alluxio_kerberos_principal = config["configurations"]["alluxio-env"][ + "alluxio_principal" + ] + smoke_user_keytab = config["configurations"]["cluster-env"]["smokeuser_keytab"] + smokeuser_principal = config["configurations"]["cluster-env"][ + "smokeuser_principal_name" + ] + alluxio_service_kerberos_keytab = config["configurations"]["alluxio-env"][ + "alluxio_service_keytab" + ] + alluxio_serive_kerberos_principal = config["configurations"]["alluxio-env"][ + "alluxio_service_principal" + ] + + +# for create_hdfs_directory +default_fs = config["configurations"]["core-site"]["fs.defaultFS"] +hdfs_site = config["configurations"]["hdfs-site"] +hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore" + +dfs_type = default("/clusterLevelParams/dfs_type", "") + +import functools + +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file=hdfs_resource_ignore_file, + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + hadoop_bin_dir=hadoop_bin_dir, + hadoop_conf_dir=hadoop_conf_dir, + principal_name=hdfs_principal_name, + hdfs_site=hdfs_site, + default_fs=default_fs, + immutable_paths=get_not_managed_resources(), + dfs_type=dfs_type, +) + + +# command +alluxio_master_pid_file = format("{alluxio_pid_dir}/{alluxio_user}-master.pid") +alluxio_master_pid_cmd = ( + "echo `ps -A -o pid,command | grep -i \"[j]ava\" | grep alluxio.master.AlluxioMaster | awk '{print $1; exit}'`> " + + alluxio_master_pid_file +) +alluxio_master_start_cmd = format( + "rm -fr {alluxio_master_pid_file} && {alluxio_home}/bin/alluxio-start.sh -a master" +) +alluxio_master_stop_cmd = format( + "{alluxio_home}/bin/alluxio-stop.sh master && rm -fr {alluxio_master_pid_file}" +) +alluxio_master_format = format("{alluxio_home}/bin/alluxio formatMaster") + +alluxio_worker_pid_file = format("{alluxio_pid_dir}/{alluxio_user}-worker.pid") +alluxio_worker_pid_cmd = ( + "echo `ps -A -o pid,command | grep -i \"[j]ava\" | grep alluxio.worker.AlluxioWorker | awk '{print $1; exit}'`> " + + alluxio_worker_pid_file +) +alluxio_worker_start_cmd = format( + "rm -fr {alluxio_worker_pid_file} && {alluxio_home}/bin/alluxio-start.sh worker" +) +alluxio_worker_stop_cmd = format( + "{alluxio_home}/bin/alluxio-stop.sh worker && rm -fr {alluxio_worker_pid_file}" +) + +sudo = AMBARI_SUDO_BINARY +alluxio_worker_mount_cmd = format("{alluxio_home}/bin/alluxio-mount.sh Mount") + +alluxio_test_cmd = format("{alluxio_home}/bin/alluxio runTests") + +if security_enabled: + kinit_principal = alluxio_serive_kerberos_principal.replace("_HOST", host_name) + alluxio_kinit_cmd = format( + "kinit -kt {alluxio_service_kerberos_keytab} {kinit_principal}" + ) + alluxio_master_start_cmd = alluxio_kinit_cmd + " && " + alluxio_master_start_cmd + alluxio_worker_start_cmd = alluxio_kinit_cmd + " && " + alluxio_worker_start_cmd diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/service_check.py new file mode 100755 index 00000000000..0d5c8da9293 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/service_check.py @@ -0,0 +1,43 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agree in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import subprocess +import time +import os + +from resource_management.core.exceptions import Fail +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import Execute +from resource_management.core.logger import Logger + + +CHECK_COMMAND_TIMEOUT_DEFAULT = 300.0 + + +class AlluxioServiceCheck(Script): + def service_check(self, env): + import params + + env.set_params(params) + + Execute(params.alluxio_test_cmd, user=params.alluxio_user) + + +if __name__ == "__main__": + AlluxioServiceCheck().execute() diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/worker.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/worker.py new file mode 100755 index 00000000000..8c8072d0c26 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/scripts/worker.py @@ -0,0 +1,147 @@ +#!/usr/bin/python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management import * +import time + + +class AlluxioWorker(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + + env.set_params(params) + + Directory( + [params.alluxio_pid_dir, params.alluxio_journal_dir], + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o775, + create_parents=True, + ) + + Directory( + [params.alluxio_log_dir, os.path.join(params.alluxio_log_dir, "user")], + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o777, + create_parents=True, + ) + + # create alluxio-site.properties in alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "alluxio-site.properties"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_site_properties), + mode=0o644, + ) + + # create alluxio-env.sh in alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "alluxio-env.sh"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_env_sh), + mode=0o775, + ) + + # create log4j2.properties alluxio install dir + File( + os.path.join(params.alluxio_conf_dir, "log4j.properties"), + owner=params.alluxio_user, + group=params.alluxio_group, + content=InlineTemplate(params.alluxio_log4j2_properties), + mode=0o644, + ) + + # masters + File( + format("{alluxio_conf_dir}/masters"), + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o644, + content=Template("masters.j2", conf_dir=params.alluxio_conf_dir), + ) + + # workers + File( + format("{alluxio_conf_dir}/workers"), + owner=params.alluxio_user, + group=params.alluxio_group, + mode=0o644, + content=Template("workers.j2", conf_dir=params.alluxio_conf_dir), + ) + + def start(self, env, upgrade_type=None): + import params + + env.set_params(params) + + self.configure(env) + + Execute(("bash", "-c", params.alluxio_worker_mount_cmd), sudo=True) + + Execute( + params.alluxio_worker_start_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + Execute( + params.alluxio_worker_pid_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + def stop(self, env, upgrade_type=None): + import params + + env.set_params(params) + self.configure(env) + + Execute( + params.alluxio_worker_stop_cmd, + user=params.alluxio_user, + environment={"JAVA_HOME": params.java_home}, + ) + + def status(self, env): + import params + + env.set_params(params) + check_process_status(params.alluxio_worker_pid_file) + + def get_user(self): + import params + + return params.alluxio_user + + def get_pid_files(self): + import params + + return [params.alluxio_worker_pid_file] + + +if __name__ == "__main__": + AlluxioWorker().execute() diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/masters.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/masters.j2 new file mode 100755 index 00000000000..8b612095170 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/masters.j2 @@ -0,0 +1,19 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{{alluxio_masters_str}} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/workers.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/workers.j2 new file mode 100755 index 00000000000..a7ffc689dfd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/package/templates/workers.j2 @@ -0,0 +1,19 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{{alluxio_workers_str}} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/quicklinks/quicklinks.json new file mode 100755 index 00000000000..46cc4ef05c4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/quicklinks/quicklinks.json @@ -0,0 +1,21 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "links": [ + { + "name": "Alluxio_WEB_UI", + "label": "Alluxio Web UI", + "requires_user_name": "false", + "component_name": "ALLUXIO_MASTER", + "url":"%@://%@:%@", + "port":{ + "http_property": "alluxio.master.web.port", + "http_default_port": "19999", + "regex": "^(\\d+)$", + "site": "alluxio-site-properties" + } + } + ] + } +} diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/role_command_order.json new file mode 100755 index 00000000000..a5b5669247f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/role_command_order.json @@ -0,0 +1,8 @@ +{ + "general_deps" : { + "_comment" : "dependencies for ALLUXIO", + "ALLUXIO_SERVICE_CHECK-SERVICE_CHECK" : ["ALLUXIO_MASTER-START","ALLUXIO_WORKER-START"], + "ALLUXIO_MASTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START","SECONDARY_NAMENODE-START", "DATANODE-START"], + "ALLUXIO_WORKER-START" : ["ALLUXIO_MASTER-START"] + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/service_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/service_advisor.py new file mode 100755 index 00000000000..08c5dcb261a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ALLUXIO/service_advisor.py @@ -0,0 +1,210 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +from ast import Param +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, "../../../../../stacks/") +PARENT_FILE = os.path.join(STACKS_DIR, "service_advisor.py") + +try: + if "BASE_SERVICE_ADVISOR" in os.environ: + PARENT_FILE = os.environ["BASE_SERVICE_ADVISOR"] + with open(PARENT_FILE, "rb") as fp: + service_advisor = imp.load_module( + "service_advisor", fp, PARENT_FILE, (".py", "rb", imp.PY_SOURCE) + ) +except Exception as e: + traceback.print_exc() + print("Failed to load parent") + + +class AlluxioServiceAdvisor(service_advisor.ServiceAdvisor): + def __init__(self, *args, **kwargs): + self.as_super = super(AlluxioServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return self.getServiceComponentCardinalityValidations(services, hosts, "ALLUXIO") + + def getServiceConfigurationRecommendations( + self, configurations, clusterData, services, hosts + ): + """ + Entry point. + Must be overriden in child class. + """ + # Logger.info("Class: %s, Method: %s. Recommending Service Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + recommender = AlluxioRecommender() + recommender.recommendAlluxioConfigurationsFromHDP33( + configurations, clusterData, services, hosts + ) + + # def getServiceConfigurationRecommendationsForSSO(self, configurations, clusterData, services, hosts): + # """ + # Entry point. + # Must be overriden in child class. + # """ + # recommender = AlluxioRecommender() + # recommender.recommendConfigurationsForSSO(configurations, clusterData, services, hosts) + + def getServiceConfigurationsValidationItems( + self, configurations, recommendedDefaults, services, hosts + ): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + # Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + return [] + + @staticmethod + def isKerberosEnabled(services, configurations): + """ + Determines if security is enabled by testing the value of core-site/hadoop.security.authentication enabled. + If the property exists and is equal to "kerberos", then is it enabled; otherwise is it assumed to be + disabled. + + :type services: dict + :param services: the dictionary containing the existing configuration values + :type configurations: dict + :param configurations: the dictionary containing the updated configuration values + :rtype: bool + :return: True or False + """ + if ( + configurations + and "core-site" in configurations + and "hadoop.security.authentication" in configurations["core-site"]["properties"] + ): + return ( + configurations["core-site"]["properties"][ + "hadoop.security.authentication" + ].lower() + == "kerberos" + ) + elif ( + services + and "core-site" in services["configurations"] + and "hadoop.security.authentication" + in services["configurations"]["core-site"]["properties"] + ): + return ( + services["configurations"]["core-site"]["properties"][ + "hadoop.security.authentication" + ].lower() + == "kerberos" + ) + else: + return False + + +class AlluxioRecommender(service_advisor.ServiceAdvisor): + """ + Alluxio Recommender suggests properties when adding the service for the first time or modifying configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(AlluxioRecommender, self) + self.as_super.__init__(*args, **kwargs) + + def recommendAlluxioConfigurationsFromHDP33( + self, configurations, clusterData, services, hosts + ): + """ + Recommend configurations for this service based on HDP 3.3. + """