SPARK-1252. On YARN, use container-log4j.properties for executors

Author: sryza

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

@@ -344,6 +344,8 @@ trait ClientBase extends Logging {
       JAVA_OPTS += " " + env("SPARK_JAVA_OPTS")
     }
 
+    JAVA_OPTS += " " + YarnSparkHadoopUtil.getLoggingArgsForContainerCommandLine()
+
     // Command for the ApplicationMaster
     val commands = List[String](
       Environment.JAVA_HOME.$() + "/bin/java" +

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala

@@ -62,6 +62,7 @@ trait ExecutorRunnableUtil extends Logging {
 
     JAVA_OPTS += " -Djava.io.tmpdir=" +
       new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR) + " "
+    JAVA_OPTS += YarnSparkHadoopUtil.getLoggingArgsForContainerCommandLine() + " "
 
     // Commenting it out for now - so that people can refer to the properties if required. Remove
     // it once cpuset version is pushed out.

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala

@@ -22,6 +22,7 @@ import org.apache.hadoop.mapred.JobConf
 import org.apache.hadoop.security.Credentials
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.api.ApplicationConstants
 import org.apache.hadoop.conf.Configuration
 import org.apache.spark.deploy.SparkHadoopUtil
 
@@ -67,3 +68,16 @@ class YarnSparkHadoopUtil extends SparkHadoopUtil {
   }
 
 }
+
+object YarnSparkHadoopUtil {
+  def getLoggingArgsForContainerCommandLine(): String = {
+    "-Dlog4j.configuration=container-log4j.properties " +
+      "-Dhadoop.root.logger=INFO,CLA " +
+      // for yarn/stable:
+      "-Dyarn.app.container.log.filesize=0 " +
+      "-Dyarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + " " +
+      // for yarn/alpha:
+      "-Dyarn.app.mapreduce.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + " " +
+      "-Dyarn.app.mapreduce.container.log.filesize=0"
+  }
+}