From 7f9ef388679e8b9a282befc3c5a031a2199d0eb0 Mon Sep 17 00:00:00 2001 From: hejiefang Date: Fri, 20 Jul 2018 17:39:11 +0800 Subject: [PATCH 1/2] [SPARK-24873]Increase switch to shielding frequent interaction reports with yarn --- .../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index f1a8df00f9c5..d47db0850f2d 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -111,7 +111,8 @@ private[spark] class YarnClientSchedulerBackend( override def run() { try { val YarnAppReport(_, state, diags) = - client.monitorApplication(appId.get, logApplicationReport = true) + client.monitorApplication(appId.get, + logApplicationReport = conf.getBoolean("spark.yarn.log.application.report", true)) logError(s"YARN application has exited unexpectedly with state $state! " + "Check the YARN application logs for more details.") diags.foreach { err => From 912878e88d684df966831c903a021b73d0ced778 Mon Sep 17 00:00:00 2001 From: hejiefang Date: Mon, 23 Jul 2018 10:51:22 +0800 Subject: [PATCH 2/2] [SPARK-24873]Increase switch to shielding frequent interaction reports with yarn --- docs/running-on-yarn.md | 7 +++++++ .../main/scala/org/apache/spark/deploy/yarn/config.scala | 5 +++++ .../scheduler/cluster/YarnClientSchedulerBackend.scala | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 0b265b0cb1b3..18d424e1b2cd 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -500,6 +500,13 @@ providers can be disabled individually by setting `spark.security.credentials.{s The default value should be enough for most deployments. + + spark.yarn.log.application.report + true + + Whether to show the interaction reports with yarn. + + ## Troubleshooting Kerberos diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala index 129084a86597..3e499c3d6f96 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala @@ -334,4 +334,9 @@ package object config { .booleanConf .createWithDefault(false) + private[spark] val SPARK_YARN_LOG_APPLICATION_REPORT = ConfigBuilder("spark.yarn.log.application.report") + .doc("Whether to show the interaction reports with yarn.") + .booleanConf + .createWithDefault(true) + } diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index d47db0850f2d..aefec79715ff 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -112,7 +112,7 @@ private[spark] class YarnClientSchedulerBackend( try { val YarnAppReport(_, state, diags) = client.monitorApplication(appId.get, - logApplicationReport = conf.getBoolean("spark.yarn.log.application.report", true)) + logApplicationReport = conf.getBoolean(SPARK_YARN_LOG_APPLICATION_REPORT, true)) logError(s"YARN application has exited unexpectedly with state $state! " + "Check the YARN application logs for more details.") diags.foreach { err =>