From 4b26fe99173c58d66cb63323044ebce53050fcfc Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 11 Oct 2022 17:40:50 +0800 Subject: [PATCH] [SPARK-40747] Support setting driver log url using env vars other than Standalone mode --- .../org/apache/spark/scheduler/SchedulerBackend.scala | 9 ++++++++- .../scheduler/cluster/StandaloneSchedulerBackend.scala | 8 -------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala index b2acdb3e12a6d..865ea15470c74 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala @@ -17,6 +17,8 @@ package org.apache.spark.scheduler +import java.util.Locale + import org.apache.spark.resource.ResourceProfile import org.apache.spark.storage.BlockManagerId @@ -73,7 +75,12 @@ private[spark] trait SchedulerBackend { * Executors tab for the driver. * @return Map containing the log names and their respective URLs */ - def getDriverLogUrls: Option[Map[String, String]] = None + def getDriverLogUrls: Option[Map[String, String]] = { + val prefix = "SPARK_DRIVER_LOG_URL_" + val driverLogUrls = sys.env.filterKeys(_.startsWith(prefix)) + .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap + if (driverLogUrls.nonEmpty) Some(driverLogUrls) else None + } /** * Get the attributes on driver. These attributes are used to replace log URLs when diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala index 9f71a0fe58ce1..467cb439a4d21 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala @@ -17,7 +17,6 @@ package org.apache.spark.scheduler.cluster -import java.util.Locale import java.util.concurrent.{Semaphore, TimeUnit} import java.util.concurrent.atomic.AtomicBoolean @@ -251,13 +250,6 @@ private[spark] class StandaloneSchedulerBackend( } } - override def getDriverLogUrls: Option[Map[String, String]] = { - val prefix = "SPARK_DRIVER_LOG_URL_" - val driverLogUrls = sys.env.filterKeys(_.startsWith(prefix)) - .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap - if (driverLogUrls.nonEmpty) Some(driverLogUrls) else None - } - private def waitForRegistration() = { registrationBarrier.acquire() }