From 8eb019145441e830e3d4afb005e9fbf10434f780 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 16 Feb 2017 17:46:49 +0800 Subject: [PATCH 1/2] update cred using spark.yarn.credentials.updateTime --- .../deploy/yarn/security/CredentialUpdater.scala | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala index 5df4fbd9c153..3faa1fe79f92 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala @@ -55,14 +55,10 @@ private[spark] class CredentialUpdater( /** Start the credential updater task */ def start(): Unit = { - val startTime = sparkConf.get(CREDENTIALS_RENEWAL_TIME) - val remainingTime = startTime - System.currentTimeMillis() - if (remainingTime <= 0) { - credentialUpdater.schedule(credentialUpdaterRunnable, 1, TimeUnit.MINUTES) - } else { - logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime millis.") - credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS) - } + val startTime = sparkConf.get(CREDENTIALS_UPDATE_TIME) + val remainingTime = Math.max(1, startTime - System.currentTimeMillis()) + logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime millis.") + credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS) } private def updateCredentialsIfRequired(): Unit = { From 211bbd3234a3641b02b38eae7bc0020ffb5b46d3 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 16 Feb 2017 18:57:43 +0800 Subject: [PATCH 2/2] change remaining time calculation back --- .../spark/deploy/yarn/security/CredentialUpdater.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala index 3faa1fe79f92..2fdb70a73c75 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala @@ -56,9 +56,13 @@ private[spark] class CredentialUpdater( /** Start the credential updater task */ def start(): Unit = { val startTime = sparkConf.get(CREDENTIALS_UPDATE_TIME) - val remainingTime = Math.max(1, startTime - System.currentTimeMillis()) - logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime millis.") - credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS) + val remainingTime = startTime - System.currentTimeMillis() + if (remainingTime <= 0) { + credentialUpdater.schedule(credentialUpdaterRunnable, 1, TimeUnit.MINUTES) + } else { + logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime millis.") + credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS) + } } private def updateCredentialsIfRequired(): Unit = {