From 2486a5a7f09a4961fdc01847c51ac02a3db6d622 Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Mon, 19 Aug 2024 16:32:25 +0800 Subject: [PATCH 1/2] [SPARK-49300][CORE]Fix Hadoop delegation token leak when tokenRenewalInterval is not set. --- .../spark/deploy/security/HadoopFSDelegationTokenProvider.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala index b47f9e5a43af..50756fd69953 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala @@ -150,6 +150,7 @@ private[deploy] class HadoopFSDelegationTokenProvider val interval = newExpiration - getIssueDate(tokenKind, identifier) logInfo(log"Renewal interval is ${MDC(TOTAL_TIME, interval)} for" + log" token ${MDC(TOKEN_KIND, tokenKind)}") + token.cancel(hadoopConf) interval }.toOption } From 44040408d572c3eb26dc9c4e86c1c8e7aa6cb672 Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Wed, 21 Aug 2024 11:02:39 +0800 Subject: [PATCH 2/2] Add some comments. --- .../spark/deploy/security/HadoopFSDelegationTokenProvider.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala index 50756fd69953..fc750b54d0b8 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala @@ -150,6 +150,8 @@ private[deploy] class HadoopFSDelegationTokenProvider val interval = newExpiration - getIssueDate(tokenKind, identifier) logInfo(log"Renewal interval is ${MDC(TOTAL_TIME, interval)} for" + log" token ${MDC(TOKEN_KIND, tokenKind)}") + // The token here is only used to obtain renewal intervals. We should cancel it in + // a timely manner to avoid causing additional pressure on the server. token.cancel(hadoopConf) interval }.toOption