From 5d198bfcde1a789b42b2125582c0b551fc48896c Mon Sep 17 00:00:00 2001 From: liujianhui Date: Sat, 24 Dec 2016 14:56:04 +0800 Subject: [PATCH 1/2] [SPARK-18994] fix bug --- .../scala/org/apache/spark/deploy/worker/Worker.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index f41efb097b4b..094189dcfcac 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -570,9 +570,11 @@ private[deploy] class Worker( finishedApps -= id appDirectories.remove(id).foreach { dirList => logInfo(s"Cleaning up local directories for application $id") - dirList.foreach { dir => - Utils.deleteRecursively(new File(dir)) - } + concurrent.future { + dirList.foreach { dir => + Utils.deleteRecursively(new File(dir)) + } + }(cleanupThreadExecutor) } shuffleService.applicationRemoved(id) } From d9e6f0167ddfccbacffbb0f05277b947708bf603 Mon Sep 17 00:00:00 2001 From: liujianhuiouc Date: Mon, 26 Dec 2016 17:26:15 +0800 Subject: [PATCH 2/2] [spark-18994] add log if remove failure --- .../src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 094189dcfcac..ee679e4e4cdc 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -574,6 +574,9 @@ private[deploy] class Worker( dirList.foreach { dir => Utils.deleteRecursively(new File(dir)) } + }(cleanupThreadExecutor).onFailure { + case e: Throwable => + logError("App dir cleanup failed: " + e.getMessage, e) }(cleanupThreadExecutor) } shuffleService.applicationRemoved(id)