S3 uploader + executor cleanup efficiency improvements

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java

@@ -72,26 +72,25 @@ public TaskCleanupResult cleanup(boolean cleanupTaskAppDirectory, boolean cleanu
       return finishTaskCleanup(dockerCleanSuccess);
     }
 
-
-    if (!cleanupTaskAppDirectory) {
-      log.info("Not finishing cleanup because taskApp directory is being preserved");
-      return TaskCleanupResult.WAITING;
-    }
-
-    boolean cleanupTaskAppDirectorySuccess = cleanupTaskAppDirectory();
     boolean logTearDownSuccess = taskLogManager.teardown();
-
-    log.info("Cleaned up task app directory ({}) and rotated logs ({})", cleanupTaskAppDirectorySuccess, logTearDownSuccess);
+    log.info("Rotated and marked logs for upload for {} ({})", taskDirectory, logTearDownSuccess);
 
     if (!cleanupLogs) {
-      log.info("Not finishing cleanup because log files will be preserved for 15 minutes after task termination");
+      log.debug("Not finishing cleanup because log files will be preserved for 15 minutes after task termination");
       return TaskCleanupResult.WAITING;
     }
 
     boolean rotatedLogfileDeleteSuccess = checkForLogrotateAdditionalFilesToDelete(taskDefinition);
-
     log.info("Deleted rotated logfiles ({})", rotatedLogfileDeleteSuccess);
 
+    if (!cleanupTaskAppDirectory) {
+      log.debug("Not finishing cleanup because taskApp directory is being preserved");
+      return TaskCleanupResult.WAITING;
+    }
+
+    boolean cleanupTaskAppDirectorySuccess = cleanupTaskAppDirectory();
+    log.info("Cleaned up task app directory ({})", cleanupTaskAppDirectorySuccess);
+
     if (logTearDownSuccess && cleanupTaskAppDirectorySuccess) {
       return finishTaskCleanup(dockerCleanSuccess);
     } else {
@@ -112,12 +111,12 @@ private TaskCleanupResult finishTaskCleanup(boolean dockerCleanSuccess) {
   public boolean cleanTaskDefinitionFile() {
     Path taskDefinitionPath = configuration.getTaskDefinitionPath(taskDefinition.getTaskId());
 
-    log.info("Successful cleanup, deleting file {}", taskDefinitionPath);
+    log.debug("Successful cleanup, deleting file {}", taskDefinitionPath);
 
     try {
       boolean deleted = Files.deleteIfExists(taskDefinitionPath);
 
-      log.info("File deleted ({})", deleted);
+      log.debug("File deleted ({})", deleted);
 
       return true;
     } catch (IOException e) {
@@ -129,7 +128,7 @@ public boolean cleanTaskDefinitionFile() {
   private boolean cleanupTaskAppDirectory() {
     final String pathToDelete = taskDefinition.getTaskAppDirectory();
 
-    log.info("Deleting: {}", pathToDelete);
+    log.debug("Deleting: {}", pathToDelete);
 
     try {
       final List<String> cmd = ImmutableList.of(

SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/JsonObjectFileHelper.java

@@ -30,7 +30,7 @@ public JsonObjectFileHelper(ObjectMapper objectMapper) {
   public <T> Optional<T> read(Path file, Logger log, Class<T> clazz) throws IOException {
     final long start = System.currentTimeMillis();
 
-    log.info("Reading {}", file);
+    log.trace("Reading {}", file);
 
     byte[] bytes = new byte[0];
 

SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/WatchServiceHelper.java

@@ -111,7 +111,7 @@ private void processWatchKey(WatchKey watchKey) throws IOException {
       WatchEvent.Kind<?> kind = event.kind();
 
       if (!watchEvents.contains(kind)) {
-        LOG.trace("Ignoring an {} event to {}", event.context());
+        LOG.trace("Ignoring an {} event to {}", event.kind(), event.context());
         continue;
       }
 

SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/server/SingularityS3DownloaderAsyncHandler.java

@@ -47,10 +47,10 @@ public S3Artifact getS3Artifact() {
   }
 
   private boolean download() throws Exception {
-    LOG.info("Beginning download {} after {}", artifactDownloadRequest, JavaUtils.duration(start));
+    LOG.trace("Beginning download {} after {}", artifactDownloadRequest, JavaUtils.duration(start));
 
     if (continuation.isExpired()) {
-      LOG.info("Continuation expired for {}, aborting...", artifactDownloadRequest.getTargetDirectory());
+      LOG.trace("Continuation expired for {}, aborting...", artifactDownloadRequest.getTargetDirectory());
       return false;
     }
 
@@ -71,7 +71,7 @@ private boolean download() throws Exception {
       artifactManager.copy(fetched, targetDirectory, artifactDownloadRequest.getS3Artifact().getFilename());
     }
 
-    LOG.info("Finishing request {} after {}", artifactDownloadRequest.getTargetDirectory(), JavaUtils.duration(start));
+    LOG.debug("Finishing request {} after {}", artifactDownloadRequest.getTargetDirectory(), JavaUtils.duration(start));
 
     getResponse().getOutputStream().close();
 

SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/server/SingularityS3DownloaderCoordinator.java

@@ -111,7 +111,7 @@ private boolean addDownloadRequest() {
         return false;
       }
 
-      LOG.info("Queing new downloader for {} ({} handlers, {} active threads, {} queue size, {} max) after {}", artifactDownloadRequest, downloadRequestToHandler.size(),
+      LOG.trace("Queing new downloader for {} ({} handlers, {} active threads, {} queue size, {} max) after {}", artifactDownloadRequest, downloadRequestToHandler.size(),
           downloadService.getActiveCount(), downloadService.getQueue().size(), configuration.getNumDownloaderThreads(), JavaUtils.duration(start));
 
       ListenableFuture<?> future = listeningDownloadWrapper.submit(newHandler);

SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java

@@ -306,7 +306,7 @@ private int checkUploads() {
       futures.put(uploader, CompletableFuture.supplyAsync(performUploadSupplier(uploader, isFinished, false), executorService));
     }
 
-    LOG.info("Waiting on {} future(s)", futures.size());
+    LOG.debug("Waiting on {} future(s)", futures.size());
 
     final long now = System.currentTimeMillis();
     final Set<SingularityUploader> expiredUploaders = Sets.newHashSetWithExpectedSize(initialExpectedSize);

SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityUploader.java

@@ -1,12 +1,14 @@
 package com.hubspot.singularity.s3uploader;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.Paths;
@@ -114,7 +116,7 @@ int uploadBatch(List<Path> toUpload) {
           context.stop();
         }
       } else {
-        LOG.info("{} is in use by another process, will retry upload later", file);
+        LOG.debug("{} is in use by another process, will retry upload later", file);
       }
     }
 
@@ -156,6 +158,8 @@ List<Path> filesToUpload(boolean isFinished) throws IOException {
           throw new RuntimeException(ioe);
         }
       });
+    } catch (UncheckedIOException|NoSuchFileException nsfe) {
+      LOG.debug("Parent file {} did not exist, skipping", directory);
     }
     return toUpload;
   }