Disable cache backup/restore if cloudcache is used (#4125)

Signed-off-by: Ben Sherman <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Jul 25, 2023 · 46e828e · 46e828e
1 parent b14674d
commit 46e828e
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 4 deletions.
diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/CacheCommand.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/CacheCommand.groovy
@@ -46,7 +46,9 @@ class CacheCommand implements PluginAbstractExec {
 
     protected void cacheBackup() {
         log.debug "Running Nextflow cache backup"
-        new CacheManager(System.getenv()).saveCacheFiles()
+        final manager = new CacheManager(System.getenv())
+        manager.saveCacheFiles()
+        manager.saveMiscFiles()
     }
 
     protected void archiveLogs(Session sess) {

diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/CacheManager.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/CacheManager.groovy
@@ -65,7 +65,10 @@ class CacheManager {
         if( !sessionUuid )
             throw new AbortOperationException("Missing target uuid - cache sync cannot be performed")
 
-        this.localCachePath = Paths.get(".nextflow/cache/${sessionUuid}")
+        // ignore the `localCachePath` when the `NXF_CLOUDCACHE_PATH` variable is set because
+        // the nextflow cache metadata is going to be managed (and stored) via the nf-cloudcache plugin
+        if( !env.containsKey('NXF_CLOUDCACHE_PATH') )
+            this.localCachePath = Paths.get(".nextflow/cache/${sessionUuid}")
 
         if( env.NXF_OUT_FILE )
             localOutFile = Paths.get(env.NXF_OUT_FILE)
@@ -80,7 +83,7 @@ class CacheManager {
     }
 
     protected void restoreCacheFiles() {
-        if( !remoteWorkDir || !sessionUuid )
+        if( !remoteWorkDir || !sessionUuid || !localCachePath )
             return
 
         if(!Files.exists(remoteCachePath)) {
@@ -100,7 +103,7 @@ class CacheManager {
     }
 
     protected void saveCacheFiles() {
-        if( !remoteWorkDir || !sessionUuid )
+        if( !remoteWorkDir || !sessionUuid || !localCachePath )
             return
 
         if( !Files.exists(localCachePath) ) {
@@ -118,7 +121,9 @@ class CacheManager {
         catch (Throwable e) {
             log.warn "Failed to backup resume metadata to remote store path: ${remoteCachePath.toUriString()} — cause: ${e}", e
         }
+    }
 
+    protected void saveMiscFiles() {
         // — upload out file
         try {
             if( localOutFile?.exists() )

diff --git a/plugins/nf-tower/src/test/io/seqera/tower/plugin/CacheManagerTest.groovy b/plugins/nf-tower/src/test/io/seqera/tower/plugin/CacheManagerTest.groovy
@@ -78,6 +78,7 @@ class CacheManagerTest extends Specification {
         tower.localCachePath.resolve('db/yyy').text = 'data yyy'
         and:
         tower.saveCacheFiles()
+        tower.saveMiscFiles()
         then:
         tower.remoteCachePath.resolve('index-foo').text == 'index foo'
         tower.remoteCachePath.resolve('db/xxx').text == 'data xxx'
@@ -99,6 +100,7 @@ class CacheManagerTest extends Specification {
         tower.localCachePath.resolve('db/delta').text = 'data delta'
         and:
         tower.saveCacheFiles()
+        tower.saveMiscFiles()
         then:
         tower.remoteCachePath.resolve('index-bar').text == 'index bar'
         tower.remoteCachePath.resolve('db/alpha').text == 'data alpha'
@@ -154,4 +156,26 @@ class CacheManagerTest extends Specification {
         cleanup:
         folder?.deleteDir()
     }
+
+    def 'should not backup/restore cache if cloudcache is enabled' () {
+        given:
+        def ENV = [
+                NXF_UUID: 'uuid',
+                NXF_WORK: '/work',
+                NXF_CLOUDCACHE_PATH: 's3://my-bucket/cache'
+        ]
+        and:
+        def tower = new CacheManager(ENV)
+
+        when:
+        tower.saveCacheFiles()
+        then:
+        0 * tower.getRemoteCachePath()
+
+        when:
+        tower.restoreCacheFiles()
+        then:
+        0 * tower.getRemoteCachePath()
+
+    }
 }