From eaaeb3ded4cb7bf6293e06a5ddd00fb38e68b5aa Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 11 Nov 2024 12:19:54 +0100 Subject: [PATCH] Fix overlapping file lock exception (#5489) [ci fast] Signed-off-by: jorgee Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../groovy/nextflow/conda/CondaCache.groovy | 23 +++++++++++++------ .../nextflow/conda/CondaCacheTest.groovy | 10 ++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy index 7d1ff901f5..17c605f19b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy @@ -22,6 +22,7 @@ import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap +import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j @@ -32,7 +33,6 @@ import nextflow.file.FileMutex import nextflow.util.CacheHelper import nextflow.util.Duration import nextflow.util.Escape -import org.yaml.snakeyaml.Yaml /** * Handle Conda environment creation and caching * @@ -166,6 +166,18 @@ class CondaCache { str.endsWith('.txt') && !str.contains('\n') } + static protected String sipHash(CharSequence data) { + Hashing + .sipHash24() + .newHasher() + .putUnencodedChars(data) + .hash() + .toString() + } + + static protected String sipHash(Path path) { + sipHash(path.toAbsolutePath().normalize().toString()) + } /** * Get the path on the file system where store a Conda environment @@ -188,11 +200,8 @@ class CondaCache { try { final path = condaEnv as Path content = path.text - final yaml = (Map)new Yaml().load(content) - if( yaml.name ) - name = yaml.name - else - name = path.baseName + name = 'env-' + sipHash(path) + } catch( NoSuchFileException e ) { throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") @@ -205,7 +214,7 @@ class CondaCache { try { final path = condaEnv as Path content = path.text - name = path.baseName + name = 'env-' + sipHash(path) } catch( NoSuchFileException e ) { throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") diff --git a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy index 637ae5623a..9a8baf952c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy @@ -89,6 +89,7 @@ class CondaCacheTest extends Specification { def cache = Spy(CondaCache) def BASE = Paths.get('/conda/envs') def ENV = folder.resolve('foo.yml') + def hash = CondaCache.sipHash(ENV) ENV.text = ''' channels: - bioconda @@ -99,13 +100,12 @@ class CondaCacheTest extends Specification { - bwa=0.7.15 ''' .stripIndent(true) // https://issues.apache.org/jira/browse/GROOVY-9423 - when: def prefix = cache.condaPrefixPath(ENV.toString()) then: 1 * cache.isYamlFilePath(ENV.toString()) 1 * cache.getCacheDir() >> BASE - prefix.toString() == '/conda/envs/foo-9416240708c49c4e627414b46a743664' + prefix.toString() == "/conda/envs/env-${hash}-9416240708c49c4e627414b46a743664" cleanup: folder?.deleteDir() @@ -118,6 +118,7 @@ class CondaCacheTest extends Specification { def cache = Spy(CondaCache) def BASE = Paths.get('/conda/envs') def ENV = Files.createTempFile('test','.yml') + def hash = CondaCache.sipHash(ENV) ENV.text = ''' name: my-env-1.1 channels: @@ -135,7 +136,7 @@ class CondaCacheTest extends Specification { then: 1 * cache.isYamlFilePath(ENV.toString()) 1 * cache.getCacheDir() >> BASE - prefix.toString() == '/conda/envs/my-env-1.1-e7fafe40ca966397a2c0d9bed7181aa7' + prefix.toString() == "/conda/envs/env-${hash}-e7fafe40ca966397a2c0d9bed7181aa7" } @@ -146,6 +147,7 @@ class CondaCacheTest extends Specification { def cache = Spy(CondaCache) def BASE = Paths.get('/conda/envs') def ENV = folder.resolve('bar.txt') + def hash = CondaCache.sipHash(ENV) ENV.text = ''' star=2.5.4a bwa=0.7.15 @@ -159,7 +161,7 @@ class CondaCacheTest extends Specification { 1 * cache.isYamlFilePath(ENV.toString()) 1 * cache.isTextFilePath(ENV.toString()) 1 * cache.getCacheDir() >> BASE - prefix.toString() == '/conda/envs/bar-8a4aa7db8ddb8ce4eb4d450d4814a437' + prefix.toString() == "/conda/envs/env-${hash}-8a4aa7db8ddb8ce4eb4d450d4814a437" cleanup: folder?.deleteDir()