From 40670f7ef85b9276aeda091eb9e1064ddb0bf3a2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 9 Dec 2024 07:59:05 +0100 Subject: [PATCH 1/6] Bump groovy 4.0.24 missing deps Signed-off-by: Paolo Di Tommaso --- build.gradle | 4 ++-- modules/nextflow/build.gradle | 12 ++++++------ modules/nf-commons/build.gradle | 2 +- modules/nf-httpfs/build.gradle | 4 ++-- plugins/nf-amazon/build.gradle | 2 +- plugins/nf-azure/build.gradle | 2 +- plugins/nf-cloudcache/build.gradle | 2 +- plugins/nf-codecommit/build.gradle | 2 +- plugins/nf-console/build.gradle | 4 ++-- plugins/nf-google/build.gradle | 2 +- plugins/nf-tower/build.gradle | 2 +- plugins/nf-wave/build.gradle | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 4b4ee6e1f2..33c367f0e0 100644 --- a/build.gradle +++ b/build.gradle @@ -109,8 +109,8 @@ allprojects { // Documentation required libraries groovyDoc 'org.fusesource.jansi:jansi:2.4.0' - groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.23" - groovyDoc "org.apache.groovy:groovy-ant:4.0.23" + groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.24" + groovyDoc "org.apache.groovy:groovy-ant:4.0.24" } test { diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index eda3d3cc84..3ada8f7662 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -21,11 +21,11 @@ dependencies { api(project(':nf-commons')) api(project(':nf-httpfs')) api "org.apache.groovy:groovy:4.0.24" - api "org.apache.groovy:groovy-nio:4.0.23" - api "org.apache.groovy:groovy-xml:4.0.23" - api "org.apache.groovy:groovy-json:4.0.23" - api "org.apache.groovy:groovy-templates:4.0.23" - api "org.apache.groovy:groovy-yaml:4.0.23" + api "org.apache.groovy:groovy-nio:4.0.24" + api "org.apache.groovy:groovy-xml:4.0.24" + api "org.apache.groovy:groovy-json:4.0.24" + api "org.apache.groovy:groovy-templates:4.0.24" + api "org.apache.groovy:groovy-yaml:4.0.24" api "org.slf4j:jcl-over-slf4j:2.0.7" api "org.slf4j:jul-to-slf4j:2.0.7" api "org.slf4j:log4j-over-slf4j:2.0.7" @@ -53,7 +53,7 @@ dependencies { testImplementation 'org.subethamail:subethasmtp:3.1.7' // test configuration - testFixturesApi ("org.apache.groovy:groovy-test:4.0.23") { exclude group: 'org.apache.groovy' } + testFixturesApi ("org.apache.groovy:groovy-test:4.0.24") { exclude group: 'org.apache.groovy' } testFixturesApi ("org.objenesis:objenesis:3.4") testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17") testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy' } diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle index 453dee0680..951bf44f23 100644 --- a/modules/nf-commons/build.gradle +++ b/modules/nf-commons/build.gradle @@ -27,7 +27,7 @@ sourceSets { dependencies { api "ch.qos.logback:logback-classic:1.4.14" api "org.apache.groovy:groovy:4.0.24" - api "org.apache.groovy:groovy-nio:4.0.23" + api "org.apache.groovy:groovy-nio:4.0.24" api "commons-lang:commons-lang:2.6" api 'com.google.guava:guava:33.0.0-jre' api 'org.pf4j:pf4j:3.12.0' diff --git a/modules/nf-httpfs/build.gradle b/modules/nf-httpfs/build.gradle index 6b9d8cee37..29fc92922c 100644 --- a/modules/nf-httpfs/build.gradle +++ b/modules/nf-httpfs/build.gradle @@ -31,11 +31,11 @@ dependencies { api project(':nf-commons') api "ch.qos.logback:logback-classic:1.4.14" api "org.apache.groovy:groovy:4.0.24" - api "org.apache.groovy:groovy-nio:4.0.23" + api "org.apache.groovy:groovy-nio:4.0.24" api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' } /* testImplementation inherited from top gradle build file */ - testImplementation "org.apache.groovy:groovy-json:4.0.23" // needed by wiremock + testImplementation "org.apache.groovy:groovy-json:4.0.24" // needed by wiremock testImplementation ('com.github.tomakehurst:wiremock:1.57') { exclude module: 'groovy-all' } testImplementation ('com.github.tomjankes:wiremock-groovy:0.2.0') { exclude module: 'groovy-all' } diff --git a/plugins/nf-amazon/build.gradle b/plugins/nf-amazon/build.gradle index 82a3709305..e8a8c75eeb 100644 --- a/plugins/nf-amazon/build.gradle +++ b/plugins/nf-amazon/build.gradle @@ -60,5 +60,5 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-azure/build.gradle b/plugins/nf-azure/build.gradle index 79a4487ae2..2116355076 100644 --- a/plugins/nf-azure/build.gradle +++ b/plugins/nf-azure/build.gradle @@ -53,5 +53,5 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-cloudcache/build.gradle b/plugins/nf-cloudcache/build.gradle index 240f43f9e6..7cfa07de4b 100644 --- a/plugins/nf-cloudcache/build.gradle +++ b/plugins/nf-cloudcache/build.gradle @@ -36,6 +36,6 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-codecommit/build.gradle b/plugins/nf-codecommit/build.gradle index 4b8c519935..b0f16607e6 100644 --- a/plugins/nf-codecommit/build.gradle +++ b/plugins/nf-codecommit/build.gradle @@ -43,5 +43,5 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-console/build.gradle b/plugins/nf-console/build.gradle index 69be75d9eb..64184c2b80 100644 --- a/plugins/nf-console/build.gradle +++ b/plugins/nf-console/build.gradle @@ -38,13 +38,13 @@ dependencies { compileOnly 'org.pf4j:pf4j:3.12.0' api("org.apache.groovy:groovy-console:4.0.21-patch.2") { transitive=false } - api("org.apache.groovy:groovy-swing:4.0.23") { transitive=false } + api("org.apache.groovy:groovy-swing:4.0.24") { transitive=false } // this is required by 'groovy-console' api("com.github.javaparser:javaparser-core:3.25.8") testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-google/build.gradle b/plugins/nf-google/build.gradle index db999f398d..954d4963ea 100644 --- a/plugins/nf-google/build.gradle +++ b/plugins/nf-google/build.gradle @@ -47,7 +47,7 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } test { diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle index 88e24e525b..347d127762 100644 --- a/plugins/nf-tower/build.gradle +++ b/plugins/nf-tower/build.gradle @@ -38,5 +38,5 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle index b6a8367f0d..7f4f40d744 100644 --- a/plugins/nf-wave/build.gradle +++ b/plugins/nf-wave/build.gradle @@ -41,5 +41,5 @@ dependencies { testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.apache.groovy:groovy:4.0.24" - testImplementation "org.apache.groovy:groovy-nio:4.0.23" + testImplementation "org.apache.groovy:groovy-nio:4.0.24" } From 3d7146835b9ac1781fb24bb0aa12f9bffe80362c Mon Sep 17 00:00:00 2001 From: Tom Sellman Date: Mon, 9 Dec 2024 14:41:27 +0000 Subject: [PATCH 2/6] Bump jgit to version 7.1.0 (#5587) (otherwise the tests fail if you have git config setting `gpg.format=ssh`) Signed-off-by: Tom Sellman --- modules/nextflow/build.gradle | 2 +- .../nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 3ada8f7662..867d5e2c5a 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -39,7 +39,7 @@ dependencies { api "com.beust:jcommander:1.35" api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' } api('org.iq80.leveldb:leveldb:0.12') - api('org.eclipse.jgit:org.eclipse.jgit:6.10.0.202406032230-r') + api('org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r') api ('javax.activation:activation:1.1.1') api ('javax.mail:mail:1.4.7') api ('org.yaml:snakeyaml:2.2') diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index d0a66de911..a1b7d57683 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -833,7 +833,7 @@ class AssetManager { protected Map refToMap(Ref ref, Map remote) { final entry = new HashMap(2) - final peel = git.getRepository().peel(ref) + final peel = git.getRepository().getRefDatabase().peel(ref) final objId = peel.getPeeledObjectId() ?: peel.getObjectId() // the branch or tag name entry.name = shortenRefName(ref.name) @@ -867,7 +867,7 @@ class AssetManager { result << (name == current ? '*' : ' ') if( level ) { - def peel = git.getRepository().peel(ref) + def peel = git.getRepository().getRefDatabase().peel(ref) def obj = peel.getPeeledObjectId() ?: peel.getObjectId() result << ' ' result << formatObjectId(obj, level == 1) From 18f7de132c0342102f7160ce97b2e68dc6b6763f Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 9 Dec 2024 15:42:15 +0100 Subject: [PATCH 3/6] Fix Google Batch hang when internal error during scheduling (#5567) Signed-off-by: jorgee Signed-off-by: Jorge Ejarque Signed-off-by: Ben Sherman Co-authored-by: Paolo Di Tommaso Co-authored-by: Ben Sherman --- .../batch/GoogleBatchTaskHandler.groovy | 40 +++++++++++++++---- .../google/batch/client/BatchClient.groovy | 5 +++ .../batch/GoogleBatchTaskHandlerTest.groovy | 38 ++++++++++++++++++ 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy index 389f2247b6..c651e4fa4e 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy @@ -60,7 +60,9 @@ import nextflow.trace.TraceRecord @CompileStatic class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { - private static Pattern EXIT_CODE_REGEX = ~/exit code 500(\d\d)/ + private static final Pattern EXIT_CODE_REGEX = ~/exit code 500(\d\d)/ + + private static final Pattern BATCH_ERROR_REGEX = ~/Batch Error: code/ private GoogleBatchExecutor executor @@ -98,6 +100,11 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { private volatile long timestamp + /** + * A flag to indicate that the job has failed without launching any tasks + */ + private volatile boolean noTaskJobfailure + GoogleBatchTaskHandler(TaskRun task, GoogleBatchExecutor executor) { super(task) this.client = executor.getClient() @@ -445,9 +452,10 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { */ protected String getTaskState() { final tasks = client.listTasks(jobId) - if( !tasks.iterator().hasNext() ) - return 'PENDING' - + if( !tasks.iterator().hasNext() ) { + // if there are no tasks checks the job status + return checkJobStatus() + } final now = System.currentTimeMillis() final delta = now - timestamp; if( !taskState || delta >= 1_000) { @@ -468,6 +476,21 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { return taskState } + protected String checkJobStatus() { + final jobStatus = client.getJobStatus(jobId) + final newState = jobStatus?.state as String + if (newState) { + taskState = newState + timestamp = System.currentTimeMillis() + if (newState == "FAILED") { + noTaskJobfailure = true + } + return taskState + } else { + return "PENDING" + } + } + static private final List RUNNING_OR_COMPLETED = ['RUNNING', 'SUCCEEDED', 'FAILED'] static private final List COMPLETED = ['SUCCEEDED', 'FAILED'] @@ -510,13 +533,14 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { protected Throwable getJobError() { try { - final status = client.getTaskStatus(jobId, taskId) - final eventsCount = status.getStatusEventsCount() - final lastEvent = eventsCount > 0 ? status.getStatusEvents(eventsCount - 1) : null + final events = noTaskJobfailure + ? client.getJobStatus(jobId).getStatusEventsList() + : client.getTaskStatus(jobId, taskId).getStatusEventsList() + final lastEvent = events?.get(events.size() - 1) log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - last event: ${lastEvent}; exit code: ${lastEvent?.taskExecution?.exitCode}" final error = lastEvent?.description - if( error && EXIT_CODE_REGEX.matcher(error).find() ) { + if( error && (EXIT_CODE_REGEX.matcher(error).find() || BATCH_ERROR_REGEX.matcher(error).find()) ) { return new ProcessException(error) } } diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchClient.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchClient.groovy index 405b249d9b..d85d42b581 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchClient.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchClient.groovy @@ -29,6 +29,7 @@ import com.google.cloud.batch.v1.BatchServiceClient import com.google.cloud.batch.v1.BatchServiceSettings import com.google.cloud.batch.v1.Job import com.google.cloud.batch.v1.JobName +import com.google.cloud.batch.v1.JobStatus import com.google.cloud.batch.v1.LocationName import com.google.cloud.batch.v1.Task import com.google.cloud.batch.v1.TaskGroupName @@ -123,6 +124,10 @@ class BatchClient { return describeTask(jobId, taskId).getStatus() } + JobStatus getJobStatus(String jobId) { + return describeJob(jobId).getStatus() + } + String getTaskState(String jobId, String taskId) { final status = getTaskStatus(jobId, taskId) return status ? status.getState().toString() : null diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy index ba21a84588..43150d17fd 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy @@ -17,6 +17,9 @@ package nextflow.cloud.google.batch +import com.google.cloud.batch.v1.JobStatus +import com.google.cloud.batch.v1.Task + import java.nio.file.Path import com.google.cloud.batch.v1.GCS @@ -581,4 +584,39 @@ class GoogleBatchTaskHandlerTest extends Specification { and: 0 * client.deleteJob('job1') >> null } + + JobStatus makeJobStatus(JobStatus.State state, String desc = null) { + final builder = JobStatus.newBuilder().setState(state) + if( desc ) { + builder.addStatusEvents( + StatusEvent.newBuilder() + .setDescription(desc) + ) + } + builder.build() + } + + def 'should check job status when no tasks in job '() { + + given: + def jobId = 'job-id' + def taskId = 'task-id' + def client = Mock(BatchClient) + def task = Mock(TaskRun) { + lazyName() >> 'foo (1)' + } + def handler = Spy(new GoogleBatchTaskHandler(jobId: jobId, taskId: taskId, client: client, task: task)) + final message = 'Job failed when Batch tries to schedule it: Batch Error: code - CODE_MACHINE_TYPE_NOT_FOUND' + when: + client.listTasks(jobId) >>> [new LinkedList(), new LinkedList()] + client.getJobStatus(jobId) >>> [ + null, + makeJobStatus(JobStatus.State.FAILED, 'Scheduling Failed'), + makeJobStatus(JobStatus.State.FAILED, message) + ] + then: + handler.getTaskState() == "PENDING" + handler.getTaskState() == "FAILED" + handler.getJobError().message == message + } } From c6434d46d619a82ec4918313aa862d5c40368635 Mon Sep 17 00:00:00 2001 From: Nico Date: Mon, 9 Dec 2024 08:06:58 -0800 Subject: [PATCH 4/6] Use printf instead of echo to write trace file (#5397) Echo commands were combined into a single block to write to $trace_file more efficiently, reducing the number of system calls. Added error handling to prevent tracing from causing workflow failures. Signed-off-by: Nico Holguin Signed-off-by: Paolo Di Tommaso Co-authored-by: nholguin Co-authored-by: Paolo Di Tommaso --- .../nextflow/executor/command-trace.txt | 57 ++++++++++--------- .../executor/test-bash-wrapper-with-trace.txt | 57 ++++++++++--------- 2 files changed, 60 insertions(+), 54 deletions(-) diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt index 8b5b2195f6..cf9d0b5d77 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt @@ -114,26 +114,28 @@ nxf_mem_watch() { done ## result struct: pid %mem vmem rss peak_vmem peak_rss - echo "%mem=${nxf_stat_ret[1]}" >> $trace_file - echo "vmem=${nxf_stat_ret[2]}" >> $trace_file - echo "rss=${nxf_stat_ret[3]}" >> $trace_file - echo "peak_vmem=${nxf_stat_ret[4]}" >> $trace_file - echo "peak_rss=${nxf_stat_ret[5]}" >> $trace_file - echo "vol_ctxt=${nxf_stat_ret[6]}" >> $trace_file - echo "inv_ctxt=${nxf_stat_ret[7]}" >> $trace_file + printf "%s\n" \ + "%mem=${nxf_stat_ret[1]}" \ + "vmem=${nxf_stat_ret[2]}" \ + "rss=${nxf_stat_ret[3]}" \ + "peak_vmem=${nxf_stat_ret[4]}" \ + "peak_rss=${nxf_stat_ret[5]}" \ + "vol_ctxt=${nxf_stat_ret[6]}" \ + "inv_ctxt=${nxf_stat_ret[7]}" >> "$trace_file" || >&2 echo "Error: Failed to append to file: $trace_file" } nxf_write_trace() { - echo "nextflow.trace/v2" > $trace_file - echo "realtime=$wall_time" >> $trace_file - echo "%cpu=$ucpu" >> $trace_file - echo "cpu_model=$cpu_model" >> $trace_file - echo "rchar=${io_stat1[0]}" >> $trace_file - echo "wchar=${io_stat1[1]}" >> $trace_file - echo "syscr=${io_stat1[2]}" >> $trace_file - echo "syscw=${io_stat1[3]}" >> $trace_file - echo "read_bytes=${io_stat1[4]}" >> $trace_file - echo "write_bytes=${io_stat1[5]}" >> $trace_file + printf "%s\n" \ + "nextflow.trace/v2" \ + "realtime=$wall_time" \ + "%cpu=$ucpu" \ + "cpu_model=$cpu_model" \ + "rchar=${io_stat1[0]}" \ + "wchar=${io_stat1[1]}" \ + "syscr=${io_stat1[2]}" \ + "syscw=${io_stat1[3]}" \ + "read_bytes=${io_stat1[4]}" \ + "write_bytes=${io_stat1[5]}" > "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" } nxf_trace_mac() { @@ -199,16 +201,17 @@ nxf_trace_linux() { local wall_time=$((end_millis-start_millis)) [ $NXF_DEBUG = 1 ] && echo "+++ STATS %CPU=$ucpu TIME=$wall_time I/O=${io_stat1[*]}" - echo "nextflow.trace/v2" > $trace_file - echo "realtime=$wall_time" >> $trace_file - echo "%cpu=$ucpu" >> $trace_file - echo "cpu_model=$cpu_model" >> $trace_file - echo "rchar=${io_stat1[0]}" >> $trace_file - echo "wchar=${io_stat1[1]}" >> $trace_file - echo "syscr=${io_stat1[2]}" >> $trace_file - echo "syscw=${io_stat1[3]}" >> $trace_file - echo "read_bytes=${io_stat1[4]}" >> $trace_file - echo "write_bytes=${io_stat1[5]}" >> $trace_file + printf "%s\n" \ + "nextflow.trace/v2" \ + "realtime=$wall_time" \ + "%cpu=$ucpu" \ + "cpu_model=$cpu_model" \ + "rchar=${io_stat1[0]}" \ + "wchar=${io_stat1[1]}" \ + "syscr=${io_stat1[2]}" \ + "syscw=${io_stat1[3]}" \ + "read_bytes=${io_stat1[4]}" \ + "write_bytes=${io_stat1[5]}" > "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" ## join nxf_mem_watch [ -e /proc/$mem_proc ] && eval "echo 'DONE' >&$mem_fd" || true diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt index 1de9614e11..5aef2f4795 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt @@ -98,26 +98,28 @@ nxf_mem_watch() { count=$((count+1)) done - echo "%mem=${nxf_stat_ret[1]}" >> $trace_file - echo "vmem=${nxf_stat_ret[2]}" >> $trace_file - echo "rss=${nxf_stat_ret[3]}" >> $trace_file - echo "peak_vmem=${nxf_stat_ret[4]}" >> $trace_file - echo "peak_rss=${nxf_stat_ret[5]}" >> $trace_file - echo "vol_ctxt=${nxf_stat_ret[6]}" >> $trace_file - echo "inv_ctxt=${nxf_stat_ret[7]}" >> $trace_file + printf "%s\n" \ + "%mem=${nxf_stat_ret[1]}" \ + "vmem=${nxf_stat_ret[2]}" \ + "rss=${nxf_stat_ret[3]}" \ + "peak_vmem=${nxf_stat_ret[4]}" \ + "peak_rss=${nxf_stat_ret[5]}" \ + "vol_ctxt=${nxf_stat_ret[6]}" \ + "inv_ctxt=${nxf_stat_ret[7]}" >> "$trace_file" || >&2 echo "Error: Failed to append to file: $trace_file" } nxf_write_trace() { - echo "nextflow.trace/v2" > $trace_file - echo "realtime=$wall_time" >> $trace_file - echo "%cpu=$ucpu" >> $trace_file - echo "cpu_model=$cpu_model" >> $trace_file - echo "rchar=${io_stat1[0]}" >> $trace_file - echo "wchar=${io_stat1[1]}" >> $trace_file - echo "syscr=${io_stat1[2]}" >> $trace_file - echo "syscw=${io_stat1[3]}" >> $trace_file - echo "read_bytes=${io_stat1[4]}" >> $trace_file - echo "write_bytes=${io_stat1[5]}" >> $trace_file + printf "%s\n" \ + "nextflow.trace/v2" \ + "realtime=$wall_time" \ + "%cpu=$ucpu" \ + "cpu_model=$cpu_model" \ + "rchar=${io_stat1[0]}" \ + "wchar=${io_stat1[1]}" \ + "syscr=${io_stat1[2]}" \ + "syscw=${io_stat1[3]}" \ + "read_bytes=${io_stat1[4]}" \ + "write_bytes=${io_stat1[5]}" > "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" } nxf_trace_mac() { @@ -173,16 +175,17 @@ nxf_trace_linux() { local wall_time=$((end_millis-start_millis)) [ $NXF_DEBUG = 1 ] && echo "+++ STATS %CPU=$ucpu TIME=$wall_time I/O=${io_stat1[*]}" - echo "nextflow.trace/v2" > $trace_file - echo "realtime=$wall_time" >> $trace_file - echo "%cpu=$ucpu" >> $trace_file - echo "cpu_model=$cpu_model" >> $trace_file - echo "rchar=${io_stat1[0]}" >> $trace_file - echo "wchar=${io_stat1[1]}" >> $trace_file - echo "syscr=${io_stat1[2]}" >> $trace_file - echo "syscw=${io_stat1[3]}" >> $trace_file - echo "read_bytes=${io_stat1[4]}" >> $trace_file - echo "write_bytes=${io_stat1[5]}" >> $trace_file + printf "%s\n" \ + "nextflow.trace/v2" \ + "realtime=$wall_time" \ + "%cpu=$ucpu" \ + "cpu_model=$cpu_model" \ + "rchar=${io_stat1[0]}" \ + "wchar=${io_stat1[1]}" \ + "syscr=${io_stat1[2]}" \ + "syscw=${io_stat1[3]}" \ + "read_bytes=${io_stat1[4]}" \ + "write_bytes=${io_stat1[5]}" > "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" [ -e /proc/$mem_proc ] && eval "echo 'DONE' >&$mem_fd" || true wait $mem_proc 2>/dev/null || true From 335701e3ee256b678e8949987ce2ea7335b50623 Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 9 Dec 2024 21:02:33 +0100 Subject: [PATCH 5/6] Fixing getRepositoryUrl when project name is not provided (#5560) Signed-off-by: jorgee Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../main/groovy/nextflow/scm/AzureRepositoryProvider.groovy | 4 +++- .../groovy/nextflow/scm/AzureRepositoryProviderTest.groovy | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy index a8cb617bf9..8435099ebf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy @@ -35,6 +35,7 @@ final class AzureRepositoryProvider extends RepositoryProvider { private String user private String repo + private String urlPath; private String continuationToken AzureRepositoryProvider(String project, ProviderConfig config=null) { @@ -42,6 +43,7 @@ final class AzureRepositoryProvider extends RepositoryProvider { Azure repo format follows Organization/Project/Repository where Project can be optional If Project is not present then Repository is used as Project (and also as Repository) */ + this.urlPath = project def tokens = project.tokenize('/') this.repo = tokens.removeLast() if( tokens.size() == 1){ @@ -164,7 +166,7 @@ final class AzureRepositoryProvider extends RepositoryProvider { /** {@inheritDoc} */ @Override String getRepositoryUrl() { - "${config.server}/$project" + "${config.server}/${urlPath}" } /** {@inheritDoc} */ diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy index 00e7cc74f0..decec42c1d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy @@ -80,7 +80,7 @@ class AzureRepositoryProviderTest extends Specification { def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject) expect: - new AzureRepositoryProvider('ORGANIZATION/PROJECT/hello', obj).getRepositoryUrl() == 'https://dev.azure.com/ORGANIZATION/PROJECT' + new AzureRepositoryProvider('ORGANIZATION/PROJECT/hello', obj).getRepositoryUrl() == 'https://dev.azure.com/ORGANIZATION/PROJECT/hello' } From b23e42cd6dea0e4ce6bba19c13968546807be5f1 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Tue, 10 Dec 2024 04:40:27 -0600 Subject: [PATCH 6/6] Fix docs for `csi` option in `pod` directive (#5585) [ci skip] Signed-off-by: Ben Sherman --- docs/reference/process.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/process.md b/docs/reference/process.md index 95a9456ef8..7383c8d5eb 100644 --- a/docs/reference/process.md +++ b/docs/reference/process.md @@ -1078,11 +1078,11 @@ The following options are available: : *Can be specified multiple times* : Mounts a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) with name and optional key to the given path. If the key is omitted, the path is interpreted as a directory and all entries in the `ConfigMap` are exposed in that path. -`csi: '', mountPath: ''` +`csi: '', mountPath: ''` : :::{versionadded} 22.11.0-edge ::: : *Can be specified multiple times* -: Mounts a [CSI ephemeral volume](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes) by name to the given path. +: Mounts a [CSI ephemeral volume](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes) with the given configuration to the given path. `emptyDir: , mountPath: ''` : :::{versionadded} 22.11.0-edge