From ad588c0866a10639b716abca5798e8ca2e135e1a Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Fri, 12 Jul 2024 05:41:47 -0500 Subject: [PATCH 1/2] Prevent default output/error logs for job arrays Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/executor/CrgExecutor.groovy | 7 +++++-- .../src/main/groovy/nextflow/executor/LsfExecutor.groovy | 5 ++++- .../src/main/groovy/nextflow/executor/PbsExecutor.groovy | 7 +++++-- .../main/groovy/nextflow/executor/PbsProExecutor.groovy | 7 +++++-- .../src/main/groovy/nextflow/executor/SgeExecutor.groovy | 7 +++++-- .../src/main/groovy/nextflow/executor/SlurmExecutor.groovy | 7 +++++-- .../test/groovy/nextflow/executor/CrgExecutorTest.groovy | 2 ++ .../test/groovy/nextflow/executor/LsfExecutorTest.groovy | 1 + .../test/groovy/nextflow/executor/PbsExecutorTest.groovy | 2 ++ .../groovy/nextflow/executor/PbsProExecutorTest.groovy | 2 ++ .../test/groovy/nextflow/executor/SgeExecutorTest.groovy | 2 ++ .../test/groovy/nextflow/executor/SlurmExecutorTest.groovy | 1 + 12 files changed, 39 insertions(+), 11 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy index 53b2ca92b1..b3cc7e1b16 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy @@ -49,10 +49,13 @@ class CrgExecutor extends SgeExecutor { result << '-N' << getJobNameFor(task) - if( task !instanceof TaskArrayRun ) { + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'y' } + result << '-j' << 'y' result << '-terse' << '' // note: directive need to be returned as pairs diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy index 775bf6dbdd..35aeb09bd8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy @@ -70,7 +70,10 @@ class LsfExecutor extends AbstractGridExecutor implements TaskArrayExecutor { */ protected List getDirectives(TaskRun task, List result) { - if( task !instanceof TaskArrayRun ) { + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString() } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy index 6f7bc7d9b4..f562afa2b9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy @@ -51,10 +51,13 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-N' << getJobNameFor(task) - if( task !instanceof TaskArrayRun ) { + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'oe' } + result << '-j' << 'oe' // the requested queue name if( task.config.queue ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy index 1672ca200d..3126e0989f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy @@ -58,10 +58,13 @@ class PbsProExecutor extends PbsExecutor { result << '-N' << getJobNameFor(task) - if( task !instanceof TaskArrayRun ) { + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'oe' } + result << '-j' << 'oe' // the requested queue name if( task.config.queue ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy index 1f1c2a6304..a44a0da429 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy @@ -45,10 +45,13 @@ class SgeExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-N' << getJobNameFor(task) - if( task !instanceof TaskArrayRun ) { + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'y' } + result << '-j' << 'y' result << '-terse' << '' // note: directive need to be returned as pairs diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy index 8210f24c8b..0e3461c1ac 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy @@ -62,8 +62,11 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-J' << getJobNameFor(task) - if( task !instanceof TaskArrayRun ) { - // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE + // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE + if( task instanceof TaskArrayRun ) { + result << '-o' << '/dev/null' + } + else { result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy index 6036be4980..f683bec657 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy @@ -351,6 +351,8 @@ class CrgExecutorTest extends Specification { executor.getHeaders(task) == ''' #$ -t 1-5 #$ -N nf-mapping_tag + #$ -o /dev/null + #$ -j y #$ -terse #$ -notify ''' diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy index 8e6baf7829..d66adf5d13 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy @@ -324,6 +324,7 @@ class LsfExecutorTest extends Specification { } then: executor.getHeaders(taskArray) == ''' + #BSUB -o /dev/null #BSUB -J "nf-mapping_hola[1-5]" ''' .stripIndent().leftTrim() diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy index 7ac672acc8..b279c7eee2 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy @@ -174,6 +174,8 @@ class PbsExecutorTest extends Specification { executor.getHeaders(taskArray) == ''' #PBS -J 0-4 #PBS -N nf-task_name + #PBS -o /dev/null + #PBS -j oe ''' .stripIndent().leftTrim() } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy index f4828dff71..b9b4ec164b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy @@ -201,6 +201,8 @@ class PbsProExecutorTest extends Specification { executor.getDirectives(task, []) == [ '-J', '0-4', '-N', 'nf-foo', + '-o', '/dev/null', + '-j', 'oe' ] } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy index 7466c83885..095ecfbf72 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy @@ -207,6 +207,8 @@ class SgeExecutorTest extends Specification { executor.getHeaders(taskArray) == ''' #$ -t 1-5 #$ -N nf-the_task_name + #$ -o /dev/null + #$ -j y #$ -terse #$ -notify ''' diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy index d1d9382d0a..4e2147eca3 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy @@ -230,6 +230,7 @@ class SlurmExecutorTest extends Specification { executor.getHeaders(taskArray) == ''' #SBATCH --array 0-4 #SBATCH -J nf-the_task_name + #SBATCH -o /dev/null #SBATCH --no-requeue #SBATCH --signal B:USR2@30 ''' From 7b8b2b0370884b6aa0203f7ca8cb22161e5c6ba7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 5 Aug 2024 12:58:16 +0200 Subject: [PATCH 2/2] Made more concise [ci fast] Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/executor/CrgExecutor.groovy | 7 +------ .../src/main/groovy/nextflow/executor/LsfExecutor.groovy | 7 +------ .../src/main/groovy/nextflow/executor/PbsExecutor.groovy | 7 +------ .../main/groovy/nextflow/executor/PbsProExecutor.groovy | 7 +------ .../src/main/groovy/nextflow/executor/SgeExecutor.groovy | 7 +------ .../src/main/groovy/nextflow/executor/SlurmExecutor.groovy | 7 +------ .../src/main/groovy/nextflow/processor/TaskArrayRun.groovy | 5 +++++ .../src/main/groovy/nextflow/processor/TaskRun.groovy | 4 ++++ .../test/groovy/nextflow/processor/TaskArrayRunTest.groovy | 7 +++++++ .../src/test/groovy/nextflow/processor/TaskRunTest.groovy | 6 ++++++ 10 files changed, 28 insertions(+), 36 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy index b3cc7e1b16..4040d6a970 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy @@ -49,12 +49,7 @@ class CrgExecutor extends SgeExecutor { result << '-N' << getJobNameFor(task) - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - } + result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG))) result << '-j' << 'y' result << '-terse' << '' // note: directive need to be returned as pairs diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy index 35aeb09bd8..e5f425a2ac 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy @@ -70,12 +70,7 @@ class LsfExecutor extends AbstractGridExecutor implements TaskArrayExecutor { */ protected List getDirectives(TaskRun task, List result) { - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString() - } + result << '-o' << (task.isArray() ? '/dev/null' : task.workDir.resolve(TaskRun.CMD_LOG).toString()) // add other parameters (if any) if( task.config.queue ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy index f562afa2b9..8e968f99c7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy @@ -51,12 +51,7 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-N' << getJobNameFor(task) - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - } + result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG))) result << '-j' << 'oe' // the requested queue name diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy index 3126e0989f..c171657224 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy @@ -58,12 +58,7 @@ class PbsProExecutor extends PbsExecutor { result << '-N' << getJobNameFor(task) - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - } + result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG))) result << '-j' << 'oe' // the requested queue name diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy index a44a0da429..a04633b8ec 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy @@ -45,12 +45,7 @@ class SgeExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-N' << getJobNameFor(task) - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - } + result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG))) result << '-j' << 'y' result << '-terse' << '' // note: directive need to be returned as pairs diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy index 0e3461c1ac..08d7e08576 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy @@ -63,12 +63,7 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor { result << '-J' << getJobNameFor(task) // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE - if( task instanceof TaskArrayRun ) { - result << '-o' << '/dev/null' - } - else { - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - } + result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG))) result << '--no-requeue' << '' // note: directive need to be returned as pairs diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy index 6d03af83f0..9621386d1c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy @@ -52,4 +52,9 @@ class TaskArrayRun extends TaskRun { return false } + @Override + final boolean isArray() { + return true + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 2189ef39d1..52670a3a1e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -734,6 +734,10 @@ class TaskRun implements Cloneable { return processor.executor?.isContainerNative() ?: false } + boolean isArray() { + return false + } + boolean isContainerEnabled() { return getContainerConfig().isEnabled() && getContainer()!=null } diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy index e64af32035..2b65335d8c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy @@ -46,4 +46,11 @@ class TaskArrayRunTest extends Specification { info == new ContainerInfo('ubuntu','ubuntu','ubuntu') } + def 'should be an array' () { + given: + def task = new TaskArrayRun() + expect: + task.isArray() + } + } diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy index 3fcaac0206..9f0cd50875 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy @@ -885,4 +885,10 @@ class TaskRunTest extends Specification { info == new ContainerInfo('ubuntu','ubuntu','ubuntu') } + def 'should not be an array' () { + given: + def task = new TaskRun() + expect: + !task.isArray() + } }