Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prevent default output/error logs for job arrays #5152

Merged
merged 3 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,8 @@ class CrgExecutor extends SgeExecutor {

result << '-N' << getJobNameFor(task)

if( task !instanceof TaskArrayRun ) {
result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
result << '-j' << 'y'
}
result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
result << '-j' << 'y'

result << '-terse' << '' // note: directive need to be returned as pairs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,7 @@ class LsfExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
*/
protected List<String> getDirectives(TaskRun task, List<String> result) {

if( task !instanceof TaskArrayRun ) {
result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString()
}
result << '-o' << (task.isArray() ? '/dev/null' : task.workDir.resolve(TaskRun.CMD_LOG).toString())

// add other parameters (if any)
if( task.config.queue ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,8 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor {

result << '-N' << getJobNameFor(task)

if( task !instanceof TaskArrayRun ) {
result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
result << '-j' << 'oe'
}
result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
result << '-j' << 'oe'

// the requested queue name
if( task.config.queue ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,8 @@ class PbsProExecutor extends PbsExecutor {

result << '-N' << getJobNameFor(task)

if( task !instanceof TaskArrayRun ) {
result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
result << '-j' << 'oe'
}
result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
result << '-j' << 'oe'

// the requested queue name
if( task.config.queue ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,8 @@ class SgeExecutor extends AbstractGridExecutor implements TaskArrayExecutor {

result << '-N' << getJobNameFor(task)

if( task !instanceof TaskArrayRun ) {
result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
result << '-j' << 'y'
}
result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
result << '-j' << 'y'

result << '-terse' << '' // note: directive need to be returned as pairs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,8 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor {

result << '-J' << getJobNameFor(task)

if( task !instanceof TaskArrayRun ) {
// -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE
result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
}
// -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE
result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))

result << '--no-requeue' << '' // note: directive need to be returned as pairs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ class TaskArrayRun extends TaskRun {
return false
}

@Override
final boolean isArray() {
return true
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,10 @@ class TaskRun implements Cloneable {
return processor.executor?.isContainerNative() ?: false
}

boolean isArray() {
return false
}

boolean isContainerEnabled() {
return getContainerConfig().isEnabled() && getContainer()!=null
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,8 @@ class CrgExecutorTest extends Specification {
executor.getHeaders(task) == '''
#$ -t 1-5
#$ -N nf-mapping_tag
#$ -o /dev/null
#$ -j y
#$ -terse
#$ -notify
'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@ class LsfExecutorTest extends Specification {
}
then:
executor.getHeaders(taskArray) == '''
#BSUB -o /dev/null
#BSUB -J "nf-mapping_hola[1-5]"
'''
.stripIndent().leftTrim()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ class PbsExecutorTest extends Specification {
executor.getHeaders(taskArray) == '''
#PBS -J 0-4
#PBS -N nf-task_name
#PBS -o /dev/null
#PBS -j oe
'''
.stripIndent().leftTrim()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,8 @@ class PbsProExecutorTest extends Specification {
executor.getDirectives(task, []) == [
'-J', '0-4',
'-N', 'nf-foo',
'-o', '/dev/null',
'-j', 'oe'
]
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,8 @@ class SgeExecutorTest extends Specification {
executor.getHeaders(taskArray) == '''
#$ -t 1-5
#$ -N nf-the_task_name
#$ -o /dev/null
#$ -j y
#$ -terse
#$ -notify
'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ class SlurmExecutorTest extends Specification {
executor.getHeaders(taskArray) == '''
#SBATCH --array 0-4
#SBATCH -J nf-the_task_name
#SBATCH -o /dev/null
#SBATCH --no-requeue
#SBATCH --signal B:USR2@30
'''
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,11 @@ class TaskArrayRunTest extends Specification {
info == new ContainerInfo('ubuntu','ubuntu','ubuntu')
}

def 'should be an array' () {
given:
def task = new TaskArrayRun()
expect:
task.isArray()
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -885,4 +885,10 @@ class TaskRunTest extends Specification {
info == new ContainerInfo('ubuntu','ubuntu','ubuntu')
}

def 'should not be an array' () {
given:
def task = new TaskRun()
expect:
!task.isArray()
}
}
Loading