Skip to content

Commit 28d975e

Browse files
ahvigilahvigil
authored
Merge branch 'master' into master
2 parents 43649da + 681ace8 commit 28d975e

File tree

20 files changed

+293
-74
lines changed

20 files changed

+293
-74
lines changed

‎docs/config.md

+12-3
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@ The following settings are available:
104104
`apptainer.noHttps`
105105
: Pull the Apptainer image with http protocol (default: `false`).
106106

107+
`apptainer.ociAutoPull`
108+
: :::{versionadded} 23.12.0-edge
109+
:::
110+
: When enabled, OCI (and Docker) container images are pulled and converted to the SIF format by the Apptainer run command, instead of Nextflow (default: `false`).
111+
107112
`apptainer.pullTimeout`
108113
: The amount of time the Apptainer pull can last, exceeding which the process is terminated (default: `20 min`).
109114

@@ -1407,11 +1412,15 @@ The following settings are available:
14071412
`singularity.noHttps`
14081413
: Pull the Singularity image with http protocol (default: `false`).
14091414

1410-
`singularity.oci`
1411-
: :::{versionadded} 23.11.0-edge
1415+
`singularity.ociAutoPull`
1416+
: :::{versionadded} 23.12.0-edge
14121417
:::
1413-
: Enable OCI-mode, that allows running native OCI-compatible containers with Singularity using `crun` or `runc` as low-level runtime. See `--oci` flag in the [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`).
1418+
: When enabled, OCI (and Docker) container images are pull and converted to a SIF image file format implicitly by the Singularity run command, instead of Nextflow. Requires Singulairty 3.11 or later (default: `false`).
14141419

1420+
`singularity.ociMode`
1421+
: :::{versionadded} 23.12.0-edge
1422+
:::
1423+
: Enable OCI-mode, that allows running native OCI compliant container image with Singularity using `crun` or `runc` as low-level runtime. Note: it requires Singulairty 4 or later. See `--oci` flag in the [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`).
14151424

14161425
`singularity.pullTimeout`
14171426
: The amount of time the Singularity pull can last, exceeding which the process is terminated (default: `20 min`).

‎modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy

+36-3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package nextflow.container
1818

1919
import groovy.transform.CompileStatic
20+
import groovy.transform.Memoized
2021
import groovy.util.logging.Slf4j
2122

2223
/**
@@ -55,8 +56,40 @@ class ContainerConfig extends LinkedHashMap {
5556
get('engine')
5657
}
5758

58-
boolean singularityOciMode() {
59-
getEngine()=='singularity' && get('oci')?.toString() == 'true'
59+
/**
60+
* Whenever Singularity or Apptainer container engine can a OCI (Docker)
61+
* image without requiring a separate OCI to SIF conversion execution (managed by Nextflow via {@link SingularityCache).
62+
*
63+
* @return
64+
* {@code true} when the OCI container can be run without an explicit OCI to SIF conversion or {@code false}
65+
* otherwise
66+
*
67+
*/
68+
boolean canRunOciImage() {
69+
if( isSingularityOciMode() )
70+
return true
71+
if( (getEngine()!='singularity' && getEngine()!='apptainer') )
72+
return false
73+
return get('ociAutoPull')?.toString()=='true'
74+
}
75+
76+
/**
77+
* Whenever the Singularity OCI-mode is enabled
78+
*
79+
* @return {@code true} when Singularity OCI-mode is enabled or {@code false} otherwise
80+
*/
81+
@Memoized
82+
boolean isSingularityOciMode() {
83+
if( getEngine()!='singularity' ) {
84+
return false
85+
}
86+
if( get('oci')?.toString()=='true' ) {
87+
log.warn "The setting `singularity.oci` is deprecated - use `singularity.ociNative` instead"
88+
return true
89+
}
90+
if( get('ociMode')?.toString()=='true' )
91+
return true
92+
return false
6093
}
6194

6295
List<String> getEnvWhitelist() {
@@ -93,7 +126,7 @@ class ContainerConfig extends LinkedHashMap {
93126
return null
94127
if( eng=='docker' || eng=='podman' )
95128
return '--rm --privileged'
96-
if( singularityOciMode() )
129+
if( isSingularityOciMode() )
97130
return '-B /dev/fuse'
98131
if( eng=='singularity' || eng=='apptainer' )
99132
return null

‎modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy

+3-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class ContainerHandler {
6767
final normalizedImageName = normalizeSingularityImageName(imageName)
6868
if( !config.isEnabled() || !normalizedImageName )
6969
return normalizedImageName
70-
if( normalizedImageName.startsWith('docker://') && config.singularityOciMode() )
70+
if( normalizedImageName.startsWith('docker://') && config.canRunOciImage() )
7171
return normalizedImageName
7272
final requiresCaching = normalizedImageName =~ IMAGE_URL_PREFIX
7373
if( ContainerInspectMode.active() && requiresCaching )
@@ -79,6 +79,8 @@ class ContainerHandler {
7979
final normalizedImageName = normalizeApptainerImageName(imageName)
8080
if( !config.isEnabled() || !normalizedImageName )
8181
return normalizedImageName
82+
if( normalizedImageName.startsWith('docker://') && config.canRunOciImage() )
83+
return normalizedImageName
8284
final requiresCaching = normalizedImageName =~ IMAGE_URL_PREFIX
8385
if( ContainerInspectMode.active() && requiresCaching )
8486
return imageName

‎modules/nextflow/src/main/groovy/nextflow/container/DockerBuilder.groovy

+6-9
Original file line numberDiff line numberDiff line change
@@ -15,29 +15,29 @@
1515
*/
1616

1717
package nextflow.container
18+
19+
1820
import groovy.transform.CompileStatic
21+
import groovy.util.logging.Slf4j
1922
/**
2023
* Helper methods to handle Docker containers
2124
*
2225
* @author Paolo Di Tommaso <[email protected]>
2326
*/
27+
@Slf4j
2428
@CompileStatic
2529
class DockerBuilder extends ContainerBuilder<DockerBuilder> {
2630

2731
private boolean sudo
2832

2933
private boolean remove = true
3034

31-
private boolean userEmulation
32-
3335
private String registry
3436

3537
private String name
3638

3739
private boolean tty
3840

39-
private static final String USER_AND_HOME_EMULATION = '-u $(id -u) -e "HOME=${HOME}" -v /etc/passwd:/etc/passwd:ro -v /etc/shadow:/etc/shadow:ro -v /etc/group:/etc/group:ro -v $HOME:$HOME'
40-
4141
private String removeCommand
4242

4343
private String killCommand
@@ -69,8 +69,8 @@ class DockerBuilder extends ContainerBuilder<DockerBuilder> {
6969
if( params.containsKey('runOptions') )
7070
addRunOptions(params.runOptions.toString())
7171

72-
if ( params.containsKey('userEmulation') )
73-
this.userEmulation = params.userEmulation?.toString() == 'true'
72+
if ( params.userEmulation?.toString() == 'true' )
73+
log.warn1("Undocumented setting `docker.userEmulation` is not supported any more - consider to remove it from your config")
7474

7575
if ( params.containsKey('remove') )
7676
this.remove = params.remove?.toString() == 'true'
@@ -150,9 +150,6 @@ class DockerBuilder extends ContainerBuilder<DockerBuilder> {
150150
if( temp )
151151
result << "-v $temp:/tmp "
152152

153-
if( userEmulation )
154-
result << USER_AND_HOME_EMULATION << ' '
155-
156153
// mount the input folders
157154
result << makeVolumes(mounts)
158155
result << '-w "$NXF_TASK_WORKDIR" '

‎modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy

+10-7
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
3939

4040
private String runCmd0
4141

42-
private Boolean oci
42+
private Boolean ociMode
4343

4444
SingularityBuilder(String name) {
4545
this.image = name
@@ -94,8 +94,11 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
9494
if( params.containsKey('readOnlyInputs') )
9595
this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true'
9696

97-
if( params.oci!=null )
98-
oci = params.oci.toString() == 'true'
97+
// note: 'oci' flag should be ignored by Apptainer sub-class
98+
if( params.oci!=null && this.class==SingularityBuilder )
99+
ociMode = params.oci.toString() == 'true'
100+
else if( params.ociMode!=null && this.class==SingularityBuilder )
101+
ociMode = params.ociMode.toString() == 'true'
99102

100103
return this
101104
}
@@ -122,11 +125,11 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
122125
if( !homeMount )
123126
result << '--no-home '
124127

125-
if( newPidNamespace && !oci )
128+
if( newPidNamespace && !ociMode )
126129
result << '--pid '
127130

128-
if( oci != null )
129-
result << (oci ? '--oci ' : '--no-oci ')
131+
if( ociMode != null )
132+
result << (ociMode ? '--oci ' : '--no-oci ')
130133

131134
if( autoMounts ) {
132135
makeVolumes(mounts, result)
@@ -154,7 +157,7 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
154157
makeEnv('TMP',result) .append(' ')
155158
makeEnv('TMPDIR',result) .append(' ')
156159
// add magic variables required by singularity to run in OCI-mode
157-
if( oci ) {
160+
if( ociMode ) {
158161
result .append('${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ')
159162
result .append('${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} ')
160163
}

‎modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy

+2-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ class PublishDir {
134134
}
135135

136136
protected Map<String,Path> getTaskInputs() {
137-
return task?.getInputFilesMap()
137+
return task ? task.getInputFilesMap() : Map.<String,Path>of()
138138
}
139139

140140
void setPath( def value ) {
@@ -294,6 +294,7 @@ class PublishDir {
294294
this.sourceDir = task.targetDir
295295
this.sourceFileSystem = sourceDir.fileSystem
296296
this.stageInMode = task.config.stageInMode
297+
this.task = task
297298

298299
apply0(files)
299300
}

‎modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy

+6-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ final class BitbucketRepositoryProvider extends RepositoryProvider {
5050

5151
@Override
5252
String getContentUrl( String path ) {
53-
final ref = revision ?: getMainBranch()
53+
final ref = revision ? getRefForRevision(revision) : getMainBranch()
5454
return "${config.endpoint}/api/2.0/repositories/$project/src/$ref/$path"
5555
}
5656

@@ -62,6 +62,11 @@ final class BitbucketRepositoryProvider extends RepositoryProvider {
6262
invokeAndParseResponse(getMainBranchUrl()) ?. mainbranch ?. name
6363
}
6464

65+
private String getRefForRevision(String revision){
66+
final resp = invokeAndParseResponse("${config.endpoint}/api/2.0/repositories/$project/refs/branches/$revision")
67+
return resp?.target?.hash
68+
}
69+
6570
@Memoized
6671
protected <T> List<T> invokeAndResponseWithPaging(String url, Closure<T> parse) {
6772
final result = new ArrayList<T>(50)

‎modules/nextflow/src/test/groovy/nextflow/container/ApptainerBuilderTest.groovy

+5
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,11 @@ class ApptainerBuilderTest extends Specification {
153153
.build()
154154
.runCommand == 'set +u; env - PATH="$PATH" ${TMP:+APPTAINERENV_TMP="$TMP"} ${TMPDIR:+APPTAINERENV_TMPDIR="$TMPDIR"} apptainer exec --no-home -B "$NXF_TASK_WORKDIR" ubuntu'
155155

156+
new ApptainerBuilder('ubuntu')
157+
.params(oci: true)
158+
.build()
159+
.runCommand == 'set +u; env - PATH="$PATH" ${TMP:+APPTAINERENV_TMP="$TMP"} ${TMPDIR:+APPTAINERENV_TMPDIR="$TMPDIR"} apptainer exec --no-home --pid -B "$NXF_TASK_WORKDIR" ubuntu'
160+
156161
}
157162

158163
def 'should mount home directory if specified' () {

‎modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy

+25-13
Original file line numberDiff line numberDiff line change
@@ -61,24 +61,34 @@ class ContainerConfigTest extends Specification {
6161

6262
}
6363

64-
65-
def 'should validate oci mode' () {
64+
@Unroll
65+
def 'should validate oci mode and direct mode' () {
6666

6767
when:
6868
def cfg = new ContainerConfig(OPTS)
6969
then:
70-
cfg.singularityOciMode() == EXPECTED
70+
cfg.isSingularityOciMode() == OCI_MODE
71+
cfg.canRunOciImage() == AUTO_PULL
7172

7273
where:
73-
OPTS | EXPECTED
74-
[:] | false
75-
[oci:false] | false
76-
[oci:true] | false
77-
[engine:'apptainer', oci:true] | false
78-
[engine:'docker', oci:true] | false
79-
[engine:'singularity'] | false
80-
[engine:'singularity', oci:false] | false
81-
[engine:'singularity', oci:true] | true
74+
OPTS | OCI_MODE | AUTO_PULL
75+
[:] | false | false
76+
[oci:true] | false | false
77+
[oci:false] | false | false
78+
[ociMode:true] | false | false
79+
and:
80+
[engine:'docker', oci:true] | false | false
81+
[engine:'singularity'] | false | false
82+
[engine:'singularity', oci:false] | false | false
83+
[engine:'singularity', ociAutoPull:false] | false | false
84+
and:
85+
[engine:'singularity', oci:true] | true | true
86+
[engine:'singularity', ociMode:true] | true | true
87+
[engine:'apptainer', oci:true] | false | false
88+
[engine:'apptainer', ociMode:true] | false | false
89+
and:
90+
[engine:'singularity', ociAutoPull:true] | false | true
91+
[engine:'apptainer', ociAutoPull:true] | false | true
8292

8393
}
8494

@@ -96,7 +106,9 @@ class ContainerConfigTest extends Specification {
96106
[engine:'podman'] | '--rm --privileged'
97107
and:
98108
[engine: 'singularity'] | null
99-
[engine: 'singularity', oci:true] | '-B /dev/fuse'
109+
[engine: 'singularity', ociMode:true] | '-B /dev/fuse'
110+
[engine: 'singularity', ociAutoPull: true] | null
111+
[engine: 'apptainer', oci:true] | null
100112
and:
101113
[engine:'docker', fusionOptions:'--cap-add foo']| '--cap-add foo'
102114
[engine:'podman', fusionOptions:'--cap-add bar']| '--cap-add bar'

‎modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy

+49-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ class ContainerHandlerTest extends Specification {
205205
def 'test normalize method for singularity' () {
206206
given:
207207
def BASE = Paths.get('/abs/path/')
208-
def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, oci:OCI, baseDir: BASE))
208+
def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, ociMode: OCI, baseDir: BASE))
209209

210210
when:
211211
def result = handler.normalizeImageName(IMAGE)
@@ -233,6 +233,54 @@ class ContainerHandlerTest extends Specification {
233233

234234
}
235235

236+
@Unroll
237+
def 'test normalize method for OCI direct mode' () {
238+
given:
239+
def BASE = Paths.get('/abs/path/')
240+
def handler = Spy(new ContainerHandler(engine: 'apptainer', enabled: true, ociAutoPull:AUTO, baseDir: BASE))
241+
242+
when:
243+
def result = handler.normalizeImageName(IMAGE)
244+
245+
then:
246+
1 * handler.normalizeApptainerImageName(IMAGE) >> NORMALIZED
247+
X * handler.createApptainerCache(handler.config, NORMALIZED) >> EXPECTED
248+
result == EXPECTED
249+
250+
where:
251+
IMAGE | NORMALIZED | ENGINE | AUTO | X | EXPECTED
252+
null | null | 'singularity' | false | 0 | null
253+
'' | null | 'singularity' | false | 0 | null
254+
'/abs/path/bar.img' | '/abs/path/bar.img' | 'singularity' | false | 0 | '/abs/path/bar.img'
255+
'/abs/path bar.img' | '/abs/path bar.img' | 'singularity' | false | 0 | '/abs/path\\ bar.img'
256+
'file:///abs/path/bar.img' | '/abs/path/bar.img' | 'singularity' | false | 0 | '/abs/path/bar.img'
257+
'foo.img' | Paths.get('foo.img').toAbsolutePath().toString() | 'singularity' | false | 0 | Paths.get('foo.img').toAbsolutePath().toString()
258+
'shub://busybox' | 'shub://busybox' | 'singularity' | false | 1 | '/path/to/busybox'
259+
'docker://library/busybox' | 'docker://library/busybox' | 'singularity' | false | 1 | '/path/to/busybox'
260+
'foo' | 'docker://foo' | 'singularity' | false | 1 | '/path/to/foo'
261+
'library://pditommaso/foo/bar.sif:latest' | 'library://pditommaso/foo/bar.sif:latest' | 'singularity' | false | 1 | '/path/to/foo-bar-latest.img'
262+
and:
263+
'docker://library/busybox' | 'docker://library/busybox' | 'singularity' | true | 0 | 'docker://library/busybox'
264+
'shub://busybox' | 'shub://busybox' | 'singularity' | true | 1 | '/path/to/busybox'
265+
266+
and:
267+
null | null | 'apptainer' | false | 0 | null
268+
'' | null | 'apptainer' | false | 0 | null
269+
'/abs/path/bar.img' | '/abs/path/bar.img' | 'apptainer' | false | 0 | '/abs/path/bar.img'
270+
'/abs/path bar.img' | '/abs/path bar.img' | 'apptainer' | false | 0 | '/abs/path\\ bar.img'
271+
'file:///abs/path/bar.img' | '/abs/path/bar.img' | 'apptainer' | false | 0 | '/abs/path/bar.img'
272+
'foo.img' | Paths.get('foo.img').toAbsolutePath().toString() | 'apptainer' | false | 0 | Paths.get('foo.img').toAbsolutePath().toString()
273+
'shub://busybox' | 'shub://busybox' | 'apptainer' | false | 1 | '/path/to/busybox'
274+
'docker://library/busybox' | 'docker://library/busybox' | 'apptainer' | false | 1 | '/path/to/busybox'
275+
'foo' | 'docker://foo' | 'apptainer' | false | 1 | '/path/to/foo'
276+
'library://pditommaso/foo/bar.sif:latest' | 'library://pditommaso/foo/bar.sif:latest' | 'apptainer' | false | 1 | '/path/to/foo-bar-latest.img'
277+
and:
278+
'docker://library/busybox' | 'docker://library/busybox' | 'apptainer' | true | 0 | 'docker://library/busybox'
279+
'shub://busybox' | 'shub://busybox' | 'apptainer' | true | 1 | '/path/to/busybox'
280+
281+
282+
}
283+
236284
def 'should not invoke caching when engine is disabled' () {
237285
given:
238286
final handler = Spy(new ContainerHandler([engine: 'singularity']))

‎modules/nextflow/src/test/groovy/nextflow/container/DockerBuilderTest.groovy

-5
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,6 @@ class DockerBuilderTest extends Specification {
9797
.build()
9898
.runCommand == 'docker run -i -v "$NXF_TASK_WORKDIR":"$NXF_TASK_WORKDIR" -w "$NXF_TASK_WORKDIR" -x --zeta busybox'
9999

100-
new DockerBuilder('busybox')
101-
.params(userEmulation:true)
102-
.build()
103-
.runCommand == 'docker run -i -u $(id -u) -e "HOME=${HOME}" -v /etc/passwd:/etc/passwd:ro -v /etc/shadow:/etc/shadow:ro -v /etc/group:/etc/group:ro -v $HOME:$HOME -v "$NXF_TASK_WORKDIR":"$NXF_TASK_WORKDIR" -w "$NXF_TASK_WORKDIR" busybox'
104-
105100
new DockerBuilder('busybox')
106101
.setName('hola')
107102
.build()

0 commit comments

Comments
 (0)