Fix Mount input file buckets in task arrays for Google Batch (#5739)

Signed-off-by: Christian Romberg <[email protected]> Signed-off-by: jorgee <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Christian Romberg <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Feb 4, 2025 · ba171fd · ba171fd
1 parent 7eea9f2
commit ba171fd
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 2 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy
@@ -112,6 +112,8 @@ class TaskBean implements Serializable, Cloneable {
 
     List<Path> arrayWorkDirs
 
+    List<Path> arrayInputFiles
+
     @PackageScope
     TaskBean() {
         shell = BashWrapperBuilder.BASH
@@ -173,6 +175,7 @@ class TaskBean implements Serializable, Cloneable {
             this.arrayIndexName = executor.getArrayIndexName()
             this.arrayIndexStart = executor.getArrayIndexStart()
             this.arrayWorkDirs = task.children.collect( h -> h.task.workDir )
+            this.arrayInputFiles = task.children.collectMany { h -> h.task.getInputFilesMap().values() }
         }
     }
 

diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy
@@ -70,6 +70,12 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
                 toContainerMount(it)
         }
 
+        // add input file mounts
+        if( bean.arrayInputFiles ) {
+            for( Path it : bean.arrayInputFiles )
+                toContainerMount(it)
+        }
+
         // remap input files to container mounted paths
         for( Map.Entry<String,Path> entry : new HashMap<>(bean.inputFiles).entrySet() ) {
             bean.inputFiles.put( entry.key, toContainerMount(entry.value, true) )

diff --git a/validation/google.config b/validation/google.config
@@ -2,7 +2,7 @@
  * do not include plugin requirements otherwise latest
  * published version will be downloaded instead of using local build
  */
-
+params.array_file_path = "gs://nf-test-arrays/file" 
 params.transcriptome = 'gs://rnaseq-nf/data/ggal/transcript.fa'
 params.reads = 'gs://rnaseq-nf/data/ggal/gut_{1,2}.fq'
 params.multiqc = 'gs://rnaseq-nf/multiqc'

diff --git a/validation/google.sh b/validation/google.sh
@@ -9,6 +9,10 @@ export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)}
 [[ $TOWER_ACCESS_TOKEN ]] && OPTS='-with-tower' || OPTS=''
 set -x
 
+$NXF_CMD -C ./google.config -q run ./test-arrays.nf > array_output
+[[ `grep 'Hi from the nf-test-array bucket!' -c array_output` == 3 ]] && echo OK || { echo 'Failed array tasks' && false; }
+
+
 $NXF_CMD -C ./google.config \
     run ./test-readspair.nf \
     -with-report \
@@ -44,7 +48,7 @@ $NXF_CMD -C ./google.config run ./test-complexpaths.nf -resume
 [[ -e 'foo/sample_(1 2).vcf' ]] || false
 
 ## run test-subdirs inputs/outputs
-$NXF_CMD -C ./gls.config run ./test-subdirs.nf
+$NXF_CMD -C ./gls.config -q run ./test-subdirs.nf
 
 ## run publishDir overwrite
 $NXF_CMD -C ./gls.config run ./test-overwrite.nf

diff --git a/validation/test-arrays.nf b/validation/test-arrays.nf
@@ -0,0 +1,19 @@
+process python_version {
+    array 3
+    input:
+    val x
+    path 'input_file'
+
+    output:
+    stdout
+
+    """
+    echo $x
+    cat $input_file
+    """
+}
+
+workflow {
+    python_version(Channel.of(1,2,3), params.array_file_path) | view
+}
+