vib-singlecell-nf · cflerin · Sep 28, 2020 · Aug 19, 2020 · Aug 19, 2020 · Aug 19, 2020
diff --git a/conf/compute_resources.config b/conf/compute_resources.config
@@ -0,0 +1,33 @@
+
+// define computing resources via process labels
+process {
+
+    /*
+      set a default compute profile
+      applies to all 'compute_resources__.*' labels:
+    */
+    withLabel: 'compute_resources__.*|compute_resources__default' {
+        cpus = 2
+        memory = '2 GB'
+        time = '1h'
+        // additional cluster options (applies to grid based executors):
+        clusterOptions = "-A cluster_account"
+    }
+
+    withLabel: 'compute_resources__minimal' {
+        cpus = 1
+        memory = '1 GB'
+    }
+
+    withLabel: 'compute_resources__mem' {
+        cpus = 4
+        memory = '30 GB'
+    }
+
+    withLabel: 'compute_resources__cpu' {
+        cpus = 20
+        memory = '30 GB'
+    }
+
+}
+
diff --git a/conf/global.config b/conf/global.config
@@ -2,7 +2,5 @@ params {
     global {
         project_name = '10x_PBMC'
         outdir = 'out'
-        qsubaccount = ''
-        qsubemail = ''
     }
-}
+}
diff --git a/conf/test__scenic.config b/conf/test__scenic.config
@@ -9,7 +9,6 @@ params {
         }
         scenic {
             filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_tiny.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_tiny.txt'
             }

diff --git a/conf/test__scenic_multiruns.config b/conf/test__scenic_multiruns.config
@@ -9,7 +9,6 @@ params {
         }
         scenic {
             filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_small.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/conf/test__single_sample_scenic.config b/conf/test__single_sample_scenic.config
@@ -27,7 +27,6 @@ params {
             }
         }
         scenic {
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/conf/test__single_sample_scenic_multiruns.config b/conf/test__single_sample_scenic_multiruns.config
@@ -28,7 +28,6 @@ params {
         }
         scenic {
             //filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_small.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/main_atac.nf b/main_atac.nf
@@ -3,12 +3,15 @@ import static groovy.json.JsonOutput.*
 nextflow.preview.dsl=2
 
 include { 
-    INIT
-} from '../utils/workflows/utils.nf' params(params)
+    INIT;
+} from './src/utils/workflows/utils' params(params)
 INIT()
+include {
+    SC__FILE_CONVERTER;
+} from './src/utils/processes/utils' params(params)
 
 include {
-    getDataChannel
+    getDataChannel;
 } from './src/channels/channels' params(params)
 
 /* 

diff --git a/nextflow.config b/nextflow.config
@@ -18,7 +18,9 @@ params {
     }
 }
 
+// load these configs first:
 includeConfig 'conf/global.config'
+includeConfig 'conf/compute_resources.config'
 
 process {
     executor = 'local'
@@ -376,4 +378,5 @@ min {
 
 includeConfig 'src/utils/conf/scope.config'
 includeConfig 'src/utils/utils.config' // utilities config
-includeConfig 'conf/generic.config'
+includeConfig 'conf/generic.config'
+
diff --git a/src/cellranger b/src/cellranger
diff --git a/src/cellranger-atac b/src/cellranger-atac
diff --git a/src/scanpy b/src/scanpy
diff --git a/src/scenic b/src/scenic
diff --git a/src/utils/processes/gtf.nf b/src/utils/processes/gtf.nf
@@ -3,7 +3,7 @@ nextflow.preview.dsl=2
 process FORMAT_GTF {
 
     publishDir "${params.global.outdir}/00.refdata", mode: 'symlink'
-    clusterOptions "-l nodes=1:ppn=${params.global.threads} -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         file(annotation)
@@ -23,7 +23,7 @@ process FORMAT_GTF {
 process FORMAT_GTF_IGENOMES {
 
     publishDir "${params.global.outdir}/00.refdata", mode: 'symlink'
-    clusterOptions "-l nodes=1:ppn=${params.global.threads} -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         file(annotation)

diff --git a/src/utils/processes/h5adAnnotate.nf b/src/utils/processes/h5adAnnotate.nf
@@ -25,7 +25,7 @@ process SC__ANNOTATE_BY_CELL_METADATA {
 
     container params.sc.scanpy.container
     publishDir "${getPublishDir(params.global.outdir,tool)}", mode: "${getMode(tool)}", overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple \
@@ -69,7 +69,7 @@ process SC__ANNOTATE_BY_SAMPLE_METADATA() {
 
     container params.sc.scanpy.container
     publishDir "${params.global.outdir}/data/intermediate", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple val(sampleId), path(f)

diff --git a/src/utils/processes/h5adExtractMetadata.nf b/src/utils/processes/h5adExtractMetadata.nf
@@ -9,7 +9,7 @@ process SC__UTILS__EXTRACT_FEATURE_METADATA {
 
     container params.sc.scanpy.container
     publishDir "${params.global.outdir}/data/intermediate", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple val(sampleId), path(f)

diff --git a/src/utils/processes/h5adSubset.nf b/src/utils/processes/h5adSubset.nf
@@ -12,7 +12,7 @@ process SC__PREPARE_OBS_FILTER {
 
     container params.sc.scanpy.container
     publishDir "${params.global.outdir}/data/intermediate", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple \
@@ -57,7 +57,7 @@ process SC__APPLY_OBS_FILTER {
 
     container params.sc.scanpy.container
     publishDir "${params.global.outdir}/data/intermediate", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple \

diff --git a/src/utils/processes/h5adToLoom.nf b/src/utils/processes/h5adToLoom.nf
@@ -8,8 +8,8 @@ binDir = !params.containsKey("test") ? "${workflow.projectDir}/src/utils/bin" :
 process SC__H5AD_TO_LOOM {
 
 	container params.sc.scanpy.container
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir "${params.global.outdir}/loom", mode: 'link', overwrite: true, saveAs: { filename -> "${sampleId}.SCope_output.loom" }
+    label 'compute_resources__mem'
 
 	input:
 		// Expects:
@@ -42,8 +42,8 @@ process SC__H5AD_TO_LOOM {
 process SC__H5AD_TO_FILTERED_LOOM {
 
 	container params.sc.scanpy.container
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink', overwrite: true
+    label 'compute_resources__mem'
 
 	input:
 		tuple val(sampleId), path(f)

diff --git a/src/utils/processes/h5adUpdate.nf b/src/utils/processes/h5adUpdate.nf
@@ -8,7 +8,7 @@ binDir = !params.containsKey("test") ? "${workflow.projectDir}/src/utils/bin" :
 process SC__H5AD_UPDATE_X_PCA {
 
 	container params.sc.scanpy.container
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__mem'
 
 	input:
 		tuple \
@@ -34,7 +34,7 @@ process SC__H5AD_UPDATE_X_PCA {
 process SC__H5AD_CLEAN {
 
 	container params.sc.scanpy.container
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__mem'
 
 	input:
 		tuple \
@@ -56,4 +56,4 @@ process SC__H5AD_CLEAN {
 			"${sampleId}.SC__H5AD_CLEAN.h5ad"
 		"""
 
-}
+}
diff --git a/src/utils/processes/h5adUpdateMetadata.nf b/src/utils/processes/h5adUpdateMetadata.nf
@@ -9,7 +9,7 @@ process SC__UTILS__UPDATE_FEATURE_METADATA_INDEX {
 
     container params.sc.scanpy.container
     publishDir "${params.global.outdir}/data/intermediate", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=2 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple val(sampleId), path(f), path(additionalMetadata)

diff --git a/src/utils/processes/reports.nf b/src/utils/processes/reports.nf
@@ -5,8 +5,8 @@ import static groovy.json.JsonOutput.*
 process UTILS__GENERATE_WORKFLOW_CONFIG_REPORT {
 
   	container params.utils.container
-  	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
   	publishDir "${params.global.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
+    label 'compute_resources__default'
 
     input:
         path(ipynb)
@@ -27,11 +27,11 @@ process UTILS__GENERATE_WORKFLOW_CONFIG_REPORT {
 process UTILS__REPORT_TO_HTML {
 
 	container params.utils.container
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
 	publishDir "${params.global.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
 	// copy final "merged_report" to notbooks root:
 	publishDir "${params.global.outdir}/notebooks", pattern: '*merged_report*', mode: 'link', overwrite: true
 	maxForks 2
+    label 'compute_resources__minimal'
 
 	input:
 		tuple \
@@ -46,4 +46,4 @@ process UTILS__REPORT_TO_HTML {
 		jupyter nbconvert ${ipynb} --to html
 		"""
 
-}
+}
diff --git a/src/utils/processes/sra.nf b/src/utils/processes/sra.nf
@@ -12,7 +12,7 @@ process GET_SRA_DB {
 
     container params.utils.container
     publishDir "${processParams.sraDbOutDir}", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=1 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     output:
         file("SRAmetadb.sqlite")
@@ -29,9 +29,9 @@ process SRA_TO_METADATA {
 
     container params.utils.container
     publishDir "${params.global.outdir}/metadata", mode: 'link', overwrite: true
-    clusterOptions "-l nodes=1:ppn=1 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     errorStrategy 'retry'
     maxRetries 5
+    label 'compute_resources__default'
 
     input:
         tuple val(sraId), val(sampleFilters)
@@ -78,7 +78,7 @@ def normalizeSRAFastQ(fastQPath, sampleName) {
 process NORMALIZE_SRA_FASTQS {
 
     // publishDir "${params.global.outdir}/data", mode: 'symlink'
-    clusterOptions "-l nodes=1:ppn=1 -l walltime=1:00:00 -A ${params.global.qsubaccount}"
+    label 'compute_resources__default'
 
     input:
         tuple val(sampleId), file(fastqs)

diff --git a/src/utils/processes/utils.nf b/src/utils/processes/utils.nf
@@ -86,8 +86,8 @@ process SC__FILE_CONVERTER {
 		container "vibsinglecellnf/sceasy:0.0.5"
 	else
         container params.sc.scanpy.container
-    clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink', overwrite: true
+    label 'compute_resources__mem'
 
     input:
         tuple \
@@ -192,6 +192,7 @@ process SC__FILE_CONVERTER {
 process SC__FILE_CONVERTER_HELP {
 
     container params.sc.scanpy.container
+    label 'compute_resources__minimal'
 
     output:
         stdout()
@@ -207,8 +208,8 @@ process SC__FILE_CONCATENATOR {
 
     cache 'deep'
     container params.sc.scanpy.container
-    clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink', overwrite: true
+    label 'compute_resources__mem'
 
     input:
         file("*")
@@ -230,8 +231,8 @@ process SC__FILE_CONCATENATOR {
 process SC__STAR_CONCATENATOR() {
 
     container params.sc.scanpy.container
-    clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink', overwrite: true
+    label 'compute_resources__mem'
 
     input:
         tuple \
@@ -274,14 +275,14 @@ process SC__PUBLISH {
         return "${tag}.${fileOutputSuffix}.${f.extension}"
     }
 
-    clusterOptions "-l nodes=1:ppn=2 -l pmem=3gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
     publishDir \
         "${getPublishDir(params.global.outdir,toolName)}", \
         mode: 'link', \
         overwrite: true, \
         saveAs: {
             filename -> "${outputFileName}" 
         }
+    label 'compute_resources__minimal'
 
 
     input:
@@ -320,8 +321,8 @@ process SC__PUBLISH {
 process COMPRESS_HDF5() {
 
 	container "vibsinglecellnf/hdf5:1.10.5-r2"
-	clusterOptions "-l nodes=1:ppn=2 -l pmem=30gb -l walltime=1:00:00 -A ${params.global.qsubaccount}"
 	publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink', overwrite: true
+    label 'compute_resources__mem'
 
     def getFileOutputSuffix = { params, fileOutputSuffix, toolName ->
         if(isParamNull(fileOutputSuffix) && isParamNull(toolName) &&
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,7 +27,6 @@ params { @@
                 }
             }
             scenic {
-                numWorkers = 2
                 grn {
                     tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
                 }
@@ Expand Down @@
+13 −5		conf/base.config
+0 −5		conf/count.config
+1 −1		conf/count_libraries.config
+2 −0		conf/mkfastq.config
+14 −14		processes/count.nf
+4 −4		processes/mkfastq.nf
+2 −8		workflows/cellRangerCountWithMetadata.nf
+11 −4		conf/base.config
+2 −6		conf/count.config
+2 −0		conf/mkfastq.config
+8 −9		processes/count.nf
+4 −3		processes/mkfastq.nf
+1 −1		processes/batch_effect_correct.nf
+2 −2		processes/cluster.nf
+1 −1		processes/dim_reduction.nf
+2 −2		processes/feature_selection.nf
+3 −3		processes/filter.nf
+3 −3		processes/marker_genes.nf
+1 −1		processes/neighborhood_graph.nf
+1 −1		processes/regress_out.nf
+5 −5		processes/reports.nf
+3 −3		processes/transform.nf
+1 −0		conf/append.config
+1 −7		conf/min/aucell.config
+0 −6		conf/min/cistarget.config
+5 −5		conf/min/grn.config
+13 −7		conf/multi_runs.config
+0 −7		conf/test_multi_runs.config
+0 −3		main.nf
+3 −14		processes/arboreto_with_multiprocessing.nf
+4 −14		processes/aucell.nf
+3 −14		processes/cistarget.nf
+5 −5		processes/loomHandler.nf
+1 −3		processes/multiruns/aggregateFeatures.nf
+1 −3		processes/multiruns/aggregateRegulons.nf
+2 −4		processes/multiruns/aucellFromFolder.nf
+1 −4		processes/multiruns/convertMotifsToRegulons.nf
+1 −2		processes/multiruns/saveToLoom.nf
+3 −2		processes/reports.nf
+28 −25		scenic.config