Merge pull request #50 from CenterForMedicalGeneticsGhent/dev

dev

Author: matthdsm

.gitignore

@@ -8,3 +8,5 @@ testing/
 testing*
 *.pyc
 null
+s3:
+output

CHANGELOG.md

@@ -3,6 +3,17 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v1.1.2
+
+- Support for bamprocessing with ELPREP
+- Improved sample grouping
+- Meta value clean up
+- bump modules
+- fix multiqc report to include all results
+- fix duplicate marking with bamsormadup
+- drop post alignment sorting, doesn't make sense as we do sort/markdup afterwards
+- Add option to use elprep for alignment postprocessing
+
 ## v1.0.2dev
 
 - Add `unaligned` denominator to unaligned samples
@@ -18,11 +29,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## v1.0dev - [06/09/2022]
 
 Initial release of CenterForMedicalGeneticsGhent/nf-cmgg-preprocessing, created with the [nf-core](https://nf-co.re/) template.
-
-### `Added`
-
-### `Fixed`
-
-### `Dependencies`
-
-### `Deprecated`

assets/multiqc_config.yml

@@ -12,3 +12,5 @@ report_section_order:
 export_plots: true
 bclconvert:
   create_undetermined_barcode_barplots: true
+extra_fn_clean_exts:
+  - ".cram"

conf/modules.config

@@ -119,8 +119,7 @@ process {
             meta.readgroup && meta.readgroup.LB ? "--library ${meta.readgroup.LB}" : "",
             meta.readgroup && meta.readgroup.PU ? "--platform-unit ${meta.readgroup.PU}" : "",
             meta.readgroup && meta.readgroup.PL ? "--platform ${meta.readgroup.PL}" : "",
-            meta.readgroup && meta.readgroup.CN ? "--sequencing-center ${meta.readgroup.CN}" : "",
-            "--sort=true"
+            meta.readgroup && meta.readgroup.CN ? "--sequencing-center ${meta.readgroup.CN}" : ""
         ].join(" ").trim()}
         publishDir = [ enabled: false ]
     }
@@ -158,8 +157,8 @@ process {
             "--fast-local",
             meta.readgroup ? "--rg-id ${meta.readgroup.ID}": "",
             meta.readgroup ? "--rg " + meta.readgroup.findResults{ it.value?.trim() ? "$it.key:$it.value" : null }.join(" --rg ") : ""
-
         ].join(" ").trim()}
+        ext.args2 = "--fast"
     }
 
     withName: "BWAMEM.*_MEM" {
@@ -171,16 +170,16 @@ process {
             "-c 250",
             meta.readgroup ? "-R \"@RG\\t" + meta.readgroup.findResults{ it.value?.trim() ? "$it.key:$it.value" : null }.join("\\t") + "\"" : ""
         ].join(" ").trim()}
+        ext.args2 = "--fast"
     }
 
     withName: DRAGMAP_ALIGN {
         ext.args = {[
             meta.readgroup ? "--RGSM \"@RG\\t" + meta.readgroup.findResults{ it.value?.trim() ? "$it.key:$it.value" : null }.join("\\t") + "\"" : ""
-
         ].join(" ").trim()}
+        ext.args2 = "--fast"
     }
 
-    // -so    : enable coordinate sorting
     // -b-    : Don't lock threads to cpu cores, improves performance when scheduling multiple jobs/node
     // -sm 10 : Allow 10GB of memory for sorting threads
     // -I     : ignore IDs that don't match in the paired-end aligner
@@ -192,7 +191,6 @@ process {
         cpus   = { check_max( 18 , "cpus" ) }
         memory = { check_max( 72.GB , "memory" ) }
         ext.args = {[
-            "-so",
             "-b-",
             "-sm 20",
             "-I",
@@ -207,6 +205,27 @@ process {
     // BAMPROCESSING
     //********************************************************************************//
 
+    withName: ELPREP_SFM {
+        ext.args = {[
+            "--mark-duplicates",
+            "--sorting-order coordinate",
+            "--optical-duplicates-pixel-distance 2500",
+            "--mark-optical-duplicates ${meta.id}.metrics.txt"
+        ].join(" ").trim()}
+        publishDir = [
+            [
+                path: { "${params.outdir}/${meta.samplename}" },
+                mode: params.publish_dir_mode,
+                pattern: "*metrics*",
+                saveAs: {filename -> filename.replace("metrics", "duplicate_metrics")}
+            ],
+        ]
+    }
+
+    withName: ".*:FASTQ_TO_CRAM:SAMTOOLS_INDEX" {
+        publishDir = [ enabled: false ]
+    }
+
     withName: BIOBAMBAM_BAMSORMADUP {
         ext.prefix = {"${meta.id}.merged"}
         ext.args = {[
@@ -250,6 +269,7 @@ process {
 
     withName: SAMTOOLS_CONVERT {
         cpus = { check_max( 9 , "cpus" ) }
+        memory = { check_max( 36.GB * task.attempt, "memory"  ) }
         // activate cram compression
         ext.args = "-C --output-fmt-option archive --verbosity 5"
     }

modules.json

@@ -7,18 +7,19 @@
                 "nf-core": {
                     "bcl2fastq": {
                         "branch": "master",
-                        "git_sha": "c62b41a07a00fee11ec73e2a0d4a1aec771a742b",
+                        "git_sha": "100f641b272ff7e309bfbb86d0c3f63d7b6c6a9b",
                         "installed_by": ["bcl_demultiplex"]
                     },
                     "bclconvert": {
                         "branch": "master",
-                        "git_sha": "c62b41a07a00fee11ec73e2a0d4a1aec771a742b",
+                        "git_sha": "100f641b272ff7e309bfbb86d0c3f63d7b6c6a9b",
                         "installed_by": ["bcl_demultiplex"]
                     },
                     "biobambam/bamsormadup": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
-                        "installed_by": ["modules"]
+                        "git_sha": "ea421dc46ceaad39d637f8793e87193061804868",
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/biobambam/bamsormadup/biobambam-bamsormadup.diff"
                     },
                     "bowtie2/align": {
                         "branch": "master",
@@ -62,7 +63,7 @@
                     },
                     "dragmap/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "5cef12bb8a747ce018bb351871bdfe118bd28dd7",
                         "installed_by": ["fastq_align_dna"]
                     },
                     "dragmap/hashtable": {
@@ -140,6 +141,11 @@
                         "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
                         "installed_by": ["bam_stats_samtools"]
                     },
+                    "samtools/index": {
+                        "branch": "master",
+                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "installed_by": ["modules"]
+                    },
                     "samtools/stats": {
                         "branch": "master",
                         "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
@@ -157,7 +163,7 @@
                     },
                     "untar": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "cc1f997fab6d8fde5dc0e6e2a310814df5b53ce7",
                         "installed_by": ["bcl_demultiplex"]
                     }
                 }

modules/local/elprep/sfm/main.nf

@@ -0,0 +1,40 @@
+process ELPREP_SFM {
+    tag "$meta.id"
+    label 'process_high'
+
+    conda "bioconda::elprep=5.1.2"
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/elprep:5.1.2--he881be0_0':
+        'quay.io/biocontainers/elprep:5.1.2--he881be0_0' }"
+
+    input:
+    tuple val(meta), path(bam, stageAs: "input/*")
+
+    output:
+    tuple val(meta), path("output/**.{bam,sam}")    ,emit: bam
+    tuple val(meta), path("*.metrics.txt")          ,optional: true, emit: metrics
+    path "versions.yml"                             ,emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    def suffix = args.contains("--output-type sam") ? "sam" : "bam"
+
+    // single end
+    def single_end_cmd = meta.single_end ? "--single-end" : ""
+
+    """
+    elprep sfm ./input output/${prefix}.${suffix} \\
+        ${single_end_cmd} \\
+        --nr-of-threads ${task.cpus} \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        elprep: \$(elprep 2>&1 | head -n2 | tail -n1 |sed 's/^.*version //;s/ compiled.*\$//')
+    END_VERSIONS
+    """
+}