Merge pull request #27 from sanger-tol/fixes_before_release

Enable organelles subworkflow
sanger-tol · Mar 22, 2024 · 85f0dbf · 85f0dbf
2 parents 547f524 + d99f5cc
commit 85f0dbf
Show file tree

Hide file tree

Showing 23 changed files with 636 additions and 187 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,13 +22,19 @@ jobs:
     name: Run pipeline with test data
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/genomeassembly') }}"
-    runs-on: ubuntu2204-16c
+    runs-on: ubuntu2204-4c
     strategy:
       matrix:
         NXF_VER:
           - "22.10.1"
           - "latest-everything"
     steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Check out pipeline code
         uses: actions/checkout@v3
 
@@ -37,6 +43,10 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
+      - name: Set up nextflow secrets
+        run: |
+          nextflow secrets set NCBI_API_KEY ${{ secrets.NCBI_API_KEY }}
+
       - name: Download test data
         run: |
           curl https://darwin.cog.sanger.ac.uk/genomeassembly_test_data.tar.gz | tar xzf -

diff --git a/assets/test.yaml b/assets/test.yaml
@@ -4,7 +4,7 @@ dataset:
     reads: /lustre/scratch123/tol/resources/nextflow/test-data/Undibacterium_unclassified/genomic_data/baUndUnlc1/10x/
   pacbio:
     reads:
-      - reads: /lustre/scratch123/tol/resources/nextflow/test-data/Undibacterium_unclassified/genomic_data/baUndUnlc1/pacbio/fasta/HiFi.reads.fasta
+      - reads: /lustre/scratch124/tol/projects/darwin/users/kk16/development/test/test/HiFi.reads.BIG.fasta
   HiC:
     reads:
       - reads: /lustre/scratch123/tol/resources/nextflow/test-data/Undibacterium_unclassified/genomic_data/baUndUnlc1/hic-arima2/41741_2#7.sub.cram
@@ -15,3 +15,4 @@ mito:
   species: Caradrina clavipalpis
   min_length: 15000
   code: 5
+  fam: /lustre/scratch124/tol/projects/darwin/users/cz3/organelle_asm/hmm_db/insecta_mito.fam
diff --git a/assets/test_github.yaml b/assets/test_github.yaml
@@ -15,3 +15,4 @@ mito:
   species: Caradrina clavipalpis
   min_length: 15000
   code: 5
+  fam: /home/runner/work/genomeassembly/genomeassembly/Undibacterium_unclassified/hmm_db/insecta_mito.fam
diff --git a/conf/modules.config b/conf/modules.config
@@ -201,19 +201,27 @@ process {
     // Set up of organelles pipeline
 
     if (params.organelles_on) {
-        withName: '.*ORGANELLES_READS:MITOHIFI_MITOHIFI' {
+        withName: '.*MITOHIFI_MITOHIFI_READS' {
             ext.args2 = '-r'
             publishDir = [
                 path: { "${params.outdir}/${meta.id}.${params.hifiasm}/mito.reads" },
-                mode: params.publish_dir_mode,
+                mode: 'copyNoFollow',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
 
-        withName: '.*ORGANELLES_CONTIGS:MITOHIFI_MITOHIFI' {
+        withName: '.*MITOHIFI_MITOHIFI_CONTIGS' {
             ext.args2 = '-c'
             publishDir = [
                 path: { "${params.outdir}/${meta.id}.${params.hifiasm}/mito" },
+                mode: 'copyNoFollow',
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            ]
+        }
+        withName: '.*OATK' {
+            ext.args = "-k1001 -c90 -Ttmp"
+            publishDir = [
+                path: { "${params.outdir}/${meta.id}.${params.hifiasm}/oatk" },
                 mode: params.publish_dir_mode,
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
@@ -223,7 +231,7 @@ process {
 
     // Set up of the purging pipeline
 
-    withName: '.*PURGE_DUPS_PRI:MINIMAP2_ALIGN_READS' {
+    withName: '.*PURGE_DUPS:MINIMAP2_ALIGN_READS' {
         //these options are from pbmm2 CSS preset
         ext.args = "-k19 -w10 -O5,56 -E4,1 -A2 -B5 -z400,50 -r2000 --lj-min-ratio 0.5"
         ext.prefix = { "${meta.id}.reads" }
@@ -234,31 +242,31 @@ process {
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:PURGEDUPS_PBCSTAT' {
+    withName: '.*PURGE_DUPS:PURGEDUPS_PBCSTAT' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:GET_CALCUTS_PARAMS' {
+    withName: '.*PURGE_DUPS:GET_CALCUTS_PARAMS' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:PURGEDUPS_CALCUTS' {
+    withName: '.*PURGE_DUPS:PURGEDUPS_CALCUTS' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:PURGEDUPS_SPLITFA' {
+    withName: '.*PURGE_DUPS:PURGEDUPS_SPLITFA' {
         ext.prefix = "self_aln"
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/split_aln" },
@@ -267,7 +275,7 @@ process {
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:MINIMAP2_ALIGN_ASSEMBLY' {
+    withName: '.*PURGE_DUPS:MINIMAP2_ALIGN_ASSEMBLY' {
         ext.args   = "-xasm5 -DP"
         ext.prefix = "self_aln"
         publishDir = [
@@ -277,7 +285,7 @@ process {
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:PURGEDUPS_PURGEDUPS' {
+    withName: '.*PURGE_DUPS:PURGEDUPS_PURGEDUPS' {
         ext.args   = "-2"
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/purge_dups" },
@@ -286,14 +294,9 @@ process {
         ]
     }
 
-    withName: '.*PURGE_DUPS_PRI:PURGEDUPS_GETSEQS' {
+    withName: '.*PURGE_DUPS:PURGEDUPS_GETSEQS' {
         ext.prefix = { "${meta.prefix}" }
         publishDir = [
-            [
-                path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/seqs" },
-                mode: params.publish_dir_mode,
-                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-            ],
             [
                 path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/" },
                 mode: params.publish_dir_mode,
@@ -302,82 +305,12 @@ process {
         ]
     }
 
-    withName: '.*PURGE_DUPS_ALT:MINIMAP2_ALIGN_READS' {
-        //these options are from pbmm2 CSS preset
-        ext.args = "-k19 -w10 -O5,56 -E4,1 -A2 -B5 -z400,50 -r2000 --lj-min-ratio 0.5"
-        ext.prefix = { "${meta.id}.reads" }
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage.htigs" },
-            mode: params.publish_dir_mode,
-            pattern: ".*paf"
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:PURGEDUPS_PBCSTAT' {
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage.htigs" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-
-    withName: '.*PURGE_DUPS_ALT:GET_CALCUTS_PARAMS' {
+    withName: '.*CAT_CAT_HAPLOTIGS' {
+        ext.prefix = { "purged.htigs.all.fa" }
         publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage.htigs" },
+            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/" },
             mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:PURGEDUPS_CALCUTS' {
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/coverage.htigs" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:PURGEDUPS_SPLITFA' {
-        ext.prefix = "self_aln"
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/split_aln.htigs" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:MINIMAP2_ALIGN_ASSEMBLY' {
-        ext.args   = "-xasm5 -DP"
-        ext.prefix = "self_aln"
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/split_aln.htigs" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:PURGEDUPS_PURGEDUPS' {
-        ext.args   = "-2"
-        publishDir = [
-            path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/purge_dups.htigs" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*PURGE_DUPS_ALT:PURGEDUPS_GETSEQS' {
-        publishDir = [
-            [
-                path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/seqs.htigs" },
-                mode: params.publish_dir_mode,
-                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-            ],
-            [
-                path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging/" },
-                mode: params.publish_dir_mode,
-                saveAs: { filename -> filename.contains('purged.fa') ? 'purged.htigs.fa' : null }
-            ]
+            pattern: '*purged.htigs.all.fa'
         ]
     }
 
@@ -391,7 +324,7 @@ process {
     }
 
     withName: '.*GENOME_STATISTICS_PURGED:GFASTATS_HAP' {
-        ext.prefix = { "purged.htigs" }
+        ext.prefix = { "purged.htigs.all" }
         publishDir = [
             path: { "${params.outdir}/${meta.id}.${params.hifiasm}/purging" },
             mode: params.publish_dir_mode,
@@ -425,9 +358,6 @@ process {
     // Set up of the polishing pipeline
     if (params.polishing_on) {
         withName: LONGRANGER_MKREF {
-            if(System.getenv('GITHUB_ACTION') != null ) {
-                container = "ghcr.io/sanger-tol/longranger:2.2.2-c3"
-            }
             publishDir = [
                 path: { "${params.outdir}/${meta.id}.${params.hifiasm}/polishing" },
                 mode: params.publish_dir_mode,
@@ -438,9 +368,6 @@ process {
         withName: LONGRANGER_ALIGN {
             // Keep in sync with `longranger_lsf_sanger.config`
             ext.args = "--disable-ui --nopreflight"
-            if(System.getenv('GITHUB_ACTION') != null ) {
-                container = "ghcr.io/sanger-tol/longranger:2.2.2-c3"
-            }
             publishDir = [
                 path: { "${params.outdir}/${meta.id}.${params.hifiasm}/polishing" },
                 mode: params.publish_dir_mode,
@@ -719,4 +646,52 @@ process {
 */
 profiles {
     longranger_lsf_sanger { includeConfig 'longranger_lsf_sanger.config' }
+    test {
+        process {
+            // Set up of the scaffolding pipeline
+            withName: 'YAHS' {
+                // Skip the initial assembly error correction step
+                ext.args = '-r 1000,2000,5000'
+            }
+
+            withName: '.*HIFIASM.*' {
+                // Skip bloom filter
+                ext.args = '--primary -f0'
+            }
+
+            withName: '.*OATK' {
+                // Set kmer size and minimal coverage
+                ext.args = "-k1001 -c5 -Ttmp"
+            }
+        }
+    }
+
+    test_github {
+        process {
+            // Set up of the scaffolding pipeline
+            withName: 'YAHS' {
+                // Skip the initial assembly error correction step
+                ext.args = '-r 1000,2000,5000'
+            }
+
+            withName: '.*HIFIASM.*' {
+                // Skip bloom filter
+                ext.args = '--primary -f0'
+            }
+
+            withName: '.*OATK' {
+                // Set kmer size and minimal coverage
+                ext.args = "-k1001 -c5 -Ttmp"
+            }
+
+            if (params.polishing_on) {
+                withName: LONGRANGER_MKREF {
+                    container = "ghcr.io/sanger-tol/longranger:2.2.2-c4"
+                }
+                withName: LONGRANGER_ALIGN {
+                    container = "ghcr.io/sanger-tol/longranger:2.2.2-c4"
+                }
+            }
+        }
+    }
 }
diff --git a/conf/test.config b/conf/test.config
@@ -22,12 +22,13 @@ params {
     // Input data
     input = "${projectDir}/assets/test.yaml"
     bed_chunks_polishing = 2
+    organelles_on = true
     polishing_on = true
     hifiasm_hic_on = true
 }
 
 process {
-    // Set up of the scffolding eipeline
+    // Set up of the scffolding pipeline
     withName: 'YAHS' {
         // Skip the initial assembly error correction step
         ext.args = '-r 1000,2000,5000'

diff --git a/conf/test_github.config b/conf/test_github.config
@@ -24,13 +24,5 @@ params {
     bed_chunks_polishing = 2
     polishing_on = false
     hifiasm_hic_on = true
-    organelles_on = false
-}
-
-process {
-    // Set up of the scffolding eipeline
-    withName: 'YAHS' {
-        // Skip the initial assembly error correction step
-        ext.args = '-r 1000,2000,5000'
-    }
+    organelles_on = true
 }
diff --git a/docs/usage.md b/docs/usage.md
@@ -112,6 +112,12 @@ nextflow run sanger-tol/genomeassembly --input assets/dataset.yaml --outdir <OUT
 
 This will launch the pipeline with the `docker` configuration profile, also using your institution profille if available (see [nf-core/configs](#nf-core_configs)). See below for more information about profiles.
 
+In case organelles subworkflow is switched on you will also need to set a nextflow secret to store the API key belonging to your user.
+
+```bash
+  nextflow secrets set TOL_API_KEY '[API key]'
+```
+
 Note that the pipeline will create the following files in your working directory:
 
 ```console

diff --git a/modules.json b/modules.json
@@ -128,14 +128,20 @@
                     "mitohifi/findmitoreference": {
                         "branch": "master",
                         "git_sha": "f52220e84bfc16a8616a5bb3d6f5bc67d601bdce",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/mitohifi/findmitoreference/mitohifi-findmitoreference.diff"
                     },
                     "mitohifi/mitohifi": {
                         "branch": "master",
                         "git_sha": "c607e74f7aa72eb7cb7cc0a1454f97d3907e8d84",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/mitohifi/mitohifi/mitohifi-mitohifi.diff"
                     },
+                    "oatk": {
+                        "branch": "master",
+                        "git_sha": "d6a146325058eb9a18da2e898a2376e1d1093052",
+                        "installed_by": ["modules"]
+                    },
                     "pretextmap": {
                         "branch": "master",
                         "git_sha": "0f8a77ff00e65eaeebc509b8156eaa983192474b",