From e4da3de0a67bb4a51fa28ded13070492b314e76e Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Wed, 27 Aug 2025 17:30:20 +0000 Subject: [PATCH 1/6] Swap fasta_index_dna to nf-test --- subworkflows/nf-core/fasta_index_dna/main.nf | 77 ++--- .../fasta_index_dna/tests/main.nf.test | 289 ++++++++++++++++++ .../fasta_index_dna/tests/main.nf.test.snap | 24 ++ tests/config/pytest_modules.yml | 3 - .../nf-core/fasta_index_dna/main.nf | 65 ---- .../nf-core/fasta_index_dna/nextflow.config | 5 - .../nf-core/fasta_index_dna/test.yml | 134 -------- 7 files changed, 345 insertions(+), 252 deletions(-) create mode 100644 subworkflows/nf-core/fasta_index_dna/tests/main.nf.test create mode 100644 subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap delete mode 100644 tests/subworkflows/nf-core/fasta_index_dna/main.nf delete mode 100644 tests/subworkflows/nf-core/fasta_index_dna/nextflow.config delete mode 100644 tests/subworkflows/nf-core/fasta_index_dna/test.yml diff --git a/subworkflows/nf-core/fasta_index_dna/main.nf b/subworkflows/nf-core/fasta_index_dna/main.nf index 3274f8aa7a85..a6924738929e 100644 --- a/subworkflows/nf-core/fasta_index_dna/main.nf +++ b/subworkflows/nf-core/fasta_index_dna/main.nf @@ -14,7 +14,7 @@ workflow FASTA_INDEX_DNA { take: ch_fasta // channel: [mandatory] [ val(meta), path(fasta) ] - ch_altliftover // channel: [mandatory, if aligner is bwamem or bwamem2 or snap] [ val(meta), path(altliftover) ] + ch_altliftover // channel: [optional, only used if aligner is snap] [ val(meta), path(altliftover) ] val_aligner // string: [mandatory] aligner [bowtie2, bwamem, bwamem2, dragmap, snap] main: @@ -22,53 +22,40 @@ workflow FASTA_INDEX_DNA { ch_aligner_index = Channel.empty() ch_versions = Channel.empty() - switch (val_aligner) { - case 'bowtie2': - BOWTIE2_BUILD(ch_fasta ) // if aligner is bowtie2 - ch_aligner_index = ch_aligner_index.mix(BOWTIE2_BUILD.out.index) - ch_versions = ch_versions.mix(BOWTIE2_BUILD.out.versions) - break - case 'bwamem': - BWAMEM1_INDEX(ch_fasta) // If aligner is bwa-mem - ch_aligner_index = ch_aligner_index - .mix( - BWAMEM1_INDEX.out.index - .join(ch_altliftover) - .map{meta, index, alt -> [meta, index + alt]} - ) - - ch_versions = ch_versions.mix(BWAMEM1_INDEX.out.versions) - break - case 'bwamem2': - BWAMEM2_INDEX(ch_fasta) // If aligner is bwa-mem2 - ch_aligner_index = ch_aligner_index - .mix( - BWAMEM2_INDEX.out.index - .join(ch_altliftover) - .map{meta, index, alt -> [meta, index + alt]} - ) - - ch_versions = ch_versions.mix(BWAMEM2_INDEX.out.versions) - break - case 'dragmap': - DRAGMAP_HASHTABLE(ch_fasta) // If aligner is dragmap - ch_aligner_index = ch_aligner_index.mix(DRAGMAP_HASHTABLE.out.hashmap) - ch_versions = ch_versions.mix(DRAGMAP_HASHTABLE.out.versions) - break - case 'snap': - ch_snap_reference = ch_fasta - .join(ch_altliftover) - .map {meta, fasta, alt -> [meta, fasta, [], [], alt]} + if(ch_altliftover && val_aligner != "snap") { + error "Liftover file is only currently supported for aligner `snap`" + } - SNAP_INDEX(ch_snap_reference) // If aligner is snap - ch_aligner_index = ch_aligner_index.mix(SNAP_INDEX.out.index) - ch_versions = ch_versions.mix(SNAP_INDEX.out.versions) - break - default: - error "Unknown aligner: ${val_aligner}" + // Handle different aligners using conditional logic + if (val_aligner == 'bowtie2') { + BOWTIE2_BUILD(ch_fasta) + ch_aligner_index = BOWTIE2_BUILD.out.index + ch_versions = BOWTIE2_BUILD.out.versions + } else if (val_aligner == 'bwamem') { + BWAMEM1_INDEX(ch_fasta) + ch_aligner_index = BWAMEM1_INDEX.out.index + ch_versions = BWAMEM1_INDEX.out.versions + } else if (val_aligner == 'bwamem2') { + BWAMEM2_INDEX(ch_fasta) + ch_aligner_index = BWAMEM2_INDEX.out.index + ch_versions = BWAMEM2_INDEX.out.versions + } else if (val_aligner == 'dragmap') { + DRAGMAP_HASHTABLE(ch_fasta) + ch_aligner_index = DRAGMAP_HASHTABLE.out.hashmap + ch_versions = DRAGMAP_HASHTABLE.out.versions + } else if (val_aligner == 'snap') { + ch_snap_reference = ch_fasta + .join(ch_altliftover) + .map { meta, fasta_, alt -> [meta, fasta_, [], [], alt] } + + SNAP_INDEX(ch_snap_reference) + ch_aligner_index = SNAP_INDEX.out.index + ch_versions = SNAP_INDEX.out.versions + } else { + error "Unknown aligner: ${val_aligner}" } emit: index = ch_aligner_index // channel: [ val(meta), path(index) ] versions = ch_versions // channel: [ path(versions.yml) ] -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test new file mode 100644 index 000000000000..e57bb8336ae9 --- /dev/null +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test @@ -0,0 +1,289 @@ +nextflow_workflow { + + name "Test Subworkflow FASTA_INDEX_DNA" + script "../main.nf" + workflow "FASTA_INDEX_DNA" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fasta_index_dna" + tag "bowtie2/build" + tag "bwa/index" + tag "bwamem2/index" + tag "dragmap/hashtable" + tag "snapaligner/index" + + test("Params: bowtie2 | generate bowtie2 index") { + + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bowtie2' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: bwamem | generate bwamem1 index") { + + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bwamem' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: bwamem2 | generate bwamem2 index") { + + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bwamem2' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: dragmap | generate dragmap hashtable") { + + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'dragmap' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out.index[1].collect { file(it).name }.toSorted(), + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: snapaligner | generate snapaligner index") { + + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'snap' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out.index[0][1].collect { file(it).name }.toSorted(), + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + + + test("Params: bowtie2 | generate bowtie2 index - stub") { + options '-stub' + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bowtie2' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: bwamem | generate bwamem1 index - stub") { + options '-stub' + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bwamem' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: bwamem2 | generate bwamem2 index - stub") { + options '-stub' + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'bwamem2' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: dragmap | generate dragmap hashtable - stub") { + options '-stub' + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'dragmap' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + test("Params: snapaligner | generate snapaligner index - stub") { + options '-stub' + when { + workflow { + """ + input[0] = Channel.value([ + [id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + ]) + input[1] = Channel.value([ + [id:'test'], + [] + ]) + input[2] = 'snap' + """ + } + } + then { + assert workflow.success + assertAll( + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } + ) + } + } + +} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap new file mode 100644 index 000000000000..624c2cc1ce73 --- /dev/null +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap @@ -0,0 +1,24 @@ +{ + "Params: snapaligner | generate snapaligner index": { + "content": [ + [ + "Genome", + "GenomeIndex", + "GenomeIndexHash", + "OverflowTable" + ], + [ + { + "FASTA_INDEX_DNA:SNAP_INDEX": { + "snapaligner": "2.0.3." + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-27T16:35:31.24373846" + } +} \ No newline at end of file diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index ffa7f0aab3cc..e69de29bb2d1 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1,3 +0,0 @@ -subworkflows/fasta_index_dna: - - subworkflows/nf-core/fasta_index_dna/** - - tests/subworkflows/nf-core/fasta_index_dna/** diff --git a/tests/subworkflows/nf-core/fasta_index_dna/main.nf b/tests/subworkflows/nf-core/fasta_index_dna/main.nf deleted file mode 100644 index 50ae5cd9f96a..000000000000 --- a/tests/subworkflows/nf-core/fasta_index_dna/main.nf +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { FASTA_INDEX_DNA } from '../../../../subworkflows/nf-core/fasta_index_dna/main.nf' - -workflow test_bowtie2_build { - fasta = Channel.value([ - [id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - ]) - altliftover = Channel.value([ - [id:'test'], - [] - ]) - FASTA_INDEX_DNA ( fasta, altliftover, 'bowtie2' ) -} - -workflow test_bwamem1_index { - fasta = Channel.value([ - [id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - ]) - altliftover = Channel.value([ - [id:'test'], - [] - ]) - FASTA_INDEX_DNA ( fasta, altliftover, 'bwamem' ) -} - -workflow test_bwamem2_index { - fasta = Channel.value([ - [id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - ]) - altliftover = Channel.value([ - [id:'test'], - [] - ]) - FASTA_INDEX_DNA ( fasta, altliftover, 'bwamem2' ) -} - -workflow test_dragmap_hashtable { - fasta = Channel.value([ - [id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - ]) - altliftover = Channel.value([ - [id:'test'], - [] - ]) - FASTA_INDEX_DNA ( fasta, altliftover, 'dragmap' ) -} - -workflow test_snap_index { - fasta = Channel.value([ - [id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - ]) - altliftover = Channel.value([ - [id:'test'], - [] - ]) - FASTA_INDEX_DNA ( fasta, altliftover, 'snap' ) -} diff --git a/tests/subworkflows/nf-core/fasta_index_dna/nextflow.config b/tests/subworkflows/nf-core/fasta_index_dna/nextflow.config deleted file mode 100644 index 8730f1c4b930..000000000000 --- a/tests/subworkflows/nf-core/fasta_index_dna/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} diff --git a/tests/subworkflows/nf-core/fasta_index_dna/test.yml b/tests/subworkflows/nf-core/fasta_index_dna/test.yml deleted file mode 100644 index 8e3e49d08cf0..000000000000 --- a/tests/subworkflows/nf-core/fasta_index_dna/test.yml +++ /dev/null @@ -1,134 +0,0 @@ -- name: fasta_index_dna test_bowtie2_build - command: nextflow run ./tests/subworkflows/nf-core/fasta_index_dna -entry test_bowtie2_build -c ./tests/config/nextflow.config - tags: - - bwa - - bwa/index - - bwamem2 - - bwamem2/index - - dragmap - - dragmap/hashtable - - bowtie2 - - bowtie2/build - - snapaligner - - snapaligner/index - - subworkflows - - subworkflows/fasta_index_dna - files: - - path: output/bowtie2/bowtie2/genome.1.bt2 - md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf - - path: output/bowtie2/bowtie2/genome.2.bt2 - md5sum: 47b153cd1319abc88dda532462651fcf - - path: output/bowtie2/bowtie2/genome.3.bt2 - md5sum: 4ed93abba181d8dfab2e303e33114777 - - path: output/bowtie2/bowtie2/genome.4.bt2 - md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: output/bowtie2/bowtie2/genome.rev.1.bt2 - md5sum: 52be6950579598a990570fbcf5372184 - - path: output/bowtie2/bowtie2/genome.rev.2.bt2 - md5sum: e3b4ef343dea4dd571642010a7d09597 - -- name: fasta_index_dna test_bwamem1_index - command: nextflow run ./tests/subworkflows/nf-core/fasta_index_dna -entry test_bwamem1_index -c ./tests/config/nextflow.config - tags: - - bowtie2 - - bowtie2/build - - bwamem2 - - bwamem2/index - - dragmap - - dragmap/hashtable - - bwa - - bwa/index - - snapaligner - - snapaligner/index - - subworkflows - - subworkflows/fasta_index_dna - files: - - path: output/bwamem1/bwa/genome.amb - md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e - - path: output/bwamem1/bwa/genome.ann - md5sum: c32e11f6c859f166c7525a9c1d583567 - - path: output/bwamem1/bwa/genome.bwt - md5sum: 0469c30a1e239dd08f68afe66fde99da - - path: output/bwamem1/bwa/genome.pac - md5sum: 983e3d2cd6f36e2546e6d25a0da78d66 - - path: output/bwamem1/bwa/genome.sa - md5sum: ab3952cabf026b48cd3eb5bccbb636d1 - -- name: fasta_index_dna test_bwamem2_index - command: nextflow run ./tests/subworkflows/nf-core/fasta_index_dna -entry test_bwamem2_index -c ./tests/config/nextflow.config - tags: - - bowtie2 - - bowtie2/build - - bwa - - bwa/index - - dragmap - - dragmap/hashtable - - bwamem2 - - bwamem2/index - - snapaligner - - snapaligner/index - - subworkflows/fasta_index_dna - files: - - path: output/bwamem2/bwamem2/genome.fasta.0123 - md5sum: b02870de80106104abcb03cd9463e7d8 - - path: output/bwamem2/bwamem2/genome.fasta.amb - md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e - - path: output/bwamem2/bwamem2/genome.fasta.ann - md5sum: c32e11f6c859f166c7525a9c1d583567 - - path: output/bwamem2/bwamem2/genome.fasta.bwt.2bit.64 - md5sum: d097a1b82dee375d41a1ea69895a9216 - - path: output/bwamem2/bwamem2/genome.fasta.pac - md5sum: 983e3d2cd6f36e2546e6d25a0da78d66 - -- name: fasta_index_dna test_dragmap_hashtable - command: nextflow run ./tests/subworkflows/nf-core/fasta_index_dna -entry test_dragmap_hashtable -c ./tests/config/nextflow.config - tags: - - bowtie2 - - bowtie2/build - - bwa - - bwa/index - - bwamem2 - - bwamem2/index - - dragmap - - dragmap/hashtable - - snapaligner - - snapaligner/index - - subworkflows - - subworkflows/fasta_index_dna - files: - - path: output/dragmap/dragmap/hash_table.cfg - - path: output/dragmap/dragmap/hash_table.cfg.bin - - path: output/dragmap/dragmap/hash_table.cmp - md5sum: bc210e5358fd65656f9aea297b59ec7d - - path: output/dragmap/dragmap/hash_table_stats.txt - - path: output/dragmap/dragmap/ref_index.bin - md5sum: 8470be9566ecee77eb4aea6a38922a66 - - path: output/dragmap/dragmap/reference.bin - md5sum: b6b5c12a42416b990cd2844de8f33c5d - - path: output/dragmap/dragmap/repeat_mask.bin - md5sum: 2439259a2fd32a1d0f4c53d585f3da3a - - path: output/dragmap/dragmap/str_table.bin - md5sum: 302e2b30993973527e69c6bcd1f093d0 - -- name: fasta_index_dna test_snap_index - command: nextflow run ./tests/subworkflows/nf-core/fasta_index_dna -entry test_snap_index -c ./tests/config/nextflow.config - tags: - - bowtie2 - - bowtie2/build - - bwa - - bwa/index - - bwamem2 - - bwamem2/index - - dragmap - - dragmap/hashtable - - snapaligner - - snapaligner/index - - subworkflows - - subworkflows/fasta_index_dna - files: - - path: output/snap/snap/Genome - md5sum: 7e189c954142ba37460332b467e34ed4 - - path: output/snap/snap/GenomeIndex - md5sum: 298da8bcb1134f7b24379a792a7a46f8 - - path: output/snap/snap/GenomeIndexHash - - path: output/snap/snap/OverflowTable From 2dc665ba295a8fcab4957647671412a39ed1a6ca Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:06:27 +0000 Subject: [PATCH 2/6] Update snapshot --- subworkflows/nf-core/fasta_index_dna/main.nf | 2 +- .../fasta_index_dna/tests/main.nf.test.snap | 54 ++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/subworkflows/nf-core/fasta_index_dna/main.nf b/subworkflows/nf-core/fasta_index_dna/main.nf index a6924738929e..08d6b741d422 100644 --- a/subworkflows/nf-core/fasta_index_dna/main.nf +++ b/subworkflows/nf-core/fasta_index_dna/main.nf @@ -22,7 +22,7 @@ workflow FASTA_INDEX_DNA { ch_aligner_index = Channel.empty() ch_versions = Channel.empty() - if(ch_altliftover && val_aligner != "snap") { + if(ch_altliftover.map{meta, file -> file} && val_aligner != "snap") { error "Liftover file is only currently supported for aligner `snap`" } diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap index 624c2cc1ce73..dde9557ce613 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap @@ -10,7 +10,7 @@ [ { "FASTA_INDEX_DNA:SNAP_INDEX": { - "snapaligner": "2.0.3." + "snapaligner": "2.0.3" } } ] @@ -19,6 +19,56 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-27T16:35:31.24373846" + "timestamp": "2025-08-27T22:34:49.21187818" + }, + "Params: snapaligner | generate snapaligner index - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "Genome:md5,0154c489847fa6b3c612032782171293", + "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", + "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", + "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + ] + ] + ], + "1": [ + "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "Genome:md5,0154c489847fa6b3c612032782171293", + "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", + "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", + "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + ] + ] + ], + "versions": [ + "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" + ] + }, + [ + { + "FASTA_INDEX_DNA:SNAP_INDEX": { + "snapaligner": "2.0.3" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-27T22:35:45.618853501" } } \ No newline at end of file From b9a2e17417c9803764e6308f77e8fd6c964b0851 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Fri, 29 Aug 2025 08:27:33 +0000 Subject: [PATCH 3/6] Add check and update tests --- subworkflows/nf-core/fasta_index_dna/main.nf | 6 +- .../fasta_index_dna/tests/main.nf.test | 20 +- .../fasta_index_dna/tests/main.nf.test.snap | 383 +++++++++++++++++- 3 files changed, 383 insertions(+), 26 deletions(-) diff --git a/subworkflows/nf-core/fasta_index_dna/main.nf b/subworkflows/nf-core/fasta_index_dna/main.nf index 08d6b741d422..5eca28169ae1 100644 --- a/subworkflows/nf-core/fasta_index_dna/main.nf +++ b/subworkflows/nf-core/fasta_index_dna/main.nf @@ -22,8 +22,10 @@ workflow FASTA_INDEX_DNA { ch_aligner_index = Channel.empty() ch_versions = Channel.empty() - if(ch_altliftover.map{meta, file -> file} && val_aligner != "snap") { - error "Liftover file is only currently supported for aligner `snap`" + if (val_aligner != "snap") { + ch_altliftover + .filter { _meta, altliftover -> altliftover != [] } + .subscribe { error "Liftover file is only currently supported for aligner `snap`" } } // Handle different aligners using conditional logic diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test index e57bb8336ae9..5495c5359df8 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test @@ -23,7 +23,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bowtie2' @@ -50,7 +50,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bwamem' @@ -77,7 +77,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bwamem2' @@ -104,7 +104,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'dragmap' @@ -131,7 +131,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'snap' @@ -160,7 +160,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bowtie2' @@ -187,7 +187,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bwamem' @@ -214,7 +214,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'bwamem2' @@ -241,7 +241,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'dragmap' @@ -268,7 +268,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [id:'test'], + [], [] ]) input[2] = 'snap' diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap index dde9557ce613..0253a68c2279 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap @@ -1,4 +1,75 @@ { + "Params: bwamem2 | generate bwamem2 index - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "genome.fasta.0123:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bwt.2bit.64:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.pac:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,7ba3169cff18b2edf7cbd2b5bffab2ae" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "genome.fasta.0123:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bwt.2bit.64:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.pac:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,7ba3169cff18b2edf7cbd2b5bffab2ae" + ] + }, + [ + { + "FASTA_INDEX_DNA:BWAMEM2_INDEX": { + "bwamem2": "2.2.1" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:46:23.833598811" + }, + "Params: dragmap | generate dragmap hashtable": { + "content": [ + [ + + ], + [ + { + "FASTA_INDEX_DNA:DRAGMAP_HASHTABLE": { + "dragmap": "1.2.1" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:45:28.475438395" + }, "Params: snapaligner | generate snapaligner index": { "content": [ [ @@ -21,7 +92,7 @@ }, "timestamp": "2025-08-27T22:34:49.21187818" }, - "Params: snapaligner | generate snapaligner index - stub": { + "Params: bowtie2 | generate bowtie2 index - stub": { "content": [ { "0": [ @@ -30,15 +101,17 @@ "id": "test" }, [ - "Genome:md5,0154c489847fa6b3c612032782171293", - "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", - "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", - "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + "genome.1.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.2.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.3.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.4.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.rev.1.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.rev.2.bt2:md5,d41d8cd98f00b204e9800998ecf8427e" ] ] ], "1": [ - "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" + "versions.yml:md5,3b091297ba4bafebe767e26c7f85a086" ], "index": [ [ @@ -46,21 +119,23 @@ "id": "test" }, [ - "Genome:md5,0154c489847fa6b3c612032782171293", - "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", - "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", - "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + "genome.1.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.2.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.3.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.4.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.rev.1.bt2:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.rev.2.bt2:md5,d41d8cd98f00b204e9800998ecf8427e" ] ] ], "versions": [ - "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" + "versions.yml:md5,3b091297ba4bafebe767e26c7f85a086" ] }, [ { - "FASTA_INDEX_DNA:SNAP_INDEX": { - "snapaligner": "2.0.3" + "FASTA_INDEX_DNA:BOWTIE2_BUILD": { + "bowtie2": "2.5.4" } } ] @@ -69,6 +144,286 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-27T22:35:45.618853501" + "timestamp": "2025-08-28T15:45:50.575444502" + }, + "Params: bwamem | generate bwamem1 index": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "genome.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.bwt:md5,0469c30a1e239dd08f68afe66fde99da", + "genome.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.sa:md5,ab3952cabf026b48cd3eb5bccbb636d1" + ] + ] + ], + "1": [ + "versions.yml:md5,d1b738f7f149345dc417a0f77ca16a1f" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "genome.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.bwt:md5,0469c30a1e239dd08f68afe66fde99da", + "genome.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.sa:md5,ab3952cabf026b48cd3eb5bccbb636d1" + ] + ] + ], + "versions": [ + "versions.yml:md5,d1b738f7f149345dc417a0f77ca16a1f" + ] + }, + [ + { + "FASTA_INDEX_DNA:BWAMEM1_INDEX": { + "bwa": "0.7.18-r1243-dirty" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:45:02.111928619" + }, + "Params: bowtie2 | generate bowtie2 index": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "genome.1.bt2:md5,cbe3d0bbea55bc57c99b4bfa25b5fbdf", + "genome.2.bt2:md5,47b153cd1319abc88dda532462651fcf", + "genome.3.bt2:md5,4ed93abba181d8dfab2e303e33114777", + "genome.4.bt2:md5,c25be5f8b0378abf7a58c8a880b87626", + "genome.rev.1.bt2:md5,52be6950579598a990570fbcf5372184", + "genome.rev.2.bt2:md5,e3b4ef343dea4dd571642010a7d09597" + ] + ] + ], + "1": [ + "versions.yml:md5,3b091297ba4bafebe767e26c7f85a086" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "genome.1.bt2:md5,cbe3d0bbea55bc57c99b4bfa25b5fbdf", + "genome.2.bt2:md5,47b153cd1319abc88dda532462651fcf", + "genome.3.bt2:md5,4ed93abba181d8dfab2e303e33114777", + "genome.4.bt2:md5,c25be5f8b0378abf7a58c8a880b87626", + "genome.rev.1.bt2:md5,52be6950579598a990570fbcf5372184", + "genome.rev.2.bt2:md5,e3b4ef343dea4dd571642010a7d09597" + ] + ] + ], + "versions": [ + "versions.yml:md5,3b091297ba4bafebe767e26c7f85a086" + ] + }, + [ + { + "FASTA_INDEX_DNA:BOWTIE2_BUILD": { + "bowtie2": "2.5.4" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:44:49.995784503" + }, + "Params: bwamem2 | generate bwamem2 index": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", + "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", + "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" + ] + ] + ], + "1": [ + "versions.yml:md5,7ba3169cff18b2edf7cbd2b5bffab2ae" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", + "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", + "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" + ] + ] + ], + "versions": [ + "versions.yml:md5,7ba3169cff18b2edf7cbd2b5bffab2ae" + ] + }, + [ + { + "FASTA_INDEX_DNA:BWAMEM2_INDEX": { + "bwamem2": "2.2.1" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:45:16.675895086" + }, + "Params: bwamem | generate bwamem1 index - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "genome.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,d1b738f7f149345dc417a0f77ca16a1f" + ], + "index": [ + [ + { + "id": "test" + }, + [ + "genome.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,d1b738f7f149345dc417a0f77ca16a1f" + ] + }, + [ + { + "FASTA_INDEX_DNA:BWAMEM1_INDEX": { + "bwa": "0.7.18-r1243-dirty" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:46:10.153060838" + }, + "Params: dragmap | generate dragmap hashtable - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + + ] + ] + ], + "1": [ + "versions.yml:md5,e0cd4e970a93682e54b81dd073448058" + ], + "index": [ + [ + { + "id": "test" + }, + [ + + ] + ] + ], + "versions": [ + "versions.yml:md5,e0cd4e970a93682e54b81dd073448058" + ] + }, + [ + { + "FASTA_INDEX_DNA:DRAGMAP_HASHTABLE": { + "dragmap": "1.2.1" + } + } + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T15:46:37.052988806" + }, + "Params: snapaligner | generate snapaligner index - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "index": [ + + ], + "versions": [ + + ] + }, + [ + + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-08-28T14:41:35.144591698" } } \ No newline at end of file From 554284936002fdac6c65f892206e641d470c63b5 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:44:01 +0000 Subject: [PATCH 4/6] Swap to combine --- subworkflows/nf-core/fasta_index_dna/main.nf | 5 +++-- subworkflows/nf-core/fasta_index_dna/tests/main.nf.test | 6 +++--- .../nf-core/fasta_index_dna/tests/main.nf.test.snap | 8 +------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/subworkflows/nf-core/fasta_index_dna/main.nf b/subworkflows/nf-core/fasta_index_dna/main.nf index 5eca28169ae1..45f2a8f7ad03 100644 --- a/subworkflows/nf-core/fasta_index_dna/main.nf +++ b/subworkflows/nf-core/fasta_index_dna/main.nf @@ -46,9 +46,10 @@ workflow FASTA_INDEX_DNA { ch_aligner_index = DRAGMAP_HASHTABLE.out.hashmap ch_versions = DRAGMAP_HASHTABLE.out.versions } else if (val_aligner == 'snap') { + ch_snap_reference = ch_fasta - .join(ch_altliftover) - .map { meta, fasta_, alt -> [meta, fasta_, [], [], alt] } + .combine(ch_altliftover) + .map { meta, fasta_, _meta2, liftover -> [meta, fasta_, [], [], liftover] } SNAP_INDEX(ch_snap_reference) ch_aligner_index = SNAP_INDEX.out.index diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test index 5495c5359df8..dee512942206 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test @@ -131,7 +131,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [], + [id:'test'], [] ]) input[2] = 'snap' @@ -142,7 +142,7 @@ nextflow_workflow { assert workflow.success assertAll( { assert snapshot( - workflow.out.index[0][1].collect { file(it).name }.toSorted(), + //workflow.out.index[0][1].collect { file(it).name }.toSorted(), workflow.out.versions.collect{ path(it).yaml } ).match() } ) @@ -268,7 +268,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ]) input[1] = Channel.value([ - [], + [[id:'test']], [] ]) input[2] = 'snap' diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap index 0253a68c2279..0bb1a259b863 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap @@ -72,12 +72,6 @@ }, "Params: snapaligner | generate snapaligner index": { "content": [ - [ - "Genome", - "GenomeIndex", - "GenomeIndexHash", - "OverflowTable" - ], [ { "FASTA_INDEX_DNA:SNAP_INDEX": { @@ -90,7 +84,7 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-27T22:34:49.21187818" + "timestamp": "2025-08-29T14:43:37.524741686" }, "Params: bowtie2 | generate bowtie2 index - stub": { "content": [ From 5a51a904dd0f77cf266fb9dc6c702ff5b8114cca Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:54:20 +0000 Subject: [PATCH 5/6] Update snapshot --- .../fasta_index_dna/tests/main.nf.test.snap | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap index 0bb1a259b863..5112e272d79e 100644 --- a/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_index_dna/tests/main.nf.test.snap @@ -398,26 +398,50 @@ "content": [ { "0": [ - + [ + { + "id": "test" + }, + [ + "Genome:md5,0154c489847fa6b3c612032782171293", + "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", + "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", + "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + ] + ] ], "1": [ - + "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" ], "index": [ - + [ + { + "id": "test" + }, + [ + "Genome:md5,0154c489847fa6b3c612032782171293", + "GenomeIndex:md5,396ff91fd60821ae64be3aabe1dc98e7", + "GenomeIndexHash:md5,93800486f6f0a848562ef44675766907", + "OverflowTable:md5,719c5fe8b14741a4afb6f9b0e270db98" + ] + ] ], "versions": [ - + "versions.yml:md5,9bcabb7f47e8ffabdcbcab4df5312716" ] }, [ - + { + "FASTA_INDEX_DNA:SNAP_INDEX": { + "snapaligner": "2.0.3" + } + } ] ], "meta": { "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-28T14:41:35.144591698" + "timestamp": "2025-08-29T14:53:31.177309529" } } \ No newline at end of file From c659cc069217ff477b653600f9d62171ba4ecbfd Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Mon, 8 Sep 2025 08:45:12 +0000 Subject: [PATCH 6/6] Remove remaining pytest files --- .github/scripts/remove_pytest.bash | 12 - .github/workflows/pytest-workflow.yml | 333 -------------------------- pytest.ini | 1 - tests/config/pytest_modules.yml | 0 tests/test_versions_yml.py | 61 ----- 5 files changed, 407 deletions(-) delete mode 100755 .github/scripts/remove_pytest.bash delete mode 100644 .github/workflows/pytest-workflow.yml delete mode 100644 pytest.ini delete mode 100644 tests/config/pytest_modules.yml delete mode 100644 tests/test_versions_yml.py diff --git a/.github/scripts/remove_pytest.bash b/.github/scripts/remove_pytest.bash deleted file mode 100755 index e1c9def9b389..000000000000 --- a/.github/scripts/remove_pytest.bash +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env cached-nix-shell -#! nix-shell -i bash -p fd yq-go - - -# Find modules that have a tests directory -tested=$(fd main.nf.test modules/) - -for module in $tested; do - clean=$(dirname $module | sed 's|/tests||' | sed 's|modules/nf-core/||') - yq -i "del(.${clean})" tests/config/pytest_modules.yml - # rm -rf "tests/${clean}" -done diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml deleted file mode 100644 index 4626d4e52b6e..000000000000 --- a/.github/workflows/pytest-workflow.yml +++ /dev/null @@ -1,333 +0,0 @@ -name: Run pytest-workflow -on: - push: - branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "24.10.2" - -jobs: - pytest-changes: - name: pytest-changes - runs-on: - - runs-on=${{ github.run_id }}-pytest-changes - - runner=4cpu-linux-x64 - - image=ubuntu22-full-x64 - outputs: - tags: ${{ steps.filter.outputs.changes }} - modules: ${{ steps.tags.outputs.modules }} - subworkflows: ${{ steps.tags.outputs.subworkflows }} - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - with: - fetch-depth: 2 # To retrieve the preceding commit. - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - token: "" - - - name: Fetch module tags - id: tags - run: | - echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.tags.outputs.modules }} - echo ${{ steps.tags.outputs.subworkflows }} - - pytest: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: pytest - needs: [pytest-changes] - if: needs.pytest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] - profile: [conda, docker, singularity] - exclude: - - tags: nf-test - - profile: conda - tags: backsub - - profile: conda - tags: bases2fastq - - profile: singularity - tags: bases2fastq - - profile: conda - tags: basicpy - - profile: conda - tags: bcl2fastq - - profile: conda - tags: bclconvert - - profile: conda - tags: bwa/aln - - profile: conda - tags: bwa/index - - profile: conda - tags: bwa/mem - - profile: conda - tags: bwa/sampe - - profile: conda - tags: bwa/samse - - profile: conda - tags: cellpose - - profile: conda - tags: cellrangerarc/mkfastq - - profile: conda - tags: cellrangeratac/mkfastq - - profile: conda - tags: checkm2/databasedownload - - profile: conda - tags: checkm2/predict - - profile: conda - tags: controlfreec/makegraph2 - - profile: conda - tags: deepcell/mesmer - - profile: conda - tags: deepvariant - - profile: conda - tags: fastani - - profile: conda - tags: fastk/fastk - - profile: conda - tags: fastk/histex - - profile: conda - tags: fastk/merge - - profile: conda - tags: fcs/fcsadaptor - - profile: conda - tags: fcs/fcsgx - - profile: conda - tags: gatk4/cnnscorevariants - - profile: conda - tags: gatk4/determinegermlinecontigploidy - - profile: singularity - tags: gatk4/determinegermlinecontigploidy - - profile: conda - tags: gatk4/germlinecnvcaller - - profile: conda - tags: gatk4/postprocessgermlinecnvcalls - - profile: conda - tags: genescopefk - - profile: conda - tags: happy/sompy - - profile: conda - tags: hlala/preparegraph - - profile: conda - tags: imputeme/vcftoprs - - profile: conda - tags: islandpath - - profile: conda - tags: manta/convertinversion - - profile: conda - tags: mcstaging/imc2mc - - profile: conda - tags: mcquant - - profile: conda - tags: medaka - - profile: conda - tags: merquryfk/katcomp - - profile: conda - tags: merquryfk/katgc - - profile: conda - tags: merquryfk/merquryfk - - profile: conda - tags: merquryfk/ploidyplot - - profile: conda - tags: minimap2/align - - profile: conda - tags: mitohifi/findmitoreference - - profile: conda - tags: mitohifi/mitohifi - - profile: conda - tags: nanoplot - - profile: conda - tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/applybqsr - - profile: conda - tags: parabricks/dbsnp - - profile: conda - tags: parabricks/deepvariant - - profile: conda - tags: parabricks/fq2bam - - profile: conda - tags: parabricks/genotypegvcf - - profile: conda - tags: parabricks/haplotypecaller - - profile: conda - tags: parabricks/indexgvcf - - profile: conda - tags: parabricks/mutectcaller - - profile: conda - tags: picard/collecthsmetrics - - profile: conda - tags: picard/collectwgsmetrics - - profile: conda - tags: sentieon/applyvarcal - - profile: conda - tags: sentieon/datametrics - - profile: conda - tags: sentieon/dnamodelapply - - profile: conda - tags: sentieon/dnascope - - profile: conda - tags: sentieon/readwriter - - profile: conda - tags: sentieon/tnfilter - - profile: conda - tags: sentieon/tnhaplotyper2 - - profile: conda - tags: sentieon/tnscope - - profile: conda - tags: sentieon/varcal - - profile: conda - tags: sentieon/wgsmetrics - - profile: conda - tags: subworkflows/bam_qc_picard - - profile: conda - tags: subworkflows/bcl_demultiplex - - profile: conda - tags: subworkflows/fasta_clean_fcs - - profile: conda - tags: svanalyzer/svbenchmark - - profile: conda - tags: svtk/standardize - - profile: conda - tags: universc - - profile: singularity - tags: universc - - profile: conda - tags: vt/decompose - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - - - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6 - with: - python-version: "3.13" - - - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 - id: cache-pip-pytest - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pytest - restore-keys: | - ${{ runner.os }}-pip-pytest - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 - with: - distribution: "temurin" - java-version: "17" - - - name: Set up Nextflow - uses: nf-core/setup-nextflow@v2 - with: - version: ${{ env.NXF_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Conda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3 - with: - auto-update-conda: true - conda-solver: libmamba - conda-remove-defaults: true - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/ubuntu/pytest_workflow_*/*/.nextflow.log - /home/ubuntu/pytest_workflow_*/*/log.out - /home/ubuntu/pytest_workflow_*/*/log.err - /home/ubuntu/pytest_workflow_*/*/work - !/home/ubuntu/pytest_workflow_*/*/work/conda - !/home/ubuntu/pytest_workflow_*/*/work/singularity - !${{ github.workspace }}/.singularity - - confirm-pass-pytest: - runs-on: - - runs-on=${{ github.run_id }}-confirm-pass-pytest - - runner=4cpu-linux-x64 - - image=ubuntu22-full-x64 - needs: [pytest-changes, pytest] - if: always() - steps: - - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index eea2c180278f..000000000000 --- a/pytest.ini +++ /dev/null @@ -1 +0,0 @@ -[pytest] diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py deleted file mode 100644 index 26e7712bb3fa..000000000000 --- a/tests/test_versions_yml.py +++ /dev/null @@ -1,61 +0,0 @@ -import re -from pathlib import Path -from textwrap import dedent - -import pytest -import yaml - - -def _get_workflow_names(): - """Get all names of all workflows which have a test.yml in the tests directory. - - To do so, recursively finds all test.yml files and parses their content. - """ - here = Path(__file__).parent.resolve() - pytest_workflow_files = here.glob("modules/**/test.yml") - for f in pytest_workflow_files: - # test_config = yaml.safe_load(f.read_text()) - test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) - for workflow in test_config: - # https://github.com/nf-core/modules/pull/1242 - added to cover tests - # that expect an error and therefore will not generate a versions.yml - if "exit_code" not in workflow: - yield workflow["name"] - - -@pytest.mark.workflow(*_get_workflow_names()) -def test_ensure_valid_version_yml(workflow_dir): - workflow_dir = Path(workflow_dir) - software_name = workflow_dir.name.split("_")[0].lower() - try: - versions_yml_file = workflow_dir / f"output/{software_name}/versions.yml" - versions_yml = versions_yml_file.read_text() - except FileNotFoundError: - raise AssertionError( - dedent( - f"""\ - `versions.yml` not found in the output directory. - Expected path: `{versions_yml_file}` - - This can have multiple reasons: - * The test-workflow failed before a `versions.yml` could be generated. - * The workflow name in `test.yml` does not start with the tool name. - """ - ) - ) - - assert ( - "END_VERSIONS" not in versions_yml - ), "END_VERSIONS detected in versions.yml. This is a sign of an ill-formatted HEREDOC" - - # Raises an exception if yaml is not valid - versions = yaml.safe_load(versions_yml) - assert ( - len(versions) == 1 - ), "The top-level of versions.yml must contain exactly one entry: the process name as dict key" - software_versions = next(iter(versions.values())) - assert len(software_versions), "There must be at least one version emitted." - for tool, version in software_versions.items(): - assert re.match( - r"^\d.*|^[a-f0-9]{40}$", str(version) - ), f"Version number for {tool} must start with a number, or be a Git SHA commit id. "