From 4cad518fd931ab1e2dc7f9f457ee1e3086a24f1d Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Fri, 29 Apr 2022 12:21:22 +0200 Subject: [PATCH 1/6] Retry TRFP independent of exit code --- modules/local/thermorawfileparser/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/thermorawfileparser/main.nf b/modules/local/thermorawfileparser/main.nf index 0860c384..f76e7402 100644 --- a/modules/local/thermorawfileparser/main.nf +++ b/modules/local/thermorawfileparser/main.nf @@ -2,6 +2,7 @@ process THERMORAWFILEPARSER { tag "$meta.id" label 'process_low' label 'process_single_thread' + label 'error_retry' conda (params.enable_conda ? "conda-forge::mono bioconda::thermorawfileparser=1.3.4" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? From 9b4ca45daba349cc13a2bfb78ab54d13697d62fb Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 2 May 2022 11:48:47 +0200 Subject: [PATCH 2/6] Run all branches in AWS full test --- .github/workflows/awsfulltest.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 1ea40b4d..2825c72e 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -7,28 +7,32 @@ on: release: types: [published] workflow_dispatch: + jobs: run-tower: name: Run AWS full tests if: github.repository == 'nf-core/quantms' runs-on: ubuntu-latest + # Do a full-scale run with data from each sequencing platform + strategy: + matrix: + mode: ["lfq", "tmt", "dia"] steps: - name: Launch workflow via tower uses: nf-core/tower-action@v3 - # TODO nf-core: You can customise AWS full pipeline tests as required - # Add full size test data (but still relatively small datasets for few samples) - # on the `test_lfq.config` test runs with only one set of parameters with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + pipeline: ${{ github.repository }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/quantms/work-${{ github.sha }} parameters: | { - "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/quantms/results-${{ github.sha }}" + "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/quantms/results-${{ github.sha }}/mode_${{ matrix.mode }}" } - profiles: test_lfq,aws_tower + profiles: test_${{ matrix.mode }},aws_tower nextflow_config: | process.errorStrategy = 'retry' process.maxRetries = 3 From c11c61f829372981d34555ded0c67e7e4b738fba Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 2 May 2022 11:49:36 +0200 Subject: [PATCH 3/6] doc AWS tests --- .github/workflows/awsfulltest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 2825c72e..92234d12 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -1,7 +1,7 @@ name: nf-core AWS full size tests # This workflow is triggered on published releases. # It can be additionally triggered manually with GitHub actions workflow dispatch button. -# It runs the -profile 'test_lfq' on AWS batch +# It runs the -profiles 'test_lfq' 'test_tmt' and 'test_dia' on AWS batch on: release: From 57da6b1c868c55add7bbcbb97ff073ae4d86499e Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 2 May 2022 13:44:39 +0200 Subject: [PATCH 4/6] remove github vars from AWS action --- .github/workflows/awsfulltest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 92234d12..8800adde 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -13,7 +13,7 @@ jobs: name: Run AWS full tests if: github.repository == 'nf-core/quantms' runs-on: ubuntu-latest - # Do a full-scale run with data from each sequencing platform + # Do a full-scale run with data from each acquisition/quantification mode strategy: matrix: mode: ["lfq", "tmt", "dia"] @@ -25,8 +25,6 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - pipeline: ${{ github.repository }} - revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/quantms/work-${{ github.sha }} parameters: | { From 84aa31ae95e2cf43a8ca2e99418f2fe42aead755 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 2 May 2022 15:25:56 +0200 Subject: [PATCH 5/6] [FIX] specify container for bash-only modules for AWS --- modules/local/preprocess_expdesign.nf | 13 +++++++++++-- subworkflows/local/create_input_channel.nf | 8 ++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/modules/local/preprocess_expdesign.nf b/modules/local/preprocess_expdesign.nf index 61638a9a..779fcf41 100644 --- a/modules/local/preprocess_expdesign.nf +++ b/modules/local/preprocess_expdesign.nf @@ -1,19 +1,28 @@ // Fixing file endings only necessary if the experimental design is user-specified +// TODO can we combine this with another step? Feels like a waste to spawn a worker for this. +// Maybe the renaming can be done in the rawfileconversion step? Or check if the OpenMS tools +// accept different file endings already? process PREPROCESS_EXPDESIGN { label 'process_very_low' label 'process_single_thread' + container "frolvlad/alpine-bash" + input: path design output: path "experimental_design.tsv", emit: ch_expdesign - path "process_experimental_design.tsv", emit: process_ch_expdesign + path "config.tsv", emit: ch_config script: """ + # since we know that we will need to convert from raw to mzML for all tools that need the design (i.e., OpenMS tools) + # we edit the design here and change the endings. sed 's/.raw\\t/.mzML\\t/I' $design > experimental_design.tsv - a=\$(grep -n '^\$' $design | head -n1| awk -F":" '{print \$1}'); sed -e ''"\${a}"',\$d' $design > process_experimental_design.tsv + + # here we extract the filenames and fake an empty config (since the config values will be deduced from the workflow params) + a=\$(grep -n '^\$' $design | head -n1| awk -F":" '{print \$1}'); sed -e ''"\${a}"',\$d' $design > config.tsv """ } diff --git a/subworkflows/local/create_input_channel.nf b/subworkflows/local/create_input_channel.nf index 256335b7..93109b8e 100644 --- a/subworkflows/local/create_input_channel.nf +++ b/subworkflows/local/create_input_channel.nf @@ -20,14 +20,14 @@ workflow CREATE_INPUT_CHANNEL { if (is_sdrf.toString().toLowerCase().contains("true")) { SDRFPARSING ( ch_sdrf_or_design ) ch_versions = ch_versions.mix(SDRFPARSING.out.version) - ch_in_design = SDRFPARSING.out.ch_sdrf_config_file + ch_config = SDRFPARSING.out.ch_sdrf_config_file ch_expdesign = SDRFPARSING.out.ch_expdesign } else { PREPROCESS_EXPDESIGN( ch_sdrf_or_design ) - ch_in_design = PREPROCESS_EXPDESIGN.out.process_ch_expdesign + ch_config = PREPROCESS_EXPDESIGN.out.ch_config - ch_expdesign = PREPROCESS_EXPDESIGN.out.ch_expdesign + ch_expdesign = PREPROCESS_EXPDESIGN.out.ch_expdesign } Set enzymes = [] @@ -38,7 +38,7 @@ workflow CREATE_INPUT_CHANNEL { wrapper.labelling_type = "" wrapper.acquisition_method = "" - ch_in_design.splitCsv(header: true, sep: '\t') + ch_config.splitCsv(header: true, sep: '\t') .map { create_meta_channel(it, is_sdrf, enzymes, files, wrapper) } .branch { ch_meta_config_dia: it[0].acquisition_method.contains("dia") From 1949e798a4b45b56130504e3810d39f72294c820 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 2 May 2022 16:56:07 +0200 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25934400..61b70837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.0.0] nfcore/quantms - [18/03/2022] - Havana +## [1.0] nfcore/quantms - [05/02/2022] - Havana Initial release of nf-core/quantms, created with the [nf-core](https://nf-co.re/) template. @@ -11,9 +11,9 @@ Initial release of nf-core/quantms, created with the [nf-core](https://nf-co.re/ - New pipeline for DDA-LFQ data analysis - New pipeline for DDA-ISO data analysis -- New datasets for DDA-LFQ and DDA-ISO data analsysis +- New datasets for DDA-LFQ and DDA-ISO data analysis - Documentation added for DDA pipeline -- First pipeline for DIA-LFQ data analsysis +- First pipeline for DIA-LFQ data analysis ### `Fixed` @@ -23,16 +23,18 @@ Initial release of nf-core/quantms, created with the [nf-core](https://nf-co.re/ The pipeline is using Nextflow DSL2, each process will be run with its own [Biocontainer](https://biocontainers.pro/#/registry). This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference. -| Dependency | Version | -| ---------------- | ---------- | -| `comet` | 2021010 | -| `msgf+` | 2022.01.07 | -| `openms` | 2.8.0 | -| `sdrf-pipelines` | 0.0.21 | -| `percolator` | 3.5 | -| `pmultiqc` | 0.0.10 | -| `luciphor` | 2020_04_03 | -| `dia-nn` | 1.8.1 | -| `msstats` | 4.2.0 | +| Dependency | Version | +| --------------------- | ---------- | +| `thermorawfileparser` | 1.3.4 | +| `comet` | 2021010 | +| `msgf+` | 2022.01.07 | +| `openms` | 2.8.0 | +| `sdrf-pipelines` | 0.0.21 | +| `percolator` | 3.5 | +| `pmultiqc` | 0.0.11 | +| `luciphor` | 2020_04_03 | +| `dia-nn` | 1.8.1 | +| `msstats` | 4.2.0 | +| `msstatstmt` | 2.2.0 | ### `Deprecated`