Merge pull request #15 from ypriverol/dev

Dev

.github/workflows/awsfulltest.yml

@@ -1,23 +1,26 @@
 name: nf-core AWS full size tests
 # This workflow is triggered on published releases.
 # It can be additionally triggered manually with GitHub actions workflow dispatch button.
-# It runs the -profile 'test_full' on AWS batch
+# It runs the -profiles 'test_lfq' 'test_tmt' and 'test_dia' on AWS batch
 
 on:
   release:
     types: [published]
   workflow_dispatch:
+
 jobs:
   run-tower:
     name: Run AWS full tests
     if: github.repository == 'nf-core/quantms'
     runs-on: ubuntu-latest
+    # Do a full-scale run with data from each acquisition/quantification mode
+    strategy:
+      matrix:
+        mode: ["lfq", "tmt", "dia"]
     steps:
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
+        uses: seqeralabs/action-tower-launch@v2
+
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
@@ -27,11 +30,11 @@ jobs:
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/quantms/results-${{ github.sha }}"
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/quantms/results-${{ github.sha }}/mode_${{ matrix.mode }}"
             }
-          profiles: test_full
+          profiles: test_${{ matrix.mode }}
 
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+      - uses: actions/upload-artifact@v4
         with:
           name: Tower debug log file
           path: |

.github/workflows/awstest.yml

@@ -1,6 +1,6 @@
 name: nf-core AWS test
 # This workflow can be triggered manually with the GitHub actions workflow dispatch button.
-# It runs the -profile 'test' on AWS batch
+# It runs the -profile 'test_tmt' on AWS batch
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ jobs:
     steps:
       # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
@@ -23,9 +23,9 @@ jobs:
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/quantms/results-test-${{ github.sha }}"
             }
-          profiles: test
+          profiles: test_tmt
 
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+      - uses: actions/upload-artifact@v4
         with:
           name: Tower debug log file
           path: |

.github/workflows/ci.yml

@@ -17,30 +17,88 @@ concurrency:
 
 jobs:
   test:
+    env:
+      NXF_ANSI_LOG: false
+      CAPSULE_LOG: none
+      TEST_PROFILE: ${{ matrix.test_profile }}
+      EXEC_PROFILE: ${{ matrix.exec_profile }}
+
     name: Run pipeline with test data
     # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/quantms') }}"
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/quantms') }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
+        # Nextflow versions
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
+        test_profile: ["test_lfq", "test_lfq_sage", "test_dia", "test_localize", "test_tmt"]
+        exec_profile: ["docker", "conda"]
+        exclude:
+          - test_profile: test_dia
+            exec_profile: conda
+          - test_profile: test_localize
+            exec_profile: conda
+          - NXF_VER: "latest-everything"
+            exec_profile: "conda"
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1
+        uses: nf-core/setup-nextflow@v1
         with:
           version: "${{ matrix.NXF_VER }}"
 
       - name: Disk space cleanup
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
+      - name: Install micromamba
+        if: matrix.exec_profile == 'conda'
+        run: |
+          wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+          echo "$(pwd)/bin" >> $GITHUB_PATH
+          echo "$(pwd)/micromamba/bin" >> $GITHUB_PATH
+          ./bin/micromamba shell init -s bash -p ./micromamba
+          echo $'channels:\n  - conda-forge\n  - bioconda\n  - defaults\nuse_lockfiles: false' >> ~/.mambarc
+
       - name: Run pipeline with test data
+        if: matrix.exec_profile != 'conda'
         # TODO nf-core: You can customise CI pipeline run tests as required
         # For example: adding multiple test runs with different parameters
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,$EXEC_PROFILE --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results
+      - name: Run pipeline with test data in conda profile (and single-threaded)
+        if: matrix.exec_profile == 'conda'
+        # TODO nf-core: You can customise CI pipeline run tests as required
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,micromamba --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results
+      - name: Gather failed logs
+        if: failure() || cancelled()
+        run: |
+          mkdir failed_logs
+          failed=$(grep "FAILED" ${TEST_PROFILE}_${EXEC_PROFILE}_results/pipeline_info/execution_trace.txt | cut -f 2)
+          while read -r line ; do cp $(ls work/${line}*/*.log) failed_logs/ | true ; done <<< "$failed"
+      - uses: actions/upload-artifact@v1
+        if: failure() || cancelled()
+        name: Upload failed logs
+        with:
+          name: failed_logs
+          path: failed_logs
+      - uses: actions/upload-artifact@v1
+        if: always()
+        name: Upload results
+        with:
+          name: ${{ env.TEST_PROFILE }}_${{ env.EXEC_PROFILE }}_results
+          path: ${{ env.TEST_PROFILE }}_${{ env.EXEC_PROFILE }}_results
+      - uses: actions/upload-artifact@v1
+        if: always()
+        name: Upload log
+        with:
+          name: nextflow.log
+          path: .nextflow.log

.gitignore

@@ -6,3 +6,16 @@ results/
 testing/
 testing*
 *.pyc
+.idea/
+.idea/*
+*.log
+/build/
+results*/
+venv/
+node_modules
+conversion_inputs
+debug_dir
+test_out
+
+lint_log.txt
+node_modules

.nf-core.yml

@@ -1 +1,8 @@
 repository_type: pipeline
+lint:
+  files_exist:
+    - conf/igenomes.config
+    - conf/test_full.config
+    - conf/test.config
+  files_unchanged:
+    - .github/PULL_REQUEST_TEMPLATE.md

.prettierignore

@@ -10,3 +10,5 @@ testing/
 testing*
 *.pyc
 bin/
+venv/
+

CHANGELOG.md

@@ -3,14 +3,139 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v1.3.0dev - [date]
+## [1.3.0dev] nfcore/quantms - [TBD] - Vaduz
+
+### `Added`
+
+### `Changed`
+
+### `Fixed`
+
+### `Dependencies`
+
+### `Parameters`
+
+### `Deprecations`
+
+## [1.2.0] nfcore/quantms - [11/02/2023] - Thimphu
+
+### `Added`
+
+- [#275 BigBio](https://github.com/bigbio/quantms/pull/275) Added support for bruker data in DIA branch.
+- [#275 BigBio](https://github.com/bigbio/quantms/pull/275) And speed-up to DIA-NN pipeline.
+- [#275 BigBio](https://github.com/bigbio/quantms/pull/275) Support for library-base search in DIA-NN pipeline.
+- [#300 BigBio](https://github.com/bigbio/quantms/pull/300) Major refactoring of LFQ-DDA MBR algorithm.
+- [#279 BigBio](https://github.com/bigbio/quantms/pull/279) Support for SAGE search engine.
+
+### `Changed`
+
+- [#314](https://github.com/bigbio/quantms/pull/314) Update for pmultiqc to pmultiqc=0.0.23
+- [#308](https://github.com/bigbio/quantms/pull/308) Update for openms to openms=3.1.0
+- Update for sdrf-pipelines to sdrf-pipelines=0.0.24
+- Update for msstats to msstats=4.2.1
+
+### `Fixed`
+
+- [#316](https://github.com/bigbio/quantms/pull/316) Fixed jar path selection of luciphoradapter and msgf+
+- Fixed bug where modification masses were not calculated correctly in DIA-NN conversion.
+- Fixed multiple bugs Pull Request [#293 BigBio](https://github.com/bigbio/quantms/pull/293), [#279 BigBio](https://github.com/bigbio/quantms/pull/279), [#265 BigBio](https://github.com/bigbio/quantms/pull/265), [#260 BigBio](https://github.com/bigbio/quantms/pull/260), [#257 BigBio](https://github.com/bigbio/quantms/pull/257)
+
+### `Dependencies`
+
+- New dependency on `sage` search engine.
+
+### `Parameters`
+
+- feature_with_id_min_score: Minimum score of a feature with a peptide identification (default: 0.10)
+- feature_without_id_min_score: Minimum score of a feature without peptide identification (transfer feature, default: 0.75)
+- lfq_intensity_threshold: Minimum intensity of a feature to be considered in the MBR algorithm (default: 1000)
+- sage_processes: Number of processes to use in SAGE search engine (default: 1)
+- diann_speclib: Path to the spectral library to use in DIA-NN (default: null)
+- convert_dotd: if convert .d file to mzml (default: false)
+
+## [1.1.1] nfcore/quantms - [03/27/23] - Berlin-Bern
+
+### `Added`
+
+- [#92](https://github.com/nf-core/quantms/pull/92) Improved output docs for mzTab
+- [#91](https://github.com/nf-core/quantms/pull/91) Added dev profile for nightly versions of OpenMS tools
+
+### `Changed`
+
+- [#88](https://github.com/nf-core/quantms/pull/88) Updated Comet version to latest release (2023010)
+
+### `Fixed`
+
+- [#93](https://github.com/nf-core/quantms/pull/93) Fixed bug in docker vs. singularity container logic in some processes.
+
+## [1.1.0] nfcore/quantms - [03/20/2023] - Berlin
+
+- Bugfixes and speed increases in the OpenMS tools due to version update to 2.9.1
+- Improvements in logging by adding many more process.ids
+- Large restructuring of DIA branch to increase parallelizability
+- Better error handling in MSstats step plus new parameter to filter for MSstats' adjusted p-value in the plots
+- More efficient parsing of mzML statistics in a separate step
+- A clearer distinction between per-run and experiment-wide FDRs with one parameter for each
+- More test profiles including larger "full" tests
+
+### `Added`
+
+- [#176](https://github.com/bigbio/quantms/pull/176) - Add name of each ID step
+- [#205](https://github.com/bigbio/quantms/pull/205) - mzTab export for DIANN outputs
+
+### `Changed`
+
+- [#169](https://github.com/bigbio/quantms/pull/169) - Restruct DIA-NN step1 : Generate an in silico predicted spectral library
+- [#178](https://github.com/bigbio/quantms/pull/178) - Restruct DIA-NN step2 : Preliminary analysis of individual raw files
+- [#179](https://github.com/bigbio/quantms/pull/179) - Restruct DIA-NN steps 3-5 to be as parallel as possible
+- [#200](https://github.com/bigbio/quantms/pull/200) - Rename MSstats/Triqler/mzTab input and output
+
+### `Fixed`
+
+- [#187](https://github.com/bigbio/quantms/pull/187) - Bug fixing in proteomicsLFQ applying FDR at PSM level
+- [#207](https://github.com/bigbio/quantms/pull/207) - Bug fixing in dissociation method translation for Luciphor
+
+### `Dependencies`
+
+- [#203](https://github.com/bigbio/quantms/pull/203) - update openms dependency -> 3.0.0dev
+- [#208](https://github.com/bigbio/quantms/pull/208) - update pmultiqc dependency -> 0.0.13. Support for DIANN in pmultiqc and enable the generation of search engine scores distributions/peptide and protein table by pmultiqc.
+
+### `Parameters`
+
+- [#193](https://github.com/bigbio/quantms/pull/193) - Set the `local_input_type` default parameter to `mzML`
+- [#212](https://github.com/bigbio/quantms/pull/212) - Set the `min_consensus_support` default parameter to `1` to filter in ConsensusID for peptides identified with both search engines
+- [#200](https://github.com/bigbio/quantms/pull/200) - Add `export_mztab` parameter to allow to run PROTEINQUANTIFIER TMT without exporting to mzTab
+
+## [1.0] nfcore/quantms - [05/02/2022] - Havana
 
 Initial release of nf-core/quantms, created with the [nf-core](https://nf-co.re/) template.
 
 ### `Added`
 
+- New pipeline for DDA-LFQ data analysis
+- New pipeline for DDA-ISO data analysis
+- New datasets for DDA-LFQ and DDA-ISO data analysis
+- Documentation added for DDA pipeline
+- First pipeline for DIA-LFQ data analysis
+
 ### `Fixed`
 
+- This is the first release - no reported issues
+
 ### `Dependencies`
 
-### `Deprecated`
+The pipeline is using Nextflow DSL2, each process will be run with its own [Biocontainer](https://biocontainers.pro/#/registry). This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference.
+
+| Dependency            | Version    |
+| --------------------- | ---------- |
+| `thermorawfileparser` | 1.3.4      |
+| `comet`               | 2021010    |
+| `msgf+`               | 2022.01.07 |
+| `openms`              | 2.9.1      |
+| `sdrf-pipelines`      | 0.0.22     |
+| `percolator`          | 3.5        |
+| `pmultiqc`            | 0.0.11     |
+| `luciphor`            | 2020_04_03 |
+| `dia-nn`              | 1.8.1      |
+| `msstats`             | 4.2.0      |
+| `msstatstmt`          | 2.2.0      |