Release version 4.1.0 of pipeline

.devcontainer/devcontainer.json

@@ -1,20 +1,20 @@
 {
     "name": "nfcore",
-    "image": "nfcore/gitpod:latest",
-    "remoteUser": "gitpod",
-    "runArgs": ["--privileged"],
+    "image": "nfcore/devcontainer:latest",
 
-    // Configure tool-specific properties.
-    "customizations": {
-        // Configure properties specific to VS Code.
-        "vscode": {
-            // Set *default* container specific settings.json values on container create.
-            "settings": {
-                "python.defaultInterpreterPath": "/opt/conda/bin/python"
-            },
+    "remoteUser": "root",
+    "privileged": true,
 
-            // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
+    "remoteEnv": {
+        // Workspace path on the host for mounting with docker-outside-of-docker
+        "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+    },
+
+    "onCreateCommand": "./.devcontainer/setup.sh",
+
+    "hostRequirements": {
+        "cpus": 4,
+        "memory": "16gb",
+        "storage": "32gb"
     }
 }

.devcontainer/setup.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Customise the terminal command prompt
+echo "export PROMPT_DIRTRIM=2" >> $HOME/.bashrc
+echo "export PS1='\[\e[3;36m\]\w ->\[\e[0m\\] '" >> $HOME/.bashrc
+export PROMPT_DIRTRIM=2
+export PS1='\[\e[3;36m\]\w ->\[\e[0m\\] '
+
+# Update Nextflow
+nextflow self-update
+
+# Update welcome message
+echo "Welcome to the nf-core/scrnaseq devcontainer!" > /usr/local/etc/vscode-dev-containers/first-run-notice.txt

.github/CONTRIBUTING.md

@@ -78,7 +78,7 @@ If you wish to contribute a new step, please use the following coding standards:
 5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool).
 6. Add sanity checks and validation for all relevant parameters.
 7. Perform local tests to validate that the new code works as expected.
-8. If applicable, add a new test command in `.github/workflow/ci.yml`.
+8. If applicable, add a new test in the `tests` directory.
 9. Update MultiQC config `assets/multiqc_config.yml` so relevant suffixes, file name clean up and module plots are in the appropriate order. If applicable, add a [MultiQC](https://https://multiqc.info/) module.
 10. Add a description of the output files and if relevant any appropriate images from the MultiQC report to `docs/output.md`.
 

.github/actions/get-shards/action.yml

@@ -0,0 +1,69 @@
+name: "Get number of shards"
+description: "Get the number of nf-test shards for the current CI job"
+inputs:
+  max_shards:
+    description: "Maximum number of shards allowed"
+    required: true
+  paths:
+    description: "Component paths to test"
+    required: false
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: false
+outputs:
+  shard:
+    description: "Array of shard numbers"
+    value: ${{ steps.shards.outputs.shard }}
+  total_shards:
+    description: "Total number of shards"
+    value: ${{ steps.shards.outputs.total_shards }}
+runs:
+  using: "composite"
+  steps:
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: ${{ env.NFT_VER }}
+    - name: Get number of shards
+      id: shards
+      shell: bash
+      run: |
+        # Run nf-test with dynamic parameter
+        nftest_output=$(nf-test test \
+          --profile +docker \
+          $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
+          --dry-run \
+          --ci \
+          --changed-since HEAD^) || {
+            echo "nf-test command failed with exit code $?"
+            echo "Full output: $nftest_output"
+            exit 1
+        }
+        echo "nf-test dry-run output: $nftest_output"
+
+        # Default values for shard and total_shards
+        shard="[]"
+        total_shards=0
+
+        # Check if there are related tests
+        if echo "$nftest_output" | grep -q 'No tests to execute'; then
+          echo "No related tests found."
+        else
+          # Extract the number of related tests
+          number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
+          if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
+            shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
+            shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
+            total_shards="$shards_to_run"
+          else
+            echo "Unexpected output format. Falling back to default values."
+          fi
+        fi
+
+        # Write to GitHub Actions outputs
+        echo "shard=$shard" >> $GITHUB_OUTPUT
+        echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
+
+        # Debugging output
+        echo "Final shard array: $shard"
+        echo "Total number of shards: $total_shards"

.github/actions/nf-test/action.yml

@@ -0,0 +1,111 @@
+name: "nf-test Action"
+description: "Runs nf-test with common setup steps"
+inputs:
+  profile:
+    description: "Profile to use"
+    required: true
+  shard:
+    description: "Shard number for this CI job"
+    required: true
+  total_shards:
+    description: "Total number of test shards(NOT the total number of matrix jobs)"
+    required: true
+  paths:
+    description: "Test paths"
+    required: true
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: false
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Nextflow
+      uses: nf-core/setup-nextflow@v2
+      with:
+        version: "${{ env.NXF_VERSION }}"
+
+    - name: Set up Python
+      uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
+      with:
+        python-version: "3.14"
+
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: "${{ env.NFT_VER }}"
+        install-pdiff: true
+
+    - name: Setup apptainer
+      if: contains(inputs.profile, 'singularity')
+      uses: eWaterCycle/setup-apptainer@main
+
+    - name: Set up Singularity
+      if: contains(inputs.profile, 'singularity')
+      shell: bash
+      run: |
+        mkdir -p $NXF_SINGULARITY_CACHEDIR
+        mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+    - name: Conda setup
+      if: contains(inputs.profile, 'conda')
+      uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
+      with:
+        auto-update-conda: true
+        conda-solver: libmamba
+        channels: conda-forge
+        channel-priority: strict
+        conda-remove-defaults: true
+
+    - name: Run nf-test
+      shell: bash
+      env:
+        NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
+      run: |
+        nf-test test \
+          --profile=+${{ inputs.profile }} \
+          $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
+          --ci \
+          --changed-since HEAD^ \
+          --verbose \
+          --tap=test.tap \
+          --shard ${{ inputs.shard }}/${{ inputs.total_shards }}
+
+          # Save the absolute path of the test.tap file to the output
+          echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT
+
+    - name: Generate test summary
+      if: always()
+      shell: bash
+      run: |
+        # Add header if it doesn't exist (using a token file to track this)
+        if [ ! -f ".summary_header" ]; then
+          echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY
+          echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY
+          touch .summary_header
+        fi
+
+        if [ -f test.tap ]; then
+          while IFS= read -r line; do
+            if [[ $line =~ ^ok ]]; then
+              test_name="${line#ok }"
+              # Remove the test number from the beginning
+              test_name="${test_name#* }"
+              echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+            elif [[ $line =~ ^not\ ok ]]; then
+              test_name="${line#not ok }"
+              # Remove the test number from the beginning
+              test_name="${test_name#* }"
+              echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+            fi
+          done < test.tap
+        else
+          echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
+        fi
+
+    - name: Clean up
+      if: always()
+      shell: bash
+      run: |
+        sudo rm -rf /home/ubuntu/tests/

.github/workflows/awsfulltest.yml

@@ -4,66 +4,46 @@ name: nf-core AWS full size tests
 # It runs the -profile 'test_full' on AWS batch
 
 on:
-  pull_request:
-    branches:
-      - main
-      - master
   workflow_dispatch:
   pull_request_review:
     types: [submitted]
+  release:
+    types: [published]
 
 jobs:
   run-platform:
     name: Run AWS full tests
-    # run only if the PR is approved by at least 2 reviewers and against the master branch or manually triggered
-    if: github.repository == 'nf-core/scrnaseq' && github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'master' || github.event_name == 'workflow_dispatch'
+    # run only if the PR is approved by at least 2 reviewers and against the master/main branch or manually triggered
+    if: github.repository == 'nf-core/scrnaseq' && github.event.review.state == 'approved' && (github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'release'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         aligner: ["simpleaf", "kallisto", "star", "cellranger"]
     steps:
-      - name: Get PR reviews
-        uses: octokit/[email protected]
-        if: github.event_name != 'workflow_dispatch'
-        id: check_approvals
-        continue-on-error: true
-        with:
-          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews?per_page=100
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check for approvals
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
-        run: |
-          echo "No review approvals found. At least 2 approvals are required to run this action automatically."
-          exit 1
-
-      - name: Check for enough approvals (>=2)
-        id: test_variables
-        if: github.event_name != 'workflow_dispatch'
+      - name: Set revision variable
+        id: revision
         run: |
-          JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}'
-          CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length')
-          test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required
+          echo "revision=${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && github.sha || 'dev' }}" >> "$GITHUB_OUTPUT"
 
       - name: Launch workflow via Seqera Platform
         uses: seqeralabs/action-tower-launch@v2
         with:
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          workspace_id: ${{ vars.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          revision: ${{ github.sha }}
-          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/scrnaseq/work-${{ github.sha }}
+          compute_env: ${{ vars.TOWER_COMPUTE_ENV }}
+          revision: ${{ steps.revision.outputs.revision }}
+          workdir: s3://${{ vars.AWS_S3_BUCKET }}/work/scrnaseq/work-${{ steps.revision.outputs.revision }}
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/scrnaseq/results-${{ github.sha }}"
+              "outdir": "s3://${{ vars.AWS_S3_BUCKET }}/scrnaseq/results-${{ steps.revision.outputs.revision }}/aligner_${{ matrix.aligner }}",
+              "aligner": "${{ matrix.aligner }}"
             }
           profiles: test_full
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: Seqera Platform debug log file
+          name: seqera_platform_debug_log_${{ matrix.aligner }}
           path: |
             seqera_platform_action_*.log
             seqera_platform_action_*.json

.github/workflows/awstest.yml

@@ -17,21 +17,21 @@ jobs:
       - name: Launch workflow via Seqera Platform
         uses: seqeralabs/action-tower-launch@v2
         with:
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          workspace_id: ${{ vars.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          compute_env: ${{ vars.TOWER_COMPUTE_ENV }}
           revision: ${{ github.sha }}
-          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/scrnaseq/work-${{ github.sha }}
+          workdir: s3://${{ vars.AWS_S3_BUCKET }}/work/scrnaseq/work-${{ github.sha }}
           parameters: |
             {
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/scrnaseq/results-test-${{ github.sha }}/aligner_${{ matrix.aligner }}",
+              "outdir": "s3://${{ vars.AWS_S3_BUCKET }}/scrnaseq/results-test-${{ github.sha }}/aligner_${{ matrix.aligner }}",
               "aligner": "${{ matrix.aligner }}"
             }
           profiles: test
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: Seqera Platform debug log file
           path: |
-            seqera_platform_action_*.log
-            seqera_platform_action_*.json
+            tower_action_*.log
+            tower_action_*.json