diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 3ef0760513..3ffd960d23 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -15,7 +15,6 @@ runs: cd create-lint-wf export NXF_WORK=$(pwd) - # Set up Nextflow - name: Install Nextflow uses: nf-core/setup-nextflow@v2 with: diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 37ab71bc3b..fa6c38ef07 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -27,14 +27,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - NXF_ANSI_LOG: false - jobs: MakeTestWorkflow: runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} env: NXF_ANSI_LOG: false + strategy: matrix: NXF_VER: diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index fffa9ffe7a..cabd4b9abe 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -51,6 +51,7 @@ jobs: needs: prepare-matrix env: NXF_ANSI_LOG: false + strategy: matrix: TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }} diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 93581b9153..53f84b72c4 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -27,15 +27,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - NXF_ANSI_LOG: false - jobs: RunTestWorkflow: # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} env: NXF_ANSI_LOG: false + strategy: matrix: NXF_VER: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 625f00d247..7061294783 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -23,6 +23,10 @@ on: description: "Pipeline to sync" type: string default: "all" + debug: + description: "Enable debug/verbose mode (true or false)" + type: boolean + default: false # Cancel if a newer run is started concurrency: @@ -62,6 +66,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 name: Check out nf-core/tools + with: + ref: ${{ github.ref_name }} - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 name: Check out nf-core/${{ matrix.pipeline }} @@ -87,6 +93,19 @@ jobs: with: version: "latest-everything" + - name: Set Git default branch from nextflow.config and set git default branch to that or "master" + + run: | + pushd nf-core/${{ matrix.pipeline }} + defaultBranch=$(grep -B5 -A5 "nextflowVersion" nextflow.config | grep "defaultBranch" | cut -d"=" -f2) + if [ -z "$defaultBranch" ]; then + defaultBranch="master" + fi + popd + echo "Default branch: $defaultBranch" + echo "defaultBranch=$defaultBranch" >> GITHUB_OUTPUT + git config --global init.defaultBranch $defaultBranch + - name: Run synchronisation if: github.repository == 'nf-core/tools' env: @@ -94,7 +113,9 @@ jobs: run: | git config --global user.email "core@nf-co.re" git config --global user.name "nf-core-bot" - nf-core --log-file sync_log_${{ matrix.pipeline }}.txt pipelines sync -d nf-core/${{ matrix.pipeline }} \ + nf-core --log-file sync_log_${{ matrix.pipeline }}.txt \ + ${{ github.event.inputs.debug == 'true' && '--verbose' || '' }} \ + pipelines sync -d nf-core/${{ matrix.pipeline }} \ --from-branch dev \ --pull-request \ --username nf-core-bot \ diff --git a/.gitpod.yml b/.gitpod.yml index efe193f35f..d5948695bf 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -9,11 +9,4 @@ tasks: vscode: extensions: - - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - - nextflow.nextflow # Nextflow syntax highlighting - - oderwat.indent-rainbow # Highlight indentation level - - streetsidesoftware.code-spell-checker # Spelling checker for source code - - charliermarsh.ruff # Code linter Ruff + - nf-core.nf-core-extensionpack # https://github.com/nf-core/vscode-extensionpack diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1494f58182..68a6fa3ed7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.1 + rev: v0.8.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..5ffdff086c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.nosetestsEnabled": false, + "python.testing.pytestArgs": ["tests", "-v", "--tb=short"], + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "python.terminal.activateEnvInCurrentTerminal": true, + "python.terminal.shellIntegration.enabled": true +} diff --git a/CHANGELOG.md b/CHANGELOG.md index da5f72c357..603dd664c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,42 @@ # nf-core/tools: Changelog -## v3.0.3dev +## v3.1.1dev + +### Template + +- Use outputs instead of the environment to pass around values between steps in the Download Test Action ([#3351](https://github.com/nf-core/tools/pull/3351)) +- Fix pre commit template ([#3358](https://github.com/nf-core/tools/pull/3358)) + +### Download + +### Linting + +### Modules + +- fix including modules.config ([#3356](https://github.com/nf-core/tools/pull/3356)) + +### Subworkflows + +### General + +- Add missing p ([#3357](https://github.com/nf-core/tools/pull/3357)) + +### Version updates + +## [v3.1.0 - Brass Boxfish](https://github.com/nf-core/tools/releases/tag/3.1.0) - [2024-12-09] + +**Highlights** + +- We added the new `contributors` field to the pipeline template `manifest`. +- The `nf-core pipelines download` command supports ORAS container URIs. +- New command `nf-core subworkflows patch`. ### Template - Keep pipeline name in version.yml file ([#3223](https://github.com/nf-core/tools/pull/3223)) - Fix Manifest DOI text ([#3224](https://github.com/nf-core/tools/pull/3224)) - Do not assume pipeline name is url ([#3225](https://github.com/nf-core/tools/pull/3225)) -- fix workflow_dispatch trigger and parse more review comments in awsfulltest ([#3235](https://github.com/nf-core/tools/pull/3235)) +- fix `workflow_dispatch` trigger and parse more review comments in awsfulltest ([#3235](https://github.com/nf-core/tools/pull/3235)) - Add resource limits to Gitpod profile([#3255](https://github.com/nf-core/tools/pull/3255)) - Fix a typo ([#3268](https://github.com/nf-core/tools/pull/3268)) - Remove `def` from `nextflow.config` and add `trace_report_suffix` param ([#3296](https://github.com/nf-core/tools/pull/3296)) @@ -15,16 +44,21 @@ - Use `params.monochrome_logs` in the template and update nf-core components ([#3310](https://github.com/nf-core/tools/pull/3310)) - Fix some typos and improve writing in `usage.md` and `CONTRIBUTING.md` ([#3302](https://github.com/nf-core/tools/pull/3302)) - Add `manifest.contributors` to `nextflow.config` ([#3311](https://github.com/nf-core/tools/pull/3311)) +- Update template components ([#3328](https://github.com/nf-core/tools/pull/3328)) +- Template: Remove mention of GRCh37 if igenomes is skipped ([#3330](https://github.com/nf-core/tools/pull/3330)) +- Be more verbose in approval check action ([#3338](https://github.com/nf-core/tools/pull/3338)) ### Download -- First steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Modify `prioritize_direct_download()` to retain Seqera Singularity https:// Container URIs and hardcode Seqera Containers into `gather_registries()` ([#3244](https://github.com/nf-core/tools/pull/3244)). +- First steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Modify `prioritize_direct_download()` to retain Seqera Singularity `https://` Container URIs and hardcode Seqera Containers into `gather_registries()` ([#3244](https://github.com/nf-core/tools/pull/3244)). - Further steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Enable limited support for `oras://` container paths (_only absolute URIs, no flexible registries like with Docker_) and prevent unnecessary image downloads for Seqera Container modules with `reconcile_seqera_container_uris()` ([#3293](https://github.com/nf-core/tools/pull/3293)). - Update dawidd6/action-download-artifact action to v7 ([#3306](https://github.com/nf-core/tools/pull/3306)) ### Linting - allow mixed `str` and `dict` entries in lint config ([#3228](https://github.com/nf-core/tools/pull/3228)) +- fix `meta_yml` linting test failing due to `module.process_name` always being `""` ([#3317](https://github.com/nf-core/tools/pull/3317)) +- fix module section regex matching wrong things ([#3321](https://github.com/nf-core/tools/pull/3321)) ### Modules @@ -33,29 +67,38 @@ ### Subworkflows - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) +- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) ### General -- Include .nf-core.yml in `nf-core pipelines bump-version` ([#3220](https://github.com/nf-core/tools/pull/3220)) +- Include `.nf-core.yml` in `nf-core pipelines bump-version` ([#3220](https://github.com/nf-core/tools/pull/3220)) - create: add shortcut to toggle all switches ([#3226](https://github.com/nf-core/tools/pull/3226)) - Remove unrelated values when saving `.nf-core` file ([#3227](https://github.com/nf-core/tools/pull/3227)) -- chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.12.0 ([#3230](https://github.com/nf-core/tools/pull/3230)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.7.0 ([#3229](https://github.com/nf-core/tools/pull/3229)) -- Update python:3.12-slim Docker digest to 032c526 ([#3232](https://github.com/nf-core/tools/pull/3232)) - use correct `--profile` options for `nf-core subworkflows test` ([#3233](https://github.com/nf-core/tools/pull/3233)) - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) +- fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.3 ([#3275](https://github.com/nf-core/tools/pull/3275)) - Add hint to solve git errors with a synced repo ([#3279](https://github.com/nf-core/tools/pull/3279)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.4 ([#3282](https://github.com/nf-core/tools/pull/3282)) +- Run pre-commit when testing linting the template pipeline ([#3280](https://github.com/nf-core/tools/pull/3280)) +- Make CLI prompt less nf-core specific ([#3326](https://github.com/nf-core/tools/pull/3326)) +- Update gitpod vscode extensions to use nf-core extension pack ([#3327](https://github.com/nf-core/tools/pull/3327)) +- Remove toList() channel operation from inside onComplete block ([#3304](https://github.com/nf-core/tools/pull/3304)) +- build: Setup VS Code tests ([#3292](https://github.com/nf-core/tools/pull/3292)) +- Don't break gitpod.yml with template string ([#3332](https://github.com/nf-core/tools/pull/3332)) +- rocrate: remove duplicated entries for name and version ([#3333](https://github.com/nf-core/tools/pull/3333)) +- rocrate: Update crate with version bump and handle new contributor field ([#3334](https://github.com/nf-core/tools/pull/3334)) +- set default_branch to master for now ([#3335](https://github.com/nf-core/tools/issues/3335)) +- Set git defaultBranch to master in sync action ([#3337](https://github.com/nf-core/tools/pull/3337)) +- Add verbose mode to sync action ([#3339](https://github.com/nf-core/tools/pull/3339)) + +### Version updates + +- chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.12.0 ([#3230](https://github.com/nf-core/tools/pull/3230)) - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) -- Update python:3.12-slim Docker digest to 2a6386a ([#3284](https://github.com/nf-core/tools/pull/3284)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.0 ([#3299](https://github.com/nf-core/tools/pull/3299)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) -- Run pre-commit when testing linting the template pipeline ([#3280](https://github.com/nf-core/tools/pull/3280)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] diff --git a/Dockerfile b/Dockerfile index dc9948ea4b..f2141145b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:2a6386ad2db20e7f55073f69a98d6da2cf9f168e05e7487d2670baeb9b7601c5 +FROM python:3.12-slim@sha256:2b0079146a74e23bf4ae8f6a28e1b484c6292f6fb904cbb51825b4a19812fcd8 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" diff --git a/docs/api/_src/api/pipelines/bump_version.md b/docs/api/_src/api/pipelines/bump_version.md index cd7dc280f6..76db67837a 100644 --- a/docs/api/_src/api/pipelines/bump_version.md +++ b/docs/api/_src/api/pipelines/bump_version.md @@ -1,4 +1,4 @@ -# nf_core.bump_version +# nf_core.pipelines.bump_version ```{eval-rst} .. automodule:: nf_core.pipelines.bump_version diff --git a/docs/api/_src/api/pipelines/create.md b/docs/api/_src/api/pipelines/create.md index 576335e951..5019a5f3c8 100644 --- a/docs/api/_src/api/pipelines/create.md +++ b/docs/api/_src/api/pipelines/create.md @@ -1,4 +1,4 @@ -# nf_core.create +# nf_core.pipelines.create ```{eval-rst} .. automodule:: nf_core.pipelines.create diff --git a/docs/api/_src/api/pipelines/download.md b/docs/api/_src/api/pipelines/download.md index 540fb92c49..afb31ddea6 100644 --- a/docs/api/_src/api/pipelines/download.md +++ b/docs/api/_src/api/pipelines/download.md @@ -1,4 +1,4 @@ -# nf_core.download +# nf_core.pipelines.download ```{eval-rst} .. automodule:: nf_core.pipelines.download diff --git a/docs/api/_src/api/pipelines/launch.md b/docs/api/_src/api/pipelines/launch.md index 0f7fc03f64..0d0260cae6 100644 --- a/docs/api/_src/api/pipelines/launch.md +++ b/docs/api/_src/api/pipelines/launch.md @@ -1,4 +1,4 @@ -# nf_core.launch +# nf_core.pipelines.launch ```{eval-rst} .. automodule:: nf_core.pipelines.launch diff --git a/docs/api/_src/api/pipelines/lint.md b/docs/api/_src/api/pipelines/lint.md index aa62c404b8..91b37c26f6 100644 --- a/docs/api/_src/api/pipelines/lint.md +++ b/docs/api/_src/api/pipelines/lint.md @@ -1,4 +1,4 @@ -# nf_core.lint +# nf_core.pipelines.lint :::{seealso} See the [Lint Tests](/docs/nf-core-tools/api_reference/dev/pipeline_lint_tests) docs for information about specific linting functions. diff --git a/docs/api/_src/api/pipelines/list.md b/docs/api/_src/api/pipelines/list.md index 7df7564544..5f404b91c3 100644 --- a/docs/api/_src/api/pipelines/list.md +++ b/docs/api/_src/api/pipelines/list.md @@ -1,4 +1,4 @@ -# nf_core.list +# nf_core.pipelines.list ```{eval-rst} .. automodule:: nf_core.pipelines.list diff --git a/docs/api/_src/api/pipelines/params-file.md b/docs/api/_src/api/pipelines/params-file.md index 06f27cc592..37e91f458a 100644 --- a/docs/api/_src/api/pipelines/params-file.md +++ b/docs/api/_src/api/pipelines/params-file.md @@ -1,4 +1,4 @@ -# nf_core.params_file +# nf_core.pipelines.params_file ```{eval-rst} .. automodule:: nf_core.pipelines.params_file diff --git a/docs/api/_src/api/pipelines/schema.md b/docs/api/_src/api/pipelines/schema.md index c885d9ed23..4ca1aab480 100644 --- a/docs/api/_src/api/pipelines/schema.md +++ b/docs/api/_src/api/pipelines/schema.md @@ -1,4 +1,4 @@ -# nf_core.schema +# nf_core.pipelines.schema ```{eval-rst} .. automodule:: nf_core.pipelines.schema diff --git a/docs/api/_src/api/pipelines/sync.md b/docs/api/_src/api/pipelines/sync.md index da1f468fe5..f78733bb7d 100644 --- a/docs/api/_src/api/pipelines/sync.md +++ b/docs/api/_src/api/pipelines/sync.md @@ -1,4 +1,4 @@ -# nf_core.sync +# nf_core.pipelines.sync ```{eval-rst} .. automodule:: nf_core.pipelines.sync diff --git a/docs/api/_src/api/pipelines/utils.md b/docs/api/_src/api/pipelines/utils.md index 86b8c3f36f..36c2ecca4d 100644 --- a/docs/api/_src/api/pipelines/utils.md +++ b/docs/api/_src/api/pipelines/utils.md @@ -1,4 +1,4 @@ -# nf_core.utils +# nf_core.pipelines.utils ```{eval-rst} .. automodule:: nf_core.pipelines.utils diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 06d4be8f75..7cfba64531 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -288,7 +288,7 @@ def command_pipelines_create(ctx, name, description, author, version, force, out @click.option( "--release", is_flag=True, - default=os.path.basename(os.path.dirname(os.environ.get("GITHUB_REF", "").strip(" '\""))) == "master" + default=Path(os.environ.get("GITHUB_REF", "").strip(" '\"")).parent.name in ["master", "main"] and os.environ.get("GITHUB_REPOSITORY", "").startswith("nf-core/") and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools", help="Execute additional checks for release-ready workflows.", @@ -2240,7 +2240,7 @@ def command_download( @click.option( "--release", is_flag=True, - default=os.path.basename(os.path.dirname(os.environ.get("GITHUB_REF", "").strip(" '\""))) == "master" + default=Path(os.environ.get("GITHUB_REF", "").strip(" '\"")).parent.name in ["master", "main"] and os.environ.get("GITHUB_REPOSITORY", "").startswith("nf-core/") and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools", help="Execute additional checks for release-ready workflows.", diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index be28a0d870..ab4cc79ae6 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -37,10 +37,10 @@ def get_repo_info(directory: Path, use_prompt: Optional[bool] = True) -> Tuple[P if not repo_type and use_prompt: log.warning("'repository_type' not defined in %s", config_fn.name) repo_type = questionary.select( - "Is this repository an nf-core pipeline or a fork of nf-core/modules?", + "Is this repository a pipeline or a modules repository?", choices=[ {"name": "Pipeline", "value": "pipeline"}, - {"name": "nf-core/modules", "value": "modules"}, + {"name": "Modules repository", "value": "modules"}, ], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 37e43a536e..81c0ba98e7 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -62,7 +62,6 @@ def __init__( # Initialize the important files self.main_nf: Path = Path(self.component_dir, "main.nf") self.meta_yml: Optional[Path] = Path(self.component_dir, "meta.yml") - self.process_name = "" self.environment_yml: Optional[Path] = Path(self.component_dir, "environment.yml") component_list = self.component_name.split("/") @@ -96,6 +95,8 @@ def __init__( self.test_yml = None self.test_main_nf = None + self.process_name: str = self._get_process_name() + def __repr__(self) -> str: return f"" @@ -169,6 +170,13 @@ def _get_included_components_in_chained_tests(self, main_nf_test: Union[Path, st included_components.append(component) return included_components + def _get_process_name(self): + with open(self.main_nf) as fh: + for line in fh: + if re.search(r"^\s*process\s*\w*\s*{", line): + return re.search(r"^\s*process\s*(\w*)\s*{.*", line).group(1) or "" + return "" + def get_inputs_from_main_nf(self) -> None: """Collect all inputs from the main.nf file.""" inputs: Any = [] # Can be 'list[list[dict[str, dict[str, str]]]]' or 'list[str]' diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 848e17130e..ba3b87f79a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -96,19 +96,19 @@ def main_nf( for line in iter_lines: if re.search(r"^\s*process\s*\w*\s*{", line) and state == "module": state = "process" - if re.search(r"input\s*:", line) and state in ["process"]: + if re.search(r"^\s*input\s*:", line) and state in ["process"]: state = "input" continue - if re.search(r"output\s*:", line) and state in ["input", "process"]: + if re.search(r"^\s*output\s*:", line) and state in ["input", "process"]: state = "output" continue - if re.search(r"when\s*:", line) and state in ["input", "output", "process"]: + if re.search(r"^\s*when\s*:", line) and state in ["input", "output", "process"]: state = "when" continue - if re.search(r"script\s*:", line) and state in ["input", "output", "when", "process"]: + if re.search(r"^\s*script\s*:", line) and state in ["input", "output", "when", "process"]: state = "script" continue - if re.search(r"shell\s*:", line) and state in ["input", "output", "when", "process"]: + if re.search(r"^\s*shell\s*:", line) and state in ["input", "output", "when", "process"]: state = "shell" continue @@ -256,7 +256,6 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): bioconda_packages = [] # Process name should be all capital letters - self.process_name = lines[0].split()[1] if all(x.upper() for x in self.process_name): self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) else: diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 37970c09e8..b08b43667f 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -64,9 +64,9 @@ These tests are run both with the latest available version of `Nextflow` and als :warning: Only in the unlikely and regretful event of a release happening with a bug. -- On your own fork, make a new branch `patch` based on `upstream/master`. +- On your own fork, make a new branch `patch` based on `upstream/main` or `upstream/master`. - Fix the bug, and bump version (X.Y.Z+1). -- Open a pull-request from `patch` to `master` with the changes. +- Open a pull-request from `patch` to `main`/`master` with the changes. {% if is_nfcore -%} diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 412f5bd3b3..f3624afc9c 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -11,7 +11,6 @@ body: - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage) {%- endif %} - - type: textarea id: description attributes: diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index c96f2dd4c2..0df95c0a40 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -8,14 +8,14 @@ These are the most common things requested on pull requests (PRs). Remember that PRs should be made against the dev branch, unless you're preparing a pipeline release. -Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) +Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/tree/{{ default_branch }}/.github/CONTRIBUTING.md) --> ## PR checklist - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! -- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/{{ default_branch }}/.github/CONTRIBUTING.md) {%- if is_nfcore %} - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 1ca2ac2c74..6805c83a27 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -1,11 +1,12 @@ name: nf-core AWS full size tests -# This workflow is triggered on PRs opened against the master branch. +# This workflow is triggered on PRs opened against the main/master branch. # It can be additionally triggered manually with GitHub actions workflow dispatch button. # It runs the -profile 'test_full' on AWS batch on: pull_request: branches: + - main - master workflow_dispatch: pull_request_review: @@ -18,19 +19,30 @@ jobs: if: github.repository == '{{ name }}' && github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'master' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: octokit/request-action@v2.x + - name: Get PR reviews + uses: octokit/request-action@v2.x if: github.event_name != 'workflow_dispatch' id: check_approvals + continue-on-error: true with: route: GET /repos/{%- raw -%}${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews?per_page=100 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - id: test_variables + + - name: Check for approvals + if: ${{ failure() && github.event_name != 'workflow_dispatch' }} + run: | + echo "No review approvals found. At least 2 approvals are required to run this action automatically." + exit 1 + + - name: Check for enough approvals (>=2) + id: test_variables if: github.event_name != 'workflow_dispatch' run: | JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}'{% endraw %} CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required diff --git a/nf_core/pipeline-template/.github/workflows/branch.yml b/nf_core/pipeline-template/.github/workflows/branch.yml index df1a627b15..110b4a5f5a 100644 --- a/nf_core/pipeline-template/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/.github/workflows/branch.yml @@ -1,15 +1,17 @@ name: nf-core branch protection -# This workflow is triggered on PRs to master branch on the repository -# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev` +# This workflow is triggered on PRs to `main`/`master` branch on the repository +# It fails when someone tries to make a PR against the nf-core `main`/`master` branch instead of `dev` on: pull_request_target: - branches: [master] + branches: + - main + - master jobs: test: runs-on: ubuntu-latest steps: - # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches + # PRs to the nf-core repo main/master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - name: Check PRs if: github.repository == '{{ name }}' run: | @@ -22,7 +24,7 @@ jobs: uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | - ## This PR is against the `master` branch :x: + ## This PR is against the `${{github.event.pull_request.base.ref}}` branch :x: * Do not close this PR * Click _Edit_ and change the `base` to `dev` @@ -32,9 +34,9 @@ jobs: Hi @${{ github.event.pull_request.user.login }}, - It looks like this pull-request is has been made against the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `master` branch. - The `master` branch on nf-core repositories should always contain code from the latest release. - Because of this, PRs to `master` are only allowed if they come from the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `dev` branch. + It looks like this pull-request is has been made against the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) ${{github.event.pull_request.base.ref}} branch. + The ${{github.event.pull_request.base.ref}} branch on nf-core repositories should always contain code from the latest release. + Because of this, PRs to ${{github.event.pull_request.base.ref}} are only allowed if they come from the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `dev` branch. You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. Note that even after this, the test will continue to show as failing until you push a new commit. diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 1bc42469c4..f270dc5411 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -2,7 +2,7 @@ name: Test successful pipeline download with 'nf-core pipelines download' # Run the workflow when: # - dispatched manually -# - when a PR is opened or reopened to master branch +# - when a PR is opened or reopened to main/master branch # - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: workflow_dispatch: @@ -17,19 +17,25 @@ on: - edited - synchronize branches: + - main - master pull_request_target: branches: + - main - master env: NXF_ANSI_LOG: false jobs: - download: - runs-on: ubuntu-latest + configure: + runs-on: ubuntu-latest{% raw %} + outputs: + REPO_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPO_LOWERCASE }} + REPOTITLE_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPOTITLE_LOWERCASE }} + REPO_BRANCH: ${{ steps.get_repo_properties.outputs.REPO_BRANCH }} steps: - - name: Install Nextflow + - name: Install Nextflow{% endraw %} uses: nf-core/setup-nextflow@v2 - name: Disk space cleanup @@ -51,22 +57,27 @@ jobs: pip install git+https://github.com/nf-core/tools.git@dev - name: Get the repository name and current branch set as environment variable + id: get_repo_properties run: | - echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} - echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" + echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT" - name: Make a cache directory for the container images run: | mkdir -p ./singularity_container_images + download: + runs-on: ubuntu-latest + needs: configure + steps: - name: Download the pipeline env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images run: | - nf-core pipelines download ${{ env.REPO_LOWERCASE }} \ - --revision ${{ env.REPO_BRANCH }} \ - --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + nf-core pipelines download ${{ needs.configure.outputs.REPO_LOWERCASE }} \ + --revision ${{ needs.configure.outputs.REPO_BRANCH }} \ + --outdir ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }} \ --compress "none" \ --container-system 'singularity' \ --container-library "quay.io" -l "docker.io" -l "community.wave.seqera.io/library/" \ @@ -74,14 +85,14 @@ jobs: --download-configuration 'yes' - name: Inspect download - run: tree ./${{ env.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} + run: tree ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} - name: Count the downloaded number of container images id: count_initial run: | image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) echo "Initial container image count: $image_count" - echo "IMAGE_COUNT_INITIAL=$image_count" >> ${GITHUB_ENV} + echo "IMAGE_COUNT_INITIAL=$image_count" >> "$GITHUB_OUTPUT" - name: Run the downloaded pipeline (stub) id: stub_run_pipeline @@ -89,27 +100,27 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + run: nextflow run ./${{needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline - if: ${{ job.steps.stub_run_pipeline.status == failure() }} + if: ${{ steps.stub_run_pipeline.outcome == 'failure' }} env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results + run: nextflow run ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -profile test,singularity --outdir ./results - name: Count the downloaded number of container images id: count_afterwards run: | image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) echo "Post-pipeline run container image count: $image_count" - echo "IMAGE_COUNT_AFTER=$image_count" >> ${GITHUB_ENV} + echo "IMAGE_COUNT_AFTER=$image_count" >> "$GITHUB_OUTPUT" - name: Compare container image counts run: | - if [ "${{ env.IMAGE_COUNT_INITIAL }}" -ne "${{ env.IMAGE_COUNT_AFTER }}" ]; then - initial_count=${{ env.IMAGE_COUNT_INITIAL }} - final_count=${{ env.IMAGE_COUNT_AFTER }} + if [ "${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }}" -ne "${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }}" ]; then + initial_count=${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }} + final_count=${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }} difference=$((final_count - initial_count)) echo "$difference additional container images were \n downloaded at runtime . The pipeline has no support for offline runs!" tree ./singularity_container_images diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 5907fb59c9..c6a2e40b87 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -3,16 +3,8 @@ tasks: - name: Update Nextflow and setup pre-commit command: | pre-commit install --install-hooks - nextflow self-update + nextflow self-update {%- if code_linters %} vscode: - extensions: # based on nf-core.nf-core-extensionpack - #{%- if code_linters -%} - - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files{% endif %} - - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - - nextflow.nextflow # Nextflow syntax highlighting - - oderwat.indent-rainbow # Highlight indentation level - - streetsidesoftware.code-spell-checker # Spelling checker for source code - - charliermarsh.ruff # Code linter Ruff + extensions: + - nf-core.nf-core-extensionpack # https://github.com/nf-core/vscode-extensionpack{% endif %} diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index 7ecc9b61cb..02ba84c006 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -16,3 +16,6 @@ testing/ testing* *.pyc bin/ +{%- if rocrate %} +ro-crate-metadata.json +{%- endif %} diff --git a/nf_core/pipeline-template/CITATIONS.md b/nf_core/pipeline-template/CITATIONS.md index 16da9a4207..c355fd6129 100644 --- a/nf_core/pipeline-template/CITATIONS.md +++ b/nf_core/pipeline-template/CITATIONS.md @@ -18,7 +18,7 @@ {%- endif %} -{% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) +{%- if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index a8f2e60546..4cd41de368 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -7,7 +7,7 @@ -{% else %} +{%- else -%} # {{ name }} @@ -48,13 +48,13 @@ workflows use the "tube map" design for that. See https://nf-co.re/docs/contributing/design_guidelines#examples for examples. --> -{% if fastqc %}1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)){% endif %} -{% if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} +{%- if fastqc %}1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)){% endif %} +{%- if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} ## Usage > [!NOTE] -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {% if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %} +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {%- if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %} -{% if citations %} +{%- if citations %} An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. -{% endif %} +{%- endif %} + {% if is_nfcore -%} You can cite the `nf-core` publication as follows: diff --git a/nf_core/pipeline-template/assets/schema_input.json b/nf_core/pipeline-template/assets/schema_input.json index 28a468adaf..0dbc03df56 100644 --- a/nf_core/pipeline-template/assets/schema_input.json +++ b/nf_core/pipeline-template/assets/schema_input.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/{{ name }}/master/assets/schema_input.json", + "$id": "https://raw.githubusercontent.com/{{ name }}/{{ default_branch }}/assets/schema_input.json", "title": "{{ name }} pipeline - params.input schema", "description": "Schema for the file provided with params.input", "type": "array", diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index 83d5d23fe3..a9be6620e1 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -2,7 +2,7 @@ ## Introduction -This document describes the output produced by the pipeline. {% if multiqc %}Most of the plots are taken from the MultiQC report, which summarises results at the end of the pipeline.{% endif %} +This document describes the output produced by the pipeline.{% if multiqc %} Most of the plots are taken from the MultiQC report, which summarises results at the end of the pipeline.{% endif %} The directories listed below will be created in the results directory after the pipeline has finished. All paths are relative to the top-level results directory. @@ -14,9 +14,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d {% if fastqc -%} -- [FastQC](#fastqc) - Raw read QC - {%- endif %} - {%- if multiqc %} +- [FastQC](#fastqc) - Raw read QC{% endif %} + {%- if multiqc -%} - [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline {%- endif %} - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -35,7 +34,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). -{%- endif %} +{%- endif -%} {% if multiqc -%} @@ -54,7 +53,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . -{%- endif %} +{%- endif -%} ### Pipeline information diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 16e6220aaf..bbc8a828c4 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -61,7 +61,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```bash -nextflow run {{ name }} --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker +nextflow run {{ name }} --input ./samplesheet.csv --outdir ./results {% if igenomes %}--genome GRCh37{% endif %} -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -93,7 +93,9 @@ with: ```yaml title="params.yaml" input: './samplesheet.csv' outdir: './results/' +{% if igenomes -%} genome: 'GRCh37' +{% endif -%} <...> ``` @@ -113,7 +115,7 @@ It is a good idea to specify the pipeline version when running the pipeline on y First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. -This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. {% if multiqc %}For example, at the bottom of the MultiQC reports.{% endif %} +This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future.{% if multiqc %} For example, at the bottom of the MultiQC reports.{% endif %} To further assist in reproducibility, you can use share and reuse [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. @@ -136,7 +138,7 @@ Several generic profiles are bundled with the pipeline which instruct the pipeli {%- if nf_core_configs %} -The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to check if your system is suported, please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to check if your system is supported, please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). {% else %} {% endif %} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 90c5728d9a..7d2761d290 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -28,7 +28,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "9a1e8bb6a5d205cf7807dcefca872a3314b2f3e6", + "git_sha": "51ae5406a030d4da1e49e4dab49756844fdd6c7a", "installed_by": ["subworkflows"] }{% if nf_schema %}, "utils_nfschema_plugin": { diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 21174bbdc5..3325af4e06 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -77,11 +77,6 @@ params { includeConfig 'conf/base.config' {%- else %} -{% if modules -%} -// Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' -{%- endif %} - process { // TODO nf-core: Check the defaults for all processes cpus = { 1 * task.attempt } @@ -291,6 +286,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' + defaultBranch = '{{ default_branch }}' nextflowVersion = '!>=24.04.2' version = '{{ version }}' doi = '' @@ -310,7 +306,7 @@ validation { command = "nextflow run {{ name }} -profile --input samplesheet.csv --outdir " fullParameter = "help_full" showHiddenParameter = "show_hidden" - {% if is_nfcore -%} + {%- if is_nfcore %} beforeText = """ -\033[2m----------------------------------------------------\033[0m- \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m @@ -326,7 +322,7 @@ validation { https://doi.org/10.1038/s41587-020-0439-x * Software dependencies - https://github.com/{{ name }}/blob/master/CITATIONS.md + https://github.com/{{ name }}/blob/{{ default_branch }}/CITATIONS.md """{% endif %} }{% if is_nfcore %} summary { @@ -335,3 +331,8 @@ validation { }{% endif %} } {%- endif %} + +{% if modules -%} +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' +{%- endif %} diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 3e59a8ba54..c28929b47d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/{{ name }}/master/nextflow_schema.json", + "$id": "https://raw.githubusercontent.com/{{ name }}/{{ default_branch }}/nextflow_schema.json", "title": "{{ name }} pipeline parameters", "description": "{{ description }}", "type": "object", @@ -243,10 +243,12 @@ { "$ref": "#/$defs/input_output_options" }, - {% if igenomes %}{ + {%- if igenomes %} + { "$ref": "#/$defs/reference_genome_options" },{% endif %} - {% if nf_core_configs %}{ + {%- if nf_core_configs %} + { "$ref": "#/$defs/institutional_config_options" },{% endif %} { diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index be5776b836..3d540600ba 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -140,6 +140,10 @@ workflow PIPELINE_COMPLETION { summary_params = [:] {%- endif %} + {%- if multiqc %} + def multiqc_reports = multiqc_report.toList() + {%- endif %} + // // Completion email and summary // @@ -153,7 +157,7 @@ workflow PIPELINE_COMPLETION { plaintext_email, outdir, monochrome_logs, - {% if multiqc %}multiqc_report.toList(){% else %}[]{% endif %} + {% if multiqc %}multiqc_reports.getVal(),{% else %}[]{% endif %} ) } {%- endif %} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 228dbff897..bfd258760d 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -203,29 +203,24 @@ def logColours(monochrome_logs=true) { return colorcodes } +// Return a single report from an object that may be a Path or List // -// Attach the multiqc report to email -// -def attachMultiqcReport(multiqc_report) { - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") - } - mqc_report = mqc_report[0] - } +def getSingleReport(multiqc_reports) { + if (multiqc_reports instanceof Path) { + return multiqc_reports + } else if (multiqc_reports instanceof List) { + if (multiqc_reports.size() == 0) { + log.warn("[${workflow.manifest.name}] No reports found from process 'MULTIQC'") + return null + } else if (multiqc_reports.size() == 1) { + return multiqc_reports.first() + } else { + log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") + return multiqc_reports.first() } + } else { + return null } - catch (Exception msg) { - log.debug(msg) - if (multiqc_report) { - log.warn("[${workflow.manifest.name}] Could not attach MultiQC report to summary email") - } - } - return mqc_report } // @@ -279,7 +274,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi email_fields['summary'] = summary << misc_fields // On success try attach the multiqc report - def mqc_report = attachMultiqcReport(multiqc_report) + def mqc_report = getSingleReport(multiqc_report) // Check if we are only sending emails on failure def email_address = email @@ -310,7 +305,8 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi if (email_address) { try { if (plaintext_email) { -new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } + new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') + } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } @@ -318,7 +314,7 @@ new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") } catch (Exception msg) { - log.debug(msg) + log.debug(msg.toString()) log.debug("Trying with mail instead of sendmail") // Catch failures and try with plaintext def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test index e43d208b1b..f117040cbd 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -79,4 +79,48 @@ nextflow_function { ) } } + + test("Test Function getSingleReport with a single file") { + function "getSingleReport" + + when { + function { + """ + input[0] = file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") } + ) + } + } + + test("Test Function getSingleReport with multiple files") { + function "getSingleReport" + + when { + function { + """ + input[0] = [ + file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/network.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/expression.tsv', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") }, + { assert !function.result.contains("network.tsv") }, + { assert !function.result.contains("expression.tsv") } + ) + } + } } diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index 3190ed70d4..664d7a22a3 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -11,6 +11,7 @@ from ruamel.yaml import YAML import nf_core.utils +from nf_core.pipelines.rocrate import ROCrate from nf_core.utils import Pipeline log = logging.getLogger(__name__) @@ -127,6 +128,10 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: yaml_key=["template", "version"], ) + # update rocrate if ro-crate is present + if Path(pipeline_obj.wf_path, "ro-crate-metadata.json").exists(): + ROCrate(pipeline_obj.wf_path).update_rocrate() + def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: """Bumps the required Nextflow version number of a pipeline. diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 61e0b63ec3..4f90ca17f9 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -22,7 +22,7 @@ from nf_core.pipelines.create_logo import create_logo from nf_core.pipelines.lint_utils import run_prettier_on_file from nf_core.pipelines.rocrate import ROCrate -from nf_core.utils import NFCoreTemplateConfig, NFCoreYamlLintConfig +from nf_core.utils import NFCoreTemplateConfig, NFCoreYamlLintConfig, custom_yaml_dumper log = logging.getLogger(__name__) @@ -57,7 +57,7 @@ def __init__( template_config: Optional[Union[CreateConfig, str, Path]] = None, organisation: str = "nf-core", from_config_file: bool = False, - default_branch: Optional[str] = None, + default_branch: str = "master", is_interactive: bool = False, ) -> None: if isinstance(template_config, CreateConfig): @@ -87,8 +87,17 @@ def __init__( # Read features yaml file self.template_features_yml = load_features_yaml() + # Set fields used by the class methods + self.no_git = no_git + self.default_branch = default_branch + self.is_interactive = is_interactive + if self.config.outdir is None: self.config.outdir = str(Path.cwd()) + + # Get the default branch name from the Git configuration + self.get_default_branch() + self.jinja_params, self.skip_areas = self.obtain_jinja_params_dict( self.config.skip_features or [], str(self.config.outdir) ) @@ -107,11 +116,6 @@ def __init__( # Set convenience variables self.name = self.config.name - - # Set fields used by the class methods - self.no_git = no_git - self.default_branch = default_branch - self.is_interactive = is_interactive self.force = self.config.force if self.config.outdir == ".": @@ -233,6 +237,7 @@ def obtain_jinja_params_dict( jinja_params["name_docker"] = jinja_params["name"].replace(jinja_params["org"], jinja_params["prefix_nodash"]) jinja_params["logo_light"] = f"{jinja_params['name_noslash']}_logo_light.png" jinja_params["logo_dark"] = f"{jinja_params['name_noslash']}_logo_dark.png" + jinja_params["default_branch"] = self.default_branch if config_yml is not None: if ( hasattr(config_yml, "lint") @@ -254,12 +259,21 @@ def obtain_jinja_params_dict( def init_pipeline(self): """Creates the nf-core pipeline.""" + # Make the new pipeline self.render_template() # Init the git repository and make the first commit if not self.no_git: self.git_init_pipeline() + # Run prettier on files + if self.config.skip_features is None or not ( + "code_linters" in self.config.skip_features or "github" in self.config.skip_features + ): + current_dir = Path.cwd() + os.chdir(self.outdir) + run_prettier_on_file([str(f) for f in self.outdir.glob("**/*")]) + os.chdir(current_dir) if self.config.is_nfcore and not self.is_interactive: log.info( @@ -292,7 +306,6 @@ def render_template(self) -> None: template_dir = Path(nf_core.__file__).parent / "pipeline-template" object_attrs = self.jinja_params object_attrs["nf_core_version"] = nf_core.__version__ - # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980 template_files = list(Path(template_dir).glob("**/*")) template_files += list(Path(template_dir).glob("*")) @@ -357,7 +370,7 @@ def render_template(self) -> None: # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() - if self.config.skip_features is None or "ro-crate" not in self.config.skip_features: + if self.config.skip_features is None or "rocrate" not in self.config.skip_features: # Create the RO-Crate metadata file rocrate_obj = ROCrate(self.outdir) rocrate_obj.create_rocrate(json_path=self.outdir / "ro-crate-metadata.json") @@ -370,12 +383,9 @@ def render_template(self) -> None: if config_fn is not None and config_yml is not None: with open(str(config_fn), "w") as fh: config_yml.template = NFCoreTemplateConfig(**self.config.model_dump(exclude_none=True)) - yaml.safe_dump(config_yml.model_dump(exclude_none=True), fh) + yaml.dump(config_yml.model_dump(exclude_none=True), fh, Dumper=custom_yaml_dumper()) log.debug(f"Dumping pipeline template yml to pipeline config file '{config_fn.name}'") - # Run prettier on files - run_prettier_on_file([str(f) for f in self.outdir.glob("**/*")]) - def fix_linting(self): """ Updates the .nf-core.yml with linting configurations @@ -403,7 +413,13 @@ def fix_linting(self): if config_fn is not None and nf_core_yml is not None: nf_core_yml.lint = NFCoreYamlLintConfig(**lint_config) with open(self.outdir / config_fn, "w") as fh: - yaml.dump(nf_core_yml.model_dump(exclude_none=True), fh, default_flow_style=False, sort_keys=False) + yaml.dump( + nf_core_yml.model_dump(exclude_none=True), + fh, + sort_keys=False, + default_flow_style=False, + Dumper=custom_yaml_dumper(), + ) def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" @@ -421,20 +437,18 @@ def make_pipeline_logo(self): force=bool(self.force), ) - def git_init_pipeline(self) -> None: - """Initialises the new pipeline as a Git repository and submits first commit. - - Raises: - UserWarning: if Git default branch is set to 'dev' or 'TEMPLATE'. - """ - default_branch: Optional[str] = self.default_branch + def get_default_branch(self) -> None: + """Gets the default branch name from the Git configuration.""" try: - default_branch = default_branch or str(git.config.GitConfigParser().get_value("init", "defaultBranch")) + self.default_branch = ( + str(git.config.GitConfigParser().get_value("init", "defaultBranch")) or "master" + ) # default to master + log.debug(f"Default branch name: {self.default_branch}") except configparser.Error: log.debug("Could not read init.defaultBranch") - if default_branch in ["dev", "TEMPLATE"]: + if self.default_branch in ["dev", "TEMPLATE"]: raise UserWarning( - f"Your Git defaultBranch '{default_branch}' is incompatible with nf-core.\n" + f"Your Git defaultBranch '{self.default_branch}' is incompatible with nf-core.\n" "'dev' and 'TEMPLATE' can not be used as default branch name.\n" "Set the default branch name with " "[white on grey23] git config --global init.defaultBranch [/]\n" @@ -442,12 +456,19 @@ def git_init_pipeline(self) -> None: "Pipeline git repository will not be initialised." ) + def git_init_pipeline(self) -> None: + """Initialises the new pipeline as a Git repository and submits first commit. + + Raises: + UserWarning: if Git default branch is set to 'dev' or 'TEMPLATE'. + """ + log.info("Initialising local pipeline git repository") repo = git.Repo.init(self.outdir) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") - if default_branch: - repo.active_branch.rename(default_branch) + if self.default_branch: + repo.active_branch.rename(self.default_branch) try: repo.git.branch("TEMPLATE") repo.git.branch("dev") diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 9841879e83..fa24debffe 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -148,6 +148,10 @@ is_nfcore: - "docs/images/nf-core-{{short_name}}_logo_light.png" - "docs/images/nf-core-{{short_name}}_logo_dark.png" - ".github/ISSUE_TEMPLATE/bug_report.yml" + - ".github/CONTRIBUTING.md" + - ".github/PULL_REQUEST_TEMPLATE.md" + - "assets/email_template.txt" + - "docs/README.md" nextflow_config: - "manifest.name" - "manifest.homePage" @@ -445,6 +449,8 @@ rocrate: linting: files_warn: - "ro-crate-metadata.json" + files_unchanged: + - ".prettierignore" vscode: skippable_paths: - ".vscode" diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index d37dce86d1..11adebce2c 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -1744,7 +1744,7 @@ def tidy_tags_and_branches(self): for tag in tags_to_remove: self.repo.delete_tag(tag) - # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") + # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "main") self.checkout(self.revision[0]) # delete unwanted heads/branches from repository diff --git a/nf_core/pipelines/lint/actions_awsfulltest.py b/nf_core/pipelines/lint/actions_awsfulltest.py index 7ea167f6c9..080ae3583e 100644 --- a/nf_core/pipelines/lint/actions_awsfulltest.py +++ b/nf_core/pipelines/lint/actions_awsfulltest.py @@ -42,7 +42,7 @@ def actions_awsfulltest(self) -> Dict[str, List[str]]: # Check that the action is only turned on for published releases try: - if wf[True]["pull_request"]["branches"] != ["master"]: + if wf[True]["pull_request"]["branches"] != ["main", "master"]: raise AssertionError() if wf[True]["pull_request_review"]["types"] != ["submitted"]: raise AssertionError() diff --git a/nf_core/pipelines/lint/version_consistency.py b/nf_core/pipelines/lint/version_consistency.py index 5fe24ed723..2f9cead83c 100644 --- a/nf_core/pipelines/lint/version_consistency.py +++ b/nf_core/pipelines/lint/version_consistency.py @@ -5,7 +5,7 @@ def version_consistency(self): """Pipeline and container version number consistency. .. note:: This test only runs when the ``--release`` flag is set for ``nf-core pipelines lint``, - or ``$GITHUB_REF`` is equal to ``master``. + or ``$GITHUB_REF`` is equal to ``main``. This lint fetches the pipeline version number from three possible locations: diff --git a/nf_core/pipelines/lint_utils.py b/nf_core/pipelines/lint_utils.py index b4c56c6007..a6b98b1899 100644 --- a/nf_core/pipelines/lint_utils.py +++ b/nf_core/pipelines/lint_utils.py @@ -97,7 +97,7 @@ def run_prettier_on_file(file: Union[Path, str, List[str]]) -> None: all_lines = [line for line in e.stdout.decode().split("\n")] files = "\n".join(all_lines[3:]) log.debug(f"The following files were modified by prettier:\n {files}") - elif e.stderr.decode(): + else: log.warning( "There was an error running the prettier pre-commit hook.\n" f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index 915f203f00..bc868273c4 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -216,6 +216,9 @@ def set_main_entity(self, main_entity_filename: str): ) self.crate.mainEntity.append_to("version", self.version, compact=True) + # remove duplicate entries for version + self.crate.mainEntity["version"] = list(set(self.crate.mainEntity["version"])) + # get keywords from nf-core website remote_workflows = requests.get("https://nf-co.re/pipelines.json").json()["remote_workflows"] # go through all remote workflows and find the one that matches the pipeline name @@ -236,6 +239,9 @@ def set_main_entity(self, main_entity_filename: str): self.crate.mainEntity.append_to("license", self.crate.license) self.crate.mainEntity.append_to("name", self.crate.name) + # remove duplicate entries for name + self.crate.mainEntity["name"] = list(set(self.crate.mainEntity["name"])) + if "dev" in self.version: self.crate.creativeWorkStatus = "InProgress" else: @@ -261,14 +267,26 @@ def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: # add author entity to crate try: - authors = self.pipeline_obj.nf_config["manifest.author"].split(",") - # remove spaces - authors = [a.strip() for a in authors] + authors = [] + if "manifest.author" in self.pipeline_obj.nf_config: + authors.extend([a.strip() for a in self.pipeline_obj.nf_config["manifest.author"].split(",")]) + if "manifest.contributor" in self.pipeline_obj.nf_config: + authors.extend( + [ + c.get("name", "").strip() + for c in self.pipeline_obj.nf_config["manifest.contributor"] + if "name" in c + ] + ) + if not authors: + raise KeyError("No authors found") # add manifest authors as maintainer to crate except KeyError: - log.error("No author field found in manifest of nextflow.config") + log.error("No author or contributor fields found in manifest of nextflow.config") return + # remove duplicates + authors = list(set(authors)) # look at git contributors for author names try: git_contributors: Set[str] = set() @@ -330,6 +348,25 @@ def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: if author in authors: wf_file.append_to("maintainer", author_entitity) + def update_rocrate(self) -> bool: + """ + Update the rocrate file + """ + # check if we need to output a json file and/or a zip file based on the file extensions + # try to find a json file + json_path: Optional[Path] = None + potential_json_path = Path(self.pipeline_dir, "ro-crate-metadata.json") + if potential_json_path.exists(): + json_path = potential_json_path + + # try to find a zip file + zip_path: Optional[Path] = None + potential_zip_path = Path(self.pipeline_dir, "ro-crate.crate.zip") + if potential_zip_path.exists(): + zip_path = potential_zip_path + + return self.create_rocrate(json_path=json_path, zip_path=zip_path) + def get_orcid(name: str) -> Optional[str]: """ diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 61fd6bc2d7..a08dd0a2d0 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -517,11 +517,13 @@ def validate_schema_title_description(self, schema=None): if "title" not in self.schema: raise AssertionError("Schema missing top-level `title` attribute") # Validate that id, title and description match the pipeline manifest - id_attr = "https://raw.githubusercontent.com/{}/master/nextflow_schema.json".format( + id_attr = "https://raw.githubusercontent.com/{}/main/nextflow_schema.json".format( self.pipeline_manifest["name"].strip("\"'") ) - if self.schema["$id"] != id_attr: - raise AssertionError(f"Schema `$id` should be `{id_attr}`\n Found `{self.schema['$id']}`") + if self.schema["$id"] not in [id_attr, id_attr.replace("/main/", "/master/")]: + raise AssertionError( + f"Schema `$id` should be `{id_attr}` or {id_attr.replace('/main/', '/master/')}. \n Found `{self.schema['$id']}`" + ) title_attr = "{} pipeline parameters".format(self.pipeline_manifest["name"].strip("\"'")) if self.schema["title"] != title_attr: diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 7ca825f04f..8e9e62430a 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -50,7 +50,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): subworkflow.warned.append( ( "test_dir_exists", - "nf-test directory is missing", + "Migrate pytest-workflow to nf-test", subworkflow.nftest_testdir, ) ) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index efb7a8e03e..43f9b8046e 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,10 +9,7 @@ import git from git.exc import GitCommandError -from nf_core.components.constants import ( - NF_CORE_MODULES_NAME, - NF_CORE_MODULES_REMOTE, -) +from nf_core.components.constants import NF_CORE_MODULES_DEFAULT_BRANCH, NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -186,7 +183,7 @@ def setup_branch(self, branch): if branch is None: # Don't bother fetching default branch if we're using nf-core if self.remote_url == NF_CORE_MODULES_REMOTE: - self.branch = "master" + self.branch = NF_CORE_MODULES_DEFAULT_BRANCH else: self.branch = self.get_default_branch() else: diff --git a/nf_core/utils.py b/nf_core/utils.py index 30b0743493..dc208c0a78 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1083,7 +1083,7 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") # Get branch information from github api - should be no need to check if the repo exists again - branch_response = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/branches") + branch_response = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/branches?per_page=100") for branch in branch_response.json(): if ( branch["name"] != "TEMPLATE" diff --git a/requirements-dev.txt b/requirements-dev.txt index aab9b1e5d7..04c6372d72 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ responses ruff Sphinx sphinx-rtd-theme -textual-dev==1.6.1 +textual-dev==1.5.1 types-PyYAML types-requests types-jsonschema diff --git a/setup.py b/setup.py index 11b3022494..b5c5de9a4f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.0.3dev" +version = "3.1.1dev" with open("README.md") as f: readme = f.read() diff --git a/tests/pipelines/lint/test_actions_awstest.py b/tests/pipelines/lint/test_actions_awstest.py index 51b55cb867..01dc9f6168 100644 --- a/tests/pipelines/lint/test_actions_awstest.py +++ b/tests/pipelines/lint/test_actions_awstest.py @@ -24,7 +24,7 @@ def test_actions_awstest_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: awstest_yml = yaml.safe_load(fh) - awstest_yml[True]["push"] = ["master"] + awstest_yml[True]["push"] = ["main"] with open(Path(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: yaml.dump(awstest_yml, fh) diff --git a/tests/pipelines/test_lint.py b/tests/pipelines/test_lint.py index ca7353d50d..f33ac3a2ee 100644 --- a/tests/pipelines/test_lint.py +++ b/tests/pipelines/test_lint.py @@ -25,7 +25,7 @@ def setUp(self) -> None: ########################## class TestPipelinesLint(TestLint): def test_run_linting_function(self): - """Run the master run_linting() function in lint.py + """Run the run_linting() function in lint.py We don't really check any of this code as it's just a series of function calls and we're testing each of those individually. This is mostly to check for syntax errors.""" diff --git a/tests/pipelines/test_rocrate.py b/tests/pipelines/test_rocrate.py index 01a77ecd76..ac86e64bdf 100644 --- a/tests/pipelines/test_rocrate.py +++ b/tests/pipelines/test_rocrate.py @@ -1,5 +1,6 @@ """Test the nf-core pipelines rocrate command""" +import json import shutil import tempfile from pathlib import Path @@ -12,6 +13,7 @@ import nf_core.pipelines.create.create import nf_core.pipelines.rocrate import nf_core.utils +from nf_core.pipelines.bump_version import bump_pipeline_version from ..test_pipelines import TestPipelines @@ -125,3 +127,36 @@ def test_rocrate_creation_for_fetchngs(self): # Clean up shutil.rmtree(tmp_dir) + + def test_update_rocrate(self): + """Run the nf-core rocrate command with a zip output""" + + assert self.rocrate_obj.create_rocrate(json_path=self.pipeline_dir, zip_path=self.pipeline_dir) + + # read the crate json file + with open(Path(self.pipeline_dir, "ro-crate-metadata.json")) as f: + crate = json.load(f) + + # check the old version + self.assertEqual(crate["@graph"][2]["version"][0], "1.0.0dev") + # check creativeWorkStatus is InProgress + self.assertEqual(crate["@graph"][0]["creativeWorkStatus"], "InProgress") + + # bump version + bump_pipeline_version(self.pipeline_obj, "1.1.0") + + # Check that the crate was created + self.assertTrue(Path(self.pipeline_dir, "ro-crate.crate.zip").exists()) + + # Check that the crate was updated + self.assertTrue(Path(self.pipeline_dir, "ro-crate-metadata.json").exists()) + + # read the crate json file + with open(Path(self.pipeline_dir, "ro-crate-metadata.json")) as f: + crate = json.load(f) + + # check that the version was updated + self.assertEqual(crate["@graph"][2]["version"][0], "1.1.0") + + # check creativeWorkStatus is Stable + self.assertEqual(crate["@graph"][0]["creativeWorkStatus"], "Stable") diff --git a/tests/pipelines/test_sync.py b/tests/pipelines/test_sync.py index 8bf8a3c4ec..9959975e27 100644 --- a/tests/pipelines/test_sync.py +++ b/tests/pipelines/test_sync.py @@ -43,14 +43,14 @@ def mocked_requests_get(url) -> MockResponse: { "state": "closed", "head": {"ref": "nf-core-template-merge-2"}, - "base": {"ref": "master"}, + "base": {"ref": "main"}, "html_url": "pr_url", } ] + [ { "state": "open", "head": {"ref": f"nf-core-template-merge-{branch_no}"}, - "base": {"ref": "master"}, + "base": {"ref": "main"}, "html_url": "pr_url", } for branch_no in range(3, 7) @@ -345,7 +345,7 @@ def test_close_open_pr(self, mock_patch, mock_post) -> None: pr: Dict[str, Union[str, Dict[str, str]]] = { "state": "open", "head": {"ref": "nf-core-template-merge-3"}, - "base": {"ref": "master"}, + "base": {"ref": "main"}, "html_url": "pr_html_url", "url": "url_to_update_pr", "comments_url": "pr_comments_url", @@ -368,7 +368,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): pr = { "state": "open", "head": {"ref": "nf-core-template-merge-3"}, - "base": {"ref": "master"}, + "base": {"ref": "main"}, "html_url": "pr_html_url", "url": "bad_url_to_update_pr", "comments_url": "pr_comments_url", diff --git a/tests/utils.py b/tests/utils.py index cffe8ba103..1aa3750d8e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -14,7 +14,7 @@ import nf_core.modules import nf_core.pipelines.create.create from nf_core import __version__ -from nf_core.utils import NFCoreTemplateConfig, NFCoreYamlConfig +from nf_core.utils import NFCoreTemplateConfig, NFCoreYamlConfig, custom_yaml_dumper TEST_DATA_DIR = Path(__file__).parent / "data" OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" @@ -138,7 +138,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: bump_version=None, ) with open(str(Path(pipeline_dir, ".nf-core.yml")), "w") as fh: - yaml.dump(nf_core_yml.model_dump(), fh) + yaml.dump(nf_core_yml.model_dump(), fh, Dumper=custom_yaml_dumper()) nf_core.pipelines.create.create.PipelineCreate( pipeline_name, "it is mine", "me", no_git=no_git, outdir=pipeline_dir, force=True