From 59d7e92f46dae48e3acd9e294b9dce1f0b2e35f9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 12:57:23 +0100 Subject: [PATCH 01/10] add more debug logging --- entrypoint.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/entrypoint.py b/entrypoint.py index 2c161b1..16d7802 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -4,16 +4,15 @@ # with changed dependencies, then return as a JSON list import argparse +import fnmatch import json import logging import os -import yaml -import logging - from enum import Enum -from git import Repo from pathlib import Path -import fnmatch + +import yaml +from git import Repo class TestTargetType(Enum): @@ -39,6 +38,7 @@ class NextflowFile: def __init__(self, path): self.path = path + logging.debug(f"Reading {self.path}") self.lines = self.read_nf_file() self.includes = self.find_include_statements() @@ -225,8 +225,10 @@ def find_tags(self) -> list[str]: result = [] for line in self.lines: if line.strip().startswith("tag "): + logging.debug(f"Found tag line: {line}") tag = line.strip().removeprefix("tag ").strip().strip("'\"").casefold() result.append(tag) + logging.debug(f"Found tags: {result}") return result def detect_if_path_is_in_test(self, path: Path) -> bool: From 7868858ce8193d467fd9c0e044958a78100e6b01 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 12:58:26 +0100 Subject: [PATCH 02/10] replace walk_up paramter, not avaiable in py<3.12 --- entrypoint.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/entrypoint.py b/entrypoint.py index 16d7802..76e7a3d 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -214,7 +214,10 @@ def find_common_path(self) -> Path: nf_path_dir = self.nextflow.path.parent.resolve() test_path_dir = self.test_path.parent.resolve() # Find diff between them - diff = nf_path_dir.relative_to(test_path_dir, walk_up=True) + try: + diff = nf_path_dir.relative_to(test_path_dir) + except ValueError: + diff = nf_path_dir # Return common path return test_path_dir.joinpath(diff).resolve() From 725c7c6c40d292fa4274b1a93f3b5cb6f9066d21 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 14:22:54 +0100 Subject: [PATCH 03/10] add exclude_tags paramter --- .github/workflows/test.yml | 16 +++++++++++++++- README.md | 12 ++++++++++++ entrypoint.py | 23 ++++++++++++++++++----- entrypoint.sh | 1 + 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 299f7d9..f4daeeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,7 +80,19 @@ jobs: root: testrepo/test head: change_module_a base: main - tags: "test" # should pick up no tests + tags: "PIPELINE" # should pick one test + + + - name: Detect tags + id: exclude_tags + uses: ./ + with: + root: testrepo/test + head: change_module_a + base: main + tags: "PIPELINE" # should pick one test + exclude_tags: "PIPELINE" # should drop the test + - name: print outputs run: | @@ -90,3 +102,5 @@ jobs: echo Directories: ${{ steps.detect_module_a_directory.outputs.components }} echo Config files: ${{ steps.detect_config_changes.outputs.components }} echo Additional include file: ${{ steps.detect_additional_include_file.outputs.components }} + echo Tags: ${{ steps.detect_tags.outputs.components }} + echo Exclude tags: ${{ steps.exclude_tags.outputs.components }} diff --git a/README.md b/README.md index 1fdf77b..f6b5988 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,18 @@ steps: tags: "gpu" ``` +You may wish to exclude tests with specific tags. You can do this by specifying the tags in the `exclude_tags` parameter. + +```yaml +steps: +- uses: actions/checkout@v4 +- uses: adamrtalbot/detect-nf-test + with: + head: ${{ github.sha }} + base: ${{ github.base_ref }} + exclude_tags: "gpu" +``` + ## Outputs A list of changed directories or files and their immediate dependencies is returned under the variable `components`. diff --git a/entrypoint.py b/entrypoint.py index 76e7a3d..3fd7e10 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -297,9 +297,14 @@ def find_matching_dependencies(self, other_nf_tests): def find_matching_tags(self, other_nf_tests): """ - Finds and returns a list of NF tests from `other_nf_tests` that have matching tags in `self.tags`. + Finds and returns a list of NF tests from `other_nf_tests` that have matching tags in `self.tags`. If exclude_tags is specified, it will exclude those tags. + """ - return [nf_test for nf_test in other_nf_tests if nf_test.tags in self.tags] + return [ + nf_test + for nf_test in other_nf_tests + if nf_test.tags in self.tags and nf_test.tags not in self.exclude_tags + ] def get_parents(self, n: int) -> Path: """ @@ -396,6 +401,9 @@ def parse_args() -> argparse.Namespace: default="", help="Tags to include.", ) + parser.add_argument( + "-xT", "--exclude_tags", type=str, default="", help="Tags to exclude." + ) return parser.parse_args() @@ -618,13 +626,18 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: nf_test for nf_test in all_nf_tests if nf_test.test_type.value in args.types ] if args.tags: - logging.debug(f"Filtering down to only relevant test tags: {args.tags}") + logging.debug(f"Filtering down to only included test tags: {args.tags}") + only_selected_nf_tests = [ + nf_test for nf_test in only_selected_nf_tests if nf_test.tags in args.tags + ] + + if args.exclude_tags: + logging.debug(f"Excluding test tags: {args.exclude_tags}") only_selected_nf_tests = [ nf_test for nf_test in only_selected_nf_tests - if nf_test.find_matching_tags(only_selected_nf_tests) + if nf_test.tags not in args.exclude_tags ] - # Go back n_parents directories, remove root from path and stringify # It's a bit much but might as well do all path manipulation in one place logging.info("Normalising test file paths") diff --git a/entrypoint.sh b/entrypoint.sh index 2641153..5f71ae2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,3 +13,4 @@ python /entrypoint.py \ ${TYPES:+"-t=$TYPES"} \ ${N_PARENTS:+"--n_parents=$N_PARENTS"} \ ${TAGS:+"-T=$TAGS"} + ${EXCLUDE_TAGS:+"-xT=$EXCLUDE_TAGS"} From 98f6e72df697531f81f6267c0cfa731419fd43cf Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 14:33:24 +0100 Subject: [PATCH 04/10] fix tags type --- entrypoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/entrypoint.py b/entrypoint.py index 3fd7e10..410f500 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -546,6 +546,8 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: logging.basicConfig(level=args.log_level) # Argparse handling of nargs is a bit rubbish. So we do it manually here. args.types = args.types.split(",") + args.tags = args.tags.split(",") + args.exclude_tags = args.exclude_tags.split(",") # Quick validation of args.types since we cant do this in argparse if any( _type not in ["function", "process", "workflow", "pipeline"] From 88832355ea0f4ccb152463478dda4c3e92670185 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 14:35:51 +0100 Subject: [PATCH 05/10] add input parameter docs --- action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/action.yml b/action.yml index 396ba66..003f4ec 100644 --- a/action.yml +++ b/action.yml @@ -35,6 +35,10 @@ inputs: description: 'List of tags to include in the output as a comma separated list. Default: "".' required: false default: '' + exclude_tags: + description: 'List of tags to exclude in the output as a comma separated list. Default: "".' + required: false + default: '' log_level: description: 'Log level for the action. Default: "INFO".' required: false From 252779a25d45032e4f7d7638f304dcc989fb8453 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 25 Nov 2024 14:44:46 +0100 Subject: [PATCH 06/10] use correct test branch --- .github/workflows/test.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08d585c..e2f5ca8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,28 +73,25 @@ jobs: head: modify_main_nextflow_file base: main - - name: Detect tags + - name: Include tags id: detect_tags uses: ./ with: root: testrepo/test - head: change_module_a + head: modify_main_nextflow_file base: main tags: "PIPELINE" # should pick one test - - - name: Detect tags + - name: Exclude tags id: exclude_tags uses: ./ with: root: testrepo/test - head: change_module_a + head: modify_main_nextflow_file base: main tags: "PIPELINE" # should pick one test exclude_tags: "PIPELINE" # should drop the test - - - name: Detect multi-line include id: multi_line_include uses: ./ From 5bf52d2250b4c916cd6215ad8276a1729128dbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 26 Nov 2024 14:49:46 +0100 Subject: [PATCH 07/10] Update entrypoint.py Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> --- entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.py b/entrypoint.py index 64d11cc..ae339c0 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -417,7 +417,7 @@ def parse_args() -> argparse.Namespace: help="Tags to include.", ) parser.add_argument( - "-xT", "--exclude_tags", type=str, default="", help="Tags to exclude." + "--exclude_tags", type=str, default="", help="Tags to exclude." ) return parser.parse_args() From f9d746ea636c54c372a8bd254af74aab48c37eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 26 Nov 2024 14:49:53 +0100 Subject: [PATCH 08/10] Update entrypoint.sh Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 5f71ae2..aa70594 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,4 +13,4 @@ python /entrypoint.py \ ${TYPES:+"-t=$TYPES"} \ ${N_PARENTS:+"--n_parents=$N_PARENTS"} \ ${TAGS:+"-T=$TAGS"} - ${EXCLUDE_TAGS:+"-xT=$EXCLUDE_TAGS"} + ${EXCLUDE_TAGS:+"--exclude_tags=$EXCLUDE_TAGS"} From e6f2ae70af3b13ebdf8c5a5d08f59c9dd86c91a6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Nov 2024 14:54:35 +0100 Subject: [PATCH 09/10] compare tags element-wise --- entrypoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/entrypoint.py b/entrypoint.py index 64d11cc..39f0e43 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -645,7 +645,9 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: if args.tags: logging.debug(f"Filtering down to only included test tags: {args.tags}") only_selected_nf_tests = [ - nf_test for nf_test in only_selected_nf_tests if nf_test.tags in args.tags + nf_test + for nf_test in only_selected_nf_tests + if any(tag in args.tags for tag in nf_test.tags) ] if args.exclude_tags: @@ -653,7 +655,7 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: only_selected_nf_tests = [ nf_test for nf_test in only_selected_nf_tests - if nf_test.tags not in args.exclude_tags + if any(tag in args.exclude_tags for tag in nf_test.tags) ] # Go back n_parents directories, remove root from path and stringify # It's a bit much but might as well do all path manipulation in one place From ee6251d1eef071180df9991bb4adfc2470e0aac2 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Thu, 28 Nov 2024 21:48:01 +1300 Subject: [PATCH 10/10] Applied include/exclude fixes --- action.yml | 1 + entrypoint.py | 33 ++++++++++++++++----------------- entrypoint.sh | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/action.yml b/action.yml index 003f4ec..1e7a5c5 100644 --- a/action.yml +++ b/action.yml @@ -59,3 +59,4 @@ runs: TYPES: ${{ inputs.types }} N_PARENTS: ${{ inputs.n_parents }} TAGS: ${{ inputs.tags }} + EXCLUDE_TAGS: ${{ inputs.exclude_tags }} diff --git a/entrypoint.py b/entrypoint.py index 4c3c8ca..63856f4 100755 --- a/entrypoint.py +++ b/entrypoint.py @@ -310,17 +310,6 @@ def find_matching_dependencies(self, other_nf_tests): if nf_test.test_name.casefold() in self.dependencies ] - def find_matching_tags(self, other_nf_tests): - """ - Finds and returns a list of NF tests from `other_nf_tests` that have matching tags in `self.tags`. If exclude_tags is specified, it will exclude those tags. - - """ - return [ - nf_test - for nf_test in other_nf_tests - if nf_test.tags in self.tags and nf_test.tags not in self.exclude_tags - ] - def get_parents(self, n: int) -> Path: """ Get the parent directory of a path n levels up. @@ -416,9 +405,7 @@ def parse_args() -> argparse.Namespace: default="", help="Tags to include.", ) - parser.add_argument( - "--exclude_tags", type=str, default="", help="Tags to exclude." - ) + parser.add_argument("--exclude_tags", type=str, default="", help="Tags to exclude.") return parser.parse_args() @@ -561,8 +548,8 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: logging.basicConfig(level=args.log_level) # Argparse handling of nargs is a bit rubbish. So we do it manually here. args.types = args.types.split(",") - args.tags = args.tags.split(",") - args.exclude_tags = args.exclude_tags.split(",") + args.tags = [tag.strip().casefold() for tag in args.tags.split(",")] + args.exclude_tags = [tag.strip().casefold() for tag in args.exclude_tags.split(",")] # Quick validation of args.types since we cant do this in argparse if any( _type not in ["function", "process", "workflow", "pipeline"] @@ -644,19 +631,31 @@ def detect_files(paths: list[Path], suffix: str) -> list[Path]: ] if args.tags: logging.debug(f"Filtering down to only included test tags: {args.tags}") + logging.debug( + f"Tests before the filter are: {[test.test_path for test in only_selected_nf_tests]}" + ) only_selected_nf_tests = [ nf_test for nf_test in only_selected_nf_tests if any(tag in args.tags for tag in nf_test.tags) ] + logging.debug( + f"Tests after the filter are: {[test.test_path for test in only_selected_nf_tests]}" + ) if args.exclude_tags: logging.debug(f"Excluding test tags: {args.exclude_tags}") + logging.debug( + f"Tests before the filter are: {[test.test_path for test in only_selected_nf_tests]}" + ) only_selected_nf_tests = [ nf_test for nf_test in only_selected_nf_tests - if any(tag in args.exclude_tags for tag in nf_test.tags) + if all(tag not in args.exclude_tags for tag in nf_test.tags) ] + logging.debug( + f"Tests after the filter are: {[test.test_path for test in only_selected_nf_tests]}" + ) # Go back n_parents directories, remove root from path and stringify # It's a bit much but might as well do all path manipulation in one place logging.info("Normalising test file paths") diff --git a/entrypoint.sh b/entrypoint.sh index aa70594..95d9724 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -12,5 +12,5 @@ python /entrypoint.py \ ${LOG_LEVEL:+"--log-level=$LOG_LEVEL"} \ ${TYPES:+"-t=$TYPES"} \ ${N_PARENTS:+"--n_parents=$N_PARENTS"} \ - ${TAGS:+"-T=$TAGS"} + ${TAGS:+"-T=$TAGS"} \ ${EXCLUDE_TAGS:+"--exclude_tags=$EXCLUDE_TAGS"}