From efb3016cf96c6db2d3be2f0818cd661e9bc788b0 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Fri, 26 Jan 2024 08:51:47 +0000 Subject: [PATCH] Lint packaged workflows (#5873) * prevent lines starting with % being marked as a problem. * add a noqa comment to Cylc lint * run tutoral linting via tutorial .validate scripts --- changes.d/5873.feat.md | 3 ++ .../tutorial/consolidation-tutorial/.validate | 1 + .../cylc-forecasting-workflow/.validate | 1 + .../etc/tutorial/retries-tutorial/.validate | 1 + .../etc/tutorial/retries-tutorial/flow.cylc | 4 +- .../tutorial/runtime-introduction/.validate | 1 + cylc/flow/scripts/lint.py | 54 ++++++++++++++++--- tests/unit/scripts/test_lint.py | 17 ++++++ 8 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 changes.d/5873.feat.md diff --git a/changes.d/5873.feat.md b/changes.d/5873.feat.md new file mode 100644 index 00000000000..0b718922607 --- /dev/null +++ b/changes.d/5873.feat.md @@ -0,0 +1,3 @@ +- Allow use of `#noqa: S001` comments to skip Cylc lint + checks for a single line. +- Stop Cylc lint objecting to `%include ` syntax. \ No newline at end of file diff --git a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate index 4ecb762f179..2c73648700e 100755 --- a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate +++ b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate @@ -18,4 +18,5 @@ set -eux +cylc lint . cylc validate . --icp=2000 diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate index 2292afd6aad..38d306164de 100755 --- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate +++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate @@ -18,6 +18,7 @@ set -eux APIKEY="$(head --lines 1 ../api-keys)" FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc lint . cylc install --workflow-name "$FLOW_NAME" --no-run-name sed -i "s/DATAPOINT_API_KEY/$APIKEY/" "$HOME/cylc-run/$FLOW_NAME/flow.cylc" cylc validate --check-circular --icp=2000 "$FLOW_NAME" diff --git a/cylc/flow/etc/tutorial/retries-tutorial/.validate b/cylc/flow/etc/tutorial/retries-tutorial/.validate index 1c532cd1120..1eb85986072 100755 --- a/cylc/flow/etc/tutorial/retries-tutorial/.validate +++ b/cylc/flow/etc/tutorial/retries-tutorial/.validate @@ -18,4 +18,5 @@ set -eux +cylc lint . cylc validate --check-circular . --icp=2000 diff --git a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc index d0e920fba0e..04b97c274c1 100644 --- a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc +++ b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc @@ -16,8 +16,8 @@ DIE_2=$((RANDOM%6 + 1)) echo "Rolled $DIE_1 and $DIE_2..." if (($DIE_1 == $DIE_2)); then - echo "doubles!" + echo "doubles!" else - exit 1 + exit 1 fi """ diff --git a/cylc/flow/etc/tutorial/runtime-introduction/.validate b/cylc/flow/etc/tutorial/runtime-introduction/.validate index 94dace9fbbc..ec6980f05da 100755 --- a/cylc/flow/etc/tutorial/runtime-introduction/.validate +++ b/cylc/flow/etc/tutorial/runtime-introduction/.validate @@ -18,6 +18,7 @@ set -eux FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)" SRC=$(cylc get-resources tutorial 2>&1 | head -n1 | awk '{print $NF}') +cylc lint . cylc install "${SRC}/runtime-introduction" --workflow-name "$FLOW_NAME" --no-run-name cylc validate --check-circular --icp=2000 "$FLOW_NAME" cylc play --no-detach --abort-if-any-task-fails "$FLOW_NAME" diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index 636d611194f..f038d569d78 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -37,6 +37,12 @@ This can be overridden by providing the "--exit-zero" flag. """ +NOQA = """ +Individual errors can be ignored using the ``# noqa`` line comment. +It is good practice to specify specific errors you wish to ignore using +``# noqa: S002 S007 U999`` +""" + TOMLDOC = """ pyproject.toml configuration:{} [cylc-lint] # any of {} @@ -399,7 +405,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: 'short': 'Item not indented.', # Non-indented items should be sections: 'url': STYLE_GUIDE + 'indentation', - FUNCTION: re.compile(r'^[^\{\[|\s]').findall + FUNCTION: re.compile(r'^[^%\{\[|\s]').findall }, "S003": { 'short': 'Top level sections should not be indented.', @@ -1066,6 +1072,33 @@ def check_cylc_file( pass +def no_qa(line: str, index: str): + """This line has a no-qa comment. + + Examples: + # No comment, no exception: + >>> no_qa('foo = bar', 'S001') + False + + # Comment, no error codes, no checking: + >>> no_qa('foo = bar # noqa', 'S001') + True + + # Comment, no relevent error codes, no checking: + >>> no_qa('foo = bar # noqa: S999, 997', 'S001') + False + + # Comment, relevent error codes, checking: + >>> no_qa('foo = bar # noqa: S001 S003', 'S001') + True + """ + NOQA = re.compile(r'.*#\s*[Nn][Oo][Qq][Aa]:?(.*)') + noqa = NOQA.findall(line) + if noqa and (noqa[0] == '' or index in noqa[0]): + return True + return False + + def lint( file_rel: Path, lines: Iterator[str], @@ -1105,9 +1138,13 @@ def lint( # run lint checks against the current line for index, check_meta in checks.items(): # Skip commented line unless check says not to. + index_str = get_index_str(check_meta, index) if ( - line.strip().startswith('#') - and not check_meta.get('evaluate commented lines', False) + ( + line.strip().startswith('#') + and not check_meta.get('evaluate commented lines', False) + ) + or no_qa(line, index_str) ): continue @@ -1139,7 +1176,7 @@ def lint( url = get_url(check_meta) yield ( - f'# [{get_index_str(check_meta, index)}]: ' + f'# [{index_str}]: ' f'{msg}\n' f'# - see {url}\n' ) @@ -1147,7 +1184,7 @@ def lint( # write a message to inform the user write( Fore.YELLOW + - f'[{get_index_str(check_meta, index)}]' + f'[{index_str}]' f' {file_rel}:{line_no}: {msg}' ) if modify: @@ -1278,7 +1315,11 @@ def target_version_check( def get_option_parser() -> COP: parser = COP( - COP_DOC + TOMLDOC.format('', str(LINT_SECTIONS)), + ( + COP_DOC + + NOQA.replace('``', '"') + + TOMLDOC.format('', str(LINT_SECTIONS)) + ), argdoc=[ COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC) ], @@ -1403,4 +1444,5 @@ def main(parser: COP, options: 'Values', target=None) -> None: # NOTE: use += so that this works with __import__ # (docstring needed for `cylc help all` output) +__doc__ += NOQA __doc__ += get_reference('all', 'rst') diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index db3ae052ca3..f3009a6ddb3 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -148,6 +148,8 @@ [[and_another_thing]] [[[remote]]] host = `rose host-select thingy` + +%include foo.cylc """ @@ -649,3 +651,18 @@ def test_indents(spaces, expect): assert expect in result else: assert not result + + +def test_noqa(): + """Comments turn of checks. + + """ + output = lint_text( + 'foo = bar#noqa\n' + 'qux = baz # noqa: S002\n' + 'buzz = food # noqa: S007\n' + 'quixotic = foolish # noqa: S007, S992 S002\n', + ['style'] + ) + assert len(output.messages) == 1 + assert 'flow.cylc:3' in output.messages[0]