From 43aeabede4252f74194904fc06f005ea7d0aba2c Mon Sep 17 00:00:00 2001 From: jaylfc Date: Mon, 17 Aug 2026 22:16:59 +0000 Subject: [PATCH] normalise-handle gate: generalise to any duplicate function/method name The duplicate-definition gate was keyed on one hardcoded name (_normalise_handle), so duplicate definitions of anything else were invisible, including shadowed test functions. Generalise it to report any top-level function or method name defined more than once in the same module, and extend the scan to cover tests/. @typing.overload stubs and @property / @.setter / @.getter / @.deleter accessor pairs remain excluded as legitimate same-name definitions, reusing the decorator-skip introduced by #308. Proven red against origin/exec/tsk-v53vta, where three shadowed test_a2a_banner_mode_* functions in tests/test_http_server.py are defined twice, and green on current master. --- .github/workflows/normalise-handle-gate.yml | 12 +- .../tsk-35ydf3-generalise-duplicate-gate.md | 2 + scripts/normalise_handle_gate.py | 120 ++++++-- tests/test_normalise_handle_gate.py | 291 +++++++++++++----- 4 files changed, 317 insertions(+), 108 deletions(-) create mode 100644 changelog.d/tsk-35ydf3-generalise-duplicate-gate.md diff --git a/.github/workflows/normalise-handle-gate.yml b/.github/workflows/normalise-handle-gate.yml index 21103a33..98f4a991 100644 --- a/.github/workflows/normalise-handle-gate.yml +++ b/.github/workflows/normalise-handle-gate.yml @@ -1,9 +1,11 @@ name: Normalise handle gate -# Fails if any module under ``taosmd/`` contains more than one definition -# of the internal helper ``_normalise_handle``. Overload stubs decorated -# with ``@typing.overload`` or ``@overload`` are excluded. Files that -# cannot be decoded as UTF-8 are skipped with a warning. +# Fails if any top-level function or method name is defined more than once +# in the same module under ``taosmd/`` or ``tests/`` (e.g. shadowed test +# functions). ``@typing.overload`` stubs and ``@property`` getter / +# ``@.setter`` / ``@.getter`` / ``@.deleter`` accessor +# pairs are excluded. Files that cannot be decoded as UTF-8 are skipped +# with a warning. # # See scripts/normalise_handle_gate.py for the implementation. @@ -23,5 +25,5 @@ jobs: run: uv python install 3.12 - name: Install deps run: uv sync --python 3.12 - - name: Check for duplicate _normalise_handle definitions + - name: Check for duplicate definitions in taosmd/ and tests/ run: python scripts/normalise_handle_gate.py diff --git a/changelog.d/tsk-35ydf3-generalise-duplicate-gate.md b/changelog.d/tsk-35ydf3-generalise-duplicate-gate.md new file mode 100644 index 00000000..d0a3e006 --- /dev/null +++ b/changelog.d/tsk-35ydf3-generalise-duplicate-gate.md @@ -0,0 +1,2 @@ +### Fixed +- Generalised the `normalise-handle-gate` CI check from the single name `_normalise_handle` to any top-level function or method name defined more than once in the same module, so shadowed test functions (and any other accidental redefinition) are reported. The scan now also covers `tests/`. `@typing.overload` stubs, `@property` getters, and `@.setter` / `@.getter` / `@.deleter` accessor pairs are still excluded as legitimate same-name definitions. diff --git a/scripts/normalise_handle_gate.py b/scripts/normalise_handle_gate.py index 143197bf..823c2c91 100644 --- a/scripts/normalise_handle_gate.py +++ b/scripts/normalise_handle_gate.py @@ -1,21 +1,36 @@ #!/usr/bin/env python3 -"""Normalise handle gate. - -Scans ``taosmd/`` for duplicate definitions of the internal helper -``_normalise_handle`` and fails if any module contains more than one. -``@typing.overload`` stubs are excluded: they are not rival copies. - -Files that cannot be decoded as UTF-8 are skipped with a warning rather -than allowed to crash the gate. +"""Duplicate-definition gate. + +Scans ``taosmd/`` and ``tests/`` for any top-level function or method name +defined more than once in the same module (scope) and fails if rival +definitions are found. The original gate only looked for the internal helper +``_normalise_handle``; this one examines every name so that shadowed test +functions -- and any other accidental redefinition -- are reported. + +``@typing.overload`` stubs, ``@property`` getters, and ``@.setter`` / +``@.getter`` / ``@.deleter`` accessories are legitimate same-name +pairs and are therefore excluded. Files that cannot be decoded as UTF-8 are +skipped with a warning rather than allowed to crash the gate. """ from __future__ import annotations import ast import sys +from collections import defaultdict +from dataclasses import dataclass, field from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent +TARGET_PATTERNS = ("taosmd/**/*.py", "tests/**/*.py") + + +@dataclass +class Duplicate: + name: str + scope: str + lines: list[int] = field(default_factory=list) + def _has_overload_decorator(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: for decorator in node.decorator_list: @@ -34,7 +49,48 @@ def _has_overload_decorator(node: ast.FunctionDef | ast.AsyncFunctionDef) -> boo return False -def _count_definitions(file_path: Path) -> int: +def _has_property_decorator(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + """``@property`` getters and ``@.setter`` / ``.getter`` / + ``.deleter`` accessories always share their name with the companion + accessor, so they are never rival definitions.""" + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name) and decorator.id == "property": + return True + if isinstance(decorator, ast.Attribute) and decorator.attr in ( + "setter", + "getter", + "deleter", + ): + return True + return False + + +def _is_exempt(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + return _has_overload_decorator(node) or _has_property_decorator(node) + + +def _collect_definitions( + body: list[ast.stmt], class_path: tuple[str, ...] = () +) -> list[tuple[str, str, int]]: + """Yield ``(scope, name, lineno)`` for every non-exempt function/method. + + ``scope`` is ``"module"`` for a top-level function, ``"class Foo.Bar"`` + for a method of ``Foo.Bar``. Nested functions (closures) are + intentionally NOT collected: they are neither top-level functions nor + methods, and same-name closures in different parents are legal. + """ + scope = "module" if not class_path else "class " + ".".join(class_path) + found: list[tuple[str, str, int]] = [] + for node in body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if not _is_exempt(node): + found.append((scope, node.name, node.lineno)) + elif isinstance(node, ast.ClassDef): + found.extend(_collect_definitions(node.body, class_path + (node.name,))) + return found + + +def _duplicate_definitions(file_path: Path) -> list[Duplicate]: try: source = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): @@ -42,34 +98,46 @@ def _count_definitions(file_path: Path) -> int: f"normalise-handle-gate: skipping {file_path}: unreadable", file=sys.stderr, ) - return 0 + return [] try: tree = ast.parse(source) except SyntaxError: - return 0 + return [] - count = 0 - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if node.name == "_normalise_handle" and not _has_overload_decorator(node): - count += 1 - return count + defs: dict[tuple[str, str], list[int]] = defaultdict(list) + for scope, name, lineno in _collect_definitions(tree.body): + defs[(scope, name)].append(lineno) + + duplicates: list[Duplicate] = [ + Duplicate(name=name, scope=scope, lines=lines) + for (scope, name), lines in defs.items() + if len(lines) > 1 + ] + duplicates.sort(key=lambda d: d.lines[0]) + return duplicates + + +def _iter_target_files() -> list[Path]: + files: list[Path] = [] + for pattern in TARGET_PATTERNS: + files.extend(REPO_ROOT.glob(pattern)) + return sorted(set(files)) def main(argv: list[str] | None = None) -> int: - failures: list[tuple[str, int]] = [] - for file_path in sorted(REPO_ROOT.glob("taosmd/**/*.py")): - count = _count_definitions(file_path) - if count > 1: - rel = file_path.relative_to(REPO_ROOT) - failures.append((str(rel), count)) + failures: list[tuple[str, Duplicate]] = [] + for file_path in _iter_target_files(): + for dup in _duplicate_definitions(file_path): + failures.append((str(file_path.relative_to(REPO_ROOT)), dup)) if failures: - print("NORMALISE_HANDLE GATE FAIL:") - for path, count in failures: + print("DUPLICATE-DEFINITION GATE FAIL:") + for path, dup in failures: + count = len(dup.lines) print( - f" {path}: found {count} definitions; at most 1 is allowed" + f" {path}: {dup.scope} '{dup.name}' defined on lines " + f"{dup.lines}; found {count} definitions; at most 1 is allowed" ) return 1 diff --git a/tests/test_normalise_handle_gate.py b/tests/test_normalise_handle_gate.py index 8d67b343..bb207337 100644 --- a/tests/test_normalise_handle_gate.py +++ b/tests/test_normalise_handle_gate.py @@ -1,28 +1,33 @@ """Tests for scripts/normalise_handle_gate.py. -Proves both directions: -- A single _normalise_handle definition lets the gate pass. -- Two rival definitions cause the gate to fail with an explicit count. -- @typing.overload stubs are excluded from the count. -- try/except ImportError fallback must leave the gate silent. -- Files that cannot be decoded as UTF-8 are skipped with a warning. +Proves the duplicate-definition gate (the successor to the narrow +``_normalise_handle``-only check): + +- It fires for ANY name defined more than once in the same scope, not just + ``_normalise_handle`` -- including shadowed test functions. +- It is per-scope: the same name in two different classes is fine, as is the + same name as a top-level function and a method. +- ``@typing.overload`` stubs, ``@property`` getters, and + ``@.setter`` / ``@.getter`` / ``@.deleter`` accessories + are legitimate same-name pairs and do not fire. +- Nested functions (closures) are not collected: they are neither top-level + functions nor methods. +- Files that cannot be decoded as UTF-8 are skipped with a warning, and + files that fail to parse are left silent. """ from __future__ import annotations -import os -import sys +import ast from pathlib import Path -import pytest - import scripts.normalise_handle_gate as nhg from scripts.normalise_handle_gate import ( - _count_definitions, + _duplicate_definitions, _has_overload_decorator, + _has_property_decorator, main, ) - # ---------------------------------------------------------------------- # unit tests for pure helpers # ---------------------------------------------------------------------- @@ -30,63 +35,181 @@ class TestHasOverloadDecorator: def test_name_overload(self): src = "@overload\ndef _normalise_handle(handle: str) -> str: ...\n" - tree = __import__('ast').parse(src) - funcs = [n for n in __import__('ast').walk(tree) if isinstance(n, __import__('ast').FunctionDef)] + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] assert _has_overload_decorator(funcs[0]) is True def test_typing_overload(self): src = "import typing\n@typing.overload\ndef _normalise_handle(handle: str) -> str: ...\n" - tree = __import__('ast').parse(src) - funcs = [n for n in __import__('ast').walk(tree) if isinstance(n, __import__('ast').FunctionDef)] + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] assert _has_overload_decorator(funcs[0]) is True def test_no_overload(self): src = "def _normalise_handle(handle: str) -> str:\n return handle\n" - tree = __import__('ast').parse(src) - funcs = [n for n in __import__('ast').walk(tree) if isinstance(n, __import__('ast').FunctionDef)] + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] assert _has_overload_decorator(funcs[0]) is False -class TestCountDefinitions: - def test_single_def_returns_one(self, tmp_path): +class TestHasPropertyDecorator: + def test_property_getter_is_exempt(self): + src = "class C:\n @property\n def x(self):\n return 1\n" + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] + assert _has_property_decorator(funcs[0]) is True + + def test_setter_is_exempt(self): + src = "class C:\n @x.setter\n def x(self, v):\n pass\n" + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] + assert _has_property_decorator(funcs[0]) is True + + def test_getter_is_exempt(self): + src = "class C:\n @x.getter\n def x(self):\n pass\n" + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] + assert _has_property_decorator(funcs[0]) is True + + def test_deleter_is_exempt(self): + src = "class C:\n @x.deleter\n def x(self):\n pass\n" + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] + assert _has_property_decorator(funcs[0]) is True + + def test_plain_function_is_not_exempt(self): + src = "def x():\n pass\n" + tree = ast.parse(src) + funcs = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)] + assert _has_property_decorator(funcs[0]) is False + + +class TestDuplicateDefinitions: + def test_single_def_has_no_duplicates(self, tmp_path): f = tmp_path / "mod.py" - f.write_text("def _normalise_handle(handle):\n pass\n") - assert _count_definitions(f) == 1 + f.write_text("def foo():\n pass\n") + assert _duplicate_definitions(f) == [] - def test_no_def_returns_zero(self, tmp_path): + def test_no_def_has_no_duplicates(self, tmp_path): f = tmp_path / "mod.py" - f.write_text("def other():\n pass\n") - assert _count_definitions(f) == 0 + f.write_text("CONST = 1\nclass C:\n pass\n") + assert _duplicate_definitions(f) == [] - def test_two_defs_return_two(self, tmp_path): + def test_two_defs_of_any_name_reported(self, tmp_path): f = tmp_path / "mod.py" f.write_text( - "def _normalise_handle(a):\n pass\n" - "def _normalise_handle(b):\n pass\n" + "def some_other_name(a):\n pass\n" + "def some_other_name(b):\n pass\n" ) - assert _count_definitions(f) == 2 + dups = _duplicate_definitions(f) + assert len(dups) == 1 + assert dups[0].name == "some_other_name" + assert dups[0].scope == "module" + assert dups[0].lines == [1, 3] - def test_overload_stubs_ignored(self, tmp_path): + def test_generalises_beyond_normalise_handle(self, tmp_path): + """The whole point of the gate: not keyed on one hardcoded name.""" f = tmp_path / "mod.py" f.write_text( - "import typing\n\n" - "@typing.overload\n" - "def _normalise_handle(handle: str) -> str: ...\n" - "\n" - "def _normalise_handle(handle):\n pass\n" + "def _normalise_handle(a):\n pass\n" + "def handle_request(b):\n pass\n" + "def handle_request(c):\n pass\n" ) - assert _count_definitions(f) == 1 + dups = _duplicate_definitions(f) + assert [d.name for d in dups] == ["handle_request"] - def test_name_overload_ignored(self, tmp_path): + def test_overload_pair_is_not_a_duplicate(self, tmp_path): f = tmp_path / "mod.py" f.write_text( "from typing import overload\n\n" "@overload\n" - "def _normalise_handle(handle: str) -> str: ...\n" - "\n" - "def _normalise_handle(handle):\n pass\n" + "def foo(x: str) -> str: ...\n" + "@overload\n" + "def foo(x: int) -> int: ...\n" + "def foo(x):\n return x\n" + ) + assert _duplicate_definitions(f) == [] + + def test_property_setter_pair_is_not_a_duplicate(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "class C:\n" + " @property\n" + " def x(self):\n" + " return self._x\n" + " @x.setter\n" + " def x(self, v):\n" + " self._x = v\n" + ) + assert _duplicate_definitions(f) == [] + + def test_all_accessor_pairings_are_not_duplicates(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "class C:\n" + " @property\n" + " def x(self):\n" + " return self._x\n" + " @x.getter\n" + " def x(self):\n" + " return self._x\n" + " @x.setter\n" + " def x(self, v):\n" + " self._x = v\n" + " @x.deleter\n" + " def x(self):\n" + " del self._x\n" + ) + assert _duplicate_definitions(f) == [] + + def test_method_shadow_inside_a_class_is_reported(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "class C:\n" + " def bar(self):\n" + " return 1\n" + " def bar(self):\n" + " return 2\n" + ) + dups = _duplicate_definitions(f) + assert len(dups) == 1 + assert dups[0].name == "bar" + assert dups[0].scope == "class C" + + def test_same_method_name_in_different_classes_is_fine(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "class A:\n" + " def bar(self):\n" + " return 1\n" + "class B:\n" + " def bar(self):\n" + " return 2\n" + ) + assert _duplicate_definitions(f) == [] + + def test_top_level_fn_and_method_same_name_is_fine(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "def foo():\n" + " pass\n" + "class C:\n" + " def foo(self):\n" + " pass\n" ) - assert _count_definitions(f) == 1 + assert _duplicate_definitions(f) == [] + + def test_nested_functions_are_not_collected(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text( + "def outer():\n" + " def inner():\n" + " pass\n" + " def inner():\n" + " pass\n" + " return inner\n" + ) + assert _duplicate_definitions(f) == [] def test_try_except_import_error_fallback_is_silent(self, tmp_path): f = tmp_path / "mod.py" @@ -96,21 +219,27 @@ def test_try_except_import_error_fallback_is_silent(self, tmp_path): "except ImportError:\n" " pass\n" "\n" - "def _normalise_handle(handle):\n pass\n" + "def foo():\n" + " pass\n" ) - assert _count_definitions(f) == 1 + assert _duplicate_definitions(f) == [] + + def test_syntax_error_is_silent(self, tmp_path): + f = tmp_path / "mod.py" + f.write_text("def foo(:\n pass\n") + assert _duplicate_definitions(f) == [] def test_non_utf8_file_skipped(self, tmp_path): f = tmp_path / "mod.py" - f.write_bytes(b"def _normalise_handle():\n pass\n\xff\xfe\n") - assert _count_definitions(f) == 0 + f.write_bytes(b"def foo():\n pass\n\xff\xfe\n") + assert _duplicate_definitions(f) == [] def test_unreadable_file_skipped(self, tmp_path): f = tmp_path / "mod.py" - f.write_text("def _normalise_handle():\n pass\n") + f.write_text("def foo():\n pass\n") f.chmod(0o000) try: - assert _count_definitions(f) == 0 + assert _duplicate_definitions(f) == [] finally: f.chmod(0o644) @@ -119,54 +248,62 @@ def test_unreadable_file_skipped(self, tmp_path): # integration tests with a temporary repo # ---------------------------------------------------------------------- -def _init_repo(tmp_path, single_def=True): +def _init_repo(tmp_path, module_rel: str, content: str): repo = tmp_path / "repo" repo.mkdir() - taosmd_dir = repo / "taosmd" - taosmd_dir.mkdir() - (taosmd_dir / "__init__.py").write_text("") - - mod = taosmd_dir / "service.py" - if single_def: - mod.write_text("def _normalise_handle(handle):\n pass\n") - else: - mod.write_text( - "def _normalise_handle(a):\n pass\n" - "def _normalise_handle(b):\n pass\n" - ) + pkg = repo / Path(module_rel).parent + pkg.mkdir(parents=True, exist_ok=True) + (pkg / "__init__.py").write_text("") + (repo / Path(module_rel)).write_text(content) return repo -class TestNormaliseHandleGateIntegration: - def test_single_def_passes(self, tmp_path, monkeypatch): - repo = _init_repo(tmp_path, single_def=True) - monkeypatch.chdir(repo) +class TestDuplicateDefinitionGateIntegration: + def test_clean_when_no_duplicates(self, tmp_path, monkeypatch, capsys): + repo = _init_repo( + tmp_path, "taosmd/service.py", "def foo():\n pass\n" + ) monkeypatch.setattr(nhg, "REPO_ROOT", repo) rc = main([]) assert rc == 0 + captured = capsys.readouterr() + assert "normalise-handle-gate: clean" in captured.out - def test_two_defs_fail_with_count(self, tmp_path, monkeypatch): - repo = _init_repo(tmp_path, single_def=False) - monkeypatch.chdir(repo) + def test_shadowed_test_in_tests_tree_fails(self, tmp_path, monkeypatch, capsys): + repo = _init_repo( + tmp_path, + "tests/test_smoke.py", + "def test_a():\n pass\n\ndef test_a():\n pass\n", + ) monkeypatch.setattr(nhg, "REPO_ROOT", repo) rc = main([]) assert rc == 1 + captured = capsys.readouterr() + assert "DUPLICATE-DEFINITION GATE FAIL" in captured.out + assert "test_a" in captured.out + assert "tests/test_smoke.py" in captured.out + assert "found 2 definitions" in captured.out - def test_main_prints_duplicate_count(self, tmp_path, monkeypatch, capsys): - repo = _init_repo(tmp_path, single_def=False) - monkeypatch.chdir(repo) + def test_rival_defs_in_taosmd_tree_fail(self, tmp_path, monkeypatch, capsys): + repo = _init_repo( + tmp_path, + "taosmd/service.py", + "def _normalise_handle(a):\n pass\n" + "def _normalise_handle(b):\n pass\n", + ) monkeypatch.setattr(nhg, "REPO_ROOT", repo) rc = main([]) assert rc == 1 captured = capsys.readouterr() - assert "NORMALISE_HANDLE GATE FAIL" in captured.out + assert "DUPLICATE-DEFINITION GATE FAIL" in captured.out + assert "_normalise_handle" in captured.out assert "found 2 definitions" in captured.out - def test_main_prints_clean_when_single(self, tmp_path, monkeypatch, capsys): - repo = _init_repo(tmp_path, single_def=True) - monkeypatch.chdir(repo) + def test_clean_output_when_passing(self, tmp_path, monkeypatch, capsys): + repo = _init_repo( + tmp_path, "taosmd/service.py", "def foo():\n pass\n" + ) monkeypatch.setattr(nhg, "REPO_ROOT", repo) - rc = main([]) - assert rc == 0 + main([]) captured = capsys.readouterr() - assert "normalise-handle-gate: clean" in captured.out + assert "fail" not in captured.out.lower()