diff --git a/.github/sync-manifest.yml b/.github/sync-manifest.yml index cbc873009b..f11369f010 100644 --- a/.github/sync-manifest.yml +++ b/.github/sync-manifest.yml @@ -253,7 +253,8 @@ scripts: # Dev dependency sync script - source: scripts/sync_dev_dependencies.py - description: "Syncs dev dependency versions from autofix-versions.env to pyproject.toml" + description: "Syncs every marker-qualified dev-tool pin and safely regenerates transitive uv requirements locks with their recorded scope" + template_sync: exact - source: scripts/check_agents_md_freshness.py description: "Warns when the generated Orchestrator AGENTS.md playbook section cites stale repo paths or commands" template_sync: exact diff --git a/.github/workflows/maint-52-sync-dev-versions.yml b/.github/workflows/maint-52-sync-dev-versions.yml index ca54618811..111a43c801 100644 --- a/.github/workflows/maint-52-sync-dev-versions.yml +++ b/.github/workflows/maint-52-sync-dev-versions.yml @@ -218,22 +218,26 @@ jobs: | tee /tmp/sync_output.txt; then if [ "$uv_lock_stale" = "true" ]; then echo "has_changes=true" >> "$GITHUB_OUTPUT" - if [ "${{ inputs.dry_run }}" != "true" ]; then - uv lock - fi else echo "has_changes=false" >> "$GITHUB_OUTPUT" fi else echo "has_changes=true" >> "$GITHUB_OUTPUT" - # Apply if not dry run - if [ "${{ inputs.dry_run }}" != "true" ]; then - python ../scripts/sync_dev_dependencies.py --apply --pre-commit - if [ -f uv.lock ]; then - uv lock - fi - fi + fi + + # Resolve even when direct pins already match: transitive drift is + # invisible to the direct-pin check and can still break clean CI. + # Dry runs resolve in this disposable checkout too; publication is + # guarded separately. Only the final tree diff means an update exists. + python -m pip install --disable-pip-version-check 'uv>=0.5' + python ../scripts/sync_dev_dependencies.py --apply --pre-commit --resolve-locks 2>&1 \ + | tee /tmp/sync_output.txt + if [ -f uv.lock ]; then uv lock; fi + if ! git diff --quiet; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "has_changes=false" >> "$GITHUB_OUTPUT" fi # Save output for PR body @@ -254,21 +258,21 @@ jobs: echo "Adding dev dependencies section to pyproject.toml..." + python -m pip install --disable-pip-version-check 'uv>=0.5' + # Use --create-if-missing to add dev deps if python ../scripts/sync_dev_dependencies.py \ --apply \ --pre-commit \ + --resolve-locks \ --create-if-missing 2>&1 \ | tee /tmp/sync_output.txt; then - if grep -q "version updates" /tmp/sync_output.txt; then + if [ -f uv.lock ]; then uv lock; fi + if ! git diff --quiet; then echo "has_changes=true" >> "$GITHUB_OUTPUT" else echo "has_changes=false" >> "$GITHUB_OUTPUT" fi - if [ "${{ inputs.dry_run }}" != "true" ] && [ -f uv.lock ]; then - python -m pip install --disable-pip-version-check 'uv>=0.5' - uv lock - fi else exit 1 fi diff --git a/docs/ops/CONSUMER_REPO_MAINTENANCE.md b/docs/ops/CONSUMER_REPO_MAINTENANCE.md index e5ff67159c..6e8c4700dd 100644 --- a/docs/ops/CONSUMER_REPO_MAINTENANCE.md +++ b/docs/ops/CONSUMER_REPO_MAINTENANCE.md @@ -1692,6 +1692,39 @@ direct `requirements.lock` pins must move in the same Workflows PR. Maint 52 also updates managed `.pre-commit-config.yaml` hook revisions and direct tool pins in a consumer's `requirements-dev.lock` when that additional generated lockfile exists. +Direct pin replacement is not lock regeneration. Maint 52 applies +`sync_dev_dependencies.py --apply --pre-commit --resolve-locks` before publishing +a changed dependency tree. For generated uv requirements locks, this replays the +recorded repository-local compile inputs and supported extras/platform options +as an argument vector (never a shell command), permitting upgrades of managed +tools and resolving their changed transitive requirements. Existing unrelated +output pins remain resolver preferences. Unsupported provenance or a solver +failure stops publication with a concrete error; it must not leave a knowingly +unsatisfiable generated delivery marked ready to merge. Manually authored +`requirements-dev.txt` without uv provenance remains a direct-pin surface. +Repository-local baseline `.txt` inputs and uv's plural `--constraints` and +`--overrides` options retain their scope. Hashed requirement continuations still +receive canonical tool constraints; previously recorded managed upgrade options +are replaced by the current canonical version. Escaped input/output destinations +fail closed. A successful unchanged lock is a no-op, not an update receipt. +Implicit group inputs use the local `pyproject.toml`; an explicit group path +must validate its own existing repository-local project, not an unrelated root +file. Before any direct apply write, validate all potential output destinations +against repository containment, including resolved symlink targets. Canonical +pyproject updates keep dependency extras before the version and preserve markers. +Inline dependency arrays recognize quoted extras brackets as requirement content, +not the end of the array, and are tested through the full pyproject update path. +Repeated marker-qualified entries are evaluated together: a canonical last entry +must not hide an earlier stale version or non-exact operator. Update every +matching occurrence while preserving its extras and environment marker; a second +apply must be a no-op. +Both apply branches derive publication readiness from the final tree diff; dry +runs perform the same resolution in the disposable checkout without publishing. +The final apply output, rather than the preliminary direct-pin check, supplies +the preview and PR change summary for transitive-only updates. +The propagation-script digest participates in the wave hash, so this repair +produces a replacement wave even when the canonical tool versions are unchanged. + Consumer repos receive those pins through `maint-52-sync-dev-versions.yml`, not the general `maint-68-sync-consumer-repos.yml` template sync. Keep `.github/workflows/autofix-versions.env` out of `.github/sync-manifest.yml` so a diff --git a/scripts/sync_dev_dependencies.py b/scripts/sync_dev_dependencies.py index 8ad818c647..6ab8d74200 100755 --- a/scripts/sync_dev_dependencies.py +++ b/scripts/sync_dev_dependencies.py @@ -23,6 +23,8 @@ import argparse import re +import shlex +import subprocess import sys from pathlib import Path from urllib.parse import urlparse @@ -116,7 +118,11 @@ def find_dev_dependencies_section(content: str) -> tuple[int, int, str] | None: return match.start(), match.end(), match.group(0) # Try inline format: dev = ["pkg1", "pkg2"] - inline_pattern = re.compile(r"^dev\s*=\s*\[(.*?)\]", re.MULTILINE) + # An extras bracket belongs to its quoted requirement, not the array end. + inline_pattern = re.compile( + r"""^dev\s*=\s*\[(?:[^\]\n"']|"(?:\\.|[^"\\])*"|'[^']*')*\]""", + re.MULTILINE, + ) match = inline_pattern.search(content) if match: return match.start(), match.end(), match.group(0) @@ -180,12 +186,14 @@ def extract_dependencies(section: str) -> list[tuple[str, str, str]]: deps = [] # Match patterns like "package>=1.0.0" or "package==1.0.0" or just "package" # Be precise: package name followed by optional version specifier - pattern = re.compile(r'"([a-zA-Z0-9_-]+)(?:(>=|==|~=|>|<|<=|!=)([^"\[\]]+))?(?:\[.*?\])?"') + pattern = re.compile( + r'"([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:(>=|==|~=|<=|!=|>|<)([^";]+))?(?:;[^\"]*)?"' + ) for match in pattern.finditer(section): package = match.group(1) operator = match.group(2) or "" - version = match.group(3) or "" + version = (match.group(3) or "").strip() deps.append((package, operator, version)) return deps @@ -209,15 +217,16 @@ def update_dependency_in_section( # Match: "package" + optional version spec, NOT followed by more pkg name chars # The negative lookahead (?!-) ensures we don't match "pytest" in "pytest-cov" pattern = re.compile( - rf'"({re.escape(package)})(?![-\w])(>=|==|~=|>|<|<=|!=)?([^"\[\]]*)?(\[.*?\])?"', + rf'"({re.escape(package)})(?![-\w])(\[[^\]]+\])?(?:(>=|==|~=|<=|!=|>|<)([^";]+))?(;[^\"]*)?"', re.IGNORECASE, ) def replacer(m: re.Match) -> str: pkg_name = m.group(1) - extras = m.group(4) or "" + extras = m.group(2) or "" + marker = m.group(5) or "" op = "==" if use_exact_pin else ">=" - return f'"{pkg_name}{op}{new_version}{extras}"' + return f'"{pkg_name}{extras}{op}{new_version}{marker}"' new_section, count = pattern.subn(replacer, section) return new_section, count > 0 @@ -299,7 +308,9 @@ def sync_pyproject( # Extract current dependencies from the section current_deps = extract_dependencies(section) - current_packages = {pkg.lower(): (pkg, op, ver) for pkg, op, ver in current_deps} + current_packages: dict[str, list[tuple[str, str, str]]] = {} + for pkg, op, ver in current_deps: + current_packages.setdefault(pkg.lower(), []).append((pkg, op, ver)) # Work on a copy of just the section new_section = section @@ -315,12 +326,18 @@ def sync_pyproject( for pkg_name in package_names: pkg_lower = pkg_name.lower() if pkg_lower in current_packages: - actual_pkg, current_op, current_ver = current_packages[pkg_lower] + occurrences = current_packages[pkg_lower] + mismatches = [ + item + for item in occurrences + if item[2] != target_version or (use_exact_pins and item[1] != "==") + ] # Normalize both the version and the operator. A dependency that # already has the target version but still uses ">=" is not in # sync with the reproducible, exact-pin contract. - if current_ver != target_version or (use_exact_pins and current_op != "=="): + if mismatches: + actual_pkg, current_op, current_ver = mismatches[0] new_section, changed = update_dependency_in_section( new_section, actual_pkg, target_version, use_exact_pins ) @@ -352,6 +369,128 @@ def _build_lockfile_targets(pins: dict[str, str]) -> dict[str, str]: return targets +def regenerate_lockfile(lockfile_path: Path, pins: dict[str, str]) -> tuple[list[str], list[str]]: + """Resolve a generated lock using its recorded inputs, never shell replay. + + Direct pin rewriting is not sufficient when a tool adds or tightens a + transitive requirement. Preserve the consumer's extras/platform choices + and existing output preferences, permitting managed tools to upgrade. + Unsupported provenance fails closed instead of guessing a lock scope. + """ + if not lockfile_path.exists(): + return [], [] + content = lockfile_path.read_text(encoding="utf-8") + command = next( + ( + line[1:].strip() + for line in content.splitlines() + if line.startswith("#") and line[1:].strip().startswith("uv pip compile ") + ), + "", + ) + # requirements-dev.txt may be a manually authored direct requirement list. + if not command and lockfile_path.suffix == ".txt": + return [], [] + try: + tokens = shlex.split(command) + if tokens[:3] != ["uv", "pip", "compile"]: + raise ValueError("missing supported uv compile provenance") + value_flags = { + "--extra", + "--group", + "--python-version", + "--python-platform", + "--no-emit-package", + "--constraints", + "-c", + "--overrides", + "--resolution", + "--exclude-newer", + "--output-file", + "-o", + "--upgrade-package", + "-P", + } + bool_flags = { + "--universal", + "--all-extras", + "--generate-hashes", + "--no-strip-extras", + "--no-strip-markers", + "--no-annotate", + "--emit-index-url", + "--emit-find-links", + } + output = None + sources = [] + canonical = _build_lockfile_targets(pins) + remove_indices = set() + upgrade_names = set() + i = 3 + while i < len(tokens): + start = i + token = tokens[i] + flag, separator, value = token.partition("=") + if flag in value_flags: + if not separator: + i += 1 + if i >= len(tokens): + raise ValueError("missing compile option value") + value = tokens[i] + if not value or value.startswith("-"): + raise ValueError("invalid compile option value") + if flag in {"--output-file", "-o"}: + if output is not None: + raise ValueError("duplicate output destination") + output = value + if flag in {"--constraints", "-c", "--overrides"}: + sources.append(value) + if flag == "--group": + group_path, qualified, group_name = value.rpartition(":") + if qualified and (not group_path or not group_name): + raise ValueError("invalid explicit group input") + sources.append(group_path if qualified else "pyproject.toml") + if flag in {"--upgrade-package", "-P"}: + package = re.match(r"[A-Za-z0-9_.-]+", value) + if package is None: + raise ValueError("invalid package upgrade") + name = package.group().lower() + if name in canonical: + upgrade_names.add(name) + remove_indices.update(range(start, i + 1)) + elif token in bool_flags: + pass + elif token == "pyproject.toml" or token.endswith((".in", ".txt")): + sources.append(token) + else: + raise ValueError("unsupported compile argument") + i += 1 + if output != str(lockfile_path) or not sources: + raise ValueError("compile output or inputs do not match this lock") + root = Path.cwd().resolve() + if lockfile_path.is_absolute() or not lockfile_path.resolve().is_relative_to(root): + raise ValueError("compile output must remain repository-local") + for source in sources: + path = Path(source) + if path.is_absolute() or not path.resolve().is_relative_to(root) or not path.is_file(): + raise ValueError("compile input must be an existing repository-local file") + present = { + match.group("name").lower() + for line in content.splitlines() + if (match := LOCKFILE_PATTERN.match(line.rstrip().removesuffix("\\").rstrip())) + } + tokens = [token for index, token in enumerate(tokens) if index not in remove_indices] + for name, version in canonical.items(): + if name in present or name in upgrade_names: + tokens.extend(["--upgrade-package", f"{name}=={version}"]) + subprocess.run(tokens, check=True) + if lockfile_path.read_text(encoding="utf-8") == content: + return [], [] + except (ValueError, OSError, subprocess.CalledProcessError) as exc: + return [], [f"{lockfile_path}: transitive lock regeneration failed ({exc})"] + return [f"{lockfile_path}: regenerated transitive dependencies"], [] + + def sync_lockfile( lockfile_path: Path, pins: dict[str, str], apply: bool = False ) -> tuple[list[str], list[str]]: @@ -412,8 +551,9 @@ def _pre_commit_repo_name(line: str) -> str | None: if parsed.hostname and parsed.hostname.lower() == "github.com": repo = parsed.path.lstrip("/") repo = repo.rstrip("/").removesuffix(".git") - if repo.lower().startswith("github.com/"): - repo = repo.split("/", 1)[1] + host, separator, path = repo.partition("/") + if separator and host.lower() == "github.com": + repo = path return repo.lower() @@ -500,6 +640,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Apply version updates to pyproject.toml and supported requirements lockfiles", ) + parser.add_argument( + "--resolve-locks", + action="store_true", + help="After applying pins, regenerate uv requirements locks with their recorded scope", + ) parser.add_argument( "--create-if-missing", action="store_true", @@ -540,6 +685,8 @@ def main(argv: list[str] | None = None) -> int: if args.check and args.apply: parser.error("--check and --apply are mutually exclusive") + if args.resolve_locks and not args.apply: + parser.error("--resolve-locks requires --apply") if not args.check and not args.apply: args.check = True # Default to check mode @@ -551,6 +698,21 @@ def main(argv: list[str] | None = None) -> int: print("Error: No pins found in env file", file=sys.stderr) return 2 + # Validate every potential output before the first direct write. A later + # resolver guard cannot undo pyproject/lock edits through an escaped symlink. + if args.apply: + root = Path.cwd().resolve() + outputs = [args.pyproject, *LOCKFILE_FILES] + if args.pre_commit: + outputs.append(PRE_COMMIT_FILE) + for output in outputs: + if not output.resolve().is_relative_to(root): + print( + f"Error: write destination must remain repository-local: {output}", + file=sys.stderr, + ) + return 2 + changes, errors = sync_pyproject( args.pyproject, pins, @@ -571,6 +733,12 @@ def main(argv: list[str] | None = None) -> int: changes.extend(pre_commit_changes) errors.extend(pre_commit_errors) + if args.resolve_locks and not errors: + for lockfile_path in LOCKFILE_FILES: + lock_changes, lock_errors = regenerate_lockfile(lockfile_path, pins) + changes.extend(lock_changes) + errors.extend(lock_errors) + if errors: for err in errors: print(f"Error: {err}", file=sys.stderr) diff --git a/templates/consumer-repo/scripts/sync_dev_dependencies.py b/templates/consumer-repo/scripts/sync_dev_dependencies.py index ca6ea8ec7c..6ab8d74200 100644 --- a/templates/consumer-repo/scripts/sync_dev_dependencies.py +++ b/templates/consumer-repo/scripts/sync_dev_dependencies.py @@ -1,34 +1,48 @@ #!/usr/bin/env python3 -"""Sync dev tool version pins from autofix-versions.env to pyproject.toml. +"""Sync dev tool version pins from autofix-versions.env to dependency surfaces. -This script updates the [project.optional-dependencies] dev section in pyproject.toml +This script updates the [project.optional-dependencies] dev section in pyproject.toml, +supported requirements lockfiles, and (when explicitly requested) managed +.pre-commit-config.yaml hook revisions to use the pinned versions from the central autofix-versions.env file. It handles both exact pins (==) and minimum version pins (>=) in pyproject.toml, converting them to exact pins for reproducibility. +If no dev dependencies section exists, it can create one with --create-if-missing. + Usage: - python sync_dev_dependencies.py --check # Verify versions match - python sync_dev_dependencies.py --apply # Update pyproject.toml - python sync_dev_dependencies.py --apply # Syncs requirements.lock automatically if it exists + python sync_dev_dependencies.py --check # Verify versions match + python sync_dev_dependencies.py --apply # Update pyproject.toml + python sync_dev_dependencies.py --apply --create-if-missing # Create dev deps if missing + python sync_dev_dependencies.py --apply # Syncs supported requirements lockfiles when present + python sync_dev_dependencies.py --apply --pre-commit # Include managed hook revisions """ from __future__ import annotations import argparse import re +import shlex +import subprocess import sys from pathlib import Path +from urllib.parse import urlparse # Default paths (can be overridden for testing) PIN_FILE = Path(".github/workflows/autofix-versions.env") PYPROJECT_FILE = Path("pyproject.toml") -LOCKFILE_FILE = Path("requirements.lock") +# Direct requirement files can be installed by CI independently of pyproject. +# Keep every supported dev-tool surface aligned with the canonical pin file. +LOCKFILE_FILES = ( + Path("requirements.lock"), + Path("requirements-dev.lock"), + Path("requirements-dev.txt"), +) +PRE_COMMIT_FILE = Path(".pre-commit-config.yaml") # Map env file keys to package names # Format: ENV_KEY -> (package_name, optional_alternative_names) -# NOTE: Only include dev tools here. Runtime dependencies (hypothesis, pyyaml, -# pydantic, jsonschema) should NOT be synced - they're managed by Dependabot. TOOL_MAPPING: dict[str, tuple[str, ...]] = { "RUFF_VERSION": ("ruff",), "BLACK_VERSION": ("black",), @@ -39,10 +53,33 @@ "PYTEST_XDIST_VERSION": ("pytest-xdist",), "COVERAGE_VERSION": ("coverage",), "DOCFORMATTER_VERSION": ("docformatter",), + "HYPOTHESIS_VERSION": ("hypothesis",), +} + +# Only version pins for tools already governed by autofix-versions.env are managed. +# The rest of a consumer's pre-commit configuration remains consumer-owned. +PRE_COMMIT_REPO_MAPPING = { + "psf/black": "BLACK_VERSION", + "astral-sh/ruff-pre-commit": "RUFF_VERSION", + "charliermarsh/ruff-pre-commit": "RUFF_VERSION", + "pre-commit/mirrors-mypy": "MYPY_VERSION", + "pycqa/isort": "ISORT_VERSION", + "pycqa/docformatter": "DOCFORMATTER_VERSION", } +# Core dev tools to include when creating a new dev section +# (subset of TOOL_MAPPING - only the most essential ones) +CORE_DEV_TOOLS = [ + "RUFF_VERSION", + "MYPY_VERSION", + "PYTEST_VERSION", + "PYTEST_COV_VERSION", +] + LOCKFILE_PATTERN = re.compile( - r"^(?P\s*)(?P[A-Za-z0-9_.-]+)==(?P[^\s#]+)(?P\s*(?:#.*)?)$" + r"^(?P\s*)(?P[A-Za-z0-9_.-]+)(?P\[[^]]+\])?" + r"(?P(?:===|==|!=|<=|>=|~=|<|>)[^\s;#]+)?" + r"(?P\s*;[^#]+?)?(?P\s*(?:#.*)?)$" ) @@ -81,7 +118,11 @@ def find_dev_dependencies_section(content: str) -> tuple[int, int, str] | None: return match.start(), match.end(), match.group(0) # Try inline format: dev = ["pkg1", "pkg2"] - inline_pattern = re.compile(r"^dev\s*=\s*\[(.*?)\]", re.MULTILINE) + # An extras bracket belongs to its quoted requirement, not the array end. + inline_pattern = re.compile( + r"""^dev\s*=\s*\[(?:[^\]\n"']|"(?:\\.|[^"\\])*"|'[^']*')*\]""", + re.MULTILINE, + ) match = inline_pattern.search(content) if match: return match.start(), match.end(), match.group(0) @@ -89,19 +130,64 @@ def find_dev_dependencies_section(content: str) -> tuple[int, int, str] | None: return None +def find_optional_dependencies_section(content: str) -> int | None: + """Find the [project.optional-dependencies] section header. + + Returns the index after the section header, or None if not found. + """ + pattern = re.compile(r"^\[project\.optional-dependencies\]\s*$", re.MULTILINE) + match = pattern.search(content) + if match: + return match.end() + return None + + +def find_project_section_end(content: str) -> int | None: + """Find a good place to insert [project.optional-dependencies]. + + Returns the index after the [project] section ends (before next section). + """ + # Find [project] section + project_match = re.search(r"^\[project\]\s*$", content, re.MULTILINE) + if not project_match: + return None + + # Find the next section header after [project] + next_section = re.search(r"^\[", content[project_match.end() :], re.MULTILINE) + if next_section: + return project_match.end() + next_section.start() + + # No next section, return end of content + return len(content) + + +def create_dev_dependencies_section(pins: dict[str, str], use_exact_pins: bool = True) -> str: + """Create a new dev dependencies section with core tools.""" + op = "==" if use_exact_pins else ">=" + deps = [] + + for env_key in CORE_DEV_TOOLS: + if env_key in pins: + pkg_name = TOOL_MAPPING[env_key][0] + version = pins[env_key] + deps.append(f' "{pkg_name}{op}{version}",') + + if not deps: + return "" + + return "dev = [\n" + "\n".join(deps) + "\n]" + + def extract_dependencies(section: str) -> list[tuple[str, str, str]]: """Extract dependencies from a dev section. Returns list of (package_name, operator, version) tuples. """ deps = [] - # Match patterns like "package>=1.0.0", "package==1.0.0", "package", or "package[extra]==1.0.0" - # Per PEP 508, extras come BEFORE version specifier: package[extra]>=1.0.0 + # Match patterns like "package>=1.0.0" or "package==1.0.0" or just "package" + # Be precise: package name followed by optional version specifier pattern = re.compile( - r'"([a-zA-Z0-9_-]+)' # package name - r"(?:\[[^\]]+\])?" # optional extras, e.g. [standard] - r"(?:(>=|==|~=|>|<|<=|!=)" # optional operator - r'([^"\[\]]+))?"' # optional version + r'"([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:(>=|==|~=|<=|!=|>|<)([^";]+))?(?:;[^\"]*)?"' ) for match in pattern.finditer(section): @@ -113,36 +199,45 @@ def extract_dependencies(section: str) -> list[tuple[str, str, str]]: return deps -def update_dependency_version( - content: str, package: str, new_version: str, use_exact_pin: bool = True +def update_dependency_in_section( + section: str, package: str, new_version: str, use_exact_pin: bool = True ) -> tuple[str, bool]: - """Update a single dependency version in the content. + """Update a single dependency version within a section. + + IMPORTANT: This only updates exact package name matches, not partial matches. + For example, "pytest" will NOT match "pytest-cov" or "pytest-xdist". - Returns (new_content, was_changed). + Returns (new_section, was_changed). """ - # Pattern to match the package with any version specifier - # Per PEP 508: extras come BEFORE version specifier (package[extra]>=1.0.0) + # Pattern to match EXACT package name with any version specifier + # The key is using word boundaries and ensuring we match the exact package + # Pattern: "package" or "package>=version" or "package[extras]>=version" + # We need to be careful not to match "pytest" when looking at "pytest-cov" + + # Match: "package" + optional version spec, NOT followed by more pkg name chars + # The negative lookahead (?!-) ensures we don't match "pytest" in "pytest-cov" pattern = re.compile( - rf'"({re.escape(package)})(\[[^\]]+\])?(?![-\w])(>=|==|~=|>|<|<=|!=)?([^"\[\]]*)?"', + rf'"({re.escape(package)})(?![-\w])(\[[^\]]+\])?(?:(>=|==|~=|<=|!=|>|<)([^";]+))?(;[^\"]*)?"', re.IGNORECASE, ) def replacer(m: re.Match) -> str: pkg_name = m.group(1) extras = m.group(2) or "" + marker = m.group(5) or "" op = "==" if use_exact_pin else ">=" - # PEP 508: extras come BEFORE version specifier - return f'"{pkg_name}{extras}{op}{new_version}"' + return f'"{pkg_name}{extras}{op}{new_version}{marker}"' - new_content, count = pattern.subn(replacer, content) - return new_content, count > 0 + new_section, count = pattern.subn(replacer, section) + return new_section, count > 0 -def sync_versions( +def sync_pyproject( pyproject_path: Path, pins: dict[str, str], apply: bool = False, use_exact_pins: bool = True, + create_if_missing: bool = False, ) -> tuple[list[str], list[str]]: """Sync versions from pin file to pyproject.toml. @@ -160,13 +255,65 @@ def sync_versions( # Find dev section section_info = find_dev_dependencies_section(content) + if not section_info: - return [], ["No dev dependencies section found in pyproject.toml"] + if not create_if_missing: + return [], ["No dev dependencies section found in pyproject.toml"] + + # Create new dev dependencies section + new_section = create_dev_dependencies_section(pins, use_exact_pins) + if not new_section: + return [], ["Could not create dev dependencies section - no pins available"] + + # Find where to insert + opt_deps_pos = find_optional_dependencies_section(content) + if opt_deps_pos is not None: + # Add after [project.optional-dependencies] header + content = content[:opt_deps_pos] + "\n" + new_section + "\n" + content[opt_deps_pos:] + else: + # Need to add [project.optional-dependencies] section + insert_pos = find_project_section_end(content) + if insert_pos is None: + # A tool-only pyproject does not declare an installable project. + # Creating project.optional-dependencies would turn it into a + # package contract and can break consumers' editable CI install. + legacy_package_file = any( + (pyproject_path.parent / name).exists() for name in ("setup.py", "setup.cfg") + ) + if not legacy_package_file and not re.search( + r"^\[(?:build-system|tool\.poetry)(?:\]|\.)", + content, + re.MULTILINE, + ): + print("Skipping package dev section in tool-only pyproject.toml") + return [], [] + return [], ["Could not find [project] section to add optional-dependencies"] + + section_to_add = "\n[project.optional-dependencies]\n" + new_section + "\n" + content = content[:insert_pos] + section_to_add + content[insert_pos:] + + op = "==" if use_exact_pins else ">=" + for env_key in CORE_DEV_TOOLS: + if env_key in pins: + pkg_name = TOOL_MAPPING[env_key][0] + changes.append(f"{pkg_name}: (new) -> {op}{pins[env_key]}") + + if apply: + pyproject_path.write_text(content, encoding="utf-8") + + return changes, errors - # Extract current dependencies - _, _, section = section_info + # Extract the section boundaries and content + section_start, section_end, section = section_info + + # Extract current dependencies from the section current_deps = extract_dependencies(section) - current_packages = {pkg.lower(): (pkg, op, ver) for pkg, op, ver in current_deps} + current_packages: dict[str, list[tuple[str, str, str]]] = {} + for pkg, op, ver in current_deps: + current_packages.setdefault(pkg.lower(), []).append((pkg, op, ver)) + + # Work on a copy of just the section + new_section = section # Check each pinned tool for env_key, package_names in TOOL_MAPPING.items(): @@ -179,12 +326,20 @@ def sync_versions( for pkg_name in package_names: pkg_lower = pkg_name.lower() if pkg_lower in current_packages: - actual_pkg, current_op, current_ver = current_packages[pkg_lower] - - # Check if version differs - if current_ver != target_version: - content, changed = update_dependency_version( - content, actual_pkg, target_version, use_exact_pins + occurrences = current_packages[pkg_lower] + mismatches = [ + item + for item in occurrences + if item[2] != target_version or (use_exact_pins and item[1] != "==") + ] + + # Normalize both the version and the operator. A dependency that + # already has the target version but still uses ">=" is not in + # sync with the reproducible, exact-pin contract. + if mismatches: + actual_pkg, current_op, current_ver = mismatches[0] + new_section, changed = update_dependency_in_section( + new_section, actual_pkg, target_version, use_exact_pins ) if changed: op = "==" if use_exact_pins else ">=" @@ -193,6 +348,10 @@ def sync_versions( ) break + # Replace the section in the full content + if new_section != section: + content = content[:section_start] + new_section + content[section_end:] + # Apply changes if requested if apply and content != original_content: pyproject_path.write_text(content, encoding="utf-8") @@ -210,9 +369,132 @@ def _build_lockfile_targets(pins: dict[str, str]) -> dict[str, str]: return targets +def regenerate_lockfile(lockfile_path: Path, pins: dict[str, str]) -> tuple[list[str], list[str]]: + """Resolve a generated lock using its recorded inputs, never shell replay. + + Direct pin rewriting is not sufficient when a tool adds or tightens a + transitive requirement. Preserve the consumer's extras/platform choices + and existing output preferences, permitting managed tools to upgrade. + Unsupported provenance fails closed instead of guessing a lock scope. + """ + if not lockfile_path.exists(): + return [], [] + content = lockfile_path.read_text(encoding="utf-8") + command = next( + ( + line[1:].strip() + for line in content.splitlines() + if line.startswith("#") and line[1:].strip().startswith("uv pip compile ") + ), + "", + ) + # requirements-dev.txt may be a manually authored direct requirement list. + if not command and lockfile_path.suffix == ".txt": + return [], [] + try: + tokens = shlex.split(command) + if tokens[:3] != ["uv", "pip", "compile"]: + raise ValueError("missing supported uv compile provenance") + value_flags = { + "--extra", + "--group", + "--python-version", + "--python-platform", + "--no-emit-package", + "--constraints", + "-c", + "--overrides", + "--resolution", + "--exclude-newer", + "--output-file", + "-o", + "--upgrade-package", + "-P", + } + bool_flags = { + "--universal", + "--all-extras", + "--generate-hashes", + "--no-strip-extras", + "--no-strip-markers", + "--no-annotate", + "--emit-index-url", + "--emit-find-links", + } + output = None + sources = [] + canonical = _build_lockfile_targets(pins) + remove_indices = set() + upgrade_names = set() + i = 3 + while i < len(tokens): + start = i + token = tokens[i] + flag, separator, value = token.partition("=") + if flag in value_flags: + if not separator: + i += 1 + if i >= len(tokens): + raise ValueError("missing compile option value") + value = tokens[i] + if not value or value.startswith("-"): + raise ValueError("invalid compile option value") + if flag in {"--output-file", "-o"}: + if output is not None: + raise ValueError("duplicate output destination") + output = value + if flag in {"--constraints", "-c", "--overrides"}: + sources.append(value) + if flag == "--group": + group_path, qualified, group_name = value.rpartition(":") + if qualified and (not group_path or not group_name): + raise ValueError("invalid explicit group input") + sources.append(group_path if qualified else "pyproject.toml") + if flag in {"--upgrade-package", "-P"}: + package = re.match(r"[A-Za-z0-9_.-]+", value) + if package is None: + raise ValueError("invalid package upgrade") + name = package.group().lower() + if name in canonical: + upgrade_names.add(name) + remove_indices.update(range(start, i + 1)) + elif token in bool_flags: + pass + elif token == "pyproject.toml" or token.endswith((".in", ".txt")): + sources.append(token) + else: + raise ValueError("unsupported compile argument") + i += 1 + if output != str(lockfile_path) or not sources: + raise ValueError("compile output or inputs do not match this lock") + root = Path.cwd().resolve() + if lockfile_path.is_absolute() or not lockfile_path.resolve().is_relative_to(root): + raise ValueError("compile output must remain repository-local") + for source in sources: + path = Path(source) + if path.is_absolute() or not path.resolve().is_relative_to(root) or not path.is_file(): + raise ValueError("compile input must be an existing repository-local file") + present = { + match.group("name").lower() + for line in content.splitlines() + if (match := LOCKFILE_PATTERN.match(line.rstrip().removesuffix("\\").rstrip())) + } + tokens = [token for index, token in enumerate(tokens) if index not in remove_indices] + for name, version in canonical.items(): + if name in present or name in upgrade_names: + tokens.extend(["--upgrade-package", f"{name}=={version}"]) + subprocess.run(tokens, check=True) + if lockfile_path.read_text(encoding="utf-8") == content: + return [], [] + except (ValueError, OSError, subprocess.CalledProcessError) as exc: + return [], [f"{lockfile_path}: transitive lock regeneration failed ({exc})"] + return [f"{lockfile_path}: regenerated transitive dependencies"], [] + + def sync_lockfile( lockfile_path: Path, pins: dict[str, str], apply: bool = False ) -> tuple[list[str], list[str]]: + """Sync direct tool pins in one supported requirements lockfile.""" if not lockfile_path.exists(): return [], [] @@ -229,13 +511,18 @@ def sync_lockfile( continue name = match.group("name") - version = match.group("version") target_version = targets.get(name.lower()) - if target_version and version != target_version: - changes.append(f"requirements.lock:{name}: {version} -> =={target_version}") + current_requirement = f"{match.group('specifier') or ''}{match.group('marker') or ''}" + target_requirement = f"=={target_version}{match.group('marker') or ''}" + if target_version and current_requirement != target_requirement: + current_version = match.group("specifier") or "(unversioned)" + if current_version.startswith("=="): + current_version = current_version[2:] + changes.append(f"{lockfile_path.name}:{name}: {current_version} -> =={target_version}") if apply: updated_lines.append( - f"{match.group('lead')}{name}=={target_version}{match.group('trail')}" + f"{match.group('lead')}{name}{match.group('extras') or ''}" + f"=={target_version}{match.group('marker') or ''}{match.group('trail')}" ) else: updated_lines.append(line) @@ -252,6 +539,93 @@ def sync_lockfile( return changes, [] +def _pre_commit_repo_name(line: str) -> str | None: + """Return the normalized repository name from a pre-commit ``repo:`` line.""" + match = re.match(r"^\s*-\s*repo:\s*(?P[^\s#]+)", line) + if not match: + return None + + repo = match.group("repo").strip().strip("\"'") + if "://" in repo: + parsed = urlparse(repo) + if parsed.hostname and parsed.hostname.lower() == "github.com": + repo = parsed.path.lstrip("/") + repo = repo.rstrip("/").removesuffix(".git") + host, separator, path = repo.partition("/") + if separator and host.lower() == "github.com": + repo = path + return repo.lower() + + +def sync_pre_commit_config( + pre_commit_path: Path, pins: dict[str, str], apply: bool = False +) -> tuple[list[str], list[str]]: + """Sync managed pre-commit hook revisions while preserving all other text. + + Pre-commit configuration is intentionally not copied wholesale to consumers. + This only changes a recognized remote hook's ``rev:`` value and preserves that + hook's existing ``v`` prefix convention. + """ + if not pre_commit_path.exists(): + return [], [] + + with pre_commit_path.open(encoding="utf-8", newline="") as pre_commit_file: + content = pre_commit_file.read() + lines = content.splitlines(keepends=True) + changes: list[str] = [] + current_env_key: str | None = None + current_repo_name: str | None = None + + for index, line in enumerate(lines): + repo_name = _pre_commit_repo_name(line) + if repo_name is not None: + current_env_key = PRE_COMMIT_REPO_MAPPING.get(repo_name) + current_repo_name = repo_name if current_env_key is not None else None + continue + + if current_env_key is None or current_env_key not in pins: + continue + + line_ending = "\r\n" if line.endswith("\r\n") else "\n" if line.endswith("\n") else "" + revision_line = line[: -len(line_ending)] if line_ending else line + rev_match = re.match( + r"^(?P\s*rev:\s*)(?P['\"]?)(?P[^\s#'\"]+)" + r"(?P=quote)(?P.*)$", + revision_line, + ) + if not rev_match: + continue + + current_value = rev_match.group("value") + target_value = pins[current_env_key] + if current_value.startswith("v"): + target_value = f"v{target_value}" + if current_value == target_value: + current_env_key = None + current_repo_name = None + continue + + changes.append( + f"{pre_commit_path.name}:{current_repo_name}: {current_value} -> {target_value}" + ) + if apply: + lines[index] = ( + f"{rev_match.group('prefix')}{rev_match.group('quote')}{target_value}" + f"{rev_match.group('quote')}{rev_match.group('suffix')}" + f"{line_ending}" + ) + current_env_key = None + current_repo_name = None + + if apply: + updated = "".join(lines) + if updated != content: + with pre_commit_path.open("w", encoding="utf-8", newline="") as pre_commit_file: + pre_commit_file.write(updated) + + return changes, [] + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Sync dev dependency versions from autofix-versions.env to pyproject.toml" @@ -264,71 +638,126 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--apply", action="store_true", - help="Apply version updates to pyproject.toml", + help="Apply version updates to pyproject.toml and supported requirements lockfiles", + ) + parser.add_argument( + "--resolve-locks", + action="store_true", + help="After applying pins, regenerate uv requirements locks with their recorded scope", + ) + parser.add_argument( + "--create-if-missing", + action="store_true", + help="Create dev dependencies section if it doesn't exist", + ) + parser.add_argument( + "--use-minimum-pins", + action="store_true", + help="Use >= instead of == for version pins", ) parser.add_argument( "--lockfile", action="store_true", - help="Force lockfile sync even if requirements.lock doesn't exist (no-op)", + help="Compatibility flag; supported requirements lockfiles are always checked", ) parser.add_argument( - "--lockfile-path", - type=Path, - default=LOCKFILE_FILE, - help="Path to lockfile (default: requirements.lock)", + "--pre-commit", + action="store_true", + help=( + "Include managed .pre-commit-config.yaml hook revisions; Maint 52 opts in " + "after the canonical dependency wave is ready" + ), ) parser.add_argument( "--pin-file", type=Path, default=PIN_FILE, - help=f"Path to pin file (default: {PIN_FILE})", + help="Path to the version pins file", ) parser.add_argument( "--pyproject", type=Path, default=PYPROJECT_FILE, - help=f"Path to pyproject.toml (default: {PYPROJECT_FILE})", + help="Path to pyproject.toml", ) args = parser.parse_args(argv) + if args.check and args.apply: + parser.error("--check and --apply are mutually exclusive") + if args.resolve_locks and not args.apply: + parser.error("--resolve-locks requires --apply") + if not args.check and not args.apply: - parser.error("Must specify either --check or --apply") + args.check = True # Default to check mode + + use_exact_pins = not args.use_minimum_pins pins = parse_env_file(args.pin_file) if not pins: print("Error: No pins found in env file", file=sys.stderr) - return 1 + return 2 + + # Validate every potential output before the first direct write. A later + # resolver guard cannot undo pyproject/lock edits through an escaped symlink. + if args.apply: + root = Path.cwd().resolve() + outputs = [args.pyproject, *LOCKFILE_FILES] + if args.pre_commit: + outputs.append(PRE_COMMIT_FILE) + for output in outputs: + if not output.resolve().is_relative_to(root): + print( + f"Error: write destination must remain repository-local: {output}", + file=sys.stderr, + ) + return 2 - changes, errors = sync_versions( + changes, errors = sync_pyproject( args.pyproject, pins, apply=args.apply, + use_exact_pins=use_exact_pins, + create_if_missing=args.create_if_missing, ) - lockfile_path = args.lockfile_path - lockfile_enabled = args.lockfile or lockfile_path.exists() - if lockfile_enabled: + for lockfile_path in LOCKFILE_FILES: lock_changes, lock_errors = sync_lockfile(lockfile_path, pins, apply=args.apply) changes.extend(lock_changes) errors.extend(lock_errors) + if args.pre_commit: + pre_commit_changes, pre_commit_errors = sync_pre_commit_config( + PRE_COMMIT_FILE, pins, apply=args.apply + ) + changes.extend(pre_commit_changes) + errors.extend(pre_commit_errors) + + if args.resolve_locks and not errors: + for lockfile_path in LOCKFILE_FILES: + lock_changes, lock_errors = regenerate_lockfile(lockfile_path, pins) + changes.extend(lock_changes) + errors.extend(lock_errors) + if errors: - for error in errors: - print(f"Error: {error}", file=sys.stderr) - return 1 + for err in errors: + print(f"Error: {err}", file=sys.stderr) + return 2 if changes: - action = "Updated" if args.apply else "Would update" - print(f"{action} the following dependencies:") + print(f"{'Applied' if args.apply else 'Found'} {len(changes)} version updates:") for change in changes: - print(f" {change}") + print(f" - {change}") + if args.check: + print("\nRun with --apply to update dependency files") return 1 + else: + print("\nāœ“ Dependency files updated") + return 0 else: - print("All dev dependencies are in sync") - - return 0 + print("āœ“ All dev dependency versions are in sync") + return 0 if __name__ == "__main__": diff --git a/tests/scripts/test_sync_dev_dependencies.py b/tests/scripts/test_sync_dev_dependencies.py index 686cc9cb5c..6724090c0b 100644 --- a/tests/scripts/test_sync_dev_dependencies.py +++ b/tests/scripts/test_sync_dev_dependencies.py @@ -1,11 +1,307 @@ from __future__ import annotations from pathlib import Path +from unittest.mock import Mock import pytest from scripts import sync_dev_dependencies as sdd +@pytest.mark.parametrize("inline", [True, False]) +@pytest.mark.parametrize( + "versions", [("==7.0", "==7.1"), ("==7.1", "==7.0"), (">=7.1", "==7.1"), ("==7.1", "==7.1")] +) +def test_sync_pyproject_updates_every_marker_occurrence(tmp_path, inline, versions): + requirements = [ + f"coverage[toml]{versions[0]}; python_version < '3.11'", + f"coverage[toml]{versions[1]}; python_version >= '3.11'", + ] + separator = ", " if inline else ",\n " + deps = separator.join(f'"{item}"' for item in requirements) + original = "[project.optional-dependencies]\ndev = [" + deps + "]\n" + if not inline: + original = "[project.optional-dependencies]\ndev = [\n " + deps + ",\n]\n" + project = tmp_path / "pyproject.toml" + project.write_text(original) + changes, errors = sdd.sync_pyproject(project, {"COVERAGE_VERSION": "7.1"}, apply=True) + assert not errors + assert bool(changes) == (versions != ("==7.1", "==7.1")) + expected = original.replace(versions[0] + "; python_version <", "==7.1; python_version <") + expected = expected.replace(versions[1] + "; python_version >=", "==7.1; python_version >=") + assert project.read_text() == expected + assert sdd.sync_pyproject(project, {"COVERAGE_VERSION": "7.1"}, apply=True) == ([], []) + + +@pytest.mark.parametrize("inline", [True, False]) +@pytest.mark.parametrize( + "requirement", + ["coverage[toml]==7.0", "coverage[toml]", "coverage[toml]>=7.0; python_version < '3.14'"], +) +def test_sync_pyproject_extras_end_to_end(tmp_path, requirement, inline): + deps = f'dev = ["{requirement}"]\n' if inline else f'dev = [\n "{requirement}",\n]\n' + project = tmp_path / "pyproject.toml" + project.write_text( + '[project]\nname="example"\nversion="1"\n[project.optional-dependencies]\n' + deps + ) + changes, errors = sdd.sync_pyproject(project, {"COVERAGE_VERSION": "7.1"}, apply=True) + assert changes and not errors + assert "coverage[toml]==7.1" in project.read_text() + if ";" in requirement: + assert "; python_version < '3.14'" in project.read_text() + + +@pytest.mark.parametrize( + "requirement", + ["coverage[toml]==7.0", "coverage[toml]", "coverage[toml]>=7.0; python_version < '3.14'"], +) +def test_sync_dependency_extras_before_version_preserves_marker(requirement): + section = f'dev = ["{requirement}"]' + assert sdd.extract_dependencies(section)[0][0] == "coverage" + updated, changed = sdd.update_dependency_in_section(section, "coverage", "7.1") + assert changed + marker = "; python_version < '3.14'" if ";" in requirement else "" + assert updated == f'dev = ["coverage[toml]==7.1{marker}"]' + + +@pytest.mark.parametrize( + "group,valid", + [ + ("nested/pyproject.toml:dev", True), + ("/outside/pyproject.toml:dev", False), + ("../outside/pyproject.toml:dev", False), + ], +) +def test_regenerate_explicit_group_validates_its_actual_input(tmp_path, monkeypatch, group, valid): + monkeypatch.chdir(tmp_path) + Path("nested").mkdir() + Path("nested/pyproject.toml").write_text("[dependency-groups]\ndev=[]\n") + # A root project must neither authorize an escaped group nor be required + # when a legitimate local group supplies its own project. + if not valid: + Path("pyproject.toml").write_text("[dependency-groups]\ndev=[]\n") + lock = Path("requirements.lock") + lock.write_text(f"# uv pip compile --group {group} -o requirements.lock\nmypy==2.4.0\n") + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + _, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert bool(errors) is not valid + assert run.called is valid + + +def test_main_rejects_escaped_write_before_any_direct_pin_mutation(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + env = Path("pins.env") + env.write_text("MYPY_VERSION=2.4.0\n") + project = Path("pyproject.toml") + project.write_text( + '[project]\nname="example"\nversion="1"\n[project.optional-dependencies]\ndev=["mypy==2.3.0"]\n' + ) + external = tmp_path.parent / "external-before-write.lock" + original = "# uv pip compile pyproject.toml -o requirements.lock\nmypy==2.3.0\n" + external.write_text(original) + Path("requirements.lock").symlink_to(external) + before = project.read_text() + assert sdd.main(["--apply", "--resolve-locks", "--pin-file", str(env)]) == 2 + assert external.read_text() == original + assert project.read_text() == before + + +def test_regenerate_lock_retains_original_extras_and_platform(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + lock = Path("requirements.lock") + lock.write_text( + "# This file was autogenerated by uv via the following command:\n" + "# uv pip compile pyproject.toml --extra=app --extra dev --universal " + "--output-file=requirements.lock\nast-serialize==0.6.0\nmypy==2.4.0\n" + ) + run = Mock( + side_effect=lambda *_args, **_kwargs: lock.write_text(lock.read_text() + "# resolved\n") + ) + monkeypatch.setattr(sdd.subprocess, "run", run) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert errors == [] + assert changes == ["requirements.lock: regenerated transitive dependencies"] + args = run.call_args.args[0] + assert args[:4] == ["uv", "pip", "compile", "pyproject.toml"] + assert "--extra=app" in args and args[args.index("--extra") + 1] == "dev" + assert "--universal" in args + assert args[-2:] == ["--upgrade-package", "mypy==2.4.0"] + assert run.call_args.kwargs == {"check": True} + + +@pytest.mark.parametrize( + "command", + [ + "pip-compile pyproject.toml -o requirements.lock", + "uv pip compile pyproject.toml -o ../requirements.lock", + "uv pip compile ../pyproject.toml -o requirements.lock", + "uv pip compile pyproject.toml -o requirements.lock ; touch owned", + "uv pip compile pyproject.toml --unknown-flag -o requirements.lock", + "uv pip compile pyproject.toml --no-header -o requirements.lock", + ], +) +def test_regenerate_lock_rejects_unsupported_provenance(tmp_path, monkeypatch, command): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + lock = Path("requirements.lock") + lock.write_text( + f"# This file was autogenerated by uv via the following command:\n# {command}\nmypy==2.4.0\n" + ) + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert changes == [] and errors + run.assert_not_called() + + +@pytest.mark.parametrize("hashed", [False, True]) +@pytest.mark.parametrize("recorded_upgrade", [False, True]) +def test_regenerate_lock_replays_own_canonical_upgrade( + tmp_path, monkeypatch, hashed, recorded_upgrade +): + monkeypatch.chdir(tmp_path) + Path("requirements.in").write_text("mypy>=2\n") + lock = Path("requirements.lock") + continuation = " \\" if hashed else "" + upgrade = "--upgrade-package mypy==2.3.0 " if recorded_upgrade else "" + lock.write_text( + "# uv pip compile requirements.in " + + upgrade + + "--generate-hashes -o requirements.lock\nmypy==2.3.0" + + continuation + + "\n" + ) + run = Mock( + side_effect=lambda *_args, **_kwargs: lock.write_text(lock.read_text() + "# resolved\n") + ) + monkeypatch.setattr(sdd.subprocess, "run", run) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert changes and not errors + args = run.call_args.args[0] + assert "mypy==2.3.0" not in args + assert args.count("mypy==2.4.0") == 1 + + +def test_regenerate_lock_retains_consumer_baseline_txt(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + baseline = Path("tests/baseline/requirements-baseline.txt") + baseline.parent.mkdir(parents=True) + baseline.write_text("example==1\n") + lock = Path("requirements.lock") + lock.write_text( + "# uv pip compile pyproject.toml tests/baseline/requirements-baseline.txt " + "--extra=dev --universal --output-file=requirements.lock\nmypy==2.4.0\n" + ) + run = Mock( + side_effect=lambda *_args, **_kwargs: lock.write_text(lock.read_text() + "# resolved\n") + ) + monkeypatch.setattr(sdd.subprocess, "run", run) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert changes and not errors + assert str(baseline) in run.call_args.args[0] + + +def test_regenerate_group_only_compile_uses_local_pyproject(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[dependency-groups]\ndev=['mypy>=2']\n") + lock = Path("requirements.lock") + lock.write_text("# uv pip compile --group dev -o requirements.lock\nmypy==2.4.0\n") + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + _, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert not errors + run.assert_called_once() + Path("pyproject.toml").unlink() + run.reset_mock() + _, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert errors + run.assert_not_called() + + +@pytest.mark.parametrize( + "repo,expected", + [ + ("github.com/psf/black", "psf/black"), + ("GITHUB.COM/psf/black", "psf/black"), + ("github.com.evil/psf/black", "github.com.evil/psf/black"), + ("evil/github.com/psf/black", "evil/github.com/psf/black"), + ], +) +def test_precommit_bare_domain_is_exact_segment(repo, expected): + assert sdd._pre_commit_repo_name("- repo: " + repo) == expected + + +def test_regenerate_unchanged_lock_is_noop(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + lock = Path("requirements.lock") + lock.write_text("# uv pip compile pyproject.toml -o requirements.lock\nmypy==2.4.0\n") + monkeypatch.setattr(sdd.subprocess, "run", Mock()) + assert sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) == ([], []) + + +@pytest.mark.parametrize("flag", ["--constraints", "--overrides"]) +def test_regenerate_plural_uv_constraints(tmp_path, monkeypatch, flag): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + Path("constraints.txt").write_text("mypy>=2\n") + lock = Path("requirements.lock") + lock.write_text( + f"# uv pip compile pyproject.toml {flag} constraints.txt -o requirements.lock\nmypy==2.4.0\n" + ) + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + _, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert not errors + run.assert_called_once() + + +def test_regenerate_lock_rejects_external_output_symlink(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + external = tmp_path.parent / "external-lock" + external.write_text("# uv pip compile pyproject.toml -o requirements.lock\nmypy==2.4.0\n") + lock = Path("requirements.lock") + lock.symlink_to(external) + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert not changes and errors + run.assert_not_called() + + +def test_regenerate_lock_fails_closed_on_solver_failure(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("pyproject.toml").write_text("[project]\nname='example'\nversion='1'\n") + lock = Path("requirements.lock") + lock.write_text( + "# This file was autogenerated by uv via the following command:\n# uv pip compile pyproject.toml -o requirements.lock\nmypy==2.4.0\n" + ) + monkeypatch.setattr( + sdd.subprocess, "run", Mock(side_effect=sdd.subprocess.CalledProcessError(1, ["uv"])) + ) + changes, errors = sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) + assert changes == [] and "failed" in errors[0] + + +def test_regenerate_manual_requirement_list_stays_direct_pin_only(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + lock = Path("requirements-dev.txt") + lock.write_text("mypy==2.4.0\n") + run = Mock() + monkeypatch.setattr(sdd.subprocess, "run", run) + assert sdd.regenerate_lockfile(lock, {"MYPY_VERSION": "2.4.0"}) == ([], []) + run.assert_not_called() + + +def test_resolve_locks_requires_apply(): + with pytest.raises(SystemExit) as error: + sdd.main(["--check", "--resolve-locks"]) + assert error.value.code == 2 + + def _write_env_file(path: Path, versions: dict[str, str]) -> None: lines = [f"{key}={value}" for key, value in versions.items()] path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/tests/workflows/test_maint52_sync_dev_versions_pr_body.py b/tests/workflows/test_maint52_sync_dev_versions_pr_body.py index 068d407105..e1e0952f08 100644 --- a/tests/workflows/test_maint52_sync_dev_versions_pr_body.py +++ b/tests/workflows/test_maint52_sync_dev_versions_pr_body.py @@ -3,6 +3,16 @@ WORKFLOW = Path(".github/workflows/maint-52-sync-dev-versions.yml") +def test_transitive_summary_captures_final_apply_not_preliminary_check(): + text = WORKFLOW.read_text() + existing = text[ + text.index("- name: Sync versions") : text.index("- name: Add dev dependencies") + ] + apply = existing[existing.index("--apply --pre-commit --resolve-locks") :] + assert "tee /tmp/sync_output.txt" in apply + assert apply.index("tee /tmp/sync_output.txt") < apply.index("cat /tmp/sync_output.txt") + + def test_dev_version_sync_pr_body_matches_changed_files(): text = WORKFLOW.read_text(encoding="utf-8") pr_scope_lines = [ @@ -45,6 +55,26 @@ def test_maint52_stages_managed_precommit_repairs(): assert "if [ -f .pre-commit-config.yaml ]; then git add .pre-commit-config.yaml; fi" in text +def test_maint52_regenerates_transitive_requirements_before_publishing(): + text = WORKFLOW.read_text(encoding="utf-8") + assert "--apply --pre-commit --resolve-locks" in text + add_section = text[text.index("- name: Add dev dependencies") : text.index("- name: Skip")] + assert "--resolve-locks" in add_section + assert add_section.index("pip install") < add_section.index("--resolve-locks") + + +def test_maint52_previews_resolution_and_uses_actual_tree_changes(): + text = WORKFLOW.read_text(encoding="utf-8") + existing = text[ + text.index("- name: Sync versions") : text.index("- name: Add dev dependencies") + ] + missing = text[text.index("- name: Add dev dependencies") : text.index("- name: Skip")] + assert "inputs.dry_run" not in existing + assert "inputs.dry_run" not in missing + assert "git diff --quiet" in existing and "git diff --quiet" in missing + assert 'grep -q "version updates"' not in missing + + def test_maint52_pr_body_reports_canonical_source_commit_and_never_proposes_upstream(): """AC: Maint 52 reports the settled source commit and never opens an upstream bump.""" text = WORKFLOW.read_text(encoding="utf-8")