diff --git a/.github/scripts/check_vendored_files.py b/.github/scripts/check_vendored_files.py index fc05377f7cab..daddbddc2926 100644 --- a/.github/scripts/check_vendored_files.py +++ b/.github/scripts/check_vendored_files.py @@ -13,7 +13,9 @@ The check mode is idempotent: existing issues are matched by label `area-vendored-sync` plus a hidden HTML marker in the body of the form -``. +``. Issues additionally carry +the area labels resolved for the entry (manifest `default_area_labels`, or the +entry's own `area_labels` override) so they land in the right triage queue. See eng/vendored-files.md for the manifest schema and reconciliation workflow. """ @@ -38,6 +40,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] MANIFEST_PATH = REPO_ROOT / "eng" / "vendored-files.json" ISSUE_LABEL = "area-vendored-sync" +AREA_LABEL_PREFIX = "Area-" ISSUE_REPO = os.environ.get("VENDORED_SYNC_REPO", "dotnet/sdk") MAX_DIFF_LINES = 300 SHA_RE = re.compile(r"^[0-9a-f]{40}$") @@ -72,21 +75,32 @@ class Entry: local_path: str notes: str sources: list[Source] + area_labels: object @classmethod - def from_dict(cls, d: dict[str, Any]) -> "Entry": + def from_dict(cls, d: dict[str, Any], default_area_labels: object) -> "Entry": + area_labels = d.get("area_labels", default_area_labels) return cls( id=d["id"], local_path=d["local_path"], notes=d.get("notes", ""), sources=[Source.from_dict(s) for s in d["sources"]], + area_labels=area_labels.copy() if isinstance(area_labels, list) else area_labels, ) + @property + def issue_labels(self) -> list[str]: + """Labels applied to this entry's drift issues, sync label first.""" + if not isinstance(self.area_labels, list): + raise ValueError(f"{self.id}: area_labels must be a list.") + return [ISSUE_LABEL, *self.area_labels] + def load_manifest() -> list[Entry]: with MANIFEST_PATH.open("r", encoding="utf-8") as f: data = json.load(f) - return [Entry.from_dict(e) for e in data["entries"]] + default_area_labels = data.get("default_area_labels", []) + return [Entry.from_dict(e, default_area_labels) for e in data["entries"]] # ---------- validation ---------- @@ -109,6 +123,13 @@ def validate(entries: list[Entry]) -> int: if not entry.sources: errors.append(f"{entry.id}: must declare at least one source.") + if not isinstance(entry.area_labels, list): + errors.append(f"{entry.id}: area_labels must be a list.") + elif not all(isinstance(label, str) and label.strip() for label in entry.area_labels): + errors.append(f"{entry.id}: area_labels must contain only non-empty strings.") + elif ISSUE_LABEL in entry.area_labels: + errors.append(f"{entry.id}: area_labels must not repeat '{ISSUE_LABEL}'.") + for index, source in enumerate(entry.sources): tag = f"{entry.id}#sources[{index}]" if not re.match(r"^[\w.-]+/[\w.-]+$", source.repo): @@ -384,7 +405,7 @@ def _list_open_sync_issues() -> list[dict[str, Any]]: "--label", ISSUE_LABEL, "--state", "open", "--limit", "200", - "--json", "number,title,body", + "--json", "number,title,body,labels", ]) if rc != 0: print(f"gh issue list failed: {err}", file=sys.stderr) @@ -404,15 +425,53 @@ def find_existing_issue(marker: str) -> dict[str, Any] | None: return None -def ensure_label() -> None: - """Ensure the sync label exists. `gh label create --force` is idempotent.""" - _gh([ - "label", "create", ISSUE_LABEL, +def ensure_labels(entries: list[Entry], create_sync_label: bool = True) -> bool: + """Ensure the labels applied to drift issues exist. + + The sync label is owned by this workflow, so it is force-created when issue + mutation is enabled. Area labels come from the manifest and are owned by the + repo's regular triage taxonomy, so they must already exist. + """ + if create_sync_label: + rc, _, err = _gh([ + "label", "create", ISSUE_LABEL, + "--repo", ISSUE_REPO, + "--description", "Drift detected between a vendored source file and its upstream copy", + "--color", "fbca04", + "--force", + ]) + if rc != 0: + print(f"Failed to ensure label '{ISSUE_LABEL}': {err}", file=sys.stderr) + return False + + rc, out, err = _gh([ + "label", "list", "--repo", ISSUE_REPO, - "--description", "Drift detected between a vendored source file and its upstream copy", - "--color", "fbca04", - "--force", + "--limit", "1000", + "--json", "name", ]) + if rc != 0: + print(f"Failed to list labels: {err}", file=sys.stderr) + return False + try: + existing_labels = {item["name"] for item in json.loads(out)} + except (json.JSONDecodeError, KeyError, TypeError): + print("Failed to parse labels returned by gh.", file=sys.stderr) + return False + + area_labels = { + label + for entry in entries + for label in entry.issue_labels[1:] + } + missing_labels = sorted(area_labels - existing_labels) + if missing_labels: + print( + f"Missing area labels in {ISSUE_REPO}: {', '.join(missing_labels)}", + file=sys.stderr, + ) + return False + return True _STATUS_TITLES = { @@ -422,15 +481,36 @@ def ensure_label() -> None: } +def _label_flags(labels: list[str]) -> list[str]: + flags: list[str] = [] + for label in labels: + flags += ["--label", label] + return flags + + +def _label_changes(issue: dict[str, Any], labels: list[str]) -> tuple[list[str], list[str]]: + present = {(item.get("name") or "") for item in (issue.get("labels") or [])} + desired = set(labels) + add = sorted(desired - present) + remove = sorted( + label + for label in present - desired + if label.startswith(AREA_LABEL_PREFIX) + ) + return add, remove + + def upsert_issue(result: DriftResult, dry_run: bool) -> None: marker = _marker(result.entry.id, result.source_index) body = _render_issue_body(result) status_text = _STATUS_TITLES.get(result.status, result.status) title = f"[vendored-sync] {result.entry.id}: {status_text} (#{result.source_index})" + labels = result.entry.issue_labels if dry_run: print(f"\n--- would create/update issue ({result.entry.id}#{result.source_index}) ---") print(f"title: {title}") + print(f"labels: {', '.join(labels)}") print(body[:2000]) print("---") return @@ -441,7 +521,7 @@ def upsert_issue(result: DriftResult, dry_run: bool) -> None: "issue", "create", "--repo", ISSUE_REPO, "--title", title, - "--label", ISSUE_LABEL, + *_label_flags(labels), "--body-file", "-", ], input_data=body) if rc != 0: @@ -450,8 +530,30 @@ def upsert_issue(result: DriftResult, dry_run: bool) -> None: print(f"Created issue for {result.entry.id}#{result.source_index}: {out.strip()}") return + # Keep the workflow-managed area routing in sync while preserving other labels. + add_labels, remove_labels = _label_changes(existing, labels) + labels_up_to_date = not add_labels and not remove_labels + if not labels_up_to_date: + rc, _, err = _gh([ + "issue", "edit", str(existing["number"]), + "--repo", ISSUE_REPO, + *[arg for label in add_labels for arg in ("--add-label", label)], + *[arg for label in remove_labels for arg in ("--remove-label", label)], + ]) + if rc != 0: + print(f"Failed to reconcile labels on issue #{existing['number']}: {err}", file=sys.stderr) + else: + changes: list[str] = [] + if add_labels: + changes.append(f"added {', '.join(add_labels)}") + if remove_labels: + changes.append(f"removed {', '.join(remove_labels)}") + print(f"Reconciled labels on issue #{existing['number']}: {'; '.join(changes)}") + labels_up_to_date = True + if (existing.get("body") or "").strip() == body.strip(): - print(f"Issue #{existing['number']} already up to date for {result.entry.id}#{result.source_index}.") + if labels_up_to_date: + print(f"Issue #{existing['number']} already up to date for {result.entry.id}#{result.source_index}.") return rc, _, err = _gh([ @@ -480,8 +582,8 @@ def cmd_check(args: argparse.Namespace) -> int: if validate(entries) != 0: return 1 - if not args.dry_run: - ensure_label() + if not ensure_labels(entries, create_sync_label=not args.dry_run): + return 1 drift_count = 0 error_count = 0 diff --git a/.github/workflows/check-vendored-files.yml b/.github/workflows/check-vendored-files.yml index cf1e4cf3b203..dd8e2f3f59f8 100644 --- a/.github/workflows/check-vendored-files.yml +++ b/.github/workflows/check-vendored-files.yml @@ -4,7 +4,8 @@ name: Check vendored source files # sources, as listed in eng/vendored-files.json. See eng/vendored-files.md. # # - Scheduled runs and manual dispatch perform full drift detection and open -# (or update) tracking issues labelled `area-vendored-sync`. +# (or update) tracking issues labelled `area-vendored-sync` plus the area +# labels declared for the entry in eng/vendored-files.json. # - Pull-request runs that touch the manifest, script, or workflow validate the # manifest structure only (no network, no issue mutation). diff --git a/eng/vendored-files.json b/eng/vendored-files.json index 522c8bf50983..ecb46e75e21b 100644 --- a/eng/vendored-files.json +++ b/eng/vendored-files.json @@ -1,5 +1,7 @@ { "$comment": "Manifest of source files copied (vendored) from other repositories into dotnet/sdk. See eng/vendored-files.md. These files are the 'dotnet test' <-> Microsoft.Testing.Platform shared source (wire contract + terminal reporter); the source of truth is microsoft/testfx, which enumerates the same set via DotnetTestProtocolContract.props and TerminalReporterContract.props.", + "$comment_labels": "Area labels applied to drift issues in addition to 'area-vendored-sync'. Every entry below is 'dotnet test'/MTP source, so the default covers them all. A future entry vendored from a different upstream for a different area must set its own 'area_labels' so it is not misrouted to MTP triage.", + "default_area_labels": ["Area-dotnet test (MTP)"], "entries": [ { "id": "dotnet-test-wire-contract-fieldids", diff --git a/eng/vendored-files.md b/eng/vendored-files.md index 50e5668e855f..cafa4f5ec315 100644 --- a/eng/vendored-files.md +++ b/eng/vendored-files.md @@ -45,11 +45,17 @@ differ". ```jsonc { + // Area labels applied to every entry's drift issues, in addition to the + // `area-vendored-sync` label. Today all entries are `dotnet test`/MTP source. + "default_area_labels": ["Area-dotnet test (MTP)"], "entries": [ { "id": "stable-kebab-case-id", "local_path": "src/path/to/Local.cs", "notes": "free-form description of local adaptations", + // Optional. Overrides `default_area_labels` for this entry. Set it when + // the entry is vendored for a different area than the manifest default. + "area_labels": ["Area-SomethingElse"], "sources": [ { "repo": "owner/repo", @@ -65,6 +71,15 @@ differ". } ``` +The drift-detection mechanism itself is upstream-agnostic — a source may point at +any repo — so area routing is manifest data rather than something hard-coded in +the script. If you vendor a file for a different area, set `area_labels` on that +entry so its issues are not misrouted to the default area's triage queue. Every +declared area label must already exist in the repository's triage taxonomy; the +workflow only creates its own `area-vendored-sync` label. Existing sync issues +are reconciled to the entry's current area-label set while retaining unrelated +labels. + A single local file may declare multiple upstream sources. For example the terminal reporter is one file in this repo (`src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs`) but many @@ -92,7 +107,8 @@ For every `(entry, source)` pair the workflow: 3. Otherwise fetches the baseline content (via the blobs API, robust to force-pushes) and the current content (via `raw.githubusercontent.com`), computes a unified diff, and opens/updates a tracking issue labelled - `area-vendored-sync` containing: + `area-vendored-sync` plus the entry's area labels (see `default_area_labels` / + `area_labels` above) containing: - links to the upstream file history, baseline blob, current blob, and the whole-repo compare URL, - the upstream-only diff (truncated at 300 lines), @@ -117,7 +133,10 @@ reconciliation PR is merged. (`gh api repos/{repo}/commits/{ref} --jq .sha`), - `baseline_blob_sha`: the upstream file's blob SHA at that ref (`gh api "repos/{repo}/contents/{path}?ref={ref}" --jq .sha`). -3. Run `python .github/scripts/check_vendored_files.py validate` locally to +3. If the file does not belong to the area in `default_area_labels`, set + `area_labels` on the entry to existing repository triage labels so its drift + issues reach the right triage queue. +4. Run `python .github/scripts/check_vendored_files.py validate` locally to confirm the structure is correct. ## Reconciling drift