From 6798d87e37bbd2932571ec29e125502fee3d9621 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:30:29 +0800 Subject: [PATCH 01/35] [TRTLLM-12838][infra] CBTS: coverage-based test selection (Tier 2 selector) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add coverage-based test selection as Tier 2 in the CBTS decision pipeline. When Tier 1 (flat rules) defers to a full run on a residual of core-Python files, Tier 2 consults the merged touch DB to narrow per-stage test lists to only the cases provably reached by the changed functions. New files: - coverage_selection/selector.py: CoverageSelector — maps changed lines to qualnames via AST, queries the DB for impacted tests, returns per-stage impacted/skippable sets. - coverage_selection/qualname_map.py: AST-based line→qualname mapping with fallback to enclosing class/module importers for class-body changes. - coverage_selection/fixture.py: pytest fixture that resolves the CBTS touch DB path for unit tests. - coverage_tier.py: apply_coverage_tier() — orchestrates the selector, applies shared-block and rule-union correctness rules, sizes splits, emits structured reasons. - rules/product_data_rule.py: ProductDataRule — scopes pure product-data PRs (configs, curated examples, golden manifests) to productdataonly. - tools/coverage_explain.py: per-test explain tool showing which changed function triggered each must-run verdict. Modified: - main.py: wire Tier 2 (--coverage-db flag, CoverageTierResult → SelectionResult fields enable_multi_gpu / coverage_dropped_stages, structured reasons dict). - L0_MergeRequest.groovy: _cbtsCoverageAudit() returns the sqlite path; pass --coverage-db to main.py when the DB is available (enforce step). - L0_Test.groovy: re-add multiGpuJobs under baseline gate when enable_multi_gpu (coverage narrows single-GPU only; multi-GPU stays on MULTI_GPU_FILE_CHANGED). - dryrun.py: --coverage-db option to replay Tier 2 over historical commits. - rules/base.py, tests_def_rule.py, waives_rule.py: minor cleanups to expose block_filters and handled_files for Tier 2 consumption. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 31 +- jenkins/L0_Test.groovy | 14 +- .../cbts/coverage_selection/qualname_map.py | 86 +++++ .../cbts/coverage_selection/selector.py | 148 +++++++++ jenkins/scripts/cbts/coverage_tier.py | 308 ++++++++++++++++++ jenkins/scripts/cbts/main.py | 123 ++++++- jenkins/scripts/cbts/rules/base.py | 1 + jenkins/scripts/cbts/rules/tests_def_rule.py | 1 + jenkins/scripts/cbts/rules/waives_rule.py | 1 + .../scripts/cbts/tools/coverage_explain.py | 147 +++++++++ jenkins/scripts/cbts/tools/dryrun.py | 83 +++-- .../cbts/tools/report_cbts_decision.py | 11 +- 12 files changed, 898 insertions(+), 56 deletions(-) create mode 100644 jenkins/scripts/cbts/coverage_selection/qualname_map.py create mode 100644 jenkins/scripts/cbts/coverage_selection/selector.py create mode 100644 jenkins/scripts/cbts/coverage_tier.py create mode 100644 jenkins/scripts/cbts/tools/coverage_explain.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 996ac079314e..93505a318b42 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -846,8 +846,8 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // Shadow audit: download the latest merged touch DB and log its health + HEAD coverage gap (diagnostic only). - _cbtsCoverageAudit(pipeline) + // Download the touch DB for audit + Tier 2 coverage-based narrowing. + def coverageDbPath = _cbtsCoverageAudit(pipeline) // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -872,10 +872,11 @@ def getCbtsResult(pipeline, testFilter, globalVars) def inputPath = "${LLM_ROOT}/cbts_input.json" writeFile file: inputPath, text: inputJson - def output = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json", - returnStdout: true, - ) + def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" + if (coverageDbPath) { + mainCmd += " --coverage-db ${coverageDbPath}" + } + def output = sh(script: mainCmd, returnStdout: true) def result = _cbtsParseSelectionResult(output) if (result.scope == null) { @@ -917,30 +918,34 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Download the latest merged touch DB and run coverage_audit.py on it; best-effort, never changes the CBTS decision. +// Download the touch DB, audit it, and return the sqlite path (or "" on failure). def _cbtsCoverageAudit(pipeline) { try { - def covDir = "${LLM_ROOT}/cbts_cov" + // All commands run from ${LLM_ROOT}; covDir and the returned path are + // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. + def covDir = "cbts_cov" def url = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-url || true", returnStdout: true, ).trim() if (!url) { - pipeline.echo("CBTS audit: no coverage DB artifact found — skipping") - return + pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") + return "" } - sh "mkdir -p ${covDir}" + sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" // wget the tarball (retrying) and extract the sqlite. trtllm_utils.llmExecStepWithRetry(pipeline, script: - "wget -nv '${url}' -O ${covDir}/cbts_pystart_report.tar.gz && " + + "cd ${LLM_ROOT} && wget -nv '${url}' -O ${covDir}/cbts_pystart_report.tar.gz && " + "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") - sh "python3 ${LLM_ROOT}/jenkins/scripts/cbts/tools/coverage_audit.py " + + sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + "--db ${covDir}/cbts_touchmap.sqlite" + return "${covDir}/cbts_touchmap.sqlite" } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") + return "" } } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 3e8e4bf8391d..76bf47975cb0 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6546,9 +6546,14 @@ def launchTestJobs(pipeline, testFilter, globalVars) // they only run when explicitly listed in affected_stages. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { - // Match the -cbts rename cbtsResizeSplits applies to narrowed stages. + // cbtsResizeSplits renames only narrowed stages (those in + // affected_stage_split_counts) to `-cbts`; affected-but-not-narrowed + // stages keep their original name, so match each per its actual key. def stageSuffix = cbts.cbts_test_db_artifact_path ? CBTS_STAGE_SUFFIX : "" - def affectedSet = (cbts.affected_stages ?: []).collect { it + stageSuffix } as Set + def narrowed = (cbts.affected_stage_split_counts ?: [:]).keySet() + def affectedSet = (cbts.affected_stages ?: []).collect { + (stageSuffix && narrowed.contains(it)) ? (it + stageSuffix) : it + } as Set def needsSanity = cbts.sanity_required def needsPerfSanity = cbts.perfsanity_required parallelJobsFiltered = parallelJobs.findAll { key, _ -> @@ -6573,6 +6578,11 @@ def launchTestJobs(pipeline, testFilter, globalVars) } else { echo "CBTS [${cbts.scope}]: empty stage set after filtering" } + // coverage tier omits multi-GPU; re-add under baseline gate + if (cbts.enable_multi_gpu && testFilter[(MULTI_GPU_FILE_CHANGED)]) { + parallelJobsFiltered += multiGpuJobs + echo "CBTS [${cbts.scope}]: multi-GPU file changed → running ${multiGpuJobs.size()} multi-GPU stage(s) at baseline" + } } if (globalVars[RUN_MODE] == "nightly_release") { diff --git a/jenkins/scripts/cbts/coverage_selection/qualname_map.py b/jenkins/scripts/cbts/coverage_selection/qualname_map.py new file mode 100644 index 000000000000..1ed3f6236cb3 --- /dev/null +++ b/jenkins/scripts/cbts/coverage_selection/qualname_map.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Map changed source lines to co_qualname strings matching the touch DB.""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass + + +@dataclass +class _Scope: + qualname: str + sig_start: int + body_start: int + body_end: int + body_attr: str + sig_attr: str + + +def _substatements(node: ast.stmt): + """Yield direct sub-statements of a compound statement (no new scope).""" + for field in ("body", "orelse", "finalbody"): + yield from getattr(node, field, None) or [] + for handler in getattr(node, "handlers", None) or []: + yield from handler.body + for case in getattr(node, "cases", None) or []: + yield from case.body + + +def _collect_scopes(tree: ast.Module) -> list[_Scope]: + scopes: list[_Scope] = [] + + def walk(stmts, prefix: str, enclosing_attr: str) -> None: + for node in stmts: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + qual = prefix + node.name + recorded = "" not in qual + sig_start = min([node.lineno, *(d.lineno for d in node.decorator_list)]) + body_start = node.body[0].lineno + body_attr = qual if recorded else enclosing_attr + scopes.append( + _Scope(qual, sig_start, body_start, node.end_lineno, body_attr, enclosing_attr) + ) + if isinstance(node, ast.ClassDef): + walk(node.body, qual + ".", body_attr) + else: + walk(node.body, qual + "..", body_attr) + else: + subs = list(_substatements(node)) + if subs: + walk(subs, prefix, enclosing_attr) + + walk(tree.body, "", "") + return scopes + + +def _attribute(line: int, scopes: list[_Scope]) -> str: + best: _Scope | None = None + for s in scopes: + if s.sig_start <= line <= s.body_end and (best is None or s.sig_start > best.sig_start): + best = s + if best is None: + return "" + return best.sig_attr if line < best.body_start else best.body_attr + + +def qualnames_for_lines(source: str, lines: set[int]) -> tuple[set[str], bool]: + """Return (qualnames, ok); ok=False when the source cannot be parsed.""" + try: + tree = ast.parse(source) + except SyntaxError: + return set(), False + scopes = _collect_scopes(tree) + return {_attribute(ln, scopes) for ln in lines}, True diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py new file mode 100644 index 000000000000..7123004682da --- /dev/null +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Coverage-based selection: changed core-Python files -> per-stage impacted/skippable sets.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from qualname_map import qualnames_for_lines +from rules._helpers import iter_diff_post_line_numbers +from touch_db import ( + _LAUNCH_MARKERS, + _MIN_FUNCS, + _SERVING_PATH_MARKERS, + _WORKER_SENTINEL, + TouchDB, + canon, + split_stage, +) + + +@dataclass +class CoverageResult: + """Per-stage coverage decision over a set of residual core-Python files.""" + + ok: bool + reason: str + impacted: dict[str, set[str]] = field(default_factory=dict) + skippable: dict[str, set[str]] = field(default_factory=dict) + n_untrusted: int = 0 + # functions with no DB rows (new/uninstrumented); bounded via importers + no_data_funcs: list[str] = field(default_factory=list) + + +class CoverageSelector: + def __init__( + self, + db: TouchDB, + repo_root: Path, + *, + worker_sentinel: str = _WORKER_SENTINEL, + launch_markers: tuple[tuple[str, str], ...] = _LAUNCH_MARKERS, + serving_path_markers: tuple[str, ...] = _SERVING_PATH_MARKERS, + min_funcs: int = _MIN_FUNCS, + ) -> None: + self.db = db + self.repo_root = Path(repo_root) + self._worker_sentinel = worker_sentinel + self._launch_markers = launch_markers + self._serving_path_markers = serving_path_markers + self._min_funcs = min_funcs + self._untrusted: set[str] | None = None + + def untrusted_tests(self) -> set[str]: + """Stage-prefixed tests with incomplete-looking capture (cached, DB-wide).""" + if self._untrusted is None: + self._untrusted = self.db.untrusted_tests( + self._worker_sentinel, + self._launch_markers, + self._serving_path_markers, + self._min_funcs, + ) + return self._untrusted + + def _impacted_tests( + self, residual_files: list[str], diffs: dict[str, str] + ) -> tuple[set[str], list[str]]: + """Return (impacted stage-prefixed tests, file::qualname symbols with no DB rows).""" + impacted: set[str] = set() + no_data: list[str] = [] + for path in residual_files: + cf = canon(path) + lines = iter_diff_post_line_numbers(diffs.get(path, "")) + source = self._read_head(path) + if not lines or source is None: + impacted |= self.db.tests_touching_file(cf) + continue + qualnames, ok = qualnames_for_lines(source, lines) + if not ok: + impacted |= self.db.tests_touching_file(cf) + continue + for qualname in sorted(qualnames): # sorted -> deterministic no_data order + tests = self.db.tests_touching_func(cf, qualname) + impacted |= tests + if not tests and qualname != "": + no_data.append(f"{cf}::{qualname}") + return impacted, no_data + + def _read_head(self, path: str) -> str | None: + try: + return (self.repo_root / path).read_text() + except (OSError, UnicodeDecodeError): + return None + + def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: + """Decide over residual files (repo-relative paths no rule claimed). + + Returns ok=False for any non-core-Python file or file absent from the DB. + """ + for path in residual_files: + cf = canon(path) + if not (path.endswith(".py") and cf.startswith("tensorrt_llm/")): + return CoverageResult(ok=False, reason=f"non-core-Python residual file: {path}") + if not self.db.file_has_touch_rows(cf): + return CoverageResult( + ok=False, reason=f"zero-touch residual file (new/uninstrumented): {path}" + ) + + impacted_tests, no_data_funcs = self._impacted_tests(residual_files, diffs) + + impacted: dict[str, set[str]] = {} + for test in impacted_tests: + stage, nodeid = split_stage(test) + if stage: + impacted.setdefault(stage, set()).add(nodeid) + + untrusted = self.untrusted_tests() + skippable: dict[str, set[str]] = {} + n_untrusted = 0 + for stage, known_nodeids in self.db.known_by_stage().items(): + imp = impacted.get(stage, set()) + keep_untrusted = {n for n in known_nodeids if f"{stage}/{n}" in untrusted} + n_untrusted += len(keep_untrusted - imp) + skippable[stage] = known_nodeids - imp - keep_untrusted + + return CoverageResult( + ok=True, + reason=( + f"{len(residual_files)} file(s) -> {len(impacted_tests)} impacted test(s); " + f"{n_untrusted} untrusted (incomplete-capture) test(s) forced to run" + ), + impacted=impacted, + skippable=skippable, + n_untrusted=n_untrusted, + no_data_funcs=no_data_funcs, + ) diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py new file mode 100644 index 000000000000..06242892299a --- /dev/null +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CBTS Tier 2: coverage-based test-db narrowing on the Tier-1 fallback residual.""" + +from __future__ import annotations + +import math +import re +import sys +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +import yaml +from blocks import ( + TARGET_SHARD_SECONDS, + Stage, + YAMLIndex, + _avg_duration, + _entry_applies_to_waive, + _entry_target, + _estimate_entries_seconds, + _target_in_filter_subtree, + block_matches_stage, +) +from rules.base import PRInputs, RuleResult + +sys.path.insert(0, str(Path(__file__).resolve().parent / "coverage_selection")) +from selector import CoverageSelector # noqa: E402 +from touch_db import TouchDB, db_key # noqa: E402 + + +def open_db(path: str) -> TouchDB: + """Open a touch DB read-only.""" + return TouchDB.open(path) + + +@dataclass +class CoverageTierResult: + affected_stages: set[str] + removed: dict[tuple[str, int], set[str]] = field(default_factory=dict) + dropped: set[str] = field(default_factory=set) + reason: str = "" + must_run_reasons: dict[str, int] = field(default_factory=dict) + detail: dict[str, object] = field(default_factory=dict) + + +# Mirrors L0_Test.groovy multiGpuJobs: pre-merge stages with N_GPUs token. +_MULTI_GPU_RE = re.compile(r"\d+_GPUs") + + +def _is_multi_gpu(stage_name: str) -> bool: + return bool(_MULTI_GPU_RE.search(stage_name)) and "Post-Merge" not in stage_name + + +def _rule_kept_entries(block, prefix_to_waives: dict[str, set[str]]) -> set[str]: + """Entries a rule's block_filters would keep.""" + kept: set[str] = set() + for t in block.tests: + target = _entry_target(t) + matched: set[str] = set() + for prefix, waives in prefix_to_waives.items(): + if _target_in_filter_subtree(target, prefix): + matched |= waives + if matched and any(_entry_applies_to_waive(t, w) for w in matched): + kept.add(t) + return kept + + +_R_SAFE = "safe" +_R_IMPACTED = "impacted" +_R_UNTRUSTED = "untrusted" +_R_NO_DATA = "no_data" +_R_RULE_KEPT = "rule_kept" +_R_COARSE = "coarse" + + +def _entry_reason( + entry: str, + served: list[str], + keep_rule: set[str], + cov, + known: dict[str, set[str]], + untrusted: set[str], +) -> str: + """Return SAFE or the must-run cause for a candidate YAML entry.""" + if entry in keep_rule: + return _R_RULE_KEPT + dbk = db_key(entry) + if dbk is None: + return _R_COARSE + for name in served: + if dbk not in known.get(name, frozenset()): + return _R_NO_DATA + if dbk in cov.impacted.get(name, frozenset()): + return _R_IMPACTED + if f"{name}/{dbk}" in untrusted: + return _R_UNTRUSTED + return _R_SAFE + + +def _build_narrowing( + cov, + stages: dict[str, Stage], + yaml_index: YAMLIndex, + rule_block_filters: dict[tuple[str, int], dict[str, set[str]]], + known: dict[str, set[str]], + untrusted: set[str], +) -> tuple[dict[tuple[str, int], set[str]], set[str], Counter]: + """Classify every candidate entry; remove only SAFE ones. + + Returns (removed per block, fully-emptied instrumented stages, must-run tally). + """ + instrumented = set(cov.skippable) + rule_kept = { + key: _rule_kept_entries(b, rule_block_filters[key]) + for b in yaml_index.blocks + if (key := (b.yaml_stem, b.block_index)) in rule_block_filters + } + + removed: dict[tuple[str, int], set[str]] = {} + must_run: Counter = Counter() + for block in yaml_index.blocks: + served = [ + s.name + for s in stages.values() + if s.yaml_stem == block.yaml_stem and block_matches_stage(block, s) + ] + # shared-block rule: prune only when every served stage is instrumented + if not served or any(name not in instrumented for name in served): + continue + key = (block.yaml_stem, block.block_index) + keep_rule = rule_kept.get(key, set()) + rm: set[str] = set() + for entry in block.tests: + reason = _entry_reason(entry, served, keep_rule, cov, known, untrusted) + if reason == _R_SAFE: + rm.add(entry) + else: + must_run[reason] += 1 + if rm: + removed[key] = rm + + dropped: set[str] = set() + for name in instrumented: + stage = stages.get(name) + if stage is None: + continue + total = kept = 0 + for block in yaml_index.blocks: + if block.yaml_stem != stage.yaml_stem or not block_matches_stage(block, stage): + continue + rm = removed.get((block.yaml_stem, block.block_index), set()) + for entry in block.tests: + total += 1 + if entry not in rm: + kept += 1 + if total > 0 and kept == 0: + dropped.add(name) + return removed, dropped, must_run + + +def apply_coverage_tier( + pr: PRInputs, + pairs: list[tuple[object, RuleResult]], + handled: set[str], + stages: dict[str, Stage], + yaml_index: YAMLIndex, + repo_root: Path, + db: TouchDB, +) -> tuple[CoverageTierResult | None, str]: + """Return (narrowing, note); narrowing is None when the tier keeps the Tier-1 result.""" + if any(r.scope is None for _, r in pairs): + return None, "coverage tier skipped: a rule forced fallback (scope=null)" + residual = sorted(set(pr.changed_files) - handled) + if not residual: + return None, "coverage tier skipped: no residual (all files handled by rules)" + + selector = CoverageSelector(db, repo_root) + cov = selector.decide(residual, pr.diffs) + if not cov.ok: + return None, f"coverage tier declined: {cov.reason}" + + rule_block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} + for _, r in pairs: + for key, prefix_to_waives in r.block_filters.items(): + dst = rule_block_filters.setdefault(key, {}) + for prefix, waives in prefix_to_waives.items(): + dst.setdefault(prefix, set()).update(waives) + + nd = "" + if cov.no_data_funcs: + shown = ", ".join(cov.no_data_funcs[:3]) + more = f" (+{len(cov.no_data_funcs) - 3})" if len(cov.no_data_funcs) > 3 else "" + nd = f"; new/uncovered function(s), bounded via importers: {shown}{more}" + + known = db.known_by_stage() + untrusted = selector.untrusted_tests() + removed, dropped, must_run_reasons = _build_narrowing( + cov, stages, yaml_index, rule_block_filters, known, untrusted + ) + # exclude multi-GPU and post-merge stages from the drop set (not coverage's to decide) + dropped = {s for s in dropped if not _is_multi_gpu(s)} + if not pr.post_merge: + dropped = {s for s in dropped if "Post-Merge" not in s} + + n_impacted = sum(len(v) for v in cov.impacted.values()) + n_removed = sum(len(v) for v in removed.values()) + narrowed = bool(removed or dropped) + if narrowed: + reason = ( + f"coverage: {len(residual)} core file(s), {n_impacted} impacted test(s), " + f"{cov.n_untrusted} untrusted forced-run; " + f"removed {n_removed} case(s), dropped {len(dropped)} single-GPU stage(s){nd}" + ) + else: + reason = ( + f"coverage: {len(residual)} core file(s), {n_impacted} impacted test(s), " + f"{cov.n_untrusted} untrusted; nothing removable (all impacted / untrusted / not-in-DB){nd}" + ) + result = CoverageTierResult( + # single-GPU only; multi-GPU re-added in Groovy under MULTI_GPU_FILE_CHANGED gate + affected_stages={s for s in stages if not _is_multi_gpu(s)} - dropped, + removed=removed, + dropped=dropped, + reason=reason, + must_run_reasons=dict(must_run_reasons), + detail={ + "source": "coverage", + "files": len(residual), + "impacted": n_impacted, + "untrusted": cov.n_untrusted, + "removed_cases": n_removed, + "dropped_stages": len(dropped), + "outcome": "narrowed" if narrowed else "nothing_removable", + **({"no_data_funcs": list(cov.no_data_funcs)} if cov.no_data_funcs else {}), + }, + ) + return result, result.reason + + +def write_coverage_test_db( + src_dir: Path, out_dir: Path, removed: dict[tuple[str, int], set[str]] +) -> None: + """Write narrowed YAMLs with removed entries dropped.""" + out_dir.mkdir(parents=True, exist_ok=True) + for stem in sorted({stem for stem, _ in removed}): + src = src_dir / f"{stem}.yml" + if not src.exists(): + continue + data = yaml.safe_load(src.read_text()) or {} + blocks = data.get(stem) + if not isinstance(blocks, list): + continue + for i, block_data in enumerate(blocks): + if not isinstance(block_data, dict): + continue + rm = removed.get((stem, i)) + if not rm: + continue + block_data["tests"] = [t for t in (block_data.get("tests") or []) if t not in rm] + (out_dir / src.name).write_text( + yaml.safe_dump(data, sort_keys=False, default_flow_style=False) + ) + + +def compute_coverage_stage_counts( + affected_stages: set[str], + stages: dict[str, Stage], + yaml_index: YAMLIndex, + removed: dict[tuple[str, int], set[str]], + durations: dict[str, float], + target_seconds: int = TARGET_SHARD_SECONDS, +) -> tuple[dict[str, int], dict[str, int]]: + """Return per-stage (kept-entry count, resized split count) for narrowed stages only.""" + avg = _avg_duration(durations) + test_counts: dict[str, int] = {} + split_counts: dict[str, int] = {} + for name in affected_stages: + stage = stages.get(name) + if stage is None: + continue + entries: list[str] = [] + had_removal = False + for block in yaml_index.blocks: + if block.yaml_stem != stage.yaml_stem or not block_matches_stage(block, stage): + continue + rm = removed.get((block.yaml_stem, block.block_index), set()) + if rm: + had_removal = True + entries.extend(t for t in block.tests if t not in rm) + if not had_removal: + continue + test_counts[name] = len(entries) + seconds = _estimate_entries_seconds(entries, durations, avg) + split_counts[name] = max(1, min(math.ceil(seconds / target_seconds), stage.total_splits)) + return test_counts, split_counts diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index ab6eed8e5ca5..e261a751d63a 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -48,6 +48,12 @@ parse_stages_from_groovy, write_filtered_test_db, ) +from coverage_tier import ( # noqa: E402 + apply_coverage_tier, + compute_coverage_stage_counts, + open_db, + write_coverage_test_db, +) from rules._helpers import strip_noop_diff_lines # noqa: E402 from rules.agent_flow_rule import AgentFlowRule # noqa: E402 from rules.auto_deploy_rule import AutoDeployRule # noqa: E402 @@ -103,7 +109,7 @@ class SelectionResult: # rolls these up; noop gives way to actionable scopes there). scopes: list[str] = field(default_factory=list) affected_stages: set[str] = field(default_factory=set) - reasons: list[str] = field(default_factory=list) + reasons: list[dict] = field(default_factory=list) block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) test_db_dir_override: Optional[str] = None # Per-stage kept-entry count (decision telemetry / diagnostics). @@ -116,6 +122,9 @@ class SelectionResult: # Aggregated `any(rule.perfsanity_relevant)`. Groovy Layer 2 keeps # *-PerfSanity-* stages only when True. perfsanity_required: bool = True + # set by coverage tier; Groovy re-adds multiGpuJobs under MULTI_GPU_FILE_CHANGED gate + enable_multi_gpu: bool = False + coverage_dropped_stages: list[str] = field(default_factory=list) def to_json(self) -> str: data = { @@ -128,10 +137,31 @@ def to_json(self) -> str: "affected_stage_split_counts": dict(self.affected_stage_split_counts), "sanity_required": self.sanity_required, "perfsanity_required": self.perfsanity_required, + "enable_multi_gpu": self.enable_multi_gpu, + "coverage_dropped_stages": sorted(self.coverage_dropped_stages), } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" +def _rule_reason(rule, r) -> dict: + """One rule's structured reason entry: `{source, **detail, blocks, stages}`.""" + return { + "source": rule.name, + **r.detail, + "blocks": len(r.block_filters), + "stages": len(r.affected_stages), + } + + +def _fmt_reason(r) -> str: + """Render a structured reason dict as one human line: `[source] k=v, ...`.""" + if not isinstance(r, dict): + return str(r) + src = r.get("source", "?") + rest = ", ".join(f"{k}={v}" for k, v in r.items() if k != "source") + return f"[{src}] {rest}" if rest else f"[{src}]" + + # Scopes that compose: a PR mixing waive + test-def + test-list edits # combines to a single "testsonly" scope rather than falling back. _TESTSONLY_FAMILY: frozenset[str] = frozenset( @@ -182,26 +212,35 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: handled: set[str] = set() for _, r in pairs: handled |= r.handled_files + # Expose for the coverage tier (Tier 2), which unions over the residual. + self.pairs = pairs + self.handled = handled + rule_reasons = [_rule_reason(rule, r) for rule, r in pairs] + unhandled = sorted(set(pr.changed_files) - handled) if unhandled: - preview = unhandled[:5] - more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" - return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) + return SelectionResult( + scope=None, + reasons=rule_reasons + + [{"source": "fallback", "reason": "unhandled_files", "files": unhandled}], + ) if not pairs: - return SelectionResult(scope=None, reasons=["No rule contributed"]) + return SelectionResult( + scope=None, + reasons=[{"source": "fallback", "reason": "no_rule_contributed"}], + ) - reasons = [f"[{rule.name}] {r.reason}" for rule, r in pairs] scope = _combine_scopes([r.scope for _, r in pairs]) if scope is None: # scope=None is a rule's force-fallback signal; attribute it, not a scope conflict. forced = sorted({rule.name for rule, r in pairs if r.scope is None}) if forced: - summary = f"Fallback forced by rule(s): {', '.join(forced)}" + fb = {"source": "fallback", "reason": "forced_by_rules", "rules": forced} else: actionable = sorted({r.scope for _, r in pairs if r.scope and r.scope != "noop"}) - summary = f"Scopes cannot be combined: {', '.join(actionable)}" - return SelectionResult(scope=None, reasons=reasons + [summary]) + fb = {"source": "fallback", "reason": "scopes_uncombinable", "scopes": actionable} + return SelectionResult(scope=None, reasons=rule_reasons + [fb]) affected_stages: set[str] = set() for _, r in pairs: @@ -216,11 +255,7 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: if not affected_stages and scope != "noop": return SelectionResult( scope=None, - reasons=reasons - + [ - "Rules fired but no stages resolved (likely YAML/waive " - "granularity mismatch); falling back to baseline." - ], + reasons=rule_reasons + [{"source": "fallback", "reason": "no_stages_resolved"}], ) # Aggregate per-block prefix->{waive_ids} across rules. @@ -238,7 +273,7 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: scope=scope, scopes=sorted({r.scope for _, r in pairs if r.scope}), affected_stages=affected_stages, - reasons=reasons, + reasons=rule_reasons, block_filters=block_filters, sanity_required=sanity_required, perfsanity_required=perfsanity_required, @@ -297,6 +332,12 @@ def main(argv: Optional[list[str]] = None) -> int: help="Override path to the Jenkins test Groovy file " "(default: /jenkins/L0_Test.groovy).", ) + parser.add_argument( + "--coverage-db", + default=None, + help="Path to cbts_touchmap.sqlite. When set, the coverage tier (Tier 2) " + "runs on fallbacks and may drop fully-safe single-GPU stages.", + ) args = parser.parse_args(argv) if args.list_needed_diffs: @@ -340,7 +381,55 @@ def main(argv: Optional[list[str]] = None) -> int: stages = parse_stages_from_groovy(groovy_path, include_post_merge=True) pr = _load_pr_inputs(input_path) rules = build_rules(yaml_index, stages, repo_root) - result = Selector(stages).run(pr, rules) + selector = Selector(stages) + result = selector.run(pr, rules) + + if args.coverage_db and result.scope is None: + note = "" + try: + db = open_db(args.coverage_db) + tier, note = apply_coverage_tier( + pr, selector.pairs, selector.handled, stages, yaml_index, repo_root, db + ) + except Exception as e: # noqa: BLE001 — CBTS must never break CI + note = f"coverage tier errored: {e}" + tier = None + if tier is not None: + result.scope = "coverage" + result.scopes = sorted( + {r.scope for _, r in selector.pairs if r.scope and r.scope != "noop"} | {"coverage"} + ) + result.affected_stages = tier.affected_stages + result.enable_multi_gpu = True + result.coverage_dropped_stages = sorted(tier.dropped) + if tier.removed: + write_coverage_test_db( + src_dir=test_db_dir, + out_dir=repo_root / "cbts_test_db", + removed=tier.removed, + ) + result.test_db_dir_override = "cbts_test_db" + durations = load_durations(repo_root / "tests/integration/defs/.test_durations") + ( + result.affected_stage_test_counts, + result.affected_stage_split_counts, + ) = compute_coverage_stage_counts( + affected_stages=set(result.affected_stages), + stages=stages, + yaml_index=yaml_index, + removed=tier.removed, + durations=durations, + ) + cov_reason = dict(tier.detail) + cov_reason["narrowed_stages"] = len(result.affected_stage_split_counts) + result.reasons = [x for x in result.reasons if x.get("source") != "fallback"] + [ + cov_reason + ] + elif note: + for x in result.reasons: + if isinstance(x, dict) and x.get("source") == "fallback": + x["coverage_declined"] = note + break # Layer 3: write narrowed test-db when any block was filtered. if result.scope is not None and result.block_filters: @@ -408,7 +497,7 @@ def _log_decision_to_stderr( if result.reasons: print(" reasons:", file=out) for r in result.reasons: - print(f" - {r}", file=out) + print(f" - {_fmt_reason(r)}", file=out) print(f" block_filters ({len(result.block_filters)} blocks):", file=out) for (yaml_stem, idx), prefix_to_waives in sorted(result.block_filters.items()): print(f" - {yaml_stem}#{idx}:", file=out) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index f4f71d0335e2..6c3d24c9c013 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -50,6 +50,7 @@ class RuleResult: # True (safe default) iff this rule's matched changes might affect perf # benchmarks. Set False when matched changes are pure test infra. perfsanity_relevant: bool = True + detail: dict[str, object] = field(default_factory=dict) class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/tests_def_rule.py b/jenkins/scripts/cbts/rules/tests_def_rule.py index 9f85039f5e01..56526c19e27e 100644 --- a/jenkins/scripts/cbts/rules/tests_def_rule.py +++ b/jenkins/scripts/cbts/rules/tests_def_rule.py @@ -339,4 +339,5 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: f"{len(block_filters)} blocks, {len(affected_stages)} stages" f"{nonarrow_note}" ), + detail={"paths": len(narrowed)}, ) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 184193c53600..976143a58fc2 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -116,4 +116,5 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: f"waives.txt: +{len(added)} / -{len(removed)} → " f"{len(block_filters)} blocks, {len(affected_stages)} stages{miss_note}" ), + detail={"added": len(added), "removed": len(removed)}, ) diff --git a/jenkins/scripts/cbts/tools/coverage_explain.py b/jenkins/scripts/cbts/tools/coverage_explain.py new file mode 100644 index 000000000000..8baa894994ec --- /dev/null +++ b/jenkins/scripts/cbts/tools/coverage_explain.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Explain a coverage-selection decision for one commit. + +For each instrumented stage, prints why each known case is kept (it entered a +changed function) or removed (it is in the DB but never entered any changed +function). The justification is the forward touch lookup — the audit view that +makes `cbts_removed_cases.txt` self-verifying. + +Example:: + + python3 jenkins/scripts/cbts/tools/coverage_explain.py \\ + --db /tmp/cbts_inspect/cbts_touchmap.sqlite --sha 890e1089 --show-kept +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +THIS = Path(__file__).resolve() +CBTS = THIS.parent.parent +sys.path.insert(0, str(CBTS)) +sys.path.insert(0, str(CBTS / "coverage_selection")) + +from qualname_map import qualnames_for_lines # noqa: E402 +from rules._helpers import iter_diff_post_line_numbers # noqa: E402 +from touch_db import TouchDB, canon, split_stage # noqa: E402 + + +def _git(repo: Path, *args: str, check: bool = True) -> str: + return subprocess.run( + ["git", *args], cwd=str(repo), capture_output=True, text=True, check=check + ).stdout + + +def _src_at(repo: Path, sha: str, path: str) -> str | None: + r = subprocess.run( + ["git", "show", f"{sha}:{path}"], cwd=str(repo), capture_output=True, text=True, check=False + ) + return r.stdout if r.returncode == 0 else None + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--db", required=True) + ap.add_argument("--sha", required=True) + ap.add_argument("--repo-root", default=str(CBTS.parents[2])) + ap.add_argument("--stage", default=None, help="limit to one stage") + ap.add_argument("--show-kept", action="store_true", help="also list kept (impacted) cases") + args = ap.parse_args(argv) + + repo = Path(args.repo_root).resolve() + db = TouchDB.open(args.db) + + files = [ + ln + for ln in _git(repo, "show", "--name-only", "--pretty=format:", args.sha).splitlines() + if ln.strip() + ] + core = [f for f in files if f.endswith(".py") and canon(f).startswith("tensorrt_llm/")] + + # Build the change's impact set: function-level (file, qualname), or (file, None) + # when a file falls back to file-level. Collect the impacted tests. + impact_funcs: set[tuple[str, str]] = set() + impact_files: set[str] = set() # file-level fallback + changed_files: set[str] = set() + impacted: set[str] = set() + for f in core: + cf = canon(f) + changed_files.add(cf) + diff = _git(repo, "diff", f"{args.sha}^", args.sha, "--", f, check=False) + lines = iter_diff_post_line_numbers(diff) + src = _src_at(repo, args.sha, f) + if not lines or src is None: + impact_files.add(cf) + impacted |= db.tests_touching_file(cf) + continue + qns, ok = qualnames_for_lines(src, lines) + if not ok: + impact_files.add(cf) + impacted |= db.tests_touching_file(cf) + continue + for q in qns: + impact_funcs.add((cf, q)) + impacted |= db.tests_touching_func(cf, q) + + print(f"commit {args.sha[:12]} — {len(core)} core file(s), impact set:") + for cf, q in sorted(impact_funcs): + print(f" {cf} :: {q}") + for cf in sorted(impact_files): + print(f" {cf} :: ") + + impacted_by_stage: dict[str, set[str]] = {} + for t in impacted: + stage, nodeid = split_stage(t) + impacted_by_stage.setdefault(stage, set()).add(nodeid) + + def entered_changed(nodeid: str, stage: str) -> tuple[int, int, list[str]]: + """(total rows, funcs entered in changed files, changed qualnames entered).""" + touched = db.files_touched_by(f"{stage}/{nodeid}") + in_changed = sum(1 for f, _ in touched if f in changed_files) + hits = [f"{f.rsplit('/', 1)[-1]}::{q}" for f, q in touched if (f, q) in impact_funcs] + hits += [f"{f.rsplit('/', 1)[-1]}::" for f, q in touched if f in impact_files] + return len(touched), in_changed, sorted(set(hits)) + + for stage in sorted(db.known_by_stage()): + if args.stage and stage != args.stage: + continue + known_s = db.known_by_stage()[stage] + imp_s = impacted_by_stage.get(stage, set()) & known_s + skip_s = known_s - imp_s + print(f"\n=== {stage} known={len(known_s)} kept={len(imp_s)} removed={len(skip_s)} ===") + if args.show_kept and imp_s: + print(" KEPT (impacted):") + for n in sorted(imp_s): + _, _, hits = entered_changed(n, stage) + print(f" {n}\n entered: {', '.join(hits) or '(file-level)'}") + print(" REMOVED (safe to skip):") + for n in sorted(skip_s): + total, in_changed, _ = entered_changed(n, stage) + if in_changed == 0: + why = f"in DB (rows={total}); never entered any changed file" + else: + why = f"in DB (rows={total}); entered {in_changed} func(s) in changed file(s), none the changed one" + print(f" {n}\n {why}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index 2293e35ebc2b..de376d716f5e 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -45,6 +45,13 @@ python3 jenkins/scripts/cbts/tools/dryrun.py \\ --range origin/main...HEAD --out /tmp/cbts_range + +Coverage tier (Tier 2) is OFF by default. To include coverage-based selection +(merged with the rules by `main.py`), download the post-merge touch DB and pass +`--coverage-db`:: + + python3 jenkins/scripts/cbts/tools/dryrun.py \\ + --window 40 --filter all --coverage-db /path/to/cbts_touchmap.sqlite """ from __future__ import annotations @@ -127,27 +134,27 @@ def _resolve_pr(subject: str, sha: str) -> tuple[str, str]: # --- run main.py ------------------------------------------------------------ -def _run_cbts(payload: dict, test_db: Path, groovy: Path, repo: Path) -> dict: +def _run_cbts( + payload: dict, test_db: Path, groovy: Path, repo: Path, coverage_db: Optional[str] = None +) -> dict: with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: json.dump(payload, f) path = f.name + argv = [ + sys.executable, + str(CBTS_MAIN), + path, + "--repo-root", + str(repo), + "--test-db", + str(test_db), + "--groovy-file", + str(groovy), + ] + if coverage_db: + argv += ["--coverage-db", coverage_db] try: - res = subprocess.run( - [ - sys.executable, - str(CBTS_MAIN), - path, - "--repo-root", - str(repo), - "--test-db", - str(test_db), - "--groovy-file", - str(groovy), - ], - capture_output=True, - text=True, - check=False, - ) + res = subprocess.run(argv, capture_output=True, text=True, check=False) finally: os.unlink(path) if res.returncode != 0: @@ -200,8 +207,18 @@ def _fmt_summary( lines.append(f"split sizing ({len(splits)} stages, kept_entries -> shards):") for s in sorted(splits): lines.append(f" - {s}: {counts.get(s, '?')} -> {splits[s]}") + dropped = result.get("coverage_dropped_stages", []) + if dropped: + lines.append(f"coverage dropped stages ({len(dropped)}, emptied -> not run):") + lines.extend(f" - {s}" for s in dropped) lines.append("reasons:") - lines.extend(f" - {r}" for r in result.get("reasons", [])) + for r in result.get("reasons", []): + if isinstance(r, dict): + src = r.get("source", "?") + rest = ", ".join(f"{k}={v}" for k, v in r.items() if k != "source") + lines.append(f" - [{src}] {rest}" if rest else f" - [{src}]") + else: + lines.append(f" - {r}") return "\n".join(lines) + "\n" @@ -260,6 +277,7 @@ def _cbts_for_snapshot( diffs: dict[str, str], post_merge: bool, pr_dir: Path, + coverage_db: Optional[str] = None, ) -> dict: """Run CBTS against the tree at `tip_sha` over `files`/`diffs`. @@ -278,7 +296,7 @@ def _cbts_for_snapshot( test_db = wt / TEST_DB_REL groovy = wt / GROOVY_REL shared_out = wt / "cbts_test_db" - result = _run_cbts(payload, test_db, groovy, wt) + result = _run_cbts(payload, test_db, groovy, wt, coverage_db) if shared_out.exists(): for yml in shared_out.glob("*.yml"): shutil.copy2(yml, pr_dir / yml.name) @@ -292,6 +310,7 @@ def _replay_one( sha: str, out_dir: Path, post_merge: bool, + coverage_db: Optional[str] = None, ) -> tuple[str, str, list[str], str, dict, bool]: subject = _git(repo, "log", "-1", "--pretty=%s", sha).stdout.strip() label, pr_url = _resolve_pr(subject, sha) @@ -299,7 +318,7 @@ def _replay_one( tests_only = _is_tests_only(files) pr_dir = _prep_pr_dir(out_dir, label) diffs = {f: _file_diff(repo, sha, f) for f in files} - result = _cbts_for_snapshot(repo, sha, files, diffs, post_merge, pr_dir) + result = _cbts_for_snapshot(repo, sha, files, diffs, post_merge, pr_dir, coverage_db) (pr_dir / "summary.txt").write_text( _fmt_summary(pr_url, sha, subject, files, result, post_merge, tests_only) ) @@ -311,6 +330,7 @@ def _replay_range( range_expr: str, out_dir: Path, post_merge: bool, + coverage_db: Optional[str] = None, ) -> tuple[str, str, list[str], str, dict, bool]: """Replay the cumulative diff of a git range as a single CBTS run. @@ -329,7 +349,7 @@ def _replay_range( pr_url = f"range {range_expr} (base={base_disp}, tip={tip_sha[:8]})" pr_dir = _prep_pr_dir(out_dir, label) diffs = {f: _file_diff_range(repo, diff_arg, f) for f in files} - result = _cbts_for_snapshot(repo, tip_sha, files, diffs, post_merge, pr_dir) + result = _cbts_for_snapshot(repo, tip_sha, files, diffs, post_merge, pr_dir, coverage_db) (pr_dir / "summary.txt").write_text( _fmt_summary(pr_url, tip_sha, subject, files, result, post_merge, tests_only) ) @@ -438,6 +458,11 @@ def main(argv: Optional[list[str]] = None) -> int: "ignores --ref/--window/--filter/--sha/--limit", ) ap.add_argument("--post-merge", action="store_true", help="set post_merge=True") + ap.add_argument( + "--coverage-db", + default=None, + help="path to cbts_touchmap.sqlite; enables the coverage tier (Tier 2) in main.py", + ) ap.add_argument("--out", default="cbts_dryrun", help="output directory (default: cbts_dryrun)") ap.add_argument( "--repo-root", @@ -457,6 +482,18 @@ def main(argv: Optional[list[str]] = None) -> int: print(f"error: cbts main.py not found at {CBTS_MAIN}", file=sys.stderr) return 2 + # Coverage tier is opt-in: it needs the post-merge touch DB, which the user + # must download separately. Default replays rules only. + if args.coverage_db: + print(f"coverage tier ON (merged with rules) via {args.coverage_db}", file=sys.stderr) + else: + print( + "coverage tier OFF (rules only). To include coverage-based selection, download the " + "post-merge cbts_pystart_report.tar.gz (.../L0_PostMerge/cbts-coverage/), extract " + "cbts_touchmap.sqlite, and pass --coverage-db .", + file=sys.stderr, + ) + # Drop stale worktree metadata from interrupted prior runs so # `git worktree add` doesn't trip on dangling entries. _git(repo, "worktree", "prune", check=False) @@ -474,7 +511,7 @@ def main(argv: Optional[list[str]] = None) -> int: if args.range_expr: print(f"Replaying cumulative range: {args.range_expr}", file=sys.stderr) try: - row = _replay_range(repo, args.range_expr, out_dir, args.post_merge) + row = _replay_range(repo, args.range_expr, out_dir, args.post_merge, args.coverage_db) except subprocess.CalledProcessError as e: print(f"git error: {e.stderr.strip()}", file=sys.stderr) return 1 @@ -502,7 +539,7 @@ def main(argv: Optional[list[str]] = None) -> int: ) for sha in shas: try: - row = _replay_one(repo, sha, out_dir, args.post_merge) + row = _replay_one(repo, sha, out_dir, args.post_merge, args.coverage_db) except subprocess.CalledProcessError as e: print(f" {sha[:8]}: git error: {e.stderr.strip()}", file=sys.stderr) continue diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index dc5127e57343..144a0a186fa1 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -76,6 +76,15 @@ def full(stage) -> int: return 0, 0 +def _fmt_reason(r) -> str: + """Render a structured reason dict as one human line: `[source] k=v, ...`.""" + if not isinstance(r, dict): + return str(r) + src = r.get("source", "?") + rest = ", ".join(f"{k}={v}" for k, v in r.items() if k != "source") + return f"[{src}] {rest}" if rest else f"[{src}]" + + def build_document( decision: dict, status: str, @@ -89,7 +98,7 @@ def build_document( affected = sorted(decision.get("affected_stages") or []) # deferred has no decision; fall back to --reason. if not reason: - reason = " | ".join(decision.get("reasons") or []) + reason = " | ".join(_fmt_reason(r) for r in decision.get("reasons") or []) case_skip_rate = (1 - cbts_cases / total_cases) if total_cases else 0.0 From 8a9ea5633db7bf46352847a9f2eb0108d90a85c1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:53:26 +0800 Subject: [PATCH 02/35] [TRTLLM-12838][infra] CBTS: key coverage selection on stage family Coverage is captured per pytest-split shard, but pytest-split assigns each entry to exactly one shard and rebalances by duration across runs, so a single shard's capture set answers "was this entry captured on this shard", not "was it ever captured on this stage". Key selection on the stage family (shard suffix stripped, shards unioned) instead: add stage_family() and known_by_family() in touch_db, untrusted_families() in the selector, and map each block's served stages through stage_family in coverage_tier. Also make the no-rows case configurable. A changed function with no DB rows was never observed, which is not the same as "no test reaches it", so --no-data-policy selects the fallback: 'file' force-runs every test entering that file (default, tightest sound bound), 'importers' only the file's touch set, 'ignore' treats it as impacting nothing. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/selector.py | 57 ++++++++++++++++--- .../cbts/coverage_selection/touch_db.py | 27 +++++++++ jenkins/scripts/cbts/coverage_tier.py | 52 +++++++++++------ jenkins/scripts/cbts/main.py | 21 ++++++- 4 files changed, 129 insertions(+), 28 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 7123004682da..a5bf65365f26 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -19,7 +19,6 @@ from pathlib import Path from qualname_map import qualnames_for_lines -from rules._helpers import iter_diff_post_line_numbers from touch_db import ( _LAUNCH_MARKERS, _MIN_FUNCS, @@ -28,22 +27,35 @@ TouchDB, canon, split_stage, + stage_family, ) +from rules._helpers import iter_diff_post_line_numbers + @dataclass class CoverageResult: - """Per-stage coverage decision over a set of residual core-Python files.""" + """Per-stage-family coverage decision over a set of residual core-Python files.""" ok: bool reason: str + # keyed by stage family (shard suffix stripped); see `touch_db.stage_family` impacted: dict[str, set[str]] = field(default_factory=dict) skippable: dict[str, set[str]] = field(default_factory=dict) n_untrusted: int = 0 - # functions with no DB rows (new/uninstrumented); bounded via importers + # functions with no DB rows (new/uninstrumented); bounded per `no_data_policy` no_data_funcs: list[str] = field(default_factory=list) +# What to do with a changed function that has no rows in the DB — it was never +# captured, so "which tests exercise it" is unknown rather than empty. +# file fall back to every test that entered any function in the file +# importers fall back to the file's `` touch set (its importers) +# ignore treat as impacting nothing +NO_DATA_POLICIES = ("file", "importers", "ignore") +DEFAULT_NO_DATA_POLICY = "file" + + class CoverageSelector: def __init__( self, @@ -54,6 +66,7 @@ def __init__( launch_markers: tuple[tuple[str, str], ...] = _LAUNCH_MARKERS, serving_path_markers: tuple[str, ...] = _SERVING_PATH_MARKERS, min_funcs: int = _MIN_FUNCS, + no_data_policy: str = DEFAULT_NO_DATA_POLICY, ) -> None: self.db = db self.repo_root = Path(repo_root) @@ -61,6 +74,7 @@ def __init__( self._launch_markers = launch_markers self._serving_path_markers = serving_path_markers self._min_funcs = min_funcs + self._no_data_policy = no_data_policy self._untrusted: set[str] | None = None def untrusted_tests(self) -> set[str]: @@ -74,6 +88,18 @@ def untrusted_tests(self) -> set[str]: ) return self._untrusted + def untrusted_families(self) -> set[str]: + """`untrusted_tests()` re-keyed to `/`. + + Untrusted on any shard means untrusted for the family — the entry runs. + """ + out: set[str] = set() + for test in self.untrusted_tests(): + stage, nodeid = split_stage(test) + if stage: + out.add(f"{stage_family(stage)}/{nodeid}") + return out + def _impacted_tests( self, residual_files: list[str], diffs: dict[str, str] ) -> tuple[set[str], list[str]]: @@ -96,8 +122,21 @@ def _impacted_tests( impacted |= tests if not tests and qualname != "": no_data.append(f"{cf}::{qualname}") + impacted |= self._no_data_fallback(cf) return impacted, no_data + def _no_data_fallback(self, cf: str) -> set[str]: + """Tests to force-run for a changed function the DB never captured. + + No rows means the function was never observed, not that no test reaches + it, so the file's own test set is the tightest sound bound available. + """ + if self._no_data_policy == "file": + return self.db.tests_touching_file(cf) + if self._no_data_policy == "importers": + return self.db.tests_touching_func(cf, "") + return set() + def _read_head(self, path: str) -> str | None: try: return (self.repo_root / path).read_text() @@ -124,16 +163,16 @@ def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageRe for test in impacted_tests: stage, nodeid = split_stage(test) if stage: - impacted.setdefault(stage, set()).add(nodeid) + impacted.setdefault(stage_family(stage), set()).add(nodeid) - untrusted = self.untrusted_tests() + untrusted = self.untrusted_families() skippable: dict[str, set[str]] = {} n_untrusted = 0 - for stage, known_nodeids in self.db.known_by_stage().items(): - imp = impacted.get(stage, set()) - keep_untrusted = {n for n in known_nodeids if f"{stage}/{n}" in untrusted} + for family, known_nodeids in self.db.known_by_family().items(): + imp = impacted.get(family, set()) + keep_untrusted = {n for n in known_nodeids if f"{family}/{n}" in untrusted} n_untrusted += len(keep_untrusted - imp) - skippable[stage] = known_nodeids - imp - keep_untrusted + skippable[family] = known_nodeids - imp - keep_untrusted return CoverageResult( ok=True, diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py index d6ff12abc686..88214b6d88b4 100644 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -35,6 +35,9 @@ # A DB test value is `/`; unit tests wrap the inner entry. _UNITTEST_WRAP_RE = re.compile(r"::test_unittests_v2\[(?P.+)\]$") +# Trailing `-` of a pytest-split shard name. +_SPLIT_SUFFIX_RE = re.compile(r"-\d+$") + # Completeness-heuristic constants consumed by `untrusted_tests()`. _WORKER_SENTINEL = "tensorrt_llm/_torch/pyexecutor/py_executor.py" _LAUNCH_MARKERS: tuple[tuple[str, str], ...] = ( @@ -57,6 +60,18 @@ def split_stage(test: str) -> tuple[str, str]: return (stage, nodeid) if sep else ("", test) +def stage_family(stage: str) -> str: + """Collapse a pytest-split shard name to its family (`A10-PyTorch-2` -> `A10-PyTorch`). + + Coverage is captured per shard, but pytest-split assigns each entry to + exactly one shard, so a stage's shards hold disjoint capture sets. Only the + family-level union answers "was this entry ever captured on this stage" — + and the shard an entry lands on is not stable across runs anyway, since + pytest-split rebalances by duration. + """ + return _SPLIT_SUFFIX_RE.sub("", stage) + + def unwrap_unittest(nodeid: str) -> Optional[str]: """Return the inner `unittest/...` entry of a wrapped unittest nodeid, else None. @@ -181,6 +196,18 @@ def known_by_stage(self) -> dict[str, set[str]]: out.setdefault(stage, set()).add(nodeid) return out + def known_by_family(self) -> dict[str, set[str]]: + """`{stage family -> {bare nodeid, ...}}` — a stage's shards unioned. + + Selection keys on this rather than `known_by_stage`: see `stage_family`. + """ + out: dict[str, set[str]] = {} + for test in self.known_tests(): + stage, nodeid = split_stage(test) + if stage: + out.setdefault(stage_family(stage), set()).add(nodeid) + return out + # -- forward lookup (debug / explain-why) -- def files_touched_by(self, test: str) -> list[tuple[str, str]]: diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index 06242892299a..1c6b52657058 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -34,11 +34,12 @@ _target_in_filter_subtree, block_matches_stage, ) + from rules.base import PRInputs, RuleResult sys.path.insert(0, str(Path(__file__).resolve().parent / "coverage_selection")) -from selector import CoverageSelector # noqa: E402 -from touch_db import TouchDB, db_key # noqa: E402 +from selector import DEFAULT_NO_DATA_POLICY, NO_DATA_POLICIES, CoverageSelector # noqa: E402,F401 +from touch_db import TouchDB, db_key, stage_family # noqa: E402 def open_db(path: str) -> TouchDB: @@ -88,24 +89,29 @@ def _rule_kept_entries(block, prefix_to_waives: dict[str, set[str]]) -> set[str] def _entry_reason( entry: str, - served: list[str], + served_families: list[str], keep_rule: set[str], cov, known: dict[str, set[str]], untrusted: set[str], ) -> str: - """Return SAFE or the must-run cause for a candidate YAML entry.""" + """Return SAFE or the must-run cause for a candidate YAML entry. + + Keyed by stage family, not by shard: pytest-split puts each entry on + exactly one shard, so a per-shard lookup would report `no_data` for every + entry of any stage split more than one way. + """ if entry in keep_rule: return _R_RULE_KEPT dbk = db_key(entry) if dbk is None: return _R_COARSE - for name in served: - if dbk not in known.get(name, frozenset()): + for family in served_families: + if dbk not in known.get(family, frozenset()): return _R_NO_DATA - if dbk in cov.impacted.get(name, frozenset()): + if dbk in cov.impacted.get(family, frozenset()): return _R_IMPACTED - if f"{name}/{dbk}" in untrusted: + if f"{family}/{dbk}" in untrusted: return _R_UNTRUSTED return _R_SAFE @@ -120,6 +126,9 @@ def _build_narrowing( ) -> tuple[dict[tuple[str, int], set[str]], set[str], Counter]: """Classify every candidate entry; remove only SAFE ones. + `cov.skippable` / `cov.impacted` / `known` / `untrusted` are keyed by stage + family, so each block's served stages are mapped through `stage_family`. + Returns (removed per block, fully-emptied instrumented stages, must-run tally). """ instrumented = set(cov.skippable) @@ -137,14 +146,15 @@ def _build_narrowing( for s in stages.values() if s.yaml_stem == block.yaml_stem and block_matches_stage(block, s) ] + served_families = sorted({stage_family(name) for name in served}) # shared-block rule: prune only when every served stage is instrumented - if not served or any(name not in instrumented for name in served): + if not served or any(f not in instrumented for f in served_families): continue key = (block.yaml_stem, block.block_index) keep_rule = rule_kept.get(key, set()) rm: set[str] = set() for entry in block.tests: - reason = _entry_reason(entry, served, keep_rule, cov, known, untrusted) + reason = _entry_reason(entry, served_families, keep_rule, cov, known, untrusted) if reason == _R_SAFE: rm.add(entry) else: @@ -153,9 +163,8 @@ def _build_narrowing( removed[key] = rm dropped: set[str] = set() - for name in instrumented: - stage = stages.get(name) - if stage is None: + for name, stage in stages.items(): + if stage_family(name) not in instrumented: continue total = kept = 0 for block in yaml_index.blocks: @@ -179,6 +188,7 @@ def apply_coverage_tier( yaml_index: YAMLIndex, repo_root: Path, db: TouchDB, + no_data_policy: str = DEFAULT_NO_DATA_POLICY, ) -> tuple[CoverageTierResult | None, str]: """Return (narrowing, note); narrowing is None when the tier keeps the Tier-1 result.""" if any(r.scope is None for _, r in pairs): @@ -187,7 +197,7 @@ def apply_coverage_tier( if not residual: return None, "coverage tier skipped: no residual (all files handled by rules)" - selector = CoverageSelector(db, repo_root) + selector = CoverageSelector(db, repo_root, no_data_policy=no_data_policy) cov = selector.decide(residual, pr.diffs) if not cov.ok: return None, f"coverage tier declined: {cov.reason}" @@ -203,10 +213,15 @@ def apply_coverage_tier( if cov.no_data_funcs: shown = ", ".join(cov.no_data_funcs[:3]) more = f" (+{len(cov.no_data_funcs) - 3})" if len(cov.no_data_funcs) > 3 else "" - nd = f"; new/uncovered function(s), bounded via importers: {shown}{more}" - - known = db.known_by_stage() - untrusted = selector.untrusted_tests() + bound = { + "file": "bounded to each file's whole test set", + "importers": "bounded via importers", + "ignore": "NOT bounded", + }[no_data_policy] + nd = f"; new/uncovered function(s), {bound}: {shown}{more}" + + known = db.known_by_family() + untrusted = selector.untrusted_families() removed, dropped, must_run_reasons = _build_narrowing( cov, stages, yaml_index, rule_block_filters, known, untrusted ) @@ -244,6 +259,7 @@ def apply_coverage_tier( "removed_cases": n_removed, "dropped_stages": len(dropped), "outcome": "narrowed" if narrowed else "nothing_removable", + "no_data_policy": no_data_policy, **({"no_data_funcs": list(cov.no_data_funcs)} if cov.no_data_funcs else {}), }, ) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index e261a751d63a..e86b91f3dadf 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -49,11 +49,14 @@ write_filtered_test_db, ) from coverage_tier import ( # noqa: E402 + DEFAULT_NO_DATA_POLICY, + NO_DATA_POLICIES, apply_coverage_tier, compute_coverage_stage_counts, open_db, write_coverage_test_db, ) + from rules._helpers import strip_noop_diff_lines # noqa: E402 from rules.agent_flow_rule import AgentFlowRule # noqa: E402 from rules.auto_deploy_rule import AutoDeployRule # noqa: E402 @@ -338,6 +341,15 @@ def main(argv: Optional[list[str]] = None) -> int: help="Path to cbts_touchmap.sqlite. When set, the coverage tier (Tier 2) " "runs on fallbacks and may drop fully-safe single-GPU stages.", ) + parser.add_argument( + "--no-data-policy", + choices=NO_DATA_POLICIES, + default=DEFAULT_NO_DATA_POLICY, + help="How to treat a changed function with no rows in the coverage DB: " + "'file' force-runs every test entering that file, 'importers' only the " + "file's touch set, 'ignore' treats it as impacting nothing " + f"(default: {DEFAULT_NO_DATA_POLICY}).", + ) args = parser.parse_args(argv) if args.list_needed_diffs: @@ -389,7 +401,14 @@ def main(argv: Optional[list[str]] = None) -> int: try: db = open_db(args.coverage_db) tier, note = apply_coverage_tier( - pr, selector.pairs, selector.handled, stages, yaml_index, repo_root, db + pr, + selector.pairs, + selector.handled, + stages, + yaml_index, + repo_root, + db, + no_data_policy=args.no_data_policy, ) except Exception as e: # noqa: BLE001 — CBTS must never break CI note = f"coverage tier errored: {e}" From 170b2de3f3f17e2dfa731458095c1beb871c25a5 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:57:00 +0800 Subject: [PATCH 03/35] [TRTLLM-12838][infra] CBTS: treat Ray stages as untrusted coverage sitecustomize opts Ray infra processes out of instrumentation, so a Ray stage's RayGPUWorker runs inside an uninstrumented default_worker.py and its tests record only the driver-side footprint. Those tests still look complete to the existing heuristics (they pass the worker-sentinel and near-empty checks), so the selector could skip them for a change their uncaptured worker path exercises. Add _UNTRUSTED_STAGE_MARKERS and match it against the stage name in untrusted_tests(), so every test on a "-Ray-" stage is forced to run. The marker matches the pipeline's own convention (L0_Test.groovy gates --run-ray on the same substring). On the 2026-07-30 touch DB this moves untrusted from 78 to 89 of 746 known tests; all 26 Ray-stage tests are covered, including three accuracy tests that carried 17-19% of the footprint their same-nodeid runs on other stages recorded. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/selector.py | 4 ++++ .../cbts/coverage_selection/touch_db.py | 19 ++++++++++++++++--- jenkins/scripts/cbts/tools/coverage_audit.py | 9 ++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index a5bf65365f26..885b1e23fc09 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -23,6 +23,7 @@ _LAUNCH_MARKERS, _MIN_FUNCS, _SERVING_PATH_MARKERS, + _UNTRUSTED_STAGE_MARKERS, _WORKER_SENTINEL, TouchDB, canon, @@ -66,6 +67,7 @@ def __init__( launch_markers: tuple[tuple[str, str], ...] = _LAUNCH_MARKERS, serving_path_markers: tuple[str, ...] = _SERVING_PATH_MARKERS, min_funcs: int = _MIN_FUNCS, + untrusted_stage_markers: tuple[str, ...] = _UNTRUSTED_STAGE_MARKERS, no_data_policy: str = DEFAULT_NO_DATA_POLICY, ) -> None: self.db = db @@ -74,6 +76,7 @@ def __init__( self._launch_markers = launch_markers self._serving_path_markers = serving_path_markers self._min_funcs = min_funcs + self._untrusted_stage_markers = untrusted_stage_markers self._no_data_policy = no_data_policy self._untrusted: set[str] | None = None @@ -85,6 +88,7 @@ def untrusted_tests(self) -> set[str]: self._launch_markers, self._serving_path_markers, self._min_funcs, + self._untrusted_stage_markers, ) return self._untrusted diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py index 88214b6d88b4..ab7153aee9c3 100644 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -45,6 +45,10 @@ ("tensorrt_llm/executor/executor.py", "GenerationExecutor.generate"), ) _SERVING_PATH_MARKERS: tuple[str, ...] = ("disaggregated/",) +# Stage-name markers whose capture is structurally partial regardless of footprint: +# `sitecustomize` opts Ray infra processes out, so a Ray stage's GPU worker lives in +# an uninstrumented `default_worker.py` and its tests carry only the driver's rows. +_UNTRUSTED_STAGE_MARKERS: tuple[str, ...] = ("-Ray-",) _MIN_FUNCS = 30 @@ -225,13 +229,15 @@ def untrusted_tests( launch_markers: tuple[tuple[str, str], ...], serving_path_markers: tuple[str, ...], min_funcs: int, + untrusted_stage_markers: tuple[str, ...] = (), ) -> set[str]: """Stage-prefixed tests whose per-test capture looks incomplete (must always run). Flags a test that drove execution/serving but is missing `worker_file` — matched by a `launch_markers` `(file, qualname_substring)` call or a - `serving_path_markers` nodeid substring — or that entered fewer than - `min_funcs` functions total. + `serving_path_markers` nodeid substring — that entered fewer than + `min_funcs` functions total, or that ran on a stage whose name contains an + `untrusted_stage_markers` substring. """ drove_execution: set[str] = set() for file, qual_substr in launch_markers: @@ -256,4 +262,11 @@ def untrusted_tests( (min_funcs,), ) } - return missing_worker | tiny + on_untrusted_stage: set[str] = set() + if untrusted_stage_markers: + on_untrusted_stage = { + test + for test in self.known_tests() + if any(marker in split_stage(test)[0] for marker in untrusted_stage_markers) + } + return missing_worker | tiny | on_untrusted_stage diff --git a/jenkins/scripts/cbts/tools/coverage_audit.py b/jenkins/scripts/cbts/tools/coverage_audit.py index b9e2309b6152..c4a61d049f62 100644 --- a/jenkins/scripts/cbts/tools/coverage_audit.py +++ b/jenkins/scripts/cbts/tools/coverage_audit.py @@ -40,6 +40,7 @@ _LAUNCH_MARKERS, _MIN_FUNCS, _SERVING_PATH_MARKERS, + _UNTRUSTED_STAGE_MARKERS, _WORKER_SENTINEL, TouchDB, db_key, @@ -115,10 +116,16 @@ def main(argv: list[str] | None = None) -> int: # -- Completeness -- untrusted = db.untrusted_tests( - _WORKER_SENTINEL, _LAUNCH_MARKERS, _SERVING_PATH_MARKERS, args.min_funcs + _WORKER_SENTINEL, + _LAUNCH_MARKERS, + _SERVING_PATH_MARKERS, + args.min_funcs, + _UNTRUSTED_STAGE_MARKERS, ) def reason(test: str) -> str: + if any(m in split_stage(test)[0] for m in _UNTRUSTED_STAGE_MARKERS): + return "untrusted stage (GPU worker uninstrumented)" if any(m in test for m in _SERVING_PATH_MARKERS): return "disagg-path (servers uninstrumented)" if footprint[test] < args.min_funcs: From f65363cffc10ea4364908d33f3a2148aae7e8581 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:59:58 +0800 Subject: [PATCH 04/35] [TRTLLM-12838][infra] CBTS: record which coverage DB build a decision used The touch DB is resolved as "the latest post-merge tarball that exists" at decision time, with no tie to the PR commit. Post-merge runs several times a day, so two /bot run invocations on the same commit routinely consult different DBs and can narrow to different test sets. Nothing recorded which one was used, leaving a decision impossible to replay. Resolve the URL once (it is already fetched for the audit) and pass it to main.py, which extracts the build number via artifact.build_from_url() and puts it in the decision as coverage_db_build; report_cbts_decision posts it as l_coverage_db_build (0 when no DB was consulted). Passing the URL rather than re-resolving avoids recording a build other than the one actually used. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 14 ++++++++------ .../scripts/cbts/coverage_selection/artifact.py | 7 +++++++ jenkins/scripts/cbts/main.py | 17 +++++++++++++++++ .../scripts/cbts/tools/report_cbts_decision.py | 2 ++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 93505a318b42..5c720265febf 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -847,7 +847,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) sh "apt-get update -qq && apt-get install -y -qq python3-yaml" // Download the touch DB for audit + Tier 2 coverage-based narrowing. - def coverageDbPath = _cbtsCoverageAudit(pipeline) + def coverageDb = _cbtsCoverageAudit(pipeline) // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -873,8 +873,8 @@ def getCbtsResult(pipeline, testFilter, globalVars) writeFile file: inputPath, text: inputJson def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" - if (coverageDbPath) { - mainCmd += " --coverage-db ${coverageDbPath}" + if (coverageDb.path) { + mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-url '${coverageDb.url}'" } def output = sh(script: mainCmd, returnStdout: true) @@ -931,7 +931,7 @@ def _cbtsCoverageAudit(pipeline) ).trim() if (!url) { pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") - return "" + return [path: "", url: ""] } sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" // wget the tarball (retrying) and extract the sqlite. @@ -940,12 +940,14 @@ def _cbtsCoverageAudit(pipeline) "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + "--db ${covDir}/cbts_touchmap.sqlite" - return "${covDir}/cbts_touchmap.sqlite" + // url rides along so main.py can record which post-merge build the DB + // came from ("latest" is resolved here, once, per run). + return [path: "${covDir}/cbts_touchmap.sqlite", url: url] } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - return "" + return [path: "", url: ""] } } diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index 74bb0159fffb..73effa7fe8ab 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -30,6 +30,7 @@ import argparse import json +import re import shutil import sys import tarfile @@ -91,6 +92,12 @@ def tarball_url(build: int, artifact_base: str = ARTIFACT_BASE) -> str: return f"{_URM}/{artifact_base}/{build}/cbts-coverage/{TARBALL_NAME}" +def build_from_url(url: str) -> Optional[int]: + """Post-merge build number encoded in a `tarball_url()`, or None if absent.""" + m = re.search(r"/(\d+)/cbts-coverage/", url or "") + return int(m.group(1)) if m else None + + def latest_tarball_url( artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index e86b91f3dadf..4d3ee71ba314 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -38,7 +38,9 @@ # Make sibling modules importable when invoked as `python3 /main.py ...`. sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent / "coverage_selection")) +from artifact import build_from_url # noqa: E402 from blocks import ( # noqa: E402 Stage, YAMLIndex, @@ -128,6 +130,10 @@ class SelectionResult: # set by coverage tier; Groovy re-adds multiGpuJobs under MULTI_GPU_FILE_CHANGED gate enable_multi_gpu: bool = False coverage_dropped_stages: list[str] = field(default_factory=list) + # Post-merge build the consulted touch DB came from. The DB is resolved as + # "latest post-merge tarball" at decision time, so two runs of the same + # commit can consult different DBs; recording it makes a decision replayable. + coverage_db_build: Optional[int] = None def to_json(self) -> str: data = { @@ -142,6 +148,7 @@ def to_json(self) -> str: "perfsanity_required": self.perfsanity_required, "enable_multi_gpu": self.enable_multi_gpu, "coverage_dropped_stages": sorted(self.coverage_dropped_stages), + "coverage_db_build": self.coverage_db_build, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -341,6 +348,13 @@ def main(argv: Optional[list[str]] = None) -> int: help="Path to cbts_touchmap.sqlite. When set, the coverage tier (Tier 2) " "runs on fallbacks and may drop fully-safe single-GPU stages.", ) + parser.add_argument( + "--coverage-db-url", + default=None, + help="Artifactory URL the --coverage-db tarball was fetched from. Only its " + "post-merge build number is used, recorded in the decision as " + "coverage_db_build so a decision can be traced back to its DB.", + ) parser.add_argument( "--no-data-policy", choices=NO_DATA_POLICIES, @@ -396,6 +410,9 @@ def main(argv: Optional[list[str]] = None) -> int: selector = Selector(stages) result = selector.run(pr, rules) + if args.coverage_db_url: + result.coverage_db_build = build_from_url(args.coverage_db_url) + if args.coverage_db and result.scope is None: note = "" try: diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index 144a0a186fa1..736ee14120a1 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -113,6 +113,8 @@ def build_document( "l_hit_stages": len(affected), "l_total_cases": total_cases, "l_cbts_cases": cbts_cases, + # Post-merge build of the consulted touch DB; 0 when no DB was used. + "l_coverage_db_build": int(decision.get("coverage_db_build") or 0), "d_case_skip_rate": round(case_skip_rate, 4), "flat_detail": { "hit_stages": affected, From 3edc41a3fa350f7714c96117509bb94b8ca72c5b Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:40:34 +0800 Subject: [PATCH 05/35] [TRTLLM-12838][infra] CBTS: align coverage_explain with selector safety gates The explain tool re-derived the selection instead of mirroring CoverageSelector.decide(), so it listed cases as removable that the selector would in fact run. Its output backs cbts_removed_cases.txt, so every divergence read as a case being safely skipped when it was not. Three gates were missing, all erring the same way (over-reporting removals): a changed file with no DB rows makes decide() refuse the whole change; untrusted tests are subtracted from skippable; and a changed function with no rows falls back to the file's test set under no_data_policy="file". Add all three: refuse-and-stop on a zero-touch file, a FORCED-KEPT section for untrusted cases (with a forced=N count in the per-stage header), and the file-level fallback with the no-data functions listed. Non-core files in the commit are now named too, since Tier-1 rules claim them and any left as residual also makes the tier refuse. Verified against the 2026-07-30 touch DB: for a py_executor.py change the tool and CoverageSelector.decide() now agree exactly (334 removable, 82 forced); a commit adding tensorrt_llm/_torch/configs/gemma4.py correctly reports the refusal instead of a removal list. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../scripts/cbts/tools/coverage_explain.py | 79 +++++++++++++++++-- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/jenkins/scripts/cbts/tools/coverage_explain.py b/jenkins/scripts/cbts/tools/coverage_explain.py index 8baa894994ec..f25ffb404e44 100644 --- a/jenkins/scripts/cbts/tools/coverage_explain.py +++ b/jenkins/scripts/cbts/tools/coverage_explain.py @@ -15,9 +15,13 @@ r"""Explain a coverage-selection decision for one commit. For each instrumented stage, prints why each known case is kept (it entered a -changed function) or removed (it is in the DB but never entered any changed -function). The justification is the forward touch lookup — the audit view that -makes `cbts_removed_cases.txt` self-verifying. +changed function), forced-kept (its capture is untrusted), or removed (it is in +the DB, entered no changed function, and its capture is trusted). The +justification is the forward touch lookup — the audit view that makes +`cbts_removed_cases.txt` self-verifying. + +Mirrors `CoverageSelector.decide()`'s safety gates: a changed file with no DB +rows refuses the whole change, and untrusted tests are never removable. Example:: @@ -38,8 +42,19 @@ sys.path.insert(0, str(CBTS / "coverage_selection")) from qualname_map import qualnames_for_lines # noqa: E402 +from touch_db import ( # noqa: E402 + _LAUNCH_MARKERS, + _MIN_FUNCS, + _SERVING_PATH_MARKERS, + _UNTRUSTED_STAGE_MARKERS, + _WORKER_SENTINEL, + TouchDB, + canon, + split_stage, + stage_family, +) + from rules._helpers import iter_diff_post_line_numbers # noqa: E402 -from touch_db import TouchDB, canon, split_stage # noqa: E402 def _git(repo: Path, *args: str, check: bool = True) -> str: @@ -75,6 +90,17 @@ def main(argv: list[str] | None = None) -> int: if ln.strip() ] core = [f for f in files if f.endswith(".py") and canon(f).startswith("tensorrt_llm/")] + non_core = [f for f in files if f not in core] + + # `CoverageSelector.decide()` refuses the whole decision when a residual file has + # no rows, so a removed list computed past that point would not be reachable. + zero_touch = [f for f in core if not db.file_has_touch_rows(canon(f))] + if zero_touch: + print(f"commit {args.sha[:12]} — coverage selection REFUSES this change:") + for f in zero_touch: + print(f" zero-touch residual file (new/uninstrumented): {f}") + print("\nNo case is removable; every stage runs in full.") + return 0 # Build the change's impact set: function-level (file, qualname), or (file, None) # when a file falls back to file-level. Collect the impacted tests. @@ -82,6 +108,7 @@ def main(argv: list[str] | None = None) -> int: impact_files: set[str] = set() # file-level fallback changed_files: set[str] = set() impacted: set[str] = set() + no_data: list[str] = [] for f in core: cf = canon(f) changed_files.add(cf) @@ -99,13 +126,42 @@ def main(argv: list[str] | None = None) -> int: continue for q in qns: impact_funcs.add((cf, q)) - impacted |= db.tests_touching_func(cf, q) + tests = db.tests_touching_func(cf, q) + impacted |= tests + # Mirrors the selector's default no_data_policy="file": a function with no + # rows was never observed, so the file's own test set is the bound used. + if not tests and q != "": + no_data.append(f"{cf}::{q}") + impact_files.add(cf) + impacted |= db.tests_touching_file(cf) print(f"commit {args.sha[:12]} — {len(core)} core file(s), impact set:") for cf, q in sorted(impact_funcs): print(f" {cf} :: {q}") for cf in sorted(impact_files): print(f" {cf} :: ") + if no_data: + print(f" ({len(no_data)} changed function(s) with no DB rows -> file-level fallback)") + for s in sorted(no_data): + print(f" no-data: {s}") + if non_core: + print( + f"\n NOTE: {len(non_core)} non-core file(s) in this commit are not evaluated here. " + "Tier-1 rules claim them; any left as residual makes the coverage tier refuse." + ) + for f in sorted(non_core): + print(f" {f}") + + # Untrusted capture is forced to run by the selector; it is never removable. + untrusted = db.untrusted_tests( + _WORKER_SENTINEL, + _LAUNCH_MARKERS, + _SERVING_PATH_MARKERS, + _MIN_FUNCS, + _UNTRUSTED_STAGE_MARKERS, + ) + # Keyed by family: untrusted on any shard means untrusted for the family. + untrusted_fam = {f"{stage_family(s)}/{n}" for s, n in map(split_stage, untrusted) if s} impacted_by_stage: dict[str, set[str]] = {} for t in impacted: @@ -125,13 +181,22 @@ def entered_changed(nodeid: str, stage: str) -> tuple[int, int, list[str]]: continue known_s = db.known_by_stage()[stage] imp_s = impacted_by_stage.get(stage, set()) & known_s - skip_s = known_s - imp_s - print(f"\n=== {stage} known={len(known_s)} kept={len(imp_s)} removed={len(skip_s)} ===") + fam = stage_family(stage) + forced_s = {n for n in known_s - imp_s if f"{fam}/{n}" in untrusted_fam} + skip_s = known_s - imp_s - forced_s + print( + f"\n=== {stage} known={len(known_s)} kept={len(imp_s)} " + f"forced={len(forced_s)} removed={len(skip_s)} ===" + ) if args.show_kept and imp_s: print(" KEPT (impacted):") for n in sorted(imp_s): _, _, hits = entered_changed(n, stage) print(f" {n}\n entered: {', '.join(hits) or '(file-level)'}") + if forced_s: + print(" FORCED-KEPT (untrusted capture; not impacted but never removable):") + for n in sorted(forced_s): + print(f" {n}") print(" REMOVED (safe to skip):") for n in sorted(skip_s): total, in_changed, _ = entered_changed(n, stage) From 8db2bd2ea365bcd833ca2180cf240f80ba80fda6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:40:46 +0800 Subject: [PATCH 06/35] [TRTLLM-12838][infra] CBTS: drop unused RuleResult.detail RuleResult.detail let a rule attach typed key/value pairs that _rule_reason merged into the decision's reasons entry, but only two of the seven rules ever populated it. Structured reasons are their own change; carrying a partially-adopted field here leaves the rules/ tree touched for a branch that is otherwise about coverage-based selection. Remove the field, the two payloads that used it (waives, testdef), and the **r.detail unpack in _rule_reason. Reasons keep {source, blocks, stages}. coverage_tier's own detail is a different dataclass and is unaffected -- the [coverage] reason still carries its full breakdown. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 3 +-- jenkins/scripts/cbts/rules/base.py | 1 - jenkins/scripts/cbts/rules/tests_def_rule.py | 1 - jenkins/scripts/cbts/rules/waives_rule.py | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 4d3ee71ba314..1d14ad3e58a3 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -154,10 +154,9 @@ def to_json(self) -> str: def _rule_reason(rule, r) -> dict: - """One rule's structured reason entry: `{source, **detail, blocks, stages}`.""" + """One rule's structured reason entry: `{source, blocks, stages}`.""" return { "source": rule.name, - **r.detail, "blocks": len(r.block_filters), "stages": len(r.affected_stages), } diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 6c3d24c9c013..f4f71d0335e2 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -50,7 +50,6 @@ class RuleResult: # True (safe default) iff this rule's matched changes might affect perf # benchmarks. Set False when matched changes are pure test infra. perfsanity_relevant: bool = True - detail: dict[str, object] = field(default_factory=dict) class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/tests_def_rule.py b/jenkins/scripts/cbts/rules/tests_def_rule.py index 56526c19e27e..9f85039f5e01 100644 --- a/jenkins/scripts/cbts/rules/tests_def_rule.py +++ b/jenkins/scripts/cbts/rules/tests_def_rule.py @@ -339,5 +339,4 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: f"{len(block_filters)} blocks, {len(affected_stages)} stages" f"{nonarrow_note}" ), - detail={"paths": len(narrowed)}, ) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 976143a58fc2..184193c53600 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -116,5 +116,4 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: f"waives.txt: +{len(added)} / -{len(removed)} → " f"{len(block_filters)} blocks, {len(affected_stages)} stages{miss_note}" ), - detail={"added": len(added), "removed": len(removed)}, ) From 885d3e4ff2cd0d2575e006a05e190a7f949672e3 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:58:47 +0800 Subject: [PATCH 07/35] [TRTLLM-12838][infra] CBTS: fix import order in the coverage selection modules `rules` is a third-party module for ruff's isort in a clean checkout, so the `from rules...` imports belong in the same block as `qualname_map`/`touch_db` rather than in a section of their own. Local runs can classify `rules` as first-party and split it out when an unrelated `rules/` directory happens to sit at the repo root, which is what produced the current ordering; CI, which has no such directory, rejects it. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/coverage_selection/selector.py | 3 +-- jenkins/scripts/cbts/coverage_tier.py | 1 - jenkins/scripts/cbts/main.py | 1 - jenkins/scripts/cbts/tools/coverage_explain.py | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 885b1e23fc09..1f1d837044dc 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -19,6 +19,7 @@ from pathlib import Path from qualname_map import qualnames_for_lines +from rules._helpers import iter_diff_post_line_numbers from touch_db import ( _LAUNCH_MARKERS, _MIN_FUNCS, @@ -31,8 +32,6 @@ stage_family, ) -from rules._helpers import iter_diff_post_line_numbers - @dataclass class CoverageResult: diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index 1c6b52657058..50ee4b037b25 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -34,7 +34,6 @@ _target_in_filter_subtree, block_matches_stage, ) - from rules.base import PRInputs, RuleResult sys.path.insert(0, str(Path(__file__).resolve().parent / "coverage_selection")) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 1d14ad3e58a3..4873ffe5fc79 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -58,7 +58,6 @@ open_db, write_coverage_test_db, ) - from rules._helpers import strip_noop_diff_lines # noqa: E402 from rules.agent_flow_rule import AgentFlowRule # noqa: E402 from rules.auto_deploy_rule import AutoDeployRule # noqa: E402 diff --git a/jenkins/scripts/cbts/tools/coverage_explain.py b/jenkins/scripts/cbts/tools/coverage_explain.py index f25ffb404e44..61d1c06b4bf2 100644 --- a/jenkins/scripts/cbts/tools/coverage_explain.py +++ b/jenkins/scripts/cbts/tools/coverage_explain.py @@ -42,6 +42,7 @@ sys.path.insert(0, str(CBTS / "coverage_selection")) from qualname_map import qualnames_for_lines # noqa: E402 +from rules._helpers import iter_diff_post_line_numbers # noqa: E402 from touch_db import ( # noqa: E402 _LAUNCH_MARKERS, _MIN_FUNCS, @@ -54,8 +55,6 @@ stage_family, ) -from rules._helpers import iter_diff_post_line_numbers # noqa: E402 - def _git(repo: Path, *args: str, check: bool = True) -> str: return subprocess.run( From f6945f192f856242366195732f53f68fd03edbca Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:55:38 +0800 Subject: [PATCH 08/35] [TRTLLM-12838][infra] CBTS: always run CPU stages, and fix their gpu-count mako Two coupled changes. blocks.py mirrored renderTestDB's mako derivation but missed the CPU special case: renderTestDB (jenkins/L0_Test.groovy:3465) hardcodes system_gpu_count=0 for CPU- stages, while the mirror left the default of 1 because the stage name carries no N_GPUs token. It also lacked the -Generic- backend mapping. Both CPU stages therefore matched no block in l0_cpu.yml, so their 32 entries were invisible to the coverage tier -- never narrowed, and never reported as served. With the fix every one of the 130 parsed stages now matches a block, up from 128. That fix alone would let the tier prune CPU entries, including down to an empty list, so the second change makes "CPU stages always run" explicit rather than an accident of the previous mismatch. L0_Test.groovy Layer 2 keeps any CBTS_ALWAYS_RUN_STAGE_PREFIX stage regardless of the decision, alongside the existing PackageSanityCheck / PerfSanity carve-outs, and coverage_tier drops those families from `instrumented` so every block they serve stays intact and the stage never renders an empty test list. Replaying this PR's own diff against the 2026-07-30 touch DB: the decision is unchanged (scope=coverage, 41 affected stages, 248 removed cases) and CPU entries stay at 32; with the always-run constraint disabled the same replay prunes them to 30. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 9 +++++++++ jenkins/scripts/cbts/coverage_tier.py | 6 +++++- jenkins/scripts/cbts/main.py | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 9ce689073682..034b20fe8d63 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -435,12 +435,17 @@ def _classify_map_var(var_name: str) -> Optional[str]: # Backend name -> mako value. Same patterns as getMakoArgsFromStageName in # jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. +# Stages CBTS always runs, whatever the decision: their entries stay in the +# test-db and they are added to every tier's affected_stages. +ALWAYS_RUN_STAGE_PREFIX = "CPU-" + _BACKEND_PATTERNS = [ ("-PyTorch-", "pytorch"), ("-CPP-", "cpp"), ("-Triton-", "triton"), ("-FMHA-", "fmha"), ("-AutoDeploy-", "autodeploy"), + ("-Generic-", "generic"), ("-Verl-", "verl"), ] @@ -488,6 +493,10 @@ def derive_mako_from_stage(stage_name: str) -> dict[str, str]: mako["gpu"] = gpu_match.group(1).lower() count_match = _GPU_COUNT_RE.search(stage_name) mako["system_gpu_count"] = count_match.group(1) if count_match else "1" + # CPU stages carry no `N_GPUs` token but run on 0-GPU machines; renderTestDB + # (jenkins/L0_Test.groovy:3465) hardcodes system_gpu_count=0 for them. + if stage_name.startswith("CPU-"): + mako["system_gpu_count"] = "0" return mako diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index 50ee4b037b25..41bc34708b1f 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -24,6 +24,7 @@ import yaml from blocks import ( + ALWAYS_RUN_STAGE_PREFIX, TARGET_SHARD_SECONDS, Stage, YAMLIndex, @@ -130,7 +131,10 @@ def _build_narrowing( Returns (removed per block, fully-emptied instrumented stages, must-run tally). """ - instrumented = set(cov.skippable) + # Always-run families are dropped from `instrumented` rather than filtered + # later: the shared-block rule below prunes a block only when every served + # stage is instrumented, so this keeps every block they serve intact. + instrumented = {f for f in cov.skippable if not f.startswith(ALWAYS_RUN_STAGE_PREFIX)} rule_kept = { key: _rule_kept_entries(b, rule_block_filters[key]) for b in yaml_index.blocks diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 4873ffe5fc79..7a6b49184939 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -42,6 +42,7 @@ from artifact import build_from_url # noqa: E402 from blocks import ( # noqa: E402 + ALWAYS_RUN_STAGE_PREFIX, Stage, YAMLIndex, compute_stage_split_counts, @@ -490,6 +491,12 @@ def main(argv: Optional[list[str]] = None) -> int: durations=durations, ) + # Stages that run whatever the decision. Added before the trigger-mode filter + # so a Post-Merge one is still dropped in pre-merge, and left out of the + # per-stage counts so Layer 2.5 neither renames nor resizes them. + if result.scope is not None: + result.affected_stages |= {s for s in stages if s.startswith(ALWAYS_RUN_STAGE_PREFIX)} + # Trigger-mode filter; recompute derived counts. pre-merge drops Post-Merge # stages; post-merge keeps both (adds Post-Merge on top, matching baseline). pre_filter_stages = set(result.affected_stages) From 45cd2c2491c77950e4a6ede4a54eca49d25d74c1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:45:25 +0800 Subject: [PATCH 09/35] [TRTLLM-12838][infra] CBTS: bound import-executed changes by file, or decline Module bodies, class bodies, and signature/decorator lines all attribute to a qualname whose code runs once per process, at import. A test only holds a row for such a qualname if its process was already being recorded at that moment, which excludes both every test served by an MPI pool worker (capture starts after the framework import settles) and every in-process import done during collection (recorded under the empty context and filtered out). The rows are a strict subset of the tests that actually executed the code, so narrowing on them alone drops the difference on every run, silently. Measured on the 2026-07-30 touch DB: llmapi/llm_args.py:: has 509 recorded holders out of 746 known tests, while 735 hold a row somewhere in that file -- the 226 missing ones (123 accuracy, 29 disagg) each recorded other qualnames in llm_args.py, proving their process did import and execute it. Resolve such a qualname against the file's own row set, which spans the gap whenever the process later entered any function in the file. When it does not -- 881 of 1149 files, mostly __init__.py and other import-only modules, where the file row set equals the qualname's -- no sound impact set exists, so decide() declines and the change runs full instead. Replaying the 2026-07-21..08-04 window (372 commits) against that DB: of the 109 commits the coverage tier narrowed, 68 still narrow, 40 decline and 1 falls to a rule tier; coverage-tier skip goes 25.5% -> 15.6% and whole-window skip 45.8% -> 42.9%. Declining every import-executed change instead, without the file-level step, leaves only 30 narrowing and drops coverage-tier skip to 12.2%. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/qualname_map.py | 28 ++++++++++ .../cbts/coverage_selection/selector.py | 51 ++++++++++++++++--- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/qualname_map.py b/jenkins/scripts/cbts/coverage_selection/qualname_map.py index 1ed3f6236cb3..54cb5d996652 100644 --- a/jenkins/scripts/cbts/coverage_selection/qualname_map.py +++ b/jenkins/scripts/cbts/coverage_selection/qualname_map.py @@ -84,3 +84,31 @@ def qualnames_for_lines(source: str, lines: set[int]) -> tuple[set[str], bool]: return set(), False scopes = _collect_scopes(tree) return {_attribute(ln, scopes) for ln in lines}, True + + +def import_executed_qualnames(source: str) -> set[str]: + """Qualnames whose code runs once, at import: `` and every class body. + + A signature or decorator line attributes to its *enclosing* scope (see + `_attribute`), so a method's `def` line lands on the class and is covered by + this set too. Unparsable source yields just ``. + """ + out = {""} + + def walk(stmts, prefix: str) -> None: + for node in stmts: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + qual = prefix + node.name + if isinstance(node, ast.ClassDef): + out.add(qual) + walk(node.body, qual + ".") + else: + walk(node.body, qual + "..") + else: + walk(list(_substatements(node)), prefix) + + try: + walk(ast.parse(source).body, "") + except SyntaxError: + pass + return out diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 1f1d837044dc..a18f4b00b3b2 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -18,7 +18,7 @@ from dataclasses import dataclass, field from pathlib import Path -from qualname_map import qualnames_for_lines +from qualname_map import import_executed_qualnames, qualnames_for_lines from rules._helpers import iter_diff_post_line_numbers from touch_db import ( _LAUNCH_MARKERS, @@ -105,8 +105,12 @@ def untrusted_families(self) -> set[str]: def _impacted_tests( self, residual_files: list[str], diffs: dict[str, str] - ) -> tuple[set[str], list[str]]: - """Return (impacted stage-prefixed tests, file::qualname symbols with no DB rows).""" + ) -> tuple[set[str], list[str], str | None]: + """Return (impacted tests, file::qualname symbols with no DB rows, decline reason). + + A non-None decline reason means the change cannot be resolved soundly and + the caller must not narrow; see `_import_executed_impact`. + """ impacted: set[str] = set() no_data: list[str] = [] for path in residual_files: @@ -120,13 +124,43 @@ def _impacted_tests( if not ok: impacted |= self.db.tests_touching_file(cf) continue + import_executed = import_executed_qualnames(source) for qualname in sorted(qualnames): # sorted -> deterministic no_data order + if qualname in import_executed: + tests, decline = self._import_executed_impact(cf, qualname, path) + if decline is not None: + return impacted, no_data, decline + impacted |= tests + continue tests = self.db.tests_touching_func(cf, qualname) impacted |= tests - if not tests and qualname != "": + if not tests: no_data.append(f"{cf}::{qualname}") impacted |= self._no_data_fallback(cf) - return impacted, no_data + return impacted, no_data, None + + def _import_executed_impact( + self, cf: str, qualname: str, path: str + ) -> tuple[set[str], str | None]: + """Resolve a changed qualname whose code runs at import time. + + Module and class bodies execute once per process, during import. A test + only holds a row for one if its process was already being recorded then, + which excludes every test served by a pool worker (those import before + capture starts) and every in-process import done at collection time + (recorded under the empty context). The rows are therefore a subset of + the tests that actually ran the code, and narrowing on them alone drops + the difference. + + The file's own row set covers the gap whenever the same process later + entered any function in the file. When it does not — a file whose only + recorded rows are the import-time ones — no sound impact set exists and + the change is declined instead. + """ + file_tests = self.db.tests_touching_file(cf) + if len(file_tests) > len(self.db.tests_touching_func(cf, qualname)): + return file_tests, None + return set(), (f"import-executed scope changed with no wider row set: {path}::{qualname}") def _no_data_fallback(self, cf: str) -> set[str]: """Tests to force-run for a changed function the DB never captured. @@ -149,7 +183,8 @@ def _read_head(self, path: str) -> str | None: def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: """Decide over residual files (repo-relative paths no rule claimed). - Returns ok=False for any non-core-Python file or file absent from the DB. + Returns ok=False for any non-core-Python file, any file absent from the DB, + and any import-executed change `_import_executed_impact` cannot bound. """ for path in residual_files: cf = canon(path) @@ -160,7 +195,9 @@ def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageRe ok=False, reason=f"zero-touch residual file (new/uninstrumented): {path}" ) - impacted_tests, no_data_funcs = self._impacted_tests(residual_files, diffs) + impacted_tests, no_data_funcs, decline = self._impacted_tests(residual_files, diffs) + if decline is not None: + return CoverageResult(ok=False, reason=decline) impacted: dict[str, set[str]] = {} for test in impacted_tests: From 35f9dd3d3124931b2dddec827273a975ec0241e0 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:24:30 +0800 Subject: [PATCH 10/35] [TRTLLM-12838][infra] CBTS: select the coverage DB by revision, and record its lag Build numbers do not order revisions. A post-merge build can be a re-run of an older commit, so walking numbers down -- what latest_tarball_url did -- can land on a DB collected earlier than a lower-numbered build's, with nothing in the log to say so. Observed in the live artifact listing: build 2880 carries the same commit as 2874, three days older than its own build timestamp. Each build publishes build_info.txt next to its artifacts, carrying `commit=`. select_tarball now reads it for every tarball-bearing build in the probe window and picks the one whose commit trails the checkout least, keeping build order only as the tie-break for builds whose commit git cannot resolve (build_info.txt is absent on some builds -- 1 of the 9 tarball-bearing builds surveyed). The chosen commit and its distance from HEAD ride through main.py into the decision as coverage_db_commit / coverage_db_lag, and into OpenSearch as s_coverage_db_commit / l_coverage_db_lag (-1 when unknown). This records staleness rather than gating on it. Measured lag between consecutive tarball-bearing builds over 2026-07-29..08-06 was 14 to 90 commits, and the newest available DB trailed main by 31 at the time of writing, so any threshold picked today would be guesswork; the recorded field is what makes calibration possible. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 32 +++-- .../cbts/coverage_selection/artifact.py | 121 +++++++++++++++--- jenkins/scripts/cbts/main.py | 31 ++++- .../cbts/tools/report_cbts_decision.py | 5 + 4 files changed, 156 insertions(+), 33 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 5c720265febf..87ff1012065e 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -874,7 +874,13 @@ def getCbtsResult(pipeline, testFilter, globalVars) def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" if (coverageDb.path) { - mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-url '${coverageDb.url}'" + mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-build ${coverageDb.build}" + if (coverageDb.commit) { + mainCmd += " --coverage-db-commit ${coverageDb.commit}" + } + if (coverageDb.lag != null) { + mainCmd += " --coverage-db-lag ${coverageDb.lag}" + } } def output = sh(script: mainCmd, returnStdout: true) @@ -925,14 +931,21 @@ def _cbtsCoverageAudit(pipeline) // All commands run from ${LLM_ROOT}; covDir and the returned path are // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. def covDir = "cbts_cov" - def url = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-url || true", + // Selection is by collected revision, not build number; the JSON also carries + // the commit and how far HEAD runs ahead of it, both recorded in the decision. + def selJson = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + + "--print-selection --repo-root ${LLM_ROOT} || true", returnStdout: true, ).trim() - if (!url) { + if (!selJson) { pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") - return [path: "", url: ""] + return [path: "", build: null, commit: "", lag: null] } + def sel = new groovy.json.JsonSlurper().parseText(selJson) + def url = sel.url + pipeline.echo("CBTS audit: coverage DB from build ${sel.build}, " + + "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind HEAD'}") sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" // wget the tarball (retrying) and extract the sqlite. trtllm_utils.llmExecStepWithRetry(pipeline, script: @@ -940,14 +953,15 @@ def _cbtsCoverageAudit(pipeline) "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + "--db ${covDir}/cbts_touchmap.sqlite" - // url rides along so main.py can record which post-merge build the DB - // came from ("latest" is resolved here, once, per run). - return [path: "${covDir}/cbts_touchmap.sqlite", url: url] + // build/commit/lag ride along so main.py can record which DB the decision + // used (resolved here, once, per run). + return [path: "${covDir}/cbts_touchmap.sqlite", build: sel.build, + commit: sel.commit ?: "", lag: sel.lag] } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - return [path: "", url: ""] + return [path: "", build: null, commit: "", lag: null] } } diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index 73effa7fe8ab..d41269d8a601 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -30,8 +30,8 @@ import argparse import json -import re import shutil +import subprocess import sys import tarfile import urllib.error @@ -43,6 +43,9 @@ ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" TARBALL_NAME = "cbts_pystart_report.tar.gz" SQLITE_NAME = "cbts_touchmap.sqlite" +# Per-build metadata the pipeline uploads next to the artifacts; `commit=` +# is the revision that build ran, and is absent on some builds. +BUILD_INFO_NAME = "build_info.txt" _URM = "https://urm.nvidia.com/artifactory" _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" @@ -92,31 +95,93 @@ def tarball_url(build: int, artifact_base: str = ARTIFACT_BASE) -> str: return f"{_URM}/{artifact_base}/{build}/cbts-coverage/{TARBALL_NAME}" -def build_from_url(url: str) -> Optional[int]: - """Post-merge build number encoded in a `tarball_url()`, or None if absent.""" - m = re.search(r"/(\d+)/cbts-coverage/", url or "") - return int(m.group(1)) if m else None +def build_info_url(build: int, artifact_base: str = ARTIFACT_BASE) -> str: + return f"{_URM}/{artifact_base}/{build}/{BUILD_INFO_NAME}" -def latest_tarball_url( +def build_commit(build: int, artifact_base: str = ARTIFACT_BASE) -> Optional[str]: + """Revision a build ran, from its `build_info.txt`, or None when unavailable.""" + status, data = _get(build_info_url(build, artifact_base)) + if status != 200 or not data: + return None + for line in data.decode("utf-8", "replace").splitlines(): + key, _, value = line.partition("=") + if key.strip() == "commit" and value.strip(): + return value.strip() + return None + + +def commit_distance(commit: str, repo_root: str = ".", ref: str = "HEAD") -> Optional[int]: + """Commits in `ref` not reachable from `commit`, or None if git cannot answer.""" + try: + out = subprocess.run( + ["git", "rev-list", "--count", f"{commit}..{ref}"], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + timeout=_TIMEOUT, + ) + except (OSError, subprocess.SubprocessError): + return None + return int(out.stdout.strip()) if out.returncode == 0 and out.stdout.strip() else None + + +def select_tarball( artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, max_probe: int = _MAX_PROBE, -) -> Optional[str]: - """URL of the newest build whose coverage tarball actually exists, or None.""" + repo_root: str = ".", +) -> Optional[dict]: + """Pick the coverage tarball collected at the newest revision. + + Build numbers do not order revisions: a build can be a re-run of an older + commit, so walking numbers down can land on a DB older than a lower-numbered + build's. Rank the probe window's tarball-bearing builds by how far their + commit trails `repo_root`'s HEAD instead, and keep the build number only as + the tie-break for builds whose commit git cannot resolve. + + Returns {url, build, commit, lag} — commit/lag are None when unavailable. + """ build = latest_build_number(jenkins_base) if build is None: print("[artifact] could not resolve latest build number", file=sys.stderr) return None - floor = max(0, build - max_probe) - while build > floor: - url = tarball_url(build, artifact_base) - if _exists(url): - return url - print(f"[artifact] build {build} has no tarball, trying {build - 1}", file=sys.stderr) - build -= 1 - print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) - return None + candidates = [] + for b in range(build, max(0, build - max_probe), -1): + url = tarball_url(b, artifact_base) + if not _exists(url): + continue + commit = build_commit(b, artifact_base) + lag = commit_distance(commit, repo_root) if commit else None + candidates.append({"url": url, "build": b, "commit": commit, "lag": lag}) + if not candidates: + print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) + return None + # Known lag first (smallest = closest to HEAD); unknown lag falls back to build order. + best = min(candidates, key=lambda c: (c["lag"] is None, c["lag"] or 0, -c["build"])) + if best["lag"] is None: + print( + f"[artifact] build {best['build']}: commit unknown, selected by build number", + file=sys.stderr, + ) + skipped = [c["build"] for c in candidates if c["build"] > best["build"]] + if skipped: + print( + f"[artifact] builds {skipped} carry an older commit than {best['build']}; skipped", + file=sys.stderr, + ) + return best + + +def latest_tarball_url( + artifact_base: str = ARTIFACT_BASE, + jenkins_base: str = _JENKINS_BASE, + max_probe: int = _MAX_PROBE, +) -> Optional[str]: + """URL of the best available coverage tarball; see `select_tarball`.""" + best = select_tarball(artifact_base, jenkins_base, max_probe) + return best["url"] if best else None def extract_touch_db(tarball: Path | str, dest_dir: Path | str) -> Optional[Path]: @@ -160,13 +225,35 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--print-url", action="store_true", help="resolve and print the tarball URL only" ) + ap.add_argument( + "--print-selection", + action="store_true", + help="resolve and print {url, build, commit, lag} as JSON", + ) ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) + ap.add_argument("--repo-root", default=".", help="repo the lag is measured against") args = ap.parse_args(argv) url = tarball_url(args.build) if args.build is not None else None + if args.print_selection: + if args.build is not None: + commit = build_commit(args.build) + best = { + "url": url, + "build": args.build, + "commit": commit, + "lag": commit_distance(commit, args.repo_root) if commit else None, + } + else: + best = select_tarball(repo_root=args.repo_root) + if best is None: + return 1 + print(json.dumps(best)) + return 0 + if args.print_url: url = url or latest_tarball_url() if url is None: diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 7a6b49184939..41306e18ffea 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -40,7 +40,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parent / "coverage_selection")) -from artifact import build_from_url # noqa: E402 from blocks import ( # noqa: E402 ALWAYS_RUN_STAGE_PREFIX, Stage, @@ -134,6 +133,10 @@ class SelectionResult: # "latest post-merge tarball" at decision time, so two runs of the same # commit can consult different DBs; recording it makes a decision replayable. coverage_db_build: Optional[int] = None + # Revision the DB was collected at, and how many commits HEAD is ahead of it. + # Both None when `build_info.txt` carried no commit for that build. + coverage_db_commit: Optional[str] = None + coverage_db_lag: Optional[int] = None def to_json(self) -> str: data = { @@ -149,6 +152,8 @@ def to_json(self) -> str: "enable_multi_gpu": self.enable_multi_gpu, "coverage_dropped_stages": sorted(self.coverage_dropped_stages), "coverage_db_build": self.coverage_db_build, + "coverage_db_commit": self.coverage_db_commit, + "coverage_db_lag": self.coverage_db_lag, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -348,11 +353,22 @@ def main(argv: Optional[list[str]] = None) -> int: "runs on fallbacks and may drop fully-safe single-GPU stages.", ) parser.add_argument( - "--coverage-db-url", + "--coverage-db-build", + type=int, default=None, - help="Artifactory URL the --coverage-db tarball was fetched from. Only its " - "post-merge build number is used, recorded in the decision as " - "coverage_db_build so a decision can be traced back to its DB.", + help="Post-merge build the --coverage-db came from; recorded in the decision " + "so it can be traced back to its DB.", + ) + parser.add_argument( + "--coverage-db-commit", + default=None, + help="Revision the --coverage-db was collected at (from the build's build_info.txt).", + ) + parser.add_argument( + "--coverage-db-lag", + type=int, + default=None, + help="Commits HEAD is ahead of --coverage-db-commit; recorded in the decision.", ) parser.add_argument( "--no-data-policy", @@ -409,8 +425,9 @@ def main(argv: Optional[list[str]] = None) -> int: selector = Selector(stages) result = selector.run(pr, rules) - if args.coverage_db_url: - result.coverage_db_build = build_from_url(args.coverage_db_url) + result.coverage_db_build = args.coverage_db_build + result.coverage_db_commit = args.coverage_db_commit + result.coverage_db_lag = args.coverage_db_lag if args.coverage_db and result.scope is None: note = "" diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index 736ee14120a1..a534d97e2fcf 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -115,6 +115,11 @@ def build_document( "l_cbts_cases": cbts_cases, # Post-merge build of the consulted touch DB; 0 when no DB was used. "l_coverage_db_build": int(decision.get("coverage_db_build") or 0), + "s_coverage_db_commit": decision.get("coverage_db_commit") or "", + # Commits HEAD ran ahead of that DB; -1 when the build carried no commit. + "l_coverage_db_lag": int( + decision["coverage_db_lag"] if decision.get("coverage_db_lag") is not None else -1 + ), "d_case_skip_rate": round(case_skip_rate, 4), "flat_detail": { "hit_stages": affected, From 53b9feae10194872e9ccc4bc4a8f12fd50d31125 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:43:21 +0800 Subject: [PATCH 11/35] [TRTLLM-12838][infra] CBTS: fetch core-file diffs, and forward enable_multi_gpu Two wiring gaps that disabled parts of the coverage tier in the pipeline while leaving them working under the dry-run harness, which supplies a diff for every changed file. --list-needed-diffs unioned only the rules' needs_diff_for, none of which match tensorrt_llm/**. getCbtsResult fetches a diff only for files matching that list, so the tier never saw one for its own inputs and every residual core file took the empty-lines path: file-level resolution instead of qualname-level, and _import_executed_impact skipped entirely. Replayed against the 2026-07-30 touch DB, a one-line change inside PyExecutor._executor_loop resolved to 366 impacted tests and 230 removed cases without the diff, against 118 and 453 with it; a tensorrt_llm/version.py change removed 216 cases where the guard would have declined. Declare the tier's own pattern so the diffs are fetched. An absent diff is now a decline rather than a file-level fallback: without the changed lines an import-executed change cannot be told apart from a function-body one, and the file's row set bounds only the latter. A diff that strips to no changed lines (comment-only) still resolves at file level, which that bound does cover. Separately, _cbtsParseSelectionResult copied eight keys and enable_multi_gpu was not among them, so cbts.enable_multi_gpu in L0_Test.groovy was always null and the multi-GPU re-add never fired. The tier deliberately omits multi-GPU stages and relies on that gate, so any change to a file in getMultiGpuFileChanged's list ran none of them. Forward the flag. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 + jenkins/scripts/cbts/blocks.py | 10 ++-- .../cbts/coverage_selection/artifact.py | 14 +---- .../cbts/coverage_selection/qualname_map.py | 7 +-- .../cbts/coverage_selection/selector.py | 58 +++++-------------- .../cbts/coverage_selection/touch_db.py | 27 ++------- jenkins/scripts/cbts/coverage_tier.py | 19 +----- jenkins/scripts/cbts/main.py | 17 +++--- .../scripts/cbts/tools/coverage_explain.py | 9 +-- jenkins/scripts/cbts/tools/dryrun.py | 3 +- 10 files changed, 42 insertions(+), 124 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 87ff1012065e..37fbdef4ba2e 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1054,6 +1054,8 @@ def _cbtsParseSelectionResult(String text) // Explicit null check preserves `false`; default True is safe. sanity_required: data.sanity_required != null ? data.sanity_required : true, perfsanity_required: data.perfsanity_required != null ? data.perfsanity_required : true, + // Coverage tier omits multi-GPU stages; L0_Test re-adds them under this flag. + enable_multi_gpu: data.enable_multi_gpu ?: false, ] } diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 034b20fe8d63..4e020b215b0d 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -433,12 +433,11 @@ def _classify_map_var(var_name: str) -> Optional[str]: return None -# Backend name -> mako value. Same patterns as getMakoArgsFromStageName in -# jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. -# Stages CBTS always runs, whatever the decision: their entries stay in the -# test-db and they are added to every tier's affected_stages. +# Stages CBTS always runs, whatever the decision. ALWAYS_RUN_STAGE_PREFIX = "CPU-" +# Backend name -> mako value. Same patterns as getMakoArgsFromStageName in +# jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. _BACKEND_PATTERNS = [ ("-PyTorch-", "pytorch"), ("-CPP-", "cpp"), @@ -493,8 +492,7 @@ def derive_mako_from_stage(stage_name: str) -> dict[str, str]: mako["gpu"] = gpu_match.group(1).lower() count_match = _GPU_COUNT_RE.search(stage_name) mako["system_gpu_count"] = count_match.group(1) if count_match else "1" - # CPU stages carry no `N_GPUs` token but run on 0-GPU machines; renderTestDB - # (jenkins/L0_Test.groovy:3465) hardcodes system_gpu_count=0 for them. + # renderTestDB hardcodes system_gpu_count=0 for CPU- stages (no N_GPUs token). if stage_name.startswith("CPU-"): mako["system_gpu_count"] = "0" diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index d41269d8a601..e15b20fed441 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -43,8 +43,7 @@ ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" TARBALL_NAME = "cbts_pystart_report.tar.gz" SQLITE_NAME = "cbts_touchmap.sqlite" -# Per-build metadata the pipeline uploads next to the artifacts; `commit=` -# is the revision that build ran, and is absent on some builds. +# Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" _URM = "https://urm.nvidia.com/artifactory" @@ -133,16 +132,7 @@ def select_tarball( max_probe: int = _MAX_PROBE, repo_root: str = ".", ) -> Optional[dict]: - """Pick the coverage tarball collected at the newest revision. - - Build numbers do not order revisions: a build can be a re-run of an older - commit, so walking numbers down can land on a DB older than a lower-numbered - build's. Rank the probe window's tarball-bearing builds by how far their - commit trails `repo_root`'s HEAD instead, and keep the build number only as - the tie-break for builds whose commit git cannot resolve. - - Returns {url, build, commit, lag} — commit/lag are None when unavailable. - """ + """Tarball of the least-trailing commit as {url, build, commit, lag}; build number breaks ties.""" build = latest_build_number(jenkins_base) if build is None: print("[artifact] could not resolve latest build number", file=sys.stderr) diff --git a/jenkins/scripts/cbts/coverage_selection/qualname_map.py b/jenkins/scripts/cbts/coverage_selection/qualname_map.py index 54cb5d996652..4c0ca84dbc36 100644 --- a/jenkins/scripts/cbts/coverage_selection/qualname_map.py +++ b/jenkins/scripts/cbts/coverage_selection/qualname_map.py @@ -87,12 +87,7 @@ def qualnames_for_lines(source: str, lines: set[int]) -> tuple[set[str], bool]: def import_executed_qualnames(source: str) -> set[str]: - """Qualnames whose code runs once, at import: `` and every class body. - - A signature or decorator line attributes to its *enclosing* scope (see - `_attribute`), so a method's `def` line lands on the class and is covered by - this set too. Unparsable source yields just ``. - """ + """Qualnames whose code runs once at import: `` and every class body.""" out = {""} def walk(stmts, prefix: str) -> None: diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index a18f4b00b3b2..7626ac9ba821 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -47,11 +47,7 @@ class CoverageResult: no_data_funcs: list[str] = field(default_factory=list) -# What to do with a changed function that has no rows in the DB — it was never -# captured, so "which tests exercise it" is unknown rather than empty. -# file fall back to every test that entered any function in the file -# importers fall back to the file's `` touch set (its importers) -# ignore treat as impacting nothing +# Fallback for a changed function with no DB rows: whole file / its importers / nothing. NO_DATA_POLICIES = ("file", "importers", "ignore") DEFAULT_NO_DATA_POLICY = "file" @@ -92,10 +88,7 @@ def untrusted_tests(self) -> set[str]: return self._untrusted def untrusted_families(self) -> set[str]: - """`untrusted_tests()` re-keyed to `/`. - - Untrusted on any shard means untrusted for the family — the entry runs. - """ + """`untrusted_tests()` re-keyed to `/`; any shard taints the family.""" out: set[str] = set() for test in self.untrusted_tests(): stage, nodeid = split_stage(test) @@ -106,22 +99,21 @@ def untrusted_families(self) -> set[str]: def _impacted_tests( self, residual_files: list[str], diffs: dict[str, str] ) -> tuple[set[str], list[str], str | None]: - """Return (impacted tests, file::qualname symbols with no DB rows, decline reason). - - A non-None decline reason means the change cannot be resolved soundly and - the caller must not narrow; see `_import_executed_impact`. - """ + """Return (impacted tests, file::qualname with no DB rows, decline reason or None).""" impacted: set[str] = set() no_data: list[str] = [] for path in residual_files: cf = canon(path) - lines = iter_diff_post_line_numbers(diffs.get(path, "")) + diff = diffs.get(path) or "" source = self._read_head(path) - if not lines or source is None: - impacted |= self.db.tests_touching_file(cf) - continue + if not diff.strip() or source is None: + # No diff means an import-executed change cannot be told apart from + # a function-body one, and the file row set bounds only the latter. + return impacted, no_data, f"no usable diff for residual file: {path}" + lines = iter_diff_post_line_numbers(diff) qualnames, ok = qualnames_for_lines(source, lines) - if not ok: + if not lines or not ok: + # Comment-only or unparsable: no qualname to resolve, file-level bound. impacted |= self.db.tests_touching_file(cf) continue import_executed = import_executed_qualnames(source) @@ -142,32 +134,14 @@ def _impacted_tests( def _import_executed_impact( self, cf: str, qualname: str, path: str ) -> tuple[set[str], str | None]: - """Resolve a changed qualname whose code runs at import time. - - Module and class bodies execute once per process, during import. A test - only holds a row for one if its process was already being recorded then, - which excludes every test served by a pool worker (those import before - capture starts) and every in-process import done at collection time - (recorded under the empty context). The rows are therefore a subset of - the tests that actually ran the code, and narrowing on them alone drops - the difference. - - The file's own row set covers the gap whenever the same process later - entered any function in the file. When it does not — a file whose only - recorded rows are the import-time ones — no sound impact set exists and - the change is declined instead. - """ + """Bound an import-time qualname by the file's row set, which under-records it; decline if no wider.""" file_tests = self.db.tests_touching_file(cf) if len(file_tests) > len(self.db.tests_touching_func(cf, qualname)): return file_tests, None return set(), (f"import-executed scope changed with no wider row set: {path}::{qualname}") def _no_data_fallback(self, cf: str) -> set[str]: - """Tests to force-run for a changed function the DB never captured. - - No rows means the function was never observed, not that no test reaches - it, so the file's own test set is the tightest sound bound available. - """ + """Tests to force-run for a changed function the DB never captured.""" if self._no_data_policy == "file": return self.db.tests_touching_file(cf) if self._no_data_policy == "importers": @@ -181,11 +155,7 @@ def _read_head(self, path: str) -> str | None: return None def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: - """Decide over residual files (repo-relative paths no rule claimed). - - Returns ok=False for any non-core-Python file, any file absent from the DB, - and any import-executed change `_import_executed_impact` cannot bound. - """ + """Decide over residual files; ok=False for non-core, not-in-DB, or unbounded import-time changes.""" for path in residual_files: cf = canon(path) if not (path.endswith(".py") and cf.startswith("tensorrt_llm/")): diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py index ab7153aee9c3..2868a15160fb 100644 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -45,9 +45,7 @@ ("tensorrt_llm/executor/executor.py", "GenerationExecutor.generate"), ) _SERVING_PATH_MARKERS: tuple[str, ...] = ("disaggregated/",) -# Stage-name markers whose capture is structurally partial regardless of footprint: -# `sitecustomize` opts Ray infra processes out, so a Ray stage's GPU worker lives in -# an uninstrumented `default_worker.py` and its tests carry only the driver's rows. +# Stage-name markers whose GPU worker is uninstrumented, so capture is partial. _UNTRUSTED_STAGE_MARKERS: tuple[str, ...] = ("-Ray-",) _MIN_FUNCS = 30 @@ -65,14 +63,7 @@ def split_stage(test: str) -> tuple[str, str]: def stage_family(stage: str) -> str: - """Collapse a pytest-split shard name to its family (`A10-PyTorch-2` -> `A10-PyTorch`). - - Coverage is captured per shard, but pytest-split assigns each entry to - exactly one shard, so a stage's shards hold disjoint capture sets. Only the - family-level union answers "was this entry ever captured on this stage" — - and the shard an entry lands on is not stable across runs anyway, since - pytest-split rebalances by duration. - """ + """Collapse a pytest-split shard name to its family (`A10-PyTorch-2` -> `A10-PyTorch`).""" return _SPLIT_SUFFIX_RE.sub("", stage) @@ -201,10 +192,7 @@ def known_by_stage(self) -> dict[str, set[str]]: return out def known_by_family(self) -> dict[str, set[str]]: - """`{stage family -> {bare nodeid, ...}}` — a stage's shards unioned. - - Selection keys on this rather than `known_by_stage`: see `stage_family`. - """ + """`{stage family -> {bare nodeid, ...}}` — a stage's shards unioned.""" out: dict[str, set[str]] = {} for test in self.known_tests(): stage, nodeid = split_stage(test) @@ -231,14 +219,7 @@ def untrusted_tests( min_funcs: int, untrusted_stage_markers: tuple[str, ...] = (), ) -> set[str]: - """Stage-prefixed tests whose per-test capture looks incomplete (must always run). - - Flags a test that drove execution/serving but is missing `worker_file` — - matched by a `launch_markers` `(file, qualname_substring)` call or a - `serving_path_markers` nodeid substring — that entered fewer than - `min_funcs` functions total, or that ran on a stage whose name contains an - `untrusted_stage_markers` substring. - """ + """Tests missing `worker_file` after driving execution, near-empty, or on an untrusted stage.""" drove_execution: set[str] = set() for file, qual_substr in launch_markers: drove_execution |= { diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index 41bc34708b1f..afe5e9a6e16d 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -95,12 +95,7 @@ def _entry_reason( known: dict[str, set[str]], untrusted: set[str], ) -> str: - """Return SAFE or the must-run cause for a candidate YAML entry. - - Keyed by stage family, not by shard: pytest-split puts each entry on - exactly one shard, so a per-shard lookup would report `no_data` for every - entry of any stage split more than one way. - """ + """Return SAFE or the must-run cause for a candidate YAML entry, keyed by stage family.""" if entry in keep_rule: return _R_RULE_KEPT dbk = db_key(entry) @@ -124,16 +119,8 @@ def _build_narrowing( known: dict[str, set[str]], untrusted: set[str], ) -> tuple[dict[tuple[str, int], set[str]], set[str], Counter]: - """Classify every candidate entry; remove only SAFE ones. - - `cov.skippable` / `cov.impacted` / `known` / `untrusted` are keyed by stage - family, so each block's served stages are mapped through `stage_family`. - - Returns (removed per block, fully-emptied instrumented stages, must-run tally). - """ - # Always-run families are dropped from `instrumented` rather than filtered - # later: the shared-block rule below prunes a block only when every served - # stage is instrumented, so this keeps every block they serve intact. + """Classify every candidate entry; return (removed per block, emptied stages, must-run tally).""" + # Dropping always-run families here keeps every block they serve out of pruning. instrumented = {f for f in cov.skippable if not f.startswith(ALWAYS_RUN_STAGE_PREFIX)} rule_kept = { key: _rule_kept_entries(b, rule_block_filters[key]) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 41306e18ffea..0e63465538cd 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -69,6 +69,10 @@ from rules.visual_gen_rule import VisualGenRule # noqa: E402 from rules.waives_rule import WaivesRule # noqa: E402 +# Files the coverage tier maps to qualnames; without their diffs it can only +# resolve at file level, and cannot tell an import-executed change apart. +COVERAGE_NEEDS_DIFF_FOR: tuple[str, ...] = ("tensorrt_llm/**/*.py",) + # --- Rule registry ----------------------------------------------------------- # Classes are used for `--list-needed-diffs` (no need to construct). @@ -129,12 +133,9 @@ class SelectionResult: # set by coverage tier; Groovy re-adds multiGpuJobs under MULTI_GPU_FILE_CHANGED gate enable_multi_gpu: bool = False coverage_dropped_stages: list[str] = field(default_factory=list) - # Post-merge build the consulted touch DB came from. The DB is resolved as - # "latest post-merge tarball" at decision time, so two runs of the same - # commit can consult different DBs; recording it makes a decision replayable. + # Post-merge build the consulted touch DB came from; makes a decision replayable. coverage_db_build: Optional[int] = None - # Revision the DB was collected at, and how many commits HEAD is ahead of it. - # Both None when `build_info.txt` carried no commit for that build. + # Revision the DB was collected at and HEAD's distance from it; None when unknown. coverage_db_commit: Optional[str] = None coverage_db_lag: Optional[int] = None @@ -382,7 +383,7 @@ def main(argv: Optional[list[str]] = None) -> int: args = parser.parse_args(argv) if args.list_needed_diffs: - patterns: set[str] = set() + patterns: set[str] = set(COVERAGE_NEEDS_DIFF_FOR) for cls in RULE_CLASSES: patterns.update(cls.needs_diff_for) for p in sorted(patterns): @@ -508,9 +509,7 @@ def main(argv: Optional[list[str]] = None) -> int: durations=durations, ) - # Stages that run whatever the decision. Added before the trigger-mode filter - # so a Post-Merge one is still dropped in pre-merge, and left out of the - # per-stage counts so Layer 2.5 neither renames nor resizes them. + # Added before the trigger-mode filter, and outside the counts Layer 2.5 resizes. if result.scope is not None: result.affected_stages |= {s for s in stages if s.startswith(ALWAYS_RUN_STAGE_PREFIX)} diff --git a/jenkins/scripts/cbts/tools/coverage_explain.py b/jenkins/scripts/cbts/tools/coverage_explain.py index 61d1c06b4bf2..82e45f81de10 100644 --- a/jenkins/scripts/cbts/tools/coverage_explain.py +++ b/jenkins/scripts/cbts/tools/coverage_explain.py @@ -91,8 +91,7 @@ def main(argv: list[str] | None = None) -> int: core = [f for f in files if f.endswith(".py") and canon(f).startswith("tensorrt_llm/")] non_core = [f for f in files if f not in core] - # `CoverageSelector.decide()` refuses the whole decision when a residual file has - # no rows, so a removed list computed past that point would not be reachable. + # `decide()` refuses the whole change here, so no removal below would be reachable. zero_touch = [f for f in core if not db.file_has_touch_rows(canon(f))] if zero_touch: print(f"commit {args.sha[:12]} — coverage selection REFUSES this change:") @@ -101,8 +100,7 @@ def main(argv: list[str] | None = None) -> int: print("\nNo case is removable; every stage runs in full.") return 0 - # Build the change's impact set: function-level (file, qualname), or (file, None) - # when a file falls back to file-level. Collect the impacted tests. + # Impact set: (file, qualname) per function, or the whole file when it falls back. impact_funcs: set[tuple[str, str]] = set() impact_files: set[str] = set() # file-level fallback changed_files: set[str] = set() @@ -127,8 +125,7 @@ def main(argv: list[str] | None = None) -> int: impact_funcs.add((cf, q)) tests = db.tests_touching_func(cf, q) impacted |= tests - # Mirrors the selector's default no_data_policy="file": a function with no - # rows was never observed, so the file's own test set is the bound used. + # Mirrors the selector's default no_data_policy="file". if not tests and q != "": no_data.append(f"{cf}::{q}") impact_files.add(cf) diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index de376d716f5e..ba17f8dc46e1 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -482,8 +482,7 @@ def main(argv: Optional[list[str]] = None) -> int: print(f"error: cbts main.py not found at {CBTS_MAIN}", file=sys.stderr) return 2 - # Coverage tier is opt-in: it needs the post-merge touch DB, which the user - # must download separately. Default replays rules only. + # Coverage tier is opt-in: it needs a separately downloaded touch DB. if args.coverage_db: print(f"coverage tier ON (merged with rules) via {args.coverage_db}", file=sys.stderr) else: From a9f5a9ba1a94bff2fcde4f568054f0bb81de0e81 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:00:33 +0800 Subject: [PATCH 12/35] [TRTLLM-12838][infra] CBTS: bound files whose patch the API omitted, and count them Diffs come from the GitHub/GitLab MR API, not git: the pipeline reads the changed-file list and each file's patch over the API to avoid the history-depth problems a shallow clone has with merge-base. The API omits the patch for binary, renamed and oversized files, and getMergeRequestOneFileChanges coerces that to an empty string. Such a file previously took the same path as a comment-only edit and resolved at file level. That is the right bound for most files but not for the ~880 of 1149 whose only recorded rows are the import-time ones, where the file row set equals the module's and so does not cover the pool-worker tests that never recorded it. Treat an absent patch as an import-time change of unknown qualname: bound it by the file row set when that is wider, decline when it is not. Files that can be bounded keep narrowing, so one oversized diff no longer forfeits the whole PR. `_import_executed_impact` becomes `_import_executed_bound` returning the set or None, since both callers now phrase their own decline text. Also count the affected files. How often the API omits a patch decides whether fetching base-branch history for a real git diff would be worth its cost, and nothing measured it: the count rides through the decision as coverage_no_diff_files and into OpenSearch as l_coverage_no_diff_files. Note the dry-run harnesses compute a diff for every file, so they never exercise this path. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/selector.py | 44 ++++++++++++------- jenkins/scripts/cbts/coverage_tier.py | 1 + jenkins/scripts/cbts/main.py | 4 ++ .../cbts/tools/report_cbts_decision.py | 2 + 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 7626ac9ba821..17e4fd2a2b52 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -45,6 +45,8 @@ class CoverageResult: n_untrusted: int = 0 # functions with no DB rows (new/uninstrumented); bounded per `no_data_policy` no_data_funcs: list[str] = field(default_factory=list) + # residual files the forge API returned no patch for (binary / rename / oversized) + no_diff_files: list[str] = field(default_factory=list) # Fallback for a changed function with no DB rows: whole file / its importers / nothing. @@ -98,18 +100,24 @@ def untrusted_families(self) -> set[str]: def _impacted_tests( self, residual_files: list[str], diffs: dict[str, str] - ) -> tuple[set[str], list[str], str | None]: - """Return (impacted tests, file::qualname with no DB rows, decline reason or None).""" + ) -> tuple[set[str], list[str], list[str], str | None]: + """Return (impacted tests, qualnames with no DB rows, files with no diff, decline reason).""" impacted: set[str] = set() no_data: list[str] = [] + no_diff: list[str] = [] for path in residual_files: cf = canon(path) diff = diffs.get(path) or "" source = self._read_head(path) if not diff.strip() or source is None: - # No diff means an import-executed change cannot be told apart from - # a function-body one, and the file row set bounds only the latter. - return impacted, no_data, f"no usable diff for residual file: {path}" + # The API omits the patch for binary / renamed / oversized files, so + # which qualname changed is unknown; bound it as an import-time one. + no_diff.append(path) + tests = self._import_executed_bound(cf, "") + if tests is None: + return impacted, no_data, no_diff, f"no usable diff, no wider row set: {path}" + impacted |= tests + continue lines = iter_diff_post_line_numbers(diff) qualnames, ok = qualnames_for_lines(source, lines) if not lines or not ok: @@ -119,9 +127,10 @@ def _impacted_tests( import_executed = import_executed_qualnames(source) for qualname in sorted(qualnames): # sorted -> deterministic no_data order if qualname in import_executed: - tests, decline = self._import_executed_impact(cf, qualname, path) - if decline is not None: - return impacted, no_data, decline + tests = self._import_executed_bound(cf, qualname) + if tests is None: + why = f"import-executed scope changed, no wider row set: {path}::{qualname}" + return impacted, no_data, no_diff, why impacted |= tests continue tests = self.db.tests_touching_func(cf, qualname) @@ -129,16 +138,14 @@ def _impacted_tests( if not tests: no_data.append(f"{cf}::{qualname}") impacted |= self._no_data_fallback(cf) - return impacted, no_data, None + return impacted, no_data, no_diff, None - def _import_executed_impact( - self, cf: str, qualname: str, path: str - ) -> tuple[set[str], str | None]: - """Bound an import-time qualname by the file's row set, which under-records it; decline if no wider.""" + def _import_executed_bound(self, cf: str, qualname: str) -> set[str] | None: + """File row set when it is wider than an import-time qualname's, else None (unbounded).""" file_tests = self.db.tests_touching_file(cf) if len(file_tests) > len(self.db.tests_touching_func(cf, qualname)): - return file_tests, None - return set(), (f"import-executed scope changed with no wider row set: {path}::{qualname}") + return file_tests + return None def _no_data_fallback(self, cf: str) -> set[str]: """Tests to force-run for a changed function the DB never captured.""" @@ -165,9 +172,11 @@ def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageRe ok=False, reason=f"zero-touch residual file (new/uninstrumented): {path}" ) - impacted_tests, no_data_funcs, decline = self._impacted_tests(residual_files, diffs) + impacted_tests, no_data_funcs, no_diff_files, decline = self._impacted_tests( + residual_files, diffs + ) if decline is not None: - return CoverageResult(ok=False, reason=decline) + return CoverageResult(ok=False, reason=decline, no_diff_files=no_diff_files) impacted: dict[str, set[str]] = {} for test in impacted_tests: @@ -191,6 +200,7 @@ def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageRe f"{n_untrusted} untrusted (incomplete-capture) test(s) forced to run" ), impacted=impacted, + no_diff_files=no_diff_files, skippable=skippable, n_untrusted=n_untrusted, no_data_funcs=no_data_funcs, diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index afe5e9a6e16d..02c847236993 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -250,6 +250,7 @@ def apply_coverage_tier( "dropped_stages": len(dropped), "outcome": "narrowed" if narrowed else "nothing_removable", "no_data_policy": no_data_policy, + "no_diff_files": len(cov.no_diff_files), **({"no_data_funcs": list(cov.no_data_funcs)} if cov.no_data_funcs else {}), }, ) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 0e63465538cd..229648070bc1 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -138,6 +138,8 @@ class SelectionResult: # Revision the DB was collected at and HEAD's distance from it; None when unknown. coverage_db_commit: Optional[str] = None coverage_db_lag: Optional[int] = None + # Residual files the forge API returned no patch for; they fall back to file level. + coverage_no_diff_files: int = 0 def to_json(self) -> str: data = { @@ -155,6 +157,7 @@ def to_json(self) -> str: "coverage_db_build": self.coverage_db_build, "coverage_db_commit": self.coverage_db_commit, "coverage_db_lag": self.coverage_db_lag, + "coverage_no_diff_files": self.coverage_no_diff_files, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -455,6 +458,7 @@ def main(argv: Optional[list[str]] = None) -> int: result.affected_stages = tier.affected_stages result.enable_multi_gpu = True result.coverage_dropped_stages = sorted(tier.dropped) + result.coverage_no_diff_files = int(tier.detail.get("no_diff_files") or 0) if tier.removed: write_coverage_test_db( src_dir=test_db_dir, diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index a534d97e2fcf..39f2c849d360 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -116,6 +116,8 @@ def build_document( # Post-merge build of the consulted touch DB; 0 when no DB was used. "l_coverage_db_build": int(decision.get("coverage_db_build") or 0), "s_coverage_db_commit": decision.get("coverage_db_commit") or "", + # Residual files whose patch the forge API omitted (binary / rename / oversized). + "l_coverage_no_diff_files": int(decision.get("coverage_no_diff_files") or 0), # Commits HEAD ran ahead of that DB; -1 when the build carried no commit. "l_coverage_db_lag": int( decision["coverage_db_lag"] if decision.get("coverage_db_lag") is not None else -1 From d5f95f24b1ecd8041470390ce13f2291180365b9 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:08:38 +0800 Subject: [PATCH 13/35] [TRTLLM-12838][infra] CBTS: use the DB's own completeness signal, and four review fixes untrusted_tests judged capture completeness from three proxies while the DB already carried the producer's own account: test_meta records each test's pytest outcome and how many processes saved coverage against how many the coordinator spawned. Read it. On the 2026-07-30 DB that flags 35 further tests -- among them a disagg case whose coordinator spawned 20 workers and saw 6 save -- each of which was previously eligible for removal. Untrusted goes 89 -> 118 of 746 and whole-window skip 43.2% -> 42.9%. Absent on older DBs, which yield an empty set. coverage_explain now delegates to CoverageSelector instead of re-deriving the decision; the selector takes an optional source reader so the tool can resolve a past commit's blobs. Re-deriving had drifted twice: it modelled the zero-touch refusal but not the import-executed one, so a change the selector declines could still be explained as removable. Verified equal on a real commit, 296 removable either way. Also: write_coverage_test_db and write_filtered_test_db clear their output directory, since a leftover YAML from an earlier run would otherwise ship inside the uploaded tarball and narrow an unrelated stage; decide() gates on the repo path rather than canon(), which matches `tensorrt_llm/` anywhere and so admitted cpp/tensorrt_llm/*.py into the package namespace; and the commit interpolated into the main.py command line is quoted, as it comes from a remote file. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- jenkins/scripts/cbts/blocks.py | 3 + .../cbts/coverage_selection/selector.py | 12 +++- .../cbts/coverage_selection/touch_db.py | 20 +++++- jenkins/scripts/cbts/coverage_tier.py | 3 + jenkins/scripts/cbts/tools/coverage_audit.py | 4 ++ .../scripts/cbts/tools/coverage_explain.py | 69 +++++-------------- 7 files changed, 57 insertions(+), 56 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 37fbdef4ba2e..39a3d67ef548 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -876,7 +876,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) if (coverageDb.path) { mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-build ${coverageDb.build}" if (coverageDb.commit) { - mainCmd += " --coverage-db-commit ${coverageDb.commit}" + mainCmd += " --coverage-db-commit '${coverageDb.commit}'" } if (coverageDb.lag != null) { mainCmd += " --coverage-db-lag ${coverageDb.lag}" diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 4e020b215b0d..552afdb03d4f 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -25,6 +25,7 @@ import json import math import re +import shutil from dataclasses import dataclass, field from fnmatch import fnmatch from pathlib import Path @@ -756,6 +757,8 @@ def write_filtered_test_db( tests are kept (prevents silent skip from typo'd waive ids or granularity mismatch). The block itself is still kept either way. """ + # Clear first: a leftover YAML from an earlier run would ship in the artifact. + shutil.rmtree(output_dir, ignore_errors=True) output_dir.mkdir(parents=True, exist_ok=True) affected_stems = {stem for stem, _ in block_filters} diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 17e4fd2a2b52..ee2585829235 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -15,6 +15,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -66,6 +67,7 @@ def __init__( min_funcs: int = _MIN_FUNCS, untrusted_stage_markers: tuple[str, ...] = _UNTRUSTED_STAGE_MARKERS, no_data_policy: str = DEFAULT_NO_DATA_POLICY, + read_source: Callable[[str], str | None] | None = None, ) -> None: self.db = db self.repo_root = Path(repo_root) @@ -75,6 +77,8 @@ def __init__( self._min_funcs = min_funcs self._untrusted_stage_markers = untrusted_stage_markers self._no_data_policy = no_data_policy + # Defaults to the checkout; callers explaining a past commit inject their own. + self._read_source = read_source or self._read_head self._untrusted: set[str] | None = None def untrusted_tests(self) -> set[str]: @@ -108,7 +112,7 @@ def _impacted_tests( for path in residual_files: cf = canon(path) diff = diffs.get(path) or "" - source = self._read_head(path) + source = self._read_source(path) if not diff.strip() or source is None: # The API omits the patch for binary / renamed / oversized files, so # which qualname changed is unknown; bound it as an import-time one. @@ -164,9 +168,11 @@ def _read_head(self, path: str) -> str | None: def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: """Decide over residual files; ok=False for non-core, not-in-DB, or unbounded import-time changes.""" for path in residual_files: - cf = canon(path) - if not (path.endswith(".py") and cf.startswith("tensorrt_llm/")): + # Gate on the repo path, not canon(): canon matches `tensorrt_llm/` anywhere, + # so cpp/tensorrt_llm/*.py would otherwise map onto the package namespace. + if not (path.endswith(".py") and path.startswith("tensorrt_llm/")): return CoverageResult(ok=False, reason=f"non-core-Python residual file: {path}") + cf = canon(path) if not self.db.file_has_touch_rows(cf): return CoverageResult( ok=False, reason=f"zero-touch residual file (new/uninstrumented): {path}" diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py index 2868a15160fb..1414bb1cb730 100644 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -209,6 +209,24 @@ def files_touched_by(self, test: str) -> list[tuple[str, str]]: for row in self._conn.execute("SELECT file, qualname FROM touch WHERE test=?", (test,)) ] + def incomplete_capture_tests(self) -> set[str]: + """Stage-prefixed tests the DB itself reports as incompletely captured. + + `test_meta` (schema_version 2) records each test's pytest outcome and how + many processes saved coverage against how many the coordinator spawned; + absent on older DBs, which yield an empty set. + """ + try: + rows = self._conn.execute( + "SELECT test FROM test_meta WHERE test != '' AND " + "(outcome IS NULL OR outcome != 'passed' OR saved_procs < expected_workers + 1)" + ) + except sqlite3.OperationalError: + return set() + # Intersect with the touch universe: rows for tests that recorded nothing + # are not selection candidates, and would break `untrusted <= known`. + return {row[0] for row in rows} & self.known_tests() + # -- coverage-completeness heuristic -- def untrusted_tests( @@ -250,4 +268,4 @@ def untrusted_tests( for test in self.known_tests() if any(marker in split_stage(test)[0] for marker in untrusted_stage_markers) } - return missing_worker | tiny | on_untrusted_stage + return missing_worker | tiny | on_untrusted_stage | self.incomplete_capture_tests() diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index 02c847236993..bd647fd21105 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -17,6 +17,7 @@ import math import re +import shutil import sys from collections import Counter from dataclasses import dataclass, field @@ -261,6 +262,8 @@ def write_coverage_test_db( src_dir: Path, out_dir: Path, removed: dict[tuple[str, int], set[str]] ) -> None: """Write narrowed YAMLs with removed entries dropped.""" + # Clear first: a leftover YAML from an earlier run would ship in the artifact. + shutil.rmtree(out_dir, ignore_errors=True) out_dir.mkdir(parents=True, exist_ok=True) for stem in sorted({stem for stem, _ in removed}): src = src_dir / f"{stem}.yml" diff --git a/jenkins/scripts/cbts/tools/coverage_audit.py b/jenkins/scripts/cbts/tools/coverage_audit.py index c4a61d049f62..f8df2354b592 100644 --- a/jenkins/scripts/cbts/tools/coverage_audit.py +++ b/jenkins/scripts/cbts/tools/coverage_audit.py @@ -123,7 +123,11 @@ def main(argv: list[str] | None = None) -> int: _UNTRUSTED_STAGE_MARKERS, ) + incomplete = db.incomplete_capture_tests() + def reason(test: str) -> str: + if test in incomplete: + return "test_meta: not passed or a spawned process saved nothing" if any(m in split_stage(test)[0] for m in _UNTRUSTED_STAGE_MARKERS): return "untrusted stage (GPU worker uninstrumented)" if any(m in test for m in _SERVING_PATH_MARKERS): diff --git a/jenkins/scripts/cbts/tools/coverage_explain.py b/jenkins/scripts/cbts/tools/coverage_explain.py index 82e45f81de10..8199e51a6c5d 100644 --- a/jenkins/scripts/cbts/tools/coverage_explain.py +++ b/jenkins/scripts/cbts/tools/coverage_explain.py @@ -43,17 +43,8 @@ from qualname_map import qualnames_for_lines # noqa: E402 from rules._helpers import iter_diff_post_line_numbers # noqa: E402 -from touch_db import ( # noqa: E402 - _LAUNCH_MARKERS, - _MIN_FUNCS, - _SERVING_PATH_MARKERS, - _UNTRUSTED_STAGE_MARKERS, - _WORKER_SENTINEL, - TouchDB, - canon, - split_stage, - stage_family, -) +from selector import CoverageSelector # noqa: E402 +from touch_db import TouchDB, canon, stage_family # noqa: E402 def _git(repo: Path, *args: str, check: bool = True) -> str: @@ -91,45 +82,35 @@ def main(argv: list[str] | None = None) -> int: core = [f for f in files if f.endswith(".py") and canon(f).startswith("tensorrt_llm/")] non_core = [f for f in files if f not in core] - # `decide()` refuses the whole change here, so no removal below would be reachable. - zero_touch = [f for f in core if not db.file_has_touch_rows(canon(f))] - if zero_touch: + # Delegate the decision itself so the gates cannot drift from the selector. + diffs = {f: _git(repo, "diff", f"{args.sha}^", args.sha, "--", f, check=False) for f in core} + selector = CoverageSelector(db, repo, read_source=lambda f: _src_at(repo, args.sha, f)) + res = selector.decide(core, diffs) + if not res.ok: print(f"commit {args.sha[:12]} — coverage selection REFUSES this change:") - for f in zero_touch: - print(f" zero-touch residual file (new/uninstrumented): {f}") + print(f" {res.reason}") print("\nNo case is removable; every stage runs in full.") return 0 - # Impact set: (file, qualname) per function, or the whole file when it falls back. + # Forward lookup for the per-case justification the selector does not return. impact_funcs: set[tuple[str, str]] = set() - impact_files: set[str] = set() # file-level fallback + impact_files: set[str] = set() changed_files: set[str] = set() - impacted: set[str] = set() - no_data: list[str] = [] for f in core: cf = canon(f) changed_files.add(cf) - diff = _git(repo, "diff", f"{args.sha}^", args.sha, "--", f, check=False) - lines = iter_diff_post_line_numbers(diff) + lines = iter_diff_post_line_numbers(diffs.get(f, "")) src = _src_at(repo, args.sha, f) - if not lines or src is None: - impact_files.add(cf) - impacted |= db.tests_touching_file(cf) - continue - qns, ok = qualnames_for_lines(src, lines) + qns, ok = qualnames_for_lines(src, lines) if (lines and src) else (set(), False) if not ok: impact_files.add(cf) - impacted |= db.tests_touching_file(cf) continue for q in qns: - impact_funcs.add((cf, q)) - tests = db.tests_touching_func(cf, q) - impacted |= tests - # Mirrors the selector's default no_data_policy="file". - if not tests and q != "": - no_data.append(f"{cf}::{q}") + if db.tests_touching_func(cf, q): + impact_funcs.add((cf, q)) + else: impact_files.add(cf) - impacted |= db.tests_touching_file(cf) + no_data = res.no_data_funcs print(f"commit {args.sha[:12]} — {len(core)} core file(s), impact set:") for cf, q in sorted(impact_funcs): @@ -148,21 +129,7 @@ def main(argv: list[str] | None = None) -> int: for f in sorted(non_core): print(f" {f}") - # Untrusted capture is forced to run by the selector; it is never removable. - untrusted = db.untrusted_tests( - _WORKER_SENTINEL, - _LAUNCH_MARKERS, - _SERVING_PATH_MARKERS, - _MIN_FUNCS, - _UNTRUSTED_STAGE_MARKERS, - ) - # Keyed by family: untrusted on any shard means untrusted for the family. - untrusted_fam = {f"{stage_family(s)}/{n}" for s, n in map(split_stage, untrusted) if s} - - impacted_by_stage: dict[str, set[str]] = {} - for t in impacted: - stage, nodeid = split_stage(t) - impacted_by_stage.setdefault(stage, set()).add(nodeid) + untrusted_fam = selector.untrusted_families() def entered_changed(nodeid: str, stage: str) -> tuple[int, int, list[str]]: """(total rows, funcs entered in changed files, changed qualnames entered).""" @@ -176,8 +143,8 @@ def entered_changed(nodeid: str, stage: str) -> tuple[int, int, list[str]]: if args.stage and stage != args.stage: continue known_s = db.known_by_stage()[stage] - imp_s = impacted_by_stage.get(stage, set()) & known_s fam = stage_family(stage) + imp_s = res.impacted.get(fam, set()) & known_s forced_s = {n for n in known_s - imp_s if f"{fam}/{n}" in untrusted_fam} skip_s = known_s - imp_s - forced_s print( From da7a44d287bbdb08e5453cf95578ee534ab062ac Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:32:51 +0800 Subject: [PATCH 14/35] [TRTLLM-12838][infra] CBTS: bound closure changes the same way as import-time ones The producer records no `` frame (cbts_pystart.py drops any qualname containing one), while _collect_scopes attributes a line inside a closure to the nearest enclosing recorded scope. So a change to a closure body resolves to the function that *created* it, and that qualname's rows are its callers -- a superset of the closure's executions only while the closure cannot outlive the call that built it. Decorator wrappers, registered callbacks and cached factories all break that: the enclosing runs once at import, the closure runs in every test. Measured on the 2026-07-30 DB, closure bodies are 14191 of 646410 lines in tensorrt_llm, and 7513 of those sit under an enclosing qualname whose row set trails the file's by more than 300 tests. Over 2026-07-21..08-06, 40 of 327 commits changed a closure body; the widest gap was sampler.py's TorchSampler._process_logprobs, whose enclosing holds 15 rows against the file's 719. A one-line edit there resolved to 15 impacted tests and 521 removed cases; it now resolves to 719 and 17. Closure attributions are the same defect as import-time ones -- a qualname whose rows do not record the changed code -- so route them through the same bound: the file's row set when wider, decline when not. `_import_executed_bound` is renamed `_underrecorded_bound` for the two cases it now serves. Whole-window skip goes 42.9% -> 42.5%, with seven commits running more and two declining. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/qualname_map.py | 26 ++++++++++++++++++- .../cbts/coverage_selection/selector.py | 25 ++++++++++++------ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/qualname_map.py b/jenkins/scripts/cbts/coverage_selection/qualname_map.py index 4c0ca84dbc36..319a01db44dc 100644 --- a/jenkins/scripts/cbts/coverage_selection/qualname_map.py +++ b/jenkins/scripts/cbts/coverage_selection/qualname_map.py @@ -66,11 +66,16 @@ def walk(stmts, prefix: str, enclosing_attr: str) -> None: return scopes -def _attribute(line: int, scopes: list[_Scope]) -> str: +def _innermost(line: int, scopes: list[_Scope]) -> _Scope | None: best: _Scope | None = None for s in scopes: if s.sig_start <= line <= s.body_end and (best is None or s.sig_start > best.sig_start): best = s + return best + + +def _attribute(line: int, scopes: list[_Scope]) -> str: + best = _innermost(line, scopes) if best is None: return "" return best.sig_attr if line < best.body_start else best.body_attr @@ -107,3 +112,22 @@ def walk(stmts, prefix: str) -> None: except SyntaxError: pass return out + + +def closure_attributed_qualnames(source: str, lines: set[int]) -> set[str]: + """Qualnames a changed line only reaches by walking out of a `` scope. + + The producer records no closure frames, so such a qualname's rows are the + enclosing function's callers, not the changed code's. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + scopes = _collect_scopes(tree) + out: set[str] = set() + for line in lines: + best = _innermost(line, scopes) + if best is not None and "" in best.qualname: + out.add(_attribute(line, scopes)) + return out diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index ee2585829235..d327772c0738 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -19,7 +19,11 @@ from dataclasses import dataclass, field from pathlib import Path -from qualname_map import import_executed_qualnames, qualnames_for_lines +from qualname_map import ( + closure_attributed_qualnames, + import_executed_qualnames, + qualnames_for_lines, +) from rules._helpers import iter_diff_post_line_numbers from touch_db import ( _LAUNCH_MARKERS, @@ -117,7 +121,7 @@ def _impacted_tests( # The API omits the patch for binary / renamed / oversized files, so # which qualname changed is unknown; bound it as an import-time one. no_diff.append(path) - tests = self._import_executed_bound(cf, "") + tests = self._underrecorded_bound(cf, "") if tests is None: return impacted, no_data, no_diff, f"no usable diff, no wider row set: {path}" impacted |= tests @@ -128,12 +132,17 @@ def _impacted_tests( # Comment-only or unparsable: no qualname to resolve, file-level bound. impacted |= self.db.tests_touching_file(cf) continue - import_executed = import_executed_qualnames(source) + # Qualnames whose rows do not record the changed code: import-time bodies + # (recorded only in whichever process imported under a test context) and + # closure attributions (the producer records no `` frame at all). + underrecorded = import_executed_qualnames(source) | closure_attributed_qualnames( + source, lines + ) for qualname in sorted(qualnames): # sorted -> deterministic no_data order - if qualname in import_executed: - tests = self._import_executed_bound(cf, qualname) + if qualname in underrecorded: + tests = self._underrecorded_bound(cf, qualname) if tests is None: - why = f"import-executed scope changed, no wider row set: {path}::{qualname}" + why = f"under-recorded qualname, no wider row set: {path}::{qualname}" return impacted, no_data, no_diff, why impacted |= tests continue @@ -144,8 +153,8 @@ def _impacted_tests( impacted |= self._no_data_fallback(cf) return impacted, no_data, no_diff, None - def _import_executed_bound(self, cf: str, qualname: str) -> set[str] | None: - """File row set when it is wider than an import-time qualname's, else None (unbounded).""" + def _underrecorded_bound(self, cf: str, qualname: str) -> set[str] | None: + """File row set when it is wider than an under-recorded qualname's, else None.""" file_tests = self.db.tests_touching_file(cf) if len(file_tests) > len(self.db.tests_touching_func(cf, qualname)): return file_tests From 8c8f0fde3995436b481102ea5c7e0e82dfa20f46 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:40:26 +0800 Subject: [PATCH 15/35] [TRTLLM-12838][infra] CBTS: condense the coverage-selection comments to one line Comment-only change across the four files this branch touched most: each multi-line block or docstring added since the last pass states its point in a single line, matching the rest of the tree. No behaviour change -- replaying 2026-07-21..08-06 gives the same decisions. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../scripts/cbts/coverage_selection/qualname_map.py | 6 +----- jenkins/scripts/cbts/coverage_selection/selector.py | 10 +++------- jenkins/scripts/cbts/coverage_selection/touch_db.py | 10 ++-------- jenkins/scripts/cbts/main.py | 3 +-- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/qualname_map.py b/jenkins/scripts/cbts/coverage_selection/qualname_map.py index 319a01db44dc..ef3aca0172ef 100644 --- a/jenkins/scripts/cbts/coverage_selection/qualname_map.py +++ b/jenkins/scripts/cbts/coverage_selection/qualname_map.py @@ -115,11 +115,7 @@ def walk(stmts, prefix: str) -> None: def closure_attributed_qualnames(source: str, lines: set[int]) -> set[str]: - """Qualnames a changed line only reaches by walking out of a `` scope. - - The producer records no closure frames, so such a qualname's rows are the - enclosing function's callers, not the changed code's. - """ + """Qualnames a changed line only reaches by walking out of a `` scope.""" try: tree = ast.parse(source) except SyntaxError: diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index d327772c0738..9767969f0355 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -118,8 +118,7 @@ def _impacted_tests( diff = diffs.get(path) or "" source = self._read_source(path) if not diff.strip() or source is None: - # The API omits the patch for binary / renamed / oversized files, so - # which qualname changed is unknown; bound it as an import-time one. + # Patch omitted (binary / renamed / oversized): bound as import-time. no_diff.append(path) tests = self._underrecorded_bound(cf, "") if tests is None: @@ -132,9 +131,7 @@ def _impacted_tests( # Comment-only or unparsable: no qualname to resolve, file-level bound. impacted |= self.db.tests_touching_file(cf) continue - # Qualnames whose rows do not record the changed code: import-time bodies - # (recorded only in whichever process imported under a test context) and - # closure attributions (the producer records no `` frame at all). + # Qualnames whose rows do not record the changed code. underrecorded = import_executed_qualnames(source) | closure_attributed_qualnames( source, lines ) @@ -177,8 +174,7 @@ def _read_head(self, path: str) -> str | None: def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: """Decide over residual files; ok=False for non-core, not-in-DB, or unbounded import-time changes.""" for path in residual_files: - # Gate on the repo path, not canon(): canon matches `tensorrt_llm/` anywhere, - # so cpp/tensorrt_llm/*.py would otherwise map onto the package namespace. + # Gate on the repo path: canon() matches `tensorrt_llm/` anywhere. if not (path.endswith(".py") and path.startswith("tensorrt_llm/")): return CoverageResult(ok=False, reason=f"non-core-Python residual file: {path}") cf = canon(path) diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py index 1414bb1cb730..4980cfc6d116 100644 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -210,12 +210,7 @@ def files_touched_by(self, test: str) -> list[tuple[str, str]]: ] def incomplete_capture_tests(self) -> set[str]: - """Stage-prefixed tests the DB itself reports as incompletely captured. - - `test_meta` (schema_version 2) records each test's pytest outcome and how - many processes saved coverage against how many the coordinator spawned; - absent on older DBs, which yield an empty set. - """ + """Tests `test_meta` reports as not passed or short of the spawned process count.""" try: rows = self._conn.execute( "SELECT test FROM test_meta WHERE test != '' AND " @@ -223,8 +218,7 @@ def incomplete_capture_tests(self) -> set[str]: ) except sqlite3.OperationalError: return set() - # Intersect with the touch universe: rows for tests that recorded nothing - # are not selection candidates, and would break `untrusted <= known`. + # Tests that recorded nothing are not selection candidates. return {row[0] for row in rows} & self.known_tests() # -- coverage-completeness heuristic -- diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 229648070bc1..da9932ac8004 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -69,8 +69,7 @@ from rules.visual_gen_rule import VisualGenRule # noqa: E402 from rules.waives_rule import WaivesRule # noqa: E402 -# Files the coverage tier maps to qualnames; without their diffs it can only -# resolve at file level, and cannot tell an import-executed change apart. +# Files the coverage tier maps to qualnames; without their diffs it stays file-level. COVERAGE_NEEDS_DIFF_FOR: tuple[str, ...] = ("tensorrt_llm/**/*.py",) # --- Rule registry ----------------------------------------------------------- From c6f127910df51ff2424c185fa84f550d6f802678 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:26:33 +0800 Subject: [PATCH 16/35] [TRTLLM-12838][infra] CBTS: document the coverage tier and refresh the READMEs The docs described the rules tier only. Add two references and point the existing READMEs at them. coverage_utils/COLLECTION.md states what the collection records -- one (test, file, qualname) set per test via sys.monitoring PY_START -- and, at length, what it does not: closure frames are dropped outright, a pool worker's import phase precedes activation so module and class bodies get no rows there, and C++ frames, comprehensions, test code, Ray GPU workers and multi-GPU stages are all outside its reach. Those omissions are what the consumer side has to work around, so they are written down rather than left to be rediscovered. coverage_selection/SELECTION.md covers the decision: the qualname attribution table (function body is the only precise case; signature lines and class bodies resolve to import-time scopes, closure bodies to their enclosing function), the three decline gates, how an under-recorded qualname is bounded by the file's row set or declined, the narrowing verdicts, and where the DB comes from. cbts/README.md gains a selection-tier table and a file map covering coverage_tier.py, coverage_selection/ and the two new tools; its fallback list gains the Tier 2 declines. coverage_utils/README.md records the closure and import-phase gaps under Granularity. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 24 +++ .../cbts/coverage_selection/SELECTION.md | 180 ++++++++++++++++++ .../scripts/cbts/coverage_utils/COLLECTION.md | 149 +++++++++++++++ jenkins/scripts/cbts/coverage_utils/README.md | 5 + 4 files changed, 358 insertions(+) create mode 100644 jenkins/scripts/cbts/coverage_selection/SELECTION.md create mode 100644 jenkins/scripts/cbts/coverage_utils/COLLECTION.md diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 31b29a32cec9..a567cb42ec44 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -5,6 +5,16 @@ run, based on what the PR changed. New rules are added in Python only. --- +## Selection tiers + +| Tier | Where | Basis | +|---|---|---| +| **1. Rules** | `rules/` | Path patterns and diffs. Each rule claims the files it understands and narrows to the blocks they affect. | +| **2. Coverage** | `coverage_selection/`, `coverage_tier.py` | Runs only on what Tier 1 left unclaimed (the *residual*), and only when every residual file is core Python present in the touch DB. Maps changed lines to qualnames and removes entries no changed function reaches. Declines to the full run otherwise. See `coverage_selection/SELECTION.md`. | + +The touch DB is produced by the post-merge collection under `coverage_utils/`; +see `coverage_utils/COLLECTION.md` for what it does and does not record. + ## Consumption layers CBTS narrows test cases only; Build always runs. @@ -75,8 +85,18 @@ jenkins/scripts/cbts/ │ ├── visual_gen_rule.py │ ├── spec_dec_rule.py │ └── out_of_scope_rule.py +├── coverage_tier.py Tier 2 entry: applies the selector to the test-db YAMLs, classifies every candidate entry +├── coverage_selection/ +│ ├── SELECTION.md how a decision is made: qualname concepts, decline gates, narrowing +│ ├── selector.py CoverageSelector.decide(): changed lines → qualnames → impacted / skippable per stage family +│ ├── qualname_map.py changed lines → co_qualname, plus the import-time and closure classifications +│ ├── touch_db.py read-only accessor over cbts_touchmap.sqlite + the untrusted-capture signals +│ └── artifact.py resolve and fetch the touch DB from Artifactory (picks by collected revision) +├── coverage_utils/ post-merge collection that produces the touch DB (see its README / COLLECTION.md) └── tools/ ├── dryrun.py replay CBTS over historical commits → per-PR summary.txt + filtered YAMLs + INDEX.md (debug only) + ├── coverage_audit.py report a touch DB's format, scale, untrusted rate and HEAD coverage gap + ├── coverage_explain.py explain one commit's decision case by case (delegates to CoverageSelector) └── report_cbts_decision.py post the decision (hit-stage count, case-level skip rate, fallback) to OpenSearch for CI-health monitoring ``` @@ -294,6 +314,10 @@ CBTS defers to the existing filter chain when: but cannot decide; e.g. testdef blast-radius cap, testlist structural YAML edit) - Combined scope is `None` (incompatible mix) +- Tier 2 declines: a residual file is not core Python, is absent from the + touch DB, or its changed qualname is under-recorded with no wider row set + (see `coverage_selection/SELECTION.md` §3-4) +- No touch DB artifact could be resolved — Tier 2 never runs - Layer 3 narrowing would empty a block — block keeps original tests - `cbts_test_db` tarball upload or download/extraction fails — renderTestDB falls back to source - Narrowed YAML missing/empty on a stage agent — renderTestDB falls back diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md new file mode 100644 index 000000000000..37c7ec7cb00a --- /dev/null +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -0,0 +1,180 @@ +# CBTS Coverage Selection — Current State + +How the touch DB is turned into a decision about which cases to run. Collection side: +`../coverage_utils/COLLECTION.md`. + +--- + +## 1. Where this sits + +``` +Tier 1 rules every changed file claimed by a rule → narrow per the rule + │ some file left unclaimed (residual) + ▼ +Tier 2 coverage residual is all core Python and all present in the DB → scope=coverage + │ any condition unmet → decline + ▼ + full fallback scope=null +``` + +Tier 2 only ever looks at the **residual**: the files no Tier 1 rule claimed. + +## 2. The qualname concepts + +Precision comes down to which qualname a changed line lands on. `qualname_map`'s attribution: + +| Changed line sits in | Attributed to | Note | +|---|---|---| +| A function or method body | `Class.method` / `func` | the only precise case | +| A module-level statement | `` | module body, executed once at import | +| A class body (class attribute) | `ClassName` | class body, executed once at import | +| **A signature or decorator line** | **the enclosing scope** | a method's `def` line lands on `ClassName`, not on the method | +| A closure body (``) | **the nearest recorded enclosing scope** | changing `inner` in `def outer(): def inner()` lands on `outer` | + +The last three are where both the precision and the correctness problems come from: **the +attributed qualname's DB rows do not represent the changed code's executions.** + +DB qualnames come from `co_qualname`, so they match what the AST derives (`Class.method`), but +comprehensions, lambdas and closures are skipped during collection, so those names do not exist +in the DB at all. + +## 3. The three decline gates + +`CoverageSelector.decide()` returns `ok=False` — Tier 2 stands down and the run is full — on any +of: + +| Gate | Condition | +|---|---| +| Non-core file | `path` is not a `.py` under `tensorrt_llm/` (checked on the **repo path**, not `canon()`) | +| File absent from the DB | `file_has_touch_rows(cf)` is false — new or uninstrumented, so "who touches it" is unknown rather than empty | +| Under-recorded with no wider bound | see the next section | + +## 4. Core: handling an under-recorded qualname + +Two kinds of qualname have DB rows that **do not cover the changed code's executions**: + +| Kind | Attributed qualname | Collection-side problem | +|---|---|---| +| Import-time (module body / class body / signature line) | itself | only recorded by tests that spawn subprocesses; a pool worker's import precedes activation | +| Closure | the enclosing function | **not recorded at all**; a closure can outlive the call that created it | + +Both go through `_underrecorded_bound(cf, qualname)`: + +``` +file rows > that qualname's rows → use the file rows (the wider bound) +otherwise → decline, run in full +``` + +### 4.1 What the comparison asks + +Whether falling back to file level actually recovers the tests that were missed. + +`tests_touching_func(f, q)` is by construction a subset of `tests_touching_file(f)`, so a strictly +larger file set means it contains tests the qualname set does not. Those extra tests are the ones +that recorded some *other* function in the same file — evidence that the file is exercised beyond +import time, which is what makes the file set a usable bound. When the two are equal the file is +only ever recorded at import time (`__init__.py` and similar, 881 of 1149 files), nothing is +recovered, and no sound bound exists. + +### 4.2 The bound is not complete + +A test that imports a file but never calls any function in it records nothing there, so even the +file set misses it. The bound is the widest one available from the data, not a proof; eliminating +the remainder means recording the pool workers' import phase — see +`../coverage_utils/COLLECTION.md` §5.2. + +### 4.3 When no diff is available + +The forge API omits the patch for binary, renamed and oversized files. The changed qualname is +then unknown, so the worst case is assumed and `` is put through the same test; the count +of such files is reported as `coverage_no_diff_files`. + +## 5. Full impact resolution + +``` +for each residual file: + ├ no usable diff → _underrecorded_bound(cf, "") bound or decline + ├ diff has no lines → whole file (the change was comments only) + └ for each changed qualname: + ├ import-time or closure → _underrecorded_bound(cf, q) bound or decline + ├ has DB rows → tests_touching_func(cf, q) precise + └ no DB rows → no_data_policy fallback (file by default) +``` + +Comments and blank lines are stripped upstream by `strip_noop_diff_lines` (`^\s*#` and empty +lines), so a change that edits a function body and adds a module-level comment is not mistaken +for a module-level change. + +## 6. Tests that are never skipped + +`untrusted_tests()` unions four signals; anything it flags runs regardless: + +| Signal | Meaning | +|---|---| +| `test_meta` reports incomplete | `outcome != passed`, or `saved_procs < expected_workers + 1` — the DB's own account | +| Drove inference but has no `py_executor` rows | the coordinator ran but the worker's coverage never arrived | +| Footprint < 30 functions | the record is near-empty | +| On a `-Ray-` stage | the GPU worker is uninstrumented, only driver-side rows exist | + +Plus **CPU stages always run** (`ALWAYS_RUN_STAGE_PREFIX`): `main.py` adds them to +`affected_stages`, and `coverage_tier` drops their families from `instrumented` so no block they +serve is ever pruned. + +## 7. From test sets to a narrowed test-db + +`coverage_tier._build_narrowing` classifies every entry of every block and removes only the `SAFE` +ones: + +| Verdict | Meaning | +|---|---| +| `rule_kept` | a Tier 1 rule already asked to keep it | +| `coarse` | the entry carries `-k` and expands to many nodeids, so there is no 1:1 DB key | +| `no_data` | the entry has no rows in the DB | +| `impacted` | it entered changed code | +| `untrusted` | see the previous section | +| **`safe`** | none of the above → **removed** | + +Entries are keyed by **stage family**: `A10-PyTorch-2` → `A10-PyTorch`. pytest-split assigns each +entry to exactly one shard and rebalances by duration, so only the family-level union answers "was +this entry ever captured on this stage". + +A block is left untouched unless every stage family it serves is instrumented — coverage is +collected on single-GPU post-merge stages only, so blocks belonging to multi-GPU or +Post-Merge-only stages are never pruned. + +Single-GPU stages left with nothing go into `coverage_dropped_stages`. Multi-GPU stages are +omitted here and re-added by Groovy under the `MULTI_GPU_FILE_CHANGED` gate. + +## 8. Where the DB comes from + +`artifact.py` picks among the post-merge artifacts: it reads each build's `build_info.txt` for +`commit=` and takes the one whose commit trails the current checkout least, keeping the build +number only as a tie-break when the commit cannot be resolved — build numbers do not order +revisions, since a build can be a re-run of an older commit. + +The chosen build, its commit and its distance from HEAD are recorded in the decision +(`coverage_db_build` / `coverage_db_commit` / `coverage_db_lag`). **These are recorded, not +gated**: observed lag varies widely even in healthy operation, so a threshold needs data first. + +## 9. Decision output + +```json +{ + "scope": "coverage", + "affected_stages": [...], + "affected_stage_test_counts": {...}, + "affected_stage_split_counts": {...}, + "test_db_dir_override": "cbts_test_db", + "enable_multi_gpu": true, + "coverage_dropped_stages": [...], + "coverage_db_build": 2887, + "coverage_db_commit": "50edd738...", + "coverage_db_lag": 11, + "coverage_no_diff_files": 0, + "reasons": [{"source": "coverage", "impacted": 118, "untrusted": 104, ...}] +} +``` + +Groovy filters stages by `affected_stages` and renders each stage's test list from +`test_db_dir_override`; when that YAML is absent the stage falls back to the source test-db, i.e. +runs in full. diff --git a/jenkins/scripts/cbts/coverage_utils/COLLECTION.md b/jenkins/scripts/cbts/coverage_utils/COLLECTION.md new file mode 100644 index 000000000000..868d3e5002f7 --- /dev/null +++ b/jenkins/scripts/cbts/coverage_utils/COLLECTION.md @@ -0,0 +1,149 @@ +# CBTS Coverage Collection — Current State + +How per-test coverage data is produced on this branch, and **what it does not record**. +Consumer side: `../coverage_selection/SELECTION.md`. File-by-file roles: `README.md`. + +--- + +## 1. What is recorded + +One set per test: **which product functions it entered**. + +``` +(test, file, qualname) + │ │ └─ the code object's co_qualname, e.g. "PyExecutor._executor_loop" + │ └─ product file, canon'd to the tensorrt_llm/... form + └─ "/" +``` + +No line numbers, no call counts, no call graph. + +## 2. How + +`sys.monitoring`'s `PY_START` event (Python 3.12+, tool id 4): + +```python +def _on_py_start(self, code, offset): + if self._in_source(code.co_filename): + qual = code.co_qualname + if "" not in qual and qual not in _SKIP_QUALNAMES: + self._data[self._ctx].add((fn, qual)) + return _MON.DISABLE +``` + +- Each code object fires once on **first entry**, then `DISABLE`s itself: zero cost afterwards. +- Each test re-arms with `restart_events()`, so every test gets its own complete function set. +- Cost scales with how many **distinct** functions were entered, not with how often. + +## 3. Crossing processes + +Three environment variables (`CBTS_COVERAGE_CONFIG` / `PYTHONPATH` / `CBTS_MARKER_FILE`) are +inherited by child processes by default, and `sitecustomize.py` installs the tracker in every +Python process at startup. No product code is modified. + +| Process | Context source | Activation | +|---|---|---| +| Outer pytest | `cbts_plugin`'s `pytest_runtest_protocol` switches it directly | at interpreter startup | +| MPI pool worker | inherited `CBTS_TEST_ID`, then polls the marker file (0.1s) | **deferred** until `tensorrt_llm` finishes importing | +| Other subprocesses (serve / example / inner pytest) | same as above | at interpreter startup | + +Processes that opt themselves out: `pip` / `setup.py` / `cmake` / `ninja` and everything they +spawn, plus Ray infrastructure processes (`default_worker.py` and friends). + +## 4. Persistence and merge + +Every process writes its own SQLite (`.cbtscov...X.pid.sqlite`), on a 5s +periodic snapshot plus `atexit`: + +| Table | Contents | +|---|---| +| `touch(test, file, qualname)` | everything this process recorded | +| `proc_meta(stage)` | the stage name | +| `test_meta(test, outcome, expected_workers)` | coordinator only: pytest outcome + how many workers it spawned | + +The merge (`pystart_report.py`) unions across processes and derives `saved_procs` — how many +per-process files contributed rows for that `(test, stage)`. Final artifact: +`cbts_touchmap.sqlite`. + +--- + +## 5. What is not recorded (important) + +This section is what forces the concessions on the consumer side. + +### 5.1 Closures — not recorded at all, **still unresolved** + +```python +if "" not in qual and ... +``` + +Any frame whose qualname contains `` is dropped outright. A decorator's `wrapper`, a +registered callback, the inner function of a cached factory — none leave a trace however often +they run. + +Measured scale: `tensorrt_llm` is 646410 lines, of which **14191 (2.2%)** are closure bodies; +7513 of those sit under an enclosing function whose row set trails the whole file's by more than +300 tests. + +The consumer side can only widen a closure change to file level (see `../coverage_selection/SELECTION.md` §4); +**the collection side itself is unfixed.** A real fix means having the producer record closure +frames (the `` segments in `co_qualname` can be kept or folded), which is follow-up work. + +### 5.2 Import phase is lost inside pool workers + +A worker's `tensorrt_llm` import happens **before deferred activation**, so module bodies +(``) and class bodies (`ClassName`) get no rows at all in a worker. + +The outer pytest's import happens before any test, under the empty context, and the merge filters +it out with `WHERE test != ''`. + +Net effect: `` and class bodies are only recorded by the tests that spawn subprocesses. +Measured, `llmapi/llm_args.py::` has **509** holders while all **746** known tests import +it from their process — the missing 226 are exactly the accuracy / disagg tests served by MPI pool +workers. + +Deferred activation is deliberate: without it, the instrumented cold-start import overruns the +`wait_shutdown` worker identity barrier. + +### 5.3 Other blind spots + +| Blind spot | Reason | +|---|---| +| C++ / nanobind implementations | PY_START only sees Python frames; the C++ side of KV cache manager, scheduler and decoder is invisible | +| Comprehensions / genexprs / lambdas | skipped explicitly by `_SKIP_QUALNAMES` | +| Test code itself | `tests/` is outside the source root | +| A Ray stage's GPU worker | `RayGPUWorker` lives in the opted-out `default_worker.py` | +| Multi-GPU / multi-node stages | phase 1 collects on single-GPU stages only | +| The last ≤5s before a worker is SIGKILLed | the periodic snapshot interval | + +--- + +## 6. The completeness signal the data carries + +`test_meta` lets the consumer decide whether a record can be trusted: + +```sql +outcome IS NULL OR outcome != 'passed' OR saved_procs < expected_workers + 1 +``` + +`expected_workers` is counted in the coordinator by the patched `MPIPoolExecutor.__init__`, +`saved_procs` is counted by the merge, and the `+1` is the coordinator itself. A mismatch means +some process's coverage was lost. + +This is the only way to detect lost data — the loss is invisible in the data itself (footprint +stays large, `py_executor` rows are still there). + +--- + +## 7. When collection runs + +`L0_MergeRequest.groovy` decides pipeline-level eligibility and `isCbtsStage()` decides each +stage: + +- official post-merge pipeline only (`ENABLE_CBTS_COVERAGE && JOB_NAME ==~ /.*PostMerge.*/`) +- not a perf stage, not a TensorRT / CPP / AutoDeploy stage +- single-GPU stages only (name carries no `-_GPUs` / `-_Nodes`) +- not listed in `CBTS_EXCLUDE_STAGES` + +The per-process files ride back inside `results-.tar.gz`; the `Test Coverage` stage merges +them all and uploads to `${UPLOAD_PATH}/cbts-coverage/cbts_pystart_report.tar.gz`. diff --git a/jenkins/scripts/cbts/coverage_utils/README.md b/jenkins/scripts/cbts/coverage_utils/README.md index 7d20735a4a35..6575725048e8 100644 --- a/jenkins/scripts/cbts/coverage_utils/README.md +++ b/jenkins/scripts/cbts/coverage_utils/README.md @@ -9,6 +9,9 @@ Capture uses `sys.monitoring` `PY_START` (Python 3.12+): each function a test en then that code object is disabled until the next test — so overhead scales with functions entered, not lines executed (far cheaper than line tracing). +`COLLECTION.md` summarises what the data model is and, importantly, what it does not record; +`../coverage_selection/SELECTION.md` covers how the consumer side uses it. + ## Files | File | Role | @@ -39,6 +42,8 @@ Non-CBTS stages get an empty `.coveragerc` and run uninstrumented. - **Integration tests**: the outer pytest carries `-p cbts_plugin`, so each test-db entry (one pytest item) is its own context. - **Unit tests** (`test_unittests_v2[entry]`): the inner pytest carries no plugin, so the whole batch runs under the one inherited `CBTS_TEST_ID` context = the test-db entry. This matches CBTS's selection granularity (entry level). - `co_qualname` gives `Class.method`, so results roll up to function → class → file. Comprehension / generator / lambda frames are skipped. +- **Closures are skipped too**: any frame whose qualname contains `` is dropped, so a decorator's `wrapper` or a registered callback leaves no row however often it runs. The consumer side compensates by widening such a change to file level; see `COLLECTION.md` §5.1. +- **A pool worker's import phase is not captured**: activation is deferred until `tensorrt_llm` has imported, so module and class bodies get no rows there. See `COLLECTION.md` §5.2. ## Output From 6e637d420b8b1b6a1903cc5076a50253ee8253cd Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:04:18 +0800 Subject: [PATCH 17/35] [TRTLLM-12838][infra] CBTS: measure coverage-DB lag via the forge compare API The DB candidates are ranked by how far main has moved past the revision each was collected at, which until now was measured with git rev-list against the workspace. That never answers in CI: trtllm_utils.checkoutSpec clones depth 1 with noTags and a single-SHA refspec, so no candidate revision is in the object store and every lag came back null -- collapsing the ranking to its build-number tie-break, which is exactly what commit-based ranking was added to avoid, since a post-merge build can be a re-run of an older commit. Fall back to GitHub's compare API (`ahead_by` on ...main) when git cannot answer. `ahead_by` covers the full range; only the response's commits array is truncated at 250. Results are cached per revision, so probing ten builds costs one call per distinct revision. The token is required rather than optional: the anonymous 60/h quota is keyed to the caller's IP and NVIDIA's shared egress already exhausts it (verified: an unauthenticated compare from a corporate address returns 403 rate-limit). Bind github-cred-trtllm-ci -- the credential getGithubMRChangedFile already uses -- around the --print-selection call and read it from GITHUB_API_TOKEN. Lag stays advisory: it is recorded in the decision, not gated on, so a 403, an unmirrored revision or a missing token degrades to the previous behaviour rather than declining the tier. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 20 ++++-- .../cbts/coverage_selection/SELECTION.md | 57 ++++++++++++++--- .../cbts/coverage_selection/artifact.py | 61 ++++++++++++++++--- jenkins/scripts/cbts/main.py | 2 +- 4 files changed, 117 insertions(+), 23 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 39a3d67ef548..646ea77603fe 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -933,11 +933,21 @@ def _cbtsCoverageAudit(pipeline) def covDir = "cbts_cov" // Selection is by collected revision, not build number; the JSON also carries // the commit and how far HEAD runs ahead of it, both recorded in the decision. - def selJson = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + - "--print-selection --repo-root ${LLM_ROOT} || true", - returnStdout: true, - ).trim() + // GITHUB_API_TOKEN: the checkout is depth-1, so lag comes from the compare API instead of git. + def selJson = "" + withCredentials([ + usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN' + ), + ]) { + selJson = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + + "--print-selection --repo-root ${LLM_ROOT} || true", + returnStdout: true, + ).trim() + } if (!selJson) { pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") return [path: "", build: null, commit: "", lag: null] diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 37c7ec7cb00a..ce6589a1dde7 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -147,14 +147,55 @@ omitted here and re-added by Groovy under the `MULTI_GPU_FILE_CHANGED` gate. ## 8. Where the DB comes from -`artifact.py` picks among the post-merge artifacts: it reads each build's `build_info.txt` for -`commit=` and takes the one whose commit trails the current checkout least, keeping the build -number only as a tie-break when the commit cannot be resolved — build numbers do not order -revisions, since a build can be a re-run of an older commit. - -The chosen build, its commit and its distance from HEAD are recorded in the decision -(`coverage_db_build` / `coverage_db_commit` / `coverage_db_lag`). **These are recorded, not -gated**: observed lag varies widely even in healthy operation, so a threshold needs data first. +Only one producer exists: the `LLM/main/L0_PostMerge` job, which uploads +`/cbts-coverage/cbts_pystart_report.tar.gz`. Every DB therefore describes some revision of +`main` (`artifact.COVERAGE_BRANCH`), and picking one means picking **which revision of `main`** +the selection reasons about. + +### 8.1 Resolution + +``` +Jenkins REST lastBuild → newest build number N +for b in N .. N-9: (_MAX_PROBE) + ranged GET the tarball → skip b if absent + GET build_info.txt, parse `commit=` → sha, or None + lag(sha) → how far main moved past it +rank by (lag known, lag ascending, build descending) +``` + +Ranking is by **revision, not build number**: a post-merge build can be a re-run of an older +commit, so the highest build number is not necessarily the newest code. The build number is only +the tie-break, and when no candidate's lag can be measured the ranking degenerates to exactly that +tie-break — which is the pre-existing behaviour, not a regression. + +### 8.2 Measuring the lag + +Two sources, in order; `artifact.db_lag()` falls through: + +| Source | Answers when | Fails when | +|---|---|---| +| `git rev-list --count ..HEAD` | the checkout has history — local runs, dev tooling | **always in CI**: `trtllm_utils.checkoutSpec` clones `depth: 1, noTags: true` with a single-SHA refspec, so no candidate revision is in the object store | +| GitHub compare `...main` → `ahead_by` | a token is bound and the revision is public | the revision has not reached the public mirror yet (404); no token (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is routinely already spent) | + +Both failing leaves `lag: null`. Each failure prints its own reason to stderr — a missing working +directory, git's own `fatal: bad object `, or the HTTP status — so the CI log distinguishes a +wiring mistake from a shallow checkout from a rate limit. + +The token comes from the `github-cred-trtllm-ci` credential, bound around the `--print-selection` +call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. `compare_distance` is cached per +revision, so probing ten builds costs one API call per *distinct* revision. + +### 8.3 What happens with the result + +The tarball is downloaded (retried), the sqlite extracted, and `coverage_audit.py` run over it; +any failure in this whole path is caught and non-fatal — `coverageDb.path` stays empty, Tier 2 +never runs, and the PR gets a full run. + +The chosen build, its commit and its lag ride into `main.py` and are recorded in the decision +(`coverage_db_build` / `coverage_db_commit` / `coverage_db_lag`; an unmeasurable lag is `null` +here and `-1` in the OpenSearch record). +**These are recorded, not gated**: observed lag varies widely even in healthy operation, so a +threshold needs data first. ## 9. Decision output diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index e15b20fed441..ddbda67a807c 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -17,12 +17,16 @@ `//cbts-coverage/cbts_pystart_report.tar.gz` (sqlite at the tar root plus `cbts_report/`). -`latest_tarball_url()` reads the newest build number from the Jenkins REST API, -then walks builds down, probing Artifactory with a 1-byte ranged GET until it -finds one whose tarball exists. - -Two entry points for the Groovy wiring: +`select_tarball()` reads the newest build number from the Jenkins REST API, walks +builds down probing Artifactory with a 1-byte ranged GET, and ranks the ones that +exist by how far `COVERAGE_BRANCH` has moved past the revision each collected +(`build_info.txt`), since a build can be a re-run of an older commit. That +distance comes from local git, or from the forge compare API when the CI +checkout is too shallow to answer; the build number is only a tie-break. + +Three entry points for the Groovy wiring: * `--print-url` — resolve and print the tarball URL only (no download). + * `--print-selection` — print `{url, build, commit, lag}` as JSON. * `--dest DIR` — download + extract, printing the local sqlite path. """ @@ -30,12 +34,14 @@ import argparse import json +import os import shutil import subprocess import sys import tarfile import urllib.error import urllib.request +from functools import lru_cache from pathlib import Path from typing import Optional @@ -46,7 +52,13 @@ # Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" +# Branch the DB is collected from; must match ARTIFACT_BASE. +COVERAGE_BRANCH = "main" +# Read by `compare_distance`; the anonymous quota is unusable from shared CI egress IPs. +GITHUB_TOKEN_ENV = "GITHUB_API_TOKEN" + _URM = "https://urm.nvidia.com/artifactory" +_GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Max builds to walk back when recent builds have no tarball. _MAX_PROBE = 10 @@ -54,9 +66,10 @@ _TIMEOUT = 15 -def _get(url: str) -> tuple[Optional[int], Optional[bytes]]: +def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: + req = urllib.request.Request(url, headers=headers or {}) try: - with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp: + with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp: return resp.status, resp.read() except urllib.error.HTTPError as e: return e.code, None @@ -126,6 +139,36 @@ def commit_distance(commit: str, repo_root: str = ".", ref: str = "HEAD") -> Opt return int(out.stdout.strip()) if out.returncode == 0 and out.stdout.strip() else None +@lru_cache(maxsize=None) +def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int]: + """Commits `branch` gained since `commit`, from the forge compare API, or None.""" + headers = {"Accept": "application/vnd.github+json"} + token = os.environ.get(GITHUB_TOKEN_ENV) + if token: + headers["Authorization"] = f"Bearer {token}" + status, data = _get(f"{_GITHUB_COMPARE}/{commit}...{branch}", headers) + if status != 200 or not data: + # 403 without a token means the shared egress IP burned the 60/h anonymous quota. + hint = " (no token: anonymous quota)" if status == 403 and not token else "" + print( + f"[artifact] compare {commit[:10]}...{branch} failed: HTTP {status}{hint}", + file=sys.stderr, + ) + return None + try: + # `ahead_by` counts the full range; only the `commits` array is truncated at 250. + return int(json.loads(data)["ahead_by"]) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as e: + print(f"[artifact] compare {commit[:10]}...{branch}: bad response: {e}", file=sys.stderr) + return None + + +def db_lag(commit: str, repo_root: str = ".") -> Optional[int]: + """How far `COVERAGE_BRANCH` moved past `commit`: local git first, then the compare API.""" + lag = commit_distance(commit, repo_root) + return lag if lag is not None else compare_distance(commit) + + def select_tarball( artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, @@ -143,7 +186,7 @@ def select_tarball( if not _exists(url): continue commit = build_commit(b, artifact_base) - lag = commit_distance(commit, repo_root) if commit else None + lag = db_lag(commit, repo_root) if commit else None candidates.append({"url": url, "build": b, "commit": commit, "lag": lag}) if not candidates: print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) @@ -235,7 +278,7 @@ def main(argv: Optional[list[str]] = None) -> int: "url": url, "build": args.build, "commit": commit, - "lag": commit_distance(commit, args.repo_root) if commit else None, + "lag": db_lag(commit, args.repo_root) if commit else None, } else: best = select_tarball(repo_root=args.repo_root) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index da9932ac8004..1f084d61d92e 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -371,7 +371,7 @@ def main(argv: Optional[list[str]] = None) -> int: "--coverage-db-lag", type=int, default=None, - help="Commits HEAD is ahead of --coverage-db-commit; recorded in the decision.", + help="Commits main gained since --coverage-db-commit; recorded in the decision.", ) parser.add_argument( "--no-data-policy", From 58c27591131ca9abc53f105ddbac299cc4332912 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:09:25 +0800 Subject: [PATCH 18/35] [TRTLLM-12838][infra] CBTS: make the compare API authoritative for the DB lag db_lag tried git first and the compare API only as a fallback, but the two were not measuring the same thing: the API compares against the tip of main while git compared against HEAD, which on a feature branch includes that branch's own commits. Worse, git reports a stale ref as a smaller number rather than an error. The same DB measured 51 commits behind HEAD, 0 behind a stale local main, and 32 behind upstream/main -- only the last is the real distance, and the first two carry no signal that they are wrong. Query the API first and drop to git only when it cannot answer (no token, no network, unmirrored revision), trying upstream/main, origin/main and main in turn so the local number means the same thing the API's does. In CI this also stops ten pointless subprocess calls, since the depth-1 checkout never has any candidate revision. The scale stays the tip of main rather than the PR's base commit: candidates have to be scored against a common head, and a PR based on a commit older than every candidate would score them all zero and collapse the ranking back to the build number. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/SELECTION.md | 20 ++++++++++++++----- .../cbts/coverage_selection/artifact.py | 17 ++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index ce6589a1dde7..af03f6606caf 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -170,16 +170,26 @@ tie-break — which is the pre-existing behaviour, not a regression. ### 8.2 Measuring the lag -Two sources, in order; `artifact.db_lag()` falls through: +The lag is always measured against the **tip of `main`**, never against the PR's own base commit: +every candidate is scored on the same scale, and a PR whose base predates all the candidates would +otherwise score them all identically and collapse the ranking back to the build number. + +Two sources; `artifact.db_lag()` falls through: | Source | Answers when | Fails when | |---|---|---| -| `git rev-list --count ..HEAD` | the checkout has history — local runs, dev tooling | **always in CI**: `trtllm_utils.checkoutSpec` clones `depth: 1, noTags: true` with a single-SHA refspec, so no candidate revision is in the object store | | GitHub compare `...main` → `ahead_by` | a token is bound and the revision is public | the revision has not reached the public mirror yet (404); no token (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is routinely already spent) | +| `git rev-list --count ..` over `upstream/main`, `origin/main`, `main` | a local clone tracks the branch — dev runs, offline | **always in CI**: `trtllm_utils.checkoutSpec` clones `depth: 1, noTags: true` with a single-SHA refspec, so no candidate revision is in the object store | + +The API is authoritative and git is the backup, not the other way round: git answers relative to +whatever local ref is named, and a ref that is merely stale returns a *smaller* number rather than +an error. On one checkout the same DB measured 51 commits behind `HEAD` (the feature branch's own +commits inflating it), 0 behind a stale local `main`, and 32 behind `upstream/main` — only the last +is the real distance, and nothing in the first two signals that they are wrong. -Both failing leaves `lag: null`. Each failure prints its own reason to stderr — a missing working -directory, git's own `fatal: bad object `, or the HTTP status — so the CI log distinguishes a -wiring mistake from a shallow checkout from a rate limit. +Both sources failing leaves `lag: null`. Each failure prints its own reason to stderr — a missing +working directory, git's own `fatal: bad object `, or the HTTP status — so the CI log +distinguishes a wiring mistake from a shallow checkout from a rate limit. The token comes from the `github-cred-trtllm-ci` credential, bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. `compare_distance` is cached per diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index ddbda67a807c..c01b8767a250 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -59,6 +59,8 @@ _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" +# Local stand-ins for COVERAGE_BRANCH, best first; a stale ref understates the lag. +_LOCAL_REFS = (f"upstream/{COVERAGE_BRANCH}", f"origin/{COVERAGE_BRANCH}", COVERAGE_BRANCH) _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Max builds to walk back when recent builds have no tarball. _MAX_PROBE = 10 @@ -123,7 +125,7 @@ def build_commit(build: int, artifact_base: str = ARTIFACT_BASE) -> Optional[str return None -def commit_distance(commit: str, repo_root: str = ".", ref: str = "HEAD") -> Optional[int]: +def commit_distance(commit: str, repo_root: str = ".", ref: str = COVERAGE_BRANCH) -> Optional[int]: """Commits in `ref` not reachable from `commit`, or None if git cannot answer.""" try: out = subprocess.run( @@ -164,9 +166,16 @@ def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int def db_lag(commit: str, repo_root: str = ".") -> Optional[int]: - """How far `COVERAGE_BRANCH` moved past `commit`: local git first, then the compare API.""" - lag = commit_distance(commit, repo_root) - return lag if lag is not None else compare_distance(commit) + """Commits `COVERAGE_BRANCH` gained since `commit`; the API is authoritative, git is the backup.""" + lag = compare_distance(commit) + if lag is not None: + return lag + # Only reached without a token or network: whichever local ref tracks the branch answers. + for ref in _LOCAL_REFS: + lag = commit_distance(commit, repo_root, ref) + if lag is not None: + return lag + return None def select_tarball( From ed2cf984558fa4431baf4ccc13dcdd43783b890a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:32:09 +0800 Subject: [PATCH 19/35] [TRTLLM-12838][infra] CBTS: drop the local-git path for the coverage-DB lag commit_distance could not fire in the pipeline: the checkout is a depth-1 clone with a single-SHA refspec, so no candidate revision is ever in the object store and the git call failed on every candidate before the compare API answered. Keeping it also kept a second, differently-defined measurement -- git answers against whatever ref it is handed, and a merely stale ref returns a smaller number rather than an error -- plus the --repo-root plumbing that existed only to feed it. Query the compare API and nothing else. Without a token the lag is null and the ranking degrades to its build-number tie-break, which is what happened anyway whenever git could not answer. Also collapse the credential binding to the file's existing one-line form and correct the comment above it: the reference point is the tip of main, not HEAD. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 17 ++----- .../cbts/coverage_selection/SELECTION.md | 48 +++++++++---------- .../cbts/coverage_selection/artifact.py | 44 ++--------------- 3 files changed, 34 insertions(+), 75 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 646ea77603fe..696b9b7f8e65 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -931,20 +931,13 @@ def _cbtsCoverageAudit(pipeline) // All commands run from ${LLM_ROOT}; covDir and the returned path are // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. def covDir = "cbts_cov" - // Selection is by collected revision, not build number; the JSON also carries - // the commit and how far HEAD runs ahead of it, both recorded in the decision. - // GITHUB_API_TOKEN: the checkout is depth-1, so lag comes from the compare API instead of git. + // Selection is by collected revision, not build number; the JSON also carries the commit + // and how far main has moved past it, both recorded in the decision. The token is what + // lets artifact.py measure that — the depth-1 checkout cannot. def selJson = "" - withCredentials([ - usernamePassword( - credentialsId: 'github-cred-trtllm-ci', - usernameVariable: 'NOT_USED_YET', - passwordVariable: 'GITHUB_API_TOKEN' - ), - ]) { + withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { selJson = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + - "--print-selection --repo-root ${LLM_ROOT} || true", + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-selection || true", returnStdout: true, ).trim() } diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index af03f6606caf..5e58f7cb662d 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -170,30 +170,30 @@ tie-break — which is the pre-existing behaviour, not a regression. ### 8.2 Measuring the lag -The lag is always measured against the **tip of `main`**, never against the PR's own base commit: -every candidate is scored on the same scale, and a PR whose base predates all the candidates would -otherwise score them all identically and collapse the ranking back to the build number. - -Two sources; `artifact.db_lag()` falls through: - -| Source | Answers when | Fails when | -|---|---|---| -| GitHub compare `...main` → `ahead_by` | a token is bound and the revision is public | the revision has not reached the public mirror yet (404); no token (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is routinely already spent) | -| `git rev-list --count ..` over `upstream/main`, `origin/main`, `main` | a local clone tracks the branch — dev runs, offline | **always in CI**: `trtllm_utils.checkoutSpec` clones `depth: 1, noTags: true` with a single-SHA refspec, so no candidate revision is in the object store | - -The API is authoritative and git is the backup, not the other way round: git answers relative to -whatever local ref is named, and a ref that is merely stale returns a *smaller* number rather than -an error. On one checkout the same DB measured 51 commits behind `HEAD` (the feature branch's own -commits inflating it), 0 behind a stale local `main`, and 32 behind `upstream/main` — only the last -is the real distance, and nothing in the first two signals that they are wrong. - -Both sources failing leaves `lag: null`. Each failure prints its own reason to stderr — a missing -working directory, git's own `fatal: bad object `, or the HTTP status — so the CI log -distinguishes a wiring mistake from a shallow checkout from a rate limit. - -The token comes from the `github-cred-trtllm-ci` credential, bound around the `--print-selection` -call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. `compare_distance` is cached per -revision, so probing ten builds costs one API call per *distinct* revision. +The lag is `ahead_by` from GitHub's compare API on `...main` — always against the **tip of +`main`**, never against the PR's own base commit: every candidate is scored on the same scale, and +a PR whose base predates all the candidates would otherwise score them all identically and collapse +the ranking back to the build number. `ahead_by` covers the full range; only the response's +`commits` array is truncated at 250. + +Since every candidate revision is a commit that already merged to `main`, it can only ever be +*behind* the tip: `behind_by` stays 0 and the lag is non-negative. A non-zero `behind_by` would +mean the revision is no longer on `main` at all (history rewritten). + +There is no local-git path. The CI checkout is `depth: 1, noTags: true` with a single-SHA refspec +(`trtllm_utils.checkoutSpec`), so no candidate revision is ever in the object store; a git +measurement would also answer against whatever ref it was given, and a merely stale ref returns a +*smaller* number rather than an error. + +The API is queried once per distinct revision (`compare_distance` is cached), so probing ten builds +is at most ten calls. It answers unless the revision has not reached the public mirror yet (404) or +the token is missing (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is +routinely already spent). Either way `lag` is `null`, the ranking degrades to its build-number +tie-break, and the reason is on stderr. + +The token comes from the `github-cred-trtllm-ci` credential — the one `getGithubMRChangedFile` +already uses — bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from +`GITHUB_API_TOKEN`. ### 8.3 What happens with the result diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index c01b8767a250..cded8336c458 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -36,7 +36,6 @@ import json import os import shutil -import subprocess import sys import tarfile import urllib.error @@ -59,8 +58,6 @@ _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" -# Local stand-ins for COVERAGE_BRANCH, best first; a stale ref understates the lag. -_LOCAL_REFS = (f"upstream/{COVERAGE_BRANCH}", f"origin/{COVERAGE_BRANCH}", COVERAGE_BRANCH) _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Max builds to walk back when recent builds have no tarball. _MAX_PROBE = 10 @@ -125,22 +122,6 @@ def build_commit(build: int, artifact_base: str = ARTIFACT_BASE) -> Optional[str return None -def commit_distance(commit: str, repo_root: str = ".", ref: str = COVERAGE_BRANCH) -> Optional[int]: - """Commits in `ref` not reachable from `commit`, or None if git cannot answer.""" - try: - out = subprocess.run( - ["git", "rev-list", "--count", f"{commit}..{ref}"], - cwd=repo_root, - capture_output=True, - text=True, - check=False, - timeout=_TIMEOUT, - ) - except (OSError, subprocess.SubprocessError): - return None - return int(out.stdout.strip()) if out.returncode == 0 and out.stdout.strip() else None - - @lru_cache(maxsize=None) def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int]: """Commits `branch` gained since `commit`, from the forge compare API, or None.""" @@ -165,24 +146,10 @@ def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int return None -def db_lag(commit: str, repo_root: str = ".") -> Optional[int]: - """Commits `COVERAGE_BRANCH` gained since `commit`; the API is authoritative, git is the backup.""" - lag = compare_distance(commit) - if lag is not None: - return lag - # Only reached without a token or network: whichever local ref tracks the branch answers. - for ref in _LOCAL_REFS: - lag = commit_distance(commit, repo_root, ref) - if lag is not None: - return lag - return None - - def select_tarball( artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, max_probe: int = _MAX_PROBE, - repo_root: str = ".", ) -> Optional[dict]: """Tarball of the least-trailing commit as {url, build, commit, lag}; build number breaks ties.""" build = latest_build_number(jenkins_base) @@ -195,16 +162,16 @@ def select_tarball( if not _exists(url): continue commit = build_commit(b, artifact_base) - lag = db_lag(commit, repo_root) if commit else None + lag = compare_distance(commit) if commit else None candidates.append({"url": url, "build": b, "commit": commit, "lag": lag}) if not candidates: print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) return None - # Known lag first (smallest = closest to HEAD); unknown lag falls back to build order. + # Known lag first (smallest = closest to the branch tip); unknown falls back to build order. best = min(candidates, key=lambda c: (c["lag"] is None, c["lag"] or 0, -c["build"])) if best["lag"] is None: print( - f"[artifact] build {best['build']}: commit unknown, selected by build number", + f"[artifact] build {best['build']}: lag unknown, selected by build number", file=sys.stderr, ) skipped = [c["build"] for c in candidates if c["build"] > best["build"]] @@ -275,7 +242,6 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) - ap.add_argument("--repo-root", default=".", help="repo the lag is measured against") args = ap.parse_args(argv) url = tarball_url(args.build) if args.build is not None else None @@ -287,10 +253,10 @@ def main(argv: Optional[list[str]] = None) -> int: "url": url, "build": args.build, "commit": commit, - "lag": db_lag(commit, args.repo_root) if commit else None, + "lag": compare_distance(commit) if commit else None, } else: - best = select_tarball(repo_root=args.repo_root) + best = select_tarball() if best is None: return 1 print(json.dumps(best)) From 8741f8a3722dcfdac9a17763ae45257cf06bb77a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:40:22 +0800 Subject: [PATCH 20/35] [TRTLLM-12838][infra] CBTS: drop artifact.py's unused download entry points --dest and --print-url were never wired: _cbtsCoverageAudit calls the script once with --print-selection and then does its own wget and tar, so fetch_latest_touch_db, extract_touch_db and latest_tarball_url had no callers in the pipeline or anywhere else in the tree. Remove them along with the two flags, leaving the module to answer the one question it is asked -- which post-merge build's DB to use. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 2 +- .../cbts/coverage_selection/artifact.py | 111 ++++-------------- 2 files changed, 22 insertions(+), 91 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index a567cb42ec44..05865023c78b 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -91,7 +91,7 @@ jenkins/scripts/cbts/ │ ├── selector.py CoverageSelector.decide(): changed lines → qualnames → impacted / skippable per stage family │ ├── qualname_map.py changed lines → co_qualname, plus the import-time and closure classifications │ ├── touch_db.py read-only accessor over cbts_touchmap.sqlite + the untrusted-capture signals -│ └── artifact.py resolve and fetch the touch DB from Artifactory (picks by collected revision) +│ └── artifact.py resolve which post-merge touch DB to use (by collected revision) ├── coverage_utils/ post-merge collection that produces the touch DB (see its README / COLLECTION.md) └── tools/ ├── dryrun.py replay CBTS over historical commits → per-PR summary.txt + filtered YAMLs + INDEX.md (debug only) diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index cded8336c458..a567bac9bd27 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -15,7 +15,8 @@ The tarball is uploaded per post-merge run to `//cbts-coverage/cbts_pystart_report.tar.gz` (sqlite at -the tar root plus `cbts_report/`). +the tar root plus `cbts_report/`). This module only resolves which one to use; +the pipeline downloads and extracts it itself. `select_tarball()` reads the newest build number from the Jenkins REST API, walks builds down probing Artifactory with a 1-byte ranged GET, and ranks the ones that @@ -24,10 +25,8 @@ distance comes from local git, or from the forge compare API when the CI checkout is too shallow to answer; the build number is only a tie-break. -Three entry points for the Groovy wiring: - * `--print-url` — resolve and print the tarball URL only (no download). - * `--print-selection` — print `{url, build, commit, lag}` as JSON. - * `--dest DIR` — download + extract, printing the local sqlite path. +`--print-selection` prints `{url, build, commit, lag}` as JSON for the Groovy +wiring; `--build` pins a candidate instead of resolving one. """ from __future__ import annotations @@ -35,19 +34,15 @@ import argparse import json import os -import shutil import sys -import tarfile import urllib.error import urllib.request from functools import lru_cache -from pathlib import Path from typing import Optional # Merged-artifact base for the main-branch L0_PostMerge job. ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" TARBALL_NAME = "cbts_pystart_report.tar.gz" -SQLITE_NAME = "cbts_touchmap.sqlite" # Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" @@ -183,57 +178,10 @@ def select_tarball( return best -def latest_tarball_url( - artifact_base: str = ARTIFACT_BASE, - jenkins_base: str = _JENKINS_BASE, - max_probe: int = _MAX_PROBE, -) -> Optional[str]: - """URL of the best available coverage tarball; see `select_tarball`.""" - best = select_tarball(artifact_base, jenkins_base, max_probe) - return best["url"] if best else None - - -def extract_touch_db(tarball: Path | str, dest_dir: Path | str) -> Optional[Path]: - """Extract `cbts_touchmap.sqlite` from a downloaded tarball; return its path.""" - dest_dir = Path(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - with tarfile.open(tarball) as tf: - member = next((m for m in tf.getmembers() if m.name.endswith(SQLITE_NAME)), None) - if member is None: - return None - member.name = SQLITE_NAME - tf.extract(member, dest_dir) - return dest_dir / SQLITE_NAME - - -def fetch_latest_touch_db(dest_dir: Path | str, url: Optional[str] = None) -> Optional[Path]: - """Download + extract the latest post-merge touch DB; return local sqlite Path or None. - - `url` pins an explicit tarball (skips latest-build resolution); any failure returns None. - """ - dest_dir = Path(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - url = url or latest_tarball_url() - if url is None: - return None - tarball = dest_dir / TARBALL_NAME - try: - with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp, open(tarball, "wb") as f: - shutil.copyfileobj(resp, f) - return extract_touch_db(tarball, dest_dir) - except OSError as e: - print(f"[artifact] download/extract failed {url}: {e}", file=sys.stderr) - return None - - def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - ap.add_argument("--dest", help="download + extract into DIR; prints the local sqlite path") - ap.add_argument( - "--print-url", action="store_true", help="resolve and print the tarball URL only" - ) ap.add_argument( "--print-selection", action="store_true", @@ -244,40 +192,23 @@ def main(argv: Optional[list[str]] = None) -> int: ) args = ap.parse_args(argv) - url = tarball_url(args.build) if args.build is not None else None - - if args.print_selection: - if args.build is not None: - commit = build_commit(args.build) - best = { - "url": url, - "build": args.build, - "commit": commit, - "lag": compare_distance(commit) if commit else None, - } - else: - best = select_tarball() - if best is None: - return 1 - print(json.dumps(best)) - return 0 - - if args.print_url: - url = url or latest_tarball_url() - if url is None: - return 1 - print(url) - return 0 - - if args.dest: - path = fetch_latest_touch_db(args.dest, url=url) - if path is None: - return 1 - print(path) - return 0 - - ap.error("one of --print-url or --dest is required") - return 2 + if not args.print_selection: + ap.error("--print-selection is required") + + if args.build is not None: + commit = build_commit(args.build) + best = { + "url": tarball_url(args.build), + "build": args.build, + "commit": commit, + "lag": compare_distance(commit) if commit else None, + } + else: + best = select_tarball() + if best is None: + return 1 + print(json.dumps(best)) + return 0 if __name__ == "__main__": From f4b587d2d28f9d44dc626795cf72869488921069 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:44:28 +0800 Subject: [PATCH 21/35] [TRTLLM-12838][infra] CBTS: condense artifact.py's header and the audit comment The module docstring still described the lag as coming from local git, a path removed two commits ago -- two earlier edits to that paragraph had silently matched nothing because they were written with different line wrapping. Restate it once, and fold the three-line comment above the credential binding into one. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 +--- .../cbts/coverage_selection/artifact.py | 22 ++++++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 696b9b7f8e65..e8f8b149af40 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -931,9 +931,7 @@ def _cbtsCoverageAudit(pipeline) // All commands run from ${LLM_ROOT}; covDir and the returned path are // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. def covDir = "cbts_cov" - // Selection is by collected revision, not build number; the JSON also carries the commit - // and how far main has moved past it, both recorded in the decision. The token is what - // lets artifact.py measure that — the depth-1 checkout cannot. + // Ranked by collected revision, not build number; the token is what measures it (depth-1 checkout cannot). def selJson = "" withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { selJson = sh( diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index a567bac9bd27..c28da53f3594 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -11,22 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Resolve and fetch the latest merged CBTS touch DB from Artifactory. +"""Resolve which post-merge CBTS touch DB to use. -The tarball is uploaded per post-merge run to -`//cbts-coverage/cbts_pystart_report.tar.gz` (sqlite at -the tar root plus `cbts_report/`). This module only resolves which one to use; -the pipeline downloads and extracts it itself. - -`select_tarball()` reads the newest build number from the Jenkins REST API, walks -builds down probing Artifactory with a 1-byte ranged GET, and ranks the ones that -exist by how far `COVERAGE_BRANCH` has moved past the revision each collected -(`build_info.txt`), since a build can be a re-run of an older commit. That -distance comes from local git, or from the forge compare API when the CI -checkout is too shallow to answer; the build number is only a tie-break. +Candidates are the recent builds of `` that have a +`cbts-coverage/cbts_pystart_report.tar.gz`, ranked by how far `COVERAGE_BRANCH` +has moved past the revision each collected (`build_info.txt`), since a build can +be a re-run of an older commit; the build number is only a tie-break. That +distance is `ahead_by` from the forge compare API — the CI checkout is depth-1, +so git cannot answer it — and needs `GITHUB_API_TOKEN`, the anonymous quota +being per-IP and exhausted by shared CI egress. `--print-selection` prints `{url, build, commit, lag}` as JSON for the Groovy -wiring; `--build` pins a candidate instead of resolving one. +wiring, which downloads and extracts the tarball itself. """ from __future__ import annotations From 66dae86328c4455e8a9732c1b6d7c9ed7cdfe4e7 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:06:50 +0800 Subject: [PATCH 22/35] [TRTLLM-12838][infra] CBTS: say the audit lag is measured against main The echo still read "behind HEAD" from when git measured the distance against the workspace. The compare API measures it against the tip of main, which is a different number in a PR build: HEAD carries the PR's own commits, main does not. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index e8f8b149af40..886abc889d4c 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -946,7 +946,7 @@ def _cbtsCoverageAudit(pipeline) def sel = new groovy.json.JsonSlurper().parseText(selJson) def url = sel.url pipeline.echo("CBTS audit: coverage DB from build ${sel.build}, " + - "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind HEAD'}") + "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind main'}") sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" // wget the tarball (retrying) and extract the sqlite. trtllm_utils.llmExecStepWithRetry(pipeline, script: From 8e2eb7f5763a56f2658ee14e73a3d8e84ab71ec7 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:56:13 +0800 Subject: [PATCH 23/35] [TRTLLM-12838][infra] CBTS: gate the coverage tier on the touch DB's freshness The DB's lag behind main was recorded but never acted on. Decline the tier past --coverage-max-lag: a DB that far behind no longer describes who touches what in the code under test, and narrowing on it risks dropping a case the change actually reaches. A lag that could not be measured is treated the same way -- freshness that cannot be shown is not assumed -- so a GitHub compare outage turns the tier off rather than letting it run unverified. The default of 100 comes from what the producer currently manages: main gains roughly 24-36 commits a day, and of the last twelve post-merge builds only four uploaded a coverage tarball, the newest of them 58 commits behind. A tighter bound would decline nearly every PR today; 100 admits the DBs that actually exist while still rejecting one several days stale. The verdict is recorded as coverage_freshness (ok / stale / unknown, empty when no DB was consulted) and posted as s_coverage_freshness, so the decline rate is queryable per cause instead of only readable inside s_reason -- which is what should drive the threshold from here. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 3 + .../cbts/coverage_selection/SELECTION.md | 20 +++++-- jenkins/scripts/cbts/main.py | 59 ++++++++++++++----- .../cbts/tools/report_cbts_decision.py | 4 +- 4 files changed, 64 insertions(+), 22 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 05865023c78b..c65d7443ecb4 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -318,6 +318,9 @@ CBTS defers to the existing filter chain when: touch DB, or its changed qualname is under-recorded with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs +- The resolved DB trails main by more than `--coverage-max-lag` commits, or by + an unmeasurable amount — Tier 2 declines (`coverage_freshness` = `stale` / + `unknown`) - Layer 3 narrowing would empty a block — block keeps original tests - `cbts_test_db` tarball upload or download/extraction fails — renderTestDB falls back to source - Narrowed YAML missing/empty on a stage agent — renderTestDB falls back diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 5e58f7cb662d..85d39db03ccb 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -201,11 +201,21 @@ The tarball is downloaded (retried), the sqlite extracted, and `coverage_audit.p any failure in this whole path is caught and non-fatal — `coverageDb.path` stays empty, Tier 2 never runs, and the PR gets a full run. -The chosen build, its commit and its lag ride into `main.py` and are recorded in the decision -(`coverage_db_build` / `coverage_db_commit` / `coverage_db_lag`; an unmeasurable lag is `null` -here and `-1` in the OpenSearch record). -**These are recorded, not gated**: observed lag varies widely even in healthy operation, so a -threshold needs data first. +The chosen build, its commit and its lag ride into `main.py`, which **gates on the lag**: past +`--coverage-max-lag` (default 100) the tier declines and the PR runs in full, on the grounds that a +DB that far behind no longer describes who touches what in the code under test. A lag that could +not be measured at all is treated the same way — freshness that cannot be shown is not assumed. + +All four land in the decision and in OpenSearch: + +| Decision field | OpenSearch | Note | +|---|---|---| +| `coverage_db_build` | `l_coverage_db_build` | 0 when no DB was consulted | +| `coverage_db_commit` | `s_coverage_db_commit` | | +| `coverage_db_lag` | `l_coverage_db_lag` | `null` / `-1` when unmeasurable | +| `coverage_freshness` | `s_coverage_freshness` | `ok` / `stale` / `unknown`, empty when no DB | + +so the decline rate is queryable per verdict rather than only readable in `s_reason`. ## 9. Decision output diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 1f084d61d92e..c541495a9768 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -137,6 +137,8 @@ class SelectionResult: # Revision the DB was collected at and HEAD's distance from it; None when unknown. coverage_db_commit: Optional[str] = None coverage_db_lag: Optional[int] = None + # Freshness verdict on that lag: ok / stale / unknown; empty when no DB was consulted. + coverage_freshness: str = "" # Residual files the forge API returned no patch for; they fall back to file level. coverage_no_diff_files: int = 0 @@ -156,11 +158,26 @@ def to_json(self) -> str: "coverage_db_build": self.coverage_db_build, "coverage_db_commit": self.coverage_db_commit, "coverage_db_lag": self.coverage_db_lag, + "coverage_freshness": self.coverage_freshness, "coverage_no_diff_files": self.coverage_no_diff_files, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" +# Tier 2 stands down past this many commits: what the DB says about who touches what +# stops describing the code under test. Tune with --coverage-max-lag. +DEFAULT_COVERAGE_MAX_LAG = 100 + + +def _coverage_freshness(lag: Optional[int], max_lag: int) -> tuple[str, str]: + """Verdict on the consulted DB's lag, plus the decline note (empty when usable).""" + if lag is None: + return "unknown", "coverage DB freshness unknown: its lag could not be measured" + if lag > max_lag: + return "stale", f"coverage DB is {lag} commit(s) behind main, over the {max_lag} limit" + return "ok", "" + + def _rule_reason(rule, r) -> dict: """One rule's structured reason entry: `{source, blocks, stages}`.""" return { @@ -373,6 +390,12 @@ def main(argv: Optional[list[str]] = None) -> int: default=None, help="Commits main gained since --coverage-db-commit; recorded in the decision.", ) + parser.add_argument( + "--coverage-max-lag", + type=int, + default=DEFAULT_COVERAGE_MAX_LAG, + help="Decline the coverage tier when the DB trails main by more than this many commits.", + ) parser.add_argument( "--no-data-policy", choices=NO_DATA_POLICIES, @@ -433,22 +456,26 @@ def main(argv: Optional[list[str]] = None) -> int: result.coverage_db_lag = args.coverage_db_lag if args.coverage_db and result.scope is None: - note = "" - try: - db = open_db(args.coverage_db) - tier, note = apply_coverage_tier( - pr, - selector.pairs, - selector.handled, - stages, - yaml_index, - repo_root, - db, - no_data_policy=args.no_data_policy, - ) - except Exception as e: # noqa: BLE001 — CBTS must never break CI - note = f"coverage tier errored: {e}" - tier = None + tier = None + result.coverage_freshness, note = _coverage_freshness( + args.coverage_db_lag, args.coverage_max_lag + ) + if not note: # the gate passed; a note here means it did not + try: + db = open_db(args.coverage_db) + tier, note = apply_coverage_tier( + pr, + selector.pairs, + selector.handled, + stages, + yaml_index, + repo_root, + db, + no_data_policy=args.no_data_policy, + ) + except Exception as e: # noqa: BLE001 — CBTS must never break CI + note = f"coverage tier errored: {e}" + tier = None if tier is not None: result.scope = "coverage" result.scopes = sorted( diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index 39f2c849d360..ef10e996f7c6 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -118,10 +118,12 @@ def build_document( "s_coverage_db_commit": decision.get("coverage_db_commit") or "", # Residual files whose patch the forge API omitted (binary / rename / oversized). "l_coverage_no_diff_files": int(decision.get("coverage_no_diff_files") or 0), - # Commits HEAD ran ahead of that DB; -1 when the build carried no commit. + # Commits main gained since that DB was collected; -1 when unmeasurable. "l_coverage_db_lag": int( decision["coverage_db_lag"] if decision.get("coverage_db_lag") is not None else -1 ), + # Freshness-gate verdict on that lag: ok / stale / unknown; empty when no DB was consulted. + "s_coverage_freshness": decision.get("coverage_freshness") or "", "d_case_skip_rate": round(case_skip_rate, 4), "flat_detail": { "hit_stages": affected, From ea65c2b3e4f9e01f08670c9e4178256ae65cdb2c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:02:30 +0800 Subject: [PATCH 24/35] [TRTLLM-12838][infra] CBTS: fail closed on import-executed changes Widening an import-executed qualname to its file's row set recovers tests but does not bound the impact: a test that imports the file and never enters any function in it records nothing there, so it is missing from the file set too. On llmapi/llm_args.py the widening goes 509 -> 735 holders against roughly 746 importers, and the gap is exactly the population the bound was supposed to cover. Decline instead, for module bodies, class bodies and signature / decorator lines, until the producer records the pool workers' import phase. A missing patch (binary / rename / oversized) and unparsable source decline for the same reason -- the changed scope cannot be established at all. Function-body changes are untouched and keep precise narrowing. This is expensive: over the last 200 main commits, 92 touched tensorrt_llm/**/*.py and 76 of those (82.6%) changed at least one import-executed line, mostly by adding a method, whose def lands on the class body. Tier 2 will therefore stand down on most core-Python PRs until import- phase capture lands, which is the correct trade while the narrowing cannot be justified from the data. Closure changes keep the widening for now. It carries the same incompleteness and is not sound either -- a decorator's wrapper is created at import time, so the enclosing scope it attributes to can itself be import-only -- but it covers 4.3% of commits rather than 82.6%. Recorded in SELECTION.md 4.1 as the next candidate. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 5 +- .../cbts/coverage_selection/SELECTION.md | 68 +++++++++---------- .../cbts/coverage_selection/selector.py | 33 +++++---- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index c65d7443ecb4..28c4606ebb82 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -315,8 +315,9 @@ CBTS defers to the existing filter chain when: YAML edit) - Combined scope is `None` (incompatible mix) - Tier 2 declines: a residual file is not core Python, is absent from the - touch DB, or its changed qualname is under-recorded with no wider row set - (see `coverage_selection/SELECTION.md` §3-4) + touch DB, has an import-executed change (module / class body, signature or + decorator line), has no usable patch, has unparsable source, or has a closure + change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs - The resolved DB trails main by more than `--coverage-max-lag` commits, or by an unmeasurable amount — Tier 2 declines (`coverage_freshness` = `stale` / diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 85d39db03ccb..9788902de66f 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -38,7 +38,7 @@ DB qualnames come from `co_qualname`, so they match what the AST derives (`Class comprehensions, lambdas and closures are skipped during collection, so those names do not exist in the DB at all. -## 3. The three decline gates +## 3. The decline gates `CoverageSelector.decide()` returns `ok=False` — Tier 2 stands down and the run is full — on any of: @@ -47,58 +47,56 @@ of: |---|---| | Non-core file | `path` is not a `.py` under `tensorrt_llm/` (checked on the **repo path**, not `canon()`) | | File absent from the DB | `file_has_touch_rows(cf)` is false — new or uninstrumented, so "who touches it" is unknown rather than empty | -| Under-recorded with no wider bound | see the next section | +| **Import-executed change** | a changed line lands on a module body, a class body, or a signature / decorator line | +| No usable patch | the forge API omitted the diff (binary / rename / oversized), so the changed scope is unknown | +| Unparsable source | the AST walk failed, so lines cannot be mapped to qualnames | +| Closure change with no wider row set | see the next section | -## 4. Core: handling an under-recorded qualname +## 4. Why import-executed changes fail closed -Two kinds of qualname have DB rows that **do not cover the changed code's executions**: +Import-phase rows are recorded only by the tests that spawn subprocesses (`../coverage_utils/COLLECTION.md` +§5.2), so a `` or `ClassName` row set is missing the tests served by MPI pool workers. Widening +to the file's row set recovers some of them but is a **recovery heuristic, not an upper bound**: a test +that imports the file and never enters any function in it records nothing there at all, so it is absent +from the file set too. Measured on `llmapi/llm_args.py`, the widening goes from 509 holders to 735 while +about 746 tests import the file — the gap does not close. -| Kind | Attributed qualname | Collection-side problem | -|---|---|---| -| Import-time (module body / class body / signature line) | itself | only recorded by tests that spawn subprocesses; a pool worker's import precedes activation | -| Closure | the enclosing function | **not recorded at all**; a closure can outlive the call that created it | +Since the missing tests cannot be enumerated from the data, these changes decline outright until the +producer records the pool workers' import phase. Ordinary function-body changes are unaffected and keep +precise narrowing. + +### 4.1 Closures -Both go through `_underrecorded_bound(cf, qualname)`: +A closure body is not recorded at all, so a changed closure is attributed to its nearest recorded +enclosing scope and put through `_underrecorded_bound(cf, qualname)`: ``` file rows > that qualname's rows → use the file rows (the wider bound) otherwise → decline, run in full ``` -### 4.1 What the comparison asks - -Whether falling back to file level actually recovers the tests that were missed. - `tests_touching_func(f, q)` is by construction a subset of `tests_touching_file(f)`, so a strictly -larger file set means it contains tests the qualname set does not. Those extra tests are the ones -that recorded some *other* function in the same file — evidence that the file is exercised beyond -import time, which is what makes the file set a usable bound. When the two are equal the file is -only ever recorded at import time (`__init__.py` and similar, 881 of 1149 files), nothing is -recovered, and no sound bound exists. - -### 4.2 The bound is not complete - -A test that imports a file but never calls any function in it records nothing there, so even the -file set misses it. The bound is the widest one available from the data, not a proof; eliminating -the remainder means recording the pool workers' import phase — see -`../coverage_utils/COLLECTION.md` §5.2. - -### 4.3 When no diff is available +larger file set contains tests the qualname set does not — tests that recorded some *other* function +in the same file, which is what makes the file exercised beyond import time. When the two are equal +the file is only ever recorded at import time (`__init__.py` and similar, 881 of 1149 files) and no +bound exists. -The forge API omits the patch for binary, renamed and oversized files. The changed qualname is -then unknown, so the worst case is assumed and `` is put through the same test; the count -of such files is reported as `coverage_no_diff_files`. +This bound carries the same incompleteness as the one removed above. It is narrower in blast radius +(4.3% of core-Python commits touch a closure, against 82.6% for import-executed lines) but it is not +sound either: a decorator's `wrapper` is created at import time, so the enclosing scope it attributes +to can itself be import-only. ## 5. Full impact resolution ``` for each residual file: - ├ no usable diff → _underrecorded_bound(cf, "") bound or decline - ├ diff has no lines → whole file (the change was comments only) + ├ no usable diff / unparsable → decline + ├ diff has no lines → whole file (the change was comments only) └ for each changed qualname: - ├ import-time or closure → _underrecorded_bound(cf, q) bound or decline - ├ has DB rows → tests_touching_func(cf, q) precise - └ no DB rows → no_data_policy fallback (file by default) + ├ import-executed → decline + ├ closure → _underrecorded_bound(cf, q) bound or decline + ├ has DB rows → tests_touching_func(cf, q) precise + └ no DB rows → no_data_policy fallback (file by default) ``` Comments and blank lines are stripped upstream by `strip_noop_diff_lines` (`^\s*#` and empty diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index 9767969f0355..ba74494357d8 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -50,7 +50,7 @@ class CoverageResult: n_untrusted: int = 0 # functions with no DB rows (new/uninstrumented); bounded per `no_data_policy` no_data_funcs: list[str] = field(default_factory=list) - # residual files the forge API returned no patch for (binary / rename / oversized) + # residual file the forge API returned no patch for (binary / rename / oversized); declines no_diff_files: list[str] = field(default_factory=list) @@ -118,28 +118,27 @@ def _impacted_tests( diff = diffs.get(path) or "" source = self._read_source(path) if not diff.strip() or source is None: - # Patch omitted (binary / renamed / oversized): bound as import-time. + # Patch omitted (binary / renamed / oversized): the changed scope is unknown. no_diff.append(path) - tests = self._underrecorded_bound(cf, "") - if tests is None: - return impacted, no_data, no_diff, f"no usable diff, no wider row set: {path}" - impacted |= tests - continue + return impacted, no_data, no_diff, f"no usable diff: {path}" lines = iter_diff_post_line_numbers(diff) qualnames, ok = qualnames_for_lines(source, lines) - if not lines or not ok: - # Comment-only or unparsable: no qualname to resolve, file-level bound. + if not ok: + return impacted, no_data, no_diff, f"unparsable source: {path}" + if not lines: + # Comment-only: nothing executable changed, so any set covers it. impacted |= self.db.tests_touching_file(cf) continue - # Qualnames whose rows do not record the changed code. - underrecorded = import_executed_qualnames(source) | closure_attributed_qualnames( - source, lines - ) + import_executed = import_executed_qualnames(source) + closures = closure_attributed_qualnames(source, lines) for qualname in sorted(qualnames): # sorted -> deterministic no_data order - if qualname in underrecorded: + if qualname in import_executed: + why = f"import-executed change, no sound bound: {path}::{qualname}" + return impacted, no_data, no_diff, why + if qualname in closures: tests = self._underrecorded_bound(cf, qualname) if tests is None: - why = f"under-recorded qualname, no wider row set: {path}::{qualname}" + why = f"closure change, no wider row set: {path}::{qualname}" return impacted, no_data, no_diff, why impacted |= tests continue @@ -151,7 +150,7 @@ def _impacted_tests( return impacted, no_data, no_diff, None def _underrecorded_bound(self, cf: str, qualname: str) -> set[str] | None: - """File row set when it is wider than an under-recorded qualname's, else None.""" + """File row set when it is wider than a closure's enclosing qualname's, else None.""" file_tests = self.db.tests_touching_file(cf) if len(file_tests) > len(self.db.tests_touching_func(cf, qualname)): return file_tests @@ -172,7 +171,7 @@ def _read_head(self, path: str) -> str | None: return None def decide(self, residual_files: list[str], diffs: dict[str, str]) -> CoverageResult: - """Decide over residual files; ok=False for non-core, not-in-DB, or unbounded import-time changes.""" + """Decide over residual files; ok=False for non-core, not-in-DB, or import-executed changes.""" for path in residual_files: # Gate on the repo path: canon() matches `tensorrt_llm/` anywhere. if not (path.endswith(".py") and path.startswith("tensorrt_llm/")): From 36ecd145f7413fcebc82f07ee58c70954b8b3847 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:07:48 +0800 Subject: [PATCH 25/35] [TRTLLM-12838][infra] CBTS: a comment-only diff impacts nothing A residual file whose whole diff survives strip_noop_diff_lines as empty changed no executable line, so it was widening the impacted set to everything touching that file for no reason. Contribute nothing instead. This is not the deletion case: iter_diff_post_line_numbers anchors `-` lines at the following post-image line, so removing code still yields line numbers and still resolves to a qualname. The branch fires only when every `+` and `-` line was blank or comment. The one way it could misread is a `#`-leading line inside a multi-line string, which the regex cannot tell from a comment. Over the last 200 main commits, 9 of 375 core-Python diffs were comment-only and none was string content. Tier 1's rules already read every diff through the same stripping, so this adds no assumption the pipeline was not already making. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/coverage_selection/SELECTION.md | 7 +++++-- jenkins/scripts/cbts/coverage_selection/selector.py | 4 +--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 9788902de66f..df211dd336dc 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -91,7 +91,7 @@ to can itself be import-only. ``` for each residual file: ├ no usable diff / unparsable → decline - ├ diff has no lines → whole file (the change was comments only) + ├ diff has no lines → contributes nothing (comment / blank only) └ for each changed qualname: ├ import-executed → decline ├ closure → _underrecorded_bound(cf, q) bound or decline @@ -101,7 +101,10 @@ for each residual file: Comments and blank lines are stripped upstream by `strip_noop_diff_lines` (`^\s*#` and empty lines), so a change that edits a function body and adds a module-level comment is not mistaken -for a module-level change. +for a module-level change. A file whose whole diff survives that stripping as empty changed no +executable line — neither added nor removed, since `-` lines anchor at the following post-image +line — so it contributes no impacted tests at all. Tier 1's rules already read diffs through the +same stripping, so this introduces no assumption the pipeline does not already make. ## 6. Tests that are never skipped diff --git a/jenkins/scripts/cbts/coverage_selection/selector.py b/jenkins/scripts/cbts/coverage_selection/selector.py index ba74494357d8..ba7778027294 100644 --- a/jenkins/scripts/cbts/coverage_selection/selector.py +++ b/jenkins/scripts/cbts/coverage_selection/selector.py @@ -126,9 +126,7 @@ def _impacted_tests( if not ok: return impacted, no_data, no_diff, f"unparsable source: {path}" if not lines: - # Comment-only: nothing executable changed, so any set covers it. - impacted |= self.db.tests_touching_file(cf) - continue + continue # comment / blank only: nothing executable changed, so nothing runs import_executed = import_executed_qualnames(source) closures = closure_attributed_qualnames(source, lines) for qualname in sorted(qualnames): # sorted -> deterministic no_data order From dd8f5185073967f6fd752e05490eabcf9e20f563 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:39:06 +0800 Subject: [PATCH 26/35] [TRTLLM-12838][infra] CBTS: keep the freshness gate out of dry-run replays A replay has no lag to measure -- the DB is whatever the operator passed and the commits are historical -- so main.py read it as unmeasurable and declined every PR, making every dry run report the gate rather than the selection logic it exists to exercise. Pass 0. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/tools/dryrun.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index ba17f8dc46e1..a0c80dffa175 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -152,7 +152,8 @@ def _run_cbts( str(groovy), ] if coverage_db: - argv += ["--coverage-db", coverage_db] + # A replay has no meaningful lag; report 0 so the freshness gate is not the thing measured. + argv += ["--coverage-db", coverage_db, "--coverage-db-lag", "0"] try: res = subprocess.run(argv, capture_output=True, text=True, check=False) finally: From ac8a60113f9bc1a3425febd76301bc068cd578ab Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:47:06 +0800 Subject: [PATCH 27/35] [TRTLLM-12838][infra] CBTS: point COLLECTION.md at what the import gap now costs Both cross-references described the consumer as widening to file level, which is no longer what happens for import-executed changes -- those decline outright. State that under 5.2, where the blind spot is described, so the reason to record the workers' import phase is visible from the producer side; and move 5.1's pointer to the closure section that still does widen. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/coverage_utils/COLLECTION.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_utils/COLLECTION.md b/jenkins/scripts/cbts/coverage_utils/COLLECTION.md index 868d3e5002f7..a17827185de2 100644 --- a/jenkins/scripts/cbts/coverage_utils/COLLECTION.md +++ b/jenkins/scripts/cbts/coverage_utils/COLLECTION.md @@ -85,9 +85,10 @@ Measured scale: `tensorrt_llm` is 646410 lines, of which **14191 (2.2%)** are cl 7513 of those sit under an enclosing function whose row set trails the whole file's by more than 300 tests. -The consumer side can only widen a closure change to file level (see `../coverage_selection/SELECTION.md` §4); -**the collection side itself is unfixed.** A real fix means having the producer record closure -frames (the `` segments in `co_qualname` can be kept or folded), which is follow-up work. +The consumer side widens a closure change to file level when that is wider, and otherwise runs in +full (`../coverage_selection/SELECTION.md` §4.1); **the collection side itself is unfixed.** A real +fix means having the producer record closure frames (the `` segments in `co_qualname` can +be kept or folded), which is follow-up work. ### 5.2 Import phase is lost inside pool workers @@ -105,6 +106,11 @@ workers. Deferred activation is deliberate: without it, the instrumented cold-start import overruns the `wait_shutdown` worker identity barrier. +This blind spot is what costs the most on the consumer side: a change landing on a module body, a +class body or a signature / decorator line cannot be bounded from these rows, so the tier declines +and the PR runs in full (`../coverage_selection/SELECTION.md` §4). Recording the workers' import +phase is what would let those changes narrow again. + ### 5.3 Other blind spots | Blind spot | Reason | From 3773747c4c6b15dce7a3a2d9f61da0c43cb9c664 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:52:48 +0800 Subject: [PATCH 28/35] [TRTLLM-12838][infra] CBTS: gate coverage freshness on the PR's base, not main's tip The freshness gate measured the DB against the tip of main, which answers "is this DB recent" rather than "does this DB still describe the code under test". CI checks out the PR head (env.gitlabMergeRequestLastCommit), so the revision the DB has to match is the PR's merge base: a DB one commit off the tip scores as fresh for a PR branched three hundred commits back, and its function-to-test edges no longer describe that code. Ranking and gating now use different numbers. The lag against main's tip still ranks candidates and reports overall freshness; a new drift measures the ranked winner against the PR's merge base and is what the gate decides on. Drift sums both sides of the compare rather than picking one. The dangerous failure is an edge the code under test has and the DB never recorded, and both directions produce it -- an older DB misses callers added since, a newer DB reflects a call path deleted since. The fail-closed bound is symmetric too and catches only whole-function absence, never a row set that is merely too narrow. Summing is also the only form that handles a diverged base: a PR targeting a release branch, which a main-collected DB does not describe at all, scores as the large number it is instead of slipping through on one small term. Direction rides along as drift_status, recorded but never weighted. Any step that cannot be answered leaves the drift null, which the gate reads as unknown and declines. The limit is --coverage-max-drift, default 30. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 31 +++++-- jenkins/scripts/cbts/README.md | 6 +- .../cbts/coverage_selection/SELECTION.md | 63 +++++++++++--- .../cbts/coverage_selection/artifact.py | 87 ++++++++++++++++--- jenkins/scripts/cbts/main.py | 64 ++++++++++---- jenkins/scripts/cbts/tools/dryrun.py | 11 ++- .../cbts/tools/report_cbts_decision.py | 8 +- 7 files changed, 217 insertions(+), 53 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 886abc889d4c..6ba6150b0417 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -847,7 +847,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) sh "apt-get update -qq && apt-get install -y -qq python3-yaml" // Download the touch DB for audit + Tier 2 coverage-based narrowing. - def coverageDb = _cbtsCoverageAudit(pipeline) + def coverageDb = _cbtsCoverageAudit(pipeline, globalVars) // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -881,6 +881,16 @@ def getCbtsResult(pipeline, testFilter, globalVars) if (coverageDb.lag != null) { mainCmd += " --coverage-db-lag ${coverageDb.lag}" } + // Absent drift declines the tier. + if (coverageDb.drift != null) { + mainCmd += " --coverage-db-drift ${coverageDb.drift}" + } + if (coverageDb.baseCommit) { + mainCmd += " --coverage-db-base-commit '${coverageDb.baseCommit}'" + } + if (coverageDb.driftStatus) { + mainCmd += " --coverage-db-drift-status '${coverageDb.driftStatus}'" + } } def output = sh(script: mainCmd, returnStdout: true) @@ -925,28 +935,31 @@ def getCbtsResult(pipeline, testFilter, globalVars) } // Download the touch DB, audit it, and return the sqlite path (or "" on failure). -def _cbtsCoverageAudit(pipeline) +def _cbtsCoverageAudit(pipeline, globalVars) { try { // All commands run from ${LLM_ROOT}; covDir and the returned path are // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. def covDir = "cbts_cov" + // The checked-out revision is the PR head; its merge base is what drift is measured against. + def prHeadArg = env.gitlabMergeRequestLastCommit ? " --pr-head ${env.gitlabMergeRequestLastCommit}" : "" // Ranked by collected revision, not build number; the token is what measures it (depth-1 checkout cannot). def selJson = "" withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { selJson = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-selection || true", + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-selection${prHeadArg} || true", returnStdout: true, ).trim() } if (!selJson) { pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") - return [path: "", build: null, commit: "", lag: null] + return [path: "", build: null, commit: "", lag: null, drift: null, baseCommit: "", driftStatus: ""] } def sel = new groovy.json.JsonSlurper().parseText(selJson) def url = sel.url pipeline.echo("CBTS audit: coverage DB from build ${sel.build}, " + - "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind main'}") + "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind main'}, " + + "${sel.drift == null ? 'drift unmeasured' : sel.drift + ' commit(s) ' + (sel.drift_status ?: '') + ' the PR base ' + (sel.base_commit ?: '')}") sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" // wget the tarball (retrying) and extract the sqlite. trtllm_utils.llmExecStepWithRetry(pipeline, script: @@ -954,15 +967,15 @@ def _cbtsCoverageAudit(pipeline) "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + "--db ${covDir}/cbts_touchmap.sqlite" - // build/commit/lag ride along so main.py can record which DB the decision - // used (resolved here, once, per run). + // build/commit/lag/drift ride along for main.py's record and drift gate. return [path: "${covDir}/cbts_touchmap.sqlite", build: sel.build, - commit: sel.commit ?: "", lag: sel.lag] + commit: sel.commit ?: "", lag: sel.lag, drift: sel.drift, + baseCommit: sel.base_commit ?: "", driftStatus: sel.drift_status ?: ""] } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - return [path: "", build: null, commit: "", lag: null] + return [path: "", build: null, commit: "", lag: null, drift: null, baseCommit: "", driftStatus: ""] } } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 28c4606ebb82..03cb5b979c02 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -319,9 +319,9 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs -- The resolved DB trails main by more than `--coverage-max-lag` commits, or by - an unmeasurable amount — Tier 2 declines (`coverage_freshness` = `stale` / - `unknown`) +- The resolved DB sits more than `--coverage-max-drift` commits from the PR's + base commit, on either side, or an unmeasurable distance from it — Tier 2 + declines (`coverage_freshness` = `stale` / `unknown`) - Layer 3 narrowing would empty a block — block keeps original tests - `cbts_test_db` tarball upload or download/extraction fails — renderTestDB falls back to source - Narrowed YAML missing/empty on a stage agent — renderTestDB falls back diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index df211dd336dc..ae44b9c3ed2b 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -169,12 +169,10 @@ commit, so the highest build number is not necessarily the newest code. The buil the tie-break, and when no candidate's lag can be measured the ranking degenerates to exactly that tie-break — which is the pre-existing behaviour, not a regression. -### 8.2 Measuring the lag +### 8.2 Measuring the lag (ranking) -The lag is `ahead_by` from GitHub's compare API on `...main` — always against the **tip of -`main`**, never against the PR's own base commit: every candidate is scored on the same scale, and -a PR whose base predates all the candidates would otherwise score them all identically and collapse -the ranking back to the build number. `ahead_by` covers the full range; only the response's +The lag is `ahead_by` from GitHub's compare API on `...main` — against the **tip of `main`**, +so every candidate is scored on one scale. `ahead_by` covers the full range; only the response's `commits` array is truncated at 250. Since every candidate revision is a commit that already merged to `main`, it can only ever be @@ -196,24 +194,64 @@ The token comes from the `github-cred-trtllm-ci` credential — the one `getGith already uses — bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. +This number ranks candidates and reports overall freshness. It is **not** what the gate decides on. + +### 8.2b Measuring the drift (gating) + +Ranking and gating ask different questions. "Which DB is freshest" is answered against the tip of +`main`; "does this DB still describe the code under test" is not — the code under test is the PR +head, which CI checks out directly (`env.gitlabMergeRequestLastCommit`), so the revision the DB has +to match is the PR's **merge base**. A DB sitting one commit off the tip says nothing useful about +a PR branched three hundred commits back, and the lag scores that case as fresh. + +The drift is `merge_base_commit.sha` from `main...`, then `ahead_by + behind_by` from +`...` — two extra calls per run, and only for the candidate ranking already +picked. Both revisions sit on the same linear history, so exactly one term is non-zero and the sum +is their plain distance. + +Summing rather than picking a side is deliberate. The one dangerous failure is an edge `(F → T)` +that the code under test really has and the DB never recorded, and **both directions produce it**: + +| DB is | relative to the PR base | how the edge goes missing | +|---|---|---| +| older | `ahead` | a caller added since, so `T` now reaches `F` and the DB never saw it | +| newer | `behind` | a call path deleted since, so the DB reflects a graph the PR base still has intact | + +The fail-closed bound is symmetric too — a function absent from the DB force-runs either way — and +it only catches whole-function absence, never a row set that is merely too narrow. So there is no +principled basis for weighting one side, and the sum is also the only form that handles a genuinely +diverged base: a PR targeting a release branch, which a `main`-collected DB does not describe at +all, scores as the large number it is instead of slipping through on one small term. + +`drift_status` (`ahead` / `behind` / `diverged` / `identical`) rides along in the same response and +is recorded, never weighted — it is there to answer empirically, later, whether one direction +actually correlates with misses. + +Any step that cannot be answered leaves the drift null, which the gate reads as `unknown` and +declines: freshness that cannot be shown is not assumed. + + ### 8.3 What happens with the result The tarball is downloaded (retried), the sqlite extracted, and `coverage_audit.py` run over it; any failure in this whole path is caught and non-fatal — `coverageDb.path` stays empty, Tier 2 never runs, and the PR gets a full run. -The chosen build, its commit and its lag ride into `main.py`, which **gates on the lag**: past -`--coverage-max-lag` (default 100) the tier declines and the PR runs in full, on the grounds that a -DB that far behind no longer describes who touches what in the code under test. A lag that could -not be measured at all is treated the same way — freshness that cannot be shown is not assumed. +The chosen build, its commit, its lag and its drift ride into `main.py`, which **gates on the +drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs in full, on +the grounds that a DB that far from the PR's base no longer describes who touches what in the code +under test. A drift that could not be measured at all is treated the same way. -All four land in the decision and in OpenSearch: +All of it lands in the decision and in OpenSearch: | Decision field | OpenSearch | Note | |---|---|---| | `coverage_db_build` | `l_coverage_db_build` | 0 when no DB was consulted | | `coverage_db_commit` | `s_coverage_db_commit` | | -| `coverage_db_lag` | `l_coverage_db_lag` | `null` / `-1` when unmeasurable | +| `coverage_db_lag` | `l_coverage_db_lag` | ranking / overall freshness; `null` / `-1` when unmeasurable | +| `coverage_db_base_commit` | `s_coverage_db_base_commit` | the PR's merge base | +| `coverage_db_drift` | `l_coverage_db_drift` | **the gated number**; `null` / `-1` when unmeasurable | +| `coverage_db_drift_status` | `s_coverage_db_drift_status` | recorded, never weighted | | `coverage_freshness` | `s_coverage_freshness` | `ok` / `stale` / `unknown`, empty when no DB | so the decline rate is queryable per verdict rather than only readable in `s_reason`. @@ -232,6 +270,9 @@ so the decline rate is queryable per verdict rather than only readable in `s_rea "coverage_db_build": 2887, "coverage_db_commit": "50edd738...", "coverage_db_lag": 11, + "coverage_db_base_commit": "9f0da65d...", + "coverage_db_drift": 47, + "coverage_db_drift_status": "behind", "coverage_no_diff_files": 0, "reasons": [{"source": "coverage", "impacted": 118, "untrusted": 104, ...}] } diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index c28da53f3594..be1dcc5d7f5f 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -21,8 +21,12 @@ so git cannot answer it — and needs `GITHUB_API_TOKEN`, the anonymous quota being per-IP and exhausted by shared CI egress. -`--print-selection` prints `{url, build, commit, lag}` as JSON for the Groovy -wiring, which downloads and extracts the tarball itself. +Ranking scores candidates against the tip of `COVERAGE_BRANCH`; gating scores the +winner against the PR's merge base, which `--pr-head` supplies. + +`--print-selection` prints `{url, build, commit, lag, base_commit, drift, +drift_status}` as JSON for the Groovy wiring, which downloads and extracts the +tarball itself. """ from __future__ import annotations @@ -114,27 +118,67 @@ def build_commit(build: int, artifact_base: str = ARTIFACT_BASE) -> Optional[str @lru_cache(maxsize=None) -def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int]: - """Commits `branch` gained since `commit`, from the forge compare API, or None.""" +def _compare(base: str, head: str) -> Optional[dict]: + """The forge's `base...head` compare payload, or None when it cannot be had.""" headers = {"Accept": "application/vnd.github+json"} token = os.environ.get(GITHUB_TOKEN_ENV) if token: headers["Authorization"] = f"Bearer {token}" - status, data = _get(f"{_GITHUB_COMPARE}/{commit}...{branch}", headers) + status, data = _get(f"{_GITHUB_COMPARE}/{base}...{head}", headers) if status != 200 or not data: # 403 without a token means the shared egress IP burned the 60/h anonymous quota. hint = " (no token: anonymous quota)" if status == 403 and not token else "" print( - f"[artifact] compare {commit[:10]}...{branch} failed: HTTP {status}{hint}", + f"[artifact] compare {base[:10]}...{head[:10]} failed: HTTP {status}{hint}", file=sys.stderr, ) return None + try: + return json.loads(data) + except json.JSONDecodeError as e: + print(f"[artifact] compare {base[:10]}...{head[:10]}: bad response: {e}", file=sys.stderr) + return None + + +def compare_distance(commit: str, branch: str = COVERAGE_BRANCH) -> Optional[int]: + """Commits `branch` gained since `commit` — ranking only; never negative.""" + payload = _compare(commit, branch) + if payload is None: + return None try: # `ahead_by` counts the full range; only the `commits` array is truncated at 250. - return int(json.loads(data)["ahead_by"]) - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as e: - print(f"[artifact] compare {commit[:10]}...{branch}: bad response: {e}", file=sys.stderr) + return int(payload["ahead_by"]) + except (KeyError, TypeError, ValueError) as e: + print(f"[artifact] compare {commit[:10]}...{branch}: no ahead_by: {e}", file=sys.stderr) + return None + + +def merge_base(head: str, branch: str = COVERAGE_BRANCH) -> Optional[str]: + """The commit `head` forked from `branch` — the revision the PR's diff is against.""" + payload = _compare(branch, head) + if payload is None: return None + sha = (payload.get("merge_base_commit") or {}).get("sha") + if not sha: + print(f"[artifact] compare {branch}...{head[:10]}: no merge_base_commit", file=sys.stderr) + return None + return sha + + +def drift(db_commit: str, base_commit: str) -> tuple[Optional[int], str]: + """Distance from the DB's revision to the PR's base; direction is recorded, not weighted.""" + payload = _compare(db_commit, base_commit) + if payload is None: + return None, "unknown" + try: + distance = int(payload["ahead_by"]) + int(payload["behind_by"]) + return distance, str(payload.get("status") or "") + except (KeyError, TypeError, ValueError) as e: + print( + f"[artifact] compare {db_commit[:10]}...{base_commit[:10]}: no ahead/behind: {e}", + file=sys.stderr, + ) + return None, "unknown" def select_tarball( @@ -174,6 +218,21 @@ def select_tarball( return best +def measure_drift(sel: dict, pr_head: Optional[str]) -> dict: + """Add `base_commit` / `drift` / `drift_status` to the ranked winner, in place.""" + sel.setdefault("base_commit", None) + sel.setdefault("drift", None) + sel.setdefault("drift_status", "unknown") + if not pr_head or not sel.get("commit"): + return sel + base = merge_base(pr_head) + if not base: + return sel + sel["base_commit"] = base + sel["drift"], sel["drift_status"] = drift(sel["commit"], base) + return sel + + def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -181,11 +240,17 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--print-selection", action="store_true", - help="resolve and print {url, build, commit, lag} as JSON", + help="resolve and print {url, build, commit, lag, base_commit, drift, drift_status} as JSON", ) ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) + ap.add_argument( + "--pr-head", + default=None, + help="PR head revision; its merge base is what drift is measured against " + "(omitted leaves drift null, which the selector declines on).", + ) args = ap.parse_args(argv) if not args.print_selection: @@ -203,7 +268,7 @@ def main(argv: Optional[list[str]] = None) -> int: best = select_tarball() if best is None: return 1 - print(json.dumps(best)) + print(json.dumps(measure_drift(best, args.pr_head))) return 0 diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index c541495a9768..dbbb070ce742 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -58,6 +58,7 @@ open_db, write_coverage_test_db, ) + from rules._helpers import strip_noop_diff_lines # noqa: E402 from rules.agent_flow_rule import AgentFlowRule # noqa: E402 from rules.auto_deploy_rule import AutoDeployRule # noqa: E402 @@ -134,10 +135,14 @@ class SelectionResult: coverage_dropped_stages: list[str] = field(default_factory=list) # Post-merge build the consulted touch DB came from; makes a decision replayable. coverage_db_build: Optional[int] = None - # Revision the DB was collected at and HEAD's distance from it; None when unknown. + # Revision the DB was collected at and main's distance from it; ranking, not the gate. coverage_db_commit: Optional[str] = None coverage_db_lag: Optional[int] = None - # Freshness verdict on that lag: ok / stale / unknown; empty when no DB was consulted. + # The PR's base and the DB's distance from it — what the freshness gate decides on. + coverage_db_base_commit: Optional[str] = None + coverage_db_drift: Optional[int] = None + coverage_db_drift_status: str = "" + # Freshness verdict on that drift: ok / stale / unknown; empty when no DB was consulted. coverage_freshness: str = "" # Residual files the forge API returned no patch for; they fall back to file level. coverage_no_diff_files: int = 0 @@ -158,23 +163,28 @@ def to_json(self) -> str: "coverage_db_build": self.coverage_db_build, "coverage_db_commit": self.coverage_db_commit, "coverage_db_lag": self.coverage_db_lag, + "coverage_db_base_commit": self.coverage_db_base_commit, + "coverage_db_drift": self.coverage_db_drift, + "coverage_db_drift_status": self.coverage_db_drift_status, "coverage_freshness": self.coverage_freshness, "coverage_no_diff_files": self.coverage_no_diff_files, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" -# Tier 2 stands down past this many commits: what the DB says about who touches what -# stops describing the code under test. Tune with --coverage-max-lag. -DEFAULT_COVERAGE_MAX_LAG = 100 +# Tier 2 stands down past this many commits between the DB's revision and the PR's base. +DEFAULT_COVERAGE_MAX_DRIFT = 30 -def _coverage_freshness(lag: Optional[int], max_lag: int) -> tuple[str, str]: - """Verdict on the consulted DB's lag, plus the decline note (empty when usable).""" - if lag is None: - return "unknown", "coverage DB freshness unknown: its lag could not be measured" - if lag > max_lag: - return "stale", f"coverage DB is {lag} commit(s) behind main, over the {max_lag} limit" +def _coverage_freshness(drift: Optional[int], max_drift: int) -> tuple[str, str]: + """Verdict on the consulted DB's drift, plus the decline note (empty when usable).""" + if drift is None: + return "unknown", "coverage DB freshness unknown: its drift could not be measured" + if drift > max_drift: + return ( + "stale", + f"coverage DB is {drift} commit(s) from the PR's base, over the {max_drift} limit", + ) return "ok", "" @@ -388,13 +398,32 @@ def main(argv: Optional[list[str]] = None) -> int: "--coverage-db-lag", type=int, default=None, - help="Commits main gained since --coverage-db-commit; recorded in the decision.", + help="Commits main gained since --coverage-db-commit; recorded only, the " + "freshness gate uses --coverage-db-drift.", + ) + parser.add_argument( + "--coverage-db-drift", + type=int, + default=None, + help="Commits between --coverage-db-commit and the PR's base; what the " + "freshness gate decides on (omitted means unmeasurable, which declines).", + ) + parser.add_argument( + "--coverage-db-base-commit", + default=None, + help="The PR's base revision the drift was measured against.", + ) + parser.add_argument( + "--coverage-db-drift-status", + default="", + help="Which side of the PR's base the DB sits on; recorded only, never weighted.", ) parser.add_argument( - "--coverage-max-lag", + "--coverage-max-drift", type=int, - default=DEFAULT_COVERAGE_MAX_LAG, - help="Decline the coverage tier when the DB trails main by more than this many commits.", + default=DEFAULT_COVERAGE_MAX_DRIFT, + help="Decline the coverage tier when the DB is more than this many commits " + "from the PR's base.", ) parser.add_argument( "--no-data-policy", @@ -454,11 +483,14 @@ def main(argv: Optional[list[str]] = None) -> int: result.coverage_db_build = args.coverage_db_build result.coverage_db_commit = args.coverage_db_commit result.coverage_db_lag = args.coverage_db_lag + result.coverage_db_drift = args.coverage_db_drift + result.coverage_db_base_commit = args.coverage_db_base_commit + result.coverage_db_drift_status = args.coverage_db_drift_status if args.coverage_db and result.scope is None: tier = None result.coverage_freshness, note = _coverage_freshness( - args.coverage_db_lag, args.coverage_max_lag + args.coverage_db_drift, args.coverage_max_drift ) if not note: # the gate passed; a note here means it did not try: diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index a0c80dffa175..93d1c6b1d5b0 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -152,8 +152,15 @@ def _run_cbts( str(groovy), ] if coverage_db: - # A replay has no meaningful lag; report 0 so the freshness gate is not the thing measured. - argv += ["--coverage-db", coverage_db, "--coverage-db-lag", "0"] + # A replay has no meaningful distance; 0 keeps the freshness gate out of it. + argv += [ + "--coverage-db", + coverage_db, + "--coverage-db-lag", + "0", + "--coverage-db-drift", + "0", + ] try: res = subprocess.run(argv, capture_output=True, text=True, check=False) finally: diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index ef10e996f7c6..a6955f7f81e2 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -122,7 +122,13 @@ def build_document( "l_coverage_db_lag": int( decision["coverage_db_lag"] if decision.get("coverage_db_lag") is not None else -1 ), - # Freshness-gate verdict on that lag: ok / stale / unknown; empty when no DB was consulted. + # Commits between that DB and the PR's base — what the gate decides on; -1 when unmeasurable. + "s_coverage_db_base_commit": decision.get("coverage_db_base_commit") or "", + "l_coverage_db_drift": int( + decision["coverage_db_drift"] if decision.get("coverage_db_drift") is not None else -1 + ), + "s_coverage_db_drift_status": decision.get("coverage_db_drift_status") or "", + # Freshness-gate verdict on that drift: ok / stale / unknown; empty when no DB was consulted. "s_coverage_freshness": decision.get("coverage_freshness") or "", "d_case_skip_rate": round(case_skip_rate, 4), "flat_detail": { From 915f085d9141443fa4a9630bc7ade2299d4f64e0 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:05:29 +0800 Subject: [PATCH 29/35] [TRTLLM-12838][infra] CBTS: pass the coverage DB's selection JSON through, not six flags Groovy destructured artifact.py's --print-selection JSON into six CLI flags, which main.py reassembled into the decision JSON -- the same data serialized three times, with the SHAs shell-quoted along the way and a new field costing an edit in both layers. Only drift is ever read: the freshness gate decides on it. The other five are record-only passthrough. So the whole blob now travels as one file, written verbatim to cbts_coverage_db.json and read via --coverage-db-meta. Six conditional appends in the Groovy collapse to one unconditional line, the audit helper's return map goes from seven keys to two, and a missing, empty or unparsable meta leaves drift null, which the gate already declines on. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 30 ++------- .../cbts/coverage_selection/SELECTION.md | 9 ++- jenkins/scripts/cbts/main.py | 64 +++++++------------ jenkins/scripts/cbts/tools/dryrun.py | 17 +++-- 4 files changed, 45 insertions(+), 75 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 6ba6150b0417..a85edae0bffa 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -874,23 +874,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" if (coverageDb.path) { - mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-build ${coverageDb.build}" - if (coverageDb.commit) { - mainCmd += " --coverage-db-commit '${coverageDb.commit}'" - } - if (coverageDb.lag != null) { - mainCmd += " --coverage-db-lag ${coverageDb.lag}" - } - // Absent drift declines the tier. - if (coverageDb.drift != null) { - mainCmd += " --coverage-db-drift ${coverageDb.drift}" - } - if (coverageDb.baseCommit) { - mainCmd += " --coverage-db-base-commit '${coverageDb.baseCommit}'" - } - if (coverageDb.driftStatus) { - mainCmd += " --coverage-db-drift-status '${coverageDb.driftStatus}'" - } + mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" } def output = sh(script: mainCmd, returnStdout: true) @@ -953,7 +937,7 @@ def _cbtsCoverageAudit(pipeline, globalVars) } if (!selJson) { pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") - return [path: "", build: null, commit: "", lag: null, drift: null, baseCommit: "", driftStatus: ""] + return [path: "", meta: ""] } def sel = new groovy.json.JsonSlurper().parseText(selJson) def url = sel.url @@ -967,15 +951,15 @@ def _cbtsCoverageAudit(pipeline, globalVars) "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + "--db ${covDir}/cbts_touchmap.sqlite" - // build/commit/lag/drift ride along for main.py's record and drift gate. - return [path: "${covDir}/cbts_touchmap.sqlite", build: sel.build, - commit: sel.commit ?: "", lag: sel.lag, drift: sel.drift, - baseCommit: sel.base_commit ?: "", driftStatus: sel.drift_status ?: ""] + // The selection JSON rides along verbatim for main.py's record and drift gate. + def metaPath = "cbts_coverage_db.json" + writeFile file: "${LLM_ROOT}/${metaPath}", text: selJson + return [path: "${covDir}/cbts_touchmap.sqlite", meta: metaPath] } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - return [path: "", build: null, commit: "", lag: null, drift: null, baseCommit: "", driftStatus: ""] + return [path: "", meta: ""] } } diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index ae44b9c3ed2b..ddfc56961681 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -237,10 +237,13 @@ The tarball is downloaded (retried), the sqlite extracted, and `coverage_audit.p any failure in this whole path is caught and non-fatal — `coverageDb.path` stays empty, Tier 2 never runs, and the PR gets a full run. -The chosen build, its commit, its lag and its drift ride into `main.py`, which **gates on the -drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs in full, on +The selection JSON is written to `cbts_coverage_db.json` verbatim and reaches `main.py` as +`--coverage-db-meta`, so a new field needs no Groovy change. `main.py` records all of it and +**gates on the drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs +in full, on the grounds that a DB that far from the PR's base no longer describes who touches what in the code -under test. A drift that could not be measured at all is treated the same way. +under test. A drift that could not be measured — including a meta file that is missing or +unreadable — is treated the same way. All of it lands in the decision and in OpenSearch: diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index dbbb070ce742..842652502154 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -176,6 +176,18 @@ def to_json(self) -> str: DEFAULT_COVERAGE_MAX_DRIFT = 30 +def _load_coverage_db_meta(path: Optional[str]) -> dict: + """artifact.py's selection JSON; empty when absent or unreadable, which declines.""" + if not path: + return {} + try: + data = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"coverage DB meta unreadable ({path}): {e}", file=sys.stderr) + return {} + return data if isinstance(data, dict) else {} + + def _coverage_freshness(drift: Optional[int], max_drift: int) -> tuple[str, str]: """Verdict on the consulted DB's drift, plus the decline note (empty when usable).""" if drift is None: @@ -383,40 +395,11 @@ def main(argv: Optional[list[str]] = None) -> int: "runs on fallbacks and may drop fully-safe single-GPU stages.", ) parser.add_argument( - "--coverage-db-build", - type=int, - default=None, - help="Post-merge build the --coverage-db came from; recorded in the decision " - "so it can be traced back to its DB.", - ) - parser.add_argument( - "--coverage-db-commit", - default=None, - help="Revision the --coverage-db was collected at (from the build's build_info.txt).", - ) - parser.add_argument( - "--coverage-db-lag", - type=int, + "--coverage-db-meta", default=None, - help="Commits main gained since --coverage-db-commit; recorded only, the " - "freshness gate uses --coverage-db-drift.", - ) - parser.add_argument( - "--coverage-db-drift", - type=int, - default=None, - help="Commits between --coverage-db-commit and the PR's base; what the " - "freshness gate decides on (omitted means unmeasurable, which declines).", - ) - parser.add_argument( - "--coverage-db-base-commit", - default=None, - help="The PR's base revision the drift was measured against.", - ) - parser.add_argument( - "--coverage-db-drift-status", - default="", - help="Which side of the PR's base the DB sits on; recorded only, never weighted.", + help="Path to artifact.py's --print-selection JSON, describing which DB " + "--coverage-db is. Its `drift` is what the freshness gate decides on; the " + "rest is recorded. Absent or unreadable declines the tier.", ) parser.add_argument( "--coverage-max-drift", @@ -480,17 +463,18 @@ def main(argv: Optional[list[str]] = None) -> int: selector = Selector(stages) result = selector.run(pr, rules) - result.coverage_db_build = args.coverage_db_build - result.coverage_db_commit = args.coverage_db_commit - result.coverage_db_lag = args.coverage_db_lag - result.coverage_db_drift = args.coverage_db_drift - result.coverage_db_base_commit = args.coverage_db_base_commit - result.coverage_db_drift_status = args.coverage_db_drift_status + meta = _load_coverage_db_meta(args.coverage_db_meta) + result.coverage_db_build = meta.get("build") + result.coverage_db_commit = meta.get("commit") + result.coverage_db_lag = meta.get("lag") + result.coverage_db_drift = meta.get("drift") + result.coverage_db_base_commit = meta.get("base_commit") + result.coverage_db_drift_status = meta.get("drift_status") or "" if args.coverage_db and result.scope is None: tier = None result.coverage_freshness, note = _coverage_freshness( - args.coverage_db_drift, args.coverage_max_drift + result.coverage_db_drift, args.coverage_max_drift ) if not note: # the gate passed; a note here means it did not try: diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index 93d1c6b1d5b0..68467e533a91 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -151,20 +151,19 @@ def _run_cbts( "--groovy-file", str(groovy), ] + meta_path = None if coverage_db: - # A replay has no meaningful distance; 0 keeps the freshness gate out of it. - argv += [ - "--coverage-db", - coverage_db, - "--coverage-db-lag", - "0", - "--coverage-db-drift", - "0", - ] + # A replay has no PR base; drift 0 keeps the freshness gate out of it. + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"drift": 0}, f) + meta_path = f.name + argv += ["--coverage-db", coverage_db, "--coverage-db-meta", meta_path] try: res = subprocess.run(argv, capture_output=True, text=True, check=False) finally: os.unlink(path) + if meta_path: + os.unlink(meta_path) if res.returncode != 0: return {"_error": res.stderr.strip(), "_returncode": res.returncode} try: From dc0e2878d1a4c61dded974535d8b1706cb3a8cac Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:32:38 +0800 Subject: [PATCH 30/35] [TRTLLM-12838][infra] CBTS: fetch the coverage DB in Python, not in the pipeline _cbtsCoverageAudit had grown to string-building a log line out of JSON fields, mkdir, wget, tar, and writeFile -- all of it shell and Groovy around a Python tool that already had the selection in hand. artifact.py --prepare DIR now does the whole fetch and prints {path, meta}. Groovy keeps only what it alone can do: bind the credential and run coverage_audit.py over the result. The helper goes from roughly forty lines to twenty, and its return map no longer has to be kept in step with the JSON. The tarball is streamed rather than buffered -- it is past 200 MB, so reading it into memory was not an option -- and gets its own socket timeout: the 15s tuned for the small metadata calls expires mid-transfer on an artifact that size. Retries replace trtllm_utils.llmExecStepWithRetry, which cannot wrap a step that no longer exists as a shell command. Verified end to end against build 2895: 226 MB fetched, the 2.2 GB sqlite unpacked, meta written, and main.py declining it as stale at drift 235. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 42 +++---- .../cbts/coverage_selection/SELECTION.md | 24 ++-- .../cbts/coverage_selection/artifact.py | 111 +++++++++++++++++- 3 files changed, 132 insertions(+), 45 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index a85edae0bffa..716bdace2793 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -847,7 +847,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) sh "apt-get update -qq && apt-get install -y -qq python3-yaml" // Download the touch DB for audit + Tier 2 coverage-based narrowing. - def coverageDb = _cbtsCoverageAudit(pipeline, globalVars) + def coverageDb = _cbtsCoverageAudit(pipeline) // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -918,43 +918,29 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Download the touch DB, audit it, and return the sqlite path (or "" on failure). -def _cbtsCoverageAudit(pipeline, globalVars) +// Fetch the touch DB, audit it, and return its `[path, meta]` (both "" on failure). +def _cbtsCoverageAudit(pipeline) { try { - // All commands run from ${LLM_ROOT}; covDir and the returned path are + // artifact.py resolves, downloads and unpacks; paths come back // ${LLM_ROOT}-relative, matching the main.py caller's `cd ${LLM_ROOT}`. - def covDir = "cbts_cov" // The checked-out revision is the PR head; its merge base is what drift is measured against. - def prHeadArg = env.gitlabMergeRequestLastCommit ? " --pr-head ${env.gitlabMergeRequestLastCommit}" : "" - // Ranked by collected revision, not build number; the token is what measures it (depth-1 checkout cannot). - def selJson = "" + def prHead = env.gitlabMergeRequestLastCommit ?: "" + def readyJson = "" withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { - selJson = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-selection${prHeadArg} || true", + readyJson = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + + "--prepare cbts_cov${prHead ? " --pr-head ${prHead}" : ""} || true", returnStdout: true, ).trim() } - if (!selJson) { - pipeline.echo("CBTS audit: no coverage DB artifact found — skipping Tier 2") + if (!readyJson) { + pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2") return [path: "", meta: ""] } - def sel = new groovy.json.JsonSlurper().parseText(selJson) - def url = sel.url - pipeline.echo("CBTS audit: coverage DB from build ${sel.build}, " + - "commit ${sel.commit ?: 'unknown'}, ${sel.lag == null ? 'lag unknown' : sel.lag + ' commit(s) behind main'}, " + - "${sel.drift == null ? 'drift unmeasured' : sel.drift + ' commit(s) ' + (sel.drift_status ?: '') + ' the PR base ' + (sel.base_commit ?: '')}") - sh "cd ${LLM_ROOT} && mkdir -p ${covDir}" - // wget the tarball (retrying) and extract the sqlite. - trtllm_utils.llmExecStepWithRetry(pipeline, script: - "cd ${LLM_ROOT} && wget -nv '${url}' -O ${covDir}/cbts_pystart_report.tar.gz && " + - "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") - sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py " + - "--db ${covDir}/cbts_touchmap.sqlite" - // The selection JSON rides along verbatim for main.py's record and drift gate. - def metaPath = "cbts_coverage_db.json" - writeFile file: "${LLM_ROOT}/${metaPath}", text: selJson - return [path: "${covDir}/cbts_touchmap.sqlite", meta: metaPath] + def ready = new groovy.json.JsonSlurper().parseText(readyJson) + sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" + return [path: ready.path, meta: ready.meta] } catch (InterruptedException e) { throw e } catch (Exception e) { diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index ddfc56961681..8720cbb05319 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -233,17 +233,19 @@ declines: freshness that cannot be shown is not assumed. ### 8.3 What happens with the result -The tarball is downloaded (retried), the sqlite extracted, and `coverage_audit.py` run over it; -any failure in this whole path is caught and non-fatal — `coverageDb.path` stays empty, Tier 2 -never runs, and the PR gets a full run. - -The selection JSON is written to `cbts_coverage_db.json` verbatim and reaches `main.py` as -`--coverage-db-meta`, so a new field needs no Groovy change. `main.py` records all of it and -**gates on the drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs -in full, on -the grounds that a DB that far from the PR's base no longer describes who touches what in the code -under test. A drift that could not be measured — including a meta file that is missing or -unreadable — is treated the same way. +`--prepare DIR` does the whole fetch in one call: select, measure, stream the tarball down +(retried, and streamed rather than buffered — it runs past 200 MB), unpack it, write the selection +JSON beside the sqlite as `cbts_coverage_db.json`, and print `{path, meta}`. Groovy is left with +the two things only it can do — bind the credential and run `coverage_audit.py` over the result — +and any failure anywhere is caught and non-fatal: `coverageDb.path` stays empty, Tier 2 never +runs, and the PR gets a full run. + +Those two paths reach `main.py` as `--coverage-db` and `--coverage-db-meta`, so a new selection +field needs no Groovy change. `main.py` records all of it and **gates on the drift**: past +`--coverage-max-drift` (default 30) the tier declines and the PR runs in full, on the grounds that +a DB that far from the PR's base no longer describes who touches what in the code under test. A +drift that could not be measured — including a meta file that is missing or unreadable — is +treated the same way. All of it lands in the decision and in OpenSearch: diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index be1dcc5d7f5f..6eafbca2fa2b 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -24,9 +24,10 @@ Ranking scores candidates against the tip of `COVERAGE_BRANCH`; gating scores the winner against the PR's merge base, which `--pr-head` supplies. -`--print-selection` prints `{url, build, commit, lag, base_commit, drift, -drift_status}` as JSON for the Groovy wiring, which downloads and extracts the -tarball itself. +Two entry points. `--print-selection` prints `{url, build, commit, lag, +base_commit, drift, drift_status}` and stops. `--prepare DIR` goes on to +download and unpack the winner, drop that JSON beside it, and print +`{path, meta}` — the two paths `main.py` needs. """ from __future__ import annotations @@ -34,15 +35,21 @@ import argparse import json import os +import shutil import sys +import tarfile import urllib.error import urllib.request from functools import lru_cache +from pathlib import Path from typing import Optional # Merged-artifact base for the main-branch L0_PostMerge job. ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" TARBALL_NAME = "cbts_pystart_report.tar.gz" +# sqlite at the tar root, and the selection JSON `prepare` drops beside it. +DB_NAME = "cbts_touchmap.sqlite" +META_NAME = "cbts_coverage_db.json" # Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" @@ -56,8 +63,12 @@ _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Max builds to walk back when recent builds have no tarball. _MAX_PROBE = 10 -# Per-request timeout in seconds. +# Per-request timeout in seconds, for the small JSON/metadata calls. _TIMEOUT = 15 +# Socket timeout for the tarball itself, which runs to hundreds of MB. +_DOWNLOAD_TIMEOUT = 300 +# Tarball download attempts. +_RETRIES = 3 def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: @@ -233,6 +244,81 @@ def measure_drift(sel: dict, pr_head: Optional[str]) -> dict: return sel +def describe(sel: dict) -> str: + """One-line account of the selection, for the CI log.""" + lag = "lag unknown" if sel.get("lag") is None else f"{sel['lag']} commit(s) behind main" + if sel.get("drift") is None: + drifted = "drift unmeasured" + else: + base = (sel.get("base_commit") or "")[:10] + drifted = f"{sel['drift']} commit(s) {sel.get('drift_status')} the PR base {base}" + return ( + f"[artifact] build {sel.get('build')}, commit {(sel.get('commit') or 'unknown')[:10]}, " + f"{lag}, {drifted}" + ) + + +def download(url: str, dest: Path, attempts: int = _RETRIES) -> Optional[Path]: + """Stream the tarball into `dest`, retrying; None when every attempt fails. + + Streamed, not buffered: the artifact runs to hundreds of MB. + """ + out = dest / TARBALL_NAME + for attempt in range(1, attempts + 1): + try: + with ( + urllib.request.urlopen(url, timeout=_DOWNLOAD_TIMEOUT) as resp, + out.open("wb") as f, + ): + shutil.copyfileobj(resp, f) + return out + except OSError as e: # HTTPError/URLError are OSError subclasses + print(f"[artifact] download {attempt}/{attempts} failed: {e}", file=sys.stderr) + return None + + +def extract(tarball: Path, dest: Path) -> bool: + """Unpack the tarball into `dest`; False when it is not readable.""" + try: + with tarfile.open(tarball, "r:gz") as tf: + # `filter` lands in 3.12 and is the default from 3.14; older runtimes lack it. + if sys.version_info >= (3, 12): + tf.extractall(dest, filter="data") + else: + tf.extractall(dest) + except (OSError, tarfile.TarError) as e: + print(f"[artifact] extract failed: {e}", file=sys.stderr) + return False + return True + + +def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: + """Resolve, measure, download and unpack the DB; `{path, meta}` or None on any failure. + + `meta` is the selection JSON on disk, which `main.py --coverage-db-meta` reads. + Paths are relative to the caller's cwd, matching the Groovy caller's `cd ${LLM_ROOT}`. + """ + sel = select_tarball() + if sel is None: + return None + measure_drift(sel, pr_head) + print(describe(sel), file=sys.stderr) + + dest = Path(dest_dir) + dest.mkdir(parents=True, exist_ok=True) + tarball = download(sel["url"], dest) + if tarball is None or not extract(tarball, dest): + return None + db = dest / DB_NAME + if not db.is_file(): + print(f"[artifact] {DB_NAME} not in the tarball", file=sys.stderr) + return None + + meta = dest / META_NAME + meta.write_text(json.dumps(sel)) + return {"path": str(db), "meta": str(meta)} + + def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -245,6 +331,12 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) + ap.add_argument( + "--prepare", + metavar="DIR", + default=None, + help="resolve, download and unpack the DB into DIR, then print {path, meta} as JSON", + ) ap.add_argument( "--pr-head", default=None, @@ -253,8 +345,15 @@ def main(argv: Optional[list[str]] = None) -> int: ) args = ap.parse_args(argv) - if not args.print_selection: - ap.error("--print-selection is required") + if not args.print_selection and not args.prepare: + ap.error("one of --print-selection / --prepare is required") + + if args.prepare: + ready = prepare(args.prepare, args.pr_head) + if ready is None: + return 1 + print(json.dumps(ready)) + return 0 if args.build is not None: commit = build_commit(args.build) From d203a288301813f6ea2e011fb33b4e9d0fb4ae5c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:42:19 +0800 Subject: [PATCH 31/35] [TRTLLM-12838][infra] CBTS: hand back artifact.py's paths instead of repacking them _cbtsCoverageAudit unpacked prepare's {path, meta} and built a fresh map of the same two keys, so the failure branches had to name them too -- and meta was dead there, the caller guarding on path alone. It now returns that map verbatim, or null. The caller guards on the map itself, which also covers artifact.py printing an empty object. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 716bdace2793..aafa30e5ec2f 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -873,7 +873,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) writeFile file: inputPath, text: inputJson def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" - if (coverageDb.path) { + if (coverageDb) { mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" } def output = sh(script: mainCmd, returnStdout: true) @@ -918,7 +918,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Fetch the touch DB, audit it, and return its `[path, meta]` (both "" on failure). +// Fetch the touch DB and audit it; artifact.py's {path, meta} verbatim, or null on failure. def _cbtsCoverageAudit(pipeline) { try { @@ -936,16 +936,16 @@ def _cbtsCoverageAudit(pipeline) } if (!readyJson) { pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2") - return [path: "", meta: ""] + return null } def ready = new groovy.json.JsonSlurper().parseText(readyJson) sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" - return [path: ready.path, meta: ready.meta] + return ready } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - return [path: "", meta: ""] + return null } } From dac0c9d3c0a9ca52ea47c7cbdcd7cfa373cb0b16 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:54:14 +0800 Subject: [PATCH 32/35] [TRTLLM-12838][infra] CBTS: drop the import-section break a stray local dir provoked A scratch `rules/` in the working tree resolves as a first-party module, so ruff's isort splits `from rules._helpers` into its own section and inserts a blank line. CI checks out clean, sees the same name as third-party, and takes the line back out. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 842652502154..df0b03a2ae1e 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -58,7 +58,6 @@ open_db, write_coverage_test_db, ) - from rules._helpers import strip_noop_diff_lines # noqa: E402 from rules.agent_flow_rule import AgentFlowRule # noqa: E402 from rules.auto_deploy_rule import AutoDeployRule # noqa: E402 From 4a9cbe3334f21359af6f0fbde0074746238e57f7 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:20:48 +0800 Subject: [PATCH 33/35] drift tolerance change to 10 Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index df0b03a2ae1e..3fb23645868a 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -172,7 +172,7 @@ def to_json(self) -> str: # Tier 2 stands down past this many commits between the DB's revision and the PR's base. -DEFAULT_COVERAGE_MAX_DRIFT = 30 +DEFAULT_COVERAGE_MAX_DRIFT = 10 def _load_coverage_db_meta(path: Optional[str]) -> dict: From a8032b51123010d8c11c560d07e172bc134fb826 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:12:04 +0800 Subject: [PATCH 34/35] [TRTLLM-12838][infra] Move CBTS filtering out of launchTestJobs Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 97 ++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 76bf47975cb0..098693652d12 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2549,6 +2549,56 @@ def cbtsResizeSplits(configs) { return resized } +// CBTS Layer 2: replace the normal stage set with the selector's affected +// stages while retaining the baseline sanity and multi-GPU gates. +def filterCbtsStageJobs(parallelJobs, parallelJobsFiltered, multiGpuJobs, testFilter) { + def cbts = testFilter[(CBTS_RESULT)] + if (cbts == null) { + return parallelJobsFiltered + } + + // cbtsResizeSplits renames only narrowed stages (those in + // affected_stage_split_counts) to `-cbts`; affected-but-not-narrowed + // stages keep their original name, so match each per its actual key. + def stageSuffix = cbts.cbts_test_db_artifact_path ? CBTS_STAGE_SUFFIX : "" + def narrowed = (cbts.affected_stage_split_counts ?: [:]).keySet() + def affectedSet = (cbts.affected_stages ?: []).collect { + (stageSuffix && narrowed.contains(it)) ? (it + stageSuffix) : it + } as Set + def needsSanity = cbts.sanity_required + def needsPerfSanity = cbts.perfsanity_required + def filtered = parallelJobs.findAll { key, _ -> + if (key.contains("-OnDemand-")) { + return false + } + if (key =~ /Post-Merge/) return affectedSet.contains(key) + return affectedSet.contains(key) || + (needsSanity && key =~ /PackageSanityCheck/) || + (needsPerfSanity && key =~ /PerfSanity/) + } + if (affectedSet.isEmpty()) { + if (filtered.isEmpty()) { + echo "CBTS [${cbts.scope}]: trigger-mode mismatch + nothing force-kept → no-op" + } else { + echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running " + + "${filtered.size()} force-kept stage(s) only" + } + } else if (filtered) { + echo "CBTS [${cbts.scope}]: limiting to ${filtered.size()} stages " + + "(sanity_required=${needsSanity}, perfsanity_required=${needsPerfSanity})" + } else { + echo "CBTS [${cbts.scope}]: empty stage set after filtering" + } + + // The coverage tier omits multi-GPU; re-add it under the baseline gate. + if (cbts.enable_multi_gpu && testFilter[(MULTI_GPU_FILE_CHANGED)]) { + filtered += multiGpuJobs + echo "CBTS [${cbts.scope}]: multi-GPU file changed → running " + + "${multiGpuJobs.size()} multi-GPU stage(s) at baseline" + } + return filtered +} + // True when an exception indicates the K8s dispatcher pod this SLURM stage runs // inside died mid-run -- kubelet eviction, container termination, or the JNLP // agent otherwise going offline. Retrying inside such a pod is futile (every @@ -6539,51 +6589,8 @@ def launchTestJobs(pipeline, testFilter, globalVars) checkStageNameSet(testFilter[(EXTRA_STAGE_LIST)], fullSet, EXTRA_STAGE_LIST) } - // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus - // PackageSanityCheck (kept iff sanity_required) and PerfSanity (kept iff - // perfsanity_required). Pure -Perf- stages run only when CBTS selects them - // (present in affected_stages). Post-Merge stages are never force-kept; - // they only run when explicitly listed in affected_stages. - def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null) { - // cbtsResizeSplits renames only narrowed stages (those in - // affected_stage_split_counts) to `-cbts`; affected-but-not-narrowed - // stages keep their original name, so match each per its actual key. - def stageSuffix = cbts.cbts_test_db_artifact_path ? CBTS_STAGE_SUFFIX : "" - def narrowed = (cbts.affected_stage_split_counts ?: [:]).keySet() - def affectedSet = (cbts.affected_stages ?: []).collect { - (stageSuffix && narrowed.contains(it)) ? (it + stageSuffix) : it - } as Set - def needsSanity = cbts.sanity_required - def needsPerfSanity = cbts.perfsanity_required - parallelJobsFiltered = parallelJobs.findAll { key, _ -> - if (key.contains("-OnDemand-")) { - return false - } - if (key =~ /Post-Merge/) return affectedSet.contains(key) - return affectedSet.contains(key) || - (needsSanity && key =~ /PackageSanityCheck/) || - (needsPerfSanity && key =~ /PerfSanity/) - } - if (affectedSet.isEmpty()) { - if (parallelJobsFiltered.isEmpty()) { - echo "CBTS [${cbts.scope}]: trigger-mode mismatch + nothing force-kept → no-op" - } else { - echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running " + - "${parallelJobsFiltered.size()} force-kept stage(s) only" - } - } else if (parallelJobsFiltered) { - echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} stages " + - "(sanity_required=${needsSanity}, perfsanity_required=${needsPerfSanity})" - } else { - echo "CBTS [${cbts.scope}]: empty stage set after filtering" - } - // coverage tier omits multi-GPU; re-add under baseline gate - if (cbts.enable_multi_gpu && testFilter[(MULTI_GPU_FILE_CHANGED)]) { - parallelJobsFiltered += multiGpuJobs - echo "CBTS [${cbts.scope}]: multi-GPU file changed → running ${multiGpuJobs.size()} multi-GPU stage(s) at baseline" - } - } + parallelJobsFiltered = filterCbtsStageJobs( + parallelJobs, parallelJobsFiltered, multiGpuJobs, testFilter) if (globalVars[RUN_MODE] == "nightly_release") { parallelJobsFiltered = sanityCheckJobs From 03a730273cdc088f5e3db52c99f5b3f95d9d39b4 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:17:55 +0800 Subject: [PATCH 35/35] [TRTLLM-12838][infra] CBTS: disable coverage tier by default Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index aafa30e5ec2f..f9fbc38b33a0 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -155,6 +155,10 @@ def DISABLE_CBTS = "disable_cbts" // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true +// Rollout switch for pre-merge Tier 2 coverage-based narrowing. Keep collection +// enabled above while this remains off so a later pilot allowlist has fresh data. +@Field +def ENABLE_CBTS_COVERAGE_TIER = false def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), @@ -846,8 +850,9 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // Download the touch DB for audit + Tier 2 coverage-based narrowing. - def coverageDb = _cbtsCoverageAudit(pipeline) + // Download the touch DB only when Tier 2 is enabled. Tier 1 rules still + // run while the coverage tier is disabled during the initial rollout. + def coverageDb = _cbtsCoverageDb(pipeline) // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -918,6 +923,17 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } +// Resolve the optional Tier 2 input behind an explicit rollout gate. Keeping +// this separate makes the follow-up pilot allowlist a small policy change. +def _cbtsCoverageDb(pipeline) +{ + if (!ENABLE_CBTS_COVERAGE_TIER) { + pipeline.echo("CBTS: coverage tier disabled — running Tier 1 only") + return null + } + return _cbtsCoverageAudit(pipeline) +} + // Fetch the touch DB and audit it; artifact.py's {path, meta} verbatim, or null on failure. def _cbtsCoverageAudit(pipeline) {