From 40cbce4bc38a7e80a5a593d54af3b1519cd1db4f Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:58:41 +0800 Subject: [PATCH 01/65] initial cbts structure Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 142 +++++++++ jenkins/L0_Test.groovy | 50 ++++ jenkins/scripts/cbts/DESIGN.md | 344 ++++++++++++++++++++++ jenkins/scripts/cbts/README.md | 161 ++++++++++ jenkins/scripts/cbts/__init__.py | 2 + jenkins/scripts/cbts/blocks.py | 290 ++++++++++++++++++ jenkins/scripts/cbts/main.py | 256 ++++++++++++++++ jenkins/scripts/cbts/rules/__init__.py | 2 + jenkins/scripts/cbts/rules/base.py | 47 +++ jenkins/scripts/cbts/rules/waives_rule.py | 130 ++++++++ 10 files changed, 1424 insertions(+) create mode 100644 jenkins/scripts/cbts/DESIGN.md create mode 100644 jenkins/scripts/cbts/README.md create mode 100644 jenkins/scripts/cbts/__init__.py create mode 100644 jenkins/scripts/cbts/blocks.py create mode 100644 jenkins/scripts/cbts/main.py create mode 100644 jenkins/scripts/cbts/rules/__init__.py create mode 100644 jenkins/scripts/cbts/rules/base.py create mode 100644 jenkins/scripts/cbts/rules/waives_rule.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 3b56dd0c4646..b3568e2e8d12 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -120,6 +120,8 @@ def AUTO_TRIGGER_TAG_LIST = "auto_trigger_tag_list" def DEBUG_MODE = "debug" @Field def DETAILED_LOG = "detailed_log" +@Field +def CBTS_RESULT = "cbts_result" def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), @@ -138,6 +140,7 @@ def testFilter = [ (DEBUG_MODE): gitlabParamsFromBot.get(DEBUG_MODE, false), (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): gitlabParamsFromBot.get(DETAILED_LOG, false), + (CBTS_RESULT): null, ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -308,6 +311,7 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) getContainerURIs().each { k, v -> globalVars[k] = v } @@ -696,6 +700,132 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { return autoTriggerTagList } +// ============================================================================ +// CBTS (Change-Based Testing Selection) +// +// Upstream decision point. Calls jenkins/scripts/cbts/main.py with the PR's +// changed_files + diffs; Python self-sources stage configs from L0_Test.groovy +// and YAML blocks from test-db. Returns a dict {scope, affected_cpu_arch, +// affected_stages, affected_tests, reasons} or null (= no decision / fall +// back to the existing filter chain). +// +// See jenkins/scripts/cbts/DESIGN.md for the three-layer consumption model. +// ============================================================================ + +def getCbtsResult(pipeline, testFilter, globalVars) +{ + def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) + if (env.alternativeTRT || isOfficialPostMergeJob) { + return null + } + + def changedFiles = getMergeRequestChangedFileList(pipeline, globalVars).unique() + if (!changedFiles) { + return null + } + + try { + // 1. Ask Python for the union of needs_diff_for patterns across all rules. + def patternsOut = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", + returnStdout: true, + ).trim() + def needsDiffFor = patternsOut ? patternsOut.readLines().collect { it.trim() }.findAll { it } : [] + + // 2. For each changed file matching a needs_diff_for pattern, pull the diff. + def diffs = [:] + for (f in changedFiles) { + if (_cbtsMatchesAnyPattern(f, needsDiffFor)) { + diffs[f] = getMergeRequestOneFileChanges(pipeline, globalVars, f) + } + } + + // 3. Write INPUT_JSON (PR data only; Python reads stages/yaml itself). + def inputJson = groovy.json.JsonOutput.toJson([ + changed_files: changedFiles, + diffs: diffs, + ]) + def inputPath = "${LLM_ROOT}/cbts_input.json" + writeFile file: inputPath, text: inputJson + + // 4. Run Python; capture stdout. + def output = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json", + returnStdout: true, + ) + + // 5. Parse stdout into the map shape consumed by Layer 1/2/3. + def result = _cbtsParseSelectionResult(output) + if (result != null) { + pipeline.echo("CBTS: scope=${result.scope}, " + + "archs=${result.affected_cpu_arch}, " + + "stages=${result.affected_stages.size()}, " + + "tests=${result.affected_tests.size()}") + } + return result + } catch (Exception e) { + pipeline.echo("CBTS failed, falling back to full run: ${e.getMessage()}") + return null + } +} + +// Simple glob matcher: supports `**` (any chars) and `*` (non-slash). +// v0 needs_diff_for is a plain path, but future rules may use globs like +// "tests/integration/defs/**/*.py", so we implement minimal glob support. +def _cbtsMatchesAnyPattern(String filePath, List patterns) +{ + for (p in patterns) { + if (filePath == p) { return true } + if (!p.contains('*')) { continue } + // Escape regex meta-chars that might appear in paths, then substitute + // `**` and `*` via a sentinel so they don't collide. + def regex = p.replace('.', '\\.') + .replace('+', '\\+') + .replace('(', '\\(') + .replace(')', '\\)') + .replace('**', 'DBLSTAR_SENTINEL') + .replace('*', '[^/]*') + .replace('DBLSTAR_SENTINEL', '.*') + if (filePath ==~ regex) { return true } + } + return false +} + +// Parse CBTS stdout (format: see DESIGN.md 4.6) into a nullable map. +// Returns null when scope=none (no decision => full run). +def _cbtsParseSelectionResult(String text) +{ + def scope = null + def archs = [] + def stages = [] + def tests = [] + def reasons = [] + for (line in text.readLines()) { + if (line.startsWith("# SCOPE:")) { + def v = line.replaceFirst(/^# SCOPE:\s*/, '').trim() + scope = (v == 'none' || v == '') ? null : v + } else if (line.startsWith("# REASON:")) { + reasons.add(line.replaceFirst(/^# REASON:\s*/, '').trim()) + } else if (line.startsWith("# AFFECTED_CPU_ARCH:")) { + def v = line.replaceFirst(/^# AFFECTED_CPU_ARCH:\s*/, '').trim() + archs = v ? v.tokenize(',').collect { it.trim() }.findAll { it } : [] + } else if (line.startsWith("# AFFECTED_STAGES:")) { + def v = line.replaceFirst(/^# AFFECTED_STAGES:\s*/, '').trim() + stages = v ? v.tokenize(',').collect { it.trim() }.findAll { it } : [] + } else if (!line.startsWith("#") && line.trim()) { + tests.add(line.trim()) + } + } + if (scope == null) { return null } + return [ + scope: scope, + affected_cpu_arch: archs, + affected_stages: stages, + affected_tests: tests, + reasons: reasons, + ] +} + def getMultiGpuFileChanged(pipeline, testFilter, globalVars) { if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { @@ -1083,6 +1213,12 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) }, "x86_64-Linux": { script { + // CBTS Layer 1: skip entire x86 track when no x86 stages are affected + def _cbts = testFilter[(CBTS_RESULT)] + if (_cbts?.scope == "waiveonly" && !("x86" in _cbts.affected_cpu_arch)) { + echo "CBTS waiveonly: no x86 stages affected, skipping x86_64-Linux track" + return + } def testStageName = "[Build-x86_64] Remote Run" stage(testStageName) { def additionalParameters = [ @@ -1194,6 +1330,12 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "SBSA build job is skipped due to Jenkins configuration or conditional pipeline run" return } + // CBTS Layer 1: skip entire SBSA track when no sbsa stages are affected + def _cbts = testFilter[(CBTS_RESULT)] + if (_cbts?.scope == "waiveonly" && !("sbsa" in _cbts.affected_cpu_arch)) { + echo "CBTS waiveonly: no sbsa stages affected, skipping SBSA-Linux track" + return + } def testStageName = "[Build-SBSA] Remote Run" stage(testStageName) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 1dcdd672f37b..148742ce34ee 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -571,6 +571,36 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } +// CBTS helper: restrict a rendered testDBList to the intersection with +// CBTS's affected_tests set. Preserves original line format (including +// ISOLATION markers and pytest args), matching by the leading token. +def filterTestDBListForCbts(String testDBList, List affectedTests, String stageName) { + def affectedSet = affectedTests as Set + def originalLines = readFile(file: testDBList).readLines() + def kept = originalLines.findAll { line -> + def trimmed = line.trim() + if (!trimmed || trimmed.startsWith("#")) { return false } + // Bare node-id / path: drop ISOLATION marker and trailing pytest args. + def bare = trimmed + if (bare.contains(" ISOLATION")) { + bare = bare.replaceAll(/\s*ISOLATION.*$/, '').trim() + } + if (bare.contains(" ")) { + bare = bare.split(" ", 2)[0] + } + return affectedSet.contains(bare) || affectedSet.contains(trimmed) + } + def filtered = testDBList.replaceAll(/\.txt$/, '_cbts_filtered.txt') + if (kept.isEmpty()) { + // Avoid `echo` with empty shell-quoted content. + sh "touch ${filtered}" + } else { + writeFile file: filtered, text: kept.join("\n") + "\n" + } + echo "CBTS Layer 3 (${stageName}): kept ${kept.size()}/${originalLines.size()} tests" + return filtered +} + def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false) { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -1817,6 +1847,8 @@ def DEBUG_MODE = "debug" @Field def DETAILED_LOG = "detailed_log" @Field +def CBTS_RESULT = "cbts_result" +@Field def testFilter = [ (REUSE_TEST): null, (REUSE_STAGE_LIST): null, @@ -1834,6 +1866,7 @@ def testFilter = [ (DEBUG_MODE): false, (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): false, + (CBTS_RESULT): null, ] @Field @@ -3091,6 +3124,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt") } + // CBTS Layer 3: within an affected stage, further restrict testDBList + // to the tests CBTS identified as affected. + def _cbts = testFilter[(CBTS_RESULT)] + if (_cbts?.scope == "waiveonly" && _cbts.affected_tests) { + testDBList = filterTestDBListForCbts(testDBList, _cbts.affected_tests, stageName) + } + // Process shard test list and create separate files for regular and isolate tests def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode) @@ -4402,6 +4442,16 @@ def launchTestJobs(pipeline, testFilter) checkStageNameSet(testFilter[(EXTRA_STAGE_LIST)], fullSet, EXTRA_STAGE_LIST) } + // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all + // existing filter rules so that unknown / no-decision paths fall through + // naturally. See jenkins/scripts/cbts/DESIGN.md for the three-layer model. + def _cbts = testFilter[(CBTS_RESULT)] + if (_cbts?.scope == "waiveonly" && _cbts.affected_stages) { + def affectedSet = _cbts.affected_stages as Set + parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) } + echo "CBTS waiveonly: limiting to ${parallelJobsFiltered.size()} affected stages" + } + echo "Check the passed GitLab bot testFilter parameters." def keysStr = parallelJobsFiltered.keySet().join(",\n") pipeline.echo "Now we will run stages: [\n${keysStr}\n]" diff --git a/jenkins/scripts/cbts/DESIGN.md b/jenkins/scripts/cbts/DESIGN.md new file mode 100644 index 000000000000..f64275268910 --- /dev/null +++ b/jenkins/scripts/cbts/DESIGN.md @@ -0,0 +1,344 @@ +# CBTS — Change-Based Testing Selection for CI + +**Status**: Draft for infra + dev review +**v0 Scope**: When a PR only changes `tests/integration/test_lists/waives.txt`, decide which stages + tests to run at block granularity. + +--- + +## 1. Core Idea + +PR changes one test in waives.txt → that test lives in some YAML block → the block's `condition` matches only certain stages → **other stages aren't scheduled**; build + test sub-jobs for the arch track (x86 / SBSA) that has no affected stages are **entirely skipped**; inside each scheduled stage, the test list is further filtered by test id. + +**Three-layer filtering**: +- **Layer 1 (arch track level)**: affected stages only on x86 → skip the entire SBSA track (SBSA build + all SBSA sub-jobs), and vice versa. +- **Layer 2 (stage level)**: within the same arch, match `block.condition` against `stage.mako` to pick which stages to schedule. +- **Layer 3 (test level)**: inside a running stage, intersect `renderTestDB`'s output with CBTS's test set. + +No changes to pytest-split, trt-test-db, or the stage-scheduling core; all new code is glue. + +--- + +## 2. v0 Scope + +- **Covers**: PRs that only change waives.txt. +- **Other changes**: any other file → fall through to the existing filter chain (equivalent to full run). +- **Semantics**: add/remove/edit a waive → run the corresponding test and the stages matching its containing block. +- **scope label**: v0 defines one value — `waiveonly` (PR only changed waive-related files). Future rules introduce new scope values (e.g. `testonly`, `modelarch`) when needed; we do not invent an abstract taxonomy upfront. + +--- + +## 3. Code Architecture + +### 3.1 File layout + +``` +jenkins/scripts/cbts/ +├── __init__.py +├── DESIGN.md +├── main.py ← Selector + SelectionResult + CLI +├── blocks.py ← Stage + Block + YAMLIndex + block_matches_stage +└── rules/ + ├── __init__.py + ├── base.py ← Rule ABC + PRInputs + RuleResult + └── waives_rule.py ← v0 rule +``` + +**4 business files + 2 `__init__.py`**. + +### 3.2 Key contracts + +```python +# rules/base.py +@dataclass +class PRInputs: + changed_files: list[str] + diffs: dict[str, str] # Groovy pre-fetches based on needs_diff_for + +@dataclass +class RuleResult: + handled_files: set[str] + tests: set[str] # Layer 3: within-stage filter + affected_stages: set[str] # Layer 2: stages to schedule + scope: str # rule-declared scope label, v0 only has "waiveonly" + reason: str + +class Rule(ABC): + name: str + needs_diff_for: list[str] = [] + @abstractmethod + def apply(self, pr: PRInputs) -> Optional[RuleResult]: ... +``` + +Note: `affected_cpu_arch` is not a RuleResult field; the Selector derives it by looking up each affected stage's `cpu_arch` in the stage map. Rules don't set it. + +```python +# blocks.py +@dataclass +class Stage: + name: str + yaml_stem: str + cpu_arch: str # "x86" / "sbsa", inferred from x86TestConfigs vs SBSATestConfigs + split_id: int + total_splits: int + mako: dict[str, str] # derived from stage name (mirrors getMakoArgsFromStageName) + +@dataclass +class Block: + yaml_stem: str + block_index: int + condition: dict # raw: {ranges, wildcards, terms} + tests: list[str] + +def block_matches_stage(block, stage) -> bool: + """Generic over YAML field names: adding a new term does not require changing this.""" +``` + +### 3.3 Why no `stages.py` + +`Stage` values are derived purely from what Python can parse out of `jenkins/L0_Test.groovy` (stage map entries + stage-name patterns). Keeping `Stage` together with `Block` in `blocks.py` avoids a module for two small dataclasses. `derive_mako_from_stage` mirrors `getMakoArgsFromStageName` on the Groovy side — single source of truth is still the Groovy file; Python only reads it. + +--- + +## 4. Jenkins Integration + +### 4.1 Injection point: alongside other testFilter setters + +In `L0_MergeRequest.groovy`, find this snippet (excerpt): + +```groovy +testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) +testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) +testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) +// NEW +testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) +``` + +`getCbtsResult` returns either `null` (no decision → full run) or `{scope, affected_cpu_arch, affected_stages, affected_tests, reasons}`. **The decision is made once and cached in `testFilter`**; the three downstream layers only read. + +### 4.2 Layer 1 — Arch-track skip + +Injection point: the `x86_64-Linux` / `SBSA-Linux` track entries in `L0_MergeRequest.groovy::launchStages()` (there is already precedent there: `if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") return` skips SBSA for docs-only PRs). + +```groovy +"x86_64-Linux": { + script { + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && !("x86" in cbts.affected_cpu_arch)) { + echo "CBTS waiveonly: no x86 stages affected, skipping x86_64-Linux track" + return + } + // existing Build-x86_64 + Test-x86_64-* logic unchanged + ... + } +}, +"SBSA-Linux": { + script { + if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") { return } // existing + // NEW: CBTS waiveonly equivalent skip + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && !("sbsa" in cbts.affected_cpu_arch)) { + echo "CBTS waiveonly: no sbsa stages affected, skipping SBSA-Linux track" + return + } + // existing Build-SBSA + Test-SBSA-* logic unchanged + ... + } +}, +``` + +**Key points**: +- The condition explicitly matches `scope == "waiveonly"`. Future scopes (`testonly` / `modelarch`) do not trigger track skips by default until their safety is evaluated and an explicit `else if` branch is added. +- Effect: an x86-only waive change → the entire SBSA track disappears from Blue Ocean (including build); and vice versa. + +### 4.3 Layer 2 — Stage-scheduling override + +Injection point: the filter chain in `L0_Test.groovy` (around lines 3714–3800). **CBTS acts as a short-circuit override appended to the end of the existing chain**; the existing logic is untouched. + +```groovy +// Existing filter chain untouched: MULTI_GPU_FILE_CHANGED / AUTO_TRIGGER_TAG_LIST / +// IS_POST_MERGE / ENABLE_SKIP_TEST / GPU_TYPE_LIST / TEST_BACKEND / ... +// Produces parallelJobsFiltered. +... + +// NEW: CBTS short-circuit override at the tail +def cbts = testFilter[(CBTS_RESULT)] +if (cbts?.scope == "waiveonly") { + parallelJobsFiltered = parallelJobs.findAll { key, _ -> key in cbts.affected_stages } + echo "CBTS waiveonly: limiting to ${cbts.affected_stages.size()} affected stages" +} +``` + +**Key points**: +- **One `if`, no `else`**: concise, no nested branches. +- **Override semantics**: the existing chain produces `parallelJobsFiltered` first; when waiveonly matches, it is replaced wholesale. +- **All fallback cases naturally don't override**: `cbts == null` / `scope == null` / unknown scope / call failure → condition false → the existing chain's result is preserved. +- **Adding a new scope**: add another parallel `if (cbts?.scope == "testonly") { ... }`; scopes don't interfere. +- **Cost**: on a waiveonly PR the existing filter chain runs once and is then overwritten (pure Groovy set ops; no IO; negligible). + +### 4.4 Layer 3 — Within-stage test filter + +In the block starting at `L0_Test.groovy:2674`, the CBTS filter is inserted **right before `processShardTestList`**, after all prep (`mergeWaivesTxt` / `reusePassedTestResults`) has completed: + +```groovy +def testDBList = renderTestDB(testList, llmSrc, stageName) +mergeWaivesTxt(pipeline, llmSrc, stageName) // existing: download merged waives.txt +// reusePassedTestResults(...) // existing: append previously-passed tests to waives + +// NEW: CBTS Layer 3 filter, single-point insertion +def cbts = testFilter[(CBTS_RESULT)] +if (cbts?.scope == "waiveonly") { + testDBList = filterTestDBList(testDBList, cbts.affected_tests) +} + +def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode) +``` + +Same explicit match on `waiveonly`; future scopes must decide independently whether to filter tests at this layer by adding an `else if` branch. + +#### Interaction with `mergeWaivesTxt`: verified consistent + +The merged waives.txt downloaded by `mergeWaivesTxt` is produced by `jenkins/scripts/mergeWaiveList.py` with the following algorithm: + +``` +merged = dedupe(PR's waives.txt ∪ TOT's waives.txt) - lines in PR's diff prefixed with `-` +``` + +This algorithm is **PR-aware**: PR additions are preserved via the union; PR removals are applied via subtraction. So the merged result **fully preserves the PR's intent on waives.txt**. + +Three scenarios, all verified: + +| Scenario | merged content | runtime | CBTS decision correctness | +|---|---|---|---| +| PR removes a waive (`-test_X`) | no test_X | test_X runs | ✓ stage runs, test is actually verified | +| PR adds a waive (`+test_X`) | contains test_X | test_X skipped | ✓ stage runs, pytest collects the test and skips correctly (verifies the waive mechanism) | +| PR edits a waive line (NVBug link) | new line only | test still skipped | ✓ matches PR intent | + +**Conclusion**: CBTS decisions based on the PR diff are naturally aligned with the runtime waive state; **no extra handling for `mergeWaivesTxt` is needed**. + +### 4.5 What `getCbtsResult` does + +1. PostMerge / alternativeTRT → return null. +2. `getMergeRequestChangedFileList` → `changed_files`, `.unique()`. +3. Ask Python `--list-needed-diffs` for patterns; for each changed file matching a pattern, call `getMergeRequestOneFileChanges` to pull its diff. +4. Write `cbts_input.json` (just `changed_files` + `diffs`), then run `python3 main.py cbts_input.json`. +5. Parse stdout, return a structured result. + +Note: the Python side **parses `jenkins/L0_Test.groovy` itself** to derive stage configs (reusing the regex approach already in `scripts/test_to_stage_mapping.py`) and loads YAMLs from `tests/integration/test_lists/test-db/`. Groovy does not need to pack `stage_map` into INPUT_JSON. This keeps the Groovy code surface minimal. + +Each stage's `cpu_arch` is inferred by tracking which map-literal (`x86TestConfigs` / `SBSATestConfigs` / ...) the entry lives in. Each stage's `mako` is derived by a Python port of `getMakoArgsFromStageName` (line ~2079) and `parseTaskConfigFromStageName` (line ~2066). Those Python helpers live in `blocks.py` with explicit "keep in sync with groovy source" comments at the top. + +### 4.6 Python ↔ Groovy IO contract + +Groovy `getCbtsResult` calls Python twice: + +1. **Get needs_diff_for patterns** (no args): Python prints the union of all rules' `needs_diff_for` patterns; Groovy uses this to decide which changed files to fetch diffs for. +2. **Make the decision** (JSON file arg): Groovy packs `changed_files` / `diffs` into INPUT_JSON; Python writes the decision to stdout. + +**INPUT_JSON** is produced by Groovy `getCbtsResult`, containing only PR data: + +```json +{ + "changed_files": ["tests/integration/test_lists/waives.txt", ...], + "diffs": { + "tests/integration/test_lists/waives.txt": "@@ -1,3 +1,4 @@\n..." + } +} +``` + +Stage configs are parsed by Python directly from `jenkins/L0_Test.groovy` (see 4.5). + +**stdout when a decision is made**: + +``` +# SCOPE: waiveonly +# REASON: [waives] waives.txt: +2 / -1 → 2 blocks, 2 stages +# AFFECTED_CPU_ARCH: x86 +# AFFECTED_STAGES: DGX_H100-4_GPUs-PyTorch-DeepSeek-1, DGX_H100-4_GPUs-PyTorch-DeepSeek-2 +examples/test_deepseek.py::test_xxx +``` + +**stdout when no decision**: + +``` +# SCOPE: none +# REASON: Unhandled files: [tensorrt_llm/llmapi/llm.py, ...] +``` + +Groovy parsing contract: +- `# SCOPE:` — scope label (`waiveonly` / `none` / future values); `none` → `testFilter[CBTS_RESULT].scope = null`. +- `# AFFECTED_CPU_ARCH:` — comma-separated, values `x86` / `sbsa`. +- `# AFFECTED_STAGES:` — comma-separated stage names. +- Non-`#` lines: one test id per line. + +Exit code 0 = decision succeeded (including `none`); non-zero → Groovy falls back to full run. + +### 4.7 Multi-rule combination (future) + +v0 has only one rule, so no combining. But `Selector` reserves a `combine_scopes(scopes)` helper: + +```python +def combine_scopes(scopes: list[str]) -> str | None: + # v0: single rule returns "waiveonly" — passthrough. + if len(set(scopes)) == 1: + return scopes[0] + # Multiple differing scopes → conservative None (full run). + # Fill in a priority table here when priority needs emerge. + return None +``` + +--- + +## 5. Fallback & Safety + +Any of the following → **fall through to the existing filter chain** (`testFilter[CBTS_RESULT] = null` or `.scope == null`): + +- PostMerge / alternativeTRT job. +- `changed_files` is empty. +- Python call fails / stdout unparsable. +- Python explicitly returns `scope: none` (unhandled files, no rule matched, multi-rule scope conflict). +- Groovy sees an unknown scope (Python upgraded before Groovy caught up). + +--- + +## 6. Extensions (future, not in v0) + +| New Rule | New modules | `needs_diff_for` | Likely scope | +|---|---|---|---| +| `test_block_rule` (test-file changes) | none; reuses `blocks.py` | `[]` | `testonly` | +| `case_level_rule` (function-level precision) | `code_analysis/ast_utils.py` + `test_extractor.py` | `["tests/integration/defs/**/*.py"]` | `testonly` | +| `model_arch_rule` (model-arch matching) | `model_arch.py` | `[]` | `modelarch` | + +**What adding a rule requires**: +1. Define a new scope value (e.g. `testonly`) and add an `else if` branch at Groovy Layer 1 specifying whether aux stages are skipped for this scope. +2. Write a Rule class on the Python side. +3. If extra PR data is needed, declare it in `needs_diff_for`. + +**What adding a rule does NOT change**: the Rule ABC, the `combine_scopes` skeleton, `blocks.py`, the CLI contract, the overall shape of `getCbtsResult`, or the Layer-2 consumption site. + +--- + +## 7. Review Highlights + +### For infra +- **Zero changes to pytest-split / trt-test-db / stage rendering.** +- **Reuses existing helpers**: `getMergeRequestChangedFileList`, `getMergeRequestOneFileChanges`, `getMakoArgsFromStageName`, stageList. +- **Injections mirror existing patterns**: + - Setter: the fourth setter on `testFilter`. + - Layer 1 consumer: one skip check at each `launchStages` track entry (isomorphic to the existing Docs-only skip). + - Layer 2 consumer: appended `if` in the `L0_Test.groovy` filter chain tail. + - Layer 3 consumer: three-line injection at `L0_Test.groovy:2674`. +- **Any failure falls back to full run.** + +### For dev +- **v0 behavior is conservative**: only narrows PRs that touch only waives.txt. +- **CI log has a reason line**: you can see why particular stages / tests were selected. +- **Local reproducibility**: pull the `cbts_input.json` CI artifact and run `python3 main.py cbts_input.json` to reproduce the decision. +- **Low bar to add a rule**: write a class with an `apply()` method. + +--- + +## 8. Open Questions + +1. **Does infra accept skipping docBuild/sanityCheck under `waiveonly`?** waives.txt is a runtime plain-text list that doesn't flow into the wheel or docs, so skipping should be safe; if the team has a hard rule of "every PR must pass doc build / wheel sanity", the fallback is to keep those two aux stages even under `waiveonly`. +2. **`filterTestDBList` in Groovy or Python?** Leaning Python (more testable). +3. **Self-check mechanism**: Python's `block_matches_stage` could drift from trt-test-db semantics. Deferred to a future revision. diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md new file mode 100644 index 000000000000..40054f7798bb --- /dev/null +++ b/jenkins/scripts/cbts/README.md @@ -0,0 +1,161 @@ +# CBTS — Change-Based Testing Selection + +Pre-merge CI test-selection tool. Looks at what the PR changed and narrows the +set of Jenkins stages + tests that actually need to run. + +For conceptual design / review notes, see [DESIGN.md](./DESIGN.md). This +README is the operational reference. + +--- + +## What it does — three layers + +Given a PR, CBTS produces a decision that gets consumed at three points in the +Jenkins pipeline: + +| Layer | Where consumed | Action | +|---|---|---| +| **1. Arch track** | `L0_MergeRequest.groovy::launchStages` (each track entry) | Skip whole x86 / SBSA track (build + all tests) when no stage on that arch is affected | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset | +| **3. Test** | `L0_Test.groovy::runLLMTestlistOnPlatformImpl` (after `renderTestDB`) | Intersect rendered `testDBList` with CBTS's `affected_tests` | + +Anything CBTS can't confidently narrow → **fallback to the existing full filter +chain**. CBTS never adds stages; it only subtracts. + +## v0 scope + +- **Only handles** `tests/integration/test_lists/waives.txt` changes +- Any other changed file → CBTS returns `scope: none` → full run +- Scope label for this case: `waiveonly` + +## File map + +``` +jenkins/scripts/cbts/ +├── DESIGN.md design doc (for review) +├── README.md this file +├── main.py CLI entry + Selector + SelectionResult +├── blocks.py YAML loading + stage parsing from groovy + condition matching +└── rules/ + ├── base.py Rule ABC + PRInputs + RuleResult + └── waives_rule.py v0's only rule +``` + +## How it's invoked (CI) + +`L0_MergeRequest.groovy::getCbtsResult` orchestrates two calls to `main.py`: + +1. `python3 main.py --list-needed-diffs` — returns `needs_diff_for` patterns + so Groovy knows which changed files to fetch diffs for (via the existing + `getMergeRequestOneFileChanges` API helper). +2. `python3 main.py cbts_input.json` — returns the decision on stdout. + +The decision is cached in `testFilter[CBTS_RESULT]` and serialized into the +child job's `testFilter` param alongside the existing filter flags. + +## Debugging a CBTS decision locally + +The input JSON that Groovy sends to Python is uploaded as a CI artifact +(`cbts_input.json` in the pipeline workspace). To reproduce a decision +locally: + +```bash +# From the repo root: +python3 jenkins/scripts/cbts/main.py cbts_input.json +``` + +Or hand-craft a minimal input: + +```bash +cat > /tmp/cbts_input.json < Optional[RuleResult]: + # Return None if the rule doesn't apply to this PR. + # Return a RuleResult otherwise. + ... + return RuleResult( + handled_files={...}, # files you claim + tests={...}, # Layer 3 test filter + affected_stages={...}, # Layer 2 stage set + scope="myscope", # your scope label + reason="why this was picked", + ) + ``` + +2. **Register in `main.py`**: + - Add the class to `RULE_CLASSES` (used by `--list-needed-diffs`) + - Add an instance to `build_rules()` with its dependencies + +3. **Decide Layer 1 / 2 / 3 behavior for your scope in Groovy**. Each layer's + consumer checks `cbts.scope == "waiveonly"` explicitly. For a new scope: + - Add an `else if (cbts.scope == "myscope")` branch in `L0_Test.groovy` + Layer 2 override (or leave unspecified → behaves as fallback / full run) + - Similarly decide Layer 1 (arch track skip) and Layer 3 (test filter) + - **Defaults are conservative**: without explicit branches, new scope + paths fall through to the existing filter chain, which is safe + +4. **Keep unit-style checks via CLI smoke tests**. See + `--list-needed-diffs` should include your new pattern; a targeted + `INPUT_JSON` with a change under your rule's scope should produce the + expected output. + +Rule ordering doesn't matter. Rules independently decide whether they apply; +`Selector` combines their `affected_stages` / `tests` via union and their +scopes via `_combine_scopes` (agreement → that scope; disagreement → `None`). + +## Fallback / safety paths + +CBTS falls back to the existing filter chain (as if it weren't there) when: + +- PostMerge job / `alternativeTRT` set +- `changed_files` is empty +- `main.py` throws / stdout is unparsable +- `scope == none` (Python's explicit "no decision" output) +- Groovy sees an unknown scope value (forward compatibility) + +No silent failures: every fallback logs an `echo` line in the CI console. + +## Keep-in-sync notes + +`blocks.py::derive_mako_from_stage` mirrors the Groovy +`getMakoArgsFromStageName` (in `jenkins/L0_Test.groovy` ~line 2079) and +`parseTaskConfigFromStageName` (~line 2066). When new backends / +orchestrators / stage-name conventions are added on the Groovy side, update +the Python constants here too. The file comments flag this explicitly. diff --git a/jenkins/scripts/cbts/__init__.py b/jenkins/scripts/cbts/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/jenkins/scripts/cbts/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py new file mode 100644 index 000000000000..9f580c00b7fa --- /dev/null +++ b/jenkins/scripts/cbts/blocks.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""YAML test-db loading and block/stage matching for CBTS. + +- `Stage` carries stage metadata (yaml_stem, cpu_arch, mako) as provided by Groovy. +- `Block` is a condition-block within a test-db YAML. +- `YAMLIndex` loads all test-db YAMLs and provides a test_id -> blocks lookup. +- `block_matches_stage` implements the trt-test-db matching semantics: + ranges / wildcards / terms. Generic over field names — no hardcoded keys. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from fnmatch import fnmatch +from pathlib import Path +from typing import Iterable, Optional + +import yaml + + +@dataclass +class Stage: + name: str + yaml_stem: str + cpu_arch: str + split_id: int + total_splits: int + mako: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Block: + yaml_stem: str + block_index: int + condition: dict + tests: list[str] + + +class YAMLIndex: + """Index of all blocks across test-db YAMLs, with reverse lookup by test id.""" + + def __init__(self) -> None: + self.blocks: list[Block] = [] + self._test_to_blocks: dict[str, list[Block]] = {} + + @classmethod + def load(cls, test_db_dir: Path) -> "YAMLIndex": + idx = cls() + for yml_path in sorted(test_db_dir.glob("l0_*.yml")): + idx._load_one(yml_path) + return idx + + def _load_one(self, yml_path: Path) -> None: + yaml_stem = yml_path.stem + data = yaml.safe_load(yml_path.read_text()) or {} + blocks_data = data.get(yaml_stem, []) or [] + for i, block_data in enumerate(blocks_data): + if not isinstance(block_data, dict): + continue + tests = block_data.get("tests") or [] + block = Block( + yaml_stem=yaml_stem, + block_index=i, + condition=block_data.get("condition") or {}, + tests=list(tests), + ) + self.blocks.append(block) + for test in tests: + # Tests are raw YAML strings; they may carry trailing options + # like ` -m "gpu2"` or ` TIMEOUT (90)`. Use the full string as + # the match key so downstream test_id extraction can match + # either the bare node_id or the options-suffixed form. + self._test_to_blocks.setdefault(test, []).append(block) + + def blocks_containing_test(self, test_id: str) -> list[Block]: + return list(self._test_to_blocks.get(test_id, [])) + + def all_test_ids(self) -> Iterable[str]: + return self._test_to_blocks.keys() + + +def _range_in(val_str: str | None, gte, lte) -> bool: + if val_str is None: + return False + try: + val = int(val_str) + except (ValueError, TypeError): + return False + if gte is not None and val < gte: + return False + if lte is not None and val > lte: + return False + return True + + +# --------------------------------------------------------------------------- +# Stage config parsing from jenkins/L0_Test.groovy +# --------------------------------------------------------------------------- + +# Matches a single entry like: +# "A10-PyTorch-1": ["a10", "l0_a10", 1, 2], +# "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], +# Same shape as scripts/test_to_stage_mapping.py::_STAGE_RE, extended to +# capture split_id / total_splits / gpu_count. +_STAGE_ENTRY_RE = re.compile( + r'"(?P[^"]+)"\s*:\s*\[' + r'\s*"(?P[^"]+)"\s*,' + r'\s*"(?P[^"]+)"' + r"(?:\s*,\s*(?P\d+))?" + r"(?:\s*,\s*(?P\d+))?" + r"(?:\s*,\s*(?P\d+))?" + r"\s*\]" +) + +# Detects assignments opening a map literal, e.g. `x86TestConfigs = [`. +# Used to track which cpu_arch bucket the stage entries below belong to. +_MAP_OPEN_RE = re.compile(r"\b(?P[A-Za-z_][A-Za-z0-9_]*)\s*=\s*\[\s*$") + + +def _classify_map_var(var_name: str) -> Optional[str]: + """Map a Groovy variable name to cpu_arch bucket, or None if unknown.""" + v = var_name + if "SBSA" in v or "aarch64" in v: + return "sbsa" + if "x86" in v or "X86" in v: + return "x86" + return None + + +# Backend name -> mako value. Same patterns as getMakoArgsFromStageName in +# jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. +_BACKEND_PATTERNS = [ + ("-PyTorch-", "pytorch"), + ("-TensorRT-", "tensorrt"), + ("-CPP-", "cpp"), + ("-Triton-", "triton"), + ("-FMHA-", "fmha"), + ("-AutoDeploy-", "autodeploy"), + ("-Verl-", "verl"), +] + +# Regex mirror of parseTaskConfigFromStageName (jenkins/L0_Test.groovy:2066): +# ([^-]+)(?:-(\d+)_GPUs)? +_GPU_NAME_RE = re.compile(r"^([^-]+)") +_GPU_COUNT_RE = re.compile(r"-(\d+)_GPUs") + + +def derive_mako_from_stage(stage_name: str) -> dict[str, str]: + """Derive a stage's mako dict purely from its name. + + KEEP IN SYNC with the Groovy source of truth: + - jenkins/L0_Test.groovy::getMakoArgsFromStageName (~line 2079) + - jenkins/L0_Test.groovy::parseTaskConfigFromStageName (~line 2066) + + Fields produced (all values are strings, matching the Groovy behavior): + stage, backend (optional), auto_trigger, orchestrator, gpu, system_gpu_count + + Runtime sysinfo keys like `linux_distribution_name` are NOT included here; + they're only available once a stage executes. `block_matches_stage` + treats missing keys as "assume match" so we over-include safely. + """ + mako: dict[str, str] = {} + mako["stage"] = "post_merge" if "Post-Merge" in stage_name else "pre_merge" + + for pat, val in _BACKEND_PATTERNS: + if pat in stage_name: + mako["backend"] = val + break + # else: no 'backend' key set -> matches any block backend term + + if "-DeepSeek-" in stage_name: + mako["auto_trigger"] = "deepseek" + elif "-GptOss-" in stage_name: + mako["auto_trigger"] = "gpt_oss" + else: + mako["auto_trigger"] = "others" + + mako["orchestrator"] = "ray" if "-Ray-" in stage_name else "mpi" + + gpu_match = _GPU_NAME_RE.match(stage_name) + if gpu_match: + # Lowercased so YAML wildcards like `*a10*` / `*h100*` match. + mako["gpu"] = gpu_match.group(1).lower() + count_match = _GPU_COUNT_RE.search(stage_name) + mako["system_gpu_count"] = count_match.group(1) if count_match else "1" + + return mako + + +def parse_stages_from_groovy( + groovy_path: Path, include_post_merge: bool = False +) -> dict[str, Stage]: + """Parse `jenkins/L0_Test.groovy` and return {stage_name -> Stage}. + + - cpu_arch is determined by which map literal (x86TestConfigs, + SBSATestConfigs, ...) the entry lives in; falls back to stage-name + heuristic if the map var is unfamiliar. + - mako is derived via `derive_mako_from_stage` (pure stage-name logic). + - Post-Merge stages are excluded by default (CBTS runs in pre-merge CI). + """ + stages: dict[str, Stage] = {} + current_arch: Optional[str] = None + + for line in groovy_path.read_text().splitlines(): + open_match = _MAP_OPEN_RE.search(line.rstrip()) + if open_match: + detected = _classify_map_var(open_match.group("var")) + if detected is not None: + current_arch = detected + + m = _STAGE_ENTRY_RE.search(line) + if not m: + continue + stage_name = m.group("stage") + if not include_post_merge and "Post-Merge" in stage_name: + continue + + # Fallback heuristic if we haven't seen an obvious map-var yet. + arch = current_arch + if arch is None: + arch = "sbsa" if ("GH200" in stage_name or "SBSA" in stage_name) else "x86" + + stages[stage_name] = Stage( + name=stage_name, + yaml_stem=m.group("yml"), + cpu_arch=arch, + split_id=int(m.group("split_id") or 1), + total_splits=int(m.group("total_splits") or 1), + mako=derive_mako_from_stage(stage_name), + ) + return stages + + +# --------------------------------------------------------------------------- +# Block <-> Stage condition matching +# --------------------------------------------------------------------------- + + +def block_matches_stage(block: Block, stage: Stage) -> bool: + """Return True iff the stage's mako satisfies the block's condition. + + Matching semantics (mirroring trt-test-db) — generic over field names: + - terms[K]: stage.mako[K] must equal block.terms[K]. + - ranges[K]: int(stage.mako[K]) must lie in [gte, lte] (either bound optional). + - wildcards[K]: stage.mako[K] must fnmatch at least one of the patterns. + + When a key referenced by the block is NOT present in stage.mako, we treat + it as "unknown -> assume match". The mako we receive is derived from the + stage name (stage/backend/auto_trigger/orchestrator/gpu/system_gpu_count) + and doesn't include runtime sysinfo fields like `linux_distribution_name` + that trt-test-db adds at execution time. Over-including in this case is + safe (might launch one extra stage); under-including would mean silently + skipping a test that should have run. + """ + cond = block.condition + if not isinstance(cond, dict): + return False + mako = stage.mako or {} + + terms = cond.get("terms") or {} + for k, v in terms.items(): + if k not in mako: + continue # unknown key -> assume match + if str(mako.get(k)) != str(v): + return False + + ranges = cond.get("ranges") or {} + for k, r in ranges.items(): + if not isinstance(r, dict): + return False + if k not in mako: + continue # unknown key -> assume match + if not _range_in(mako.get(k), r.get("gte"), r.get("lte")): + return False + + wildcards = cond.get("wildcards") or {} + for k, patterns in wildcards.items(): + if k not in mako: + continue # unknown key -> assume match + val = str(mako.get(k)).lower() + if isinstance(patterns, str): + patterns = [patterns] + # Case-insensitive match — trt-test-db accepts uppercase mako values + # (e.g. gpu="A10") against lowercase wildcards (e.g. "*a10*"). + if not any(fnmatch(val, str(p).lower()) for p in patterns): + return False + + return True diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py new file mode 100644 index 000000000000..1b6559123c8f --- /dev/null +++ b/jenkins/scripts/cbts/main.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CBTS entry point — consumed by Jenkins Groovy helper `getCbtsResult`. + +Two invocation modes (see DESIGN.md for full context): + + python3 main.py --list-needed-diffs + Print the union of all rules' `needs_diff_for` patterns, one per line. + Groovy uses this to decide which changed files to fetch diffs for. + + python3 main.py INPUT_JSON + Run decision logic. Groovy passes a JSON file containing only PR data: + - changed_files: list[str] + - diffs: {path: diff_content} + Python self-sources everything else from the repo: + - stage configs: parsed from jenkins/L0_Test.groovy + - test-db YAMLs: loaded from tests/integration/test_lists/test-db/ + Output is a text blob on stdout, with a `# SCOPE:` header and, when a + rule is active, `# AFFECTED_CPU_ARCH`, `# AFFECTED_STAGES`, and one + test id per line. Parsed by `parseSelectionResult` on the Groovy side. + +Invocation assumes the current working directory is the TRT-LLM repo root, +or that --repo-root is passed explicitly. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# Make sibling modules importable when invoked as `python3 /main.py ...`. +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from blocks import Stage, YAMLIndex, parse_stages_from_groovy # noqa: E402 +from rules.base import PRInputs, Rule, RuleResult # noqa: E402 +from rules.waives_rule import WaivesRule # noqa: E402 + +# --- Rule registry ----------------------------------------------------------- + +# Classes are used for `--list-needed-diffs` (no need to construct). +RULE_CLASSES: list[type[Rule]] = [WaivesRule] + + +def build_rules(yaml_index: YAMLIndex, stages: dict[str, Stage]) -> list[Rule]: + """Instantiate rules with their dependencies. + + Add a new rule: append a new line here and a new class to RULE_CLASSES. + """ + return [WaivesRule(yaml_index, stages)] + + +# --- Selector --------------------------------------------------------------- + + +@dataclass +class SelectionResult: + """Final aggregated decision.""" + + scope: Optional[str] + affected_stages: set[str] = field(default_factory=set) + affected_cpu_arch: set[str] = field(default_factory=set) + tests: set[str] = field(default_factory=set) + reasons: list[str] = field(default_factory=list) + + def to_text(self) -> str: + if self.scope is None: + reason = "; ".join(self.reasons) if self.reasons else "no decision" + return f"# SCOPE: none\n# REASON: {reason}\n" + + lines = [f"# SCOPE: {self.scope}"] + for r in self.reasons: + lines.append(f"# REASON: {r}") + lines.append(f"# AFFECTED_CPU_ARCH: {', '.join(sorted(self.affected_cpu_arch))}") + lines.append(f"# AFFECTED_STAGES: {', '.join(sorted(self.affected_stages))}") + lines.extend(sorted(self.tests)) + return "\n".join(lines) + "\n" + + +def _combine_scopes(scopes: list[str]) -> Optional[str]: + """Combine scope labels from multiple rules. + + v0: single rule => passthrough. + Multi-rule future: when all scopes agree, use that scope; otherwise return + None (no-decision / full run) until an explicit priority table is added. + """ + if not scopes: + return None + if len(set(scopes)) == 1: + return scopes[0] + return None + + +class Selector: + def run( + self, + pr: PRInputs, + rules: list[Rule], + stages: dict[str, Stage], + ) -> SelectionResult: + # 1. Run all rules, keep (rule, result) pairs that apply. + pairs: list[tuple[Rule, RuleResult]] = [] + for rule in rules: + result = rule.apply(pr) + if result is not None: + pairs.append((rule, result)) + + # 2. Coverage check: any changed file not handled by any rule -> no decision. + handled: set[str] = set() + for _, r in pairs: + handled |= r.handled_files + unhandled = sorted(set(pr.changed_files) - handled) + if unhandled: + preview = unhandled[:5] + more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" + return SelectionResult( + scope=None, + reasons=[f"Unhandled files: {preview}{more}"], + ) + + if not pairs: + # No changed_files and no rule applied -> no decision. + return SelectionResult( + scope=None, + reasons=["No rule contributed"], + ) + + # 3. Combine scopes across rules. + scope = _combine_scopes([r.scope for _, r in pairs]) + if scope is None: + return SelectionResult( + scope=None, + reasons=[f"[{rule.name}] {r.reason}" for rule, r in pairs] + + ["Scopes cannot be combined"], + ) + + # 4. Union stages and tests across rules; derive affected_cpu_arch from stages. + affected_stages: set[str] = set() + tests: set[str] = set() + for _, r in pairs: + affected_stages |= r.affected_stages + tests |= r.tests + + affected_cpu_arch: set[str] = set() + for name in affected_stages: + stage = stages.get(name) + if stage is not None: + affected_cpu_arch.add(stage.cpu_arch) + + reasons = [f"[{rule.name}] {r.reason}" for rule, r in pairs] + return SelectionResult( + scope=scope, + affected_stages=affected_stages, + affected_cpu_arch=affected_cpu_arch, + tests=tests, + reasons=reasons, + ) + + +# --- Input loading ---------------------------------------------------------- + + +def _load_pr_inputs(input_json_path: Path) -> PRInputs: + data = json.loads(input_json_path.read_text()) + return PRInputs( + changed_files=list(data.get("changed_files", [])), + diffs=dict(data.get("diffs", {})), + ) + + +# --- CLI -------------------------------------------------------------------- + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="CBTS — Change-Based Testing Selection for TRT-LLM CI", + ) + parser.add_argument( + "input_json", + nargs="?", + help="Path to the INPUT_JSON file prepared by Groovy `getCbtsResult`.", + ) + parser.add_argument( + "--list-needed-diffs", + action="store_true", + help="Print the union of all rules' needs_diff_for patterns and exit.", + ) + parser.add_argument( + "--repo-root", + default=".", + help="Path to the TRT-LLM repo root (default: current working directory).", + ) + parser.add_argument( + "--test-db", + default=None, + help="Override path to the test-db YAML directory " + "(default: /tests/integration/test_lists/test-db).", + ) + parser.add_argument( + "--groovy-file", + default=None, + help="Override path to the Jenkins test Groovy file " + "(default: /jenkins/L0_Test.groovy).", + ) + args = parser.parse_args(argv) + + if args.list_needed_diffs: + patterns: set[str] = set() + for cls in RULE_CLASSES: + patterns.update(cls.needs_diff_for) + for p in sorted(patterns): + print(p) + return 0 + + if not args.input_json: + print( + "error: INPUT_JSON is required (or pass --list-needed-diffs)", + file=sys.stderr, + ) + return 2 + + input_path = Path(args.input_json) + if not input_path.is_file(): + print(f"error: INPUT_JSON not found: {input_path}", file=sys.stderr) + return 2 + + repo_root = Path(args.repo_root).resolve() + test_db_dir = ( + Path(args.test_db) if args.test_db else repo_root / "tests/integration/test_lists/test-db" + ) + groovy_path = ( + Path(args.groovy_file) if args.groovy_file else repo_root / "jenkins/L0_Test.groovy" + ) + + if not test_db_dir.is_dir(): + print(f"error: test-db directory not found: {test_db_dir}", file=sys.stderr) + return 2 + if not groovy_path.is_file(): + print(f"error: Jenkins groovy file not found: {groovy_path}", file=sys.stderr) + return 2 + + yaml_index = YAMLIndex.load(test_db_dir) + stages = parse_stages_from_groovy(groovy_path) + pr = _load_pr_inputs(input_path) + rules = build_rules(yaml_index, stages) + result = Selector().run(pr, rules, stages) + sys.stdout.write(result.to_text()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jenkins/scripts/cbts/rules/__init__.py b/jenkins/scripts/cbts/rules/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/jenkins/scripts/cbts/rules/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py new file mode 100644 index 000000000000..58c1b912febe --- /dev/null +++ b/jenkins/scripts/cbts/rules/base.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Rule contract and shared data types for CBTS.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class PRInputs: + """Inputs about the PR that rules can query.""" + + changed_files: list[str] + diffs: dict[str, str] = field(default_factory=dict) + + +@dataclass +class RuleResult: + """What a single rule contributes when it applies to a PR.""" + + handled_files: set[str] + tests: set[str] + affected_stages: set[str] + scope: str + reason: str + + +class Rule(ABC): + """Base class for all CBTS rules. + + A rule declares: + - `name`: identifier used in logs/reasons + - `needs_diff_for`: file paths / glob patterns whose diffs this rule consumes + (Groovy uses this to decide which files to fetch diffs for) + + Subclasses implement `apply(pr)` returning either None (not applicable) or + a RuleResult. + """ + + name: str = "" + needs_diff_for: list[str] = [] + + @abstractmethod + def apply(self, pr: PRInputs) -> Optional[RuleResult]: ... diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py new file mode 100644 index 000000000000..04a8f3bacb11 --- /dev/null +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""WaivesRule — v0 rule for changes to waives.txt.""" + +from __future__ import annotations + +import re +from typing import Optional + +from blocks import Stage, YAMLIndex, block_matches_stage + +from .base import PRInputs, Rule, RuleResult + +WAIVES_FILE = "tests/integration/test_lists/waives.txt" + +# Strip GPU/platform prefixes like "full:GH200/" or "full:sm100/" at the start. +_PREFIX_RE = re.compile(r"^full:[^/]+/") + +# Split trailing annotations (SKIP / TIMEOUT / comments) from the test id. +# A waive line typically looks like: +# SKIP (reason) +# SKIP # url +# # just a comment +# -k "expr" SKIP (reason) +# We keep the whole " [-m/-k ...]" portion as the identifier, since +# YAML entries can include the same -m/-k suffixes. +_SUFFIX_RE = re.compile(r"\s+(SKIP|TIMEOUT)\b.*$") + + +def _extract_test_id(line: str) -> Optional[str]: + """Extract the test identifier from a waives.txt line. + + Returns None if the line doesn't look like a waive entry (empty, pure + comment, etc). + """ + s = line.strip() + if not s or s.startswith("#"): + return None + # Drop the "SKIP ..." / "TIMEOUT ..." trailing annotation if present. + s = _SUFFIX_RE.sub("", s).strip() + # Drop trailing " # comment" if any. + if "#" in s: + s = s.split("#", 1)[0].strip() + if not s: + return None + return s + + +def _strip_prefix(test_id: str) -> str: + """Strip leading "full:/" prefix if any.""" + return _PREFIX_RE.sub("", test_id) + + +def parse_waives_diff(diff: str) -> tuple[set[str], set[str]]: + """Parse a unified diff of waives.txt. + + Returns (added, removed) sets of test identifiers, with "full:..." prefixes + stripped so they can match YAML entries directly. + """ + added: set[str] = set() + removed: set[str] = set() + for line in diff.splitlines(): + if not line or line.startswith(("+++", "---", "@@")): + continue + sign, body = line[0], line[1:] + if sign not in ("+", "-"): + continue + tid = _extract_test_id(body) + if tid is None: + continue + tid = _strip_prefix(tid) + (added if sign == "+" else removed).add(tid) + return added, removed + + +class WaivesRule(Rule): + name = "waives" + needs_diff_for = [WAIVES_FILE] + + def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]) -> None: + self.yaml_index = yaml_index + self.stages = stages + + def apply(self, pr: PRInputs) -> Optional[RuleResult]: + if WAIVES_FILE not in pr.changed_files: + return None + + diff = pr.diffs.get(WAIVES_FILE, "") + added, removed = parse_waives_diff(diff) + changed_test_ids = added | removed + if not changed_test_ids: + # PR touched waives.txt but diff has no parseable test ids (e.g. + # a pure comment edit). Still claim handling — no stages needed. + return RuleResult( + handled_files={WAIVES_FILE}, + tests=set(), + affected_stages=set(), + scope="waiveonly", + reason="waives.txt: no actionable test ids in diff", + ) + + # Reverse-lookup: test id -> containing blocks, deduped. + seen_block_keys: set[tuple[str, int]] = set() + affected_blocks = [] + for tid in changed_test_ids: + for block in self.yaml_index.blocks_containing_test(tid): + key = (block.yaml_stem, block.block_index) + if key not in seen_block_keys: + seen_block_keys.add(key) + affected_blocks.append(block) + + # For each block, find stages whose mako matches its condition. + affected_stage_names: set[str] = set() + for block in affected_blocks: + for stage_name, stage in self.stages.items(): + if stage.yaml_stem != block.yaml_stem: + continue + if block_matches_stage(block, stage): + affected_stage_names.add(stage_name) + + return RuleResult( + handled_files={WAIVES_FILE}, + tests=changed_test_ids, + affected_stages=affected_stage_names, + scope="waiveonly", + reason=( + f"waives.txt: +{len(added)} / -{len(removed)} → " + f"{len(affected_blocks)} blocks, {len(affected_stage_names)} stages" + ), + ) From f010300ca021325ef9dc5cb4c6506ba83085470d Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 20:46:09 +0800 Subject: [PATCH 02/65] simplify Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 70 ++++++--------------- jenkins/L0_Test.groovy | 12 ++-- jenkins/scripts/cbts/DESIGN.md | 40 +++++++----- jenkins/scripts/cbts/README.md | 16 ++--- jenkins/scripts/cbts/blocks.py | 6 +- jenkins/scripts/cbts/main.py | 76 ++++++++--------------- jenkins/scripts/cbts/rules/base.py | 4 +- jenkins/scripts/cbts/rules/waives_rule.py | 14 ++--- 8 files changed, 92 insertions(+), 146 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index b3568e2e8d12..7d6816727d8f 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -764,65 +764,31 @@ def getCbtsResult(pipeline, testFilter, globalVars) } return result } catch (Exception e) { - pipeline.echo("CBTS failed, falling back to full run: ${e.getMessage()}") + pipeline.echo("CBTS failed, falling back to full run: ${e}") return null } } -// Simple glob matcher: supports `**` (any chars) and `*` (non-slash). -// v0 needs_diff_for is a plain path, but future rules may use globs like -// "tests/integration/defs/**/*.py", so we implement minimal glob support. +// v0 needs_diff_for entries are exact file paths. When a future rule adds +// a real glob (e.g. "tests/integration/defs/**/*.py"), extend this to use +// Ant-style matching (hudson.util.AntPathMatcher). def _cbtsMatchesAnyPattern(String filePath, List patterns) { - for (p in patterns) { - if (filePath == p) { return true } - if (!p.contains('*')) { continue } - // Escape regex meta-chars that might appear in paths, then substitute - // `**` and `*` via a sentinel so they don't collide. - def regex = p.replace('.', '\\.') - .replace('+', '\\+') - .replace('(', '\\(') - .replace(')', '\\)') - .replace('**', 'DBLSTAR_SENTINEL') - .replace('*', '[^/]*') - .replace('DBLSTAR_SENTINEL', '.*') - if (filePath ==~ regex) { return true } - } - return false + return patterns.contains(filePath) } -// Parse CBTS stdout (format: see DESIGN.md 4.6) into a nullable map. -// Returns null when scope=none (no decision => full run). +// Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3, or null +// when the Python side explicitly returned scope=null (no decision). def _cbtsParseSelectionResult(String text) { - def scope = null - def archs = [] - def stages = [] - def tests = [] - def reasons = [] - for (line in text.readLines()) { - if (line.startsWith("# SCOPE:")) { - def v = line.replaceFirst(/^# SCOPE:\s*/, '').trim() - scope = (v == 'none' || v == '') ? null : v - } else if (line.startsWith("# REASON:")) { - reasons.add(line.replaceFirst(/^# REASON:\s*/, '').trim()) - } else if (line.startsWith("# AFFECTED_CPU_ARCH:")) { - def v = line.replaceFirst(/^# AFFECTED_CPU_ARCH:\s*/, '').trim() - archs = v ? v.tokenize(',').collect { it.trim() }.findAll { it } : [] - } else if (line.startsWith("# AFFECTED_STAGES:")) { - def v = line.replaceFirst(/^# AFFECTED_STAGES:\s*/, '').trim() - stages = v ? v.tokenize(',').collect { it.trim() }.findAll { it } : [] - } else if (!line.startsWith("#") && line.trim()) { - tests.add(line.trim()) - } - } - if (scope == null) { return null } + def data = new groovy.json.JsonSlurper().parseText(text) + if (data.scope == null) { return null } return [ - scope: scope, - affected_cpu_arch: archs, - affected_stages: stages, - affected_tests: tests, - reasons: reasons, + scope: data.scope, + affected_cpu_arch: data.affected_cpu_arch ?: [], + affected_stages: data.affected_stages ?: [], + affected_tests: data.tests ?: [], + reasons: data.reasons ?: [], ] } @@ -1214,8 +1180,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) "x86_64-Linux": { script { // CBTS Layer 1: skip entire x86 track when no x86 stages are affected - def _cbts = testFilter[(CBTS_RESULT)] - if (_cbts?.scope == "waiveonly" && !("x86" in _cbts.affected_cpu_arch)) { + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && !("x86" in cbts.affected_cpu_arch)) { echo "CBTS waiveonly: no x86 stages affected, skipping x86_64-Linux track" return } @@ -1331,8 +1297,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) return } // CBTS Layer 1: skip entire SBSA track when no sbsa stages are affected - def _cbts = testFilter[(CBTS_RESULT)] - if (_cbts?.scope == "waiveonly" && !("sbsa" in _cbts.affected_cpu_arch)) { + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && !("sbsa" in cbts.affected_cpu_arch)) { echo "CBTS waiveonly: no sbsa stages affected, skipping SBSA-Linux track" return } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 148742ce34ee..9b9096389ae0 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3126,9 +3126,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // CBTS Layer 3: within an affected stage, further restrict testDBList // to the tests CBTS identified as affected. - def _cbts = testFilter[(CBTS_RESULT)] - if (_cbts?.scope == "waiveonly" && _cbts.affected_tests) { - testDBList = filterTestDBListForCbts(testDBList, _cbts.affected_tests, stageName) + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && cbts.affected_tests) { + testDBList = filterTestDBListForCbts(testDBList, cbts.affected_tests, stageName) } // Process shard test list and create separate files for regular and isolate tests @@ -4445,9 +4445,9 @@ def launchTestJobs(pipeline, testFilter) // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all // existing filter rules so that unknown / no-decision paths fall through // naturally. See jenkins/scripts/cbts/DESIGN.md for the three-layer model. - def _cbts = testFilter[(CBTS_RESULT)] - if (_cbts?.scope == "waiveonly" && _cbts.affected_stages) { - def affectedSet = _cbts.affected_stages as Set + def cbts = testFilter[(CBTS_RESULT)] + if (cbts?.scope == "waiveonly" && cbts.affected_stages) { + def affectedSet = cbts.affected_stages as Set parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) } echo "CBTS waiveonly: limiting to ${parallelJobsFiltered.size()} affected stages" } diff --git a/jenkins/scripts/cbts/DESIGN.md b/jenkins/scripts/cbts/DESIGN.md index f64275268910..6696d996d0ab 100644 --- a/jenkins/scripts/cbts/DESIGN.md +++ b/jenkins/scripts/cbts/DESIGN.md @@ -247,30 +247,36 @@ Groovy `getCbtsResult` calls Python twice: Stage configs are parsed by Python directly from `jenkins/L0_Test.groovy` (see 4.5). -**stdout when a decision is made**: +**stdout is a JSON blob**: -``` -# SCOPE: waiveonly -# REASON: [waives] waives.txt: +2 / -1 → 2 blocks, 2 stages -# AFFECTED_CPU_ARCH: x86 -# AFFECTED_STAGES: DGX_H100-4_GPUs-PyTorch-DeepSeek-1, DGX_H100-4_GPUs-PyTorch-DeepSeek-2 -examples/test_deepseek.py::test_xxx +```json +{ + "scope": "waiveonly", + "affected_cpu_arch": ["x86"], + "affected_stages": [ + "DGX_H100-4_GPUs-PyTorch-DeepSeek-1", + "DGX_H100-4_GPUs-PyTorch-DeepSeek-2" + ], + "tests": ["examples/test_deepseek.py::test_xxx"], + "reasons": ["[waives] waives.txt: +2 / -1 → 2 blocks, 2 stages"] +} ``` -**stdout when no decision**: +When there is no decision, `scope` is `null` (Groovy parses this as "fall back"): -``` -# SCOPE: none -# REASON: Unhandled files: [tensorrt_llm/llmapi/llm.py, ...] +```json +{ + "scope": null, + "affected_cpu_arch": [], + "affected_stages": [], + "tests": [], + "reasons": ["Unhandled files: [tensorrt_llm/llmapi/llm.py, ...]"] +} ``` -Groovy parsing contract: -- `# SCOPE:` — scope label (`waiveonly` / `none` / future values); `none` → `testFilter[CBTS_RESULT].scope = null`. -- `# AFFECTED_CPU_ARCH:` — comma-separated, values `x86` / `sbsa`. -- `# AFFECTED_STAGES:` — comma-separated stage names. -- Non-`#` lines: one test id per line. +Groovy consumes it via `JsonSlurper` in `_cbtsParseSelectionResult`; `scope == null` maps to `testFilter[CBTS_RESULT] = null`. -Exit code 0 = decision succeeded (including `none`); non-zero → Groovy falls back to full run. +Exit code 0 = decision succeeded (including the null-scope case); non-zero → Groovy falls back to full run. ### 4.7 Multi-rule combination (future) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 40054f7798bb..65856cc13181 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -79,14 +79,16 @@ EOF python3 jenkins/scripts/cbts/main.py /tmp/cbts_input.json ``` -Output format (see DESIGN.md §4.6): +Output is a JSON blob on stdout (see DESIGN.md §4.6): -``` -# SCOPE: waiveonly -# REASON: [waives] waives.txt: +1 / -0 → 1 blocks, 2 stages -# AFFECTED_CPU_ARCH: x86 -# AFFECTED_STAGES: A10-PyTorch-1, A10-PyTorch-2 -unittest/utils/test_util.py +```json +{ + "scope": "waiveonly", + "affected_cpu_arch": ["x86"], + "affected_stages": ["A10-PyTorch-1", "A10-PyTorch-2"], + "tests": ["unittest/utils/test_util.py"], + "reasons": ["[waives] waives.txt: +1 / -0 → 1 blocks, 2 stages"] +} ``` ## Adding a new rule diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 9f580c00b7fa..d34cd118bdd7 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -121,10 +121,10 @@ def _range_in(val_str: str | None, gte, lte) -> bool: def _classify_map_var(var_name: str) -> Optional[str]: """Map a Groovy variable name to cpu_arch bucket, or None if unknown.""" - v = var_name - if "SBSA" in v or "aarch64" in v: + v = var_name.lower() + if "sbsa" in v or "aarch64" in v: return "sbsa" - if "x86" in v or "X86" in v: + if "x86" in v: return "x86" return None diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 1b6559123c8f..6cdb886dea4a 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -16,9 +16,9 @@ Python self-sources everything else from the repo: - stage configs: parsed from jenkins/L0_Test.groovy - test-db YAMLs: loaded from tests/integration/test_lists/test-db/ - Output is a text blob on stdout, with a `# SCOPE:` header and, when a - rule is active, `# AFFECTED_CPU_ARCH`, `# AFFECTED_STAGES`, and one - test id per line. Parsed by `parseSelectionResult` on the Groovy side. + Output is a JSON blob on stdout with fields `scope`, `affected_cpu_arch`, + `affected_stages`, `tests`, `reasons`. Consumed by `_cbtsParseSelectionResult` + on the Groovy side. Invocation assumes the current working directory is the TRT-LLM repo root, or that --repo-root is passed explicitly. @@ -47,10 +47,6 @@ def build_rules(yaml_index: YAMLIndex, stages: dict[str, Stage]) -> list[Rule]: - """Instantiate rules with their dependencies. - - Add a new rule: append a new line here and a new class to RULE_CLASSES. - """ return [WaivesRule(yaml_index, stages)] @@ -67,18 +63,15 @@ class SelectionResult: tests: set[str] = field(default_factory=set) reasons: list[str] = field(default_factory=list) - def to_text(self) -> str: - if self.scope is None: - reason = "; ".join(self.reasons) if self.reasons else "no decision" - return f"# SCOPE: none\n# REASON: {reason}\n" - - lines = [f"# SCOPE: {self.scope}"] - for r in self.reasons: - lines.append(f"# REASON: {r}") - lines.append(f"# AFFECTED_CPU_ARCH: {', '.join(sorted(self.affected_cpu_arch))}") - lines.append(f"# AFFECTED_STAGES: {', '.join(sorted(self.affected_stages))}") - lines.extend(sorted(self.tests)) - return "\n".join(lines) + "\n" + def to_json(self) -> str: + data = { + "scope": self.scope, + "affected_cpu_arch": sorted(self.affected_cpu_arch), + "affected_stages": sorted(self.affected_stages), + "tests": sorted(self.tests), + "reasons": list(self.reasons), + } + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" def _combine_scopes(scopes: list[str]) -> Optional[str]: @@ -96,20 +89,16 @@ def _combine_scopes(scopes: list[str]) -> Optional[str]: class Selector: - def run( - self, - pr: PRInputs, - rules: list[Rule], - stages: dict[str, Stage], - ) -> SelectionResult: - # 1. Run all rules, keep (rule, result) pairs that apply. + def __init__(self, stages: dict[str, Stage]) -> None: + self.stages = stages + + def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: pairs: list[tuple[Rule, RuleResult]] = [] for rule in rules: result = rule.apply(pr) if result is not None: pairs.append((rule, result)) - # 2. Coverage check: any changed file not handled by any rule -> no decision. handled: set[str] = set() for _, r in pairs: handled |= r.handled_files @@ -117,41 +106,26 @@ def run( if unhandled: preview = unhandled[:5] more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" - return SelectionResult( - scope=None, - reasons=[f"Unhandled files: {preview}{more}"], - ) + return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) if not pairs: - # No changed_files and no rule applied -> no decision. - return SelectionResult( - scope=None, - reasons=["No rule contributed"], - ) + return SelectionResult(scope=None, reasons=["No rule contributed"]) - # 3. Combine scopes across rules. + reasons = [f"[{rule.name}] {r.reason}" for rule, r in pairs] scope = _combine_scopes([r.scope for _, r in pairs]) if scope is None: - return SelectionResult( - scope=None, - reasons=[f"[{rule.name}] {r.reason}" for rule, r in pairs] - + ["Scopes cannot be combined"], - ) + return SelectionResult(scope=None, reasons=reasons + ["Scopes cannot be combined"]) - # 4. Union stages and tests across rules; derive affected_cpu_arch from stages. affected_stages: set[str] = set() tests: set[str] = set() for _, r in pairs: affected_stages |= r.affected_stages tests |= r.tests - affected_cpu_arch: set[str] = set() - for name in affected_stages: - stage = stages.get(name) - if stage is not None: - affected_cpu_arch.add(stage.cpu_arch) + affected_cpu_arch = { + self.stages[name].cpu_arch for name in affected_stages if name in self.stages + } - reasons = [f"[{rule.name}] {r.reason}" for rule, r in pairs] return SelectionResult( scope=scope, affected_stages=affected_stages, @@ -247,8 +221,8 @@ def main(argv: Optional[list[str]] = None) -> int: stages = parse_stages_from_groovy(groovy_path) pr = _load_pr_inputs(input_path) rules = build_rules(yaml_index, stages) - result = Selector().run(pr, rules, stages) - sys.stdout.write(result.to_text()) + result = Selector(stages).run(pr, rules) + sys.stdout.write(result.to_json()) return 0 diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 58c1b912febe..b65d9cd7fcf7 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -5,7 +5,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional @@ -14,7 +14,7 @@ class PRInputs: """Inputs about the PR that rules can query.""" changed_files: list[str] - diffs: dict[str, str] = field(default_factory=dict) + diffs: dict[str, str] @dataclass diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 04a8f3bacb11..475fe9eddbda 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -79,7 +79,11 @@ class WaivesRule(Rule): def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]) -> None: self.yaml_index = yaml_index - self.stages = stages + # Group stages by YAML stem so block->stage lookup is O(stages_in_yaml) + # instead of O(total_stages) per block. + self._stages_by_yaml: dict[str, list[tuple[str, Stage]]] = {} + for name, stage in stages.items(): + self._stages_by_yaml.setdefault(stage.yaml_stem, []).append((name, stage)) def apply(self, pr: PRInputs) -> Optional[RuleResult]: if WAIVES_FILE not in pr.changed_files: @@ -89,8 +93,6 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: added, removed = parse_waives_diff(diff) changed_test_ids = added | removed if not changed_test_ids: - # PR touched waives.txt but diff has no parseable test ids (e.g. - # a pure comment edit). Still claim handling — no stages needed. return RuleResult( handled_files={WAIVES_FILE}, tests=set(), @@ -99,7 +101,6 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: reason="waives.txt: no actionable test ids in diff", ) - # Reverse-lookup: test id -> containing blocks, deduped. seen_block_keys: set[tuple[str, int]] = set() affected_blocks = [] for tid in changed_test_ids: @@ -109,12 +110,9 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: seen_block_keys.add(key) affected_blocks.append(block) - # For each block, find stages whose mako matches its condition. affected_stage_names: set[str] = set() for block in affected_blocks: - for stage_name, stage in self.stages.items(): - if stage.yaml_stem != block.yaml_stem: - continue + for stage_name, stage in self._stages_by_yaml.get(block.yaml_stem, []): if block_matches_stage(block, stage): affected_stage_names.add(stage_name) From ae2c0aa4b5a27dd3f51ac8cecfab55ed9ffc42a1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:02:05 +0800 Subject: [PATCH 03/65] [None][chore] align CBTS copyright headers with CODING_GUIDELINES Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/__init__.py | 15 +++++++++++++-- jenkins/scripts/cbts/blocks.py | 15 +++++++++++++-- jenkins/scripts/cbts/main.py | 15 +++++++++++++-- jenkins/scripts/cbts/rules/__init__.py | 15 +++++++++++++-- jenkins/scripts/cbts/rules/base.py | 15 +++++++++++++-- jenkins/scripts/cbts/rules/waives_rule.py | 15 +++++++++++++-- 6 files changed, 78 insertions(+), 12 deletions(-) diff --git a/jenkins/scripts/cbts/__init__.py b/jenkins/scripts/cbts/__init__.py index 52a7a9daf028..c03552b02d99 100644 --- a/jenkins/scripts/cbts/__init__.py +++ b/jenkins/scripts/cbts/__init__.py @@ -1,2 +1,13 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index d34cd118bdd7..0ff83be8563c 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -1,5 +1,16 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """YAML test-db loading and block/stage matching for CBTS. - `Stage` carries stage metadata (yaml_stem, cpu_arch, mako) as provided by Groovy. diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 6cdb886dea4a..3e67f662d94b 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -1,6 +1,17 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CBTS entry point — consumed by Jenkins Groovy helper `getCbtsResult`. Two invocation modes (see DESIGN.md for full context): diff --git a/jenkins/scripts/cbts/rules/__init__.py b/jenkins/scripts/cbts/rules/__init__.py index 52a7a9daf028..c03552b02d99 100644 --- a/jenkins/scripts/cbts/rules/__init__.py +++ b/jenkins/scripts/cbts/rules/__init__.py @@ -1,2 +1,13 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index b65d9cd7fcf7..ebdf2246b2dc 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -1,5 +1,16 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Rule contract and shared data types for CBTS.""" from __future__ import annotations diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 475fe9eddbda..0062e73acaa6 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -1,5 +1,16 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """WaivesRule — v0 rule for changes to waives.txt.""" from __future__ import annotations From 084340daa408f838a06e73af72815e0b05a8deb5 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:18:46 +0800 Subject: [PATCH 04/65] [None][chore] drop empty CBTS __init__.py files; use namespace packages Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/DESIGN.md | 5 ++--- jenkins/scripts/cbts/__init__.py | 13 ------------- jenkins/scripts/cbts/rules/__init__.py | 13 ------------- 3 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 jenkins/scripts/cbts/__init__.py delete mode 100644 jenkins/scripts/cbts/rules/__init__.py diff --git a/jenkins/scripts/cbts/DESIGN.md b/jenkins/scripts/cbts/DESIGN.md index 6696d996d0ab..7c4228a54b31 100644 --- a/jenkins/scripts/cbts/DESIGN.md +++ b/jenkins/scripts/cbts/DESIGN.md @@ -33,17 +33,16 @@ No changes to pytest-split, trt-test-db, or the stage-scheduling core; all new c ``` jenkins/scripts/cbts/ -├── __init__.py ├── DESIGN.md +├── README.md ├── main.py ← Selector + SelectionResult + CLI ├── blocks.py ← Stage + Block + YAMLIndex + block_matches_stage └── rules/ - ├── __init__.py ├── base.py ← Rule ABC + PRInputs + RuleResult └── waives_rule.py ← v0 rule ``` -**4 business files + 2 `__init__.py`**. +**4 Python files** + 2 docs. No `__init__.py` files — directories are used as Python 3.3+ namespace packages, matching the `jenkins/scripts/` convention (sibling directories like `jenkins/scripts/perf/` are structured the same way). ### 3.2 Key contracts diff --git a/jenkins/scripts/cbts/__init__.py b/jenkins/scripts/cbts/__init__.py deleted file mode 100644 index c03552b02d99..000000000000 --- a/jenkins/scripts/cbts/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/jenkins/scripts/cbts/rules/__init__.py b/jenkins/scripts/cbts/rules/__init__.py deleted file mode 100644 index c03552b02d99..000000000000 --- a/jenkins/scripts/cbts/rules/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. From 9ede6d3b2c4dc5a58d65653208b0ef12c74164b2 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:46:55 +0800 Subject: [PATCH 05/65] [None][chore] drop CBTS Layer 3 per-test filter; run full block for safety Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 44 +++++++--------------------------- jenkins/scripts/cbts/DESIGN.md | 36 +++++++++++++--------------- jenkins/scripts/cbts/README.md | 22 +++++++++++------ 3 files changed, 40 insertions(+), 62 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9b9096389ae0..c6a537217626 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -571,36 +571,6 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } -// CBTS helper: restrict a rendered testDBList to the intersection with -// CBTS's affected_tests set. Preserves original line format (including -// ISOLATION markers and pytest args), matching by the leading token. -def filterTestDBListForCbts(String testDBList, List affectedTests, String stageName) { - def affectedSet = affectedTests as Set - def originalLines = readFile(file: testDBList).readLines() - def kept = originalLines.findAll { line -> - def trimmed = line.trim() - if (!trimmed || trimmed.startsWith("#")) { return false } - // Bare node-id / path: drop ISOLATION marker and trailing pytest args. - def bare = trimmed - if (bare.contains(" ISOLATION")) { - bare = bare.replaceAll(/\s*ISOLATION.*$/, '').trim() - } - if (bare.contains(" ")) { - bare = bare.split(" ", 2)[0] - } - return affectedSet.contains(bare) || affectedSet.contains(trimmed) - } - def filtered = testDBList.replaceAll(/\.txt$/, '_cbts_filtered.txt') - if (kept.isEmpty()) { - // Avoid `echo` with empty shell-quoted content. - sh "touch ${filtered}" - } else { - writeFile file: filtered, text: kept.join("\n") + "\n" - } - echo "CBTS Layer 3 (${stageName}): kept ${kept.size()}/${originalLines.size()} tests" - return filtered -} - def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false) { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -3124,12 +3094,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt") } - // CBTS Layer 3: within an affected stage, further restrict testDBList - // to the tests CBTS identified as affected. - def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && cbts.affected_tests) { - testDBList = filterTestDBListForCbts(testDBList, cbts.affected_tests, stageName) - } + // NOTE: CBTS intentionally does NOT filter testDBList here. Layer 2 has + // already narrowed stages to those whose mako matches an affected block's + // condition; within each such stage we run the FULL rendered testDBList + // (all blocks matching the stage's mako) rather than restricting to the + // specific changed test ids. This over-includes by design: if a waive is + // wrong (e.g. depends on other tests, or the node id has a typo), running + // only the single changed test would not surface the problem. The extra + // per-stage test time is accepted as the cost of CI robustness. // Process shard test list and create separate files for regular and isolate tests def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode) diff --git a/jenkins/scripts/cbts/DESIGN.md b/jenkins/scripts/cbts/DESIGN.md index 7c4228a54b31..1f757f4c4d92 100644 --- a/jenkins/scripts/cbts/DESIGN.md +++ b/jenkins/scripts/cbts/DESIGN.md @@ -7,12 +7,13 @@ ## 1. Core Idea -PR changes one test in waives.txt → that test lives in some YAML block → the block's `condition` matches only certain stages → **other stages aren't scheduled**; build + test sub-jobs for the arch track (x86 / SBSA) that has no affected stages are **entirely skipped**; inside each scheduled stage, the test list is further filtered by test id. +PR changes one test in waives.txt → that test lives in some YAML block → the block's `condition` matches only certain stages → **other stages aren't scheduled**; build + test sub-jobs for the arch track (x86 / SBSA) that has no affected stages are **entirely skipped**. -**Three-layer filtering**: +**Two-layer filtering**: - **Layer 1 (arch track level)**: affected stages only on x86 → skip the entire SBSA track (SBSA build + all SBSA sub-jobs), and vice versa. - **Layer 2 (stage level)**: within the same arch, match `block.condition` against `stage.mako` to pick which stages to schedule. -- **Layer 3 (test level)**: inside a running stage, intersect `renderTestDB`'s output with CBTS's test set. + +Inside each selected stage, CBTS **intentionally does NOT filter further**. The stage runs its normal rendered testDBList (all blocks whose conditions match the stage's mako) rather than narrowing to the specific changed test id. Rationale: if a waive is wrong (depends on another test, or has a typo), running the whole block catches it; running only the single affected test would silently miss the problem. See 4.4 for the full argument. No changes to pytest-split, trt-test-db, or the stage-scheduling core; all new code is glue. @@ -56,7 +57,7 @@ class PRInputs: @dataclass class RuleResult: handled_files: set[str] - tests: set[str] # Layer 3: within-stage filter + tests: set[str] # changed test ids (used internally to find blocks; also logged) affected_stages: set[str] # Layer 2: stages to schedule scope: str # rule-declared scope label, v0 only has "waiveonly" reason: str @@ -174,25 +175,22 @@ if (cbts?.scope == "waiveonly") { - **Adding a new scope**: add another parallel `if (cbts?.scope == "testonly") { ... }`; scopes don't interfere. - **Cost**: on a waiveonly PR the existing filter chain runs once and is then overwritten (pure Groovy set ops; no IO; negligible). -### 4.4 Layer 3 — Within-stage test filter +### 4.4 Why no within-stage test filter (intentional over-inclusion) -In the block starting at `L0_Test.groovy:2674`, the CBTS filter is inserted **right before `processShardTestList`**, after all prep (`mergeWaivesTxt` / `reusePassedTestResults`) has completed: +After Layer 2 narrows to affected stages, CBTS **does not** filter each stage's testDBList down to the specific changed test ids. The stage runs its full rendered testDBList (all blocks whose conditions match the stage's mako). -```groovy -def testDBList = renderTestDB(testList, llmSrc, stageName) -mergeWaivesTxt(pipeline, llmSrc, stageName) // existing: download merged waives.txt -// reusePassedTestResults(...) // existing: append previously-passed tests to waives +The `L0_Test.groovy:2674` injection point carries only a comment explaining the deliberate omission; `processShardTestList` is invoked with the untouched testDBList. -// NEW: CBTS Layer 3 filter, single-point insertion -def cbts = testFilter[(CBTS_RESULT)] -if (cbts?.scope == "waiveonly") { - testDBList = filterTestDBList(testDBList, cbts.affected_tests) -} +**Why over-include at the block level**: +- A waive change can be *wrong* in subtle ways: the waived test id may contain a typo and silently match nothing; the waived test may be paired with other tests via shared fixtures or ordering; the waive reason may stop applying because of an unrelated change. +- Running only the single changed test would not surface any of those failure modes — the test either runs once and passes, or is skipped by the updated waive, leaving the regression invisible. +- Running the full block (which is what the stage's testDBList already contains) gives CI a fair chance to catch waive mistakes via the neighbouring tests. -def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode) -``` +**Cost we accept**: extra per-stage test runtime on affected stages. This is justified because: +- The dominant CBTS win comes from Layers 1 + 2 (skipping entire tracks / unaffected stages). Within-stage filtering is only a marginal efficiency. +- Waive-change PRs are rare; over-running on those PRs is acceptable. -Same explicit match on `waiveonly`; future scopes must decide independently whether to filter tests at this layer by adding an `else if` branch. +If a future scope truly needs test-level narrowing (e.g. a rule that is certain about its dependencies), it can extend `L0_Test.groovy:2674` with an `if (cbts?.scope == "its_scope") { ... }` branch at that point. The `affected_tests` value is already available in `testFilter[CBTS_RESULT].affected_tests` for any future scope to consume. #### Interaction with `mergeWaivesTxt`: verified consistent @@ -331,7 +329,7 @@ Any of the following → **fall through to the existing filter chain** (`testFil - Setter: the fourth setter on `testFilter`. - Layer 1 consumer: one skip check at each `launchStages` track entry (isomorphic to the existing Docs-only skip). - Layer 2 consumer: appended `if` in the `L0_Test.groovy` filter chain tail. - - Layer 3 consumer: three-line injection at `L0_Test.groovy:2674`. + - `L0_Test.groovy:2674`: only a comment clarifying why CBTS does not filter testDBList at stage time (see 4.4). - **Any failure falls back to full run.** ### For dev diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 65856cc13181..3b082e196ad9 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -8,20 +8,25 @@ README is the operational reference. --- -## What it does — three layers +## What it does — two layers -Given a PR, CBTS produces a decision that gets consumed at three points in the +Given a PR, CBTS produces a decision that gets consumed at two points in the Jenkins pipeline: | Layer | Where consumed | Action | |---|---|---| | **1. Arch track** | `L0_MergeRequest.groovy::launchStages` (each track entry) | Skip whole x86 / SBSA track (build + all tests) when no stage on that arch is affected | | **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset | -| **3. Test** | `L0_Test.groovy::runLLMTestlistOnPlatformImpl` (after `renderTestDB`) | Intersect rendered `testDBList` with CBTS's `affected_tests` | Anything CBTS can't confidently narrow → **fallback to the existing full filter chain**. CBTS never adds stages; it only subtracts. +**No within-stage test filtering by design.** Once Layer 2 picks the affected +stages, each stage runs its full rendered testDBList (all blocks matching the +stage's mako). Running the whole block, not just the single changed test id, +is deliberately over-inclusive — if a waive is wrong, the neighbouring tests +give CI a chance to catch it. See DESIGN.md §4.4 for the full rationale. + ## v0 scope - **Only handles** `tests/integration/test_lists/waives.txt` changes @@ -114,7 +119,7 @@ Output is a JSON blob on stdout (see DESIGN.md §4.6): ... return RuleResult( handled_files={...}, # files you claim - tests={...}, # Layer 3 test filter + tests={...}, # changed test ids (logged; not filtered at stage time) affected_stages={...}, # Layer 2 stage set scope="myscope", # your scope label reason="why this was picked", @@ -125,11 +130,14 @@ Output is a JSON blob on stdout (see DESIGN.md §4.6): - Add the class to `RULE_CLASSES` (used by `--list-needed-diffs`) - Add an instance to `build_rules()` with its dependencies -3. **Decide Layer 1 / 2 / 3 behavior for your scope in Groovy**. Each layer's +3. **Decide Layer 1 / 2 behavior for your scope in Groovy**. Each layer's consumer checks `cbts.scope == "waiveonly"` explicitly. For a new scope: - - Add an `else if (cbts.scope == "myscope")` branch in `L0_Test.groovy` + - Add an `if (cbts.scope == "myscope")` branch in `L0_Test.groovy` Layer 2 override (or leave unspecified → behaves as fallback / full run) - - Similarly decide Layer 1 (arch track skip) and Layer 3 (test filter) + - Similarly decide Layer 1 (arch track skip) + - If your scope genuinely needs within-stage test filtering, add that at + `L0_Test.groovy:2674` (currently only a comment; see DESIGN.md §4.4 for + why `waiveonly` deliberately skips this) - **Defaults are conservative**: without explicit branches, new scope paths fall through to the existing filter chain, which is safe From 5a29bec2f0ef6a729da8e8a84e21e76d6150fc9e Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:53:25 +0800 Subject: [PATCH 06/65] [None][chore] delete CBTS DESIGN.md; consolidate into README.md Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- jenkins/L0_Test.groovy | 2 +- jenkins/scripts/cbts/DESIGN.md | 347 --------------------------------- jenkins/scripts/cbts/README.md | 67 ++----- jenkins/scripts/cbts/main.py | 2 +- 5 files changed, 21 insertions(+), 399 deletions(-) delete mode 100644 jenkins/scripts/cbts/DESIGN.md diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 7d6816727d8f..89fb682d78f6 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -709,7 +709,7 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { // affected_stages, affected_tests, reasons} or null (= no decision / fall // back to the existing filter chain). // -// See jenkins/scripts/cbts/DESIGN.md for the three-layer consumption model. +// See jenkins/scripts/cbts/README.md for the two-layer consumption model. // ============================================================================ def getCbtsResult(pipeline, testFilter, globalVars) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index c6a537217626..7cfb80c9eb30 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4416,7 +4416,7 @@ def launchTestJobs(pipeline, testFilter) // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all // existing filter rules so that unknown / no-decision paths fall through - // naturally. See jenkins/scripts/cbts/DESIGN.md for the three-layer model. + // naturally. See jenkins/scripts/cbts/README.md for the two-layer model. def cbts = testFilter[(CBTS_RESULT)] if (cbts?.scope == "waiveonly" && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set diff --git a/jenkins/scripts/cbts/DESIGN.md b/jenkins/scripts/cbts/DESIGN.md deleted file mode 100644 index 1f757f4c4d92..000000000000 --- a/jenkins/scripts/cbts/DESIGN.md +++ /dev/null @@ -1,347 +0,0 @@ -# CBTS — Change-Based Testing Selection for CI - -**Status**: Draft for infra + dev review -**v0 Scope**: When a PR only changes `tests/integration/test_lists/waives.txt`, decide which stages + tests to run at block granularity. - ---- - -## 1. Core Idea - -PR changes one test in waives.txt → that test lives in some YAML block → the block's `condition` matches only certain stages → **other stages aren't scheduled**; build + test sub-jobs for the arch track (x86 / SBSA) that has no affected stages are **entirely skipped**. - -**Two-layer filtering**: -- **Layer 1 (arch track level)**: affected stages only on x86 → skip the entire SBSA track (SBSA build + all SBSA sub-jobs), and vice versa. -- **Layer 2 (stage level)**: within the same arch, match `block.condition` against `stage.mako` to pick which stages to schedule. - -Inside each selected stage, CBTS **intentionally does NOT filter further**. The stage runs its normal rendered testDBList (all blocks whose conditions match the stage's mako) rather than narrowing to the specific changed test id. Rationale: if a waive is wrong (depends on another test, or has a typo), running the whole block catches it; running only the single affected test would silently miss the problem. See 4.4 for the full argument. - -No changes to pytest-split, trt-test-db, or the stage-scheduling core; all new code is glue. - ---- - -## 2. v0 Scope - -- **Covers**: PRs that only change waives.txt. -- **Other changes**: any other file → fall through to the existing filter chain (equivalent to full run). -- **Semantics**: add/remove/edit a waive → run the corresponding test and the stages matching its containing block. -- **scope label**: v0 defines one value — `waiveonly` (PR only changed waive-related files). Future rules introduce new scope values (e.g. `testonly`, `modelarch`) when needed; we do not invent an abstract taxonomy upfront. - ---- - -## 3. Code Architecture - -### 3.1 File layout - -``` -jenkins/scripts/cbts/ -├── DESIGN.md -├── README.md -├── main.py ← Selector + SelectionResult + CLI -├── blocks.py ← Stage + Block + YAMLIndex + block_matches_stage -└── rules/ - ├── base.py ← Rule ABC + PRInputs + RuleResult - └── waives_rule.py ← v0 rule -``` - -**4 Python files** + 2 docs. No `__init__.py` files — directories are used as Python 3.3+ namespace packages, matching the `jenkins/scripts/` convention (sibling directories like `jenkins/scripts/perf/` are structured the same way). - -### 3.2 Key contracts - -```python -# rules/base.py -@dataclass -class PRInputs: - changed_files: list[str] - diffs: dict[str, str] # Groovy pre-fetches based on needs_diff_for - -@dataclass -class RuleResult: - handled_files: set[str] - tests: set[str] # changed test ids (used internally to find blocks; also logged) - affected_stages: set[str] # Layer 2: stages to schedule - scope: str # rule-declared scope label, v0 only has "waiveonly" - reason: str - -class Rule(ABC): - name: str - needs_diff_for: list[str] = [] - @abstractmethod - def apply(self, pr: PRInputs) -> Optional[RuleResult]: ... -``` - -Note: `affected_cpu_arch` is not a RuleResult field; the Selector derives it by looking up each affected stage's `cpu_arch` in the stage map. Rules don't set it. - -```python -# blocks.py -@dataclass -class Stage: - name: str - yaml_stem: str - cpu_arch: str # "x86" / "sbsa", inferred from x86TestConfigs vs SBSATestConfigs - split_id: int - total_splits: int - mako: dict[str, str] # derived from stage name (mirrors getMakoArgsFromStageName) - -@dataclass -class Block: - yaml_stem: str - block_index: int - condition: dict # raw: {ranges, wildcards, terms} - tests: list[str] - -def block_matches_stage(block, stage) -> bool: - """Generic over YAML field names: adding a new term does not require changing this.""" -``` - -### 3.3 Why no `stages.py` - -`Stage` values are derived purely from what Python can parse out of `jenkins/L0_Test.groovy` (stage map entries + stage-name patterns). Keeping `Stage` together with `Block` in `blocks.py` avoids a module for two small dataclasses. `derive_mako_from_stage` mirrors `getMakoArgsFromStageName` on the Groovy side — single source of truth is still the Groovy file; Python only reads it. - ---- - -## 4. Jenkins Integration - -### 4.1 Injection point: alongside other testFilter setters - -In `L0_MergeRequest.groovy`, find this snippet (excerpt): - -```groovy -testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) -testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) -testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) -// NEW -testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) -``` - -`getCbtsResult` returns either `null` (no decision → full run) or `{scope, affected_cpu_arch, affected_stages, affected_tests, reasons}`. **The decision is made once and cached in `testFilter`**; the three downstream layers only read. - -### 4.2 Layer 1 — Arch-track skip - -Injection point: the `x86_64-Linux` / `SBSA-Linux` track entries in `L0_MergeRequest.groovy::launchStages()` (there is already precedent there: `if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") return` skips SBSA for docs-only PRs). - -```groovy -"x86_64-Linux": { - script { - def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && !("x86" in cbts.affected_cpu_arch)) { - echo "CBTS waiveonly: no x86 stages affected, skipping x86_64-Linux track" - return - } - // existing Build-x86_64 + Test-x86_64-* logic unchanged - ... - } -}, -"SBSA-Linux": { - script { - if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") { return } // existing - // NEW: CBTS waiveonly equivalent skip - def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && !("sbsa" in cbts.affected_cpu_arch)) { - echo "CBTS waiveonly: no sbsa stages affected, skipping SBSA-Linux track" - return - } - // existing Build-SBSA + Test-SBSA-* logic unchanged - ... - } -}, -``` - -**Key points**: -- The condition explicitly matches `scope == "waiveonly"`. Future scopes (`testonly` / `modelarch`) do not trigger track skips by default until their safety is evaluated and an explicit `else if` branch is added. -- Effect: an x86-only waive change → the entire SBSA track disappears from Blue Ocean (including build); and vice versa. - -### 4.3 Layer 2 — Stage-scheduling override - -Injection point: the filter chain in `L0_Test.groovy` (around lines 3714–3800). **CBTS acts as a short-circuit override appended to the end of the existing chain**; the existing logic is untouched. - -```groovy -// Existing filter chain untouched: MULTI_GPU_FILE_CHANGED / AUTO_TRIGGER_TAG_LIST / -// IS_POST_MERGE / ENABLE_SKIP_TEST / GPU_TYPE_LIST / TEST_BACKEND / ... -// Produces parallelJobsFiltered. -... - -// NEW: CBTS short-circuit override at the tail -def cbts = testFilter[(CBTS_RESULT)] -if (cbts?.scope == "waiveonly") { - parallelJobsFiltered = parallelJobs.findAll { key, _ -> key in cbts.affected_stages } - echo "CBTS waiveonly: limiting to ${cbts.affected_stages.size()} affected stages" -} -``` - -**Key points**: -- **One `if`, no `else`**: concise, no nested branches. -- **Override semantics**: the existing chain produces `parallelJobsFiltered` first; when waiveonly matches, it is replaced wholesale. -- **All fallback cases naturally don't override**: `cbts == null` / `scope == null` / unknown scope / call failure → condition false → the existing chain's result is preserved. -- **Adding a new scope**: add another parallel `if (cbts?.scope == "testonly") { ... }`; scopes don't interfere. -- **Cost**: on a waiveonly PR the existing filter chain runs once and is then overwritten (pure Groovy set ops; no IO; negligible). - -### 4.4 Why no within-stage test filter (intentional over-inclusion) - -After Layer 2 narrows to affected stages, CBTS **does not** filter each stage's testDBList down to the specific changed test ids. The stage runs its full rendered testDBList (all blocks whose conditions match the stage's mako). - -The `L0_Test.groovy:2674` injection point carries only a comment explaining the deliberate omission; `processShardTestList` is invoked with the untouched testDBList. - -**Why over-include at the block level**: -- A waive change can be *wrong* in subtle ways: the waived test id may contain a typo and silently match nothing; the waived test may be paired with other tests via shared fixtures or ordering; the waive reason may stop applying because of an unrelated change. -- Running only the single changed test would not surface any of those failure modes — the test either runs once and passes, or is skipped by the updated waive, leaving the regression invisible. -- Running the full block (which is what the stage's testDBList already contains) gives CI a fair chance to catch waive mistakes via the neighbouring tests. - -**Cost we accept**: extra per-stage test runtime on affected stages. This is justified because: -- The dominant CBTS win comes from Layers 1 + 2 (skipping entire tracks / unaffected stages). Within-stage filtering is only a marginal efficiency. -- Waive-change PRs are rare; over-running on those PRs is acceptable. - -If a future scope truly needs test-level narrowing (e.g. a rule that is certain about its dependencies), it can extend `L0_Test.groovy:2674` with an `if (cbts?.scope == "its_scope") { ... }` branch at that point. The `affected_tests` value is already available in `testFilter[CBTS_RESULT].affected_tests` for any future scope to consume. - -#### Interaction with `mergeWaivesTxt`: verified consistent - -The merged waives.txt downloaded by `mergeWaivesTxt` is produced by `jenkins/scripts/mergeWaiveList.py` with the following algorithm: - -``` -merged = dedupe(PR's waives.txt ∪ TOT's waives.txt) - lines in PR's diff prefixed with `-` -``` - -This algorithm is **PR-aware**: PR additions are preserved via the union; PR removals are applied via subtraction. So the merged result **fully preserves the PR's intent on waives.txt**. - -Three scenarios, all verified: - -| Scenario | merged content | runtime | CBTS decision correctness | -|---|---|---|---| -| PR removes a waive (`-test_X`) | no test_X | test_X runs | ✓ stage runs, test is actually verified | -| PR adds a waive (`+test_X`) | contains test_X | test_X skipped | ✓ stage runs, pytest collects the test and skips correctly (verifies the waive mechanism) | -| PR edits a waive line (NVBug link) | new line only | test still skipped | ✓ matches PR intent | - -**Conclusion**: CBTS decisions based on the PR diff are naturally aligned with the runtime waive state; **no extra handling for `mergeWaivesTxt` is needed**. - -### 4.5 What `getCbtsResult` does - -1. PostMerge / alternativeTRT → return null. -2. `getMergeRequestChangedFileList` → `changed_files`, `.unique()`. -3. Ask Python `--list-needed-diffs` for patterns; for each changed file matching a pattern, call `getMergeRequestOneFileChanges` to pull its diff. -4. Write `cbts_input.json` (just `changed_files` + `diffs`), then run `python3 main.py cbts_input.json`. -5. Parse stdout, return a structured result. - -Note: the Python side **parses `jenkins/L0_Test.groovy` itself** to derive stage configs (reusing the regex approach already in `scripts/test_to_stage_mapping.py`) and loads YAMLs from `tests/integration/test_lists/test-db/`. Groovy does not need to pack `stage_map` into INPUT_JSON. This keeps the Groovy code surface minimal. - -Each stage's `cpu_arch` is inferred by tracking which map-literal (`x86TestConfigs` / `SBSATestConfigs` / ...) the entry lives in. Each stage's `mako` is derived by a Python port of `getMakoArgsFromStageName` (line ~2079) and `parseTaskConfigFromStageName` (line ~2066). Those Python helpers live in `blocks.py` with explicit "keep in sync with groovy source" comments at the top. - -### 4.6 Python ↔ Groovy IO contract - -Groovy `getCbtsResult` calls Python twice: - -1. **Get needs_diff_for patterns** (no args): Python prints the union of all rules' `needs_diff_for` patterns; Groovy uses this to decide which changed files to fetch diffs for. -2. **Make the decision** (JSON file arg): Groovy packs `changed_files` / `diffs` into INPUT_JSON; Python writes the decision to stdout. - -**INPUT_JSON** is produced by Groovy `getCbtsResult`, containing only PR data: - -```json -{ - "changed_files": ["tests/integration/test_lists/waives.txt", ...], - "diffs": { - "tests/integration/test_lists/waives.txt": "@@ -1,3 +1,4 @@\n..." - } -} -``` - -Stage configs are parsed by Python directly from `jenkins/L0_Test.groovy` (see 4.5). - -**stdout is a JSON blob**: - -```json -{ - "scope": "waiveonly", - "affected_cpu_arch": ["x86"], - "affected_stages": [ - "DGX_H100-4_GPUs-PyTorch-DeepSeek-1", - "DGX_H100-4_GPUs-PyTorch-DeepSeek-2" - ], - "tests": ["examples/test_deepseek.py::test_xxx"], - "reasons": ["[waives] waives.txt: +2 / -1 → 2 blocks, 2 stages"] -} -``` - -When there is no decision, `scope` is `null` (Groovy parses this as "fall back"): - -```json -{ - "scope": null, - "affected_cpu_arch": [], - "affected_stages": [], - "tests": [], - "reasons": ["Unhandled files: [tensorrt_llm/llmapi/llm.py, ...]"] -} -``` - -Groovy consumes it via `JsonSlurper` in `_cbtsParseSelectionResult`; `scope == null` maps to `testFilter[CBTS_RESULT] = null`. - -Exit code 0 = decision succeeded (including the null-scope case); non-zero → Groovy falls back to full run. - -### 4.7 Multi-rule combination (future) - -v0 has only one rule, so no combining. But `Selector` reserves a `combine_scopes(scopes)` helper: - -```python -def combine_scopes(scopes: list[str]) -> str | None: - # v0: single rule returns "waiveonly" — passthrough. - if len(set(scopes)) == 1: - return scopes[0] - # Multiple differing scopes → conservative None (full run). - # Fill in a priority table here when priority needs emerge. - return None -``` - ---- - -## 5. Fallback & Safety - -Any of the following → **fall through to the existing filter chain** (`testFilter[CBTS_RESULT] = null` or `.scope == null`): - -- PostMerge / alternativeTRT job. -- `changed_files` is empty. -- Python call fails / stdout unparsable. -- Python explicitly returns `scope: none` (unhandled files, no rule matched, multi-rule scope conflict). -- Groovy sees an unknown scope (Python upgraded before Groovy caught up). - ---- - -## 6. Extensions (future, not in v0) - -| New Rule | New modules | `needs_diff_for` | Likely scope | -|---|---|---|---| -| `test_block_rule` (test-file changes) | none; reuses `blocks.py` | `[]` | `testonly` | -| `case_level_rule` (function-level precision) | `code_analysis/ast_utils.py` + `test_extractor.py` | `["tests/integration/defs/**/*.py"]` | `testonly` | -| `model_arch_rule` (model-arch matching) | `model_arch.py` | `[]` | `modelarch` | - -**What adding a rule requires**: -1. Define a new scope value (e.g. `testonly`) and add an `else if` branch at Groovy Layer 1 specifying whether aux stages are skipped for this scope. -2. Write a Rule class on the Python side. -3. If extra PR data is needed, declare it in `needs_diff_for`. - -**What adding a rule does NOT change**: the Rule ABC, the `combine_scopes` skeleton, `blocks.py`, the CLI contract, the overall shape of `getCbtsResult`, or the Layer-2 consumption site. - ---- - -## 7. Review Highlights - -### For infra -- **Zero changes to pytest-split / trt-test-db / stage rendering.** -- **Reuses existing helpers**: `getMergeRequestChangedFileList`, `getMergeRequestOneFileChanges`, `getMakoArgsFromStageName`, stageList. -- **Injections mirror existing patterns**: - - Setter: the fourth setter on `testFilter`. - - Layer 1 consumer: one skip check at each `launchStages` track entry (isomorphic to the existing Docs-only skip). - - Layer 2 consumer: appended `if` in the `L0_Test.groovy` filter chain tail. - - `L0_Test.groovy:2674`: only a comment clarifying why CBTS does not filter testDBList at stage time (see 4.4). -- **Any failure falls back to full run.** - -### For dev -- **v0 behavior is conservative**: only narrows PRs that touch only waives.txt. -- **CI log has a reason line**: you can see why particular stages / tests were selected. -- **Local reproducibility**: pull the `cbts_input.json` CI artifact and run `python3 main.py cbts_input.json` to reproduce the decision. -- **Low bar to add a rule**: write a class with an `apply()` method. - ---- - -## 8. Open Questions - -1. **Does infra accept skipping docBuild/sanityCheck under `waiveonly`?** waives.txt is a runtime plain-text list that doesn't flow into the wheel or docs, so skipping should be safe; if the team has a hard rule of "every PR must pass doc build / wheel sanity", the fallback is to keep those two aux stages even under `waiveonly`. -2. **`filterTestDBList` in Groovy or Python?** Leaning Python (more testable). -3. **Self-check mechanism**: Python's `block_matches_stage` could drift from trt-test-db semantics. Deferred to a future revision. diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 3b082e196ad9..0ba8a8c01eff 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -1,10 +1,7 @@ # CBTS — Change-Based Testing Selection Pre-merge CI test-selection tool. Looks at what the PR changed and narrows the -set of Jenkins stages + tests that actually need to run. - -For conceptual design / review notes, see [DESIGN.md](./DESIGN.md). This -README is the operational reference. +set of Jenkins stages that actually need to run. --- @@ -24,8 +21,10 @@ chain**. CBTS never adds stages; it only subtracts. **No within-stage test filtering by design.** Once Layer 2 picks the affected stages, each stage runs its full rendered testDBList (all blocks matching the stage's mako). Running the whole block, not just the single changed test id, -is deliberately over-inclusive — if a waive is wrong, the neighbouring tests -give CI a chance to catch it. See DESIGN.md §4.4 for the full rationale. +is deliberately over-inclusive — if a waive is wrong (depends on another test, +or the node id has a typo that silently matches nothing), running only the +changed test would not surface the problem. The extra per-stage test time is +accepted as the cost of CI robustness. ## v0 scope @@ -37,7 +36,6 @@ give CI a chance to catch it. See DESIGN.md §4.4 for the full rationale. ``` jenkins/scripts/cbts/ -├── DESIGN.md design doc (for review) ├── README.md this file ├── main.py CLI entry + Selector + SelectionResult ├── blocks.py YAML loading + stage parsing from groovy + condition matching @@ -58,33 +56,7 @@ jenkins/scripts/cbts/ The decision is cached in `testFilter[CBTS_RESULT]` and serialized into the child job's `testFilter` param alongside the existing filter flags. -## Debugging a CBTS decision locally - -The input JSON that Groovy sends to Python is uploaded as a CI artifact -(`cbts_input.json` in the pipeline workspace). To reproduce a decision -locally: - -```bash -# From the repo root: -python3 jenkins/scripts/cbts/main.py cbts_input.json -``` - -Or hand-craft a minimal input: - -```bash -cat > /tmp/cbts_input.json < Date: Thu, 23 Apr 2026 22:23:19 +0800 Subject: [PATCH 07/65] [None][chore] document CBTS rules inventory in rules/README.md Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 1 + jenkins/scripts/cbts/rules/README.md | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 jenkins/scripts/cbts/rules/README.md diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 0ba8a8c01eff..f65efad64ffc 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -40,6 +40,7 @@ jenkins/scripts/cbts/ ├── main.py CLI entry + Selector + SelectionResult ├── blocks.py YAML loading + stage parsing from groovy + condition matching └── rules/ + ├── README.md per-rule logic summary (scope, triggers, matching) ├── base.py Rule ABC + PRInputs + RuleResult └── waives_rule.py v0's only rule ``` diff --git a/jenkins/scripts/cbts/rules/README.md b/jenkins/scripts/cbts/rules/README.md new file mode 100644 index 000000000000..4671c99eb407 --- /dev/null +++ b/jenkins/scripts/cbts/rules/README.md @@ -0,0 +1,10 @@ +# rules/ + +One rule per file; each inherits from `Rule` in `base.py`. See the +top-level [README](../README.md) for the overall CBTS architecture. + +## Current rules + +| File | Class | Scope | Triggers on | What it picks | +|---|---|---|---|---| +| `waives_rule.py` | `WaivesRule` | `waiveonly` | PR changes `tests/integration/test_lists/waives.txt` | For each added/removed test id in the diff: look it up in the test-db YAML, pick stages whose `mako` matches the containing block's `condition`. | From 413495dc580577bdc85a2d21b04e34a43f594856 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 22:25:05 +0800 Subject: [PATCH 08/65] [None][test] validate CBTS in this PR via waive edit+add (revert before merge) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2772c080b8f0..6a0f612c897e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -264,6 +264,9 @@ full:H20/accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp8 full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,2000-reqs:10-ep:4-tp:8-gpus:8] SKIP (https://nvbugs/5150255) full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:512-input_output_len:128,128-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) +triton_server/test_triton.py::test_gpt_2b_ib_lora[gpt-2b-ib-lora] SKIP (https://nvbugs/5470830) +unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781, touched for CBTS validation) +triton_server/test_triton.py::test_llava[llava] SKIP (https://nvbugs/5547414) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5948435) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/5961814) From 0d5b0f014a48336983fde4951bcc46e271e7e2c5 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 23 Apr 2026 22:36:51 +0800 Subject: [PATCH 09/65] [None][chore] CBTS defers to explicit /bot run flags; activates only on bare run Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 24 ++++++++++++++++++++++++ jenkins/scripts/cbts/README.md | 11 +++++++++++ 2 files changed, 35 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 89fb682d78f6..9f3f1405e3d7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -719,6 +719,13 @@ def getCbtsResult(pipeline, testFilter, globalVars) return null } + // CBTS only activates on bare `/bot run`. If the user specified any + // stage-selection flag, defer entirely to their explicit choice. + if (_cbtsUserSpecifiedAnyBotFlag(testFilter)) { + pipeline.echo("CBTS: user-specified /bot run flag detected, deferring (CBTS not applied)") + return null + } + def changedFiles = getMergeRequestChangedFileList(pipeline, globalVars).unique() if (!changedFiles) { return null @@ -777,6 +784,23 @@ def _cbtsMatchesAnyPattern(String filePath, List patterns) return patterns.contains(filePath) } +// Detect whether the user supplied any stage-selection flag via `/bot run`. +// CBTS defers entirely to the user in that case. Logging flags like --debug +// and --detailed-log are orthogonal and intentionally excluded. +def _cbtsUserSpecifiedAnyBotFlag(testFilter) +{ + return testFilter[(REUSE_TEST)] != null || + testFilter[(REUSE_STAGE_LIST)] != null || + testFilter[(ENABLE_SKIP_TEST)] || + testFilter[(TEST_STAGE_LIST)] != null || + testFilter[(EXTRA_STAGE_LIST)] != null || + testFilter[(GPU_TYPE_LIST)] != null || + testFilter[(TEST_BACKEND)] != null || + testFilter[(ADD_MULTI_GPU_TEST)] || + testFilter[(ONLY_MULTI_GPU_TEST)] || + testFilter[(DISABLE_MULTI_GPU_TEST)] +} + // Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3, or null // when the Python side explicitly returned scope=null (no decision). def _cbtsParseSelectionResult(String text) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index f65efad64ffc..dfbf5646979d 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -45,6 +45,17 @@ jenkins/scripts/cbts/ └── waives_rule.py v0's only rule ``` +## When CBTS activates + +CBTS runs only on **bare `/bot run`** invocations. If the user provides any +stage-selection flag — `--stage-list`, `--extra-stage`, `--gpu-type`, +`--backend-mode`, `--skip-test`, `--add-multi-gpu-test`, +`--only-multi-gpu-test`, `--disable-multi-gpu-test`, `--reuse-test`, or +`--reuse-stage-list` — `getCbtsResult` returns `null` immediately and CBTS +stays out of the way. The existing filter chain drives test selection +exactly as before. Logging flags (`--debug`, `--detailed-log`) are +orthogonal and do NOT disable CBTS. + ## How it's invoked (CI) `L0_MergeRequest.groovy::getCbtsResult` orchestrates two calls to `main.py`: From 2a9e0dfffb4fd8b12ca4836d4c688c42b2fb36fe Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 10:40:47 +0800 Subject: [PATCH 10/65] [None][chore] CBTS: pip install pyyaml in getCbtsResult before first Python call Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 9f3f1405e3d7..4251bc79eab7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -732,6 +732,10 @@ def getCbtsResult(pipeline, testFilter, globalVars) } try { + // 0. Ensure pyyaml is available on the Jenkins agent (blocks.py needs it + // to parse test-db YAMLs). + sh "pip3 install --quiet pyyaml" + // 1. Ask Python for the union of needs_diff_for patterns across all rules. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", From ed3fc70c0db3bcd76b8df286b50b5b76a213f0f6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:57:13 +0800 Subject: [PATCH 11/65] [None][chore] CBTS: don't treat REUSE_* as user flag (auto-set on bot re-runs) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 12 +++++++----- jenkins/scripts/cbts/README.md | 24 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 4251bc79eab7..97fb1c5faabc 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -789,13 +789,15 @@ def _cbtsMatchesAnyPattern(String filePath, List patterns) } // Detect whether the user supplied any stage-selection flag via `/bot run`. -// CBTS defers entirely to the user in that case. Logging flags like --debug -// and --detailed-log are orthogonal and intentionally excluded. +// CBTS defers entirely to the user in that case. Excluded on purpose (match +// the same convention used by `enableUpdateGitlabStatus` above): +// - REUSE_TEST / REUSE_STAGE_LIST: retry semantics, auto-populated by the +// bot on re-runs; compose fine with CBTS (CBTS picks stages, reuse skips +// ones already passed). +// - DEBUG_MODE / DETAILED_LOG: logging verbosity, orthogonal to selection. def _cbtsUserSpecifiedAnyBotFlag(testFilter) { - return testFilter[(REUSE_TEST)] != null || - testFilter[(REUSE_STAGE_LIST)] != null || - testFilter[(ENABLE_SKIP_TEST)] || + return testFilter[(ENABLE_SKIP_TEST)] || testFilter[(TEST_STAGE_LIST)] != null || testFilter[(EXTRA_STAGE_LIST)] != null || testFilter[(GPU_TYPE_LIST)] != null || diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index dfbf5646979d..2a5dc369fcfc 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -47,14 +47,22 @@ jenkins/scripts/cbts/ ## When CBTS activates -CBTS runs only on **bare `/bot run`** invocations. If the user provides any -stage-selection flag — `--stage-list`, `--extra-stage`, `--gpu-type`, -`--backend-mode`, `--skip-test`, `--add-multi-gpu-test`, -`--only-multi-gpu-test`, `--disable-multi-gpu-test`, `--reuse-test`, or -`--reuse-stage-list` — `getCbtsResult` returns `null` immediately and CBTS -stays out of the way. The existing filter chain drives test selection -exactly as before. Logging flags (`--debug`, `--detailed-log`) are -orthogonal and do NOT disable CBTS. +CBTS activates when the user runs `/bot run` without a stage-selection +flag. If the user provides any of `--stage-list`, `--extra-stage`, +`--gpu-type`, `--backend-mode`, `--skip-test`, `--add-multi-gpu-test`, +`--only-multi-gpu-test`, or `--disable-multi-gpu-test`, `getCbtsResult` +returns `null` immediately and the existing filter chain drives test +selection exactly as before. + +**Not considered user flags** (CBTS still activates): +- `--reuse-test` / `--reuse-stage-list` — retry semantics; the bot + auto-populates these on re-runs of the same PR. They compose fine with + CBTS (CBTS picks stages, reuse further skips stages that already passed). +- `--debug` / `--detailed-log` — logging verbosity; orthogonal. + +This matches the convention already used by `enableUpdateGitlabStatus` in +`L0_MergeRequest.groovy` for distinguishing "default run" from +"user-customized run". ## How it's invoked (CI) From 3190378f837a5046e0185f7aaecf355fc886e3d5 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:38:45 +0800 Subject: [PATCH 12/65] [None][chore] CBTS: log which specific flag triggered the defer, for diagnostics Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 51 +++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 97fb1c5faabc..f4febd1ff3d7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -721,8 +721,9 @@ def getCbtsResult(pipeline, testFilter, globalVars) // CBTS only activates on bare `/bot run`. If the user specified any // stage-selection flag, defer entirely to their explicit choice. - if (_cbtsUserSpecifiedAnyBotFlag(testFilter)) { - pipeline.echo("CBTS: user-specified /bot run flag detected, deferring (CBTS not applied)") + def triggeredFlags = _cbtsTriggeredUserFlags(testFilter) + if (!triggeredFlags.isEmpty()) { + pipeline.echo("CBTS: user-specified /bot run flag detected, deferring. Triggered by: ${triggeredFlags.join(', ')}") return null } @@ -788,23 +789,41 @@ def _cbtsMatchesAnyPattern(String filePath, List patterns) return patterns.contains(filePath) } -// Detect whether the user supplied any stage-selection flag via `/bot run`. -// CBTS defers entirely to the user in that case. Excluded on purpose (match -// the same convention used by `enableUpdateGitlabStatus` above): +// Return a list of stage-selection flags the user set via `/bot run` (empty +// list means bare run). CBTS defers entirely when this is non-empty. +// Excluded on purpose (match the convention used by `enableUpdateGitlabStatus` +// above): // - REUSE_TEST / REUSE_STAGE_LIST: retry semantics, auto-populated by the -// bot on re-runs; compose fine with CBTS (CBTS picks stages, reuse skips -// ones already passed). +// bot on re-runs; compose fine with CBTS. // - DEBUG_MODE / DETAILED_LOG: logging verbosity, orthogonal to selection. -def _cbtsUserSpecifiedAnyBotFlag(testFilter) +def _cbtsTriggeredUserFlags(testFilter) { - return testFilter[(ENABLE_SKIP_TEST)] || - testFilter[(TEST_STAGE_LIST)] != null || - testFilter[(EXTRA_STAGE_LIST)] != null || - testFilter[(GPU_TYPE_LIST)] != null || - testFilter[(TEST_BACKEND)] != null || - testFilter[(ADD_MULTI_GPU_TEST)] || - testFilter[(ONLY_MULTI_GPU_TEST)] || - testFilter[(DISABLE_MULTI_GPU_TEST)] + def flags = [] + if (testFilter[(ENABLE_SKIP_TEST)]) { + flags << "ENABLE_SKIP_TEST=${testFilter[(ENABLE_SKIP_TEST)]}" + } + if (testFilter[(TEST_STAGE_LIST)] != null) { + flags << "TEST_STAGE_LIST=${testFilter[(TEST_STAGE_LIST)]}" + } + if (testFilter[(EXTRA_STAGE_LIST)] != null) { + flags << "EXTRA_STAGE_LIST=${testFilter[(EXTRA_STAGE_LIST)]}" + } + if (testFilter[(GPU_TYPE_LIST)] != null) { + flags << "GPU_TYPE_LIST=${testFilter[(GPU_TYPE_LIST)]}" + } + if (testFilter[(TEST_BACKEND)] != null) { + flags << "TEST_BACKEND=${testFilter[(TEST_BACKEND)]}" + } + if (testFilter[(ADD_MULTI_GPU_TEST)]) { + flags << "ADD_MULTI_GPU_TEST=${testFilter[(ADD_MULTI_GPU_TEST)]}" + } + if (testFilter[(ONLY_MULTI_GPU_TEST)]) { + flags << "ONLY_MULTI_GPU_TEST=${testFilter[(ONLY_MULTI_GPU_TEST)]}" + } + if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { + flags << "DISABLE_MULTI_GPU_TEST=${testFilter[(DISABLE_MULTI_GPU_TEST)]}" + } + return flags } // Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3, or null From b5db1156fbcd98ceb934a34be9a504146e4516da Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:46:36 +0800 Subject: [PATCH 13/65] [None][test] TESTING: WaivesRule claims all changed_files so CBTS fires on infra PR (revert before merge) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/rules/waives_rule.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 0062e73acaa6..05ac7ebf1796 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -105,7 +105,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: changed_test_ids = added | removed if not changed_test_ids: return RuleResult( - handled_files={WAIVES_FILE}, + # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim all + # changed files so CBTS fires on the cbts-v0 PR that also edits + # CBTS infra files. + handled_files=set(pr.changed_files), tests=set(), affected_stages=set(), scope="waiveonly", @@ -128,7 +131,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: affected_stage_names.add(stage_name) return RuleResult( - handled_files={WAIVES_FILE}, + # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim all + # changed files so CBTS fires on the cbts-v0 PR that also edits + # CBTS infra files. + handled_files=set(pr.changed_files), tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", From a6eabf6021a968caf57b369e17763908e896d095 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:49:47 +0800 Subject: [PATCH 14/65] [None][chore] CBTS: log specific defer reason for every branch (not just user-flag) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index f4febd1ff3d7..af72e348d0b0 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -716,6 +716,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { + pipeline.echo("CBTS: deferring — post-merge job or alternativeTRT set") return null } @@ -723,12 +724,13 @@ def getCbtsResult(pipeline, testFilter, globalVars) // stage-selection flag, defer entirely to their explicit choice. def triggeredFlags = _cbtsTriggeredUserFlags(testFilter) if (!triggeredFlags.isEmpty()) { - pipeline.echo("CBTS: user-specified /bot run flag detected, deferring. Triggered by: ${triggeredFlags.join(', ')}") + pipeline.echo("CBTS: deferring — user-specified /bot run flag(s): ${triggeredFlags.join(', ')}") return null } def changedFiles = getMergeRequestChangedFileList(pipeline, globalVars).unique() if (!changedFiles) { + pipeline.echo("CBTS: deferring — no changed files detected") return null } @@ -768,12 +770,15 @@ def getCbtsResult(pipeline, testFilter, globalVars) // 5. Parse stdout into the map shape consumed by Layer 1/2/3. def result = _cbtsParseSelectionResult(output) - if (result != null) { - pipeline.echo("CBTS: scope=${result.scope}, " + - "archs=${result.affected_cpu_arch}, " + - "stages=${result.affected_stages.size()}, " + - "tests=${result.affected_tests.size()}") + if (result.scope == null) { + pipeline.echo("CBTS: deferring — Python returned scope=null. " + + "Reasons: ${result.reasons.join('; ')}") + return null } + pipeline.echo("CBTS: scope=${result.scope}, " + + "archs=${result.affected_cpu_arch}, " + + "stages=${result.affected_stages.size()}, " + + "tests=${result.affected_tests.size()}") return result } catch (Exception e) { pipeline.echo("CBTS failed, falling back to full run: ${e}") @@ -826,12 +831,12 @@ def _cbtsTriggeredUserFlags(testFilter) return flags } -// Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3, or null -// when the Python side explicitly returned scope=null (no decision). +// Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3. Always +// returns a map; `scope == null` means "no decision" (caller should log the +// reasons and treat as defer). def _cbtsParseSelectionResult(String text) { def data = new groovy.json.JsonSlurper().parseText(text) - if (data.scope == null) { return null } return [ scope: data.scope, affected_cpu_arch: data.affected_cpu_arch ?: [], From 75d7ca287ecec3891683adb25be81ebfda5beb03 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:31:36 +0800 Subject: [PATCH 15/65] [None][fix] CBTS: install pyyaml via apt-get (buildpack-deps has no pip3) The setup pod uses buildpack-deps:trixie-scm, which does not ship pip3 by default, so `pip3 install --quiet pyyaml` fails with `pip3: not found` and CBTS falls back to a full run. Install the Debian python3-yaml package directly instead. It provides PyYAML without needing to bootstrap pip first, matching the style of launchReleaseCheck which uses apt-get for its Python deps. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index af72e348d0b0..c3dea06d1594 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -736,8 +736,9 @@ def getCbtsResult(pipeline, testFilter, globalVars) try { // 0. Ensure pyyaml is available on the Jenkins agent (blocks.py needs it - // to parse test-db YAMLs). - sh "pip3 install --quiet pyyaml" + // to parse test-db YAMLs). buildpack-deps has no pip3 by default, + // so install the Debian python3-yaml package directly. + sh "apt-get update -qq && apt-get install -y -qq python3-yaml" // 1. Ask Python for the union of needs_diff_for patterns across all rules. def patternsOut = sh( From 4c67e02297bf3e63d5bce53c2de15ba48dabc44f Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:37:48 +0800 Subject: [PATCH 16/65] [None][chore] CBTS: under --post-merge, narrow Layer 2 to post-merge hits only Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 11 ++++++++++- jenkins/scripts/cbts/README.md | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 7cfb80c9eb30..58185c1446f8 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4421,7 +4421,16 @@ def launchTestJobs(pipeline, testFilter) if (cbts?.scope == "waiveonly" && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) } - echo "CBTS waiveonly: limiting to ${parallelJobsFiltered.size()} affected stages" + // Under `/bot run --post-merge`, keep only post-merge hits; if none, + // no-op (no fallback to full post-merge). IS_POST_MERGE is also true + // for the official PostMerge pipeline, but getCbtsResult() defers + // there, so reading it here is equivalent to "user passed --post-merge". + if (testFilter[(IS_POST_MERGE)]) { + parallelJobsFiltered = parallelJobsFiltered.findAll { it.key.contains("Post-Merge") } + echo "CBTS waiveonly (--post-merge): keeping ${parallelJobsFiltered.size()} affected post-merge stages" + } else { + echo "CBTS waiveonly: limiting to ${parallelJobsFiltered.size()} affected stages" + } } echo "Check the passed GitLab bot testFilter parameters." diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 2a5dc369fcfc..42d46bea4002 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -59,6 +59,10 @@ selection exactly as before. auto-populates these on re-runs of the same PR. They compose fine with CBTS (CBTS picks stages, reuse further skips stages that already passed). - `--debug` / `--detailed-log` — logging verbosity; orthogonal. +- `--post-merge` — scopes the run to post-merge stages. CBTS still + activates; Layer 2 then narrows the affected set to post-merge hits + only. If no post-merge stage is hit, the run is a no-op (no fallback + to the full post-merge baseline). This matches the convention already used by `enableUpdateGitlabStatus` in `L0_MergeRequest.groovy` for distinguishing "default run" from From a9cbdebb4a3865a175a3ea811a9f84450bb5ad77 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:08:14 +0800 Subject: [PATCH 17/65] [None][chore] CBTS: fix RUF012 mutable default in Rule base class Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/rules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index ebdf2246b2dc..64647ab824d4 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -52,7 +52,7 @@ class Rule(ABC): """ name: str = "" - needs_diff_for: list[str] = [] + needs_diff_for: tuple[str, ...] = () @abstractmethod def apply(self, pr: PRInputs) -> Optional[RuleResult]: ... From 0a377a5416eee74a676fa36ba07bcbfbf79ca7d1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:09:59 +0800 Subject: [PATCH 18/65] [None][fix] CBTS: index normalized test ids so TIMEOUT-decorated YAML entries match waives.txt lookups Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 33 ++++++++++++++--- jenkins/scripts/cbts/rules/waives_rule.py | 44 ++++++----------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 0ff83be8563c..d26e81423d53 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -49,6 +49,29 @@ class Block: tests: list[str] +# Strip trailing ` SKIP ...` / ` TIMEOUT ...` annotations from a test id. +# YAML entries can carry `TIMEOUT (n)`, waives.txt entries can carry both; +# the lookup must hit either form so we normalize on both sides. +_TEST_ID_SUFFIX_RE = re.compile(r"\s+(SKIP|TIMEOUT)\b.*$") + +# Strip leading `full:/` platform prefix used in waives.txt. +_TEST_ID_PREFIX_RE = re.compile(r"^full:[^/]+/") + + +def normalize_test_id(test_id: str) -> str: + """Canonical form for cross-referencing test-db YAML and waives.txt. + + Strips trailing `SKIP`/`TIMEOUT` annotations, trailing `# comment`, and + leading `full:/` prefix. `YAMLIndex` indexes both the raw and the + normalized form; `rules.waives_rule` looks up by this normalization. + """ + s = test_id.strip() + s = _TEST_ID_SUFFIX_RE.sub("", s).strip() + if "#" in s: + s = s.split("#", 1)[0].strip() + return _TEST_ID_PREFIX_RE.sub("", s) + + class YAMLIndex: """Index of all blocks across test-db YAMLs, with reverse lookup by test id.""" @@ -78,12 +101,14 @@ def _load_one(self, yml_path: Path) -> None: tests=list(tests), ) self.blocks.append(block) + # Index each test under both its raw YAML string (which may carry + # ` -m "gpu2"`, ` TIMEOUT (90)`, etc.) and its normalized form, so + # waives.txt lookups — which strip SKIP/TIMEOUT — still resolve. for test in tests: - # Tests are raw YAML strings; they may carry trailing options - # like ` -m "gpu2"` or ` TIMEOUT (90)`. Use the full string as - # the match key so downstream test_id extraction can match - # either the bare node_id or the options-suffixed form. self._test_to_blocks.setdefault(test, []).append(block) + normalized = normalize_test_id(test) + if normalized and normalized != test: + self._test_to_blocks.setdefault(normalized, []).append(block) def blocks_containing_test(self, test_id: str) -> list[Block]: return list(self._test_to_blocks.get(test_id, [])) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 05ac7ebf1796..bf5cfa0253a0 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -15,58 +15,35 @@ from __future__ import annotations -import re from typing import Optional -from blocks import Stage, YAMLIndex, block_matches_stage +from blocks import Stage, YAMLIndex, block_matches_stage, normalize_test_id from .base import PRInputs, Rule, RuleResult WAIVES_FILE = "tests/integration/test_lists/waives.txt" -# Strip GPU/platform prefixes like "full:GH200/" or "full:sm100/" at the start. -_PREFIX_RE = re.compile(r"^full:[^/]+/") - -# Split trailing annotations (SKIP / TIMEOUT / comments) from the test id. -# A waive line typically looks like: -# SKIP (reason) -# SKIP # url -# # just a comment -# -k "expr" SKIP (reason) -# We keep the whole " [-m/-k ...]" portion as the identifier, since -# YAML entries can include the same -m/-k suffixes. -_SUFFIX_RE = re.compile(r"\s+(SKIP|TIMEOUT)\b.*$") - def _extract_test_id(line: str) -> Optional[str]: - """Extract the test identifier from a waives.txt line. + """Extract the normalized test identifier from a waives.txt line. - Returns None if the line doesn't look like a waive entry (empty, pure - comment, etc). + Returns None if the line doesn't look like a waive entry (empty / pure + comment line). Trailing `SKIP`/`TIMEOUT` annotations, `# comment`s, and + leading `full:/` prefix are stripped via `normalize_test_id` so the + result matches the same key used by `YAMLIndex`. """ s = line.strip() if not s or s.startswith("#"): return None - # Drop the "SKIP ..." / "TIMEOUT ..." trailing annotation if present. - s = _SUFFIX_RE.sub("", s).strip() - # Drop trailing " # comment" if any. - if "#" in s: - s = s.split("#", 1)[0].strip() - if not s: - return None - return s - - -def _strip_prefix(test_id: str) -> str: - """Strip leading "full:/" prefix if any.""" - return _PREFIX_RE.sub("", test_id) + s = normalize_test_id(s) + return s or None def parse_waives_diff(diff: str) -> tuple[set[str], set[str]]: """Parse a unified diff of waives.txt. - Returns (added, removed) sets of test identifiers, with "full:..." prefixes - stripped so they can match YAML entries directly. + Returns (added, removed) sets of normalized test identifiers ready to look + up against `YAMLIndex.blocks_containing_test`. """ added: set[str] = set() removed: set[str] = set() @@ -79,7 +56,6 @@ def parse_waives_diff(diff: str) -> tuple[set[str], set[str]]: tid = _extract_test_id(body) if tid is None: continue - tid = _strip_prefix(tid) (added if sign == "+" else removed).add(tid) return added, removed From 0f4d1eda6f9a44561639f76a7d0dace9a317167a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:15:53 +0800 Subject: [PATCH 19/65] [None][fix] CBTS: reset cpu_arch on unfamiliar Groovy map openings Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index d26e81423d53..409ef1d32055 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -242,9 +242,10 @@ def parse_stages_from_groovy( for line in groovy_path.read_text().splitlines(): open_match = _MAP_OPEN_RE.search(line.rstrip()) if open_match: - detected = _classify_map_var(open_match.group("var")) - if detected is not None: - current_arch = detected + # Reset on every map opening — including unfamiliar ones — so a + # later map between classified sections can't inherit a stale + # arch. Unknown maps fall back to the stage-name heuristic below. + current_arch = _classify_map_var(open_match.group("var")) m = _STAGE_ENTRY_RE.search(line) if not m: From 27e17d78d0c918740e19f10181841906cc1b889f Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:35:23 +0800 Subject: [PATCH 20/65] [None][chore] CBTS: make Groovy consumers scope-agnostic so new rules need no Groovy changes Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 15 +++++++++------ jenkins/L0_Test.groovy | 9 +++++---- jenkins/scripts/cbts/README.md | 22 ++++++++++++---------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c3dea06d1594..04a159487a5a 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1234,10 +1234,12 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) }, "x86_64-Linux": { script { - // CBTS Layer 1: skip entire x86 track when no x86 stages are affected + // CBTS Layer 1: skip entire x86 track when no x86 stages are affected. + // Scope-agnostic: any non-null cbts result means CBTS produced a decision; + // we trust its affected_cpu_arch regardless of which rule fired. def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && !("x86" in cbts.affected_cpu_arch)) { - echo "CBTS waiveonly: no x86 stages affected, skipping x86_64-Linux track" + if (cbts != null && !("x86" in cbts.affected_cpu_arch)) { + echo "CBTS [${cbts.scope}]: no x86 stages affected, skipping x86_64-Linux track" return } def testStageName = "[Build-x86_64] Remote Run" @@ -1351,10 +1353,11 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "SBSA build job is skipped due to Jenkins configuration or conditional pipeline run" return } - // CBTS Layer 1: skip entire SBSA track when no sbsa stages are affected + // CBTS Layer 1: skip entire SBSA track when no sbsa stages are affected. + // Scope-agnostic — see x86 track above for the rationale. def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && !("sbsa" in cbts.affected_cpu_arch)) { - echo "CBTS waiveonly: no sbsa stages affected, skipping SBSA-Linux track" + if (cbts != null && !("sbsa" in cbts.affected_cpu_arch)) { + echo "CBTS [${cbts.scope}]: no sbsa stages affected, skipping SBSA-Linux track" return } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 58185c1446f8..050611bcf196 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4416,9 +4416,10 @@ def launchTestJobs(pipeline, testFilter) // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all // existing filter rules so that unknown / no-decision paths fall through - // naturally. See jenkins/scripts/cbts/README.md for the two-layer model. + // naturally. Scope-agnostic: any non-null cbts result with affected_stages + // is treated as actionable. See jenkins/scripts/cbts/README.md. def cbts = testFilter[(CBTS_RESULT)] - if (cbts?.scope == "waiveonly" && cbts.affected_stages) { + if (cbts != null && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) } // Under `/bot run --post-merge`, keep only post-merge hits; if none, @@ -4427,9 +4428,9 @@ def launchTestJobs(pipeline, testFilter) // there, so reading it here is equivalent to "user passed --post-merge". if (testFilter[(IS_POST_MERGE)]) { parallelJobsFiltered = parallelJobsFiltered.findAll { it.key.contains("Post-Merge") } - echo "CBTS waiveonly (--post-merge): keeping ${parallelJobsFiltered.size()} affected post-merge stages" + echo "CBTS [${cbts.scope}] (--post-merge): keeping ${parallelJobsFiltered.size()} affected post-merge stages" } else { - echo "CBTS waiveonly: limiting to ${parallelJobsFiltered.size()} affected stages" + echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} affected stages" } } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 42d46bea4002..0f02d1e59467 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -128,16 +128,18 @@ Python stdout is a JSON blob: - Add the class to `RULE_CLASSES` (used by `--list-needed-diffs`). - Add an instance to `build_rules()` with its dependencies. -3. **Decide Layer 1 / 2 behavior for your scope in Groovy**. Each layer's - consumer checks `cbts.scope == "waiveonly"` explicitly. For a new scope: - - Add an `if (cbts.scope == "myscope")` branch in `L0_Test.groovy` - Layer 2 override (or leave unspecified → behaves as fallback / full run). - - Similarly decide Layer 1 (arch track skip). - - If your scope genuinely needs within-stage test filtering, add that - logic at `L0_Test.groovy:2674` (currently only a comment; `waiveonly` - deliberately skips this, see the rationale in the first section). - - **Defaults are conservative**: without explicit branches, new scope - paths fall through to the existing filter chain, which is safe. +3. **No Groovy changes needed**. Layer 1 (arch track skip) and Layer 2 + (stage filter) consume `affected_cpu_arch` / `affected_stages` + regardless of `scope` — the label is propagated to logs but does not + gate behavior. Empty `affected_stages` falls through to the existing + filter chain (safe default). + + Exceptions that still require Groovy edits: + - **Within-stage test filtering**: if your rule needs to drop + individual tests inside a stage rather than dropping whole stages, + add that logic at `L0_Test.groovy:2674` (currently only a comment). + `waiveonly` deliberately skips this — see the first section for the + rationale. Rule ordering doesn't matter. Rules independently decide whether they apply; `Selector` combines their `affected_stages` via union and their scopes via From 6f101ffce2c187bce0f4c23af1b5e7c46ec1e70b Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:36:43 +0800 Subject: [PATCH 21/65] [None][chore] CBTS: support Ant glob in needs_diff_for so future rules can match file globs Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 04a159487a5a..77517ccc7367 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -787,12 +787,20 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// v0 needs_diff_for entries are exact file paths. When a future rule adds -// a real glob (e.g. "tests/integration/defs/**/*.py"), extend this to use -// Ant-style matching (hudson.util.AntPathMatcher). +// Match a changed file path against a rule's needs_diff_for patterns using +// Ant-style globs (hudson.util.AntPathMatcher). Examples: +// "tests/integration/test_lists/waives.txt" - exact path +// "tests/integration/defs/**/*.py" - all py files under defs/ +// "cpp/tensorrt_llm/kernels/**" - any file under kernels/ +// Exact paths are still valid Ant patterns (matcher.match returns true on +// equal strings), so existing rules with literal-path needs_diff_for keep +// working without changes. +@Field +def _cbtsAntPathMatcher = new hudson.util.AntPathMatcher() + def _cbtsMatchesAnyPattern(String filePath, List patterns) { - return patterns.contains(filePath) + return patterns.any { _cbtsAntPathMatcher.match(it, filePath) } } // Return a list of stage-selection flags the user set via `/bot run` (empty From f2e650219ccc7ebdd0f73de1e8efe6286a989f97 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:48:28 +0800 Subject: [PATCH 22/65] [None][doc] CBTS: tighten READMEs and reflect scope-agnostic + Ant glob changes Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 143 +++++++++++---------------- jenkins/scripts/cbts/rules/README.md | 2 +- 2 files changed, 57 insertions(+), 88 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 0f02d1e59467..fe6a92e6b235 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -1,36 +1,32 @@ # CBTS — Change-Based Testing Selection Pre-merge CI test-selection tool. Looks at what the PR changed and narrows the -set of Jenkins stages that actually need to run. +set of Jenkins stages that actually need to run. **Adding new rules is +Python-only — Layer 1/2 in Groovy are scope-agnostic and consume the data +directly.** --- -## What it does — two layers +## Two consumption layers -Given a PR, CBTS produces a decision that gets consumed at two points in the -Jenkins pipeline: - -| Layer | Where consumed | Action | +| Layer | Where | Action | |---|---|---| -| **1. Arch track** | `L0_MergeRequest.groovy::launchStages` (each track entry) | Skip whole x86 / SBSA track (build + all tests) when no stage on that arch is affected | +| **1. Arch track** | `L0_MergeRequest.groovy::launchStages` | Skip x86 / SBSA track when no stage on that arch is affected | | **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset | -Anything CBTS can't confidently narrow → **fallback to the existing full filter -chain**. CBTS never adds stages; it only subtracts. +CBTS only **subtracts** stages, never adds. Anything it can't narrow → full +fallback to the existing filter chain. -**No within-stage test filtering by design.** Once Layer 2 picks the affected -stages, each stage runs its full rendered testDBList (all blocks matching the -stage's mako). Running the whole block, not just the single changed test id, -is deliberately over-inclusive — if a waive is wrong (depends on another test, -or the node id has a typo that silently matches nothing), running only the -changed test would not surface the problem. The extra per-stage test time is -accepted as the cost of CI robustness. +**No within-stage test filtering by design.** Each picked stage runs its full +testDBList. Filtering down to just the changed test would mask wrong waives +(broken deps, typo'd node ids silently matching nothing) — the extra per-stage +time buys robustness. ## v0 scope -- **Only handles** `tests/integration/test_lists/waives.txt` changes -- Any other changed file → CBTS returns `scope: none` → full run -- Scope label for this case: `waiveonly` +- **Only handles** `tests/integration/test_lists/waives.txt` changes (`scope: waiveonly`). +- Anything else → `scope: none` → full run. +- **v1+ rules can be added in Python alone** (no Groovy edits). ## File map @@ -38,49 +34,34 @@ accepted as the cost of CI robustness. jenkins/scripts/cbts/ ├── README.md this file ├── main.py CLI entry + Selector + SelectionResult -├── blocks.py YAML loading + stage parsing from groovy + condition matching +├── blocks.py YAML loading + stage parsing + test-id normalization └── rules/ - ├── README.md per-rule logic summary (scope, triggers, matching) + ├── README.md per-rule logic summary ├── base.py Rule ABC + PRInputs + RuleResult └── waives_rule.py v0's only rule ``` ## When CBTS activates -CBTS activates when the user runs `/bot run` without a stage-selection -flag. If the user provides any of `--stage-list`, `--extra-stage`, -`--gpu-type`, `--backend-mode`, `--skip-test`, `--add-multi-gpu-test`, -`--only-multi-gpu-test`, or `--disable-multi-gpu-test`, `getCbtsResult` -returns `null` immediately and the existing filter chain drives test -selection exactly as before. - -**Not considered user flags** (CBTS still activates): -- `--reuse-test` / `--reuse-stage-list` — retry semantics; the bot - auto-populates these on re-runs of the same PR. They compose fine with - CBTS (CBTS picks stages, reuse further skips stages that already passed). -- `--debug` / `--detailed-log` — logging verbosity; orthogonal. -- `--post-merge` — scopes the run to post-merge stages. CBTS still - activates; Layer 2 then narrows the affected set to post-merge hits - only. If no post-merge stage is hit, the run is a no-op (no fallback - to the full post-merge baseline). - -This matches the convention already used by `enableUpdateGitlabStatus` in -`L0_MergeRequest.groovy` for distinguishing "default run" from -"user-customized run". - -## How it's invoked (CI) +Bare `/bot run`. The following stage-selection flags make `getCbtsResult` +return `null` and let the existing filter chain take over: `--stage-list`, +`--extra-stage`, `--gpu-type`, `--backend-mode`, `--skip-test`, +`--add-multi-gpu-test`, `--only-multi-gpu-test`, `--disable-multi-gpu-test`. -`L0_MergeRequest.groovy::getCbtsResult` orchestrates two calls to `main.py`: +**Compatible** (CBTS still activates): +- `--reuse-test` / `--reuse-stage-list` — auto-populated by the bot on re-runs. +- `--debug` / `--detailed-log` — logging only, orthogonal. +- `--post-merge` — Layer 2 narrows the affected set to post-merge hits only. + No post-merge hit → no-op (no fallback to full post-merge baseline). -1. `python3 main.py --list-needed-diffs` — returns `needs_diff_for` patterns - so Groovy knows which changed files to fetch diffs for (via the existing - `getMergeRequestOneFileChanges` API helper). -2. `python3 main.py cbts_input.json` — returns the decision on stdout. +## How it's invoked (CI) -The decision is cached in `testFilter[CBTS_RESULT]` and serialized into the -child job's `testFilter` param alongside the existing filter flags. +`getCbtsResult` calls `main.py` twice: -Python stdout is a JSON blob: +1. `main.py --list-needed-diffs` → patterns whose diffs Groovy fetches. + Patterns are **Ant-style globs** (`tests/**/*.py`, `cpp/kernels/**`, exact + paths), matched via `hudson.util.AntPathMatcher`. +2. `main.py cbts_input.json` → decision JSON on stdout: ```json { @@ -92,7 +73,8 @@ Python stdout is a JSON blob: } ``` -`scope: null` means "no decision, fall back to the existing filter chain". +`scope: null` → no decision, full fallback. Groovy doesn't gate on the scope +value — it's metadata for logs and multi-rule combining only. ## Adding a new rule @@ -105,62 +87,49 @@ Python stdout is a JSON blob: class MyRule(Rule): name = "myrule" - needs_diff_for = ["path/or/glob/**/*.py"] # files whose diffs you need + needs_diff_for = ("tests/**/*.py",) # Ant globs; tuple per RUF012 def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]): self.yaml_index = yaml_index self.stages = stages def apply(self, pr: PRInputs) -> Optional[RuleResult]: - # Return None if the rule doesn't apply to this PR. - # Return a RuleResult otherwise. ... return RuleResult( - handled_files={...}, # files you claim - tests={...}, # changed test ids (logged; not filtered at stage time) - affected_stages={...}, # Layer 2 stage set - scope="myscope", # your scope label - reason="why this was picked", + handled_files={...}, + tests={...}, + affected_stages={...}, + scope="myscope", + reason="why this fired", ) ``` -2. **Register in `main.py`**: - - Add the class to `RULE_CLASSES` (used by `--list-needed-diffs`). - - Add an instance to `build_rules()` with its dependencies. - -3. **No Groovy changes needed**. Layer 1 (arch track skip) and Layer 2 - (stage filter) consume `affected_cpu_arch` / `affected_stages` - regardless of `scope` — the label is propagated to logs but does not - gate behavior. Empty `affected_stages` falls through to the existing - filter chain (safe default). +2. **Register in `main.py`**: add to `RULE_CLASSES` and `build_rules()`. - Exceptions that still require Groovy edits: - - **Within-stage test filtering**: if your rule needs to drop - individual tests inside a stage rather than dropping whole stages, - add that logic at `L0_Test.groovy:2674` (currently only a comment). - `waiveonly` deliberately skips this — see the first section for the - rationale. +3. **No Groovy edits needed.** Exception: if your rule needs to drop + *individual tests inside* a stage (vs whole stages), add the hook at + `L0_Test.groovy:2674` — but `waiveonly` deliberately skips this, see + the "no within-stage filtering" note above. -Rule ordering doesn't matter. Rules independently decide whether they apply; -`Selector` combines their `affected_stages` via union and their scopes via -`_combine_scopes` (agreement → that scope; disagreement → `None`). +Rule order is irrelevant. `Selector` unions `affected_stages`; scopes are +combined via `_combine_scopes` (all-agree → that scope; disagreement → `None`). -## Fallback / safety paths +## Fallback paths -CBTS falls back to the existing filter chain (as if it weren't there) when: +CBTS falls back to the existing filter chain when: - PostMerge job / `alternativeTRT` set - `changed_files` is empty - `main.py` throws / stdout is unparsable -- `scope == none` (Python's explicit "no decision" output) -- Groovy sees an unknown scope value (forward compatibility) +- Python returns `scope: null` ("no decision") +- `affected_stages` is empty (Layer 2 no-op) -No silent failures: every fallback logs an `echo` line in the CI console. +Every fallback logs an `echo` line — no silent failures. ## Keep-in-sync notes `blocks.py::derive_mako_from_stage` mirrors the Groovy -`getMakoArgsFromStageName` (in `jenkins/L0_Test.groovy` ~line 2079) and -`parseTaskConfigFromStageName` (~line 2066). When new backends / -orchestrators / stage-name conventions are added on the Groovy side, update -the Python constants here too. The file comments flag this explicitly. +`getMakoArgsFromStageName` (`L0_Test.groovy` ~line 2079) and +`parseTaskConfigFromStageName` (~line 2066). New backends / orchestrators / +stage-name conventions on the Groovy side need a matching Python update — +file comments flag this. diff --git a/jenkins/scripts/cbts/rules/README.md b/jenkins/scripts/cbts/rules/README.md index 4671c99eb407..7cdb66a1aa8b 100644 --- a/jenkins/scripts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/rules/README.md @@ -7,4 +7,4 @@ top-level [README](../README.md) for the overall CBTS architecture. | File | Class | Scope | Triggers on | What it picks | |---|---|---|---|---| -| `waives_rule.py` | `WaivesRule` | `waiveonly` | PR changes `tests/integration/test_lists/waives.txt` | For each added/removed test id in the diff: look it up in the test-db YAML, pick stages whose `mako` matches the containing block's `condition`. | +| `waives_rule.py` | `WaivesRule` | `waiveonly` | PR changes `tests/integration/test_lists/waives.txt` | For each added/removed test id (after `blocks.normalize_test_id` strips `SKIP`/`TIMEOUT`/`full:/` decorations): look it up in the test-db YAML, pick stages whose `mako` matches the containing block's `condition`. | From 99dd20bdefb2017ee2e1be4242c580a350a11ab6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:04:37 +0800 Subject: [PATCH 23/65] [None][chore] CBTS: simplify defer-flag list and clarify two supported usages Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 48 +++++++++++----------------------- jenkins/L0_Test.groovy | 9 ------- jenkins/scripts/cbts/README.md | 25 +++++++++++------- 3 files changed, 30 insertions(+), 52 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 77517ccc7367..7328930db0ac 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -803,41 +803,23 @@ def _cbtsMatchesAnyPattern(String filePath, List patterns) return patterns.any { _cbtsAntPathMatcher.match(it, filePath) } } -// Return a list of stage-selection flags the user set via `/bot run` (empty -// list means bare run). CBTS defers entirely when this is non-empty. -// Excluded on purpose (match the convention used by `enableUpdateGitlabStatus` -// above): -// - REUSE_TEST / REUSE_STAGE_LIST: retry semantics, auto-populated by the -// bot on re-runs; compose fine with CBTS. -// - DEBUG_MODE / DETAILED_LOG: logging verbosity, orthogonal to selection. +// CBTS only activates on `/bot run` and `/bot run --post-merge`. Any other +// stage-selection flag makes it defer to the user's explicit choice. +// Orthogonal flags (REUSE_*, DEBUG_MODE, DETAILED_LOG) and IS_POST_MERGE are +// intentionally not in this list — they either don't affect stage selection +// or are handled specially in Layer 2. Adding a new stage-selection flag in +// the future means adding one entry here; nothing else changes. +// +// All defer flags default to falsy (false / null), so a single truthy check +// captures both boolean and list-typed flags. def _cbtsTriggeredUserFlags(testFilter) { - def flags = [] - if (testFilter[(ENABLE_SKIP_TEST)]) { - flags << "ENABLE_SKIP_TEST=${testFilter[(ENABLE_SKIP_TEST)]}" - } - if (testFilter[(TEST_STAGE_LIST)] != null) { - flags << "TEST_STAGE_LIST=${testFilter[(TEST_STAGE_LIST)]}" - } - if (testFilter[(EXTRA_STAGE_LIST)] != null) { - flags << "EXTRA_STAGE_LIST=${testFilter[(EXTRA_STAGE_LIST)]}" - } - if (testFilter[(GPU_TYPE_LIST)] != null) { - flags << "GPU_TYPE_LIST=${testFilter[(GPU_TYPE_LIST)]}" - } - if (testFilter[(TEST_BACKEND)] != null) { - flags << "TEST_BACKEND=${testFilter[(TEST_BACKEND)]}" - } - if (testFilter[(ADD_MULTI_GPU_TEST)]) { - flags << "ADD_MULTI_GPU_TEST=${testFilter[(ADD_MULTI_GPU_TEST)]}" - } - if (testFilter[(ONLY_MULTI_GPU_TEST)]) { - flags << "ONLY_MULTI_GPU_TEST=${testFilter[(ONLY_MULTI_GPU_TEST)]}" - } - if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { - flags << "DISABLE_MULTI_GPU_TEST=${testFilter[(DISABLE_MULTI_GPU_TEST)]}" - } - return flags + def deferFlags = [ + ENABLE_SKIP_TEST, TEST_STAGE_LIST, EXTRA_STAGE_LIST, GPU_TYPE_LIST, + TEST_BACKEND, ADD_MULTI_GPU_TEST, ONLY_MULTI_GPU_TEST, DISABLE_MULTI_GPU_TEST, + ] + return deferFlags.findAll { testFilter[it] } + .collect { "${it}=${testFilter[it]}" } } // Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3. Always diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 050611bcf196..8c7994a92189 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3094,15 +3094,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt") } - // NOTE: CBTS intentionally does NOT filter testDBList here. Layer 2 has - // already narrowed stages to those whose mako matches an affected block's - // condition; within each such stage we run the FULL rendered testDBList - // (all blocks matching the stage's mako) rather than restricting to the - // specific changed test ids. This over-includes by design: if a waive is - // wrong (e.g. depends on other tests, or the node id has a typo), running - // only the single changed test would not surface the problem. The extra - // per-stage test time is accepted as the cost of CI robustness. - // Process shard test list and create separate files for regular and isolate tests def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index fe6a92e6b235..ca6f240cbd31 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -43,16 +43,21 @@ jenkins/scripts/cbts/ ## When CBTS activates -Bare `/bot run`. The following stage-selection flags make `getCbtsResult` -return `null` and let the existing filter chain take over: `--stage-list`, -`--extra-stage`, `--gpu-type`, `--backend-mode`, `--skip-test`, -`--add-multi-gpu-test`, `--only-multi-gpu-test`, `--disable-multi-gpu-test`. - -**Compatible** (CBTS still activates): -- `--reuse-test` / `--reuse-stage-list` — auto-populated by the bot on re-runs. -- `--debug` / `--detailed-log` — logging only, orthogonal. -- `--post-merge` — Layer 2 narrows the affected set to post-merge hits only. - No post-merge hit → no-op (no fallback to full post-merge baseline). +CBTS narrows test selection in **two usages only**: + +- `/bot run` — full pre-merge with CBTS narrowing. +- `/bot run --post-merge` — post-merge with CBTS narrowing. Layer 2 keeps + only post-merge hits; no post-merge hit → no-op (no fallback to full + post-merge baseline). + +Any other **stage-selection** flag makes `getCbtsResult` return `null` and +the existing filter chain takes over: `--stage-list`, `--extra-stage`, +`--gpu-type`, `--test-backend`, `--skip-test`, `--add-multi-gpu-test`, +`--only-multi-gpu-test`, `--disable-multi-gpu-test`. + +**Orthogonal** flags don't change stage selection and don't affect CBTS: +`--reuse-test`, `--disable-reuse-test`, `--debug`, `--detailed-log`, +`--disable-fail-fast`, `--high-priority`. ## How it's invoked (CI) From 0ad62102c40418055a40f44f4eea33bb6122b179 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:20:44 +0800 Subject: [PATCH 24/65] [None][fix] CBTS: include post-merge stages in main.py so post-merge-only waives resolve Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index a05504394ab0..0b9272af44be 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -229,7 +229,10 @@ def main(argv: Optional[list[str]] = None) -> int: return 2 yaml_index = YAMLIndex.load(test_db_dir) - stages = parse_stages_from_groovy(groovy_path) + # Include post-merge stages so waives on post-merge-only tests resolve. + # Layer 2 in L0_Test.groovy decides what to run with the post-merge + # subset based on the user's --post-merge flag. + stages = parse_stages_from_groovy(groovy_path, include_post_merge=True) pr = _load_pr_inputs(input_path) rules = build_rules(yaml_index, stages) result = Selector(stages).run(pr, rules) From d5696691f6e1367a0d0d833687659e997cb9b7b3 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:57:41 +0800 Subject: [PATCH 25/65] [None][fix] CBTS: implement Ant glob in pure Groovy (hudson.util.AntPathMatcher unavailable in Jenkins sandbox) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 37 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 7328930db0ac..79f0a61d6c55 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -787,20 +787,37 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Match a changed file path against a rule's needs_diff_for patterns using -// Ant-style globs (hudson.util.AntPathMatcher). Examples: -// "tests/integration/test_lists/waives.txt" - exact path -// "tests/integration/defs/**/*.py" - all py files under defs/ -// "cpp/tensorrt_llm/kernels/**" - any file under kernels/ -// Exact paths are still valid Ant patterns (matcher.match returns true on -// equal strings), so existing rules with literal-path needs_diff_for keep +// Translate an Ant-style glob to a regex. +// **/ zero or more path segments +// ** any chars (including /) +// * any chars except / +// ? single char except / +// Implemented in pure Groovy — `hudson.util.AntPathMatcher` is not visible +// to the Jenkins script sandbox classpath. Exact paths (no glob meta) round- +// trip to a literal regex, so existing rules with literal needs_diff_for keep // working without changes. -@Field -def _cbtsAntPathMatcher = new hudson.util.AntPathMatcher() +def _cbtsGlobToRegex(String glob) +{ + // 1. Escape regex specials, except glob metas (* and ?) which we handle below. + def escaped = glob.collect { c -> + (c == '*' || c == '?') ? c + : ('.+()[]{}|^$\\'.contains(c) ? '\\' + c : c) + }.join('') + // 2. Translate glob metas. Use unambiguous text sentinels so cascading + // replaces don't double-match (Jenkins sandbox struggles with \u-escaped + // control chars in some Groovy versions). + return '^' + escaped + .replace('**/', '__CBTSDOUBLESLASH__') + .replace('**', '__CBTSDOUBLESTAR__') + .replace('*', '[^/]*') + .replace('?', '[^/]') + .replace('__CBTSDOUBLESLASH__', '(?:.*/)?') + .replace('__CBTSDOUBLESTAR__', '.*') + '$' +} def _cbtsMatchesAnyPattern(String filePath, List patterns) { - return patterns.any { _cbtsAntPathMatcher.match(it, filePath) } + return patterns.any { filePath ==~ _cbtsGlobToRegex(it) } } // CBTS only activates on `/bot run` and `/bot run --post-merge`. Any other From 7a2b36eb81ff8e9778a1c91aaff7bfc711f2315c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:12:35 +0800 Subject: [PATCH 26/65] =?UTF-8?q?[None][fix]=20CBTS:=20avoid=20\u=20in=20c?= =?UTF-8?q?omment=20=E2=80=94=20Groovy=20lexer=20treats=20it=20as=20unicod?= =?UTF-8?q?e=20escape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 79f0a61d6c55..219bff324a84 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -804,8 +804,8 @@ def _cbtsGlobToRegex(String glob) : ('.+()[]{}|^$\\'.contains(c) ? '\\' + c : c) }.join('') // 2. Translate glob metas. Use unambiguous text sentinels so cascading - // replaces don't double-match (Jenkins sandbox struggles with \u-escaped - // control chars in some Groovy versions). + // replaces don't double-match. Avoid unicode-escape placeholders + // because the Groovy lexer expands those even inside string literals. return '^' + escaped .replace('**/', '__CBTSDOUBLESLASH__') .replace('**', '__CBTSDOUBLESTAR__') From ed522e3eca5cfd1b50bb447bff2a341a3e549242 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 28 Apr 2026 10:05:15 +0800 Subject: [PATCH 27/65] [None][chore] CBTS: revert testing artifacts; WaivesRule fires only when waives.txt is the lone changed file Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/rules/waives_rule.py | 10 ++-------- tests/integration/test_lists/waives.txt | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index bf5cfa0253a0..e1cea98bb301 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -81,10 +81,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: changed_test_ids = added | removed if not changed_test_ids: return RuleResult( - # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim all - # changed files so CBTS fires on the cbts-v0 PR that also edits - # CBTS infra files. - handled_files=set(pr.changed_files), + handled_files={WAIVES_FILE}, tests=set(), affected_stages=set(), scope="waiveonly", @@ -107,10 +104,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: affected_stage_names.add(stage_name) return RuleResult( - # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim all - # changed files so CBTS fires on the cbts-v0 PR that also edits - # CBTS infra files. - handled_files=set(pr.changed_files), + handled_files={WAIVES_FILE}, tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6a0f612c897e..cc1298fe2904 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -265,7 +265,7 @@ full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-floa full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:512-input_output_len:128,128-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) triton_server/test_triton.py::test_gpt_2b_ib_lora[gpt-2b-ib-lora] SKIP (https://nvbugs/5470830) -unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781, touched for CBTS validation) +unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781) triton_server/test_triton.py::test_llava[llava] SKIP (https://nvbugs/5547414) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5948435) From 747608bbefbf2af0acc6561afdb1f647df2d0a00 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 28 Apr 2026 22:05:35 +0800 Subject: [PATCH 28/65] [None][fix] CBTS: parent-chain test-id lookup + scope=None safety net for empty stages Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 81 ++++++++++++++++++++++++++++++---- jenkins/scripts/cbts/main.py | 15 +++++++ 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 409ef1d32055..f1dbdbb8e6da 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -72,6 +72,51 @@ def normalize_test_id(test_id: str) -> str: return _TEST_ID_PREFIX_RE.sub("", s) +# Detect a pytest-style option flag (e.g. ` -k`, ` -m`) inside a YAML target +# spec. Used to peel `-k "..."` / `-m "..."` off so the bare path/node-id can +# be indexed as an additional lookup key — that lets a fine-grained waive id +# match a coarse YAML entry like `dir/x.py -k "deepseek"`. +_PYTEST_OPTION_RE = re.compile(r"\s+-[a-zA-Z]\b") + + +def _strip_pytest_options(s: str) -> str: + """Return `s` with any pytest option flag (and everything after it) removed. + + `dir/x.py -k "deepseek"` -> `dir/x.py` + `x.py::test_y -m "gpu1"` -> `x.py::test_y` + `x.py` -> `x.py` (unchanged) + """ + m = _PYTEST_OPTION_RE.search(s) + return s[: m.start()].rstrip() if m else s + + +def _iter_parent_ids(test_id: str): + """Yield ancestor forms of a pytest target id, most-specific to least. + + `dir/x.py::TestC::test_m[a-b]` yields: + `dir/x.py::TestC::test_m` (strip `[params]`) + `dir/x.py::TestC` (strip `::test_m`) + `dir/x.py` (strip `::TestC`) + `dir` (strip `/x.py`) + + Lets `blocks_containing_test` match coarser YAML entries (file, class, + directory) when the waive id is finer-grained. + """ + s = test_id + if "[" in s: + s = s.rsplit("[", 1)[0] + if s: + yield s + while "::" in s: + s = s.rsplit("::", 1)[0] + if s: + yield s + while "/" in s: + s = s.rsplit("/", 1)[0] + if s: + yield s + + class YAMLIndex: """Index of all blocks across test-db YAMLs, with reverse lookup by test id.""" @@ -101,17 +146,37 @@ def _load_one(self, yml_path: Path) -> None: tests=list(tests), ) self.blocks.append(block) - # Index each test under both its raw YAML string (which may carry - # ` -m "gpu2"`, ` TIMEOUT (90)`, etc.) and its normalized form, so - # waives.txt lookups — which strip SKIP/TIMEOUT — still resolve. + # Index each test under up to three keys so a waive id can match + # YAML entries written at any granularity: + # 1. raw YAML string (with TIMEOUT/markers as written) + # 2. normalized form (SKIP/TIMEOUT/full:gpu prefix stripped) + # 3. target-only form (pytest options like `-k "..."` stripped) + # Waive-side lookup walks the pytest node-id parent chain, so a + # coarse YAML entry (file / dir / `dir/x.py -k "kw"`) matches a + # fine-grained waive (`dir/x.py::TestC::test_m[params]`). for test in tests: - self._test_to_blocks.setdefault(test, []).append(block) - normalized = normalize_test_id(test) - if normalized and normalized != test: - self._test_to_blocks.setdefault(normalized, []).append(block) + seen: set[str] = set() + for key in ( + test, + normalize_test_id(test), + _strip_pytest_options(normalize_test_id(test)), + ): + if key and key not in seen: + seen.add(key) + self._test_to_blocks.setdefault(key, []).append(block) def blocks_containing_test(self, test_id: str) -> list[Block]: - return list(self._test_to_blocks.get(test_id, [])) + # Exact match first, then walk the pytest node-id parent chain to + # also catch coarser YAML entries (file / class / directory). + blocks = list(self._test_to_blocks.get(test_id, [])) + seen_block_keys = {(b.yaml_stem, b.block_index) for b in blocks} + for parent in _iter_parent_ids(test_id): + for b in self._test_to_blocks.get(parent, []): + key = (b.yaml_stem, b.block_index) + if key not in seen_block_keys: + seen_block_keys.add(key) + blocks.append(b) + return blocks def all_test_ids(self) -> Iterable[str]: return self._test_to_blocks.keys() diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 0b9272af44be..0d1b40ca0c8c 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -133,6 +133,21 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: affected_stages |= r.affected_stages tests |= r.tests + # Safety net: if rules fired but no Jenkins stages resolved (waive ids + # missed both exact and parent-chain lookups in the YAML index), fall + # back to baseline. Returning a non-None scope with empty stages would + # let Layer 1 in Groovy mistake `affected_cpu_arch=∅` for "no arch + # needed" and silently skip both x86 and SBSA tracks. + if not affected_stages: + return SelectionResult( + scope=None, + reasons=reasons + + [ + "Rules fired but no stages resolved (likely YAML/waive " + "granularity mismatch); falling back to baseline." + ], + ) + affected_cpu_arch = { self.stages[name].cpu_arch for name in affected_stages if name in self.stages } From 14db4044b782c816b9481722a1d3881ab00c5b4c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 09:28:25 +0800 Subject: [PATCH 29/65] [None][feat] CBTS Layer 3: within-stage test filtering via filtered tmp test-db Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 1 + jenkins/L0_Test.groovy | 8 + jenkins/scripts/cbts/blocks.py | 225 +++++++++++++++++++--- jenkins/scripts/cbts/main.py | 36 +++- jenkins/scripts/cbts/rules/base.py | 15 +- jenkins/scripts/cbts/rules/waives_rule.py | 31 ++- 6 files changed, 285 insertions(+), 31 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 219bff324a84..b446a9fc8711 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -851,6 +851,7 @@ def _cbtsParseSelectionResult(String text) affected_stages: data.affected_stages ?: [], affected_tests: data.tests ?: [], reasons: data.reasons ?: [], + test_db_dir_override: data.test_db_dir_override, // Layer 3: tmp test-db path ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 8c7994a92189..6470ff829d95 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2567,7 +2567,15 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" + // CBTS Layer 3: when CBTS provided a tmp test-db dir (each affected + // block's tests narrowed to entries in the per-block filter prefix + // subtree), point trt-test-db at it. Otherwise use the source test-db. + def cbts = testFilter[(CBTS_RESULT)] def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" + if (cbts != null && cbts.test_db_dir_override) { + testDBPath = "${llmSrc}/${cbts.test_db_dir_override}" + echo "CBTS [${cbts.scope}]: rendering test list from filtered test-db at ${testDBPath}" + } def testList = "${llmSrc}/${testContext}.txt" def testDBQueryCmd = [ "trt-test-db", diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index f1dbdbb8e6da..d35dc8dab5c1 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -99,8 +99,8 @@ def _iter_parent_ids(test_id: str): `dir/x.py` (strip `::TestC`) `dir` (strip `/x.py`) - Lets `blocks_containing_test` match coarser YAML entries (file, class, - directory) when the waive id is finer-grained. + Lets the lookup match coarser YAML entries (file, class, directory) when + the waive id is finer-grained. """ s = test_id if "[" in s: @@ -117,6 +117,78 @@ def _iter_parent_ids(test_id: str): yield s +# pytest's `-k` keyword expressions can be a single word like `"deepseek"` or +# a boolean expression like `"a and not b"`. We tokenize on identifier chars +# and strip the boolean operators, then check substring against the waive id. +# Over-includes for complex expressions; never under-includes. +_K_VALUE_RE = re.compile(r'\s+-k\s+"([^"]*)"') +_M_FLAG_RE = re.compile(r"\s+-m\b") +_K_RESERVED_WORDS = {"and", "or", "not"} + + +def _extract_k_keyword(entry: str) -> Optional[str]: + """Return the raw `-k ""` keyword text, or None.""" + m = _K_VALUE_RE.search(entry) + return m.group(1) if m else None + + +def _entry_applies_to_waive(entry: str, waive_id: str) -> bool: + """Best-effort check: would this entry's pytest options run the waived test. + + Skips ancestor-level entries whose `-k` filter excludes the waive (e.g. + `file.py -k "weights"` shouldn't match a `test_biases[a]` waive). + + `-k ""`: extract identifier tokens, drop boolean operators, accept + if any remaining token appears in waive id. + `-m ""`: always accept (markers are runtime metadata, can't verify + from the test id string alone). + No options: always accept. + """ + kw = _extract_k_keyword(entry) + if kw is None: + return True # no -k → unconstrained + tokens = re.findall(r"[A-Za-z_]\w*", kw) + real = [t for t in tokens if t.lower() not in _K_RESERVED_WORDS] + if not real: + return True # all-reserved expression → can't decide; over-include + return any(t in waive_id for t in real) + + +def _strip_params(s: str) -> str: + """Strip `[params]` suffix if present. + + `file.py::test_x[a]` → `file.py::test_x` + `file.py::test_x` → `file.py::test_x` + """ + return s.rsplit("[", 1)[0] if "[" in s else s + + +def _entry_target(entry: str) -> str: + """Canonical "target" key for indexing/lookup. + + Strips SKIP/TIMEOUT/full:gpu, pytest options (-k/-m), and `[params]` suffix. + + `file.py::TestC::test_m[a-b] TIMEOUT (90)` → `file.py::TestC::test_m` + `file.py -k "kw"` → `file.py` + """ + return _strip_params(_strip_pytest_options(normalize_test_id(entry))) + + +def _target_in_filter_subtree(target: str, filter_prefix: str) -> bool: + """True iff `target` is in `filter_prefix`'s subtree. + + `target` matches when it is `filter_prefix` itself or a descendant of it + (params / method / file / dir component below) in the pytest tree. + """ + if target == filter_prefix: + return True + return ( + target.startswith(filter_prefix + "[") + or target.startswith(filter_prefix + "::") + or target.startswith(filter_prefix + "/") + ) + + class YAMLIndex: """Index of all blocks across test-db YAMLs, with reverse lookup by test id.""" @@ -146,37 +218,77 @@ def _load_one(self, yml_path: Path) -> None: tests=list(tests), ) self.blocks.append(block) - # Index each test under up to three keys so a waive id can match + # Index each test under up to four keys so a waive id can match # YAML entries written at any granularity: - # 1. raw YAML string (with TIMEOUT/markers as written) - # 2. normalized form (SKIP/TIMEOUT/full:gpu prefix stripped) - # 3. target-only form (pytest options like `-k "..."` stripped) - # Waive-side lookup walks the pytest node-id parent chain, so a - # coarse YAML entry (file / dir / `dir/x.py -k "kw"`) matches a - # fine-grained waive (`dir/x.py::TestC::test_m[params]`). + # 1. raw YAML string (with TIMEOUT/markers as written) + # 2. normalized form (SKIP/TIMEOUT/full:gpu prefix stripped) + # 3. target-w/-options (pytest options `-k "..."` stripped) + # 4. canonical target (also `[params]` stripped — the level + # the lookup walks against) + # Waive-side lookup walks the pytest node-id parent chain. A + # coarse YAML entry (file / dir / `dir/x.py -k "kw"`) and a fine + # waive (`dir/x.py::TestC::test_m[params]`) meet in the middle + # via key #4. for test in tests: seen: set[str] = set() + norm = normalize_test_id(test) for key in ( test, - normalize_test_id(test), - _strip_pytest_options(normalize_test_id(test)), + norm, + _strip_pytest_options(norm), + _strip_params(_strip_pytest_options(norm)), ): if key and key not in seen: seen.add(key) self._test_to_blocks.setdefault(key, []).append(block) - def blocks_containing_test(self, test_id: str) -> list[Block]: - # Exact match first, then walk the pytest node-id parent chain to - # also catch coarser YAML entries (file / class / directory). - blocks = list(self._test_to_blocks.get(test_id, [])) - seen_block_keys = {(b.yaml_stem, b.block_index) for b in blocks} - for parent in _iter_parent_ids(test_id): - for b in self._test_to_blocks.get(parent, []): - key = (b.yaml_stem, b.block_index) - if key not in seen_block_keys: - seen_block_keys.add(key) - blocks.append(b) - return blocks + def find_match_for_waive(self, waive_id: str) -> Optional[tuple[str, list[Block]]]: + """Walk up the pytest parent chain; first level with a match wins. + + Returns (level, blocks) for the first level whose YAML index has at + least one matching entry, or None when even the root level misses. + + Detail: + (level, blocks): `level` is the YAML target string where the match + was found; `blocks` are the affected blocks at + that level. + None: no level matched all the way to the root — + caller should treat as fallback (CBTS exits, + baseline runs). + + An entry "matches" at a level when its target (after stripping pytest + options) equals the level AND its `-k` filter (if any) actually applies + to the waive id. + """ + # Step 1+2: normalize the waive id, strip [params] to get the start. + # `file.py::TestC::test_m[a-b]` → `file.py::TestC::test_m` + # `file.py::TestC::test_m` → unchanged (already at function level) + # `file.py::TestC` → unchanged (class) + # `file.py` → unchanged (file) + target = _strip_pytest_options(normalize_test_id(waive_id)) + if "[" in target: + target = target.rsplit("[", 1)[0] + + # Step 3: walk up until something matches. + for level in [target, *_iter_parent_ids(target)]: + candidates = self._test_to_blocks.get(level, []) + matched: list[Block] = [] + seen_keys: set[tuple[str, int]] = set() + for block in candidates: + key = (block.yaml_stem, block.block_index) + if key in seen_keys: + continue + # Verify at least one entry in this block has canonical + # target == level AND its -k constraint (if any) applies to + # the waive. Canonical = strip pytest options and [params]. + for raw in block.tests: + if _entry_target(raw) == level and _entry_applies_to_waive(raw, waive_id): + matched.append(block) + seen_keys.add(key) + break + if matched: + return level, matched + return None def all_test_ids(self) -> Iterable[str]: return self._test_to_blocks.keys() @@ -390,3 +502,70 @@ def block_matches_stage(block: Block, stage: Stage) -> bool: return False return True + + +# --------------------------------------------------------------------------- +# CBTS Layer 3: filtered test-db YAML generation +# --------------------------------------------------------------------------- + + +def write_filtered_test_db( + src_dir: Path, + output_dir: Path, + block_filters: dict[tuple[str, int], set[str]], +) -> None: + """Generate a tmp test-db dir narrowed by CBTS Layer 3. + + Contains only the YAMLs whose blocks were affected, with each affected + block's `tests:` array filtered to entries in the per-block filter prefix + subtree. + + Layer 3 narrowing: trt-test-db invoked with `-d ` will produce + a smaller testDBList for each affected stage, since the YAML it reads has + fewer tests in the matched block. + + `block_filters` keys: (yaml_stem, block_index) of affected blocks. + Values: set of filter prefix strings that should keep tests in their + subtree. + + Unaffected blocks (not in `block_filters`) are written through unchanged. + Unaffected YAML files are NOT written — only stages whose YAML appears in + `block_filters` will be running anyway (Layer 2 already filtered). + + Safety: if filtering would empty a block's tests, the original tests are + kept (prevents silent skip when a YAML/waive granularity mismatch slips + through). + """ + output_dir.mkdir(parents=True, exist_ok=True) + + affected_stems = {stem for stem, _ in block_filters} + for stem in sorted(affected_stems): + src = src_dir / f"{stem}.yml" + if not src.exists(): + continue + data = yaml.safe_load(src.read_text()) or {} + + for ctx_blocks in data.values(): + if not isinstance(ctx_blocks, list): + continue + for i, block_data in enumerate(ctx_blocks): + if not isinstance(block_data, dict): + continue + key = (stem, i) + if key not in block_filters: + continue + original = block_data.get("tests") or [] + filters = block_filters[key] + kept = [ + t + for t in original + if any(_target_in_filter_subtree(_entry_target(t), f) for f in filters) + ] + # Safety: empty filter result → fallback to original (prevents + # silent skip from typo'd waive ids or granularity mismatch). + if kept: + block_data["tests"] = kept + + (output_dir / src.name).write_text( + yaml.safe_dump(data, sort_keys=False, default_flow_style=False) + ) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 0d1b40ca0c8c..b5b99b8b7eaa 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -47,7 +47,7 @@ # Make sibling modules importable when invoked as `python3 /main.py ...`. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from blocks import Stage, YAMLIndex, parse_stages_from_groovy # noqa: E402 +from blocks import Stage, YAMLIndex, parse_stages_from_groovy, write_filtered_test_db # noqa: E402 from rules.base import PRInputs, Rule, RuleResult # noqa: E402 from rules.waives_rule import WaivesRule # noqa: E402 @@ -66,13 +66,22 @@ def build_rules(yaml_index: YAMLIndex, stages: dict[str, Stage]) -> list[Rule]: @dataclass class SelectionResult: - """Final aggregated decision.""" + """Final aggregated decision. + + `block_filters` and `test_db_dir_override` drive CBTS Layer 3 (within-stage + test filtering). After the Selector aggregates per-rule `block_filters`, + `main.py` writes a tmp test-db dir with each affected block's `tests:` + array narrowed to entries in the per-block filter prefix subtree, then + sets `test_db_dir_override` so Groovy points trt-test-db at it. + """ scope: Optional[str] affected_stages: set[str] = field(default_factory=set) affected_cpu_arch: set[str] = field(default_factory=set) tests: set[str] = field(default_factory=set) reasons: list[str] = field(default_factory=list) + block_filters: dict[tuple[str, int], set[str]] = field(default_factory=dict) + test_db_dir_override: Optional[str] = None def to_json(self) -> str: data = { @@ -81,6 +90,7 @@ def to_json(self) -> str: "affected_stages": sorted(self.affected_stages), "tests": sorted(self.tests), "reasons": list(self.reasons), + "test_db_dir_override": self.test_db_dir_override, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -152,12 +162,20 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: self.stages[name].cpu_arch for name in affected_stages if name in self.stages } + # Aggregate per-block filter prefix sets across rules. Same block keyed + # by multiple rules: union the filter prefixes. + block_filters: dict[tuple[str, int], set[str]] = {} + for _, r in pairs: + for key, filters in r.block_filters.items(): + block_filters.setdefault(key, set()).update(filters) + return SelectionResult( scope=scope, affected_stages=affected_stages, affected_cpu_arch=affected_cpu_arch, tests=tests, reasons=reasons, + block_filters=block_filters, ) @@ -251,6 +269,20 @@ def main(argv: Optional[list[str]] = None) -> int: pr = _load_pr_inputs(input_path) rules = build_rules(yaml_index, stages) result = Selector(stages).run(pr, rules) + + # Layer 3: if any block has filter prefixes, write a tmp test-db so + # trt-test-db downstream renders a narrower testDBList for the affected + # stages. The path is relative to repo_root so Groovy can resolve it as + # `${LLM_ROOT}/cbts_test_db`. + if result.scope is not None and result.block_filters: + out_dir_name = "cbts_test_db" + write_filtered_test_db( + src_dir=test_db_dir, + output_dir=repo_root / out_dir_name, + block_filters=result.block_filters, + ) + result.test_db_dir_override = out_dir_name + sys.stdout.write(result.to_json()) return 0 diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 64647ab824d4..6e027bc4ed04 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -16,7 +16,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional @@ -30,13 +30,22 @@ class PRInputs: @dataclass class RuleResult: - """What a single rule contributes when it applies to a PR.""" + """What a single rule contributes when it applies to a PR. + + `block_filters` (CBTS Layer 3): per-block set of filter prefixes. Each + affected block (keyed by `(yaml_stem, block_index)`) maps to the filter + levels at which its waive(s) hit. The Selector aggregates this across + rules and uses it to write a tmp test-db with each affected block's + `tests:` array narrowed to entries in any filter prefix's subtree. + Empty when the rule doesn't produce Layer 3 narrowing. + """ handled_files: set[str] tests: set[str] affected_stages: set[str] - scope: str + scope: Optional[str] reason: str + block_filters: dict[tuple[str, int], set[str]] = field(default_factory=dict) class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index e1cea98bb301..f0d24f869a7e 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -17,7 +17,7 @@ from typing import Optional -from blocks import Stage, YAMLIndex, block_matches_stage, normalize_test_id +from blocks import Block, Stage, YAMLIndex, block_matches_stage, normalize_test_id from .base import PRInputs, Rule, RuleResult @@ -88,15 +88,39 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: reason="waives.txt: no actionable test ids in diff", ) + # For each waive, walk the parent chain looking for the first level + # where a YAML entry actually applies (-k keyword check included). + # Any unmatchable waive triggers full fallback — better safe than to + # silently drop CI for a typo'd or out-of-tree waive id. + block_filters: dict[tuple[str, int], set[str]] = {} + affected_blocks: list[Block] = [] seen_block_keys: set[tuple[str, int]] = set() - affected_blocks = [] + misses: list[str] = [] + for tid in changed_test_ids: - for block in self.yaml_index.blocks_containing_test(tid): + match = self.yaml_index.find_match_for_waive(tid) + if match is None: + misses.append(tid) + continue + level, blocks = match + for block in blocks: key = (block.yaml_stem, block.block_index) + block_filters.setdefault(key, set()).add(level) if key not in seen_block_keys: seen_block_keys.add(key) affected_blocks.append(block) + if misses: + preview = ", ".join(sorted(misses)[:3]) + more = f" (+{len(misses) - 3} more)" if len(misses) > 3 else "" + return RuleResult( + handled_files={WAIVES_FILE}, + tests=changed_test_ids, + affected_stages=set(), + scope=None, # Selector treats this as "no decision" → fallback + reason=f"waives.txt: {len(misses)} unmatchable waive(s): {preview}{more}", + ) + affected_stage_names: set[str] = set() for block in affected_blocks: for stage_name, stage in self._stages_by_yaml.get(block.yaml_stem, []): @@ -108,6 +132,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", + block_filters=block_filters, reason=( f"waives.txt: +{len(added)} / -{len(removed)} → " f"{len(affected_blocks)} blocks, {len(affected_stage_names)} stages" From 0a5eae3b616a34b71e7b3df4ddd8fb58c607a4a8 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 09:38:28 +0800 Subject: [PATCH 30/65] [None][doc] CBTS: document Layer 3 (within-stage filtering) in READMEs Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 82 ++++++++++++++++++++-------- jenkins/scripts/cbts/rules/README.md | 2 +- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index ca6f240cbd31..a965628c0ad3 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -1,26 +1,22 @@ # CBTS — Change-Based Testing Selection Pre-merge CI test-selection tool. Looks at what the PR changed and narrows the -set of Jenkins stages that actually need to run. **Adding new rules is -Python-only — Layer 1/2 in Groovy are scope-agnostic and consume the data -directly.** +set of Jenkins stages — and the tests inside each stage — that actually need +to run. **Adding new rules is Python-only — Groovy is scope-agnostic and +consumes the data directly.** --- -## Two consumption layers +## Three consumption layers | Layer | Where | Action | |---|---|---| | **1. Arch track** | `L0_MergeRequest.groovy::launchStages` | Skip x86 / SBSA track when no stage on that arch is affected | | **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset | +| **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the CBTS-narrowed tmp test-db (each affected block's `tests:` array filtered to the per-block filter prefix subtree) | -CBTS only **subtracts** stages, never adds. Anything it can't narrow → full -fallback to the existing filter chain. - -**No within-stage test filtering by design.** Each picked stage runs its full -testDBList. Filtering down to just the changed test would mask wrong waives -(broken deps, typo'd node ids silently matching nothing) — the extra per-stage -time buys robustness. +CBTS only **subtracts** stages and tests, never adds. Anything it can't +narrow → full fallback to the existing filter chain. ## v0 scope @@ -34,13 +30,39 @@ time buys robustness. jenkins/scripts/cbts/ ├── README.md this file ├── main.py CLI entry + Selector + SelectionResult -├── blocks.py YAML loading + stage parsing + test-id normalization +├── blocks.py YAML index + lookup + filtered tmp test-db generation └── rules/ ├── README.md per-rule logic summary ├── base.py Rule ABC + PRInputs + RuleResult └── waives_rule.py v0's only rule ``` +## Lookup algorithm: parent chain with first-match wins + +Per waive id, `YAMLIndex.find_match_for_waive` walks the pytest tree from the +waive towards the root. The first level whose YAML has a matching entry wins; +that level becomes the **filter prefix** the block uses for Layer 3. + +``` +waive id (raw) + ↓ normalize strip SKIP/TIMEOUT/full:gpu/comments + ↓ strip [params] if present +target_lookup (function-level when waive was parametrized; otherwise + class/file/dir level — waive's own granularity) + ↓ try YAML at this level + hit → matched: filter prefix = level + miss → strip one level up (::method → ::class → /file → /dir → ...) + and retry + ↓ all levels miss → fallback: rule emits scope=None, baseline runs +``` + +An entry "matches" at a level when its **canonical target** (entry with +`SKIP`/`TIMEOUT`/`full:gpu`, pytest options `-k "..."` / `-m "..."`, and +`[params]` all stripped) equals the level **and** any `-k` keyword filter the +entry carries actually contains an identifier present in the waive id. +`-m` markers are unverifiable from a string and always pass (over-include +when in doubt). + ## When CBTS activates CBTS narrows test selection in **two usages only**: @@ -66,7 +88,14 @@ the existing filter chain takes over: `--stage-list`, `--extra-stage`, 1. `main.py --list-needed-diffs` → patterns whose diffs Groovy fetches. Patterns are **Ant-style globs** (`tests/**/*.py`, `cpp/kernels/**`, exact paths), matched via `hudson.util.AntPathMatcher`. -2. `main.py cbts_input.json` → decision JSON on stdout: +2. `main.py cbts_input.json` → decision JSON on stdout. If any block was + narrowed, also writes `${LLM_ROOT}/cbts_test_db/` containing only the + affected YAMLs with their filtered `tests:` arrays. Each kept entry + preserves `TIMEOUT (n)`, `ISOLATION`, `-k "..."`, `-m "..."` verbatim + (YAML-level `# comments` are dropped by PyYAML round-trip but no + functional info is lost). + +Decision JSON: ```json { @@ -74,12 +103,15 @@ the existing filter chain takes over: `--stage-list`, `--extra-stage`, "affected_cpu_arch": ["x86"], "affected_stages": ["A10-PyTorch-1", "A10-PyTorch-2"], "tests": ["unittest/utils/test_util.py"], - "reasons": ["[waives] waives.txt: +1 / -0 → 1 blocks, 2 stages"] + "reasons": ["[waives] waives.txt: +1 / -0 → 1 blocks, 2 stages"], + "test_db_dir_override": "cbts_test_db" } ``` -`scope: null` → no decision, full fallback. Groovy doesn't gate on the scope -value — it's metadata for logs and multi-rule combining only. +- `scope: null` → no decision, full fallback. Groovy doesn't gate on the + scope value — it's metadata for logs and multi-rule combining only. +- `test_db_dir_override: null` → no Layer 3 narrowing; trt-test-db reads + the source `tests/integration/test_lists/test-db/` as before. ## Adding a new rule @@ -106,18 +138,20 @@ value — it's metadata for logs and multi-rule combining only. affected_stages={...}, scope="myscope", reason="why this fired", + # Optional Layer 3 contribution: per-block filter prefixes. + # Selector unions across rules and writes the tmp test-db. + block_filters={(yaml_stem, block_index): {filter_prefix}, ...}, ) ``` 2. **Register in `main.py`**: add to `RULE_CLASSES` and `build_rules()`. -3. **No Groovy edits needed.** Exception: if your rule needs to drop - *individual tests inside* a stage (vs whole stages), add the hook at - `L0_Test.groovy:2674` — but `waiveonly` deliberately skips this, see - the "no within-stage filtering" note above. +3. **No Groovy edits needed.** Layer 1 / 2 / 3 are scope-agnostic and consume + `affected_cpu_arch` / `affected_stages` / `block_filters` directly. -Rule order is irrelevant. `Selector` unions `affected_stages`; scopes are -combined via `_combine_scopes` (all-agree → that scope; disagreement → `None`). +Rule order is irrelevant. `Selector` unions `affected_stages` and +`block_filters`; scopes are combined via `_combine_scopes` (all-agree → that +scope; disagreement → `None`). ## Fallback paths @@ -127,7 +161,11 @@ CBTS falls back to the existing filter chain when: - `changed_files` is empty - `main.py` throws / stdout is unparsable - Python returns `scope: null` ("no decision") +- A waive id misses every level up to the root in `find_match_for_waive` + (likely typo'd or out-of-tree id) — the rule emits `scope: null` - `affected_stages` is empty (Layer 2 no-op) +- Layer 3 filter would empty a block's `tests:` array — that block keeps + its original tests instead (per-block safety net) Every fallback logs an `echo` line — no silent failures. diff --git a/jenkins/scripts/cbts/rules/README.md b/jenkins/scripts/cbts/rules/README.md index 7cdb66a1aa8b..587101fd28b3 100644 --- a/jenkins/scripts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/rules/README.md @@ -7,4 +7,4 @@ top-level [README](../README.md) for the overall CBTS architecture. | File | Class | Scope | Triggers on | What it picks | |---|---|---|---|---| -| `waives_rule.py` | `WaivesRule` | `waiveonly` | PR changes `tests/integration/test_lists/waives.txt` | For each added/removed test id (after `blocks.normalize_test_id` strips `SKIP`/`TIMEOUT`/`full:/` decorations): look it up in the test-db YAML, pick stages whose `mako` matches the containing block's `condition`. | +| `waives_rule.py` | `WaivesRule` | `waiveonly` | PR changes `tests/integration/test_lists/waives.txt` | For each added/removed test id, calls `YAMLIndex.find_match_for_waive` to walk the pytest parent chain (function → class → file → dir → ...) until a YAML entry matches. The matched level becomes that block's Layer 3 filter prefix; stages whose `mako` matches the block's `condition` go into `affected_stages`. Any waive that misses every level → `scope=None` (full fallback). | From e5d177d790782ecd3b3294ac5921f14d1cd19f5a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 10:08:46 +0800 Subject: [PATCH 31/65] [None][fix] CBTS: tolerate SLURM configs with extra positional args in _STAGE_ENTRY_RE Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index d35dc8dab5c1..cbe5c2288c1c 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -315,8 +315,10 @@ def _range_in(val_str: str | None, gte, lte) -> bool: # Matches a single entry like: # "A10-PyTorch-1": ["a10", "l0_a10", 1, 2], # "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], +# "DGX_B200-PyTorch-1": ["auto:dgx-b200-flex", "l0_b200", 1, 3, 1, 1, true], # Same shape as scripts/test_to_stage_mapping.py::_STAGE_RE, extended to -# capture split_id / total_splits / gpu_count. +# capture split_id / total_splits / gpu_count and tolerate any number of +# trailing positional args (e.g. SLURM configs append a boolean flag). _STAGE_ENTRY_RE = re.compile( r'"(?P[^"]+)"\s*:\s*\[' r'\s*"(?P[^"]+)"\s*,' @@ -324,6 +326,7 @@ def _range_in(val_str: str | None, gte, lte) -> bool: r"(?:\s*,\s*(?P\d+))?" r"(?:\s*,\s*(?P\d+))?" r"(?:\s*,\s*(?P\d+))?" + r"(?:\s*,[^\]]*)?" # tolerate trailing positional args before `]` r"\s*\]" ) From 9a095b43e348a30174c23c51c49d3611f6ccddb4 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:10:24 +0800 Subject: [PATCH 32/65] [None][fix] CBTS Layer 3: tolerate empty shards after narrowing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Layer 3 narrows a stage's test-db block below its configured split count, some shards end up with 0 tests. Other shards still run the affected tests, so this is expected — only raise the existing "No tests were executed" sanity-check error when CBTS isn't active. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 6470ff829d95..258f9a1b79ae 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3209,7 +3209,16 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } if (noRegularTests && noIsolateTests) { - error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." + // CBTS Layer 3 may narrow a block's tests below the stage's + // split count, leaving some shards with 0 tests. That's + // expected — other shards run the affected tests. Only + // raise the sanity-check error when CBTS isn't active. + def cbtsActive = testFilter[(CBTS_RESULT)]?.test_db_dir_override + if (cbtsActive) { + echo "CBTS Layer 3: shard ${splitId}/${splits} got 0 tests after narrowing — other shards run the affected tests. Marking as success." + } else { + error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." + } } } } From 1a2fa0d183729e3606e034fdff5cd7c6fef75fb1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:11:37 +0800 Subject: [PATCH 33/65] [None][test] TESTING: enable CBTS on infra-only PR + Layer 3 e2e waives Two test-only changes (revert before merge): 1. WaivesRule: temporarily claim every changed file in handled_files so CBTS fires on this PR even though it edits CBTS infra alongside waives.txt. (Default behavior only fires when waives.txt is the lone changed file.) 2. waives.txt: 3 SKIP entries to exercise Layer 3 within-stage filtering end-to-end across param/function-only/-k-keyword cases. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/rules/waives_rule.py | 15 +- tests/integration/test_lists/waives.txt | 176 ++++++++++++++++++++++ 2 files changed, 188 insertions(+), 3 deletions(-) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index f0d24f869a7e..e5bbb4c3b913 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -81,7 +81,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: changed_test_ids = added | removed if not changed_test_ids: return RuleResult( - handled_files={WAIVES_FILE}, + # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every + # changed file so CBTS fires on this PR even though it edits CBTS + # infra files alongside waives.txt. + handled_files=set(pr.changed_files), tests=set(), affected_stages=set(), scope="waiveonly", @@ -114,7 +117,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: preview = ", ".join(sorted(misses)[:3]) more = f" (+{len(misses) - 3} more)" if len(misses) > 3 else "" return RuleResult( - handled_files={WAIVES_FILE}, + # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every + # changed file so CBTS fires on this PR even though it edits CBTS + # infra files alongside waives.txt. + handled_files=set(pr.changed_files), tests=changed_test_ids, affected_stages=set(), scope=None, # Selector treats this as "no decision" → fallback @@ -128,7 +134,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: affected_stage_names.add(stage_name) return RuleResult( - handled_files={WAIVES_FILE}, + # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every + # changed file so CBTS fires on this PR even though it edits CBTS + # infra files alongside waives.txt. + handled_files=set(pr.changed_files), tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index cc1298fe2904..fda46013038d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -453,3 +453,179 @@ verl/test_verl_cases.py::test_async_server SKIP (https://nvbugs/5981833) verl/test_verl_cases.py::test_rollout_utils SKIP (https://nvbugs/5981833) visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark SKIP (https://nvbugs/6050483) visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos] SKIP (https://nvbugs/6050483) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) +accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[4gpus] SKIP (https://nvbugs/6069790) +accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5981293) +accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/5981293) +disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) +disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) +disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) +disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6050489) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6050489) +unittest/llmapi/test_llm_pytorch.py::test_llm_disagg_streaming_gen_cancelled SKIP (https://nvbugs/6078431) +unittest/auto_deploy/singlegpu/transformations/library/test_mrope_delta_cache.py::test_qwen_registry_configs_explicitly_enable_mrope_delta_cache SKIP (https://nvbugs/6078421) +accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6080024) +llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_eagle3 SKIP (https://nvbugs/6075431) +llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp SKIP (https://nvbugs/6079440) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6079919) +disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput] SKIP (https://nvbugs/6084764) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6084824) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=0] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp] SKIP (https://nvbugs/6084568) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6088149) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6088149) +accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 SKIP (https://nvbugs/6070857) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6094071) +accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6094070) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dummy_load_format SKIP (https://nvbugs/6094072) +cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] SKIP (https://nvbugs/6093820) +cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] SKIP (https://nvbugs/6093820) +accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm] SKIP (https://nvbugs/6094208) +accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-1-trtllm] SKIP (https://nvbugs/6094208) +accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] SKIP (https://nvbugs/6093713) +accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6093713) +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_on] SKIP (https://nvbugs/6094068) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] SKIP (https://nvbugs/6093715) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) +disaggregated/test_auto_scaling.py::test_service_discovery[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_service_discovery[http-load_balancing] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[http-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_service_discovery[etcd-load_balancing] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_worker_restart[http-kv_cache_aware] SKIP (https://nvbugs/6094100) +disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin] SKIP (https://nvbugs/6094100) +disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM] SKIP (https://nvbugs/6095421) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6095421) +perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3] SKIP (https://nvbugs/6095700) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6095851) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6098790) +disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True] SKIP (https://nvbugs/6094066) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler SKIP (https://nvbugs/6105765) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse SKIP (https://nvbugs/6105765) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse SKIP (https://nvbugs/6105765) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler SKIP (https://nvbugs/6105765) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse SKIP (https://nvbugs/6105765) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available SKIP (https://nvbugs/6106174) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse SKIP (https://nvbugs/6106174) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse SKIP (https://nvbugs/6106174) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse SKIP (https://nvbugs/6106174) +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse SKIP (https://nvbugs/6106174) +accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] SKIP (https://nvbugs/5921674) +accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/5955803) +full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16 SKIP (https://nvbugs/6069543) +perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_8k1k] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_1k1k] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6110326) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6110326) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] SKIP (https://nvbugs/6113016) +disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6011317) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto] SKIP (https://nvbugs/5596343) +unittest/_torch/speculative/test_eagle3.py::test_llama_eagle3_dynamic_tree[True-False] SKIP (https://nvbugs/6113021) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-no_overlap_scheduler] SKIP (https://nvbugs/6114821) +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4] SKIP (https://nvbugs/6110074) +test_doc.py::test_url_validity SKIP (https://nvbugs/6109719) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6115832) +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] SKIP (https://nvbugs/6112508) +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] SKIP (https://nvbugs/6112508) +accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-trtllm] SKIP (https://nvbugs/6112500) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6112497) +disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114140) +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6094102) +disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114142) +disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114139) +accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype SKIP (https://nvbugs/6114464) +disaggregated/test_disaggregated.py::test_disaggregated_trtllm_sampler[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114141) +disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) +disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_gentp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) +disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114612) +test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6114608) +test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6114608) +test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] SKIP (https://nvbugs/6114608) +test_e2e.py::test_multi_nodes_eval[nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-tp16-mmlu] SKIP (https://nvbugs/6114608) +test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https://nvbugs/6115560) +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] SKIP (https://nvbugs/6116088) +test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] SKIP (https://nvbugs/6115562) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False] SKIP (https://nvbugs/5981122) +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6117811) +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] SKIP (https://nvbugs/6117811) +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] SKIP (https://nvbugs/6117811) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True] SKIP (https://nvbugs/6117816) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler] SKIP (https://nvbugs/6120553) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] SKIP (https://nvbugs/6120553) +accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer] SKIP (https://nvbugs/6117814) +accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm] SKIP (https://nvbugs/6117814) +accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True] SKIP (https://nvbugs/6117816) +accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True] SKIP (https://nvbugs/6117816) +unittest/tools/test_layer_wise_benchmarks.py::test_performance_alignment[1] SKIP (https://nvbugs/6127669) +full:H100_PCIe/unittest/auto_deploy/standalone SKIP (https://nvbugs/6129630) +full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6128420) +full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] SKIP (https://nvbugs/6128419) +unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py::test_cp_tp_broadcast_object[tp_cp_broadcast-list] SKIP (https://nvbugs/6132301) +perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k] SKIP (https://nvbugs/6133067) +disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0] SKIP (CBTS Layer 3 test - case A param, revert before merge) +test_e2e.py::test_get_ci_container_port SKIP (CBTS Layer 3 test - case B function-only, revert before merge) +unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[CUTLASS-fp8-tp4] SKIP (CBTS Layer 3 test - case E -k keyword, revert before merge) From db17f77fd94defebd379071ceb7137a9f313ce8e Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:53:30 +0800 Subject: [PATCH 34/65] [None][fix] CBTS: exclude perf stages from Layer 2/3 CBTS is a pre-merge correctness optimization. Perf stages have their own trigger model (nightly / post-merge) and require their full test list to produce a stable baseline, so they should run independently of CBTS rather than be narrowed by it. Two guards added: - Layer 2 (launchTestJobs): when filtering parallelJobs by affected_stages, also exclude any stage whose key matches /Perf/. - Layer 3 (renderTestDB): when picking the test-db source dir, ignore cbts.test_db_dir_override on perfMode stages and fall back to the unfiltered test-db. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 258f9a1b79ae..d7e205137fa3 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2567,12 +2567,11 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" - // CBTS Layer 3: when CBTS provided a tmp test-db dir (each affected - // block's tests narrowed to entries in the per-block filter prefix - // subtree), point trt-test-db at it. Otherwise use the source test-db. + // CBTS Layer 3: use the narrowed test-db when CBTS provides one; + // perf stages keep the source test-db for stable baselines. def cbts = testFilter[(CBTS_RESULT)] def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" - if (cbts != null && cbts.test_db_dir_override) { + if (cbts != null && cbts.test_db_dir_override && !perfMode) { testDBPath = "${llmSrc}/${cbts.test_db_dir_override}" echo "CBTS [${cbts.scope}]: rendering test list from filtered test-db at ${testDBPath}" } @@ -4423,17 +4422,17 @@ def launchTestJobs(pipeline, testFilter) } // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all - // existing filter rules so that unknown / no-decision paths fall through - // naturally. Scope-agnostic: any non-null cbts result with affected_stages - // is treated as actionable. See jenkins/scripts/cbts/README.md. + // existing filter rules so unknown / no-decision paths fall through + // naturally. Perf stages are excluded — they have their own trigger + // model and need full test lists. See jenkins/scripts/cbts/README.md. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set - parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) } + parallelJobsFiltered = parallelJobs.findAll { key, _ -> + affectedSet.contains(key) && !(key =~ /Perf/) + } // Under `/bot run --post-merge`, keep only post-merge hits; if none, - // no-op (no fallback to full post-merge). IS_POST_MERGE is also true - // for the official PostMerge pipeline, but getCbtsResult() defers - // there, so reading it here is equivalent to "user passed --post-merge". + // no-op (no fallback to full post-merge). if (testFilter[(IS_POST_MERGE)]) { parallelJobsFiltered = parallelJobsFiltered.findAll { it.key.contains("Post-Merge") } echo "CBTS [${cbts.scope}] (--post-merge): keeping ${parallelJobsFiltered.size()} affected post-merge stages" From 8058125c9ae035315aaf45a91e9a0a9122006531 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 17:30:28 +0800 Subject: [PATCH 35/65] [None][test] TESTING: skip Debug build smoke stage to unblock CBTS testing under OOM (revert before merge) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/Build.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 4798c4695ab3..011f1cf43fc4 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -550,7 +550,9 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) }]} parallelJobs.failFast = enableFailFast - if (cpu_arch == X86_64_TRIPLE && !reuseArtifactPath) { + // TEMP(cbts-v0): skip "Build With Build Type Debug" smoke stage to unblock CBTS testing under OOM. + // REVERT BEFORE MERGE — this stage is the gate that prevents Debug build from rotting. + if (false && cpu_arch == X86_64_TRIPLE && !reuseArtifactPath) { def key = "Build With Build Type Debug" parallelJobs += [ (key): { From 718b08a1d4b8e61ba2bbf1b8a36e26362fcc73fe Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 29 Apr 2026 22:18:01 +0800 Subject: [PATCH 36/65] [None][fix] CBTS: drop redundant perfMode guard in renderTestDB renderTestDB() did not declare a perfMode parameter, so the !perfMode check in the test-db-override branch raised MissingPropertyException (WorkflowScript: No such property: perfMode) at runtime. The guard was redundant: Layer 2 (launchTestJobs) already filters out any stage whose key matches /Perf/ when CBTS provides affected_stages, so perf stages never reach renderTestDB with cbts != null. Removing the guard fixes the exception without changing any actual behavior. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d7e205137fa3..6cf9fada14ee 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2567,11 +2567,12 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" - // CBTS Layer 3: use the narrowed test-db when CBTS provides one; - // perf stages keep the source test-db for stable baselines. + // CBTS Layer 3: use the narrowed test-db when CBTS provides one. + // Perf stages are excluded at Layer 2 (launchTestJobs) and never + // reach this path with cbts != null, so no perfMode guard is needed here. def cbts = testFilter[(CBTS_RESULT)] def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" - if (cbts != null && cbts.test_db_dir_override && !perfMode) { + if (cbts != null && cbts.test_db_dir_override) { testDBPath = "${llmSrc}/${cbts.test_db_dir_override}" echo "CBTS [${cbts.scope}]: rendering test list from filtered test-db at ${testDBPath}" } From dd1a5abd06792348e1488023c7916d5fa67387eb Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:05:51 +0800 Subject: [PATCH 37/65] =?UTF-8?q?[None][test]=20TESTING:=20CBTS=20diagnost?= =?UTF-8?q?ics=20=E2=80=94=20log=20affected=5Fstages=20and=20renderTestDB?= =?UTF-8?q?=20hit=20count?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two diagnostic echoes to make CBTS narrowing decisions visible in CI logs: - launchTestJobs (Layer 2): print the full affected_stages set sorted before /Perf/ exclusion runs, plus the count of dropped stages, so empty or over-eager scoping is obvious at a glance. - renderTestDB (Layer 3): after trt-test-db, print stage, context, the test-db source label (CBTS-narrowed vs source), the dir, and the rendered test count — to pinpoint whether an empty per-stage list is caused upstream of processShardTestList cleaning. No control-flow changes. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 6cf9fada14ee..0e5939ecc15e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2591,6 +2591,11 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { ].join(" ") sh(label: "Render test list from test-db", script: testDBQueryCmd) + // CBTS diagnostics: show how many tests survived the trt-test-db query + // and which test-db source was used, to make empty-list bugs visible. + def testCount = sh(returnStdout: true, script: "wc -l < ${testList} | tr -d ' '").trim() + def testDBLabel = (cbts != null && cbts.test_db_dir_override) ? "CBTS-narrowed [${cbts.scope}]" : "source" + echo "renderTestDB: stage=${stageName} context=${testContext} test-db=${testDBLabel} dir=${testDBPath} -> ${testCount} tests" sh(script: "cat ${testList}") return testList @@ -4429,9 +4434,15 @@ def launchTestJobs(pipeline, testFilter) def cbts = testFilter[(CBTS_RESULT)] if (cbts != null && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set + // CBTS diagnostics: show what CBTS actually decided BEFORE the + // /Perf/ exclusion and post-merge narrowing, so empty / over-eager + // affected_stages are visible at a glance. + echo "CBTS [${cbts.scope}]: affected_stages (${affectedSet.size()}):\n ${affectedSet.sort().join('\n ')}" parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) && !(key =~ /Perf/) } + def droppedStages = (parallelJobs.keySet() - parallelJobsFiltered.keySet()).sort() + echo "CBTS [${cbts.scope}]: dropped ${droppedStages.size()} stages from parallelJobs (not in affected_stages or matched /Perf/)" // Under `/bot run --post-merge`, keep only post-merge hits; if none, // no-op (no fallback to full post-merge). if (testFilter[(IS_POST_MERGE)]) { From be167c7fdd02942a1edafefab6e039ccd2cc6b2b Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:11:02 +0800 Subject: [PATCH 38/65] =?UTF-8?q?[None][test]=20TESTING:=20CBTS=20Python?= =?UTF-8?q?=20=E2=80=94=20dump=20full=20decision=20to=20stderr=20for=20dia?= =?UTF-8?q?gnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add _log_decision_to_stderr() called right before main.py writes its stdout JSON. Logs scope, reasons, affected_cpu_arch, affected_tests, block_filters (per yaml_stem#idx with their filter prefixes), and affected_stages with each stage annotated by its yaml_stem. Goes to stderr so the stdout JSON contract Groovy getCbtsResult relies on is preserved; Jenkins console captures stderr automatically. Richer than the Groovy-side echoes because we have access to the unfiltered SelectionResult here (before the /Perf/ exclusion in launchTestJobs and before any per-stage trt-test-db rendering). Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index b5b99b8b7eaa..51515d8a5b3b 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -283,9 +283,42 @@ def main(argv: Optional[list[str]] = None) -> int: ) result.test_db_dir_override = out_dir_name + _log_decision_to_stderr(stages, result) sys.stdout.write(result.to_json()) return 0 +def _log_decision_to_stderr(stages: dict[str, Stage], result: SelectionResult) -> None: + """Dump the full CBTS decision to stderr for Jenkins console diagnostics. + + stdout carries the JSON consumed by Groovy `getCbtsResult`, so all + human-readable detail goes to stderr to avoid corrupting that contract. + Each affected stage is annotated with its yaml_stem so blocks-vs-stages + mismatches (e.g. a stage matched by an unexpected YAML) are obvious. + """ + out = sys.stderr + print("=" * 64, file=out) + print("CBTS decision (diagnostic; stderr only):", file=out) + print(f" scope: {result.scope}", file=out) + print(f" test_db_dir_override: {result.test_db_dir_override}", file=out) + print(f" affected_cpu_arch: {sorted(result.affected_cpu_arch)}", file=out) + if result.reasons: + print(" reasons:", file=out) + for r in result.reasons: + print(f" - {r}", file=out) + print(f" affected_tests ({len(result.tests)}):", file=out) + for t in sorted(result.tests): + print(f" - {t}", file=out) + print(f" block_filters ({len(result.block_filters)} blocks):", file=out) + for (yaml_stem, idx), prefixes in sorted(result.block_filters.items()): + print(f" - {yaml_stem}#{idx}: {sorted(prefixes)}", file=out) + print(f" affected_stages ({len(result.affected_stages)}):", file=out) + for name in sorted(result.affected_stages): + stage = stages.get(name) + annotation = f" [yaml_stem={stage.yaml_stem}]" if stage else "" + print(f" - {name}{annotation}", file=out) + print("=" * 64, file=out) + + if __name__ == "__main__": sys.exit(main()) From fba46128cccb1af5060866383cd569f7693203cb Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:19:40 +0800 Subject: [PATCH 39/65] [None][test] TESTING: drop redundant Layer 2 echoes (Python now covers this) Python main.py logs the full SelectionResult to stderr (scope, reasons, block_filters, affected_stages with yaml_stem annotation), so the Layer 2 affected_stages dump and dropped-count echo I added previously are redundant. The pre-existing 'limiting to N affected stages' + 'Now we will run stages' echoes already convey the post-/Perf/ runtime view. Keeping the renderTestDB echo since it shows the trt-test-db render count per stage at runtime, which Python cannot observe. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0e5939ecc15e..c0cdacb53bbc 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4434,15 +4434,9 @@ def launchTestJobs(pipeline, testFilter) def cbts = testFilter[(CBTS_RESULT)] if (cbts != null && cbts.affected_stages) { def affectedSet = cbts.affected_stages as Set - // CBTS diagnostics: show what CBTS actually decided BEFORE the - // /Perf/ exclusion and post-merge narrowing, so empty / over-eager - // affected_stages are visible at a glance. - echo "CBTS [${cbts.scope}]: affected_stages (${affectedSet.size()}):\n ${affectedSet.sort().join('\n ')}" parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) && !(key =~ /Perf/) } - def droppedStages = (parallelJobs.keySet() - parallelJobsFiltered.keySet()).sort() - echo "CBTS [${cbts.scope}]: dropped ${droppedStages.size()} stages from parallelJobs (not in affected_stages or matched /Perf/)" // Under `/bot run --post-merge`, keep only post-merge hits; if none, // no-op (no fallback to full post-merge). if (testFilter[(IS_POST_MERGE)]) { From 254d2c928c95f86638f1c87905dd62d43ad54b75 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:40:06 +0800 Subject: [PATCH 40/65] [None][fix] CBTS Layer 3: drop unaffected blocks from narrowed test-db MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_filtered_test_db previously kept blocks not in block_filters 'written through unchanged' on the rationale that only stages mapped to affected blocks would Layer-2-pass and run them anyway. That assumption breaks under '/bot run --post-merge': a post-merge stage in the same yaml_stem can match a post_merge block that was never touched by any waive, running tests CBTS did not select. Fix: rebuild each context's block list to only include blocks listed in block_filters. Verified locally on the 3-waive testing PR — l0_b200.yml narrows from 6 blocks to 1, l0_h100.yml from many to 2, matching the affected blocks reported by Selector exactly. No change to the test-narrowing logic for kept blocks. Safety fallback (empty filter result -> keep original tests) preserved. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index cbe5c2288c1c..2808840a0432 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -519,25 +519,25 @@ def write_filtered_test_db( ) -> None: """Generate a tmp test-db dir narrowed by CBTS Layer 3. - Contains only the YAMLs whose blocks were affected, with each affected - block's `tests:` array filtered to entries in the per-block filter prefix - subtree. - - Layer 3 narrowing: trt-test-db invoked with `-d ` will produce - a smaller testDBList for each affected stage, since the YAML it reads has - fewer tests in the matched block. + Each output YAML contains ONLY the blocks listed in `block_filters` for + that stem, with each affected block's `tests:` array filtered to entries + in the per-block filter prefix subtree. Unaffected blocks and unaffected + YAML files are dropped entirely. + + Why drop unaffected blocks rather than write them through unchanged: + Layer 3's contract is "only run tests touched by the affected blocks". If + we kept post_merge / other-backend blocks that this PR never touched, a + `/bot run --post-merge` could activate them on stages whose mako happens + to match — running tests CBTS never selected. Dropping them keeps Layer + 3's narrowing semantically tight. `block_filters` keys: (yaml_stem, block_index) of affected blocks. Values: set of filter prefix strings that should keep tests in their subtree. - Unaffected blocks (not in `block_filters`) are written through unchanged. - Unaffected YAML files are NOT written — only stages whose YAML appears in - `block_filters` will be running anyway (Layer 2 already filtered). - - Safety: if filtering would empty a block's tests, the original tests are - kept (prevents silent skip when a YAML/waive granularity mismatch slips - through). + Safety: if filtering would empty an affected block's tests, the original + tests are kept (prevents silent skip from typo'd waive ids or granularity + mismatch). The block itself is still kept either way. """ output_dir.mkdir(parents=True, exist_ok=True) @@ -548,14 +548,16 @@ def write_filtered_test_db( continue data = yaml.safe_load(src.read_text()) or {} - for ctx_blocks in data.values(): + for ctx_key, ctx_blocks in list(data.items()): if not isinstance(ctx_blocks, list): continue + new_blocks = [] for i, block_data in enumerate(ctx_blocks): if not isinstance(block_data, dict): continue key = (stem, i) if key not in block_filters: + # Drop unaffected blocks (see docstring rationale). continue original = block_data.get("tests") or [] filters = block_filters[key] @@ -568,6 +570,8 @@ def write_filtered_test_db( # silent skip from typo'd waive ids or granularity mismatch). if kept: block_data["tests"] = kept + new_blocks.append(block_data) + data[ctx_key] = new_blocks (output_dir / src.name).write_text( yaml.safe_dump(data, sort_keys=False, default_flow_style=False) From 254cddfdb2960903c9f4729c05740ae952685122 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:49:48 +0800 Subject: [PATCH 41/65] [None][fix] CBTS Layer 3: -k keyword guard against parent-chain over-include MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a waive like 'func[CUTLASS-fp8-tp4]' has no exact YAML entry, the parent-chain fallback resolves it to the function-level prefix 'func'. Subtree match alone then keeps every YAML entry under 'func', including 'func -k "TRTLLM"', 'func -k "CUTEDSL"', etc. — even though pytest's -k filter on those keywords would never pick up the [CUTLASS-...] case. Schema change (no JSON-protocol break — block_filters is internal): RuleResult.block_filters dict[(stem,idx), set[prefix]] -> dict[(stem,idx), dict[prefix, set[waive_id]]] SelectionResult.block_filters: same change Selector.run aggregation: union waive_ids per (block, prefix) WaivesRule.apply: record (prefix -> {waive_id}) instead of just adding prefix to a set write_filtered_test_db then narrows each kept entry twice: 1. Subtree match against any prefix (existing). 2. -k keyword guard via _entry_applies_to_waive against any waive id that resolved to a matching prefix. Entries without -k pass through. Verified locally on the 3-waive testing PR: l0_b300.yml goes from 4 entries (-k CUTLASS/TRTLLM/CUTEDSL/DEEPGEMM) to just -k "CUTLASS". Equivalent narrowing on l0_b200.yml and l0_h100.yml block #0. Stderr diagnostic logs the prefix -> waive-id mapping so the chain is visible in CI logs. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/blocks.py | 37 +++++++++++++++++------ jenkins/scripts/cbts/main.py | 20 +++++++----- jenkins/scripts/cbts/rules/base.py | 13 ++++---- jenkins/scripts/cbts/rules/waives_rule.py | 7 +++-- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 2808840a0432..a9a72937065c 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -515,7 +515,7 @@ def block_matches_stage(block: Block, stage: Stage) -> bool: def write_filtered_test_db( src_dir: Path, output_dir: Path, - block_filters: dict[tuple[str, int], set[str]], + block_filters: dict[tuple[str, int], dict[str, set[str]]], ) -> None: """Generate a tmp test-db dir narrowed by CBTS Layer 3. @@ -524,6 +524,15 @@ def write_filtered_test_db( in the per-block filter prefix subtree. Unaffected blocks and unaffected YAML files are dropped entirely. + Two narrowing checks per entry: + 1. Subtree match — entry's target lives under at least one filter prefix + 2. -k keyword guard — if the entry uses `func -k "K"`, drop it unless + `K` would actually run at least one of the waives that resolved to + the matching prefix(es). Without this guard, a parent-chain fallback + (e.g. waive `func[CUTLASS-fp8-tp4]` -> prefix `func`) over-includes + every `-k` variant of `func` even though only `-k "CUTLASS"` would + pick up the waived test at runtime. + Why drop unaffected blocks rather than write them through unchanged: Layer 3's contract is "only run tests touched by the affected blocks". If we kept post_merge / other-backend blocks that this PR never touched, a @@ -532,8 +541,8 @@ def write_filtered_test_db( 3's narrowing semantically tight. `block_filters` keys: (yaml_stem, block_index) of affected blocks. - Values: set of filter prefix strings that should keep tests in their - subtree. + Values: {filter_prefix: {waive_id, ...}} — prefix governs subtree match; + waive ids are consulted by the -k keyword guard above. Safety: if filtering would empty an affected block's tests, the original tests are kept (prevents silent skip from typo'd waive ids or granularity @@ -560,12 +569,22 @@ def write_filtered_test_db( # Drop unaffected blocks (see docstring rationale). continue original = block_data.get("tests") or [] - filters = block_filters[key] - kept = [ - t - for t in original - if any(_target_in_filter_subtree(_entry_target(t), f) for f in filters) - ] + prefix_to_waives = block_filters[key] + kept = [] + for t in original: + target = _entry_target(t) + matched_waives: set[str] = set() + for prefix, waives in prefix_to_waives.items(): + if _target_in_filter_subtree(target, prefix): + matched_waives |= waives + if not matched_waives: + continue + # -k keyword guard: drop entries whose `-k` filter + # excludes every matching waive. Entries without `-k` + # are always kept (`_entry_applies_to_waive` returns + # True when no keyword is present). + if any(_entry_applies_to_waive(t, w) for w in matched_waives): + kept.append(t) # Safety: empty filter result → fallback to original (prevents # silent skip from typo'd waive ids or granularity mismatch). if kept: diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 51515d8a5b3b..8f18ad51d475 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -80,7 +80,7 @@ class SelectionResult: affected_cpu_arch: set[str] = field(default_factory=set) tests: set[str] = field(default_factory=set) reasons: list[str] = field(default_factory=list) - block_filters: dict[tuple[str, int], set[str]] = field(default_factory=dict) + block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) test_db_dir_override: Optional[str] = None def to_json(self) -> str: @@ -162,12 +162,14 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: self.stages[name].cpu_arch for name in affected_stages if name in self.stages } - # Aggregate per-block filter prefix sets across rules. Same block keyed - # by multiple rules: union the filter prefixes. - block_filters: dict[tuple[str, int], set[str]] = {} + # Aggregate per-block prefix->{waive_ids} maps across rules. Same + # block keyed by multiple rules: union the waive_ids per prefix. + block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} for _, r in pairs: - for key, filters in r.block_filters.items(): - block_filters.setdefault(key, set()).update(filters) + for key, prefix_to_waives in r.block_filters.items(): + dst = block_filters.setdefault(key, {}) + for prefix, waives in prefix_to_waives.items(): + dst.setdefault(prefix, set()).update(waives) return SelectionResult( scope=scope, @@ -310,8 +312,10 @@ def _log_decision_to_stderr(stages: dict[str, Stage], result: SelectionResult) - for t in sorted(result.tests): print(f" - {t}", file=out) print(f" block_filters ({len(result.block_filters)} blocks):", file=out) - for (yaml_stem, idx), prefixes in sorted(result.block_filters.items()): - print(f" - {yaml_stem}#{idx}: {sorted(prefixes)}", file=out) + for (yaml_stem, idx), prefix_to_waives in sorted(result.block_filters.items()): + print(f" - {yaml_stem}#{idx}:", file=out) + for prefix, waives in sorted(prefix_to_waives.items()): + print(f" {prefix} <- {sorted(waives)}", file=out) print(f" affected_stages ({len(result.affected_stages)}):", file=out) for name in sorted(result.affected_stages): stage = stages.get(name) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 6e027bc4ed04..47221a64cd7b 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -32,11 +32,12 @@ class PRInputs: class RuleResult: """What a single rule contributes when it applies to a PR. - `block_filters` (CBTS Layer 3): per-block set of filter prefixes. Each - affected block (keyed by `(yaml_stem, block_index)`) maps to the filter - levels at which its waive(s) hit. The Selector aggregates this across - rules and uses it to write a tmp test-db with each affected block's - `tests:` array narrowed to entries in any filter prefix's subtree. + `block_filters` (CBTS Layer 3): per-block map of filter prefix -> set of + waive ids that resolved to that prefix. Each affected block (keyed by + `(yaml_stem, block_index)`) maps to {prefix: {waive_id, ...}}. The + Selector aggregates this across rules and `write_filtered_test_db` + uses both the prefix (subtree match) AND the waive ids (to skip YAML + entries whose `-k ""` filter doesn't match the waived test). Empty when the rule doesn't produce Layer 3 narrowing. """ @@ -45,7 +46,7 @@ class RuleResult: affected_stages: set[str] scope: Optional[str] reason: str - block_filters: dict[tuple[str, int], set[str]] = field(default_factory=dict) + block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index e5bbb4c3b913..45e0731b8a31 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -95,7 +95,10 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: # where a YAML entry actually applies (-k keyword check included). # Any unmatchable waive triggers full fallback — better safe than to # silently drop CI for a typo'd or out-of-tree waive id. - block_filters: dict[tuple[str, int], set[str]] = {} + # Record (prefix -> {waive_ids that resolved to it}) per block so + # write_filtered_test_db can re-check `-k` keywords against the + # original waive ids when narrowing entries. + block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} affected_blocks: list[Block] = [] seen_block_keys: set[tuple[str, int]] = set() misses: list[str] = [] @@ -108,7 +111,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: level, blocks = match for block in blocks: key = (block.yaml_stem, block.block_index) - block_filters.setdefault(key, set()).add(level) + block_filters.setdefault(key, {}).setdefault(level, set()).add(tid) if key not in seen_block_keys: seen_block_keys.add(key) affected_blocks.append(block) From b12231c4bae62fc406320f9016d3a44c0555f3f6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 14:36:37 +0800 Subject: [PATCH 42/65] [None][fix] CBTS Layer 3: regenerate cbts_test_db on each stage agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CBTS Layer 3 was producing 0 tests for every affected stage because the narrowed test-db never reached the L0_Test agents. getCbtsResult runs main.py on the L0_MergeRequest pipeline node and writes cbts_test_db/ into that agent's workspace. launchJob then triggers L0_Test as a downstream Jenkins job in a separate Kubernetes pod, which only receives the testFilter JSON parameter — the on-disk cbts_test_db/ dir is left behind. trt-test-db -d silently writes 0 lines, so every stage rendered an empty test list. Fix piggybacks the input JSON through testFilter and regenerates the narrowed dir on each stage agent: - L0_MergeRequest.groovy:getCbtsResult now sets result.cbts_input_json to the JsonOutput.toJson({changed_files, diffs}) string already written to cbts_input.json on its own agent. - L0_Test.groovy:renderTestDB checks for cbts_test_db/ on this pod and, if absent, installs python3-yaml, materializes cbts_input.json, and runs main.py. Output is deterministic so each pod's result matches what L0_MergeRequest produced. Idempotent guard avoids re-running when multiple stages share the same workspace. Plus a fallback: if the per-context narrowed YAML is missing or empty (main.py failed, yaml_stem not in affected_stems, etc.), point trt-test-db back at tests/integration/test_lists/test-db so the stage runs the full list rather than going silent. Both apt-get and main.py invocations use '|| true' so a regen failure never crashes the stage; the fallback test-db path picks up the slack. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 5 +++++ jenkins/L0_Test.groovy | 25 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index b446a9fc8711..a3c5571508f3 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -776,6 +776,11 @@ def getCbtsResult(pipeline, testFilter, globalVars) "Reasons: ${result.reasons.join('; ')}") return null } + // Layer 3 cross-job seed: piggyback the input JSON on testFilter so + // each L0_Test stage agent can re-run main.py locally and regenerate + // its own copy of cbts_test_db/. The directory written here lives on + // the L0_MergeRequest agent and never reaches downstream pods. + result.cbts_input_json = inputJson pipeline.echo("CBTS: scope=${result.scope}, " + "archs=${result.affected_cpu_arch}, " + "stages=${result.affected_stages.size()}, " + diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index c0cdacb53bbc..fff310803011 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2571,10 +2571,31 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { // Perf stages are excluded at Layer 2 (launchTestJobs) and never // reach this path with cbts != null, so no perfMode guard is needed here. def cbts = testFilter[(CBTS_RESULT)] + // Regenerate cbts_test_db/ on this stage's agent. L0_MergeRequest's + // getCbtsResult ran main.py on its own pod and produced the dir there, + // but L0_Test stages run in separate Kubernetes pods that never receive + // that dir. Re-running main.py here with the piggybacked input JSON is + // deterministic — output matches what L0_MergeRequest produced. + // Idempotent: only runs when cbts_test_db/ doesn't already exist. + if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json) { + def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" + def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() + if (dirExists != "yes") { + sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" + writeFile file: "${llmSrc}/cbts_input.json", text: cbts.cbts_input_json + sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py cbts_input.json > /dev/null 2>&1 || true" + } + } def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" if (cbts != null && cbts.test_db_dir_override) { - testDBPath = "${llmSrc}/${cbts.test_db_dir_override}" - echo "CBTS [${cbts.scope}]: rendering test list from filtered test-db at ${testDBPath}" + def overrideYaml = "${llmSrc}/${cbts.test_db_dir_override}/${testContext}.yml" + def overrideOk = sh(returnStdout: true, script: "test -s ${overrideYaml} && echo yes || echo no").trim() + if (overrideOk == "yes") { + testDBPath = "${llmSrc}/${cbts.test_db_dir_override}" + echo "CBTS [${cbts.scope}]: rendering test list from filtered test-db at ${testDBPath}" + } else { + echo "CBTS [${cbts.scope}]: ${overrideYaml} missing/empty -- falling back to source test-db" + } } def testList = "${llmSrc}/${testContext}.txt" def testDBQueryCmd = [ From 9726ce6b8c53c9059666768e22a3a5db91bf819e Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 20:36:50 +0800 Subject: [PATCH 43/65] [None][fix] CBTS Layer 3: write cbts_input.json via shell base64 to bypass Jenkins sandbox writeFile() failed on the L0_Test agent with java.nio.file.AccessDeniedException: /home/jenkins/.../TensorRT-LLM/src/cbts_input.json because Jenkins's pipeline sandbox file-IO check rejected the absolute workspace path even though the directory itself was writable for the container user. Encode the input JSON as base64 in Groovy (no shell-meta chars) and decode it via 'base64 -d' inside an sh heredoc. The shell write runs as the container user and avoids the sandbox altogether. The decoded file content is byte-identical to what writeFile would have produced. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fff310803011..dee9586a685b 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2581,8 +2581,19 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() if (dirExists != "yes") { + // writeFile() goes through Jenkins's sandbox file IO and was + // hitting AccessDeniedException on absolute workspace paths. + // Encode the JSON as base64 in Groovy and decode in-shell so + // the file is written by the container user, bypassing the + // sandbox entirely. base64 only contains [A-Za-z0-9+/=] so + // the heredoc body has no shell-meta hazards. + def encodedInput = cbts.cbts_input_json.bytes.encodeBase64().toString() sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" - writeFile file: "${llmSrc}/cbts_input.json", text: cbts.cbts_input_json + sh """\ +base64 -d > ${llmSrc}/cbts_input.json << 'CBTS_INPUT_B64_EOF' +${encodedInput} +CBTS_INPUT_B64_EOF +""" sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py cbts_input.json > /dev/null 2>&1 || true" } } From e5dd808cde12292f772c11d6a121fe3d724afa59 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:02:09 +0800 Subject: [PATCH 44/65] [None][fix] CBTS Layer 3: cap cbts_input_json piggyback at 256 KB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cbts_input_json travels to L0_Test as part of testFilter, which Jenkins serializes into a build parameter and exposes to downstream shell. Past experience with CACHED_CHANGED_FILE_LIST shows that large parameters can trigger 'Argument list too long' on the downstream invocation, so we cannot rely on the kernel ARG_MAX (typically ~2 MB) as a soft limit. Add a 256 KB hard cap. Below that, piggyback as before. Above it, log the actual size and skip the piggyback — Layer 2's affected_stages filter still narrows which stages run, and renderTestDB's fallback then uses the source test-db for the affected stages so they don't go silent. Logs the size on every CBTS run so we can monitor growth and adjust the threshold if needed. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index a3c5571508f3..ca634bc6dfea 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -780,7 +780,24 @@ def getCbtsResult(pipeline, testFilter, globalVars) // each L0_Test stage agent can re-run main.py locally and regenerate // its own copy of cbts_test_db/. The directory written here lives on // the L0_MergeRequest agent and never reaches downstream pods. - result.cbts_input_json = inputJson + // + // Hard cap to keep us well below ARG_MAX (~2 MB on most kernels) and + // Jenkins's per-parameter handling. CACHED_CHANGED_FILE_LIST already + // hits "Argument list too long" at smaller sizes (see comment on + // launchJob), so 256 KB is conservative. If we exceed it, drop the + // piggyback — Layer 2 stage filtering still applies, and renderTestDB + // falls back to the source test-db automatically. + final int CBTS_INPUT_PIGGYBACK_MAX_BYTES = 256000 + def inputJsonSize = inputJson.length() + if (inputJsonSize <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { + result.cbts_input_json = inputJson + pipeline.echo("CBTS Layer 3: cbts_input_json piggyback enabled (${inputJsonSize} bytes)") + } else { + pipeline.echo("CBTS Layer 3: cbts_input_json is ${inputJsonSize} bytes, " + + "exceeds ${CBTS_INPUT_PIGGYBACK_MAX_BYTES}-byte piggyback limit; " + + "downstream stages will fall back to source test-db " + + "(Layer 2 stage filtering still applies)") + } pipeline.echo("CBTS: scope=${result.scope}, " + "archs=${result.affected_cpu_arch}, " + "stages=${result.affected_stages.size()}, " + From a72c4dd30f9deb1cc528694378b5fb2dfef66c00 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:13:38 +0800 Subject: [PATCH 45/65] [None][refactor] CBTS Layer 3: replace base64 hack with Utils.createTempLocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix (write cbts_input.json via base64 + sh heredoc) worked but was idiosyncratic — it grew out of debugging writeFile()'s AccessDeniedException without consulting the rest of L0_Test.groovy. The codebase already has a JNLP-friendly temp-file convention used by the SLURM path (scriptLaunchPathLocal, scriptSubmitPathLocal, scriptTrackPathLocal etc.): def pathLocal = Utils.createTempLocation(pipeline, './name') pipeline.writeFile(file: pathLocal, text: contents) Utils.createTempLocation returns a path that the JNLP container's jenkins user can write to, so pipeline.writeFile no longer collides with the root-owned / tree the build container unpacked. Pass the absolute temp path to main.py — its argparse already accepts any path for input_json (Path(args.input_json)). The output dir cbts_test_db/ still lands at /cbts_test_db/ because main.py defaults --repo-root to its cwd, and we 'cd ' before invoking. No behavior change. Just style and consistency. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index dee9586a685b..dfdff63e2259 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2581,20 +2581,14 @@ def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() if (dirExists != "yes") { - // writeFile() goes through Jenkins's sandbox file IO and was - // hitting AccessDeniedException on absolute workspace paths. - // Encode the JSON as base64 in Groovy and decode in-shell so - // the file is written by the container user, bypassing the - // sandbox entirely. base64 only contains [A-Za-z0-9+/=] so - // the heredoc body has no shell-meta hazards. - def encodedInput = cbts.cbts_input_json.bytes.encodeBase64().toString() + // Write input JSON to a JNLP-writable temp location instead of + // ${llmSrc}/, which the build container created as root and + // would reject writeFile() with AccessDeniedException. Matches + // the scriptLaunch*PathLocal pattern used in the SLURM path. + def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") + pipeline.writeFile(file: cbtsInputLocal, text: cbts.cbts_input_json) sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" - sh """\ -base64 -d > ${llmSrc}/cbts_input.json << 'CBTS_INPUT_B64_EOF' -${encodedInput} -CBTS_INPUT_B64_EOF -""" - sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py cbts_input.json > /dev/null 2>&1 || true" + sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py ${cbtsInputLocal} > /dev/null 2>&1 || true" } } def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" From 9cd8da2516abed1d374286776f91dfc201584afb Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:38:56 +0800 Subject: [PATCH 46/65] [None][fix] CBTS Layer 3: pass pipeline through renderTestDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Utils.createTempLocation(pipeline, ...) and pipeline.writeFile(...) need the script-context pipeline binding to dispatch the underlying pwd() and writeFile() steps. renderTestDB() lacked a pipeline parameter, so the bare 'pipeline' identifier inside its body resolved to the declarative ModelInterpreter delegate, which has no such methods — producing 'No signature of method: ...ModelInterpreter.pwd()'. Add pipeline as the first positional parameter and propagate it from both call sites (runLLMTestlistWithSbatch and runLLMTestlistOnPlatformImpl both already receive pipeline themselves). No behavior change beyond fixing the dispatch lookup. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index dfdff63e2259..23947b46c038 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1282,7 +1282,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // line is "Mako options:", maybe we can make it more generic, which // if the line cannot be split by "=", just ignore that line. def makoOptsJson = transformMakoArgsToJson(["Mako options:"] + makoArgs) - def testListPathLocal = renderTestDB(testList, llmSrcLocal, stageName, makoOptsJson) + def testListPathLocal = renderTestDB(pipeline, testList, llmSrcLocal, stageName, makoOptsJson) Utils.copyFileToRemoteHost( pipeline, remote, @@ -2557,7 +2557,7 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { return makoArgs } -def renderTestDB(testContext, llmSrc, stageName, preDefinedMakoOpts=null) { +def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=null) { def makoOpts = preDefinedMakoOpts if (!makoOpts) { @@ -3123,7 +3123,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def noRegularTests = false def noIsolateTests = false def rerunFailed = false - def testDBList = renderTestDB(testList, llmSrc, stageName) + def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName) // Download and Merge waives.txt mergeWaivesTxt(pipeline, llmSrc, stageName) From 7c515547aabb3610526d0e6aa5ecba0d42728124 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 10:45:02 +0800 Subject: [PATCH 47/65] [None][fix] CBTS Layer 3: tolerate pytest exit 5 on SLURM path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The K8s test path already tolerates 'no tests in shard after narrowing' via the noRegularTests && noIsolateTests check at runLLMTestlistOnPlatformImpl (commit 9820ed65c). The SLURM path (runLLMTestlistWithSbatch -> slurm_run.sh) had no equivalent guard, so pytest-split assigning 0 cases to some group exited 5 and the SLURM job went FAILED. Two changes mirror the K8s tolerance: - runLLMTestlistWithSbatch computes cbtsActive from testFilter[CBTS_RESULT].test_db_dir_override and exports it into the generated slurm_launch.sh alongside the existing stageName / perfMode / pytestCommand env vars. - slurm_run.sh, immediately after capturing pytest_exit_code, rewrites exit 5 to 0 when cbtsActive == 'true'. This treats a 0-test shard the same way the K8s path's noRegularTests && noIsolateTests branch does — log it and continue, since other shards run the affected tests. Hit on DGX_B200-PyTorch-2 group 2/3 with the 3-waive testing PR: the narrowed l0_b200 block had 1 case (test_moe_backend -k 'CUTLASS'), pytest-split assigned 0 to group 2, pytest exited 5, slurm job FAILED. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 6 ++++++ jenkins/scripts/slurm_run.sh | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 23947b46c038..84b32bcf096e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1436,6 +1436,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "export ${varName}=\"${escapedValue}\"" }.join('\n') + // CBTS Layer 3 may narrow this stage's shard list below pytest-split's + // --splits count, leaving some shards with 0 tests → pytest exits 5. + // slurm_run.sh tolerates exit 5 only when this env var is "true". + def cbtsActive = testFilter[(CBTS_RESULT)]?.test_db_dir_override ? "true" : "false" + def scriptLaunchPrefix = """#!/bin/bash #SBATCH ${exemptionComment} #SBATCH --output=${slurmJobLogPath} @@ -1458,6 +1463,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" + export cbtsActive=$cbtsActive export HF_TOKEN=$HF_TOKEN export NVIDIA_IMEX_CHANNELS=\${NVIDIA_IMEX_CHANNELS:-0} export NVIDIA_VISIBLE_DEVICES=\${NVIDIA_VISIBLE_DEVICES:-\$(seq -s, 0 \$((\$(nvidia-smi --query-gpu=count -i 0 --format=csv,noheader)-1)))} diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index a8ca9b006eba..fd50b92905ff 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -113,6 +113,16 @@ eval $pytestCommand pytest_exit_code=$? echo "Rank${SLURM_PROCID} Pytest finished execution with exit code $pytest_exit_code" +# CBTS Layer 3 may narrow this shard's test list below pytest-split's +# --splits count, leaving some groups with 0 tests. pytest then exits 5 +# (no tests collected). Treat exit 5 as success when CBTS is active — +# other shards run the actual affected tests. This mirrors the K8s-path +# tolerance in runLLMTestlistOnPlatformImpl (noRegularTests && noIsolateTests). +if [ "$pytest_exit_code" -eq 5 ] && [ "${cbtsActive:-false}" = "true" ]; then + echo "Rank${SLURM_PROCID} CBTS Layer 3: 0 tests in this shard after narrowing — marking as success" + pytest_exit_code=0 +fi + # DEBUG: Diagnose intermittent "unrecognized arguments" failure (Exit Code 4) # Remove this after the issue is resolved if [ $pytest_exit_code -eq 4 ]; then From c5b52dbe825028ec23777d5c5b9db4946d708298 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 13:02:27 +0800 Subject: [PATCH 48/65] [None][feat] CBTS Layer 3: collapse pytest-split to splits=1 below 20 cases Without this change, when CBTS narrows an affected block to a small number of cases (e.g. 1), the stage's default pytest-split splits (typically 3) still allocate that many machines: only one shard runs the case and the other two start up, download wheel for ~10 min, then exit pytest=5 with 'no tests collected'. Wasteful and noisy. Add a per-stage narrowed-count heuristic with a hard-coded threshold of 20: when the count is below 20, collapse the stage to splits=1 (only group 1 runs everything; groups 2..N skip without allocating machines). At or above 20, the default splits stand and pytest-split parallelizes normally. Implementation: - blocks.py: new compute_stage_test_counts() helper that mirrors write_filtered_test_db's keep filter to count surviving entries per affected stage (sum across blocks the stage's mako matches). - main.py: SelectionResult gains affected_stage_test_counts; populated after write_filtered_test_db; serialized into the stdout JSON. - L0_MergeRequest.groovy:_cbtsParseSelectionResult: read the new field. - L0_Test.groovy: _cbtsMaybeCollapseSplits() helper consults the count and the 20-case threshold; called at the entry of runLLMTestlistOnSlurm and runLLMTestlistOnPlatform. Returns either skip=true (caller does early return; no machine allocated) or new splits/splitId values (caller overwrites locals before existing dispatch). Verified locally on the 3-waive testing PR: every affected stage has count in {1, 5}, all collapse to splits=1; only group 1 of each stage will run, no machines wasted on empty groups. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 +++ jenkins/L0_Test.groovy | 46 ++++++++++++++++++++++++++++++ jenkins/scripts/cbts/blocks.py | 51 ++++++++++++++++++++++++++++++++++ jenkins/scripts/cbts/main.py | 28 ++++++++++++++++++- jenkins/scripts/slurm_run.sh | 7 ++++- 5 files changed, 134 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index ca634bc6dfea..6dbeb4912922 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -874,6 +874,10 @@ def _cbtsParseSelectionResult(String text) affected_tests: data.tests ?: [], reasons: data.reasons ?: [], test_db_dir_override: data.test_db_dir_override, // Layer 3: tmp test-db path + // Layer 3 split-collapse heuristic: per-stage narrowed test count. + // launchTestJobs reads this to drop excess pytest-split groups when + // the affected stage's narrowed count falls below the 20-test threshold. + affected_stage_test_counts: data.affected_stage_test_counts ?: [:], ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 84b32bcf096e..a9cd44a2a355 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1698,8 +1698,47 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } } +// CBTS Layer 3 split-collapse heuristic: when the affected stage's +// narrowed test count (computed by main.py compute_stage_test_counts) +// is below this threshold, collapse pytest-split's splits to 1 — only +// group 1 runs everything; groups 2..N skip and don't even allocate a +// machine. Above the threshold, splits stay at the stage's default and +// pytest-split parallelizes normally. +def _CBTS_SPLIT_COLLAPSE_THRESHOLD = 20 + +// Decide whether to collapse this stage's splits or skip the group entirely. +// Returns a map [skip: bool, splits: int, splitId: int]. Callers should +// `return` early when skip == true and otherwise overwrite splits/splitId +// with the returned values before continuing their existing dispatch logic. +def _cbtsMaybeCollapseSplits(stageName, splitId, splits) { + def cbts = testFilter[(CBTS_RESULT)] + def counts = cbts?.affected_stage_test_counts + if (!counts) { + return [skip: false, splits: splits, splitId: splitId] + } + def count = counts[stageName] + if (count == null || count >= _CBTS_SPLIT_COLLAPSE_THRESHOLD) { + return [skip: false, splits: splits, splitId: splitId] + } + if (splitId > 1) { + echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< ${_CBTS_SPLIT_COLLAPSE_THRESHOLD}), skipping group ${splitId}/${splits}" + return [skip: true, splits: splits, splitId: splitId] + } + if (splits > 1) { + echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< ${_CBTS_SPLIT_COLLAPSE_THRESHOLD}), collapsing splits=${splits} → 1" + } + return [skip: false, splits: 1, splitId: 1] +} + def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312") { + def collapse = _cbtsMaybeCollapseSplits(stageName, splitId, splits) + if (collapse.skip) { + return + } + splits = collapse.splits + splitId = collapse.splitId + echo "Run Slurm job with native sbatch: $runWithSbatch" def attempt = 0 @@ -3338,6 +3377,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // and junit() for intermediate retryable failures). def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", postTag="", typeCheck=false, boolean isFinalAttempt=true) { + def collapse = _cbtsMaybeCollapseSplits(stageName, splitId, splits) + if (collapse.skip) { + return + } + splits = collapse.splits + splitId = collapse.splitId + cacheErrorAndUploadResult(stageName, { runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, typeCheck) }, { diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index a9a72937065c..b87be2efe72b 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -507,6 +507,57 @@ def block_matches_stage(block: Block, stage: Stage) -> bool: return True +# --------------------------------------------------------------------------- +# CBTS Layer 3 split-count heuristic: per-stage narrowed test count +# --------------------------------------------------------------------------- + + +def compute_stage_test_counts( + yaml_index: "YAMLIndex", + stages: dict[str, "Stage"], + affected_stages: set[str], + block_filters: dict[tuple[str, int], dict[str, set[str]]], +) -> dict[str, int]: + """Sum the kept-test count per affected stage across matching blocks. + + Used by Groovy launchTestJobs to decide whether to collapse the stage's + pytest-split splits to 1 (when narrowed_count < 20). The keep filter + here mirrors the one in `write_filtered_test_db` exactly so the count + matches what trt-test-db will eventually render. + """ + block_by_key: dict[tuple[str, int], Block] = { + (b.yaml_stem, b.block_index): b for b in yaml_index.blocks + } + counts: dict[str, int] = {} + for stage_name in affected_stages: + stage = stages.get(stage_name) + if stage is None: + continue + total = 0 + for (yaml_stem, idx), prefix_to_waives in block_filters.items(): + if yaml_stem != stage.yaml_stem: + continue + block = block_by_key.get((yaml_stem, idx)) + if block is None or not block_matches_stage(block, stage): + continue + kept: list[str] = [] + for t in block.tests: + target = _entry_target(t) + matched_waives: set[str] = set() + for prefix, waives in prefix_to_waives.items(): + if _target_in_filter_subtree(target, prefix): + matched_waives |= waives + if not matched_waives: + continue + if any(_entry_applies_to_waive(t, w) for w in matched_waives): + kept.append(t) + # Safety fallback mirrors write_filtered_test_db: when the + # narrowing would empty the block, the original tests stay. + total += len(kept) if kept else len(block.tests) + counts[stage_name] = total + return counts + + # --------------------------------------------------------------------------- # CBTS Layer 3: filtered test-db YAML generation # --------------------------------------------------------------------------- diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 8f18ad51d475..ccd324851e99 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -47,7 +47,13 @@ # Make sibling modules importable when invoked as `python3 /main.py ...`. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from blocks import Stage, YAMLIndex, parse_stages_from_groovy, write_filtered_test_db # noqa: E402 +from blocks import ( # noqa: E402 + Stage, + YAMLIndex, + compute_stage_test_counts, + parse_stages_from_groovy, + write_filtered_test_db, +) from rules.base import PRInputs, Rule, RuleResult # noqa: E402 from rules.waives_rule import WaivesRule # noqa: E402 @@ -82,6 +88,10 @@ class SelectionResult: reasons: list[str] = field(default_factory=list) block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) test_db_dir_override: Optional[str] = None + # Per-stage narrowed test count (sum across blocks the stage's mako + # matches, post-keep-filter). Groovy launchTestJobs uses this to + # collapse splits to 1 when the count is below the 20-test threshold. + affected_stage_test_counts: dict[str, int] = field(default_factory=dict) def to_json(self) -> str: data = { @@ -91,6 +101,7 @@ def to_json(self) -> str: "tests": sorted(self.tests), "reasons": list(self.reasons), "test_db_dir_override": self.test_db_dir_override, + "affected_stage_test_counts": dict(self.affected_stage_test_counts), } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -284,6 +295,14 @@ def main(argv: Optional[list[str]] = None) -> int: block_filters=result.block_filters, ) result.test_db_dir_override = out_dir_name + # Per-stage narrowed test count for the launchTestJobs split-collapse + # heuristic (collapse pytest-split to splits=1 when count < 20). + result.affected_stage_test_counts = compute_stage_test_counts( + yaml_index=yaml_index, + stages=stages, + affected_stages=set(result.affected_stages), + block_filters=result.block_filters, + ) _log_decision_to_stderr(stages, result) sys.stdout.write(result.to_json()) @@ -316,6 +335,13 @@ def _log_decision_to_stderr(stages: dict[str, Stage], result: SelectionResult) - print(f" - {yaml_stem}#{idx}:", file=out) for prefix, waives in sorted(prefix_to_waives.items()): print(f" {prefix} <- {sorted(waives)}", file=out) + if result.affected_stage_test_counts: + print( + f" affected_stage_test_counts ({len(result.affected_stage_test_counts)}):", + file=out, + ) + for name, count in sorted(result.affected_stage_test_counts.items()): + print(f" - {name}: {count}", file=out) print(f" affected_stages ({len(result.affected_stages)}):", file=out) for name in sorted(result.affected_stages): stage = stages.get(name) diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index fd50b92905ff..9fba0ee66900 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -102,8 +102,13 @@ if [ "${SLURM_JOB_NUM_NODES:-1}" -eq 1 ] || \ done fi -# Turn off "exit on error" so the following lines always run +# Turn off "exit on error" so the following lines always run. +# `set -E` (errtrace, set on line 4) makes the ERR trap fire even after +# `set +e`, so without removing the trap a non-zero pytest exit would +# trigger the trap's `exit $rc` and skip the rest of this script +# (including the CBTS exit-5 tolerance below). set +e +trap - ERR pytest_exit_code=0 perf_check_exit_code=0 From ef0e9b4066eb1ed325a73c89752073c0457facb2 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 15:24:50 +0800 Subject: [PATCH 49/65] [None][fix] CBTS: inline split-collapse threshold to fix MissingPropertyException The previous commit declared the threshold via top-level `def _CBTS_SPLIT_COLLAPSE_THRESHOLD = 20` and referenced it inside the helper method. Groovy script-level `def` does not enter the binding visible to method bodies (same trap that bit pwd() / pipeline earlier), so runtime threw groovy.lang.MissingPropertyException: No such property: _CBTS_SPLIT_COLLAPSE_THRESHOLD for class: WorkflowScript Inline the literal 20 in the three reference sites and drop the top-level def. The value is only used by this single helper, so inlining is the simplest fix and avoids reintroducing the binding issue. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a9cd44a2a355..8734e438b2e1 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1698,16 +1698,17 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } } -// CBTS Layer 3 split-collapse heuristic: when the affected stage's -// narrowed test count (computed by main.py compute_stage_test_counts) -// is below this threshold, collapse pytest-split's splits to 1 — only -// group 1 runs everything; groups 2..N skip and don't even allocate a -// machine. Above the threshold, splits stay at the stage's default and -// pytest-split parallelizes normally. -def _CBTS_SPLIT_COLLAPSE_THRESHOLD = 20 - -// Decide whether to collapse this stage's splits or skip the group entirely. -// Returns a map [skip: bool, splits: int, splitId: int]. Callers should +// CBTS Layer 3 split-collapse heuristic: when the affected stage's narrowed +// test count (from main.py compute_stage_test_counts) is below 20, collapse +// pytest-split's splits to 1 — only group 1 runs everything; groups 2..N +// skip and don't allocate a machine. At/above 20, splits stay at the +// stage's default and pytest-split parallelizes normally. +// +// The 20 below is hard-coded inline rather than a top-level constant +// because Groovy script-level `def` is not visible from method bodies +// without `@Field`, and we don't otherwise need this value elsewhere. +// +// Returns [skip: bool, splits: int, splitId: int]. Callers should // `return` early when skip == true and otherwise overwrite splits/splitId // with the returned values before continuing their existing dispatch logic. def _cbtsMaybeCollapseSplits(stageName, splitId, splits) { @@ -1717,15 +1718,15 @@ def _cbtsMaybeCollapseSplits(stageName, splitId, splits) { return [skip: false, splits: splits, splitId: splitId] } def count = counts[stageName] - if (count == null || count >= _CBTS_SPLIT_COLLAPSE_THRESHOLD) { + if (count == null || count >= 20) { return [skip: false, splits: splits, splitId: splitId] } if (splitId > 1) { - echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< ${_CBTS_SPLIT_COLLAPSE_THRESHOLD}), skipping group ${splitId}/${splits}" + echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< 20), skipping group ${splitId}/${splits}" return [skip: true, splits: splits, splitId: splitId] } if (splits > 1) { - echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< ${_CBTS_SPLIT_COLLAPSE_THRESHOLD}), collapsing splits=${splits} → 1" + echo "CBTS [${cbts.scope}]: ${stageName} narrowed to ${count} (< 20), collapsing splits=${splits} → 1" } return [skip: false, splits: 1, splitId: 1] } From 0c3c5b95d8f9f1cd36b9c52cfe86fab1a6daaae8 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 18:37:34 +0800 Subject: [PATCH 50/65] [None][refactor] CBTS Layer 3: drop empty-shard safety nets superseded by split-collapse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split-collapse heuristic (commit 2e0fd25be / ee7cdd9e9) makes empty shards impossible in the CBTS path — count<20 collapses to splits=1 (single group runs all narrowed cases), and count>=20 keeps default splits with at least 7 cases per group. Either way, pytest never sees 'no tests collected' (exit 5) and no shard ends up empty. Three previously-added safety nets are therefore dead code: - slurm_run.sh: 'trap - ERR' workaround and CBTS exit-5 if-block. Both were added to tolerate empty SLURM shards. Restore the file to its state before the CBTS work touched it (the underlying 'set -E + ERR trap + set +e' bug is a pre-existing latent issue in this repo; scope-creep fixing it here is unnecessary now). - L0_Test.groovy:runLLMTestlistWithSbatch: drop the 'def cbtsActive' Groovy local and the corresponding 'export cbtsActive=$cbtsActive' in the slurm_launch.sh heredoc. They only fed the slurm_run.sh if-block above. - L0_Test.groovy:runLLMTestlistOnPlatformImpl 'noRegularTests && noIsolateTests' cbtsActive branch (commit 9820ed65c). Restored to the original sanity-check 'error "No tests were executed..."' because the K8s path also can no longer hit the empty-shard condition, and a real misconfiguration outside CBTS still deserves a hard error. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 17 +---------------- jenkins/scripts/slurm_run.sh | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 8734e438b2e1..244f69383d10 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1436,11 +1436,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "export ${varName}=\"${escapedValue}\"" }.join('\n') - // CBTS Layer 3 may narrow this stage's shard list below pytest-split's - // --splits count, leaving some shards with 0 tests → pytest exits 5. - // slurm_run.sh tolerates exit 5 only when this env var is "true". - def cbtsActive = testFilter[(CBTS_RESULT)]?.test_db_dir_override ? "true" : "false" - def scriptLaunchPrefix = """#!/bin/bash #SBATCH ${exemptionComment} #SBATCH --output=${slurmJobLogPath} @@ -1463,7 +1458,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" - export cbtsActive=$cbtsActive export HF_TOKEN=$HF_TOKEN export NVIDIA_IMEX_CHANNELS=\${NVIDIA_IMEX_CHANNELS:-0} export NVIDIA_VISIBLE_DEVICES=\${NVIDIA_VISIBLE_DEVICES:-\$(seq -s, 0 \$((\$(nvidia-smi --query-gpu=count -i 0 --format=csv,noheader)-1)))} @@ -3286,16 +3280,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } if (noRegularTests && noIsolateTests) { - // CBTS Layer 3 may narrow a block's tests below the stage's - // split count, leaving some shards with 0 tests. That's - // expected — other shards run the affected tests. Only - // raise the sanity-check error when CBTS isn't active. - def cbtsActive = testFilter[(CBTS_RESULT)]?.test_db_dir_override - if (cbtsActive) { - echo "CBTS Layer 3: shard ${splitId}/${splits} got 0 tests after narrowing — other shards run the affected tests. Marking as success." - } else { - error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." - } + error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." } } } diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index 9fba0ee66900..a8ca9b006eba 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -102,13 +102,8 @@ if [ "${SLURM_JOB_NUM_NODES:-1}" -eq 1 ] || \ done fi -# Turn off "exit on error" so the following lines always run. -# `set -E` (errtrace, set on line 4) makes the ERR trap fire even after -# `set +e`, so without removing the trap a non-zero pytest exit would -# trigger the trap's `exit $rc` and skip the rest of this script -# (including the CBTS exit-5 tolerance below). +# Turn off "exit on error" so the following lines always run set +e -trap - ERR pytest_exit_code=0 perf_check_exit_code=0 @@ -118,16 +113,6 @@ eval $pytestCommand pytest_exit_code=$? echo "Rank${SLURM_PROCID} Pytest finished execution with exit code $pytest_exit_code" -# CBTS Layer 3 may narrow this shard's test list below pytest-split's -# --splits count, leaving some groups with 0 tests. pytest then exits 5 -# (no tests collected). Treat exit 5 as success when CBTS is active — -# other shards run the actual affected tests. This mirrors the K8s-path -# tolerance in runLLMTestlistOnPlatformImpl (noRegularTests && noIsolateTests). -if [ "$pytest_exit_code" -eq 5 ] && [ "${cbtsActive:-false}" = "true" ]; then - echo "Rank${SLURM_PROCID} CBTS Layer 3: 0 tests in this shard after narrowing — marking as success" - pytest_exit_code=0 -fi - # DEBUG: Diagnose intermittent "unrecognized arguments" failure (Exit Code 4) # Remove this after the issue is resolved if [ $pytest_exit_code -eq 4 ]; then From a72593c71c89a433fc8008c01bbfa56a65b7ef50 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 18:59:38 +0800 Subject: [PATCH 51/65] [None][doc] CBTS: bring README up to date with recent Layer 3 work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README had drifted from the code over the past week. Bring it back in sync with eight gaps: - Consumption layers table now includes Layer 2.5 (split-collapse) and notes that Perf stages are excluded from Layer 2 narrowing. - Layer 3 row records that unaffected blocks are now dropped from the narrowed YAML (commit 2d6d9bc05) — the prior 'each block's tests array filtered' wording understated this. - block_filters schema in 'Adding a new rule' switches from set[prefix] to dict[prefix, set[waive_id]] (commit cbe0c1f19) so rule authors emit data the -k keyword guard can reuse. - New section 'Cross-job seed for stage agents' documents the cbts_input_json piggyback through testFilter, renderTestDB's Utils.createTempLocation + main.py rerun, and the 256 KB cap. - New section 'Split-collapse heuristic (Layer 2.5)' explains the threshold-20 collapse, the splitId == 1 / > 1 dispatch, and where the count comes from (compute_stage_test_counts). - Decision JSON example gains affected_stage_test_counts and a bullet describing it. - Lookup-algorithm section spells out that the -k keyword guard runs twice — once at lookup, again at write — with an example. - Fallback paths picks up two new bullets: 256 KB cap exceeded; and narrowed YAML missing/empty on the stage agent. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 102 +++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 17 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index a965628c0ad3..002ae247b2ce 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -7,13 +7,14 @@ consumes the data directly.** --- -## Three consumption layers +## Consumption layers | Layer | Where | Action | |---|---|---| | **1. Arch track** | `L0_MergeRequest.groovy::launchStages` | Skip x86 / SBSA track when no stage on that arch is affected | -| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset | -| **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the CBTS-narrowed tmp test-db (each affected block's `tests:` array filtered to the per-block filter prefix subtree) | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset (Perf stages excluded — they have their own trigger model and need full lists) | +| **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOnSlurm` and `runLLMTestlistOnPlatform` entries | When the affected stage's narrowed test count is < 20, collapse pytest-split's splits to 1 — only group 1 runs everything; groups 2..N skip without allocating a machine. At/above 20 the stage's default splits stand and pytest-split parallelizes normally. | +| **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the CBTS-narrowed tmp test-db. Each affected block's `tests:` array is filtered to entries in the per-block filter prefix subtree, **and unaffected blocks are dropped entirely** so a `/bot run --post-merge` can't accidentally activate post-merge blocks the PR never touched. | CBTS only **subtracts** stages and tests, never adds. Anything it can't narrow → full fallback to the existing filter chain. @@ -30,7 +31,7 @@ narrow → full fallback to the existing filter chain. jenkins/scripts/cbts/ ├── README.md this file ├── main.py CLI entry + Selector + SelectionResult -├── blocks.py YAML index + lookup + filtered tmp test-db generation +├── blocks.py YAML index + lookup + filtered tmp test-db generation + per-stage count └── rules/ ├── README.md per-rule logic summary ├── base.py Rule ABC + PRInputs + RuleResult @@ -41,7 +42,9 @@ jenkins/scripts/cbts/ Per waive id, `YAMLIndex.find_match_for_waive` walks the pytest tree from the waive towards the root. The first level whose YAML has a matching entry wins; -that level becomes the **filter prefix** the block uses for Layer 3. +that level becomes the **filter prefix** the block uses for Layer 3. Each +prefix remembers the originating waive id(s) so `write_filtered_test_db` can +re-apply the `-k` keyword guard when narrowing. ``` waive id (raw) @@ -50,7 +53,7 @@ waive id (raw) target_lookup (function-level when waive was parametrized; otherwise class/file/dir level — waive's own granularity) ↓ try YAML at this level - hit → matched: filter prefix = level + hit → matched: filter prefix = level (recorded with the originating waive id) miss → strip one level up (::method → ::class → /file → /dir → ...) and retry ↓ all levels miss → fallback: rule emits scope=None, baseline runs @@ -63,6 +66,15 @@ entry carries actually contains an identifier present in the waive id. `-m` markers are unverifiable from a string and always pass (over-include when in doubt). +The `-k` keyword guard is applied **twice** by design: + +1. **At lookup** (`find_match_for_waive`) — to decide whether the entry + contributes to a block's filter prefix. +2. **At write** (`write_filtered_test_db`) — to drop sibling `-k "..."` + entries that survive prefix-subtree match but whose keyword can't pick + up the waived test (e.g., waive `func[CUTLASS-fp8-tp4]` keeps + `-k "CUTLASS"` but not `-k "TRTLLM"`). + ## When CBTS activates CBTS narrows test selection in **two usages only**: @@ -83,17 +95,17 @@ the existing filter chain takes over: `--stage-list`, `--extra-stage`, ## How it's invoked (CI) -`getCbtsResult` calls `main.py` twice: +`getCbtsResult` calls `main.py` twice on the L0_MergeRequest agent: 1. `main.py --list-needed-diffs` → patterns whose diffs Groovy fetches. Patterns are **Ant-style globs** (`tests/**/*.py`, `cpp/kernels/**`, exact paths), matched via `hudson.util.AntPathMatcher`. 2. `main.py cbts_input.json` → decision JSON on stdout. If any block was narrowed, also writes `${LLM_ROOT}/cbts_test_db/` containing only the - affected YAMLs with their filtered `tests:` arrays. Each kept entry - preserves `TIMEOUT (n)`, `ISOLATION`, `-k "..."`, `-m "..."` verbatim - (YAML-level `# comments` are dropped by PyYAML round-trip but no - functional info is lost). + affected YAMLs with only their affected blocks (others dropped). Each + kept entry preserves `TIMEOUT (n)`, `ISOLATION`, `-k "..."`, `-m "..."` + verbatim (YAML-level `# comments` are dropped by PyYAML round-trip but + no functional info is lost). Decision JSON: @@ -104,7 +116,8 @@ Decision JSON: "affected_stages": ["A10-PyTorch-1", "A10-PyTorch-2"], "tests": ["unittest/utils/test_util.py"], "reasons": ["[waives] waives.txt: +1 / -0 → 1 blocks, 2 stages"], - "test_db_dir_override": "cbts_test_db" + "test_db_dir_override": "cbts_test_db", + "affected_stage_test_counts": {"A10-PyTorch-1": 5, "A10-PyTorch-2": 5} } ``` @@ -112,6 +125,48 @@ Decision JSON: scope value — it's metadata for logs and multi-rule combining only. - `test_db_dir_override: null` → no Layer 3 narrowing; trt-test-db reads the source `tests/integration/test_lists/test-db/` as before. +- `affected_stage_test_counts` → per-stage post-keep-filter test count. + Drives Layer 2.5 split-collapse below. + +## Cross-job seed for stage agents + +The `cbts_test_db/` written above lives on the L0_MergeRequest pipeline pod +and never reaches downstream `L0_Test-*` jobs (separate Kubernetes pods / +SLURM nodes). To make the narrowed test-db available to each stage agent +without a cross-job stash: + +1. `getCbtsResult` puts the **input JSON itself** (`changed_files` + diffs) + into `result.cbts_input_json`, which rides along inside `testFilter` as + a normal build parameter. +2. `renderTestDB` on the stage agent receives it, writes a temp + `cbts_input.json` (via `Utils.createTempLocation` → JNLP-writable + path), and re-runs `python3 jenkins/scripts/cbts/main.py ` so + the narrowed `cbts_test_db/` materializes locally alongside the + source. main.py is deterministic, so each agent ends up with a + byte-identical copy of what L0_MergeRequest produced. +3. trt-test-db then queries `cbts_test_db/` as usual. + +**Size cap.** `cbts_input_json` is dropped from the piggyback when its +size exceeds 256 KB (well below ARG_MAX). Layer 2 stage filtering still +applies, but Layer 3 narrowing on each stage agent silently degrades to +"no override" and `renderTestDB` falls back to the source test-db. + +## Split-collapse heuristic (Layer 2.5) + +When the affected stage's narrowed test count is below the hard-coded +threshold of **20** (in `_cbtsMaybeCollapseSplits`): + +- `splitId == 1` → keep, override `splits = 1` so this single agent runs + the full narrowed list. +- `splitId > 1` → early `return`; no agent allocated. + +At/above the threshold, the stage's default splits stand and pytest-split +parallelizes normally. + +The per-stage count is computed by `blocks.compute_stage_test_counts`, +which sums kept entries across blocks the stage's mako matches. The same +keep filter as `write_filtered_test_db` is applied so the count matches +what trt-test-db will eventually render. ## Adding a new rule @@ -138,16 +193,24 @@ Decision JSON: affected_stages={...}, scope="myscope", reason="why this fired", - # Optional Layer 3 contribution: per-block filter prefixes. - # Selector unions across rules and writes the tmp test-db. - block_filters={(yaml_stem, block_index): {filter_prefix}, ...}, + # Optional Layer 3 contribution: per-block prefix → set of + # waive ids that resolved to it. Selector unions across + # rules; write_filtered_test_db uses both the prefix + # (subtree match) AND the waive ids (-k keyword guard). + block_filters={ + (yaml_stem, block_index): { + filter_prefix: {originating_waive_id, ...}, + }, + ... + }, ) ``` 2. **Register in `main.py`**: add to `RULE_CLASSES` and `build_rules()`. -3. **No Groovy edits needed.** Layer 1 / 2 / 3 are scope-agnostic and consume - `affected_cpu_arch` / `affected_stages` / `block_filters` directly. +3. **No Groovy edits needed.** Layers 1 / 2 / 2.5 / 3 are scope-agnostic and + consume `affected_cpu_arch` / `affected_stages` / `block_filters` / + `affected_stage_test_counts` directly. Rule order is irrelevant. `Selector` unions `affected_stages` and `block_filters`; scopes are combined via `_combine_scopes` (all-agree → that @@ -166,6 +229,11 @@ CBTS falls back to the existing filter chain when: - `affected_stages` is empty (Layer 2 no-op) - Layer 3 filter would empty a block's `tests:` array — that block keeps its original tests instead (per-block safety net) +- `cbts_input_json` exceeds the 256 KB piggyback cap — Layer 3 narrowing + is dropped per stage; renderTestDB falls back to source test-db +- The narrowed YAML for this stage's testContext is missing or empty on + the stage agent (e.g., main.py regen failed) — renderTestDB falls back + to source test-db Every fallback logs an `echo` line — no silent failures. From be4389bf062de9e66c2e560da8f2c94f7817fb2a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 1 May 2026 19:01:30 +0800 Subject: [PATCH 52/65] [None][chore] revert TEMP Debug build skip workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the `false &&` short-circuit added during CBTS testing to unblock CI under SLURM OOM (commit df6c083b5). The Debug build smoke stage is the gate that keeps Debug compilation from rotting; restoring it is required before merge. Stage keeps its original opt-out via `reuseArtifactPath`: when a PR reuses a prior build's artifacts, the Debug smoke stage still skips — no behavior change there. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/Build.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 011f1cf43fc4..4798c4695ab3 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -550,9 +550,7 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) }]} parallelJobs.failFast = enableFailFast - // TEMP(cbts-v0): skip "Build With Build Type Debug" smoke stage to unblock CBTS testing under OOM. - // REVERT BEFORE MERGE — this stage is the gate that prevents Debug build from rotting. - if (false && cpu_arch == X86_64_TRIPLE && !reuseArtifactPath) { + if (cpu_arch == X86_64_TRIPLE && !reuseArtifactPath) { def key = "Build With Build Type Debug" parallelJobs += [ (key): { From bc9b41fa9d6b9e0b8b5dd81dedf6e142c2ac9f4f Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Sat, 2 May 2026 18:49:00 +0800 Subject: [PATCH 53/65] [None][chore] revert TESTING entries used to validate CBTS Layer 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts commit df790e11d's two test-only changes now that the end-to-end CBTS Layer 3 work has been validated: - waives.txt: drop the 3 SKIP entries that were added to exercise the param / function-only / -k keyword variants of Layer 3 narrowing. - waives_rule.py: WaivesRule.apply restores handled_files={WAIVES_FILE} in all three RuleResult sites (instead of claiming every changed file). With this restored, CBTS only fires on PRs whose ONLY change is waives.txt — the original v0 contract — instead of the testing hack that made it fire on this PR's CBTS-infra commits too. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/rules/waives_rule.py | 15 +++------------ tests/integration/test_lists/waives.txt | 3 --- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 45e0731b8a31..3d4fa4e287ad 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -81,10 +81,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: changed_test_ids = added | removed if not changed_test_ids: return RuleResult( - # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every - # changed file so CBTS fires on this PR even though it edits CBTS - # infra files alongside waives.txt. - handled_files=set(pr.changed_files), + handled_files={WAIVES_FILE}, tests=set(), affected_stages=set(), scope="waiveonly", @@ -120,10 +117,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: preview = ", ".join(sorted(misses)[:3]) more = f" (+{len(misses) - 3} more)" if len(misses) > 3 else "" return RuleResult( - # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every - # changed file so CBTS fires on this PR even though it edits CBTS - # infra files alongside waives.txt. - handled_files=set(pr.changed_files), + handled_files={WAIVES_FILE}, tests=changed_test_ids, affected_stages=set(), scope=None, # Selector treats this as "no decision" → fallback @@ -137,10 +131,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: affected_stage_names.add(stage_name) return RuleResult( - # TESTING ONLY (revert to {WAIVES_FILE} before merge): claim every - # changed file so CBTS fires on this PR even though it edits CBTS - # infra files alongside waives.txt. - handled_files=set(pr.changed_files), + handled_files={WAIVES_FILE}, tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index fda46013038d..9854e59935d7 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -626,6 +626,3 @@ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] SKIP (https://nvbugs/6128419) unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py::test_cp_tp_broadcast_object[tp_cp_broadcast-list] SKIP (https://nvbugs/6132301) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k] SKIP (https://nvbugs/6133067) -disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0] SKIP (CBTS Layer 3 test - case A param, revert before merge) -test_e2e.py::test_get_ci_container_port SKIP (CBTS Layer 3 test - case B function-only, revert before merge) -unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[CUTLASS-fp8-tp4] SKIP (CBTS Layer 3 test - case E -k keyword, revert before merge) From 23706272be35c02b2fee3987015a9f216db8706b Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Sat, 2 May 2026 19:52:55 +0800 Subject: [PATCH 54/65] [None][feat] CBTS: add /bot run --disable-cbts flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default off (CBTS stays on). When set, getCbtsResult returns null immediately so the existing filter chain runs the full baseline — useful when CBTS mis-selects or for sanity full runs. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 8 ++++++++ jenkins/scripts/cbts/README.md | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 6dbeb4912922..c193dd372ebd 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -122,6 +122,8 @@ def DEBUG_MODE = "debug" def DETAILED_LOG = "detailed_log" @Field def CBTS_RESULT = "cbts_result" +@Field +def DISABLE_CBTS = "disable_cbts" // /bot run --disable-cbts → skip CBTS entirely; default off (CBTS on). def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), @@ -141,6 +143,7 @@ def testFilter = [ (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): gitlabParamsFromBot.get(DETAILED_LOG, false), (CBTS_RESULT): null, + (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -714,6 +717,11 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { def getCbtsResult(pipeline, testFilter, globalVars) { + if (testFilter[(DISABLE_CBTS)]) { + pipeline.echo("CBTS: deferring — user passed --disable-cbts") + return null + } + def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { pipeline.echo("CBTS: deferring — post-merge job or alternativeTRT set") diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 002ae247b2ce..59e956decd29 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -77,13 +77,17 @@ The `-k` keyword guard is applied **twice** by design: ## When CBTS activates -CBTS narrows test selection in **two usages only**: +CBTS is **on by default** and narrows test selection in **two usages only**: - `/bot run` — full pre-merge with CBTS narrowing. - `/bot run --post-merge` — post-merge with CBTS narrowing. Layer 2 keeps only post-merge hits; no post-merge hit → no-op (no fallback to full post-merge baseline). +Pass `/bot run --disable-cbts` (or combine with `--post-merge`) to opt out +and run the full baseline filter chain — useful when you suspect CBTS +mis-selection or want a sanity full run. + Any other **stage-selection** flag makes `getCbtsResult` return `null` and the existing filter chain takes over: `--stage-list`, `--extra-stage`, `--gpu-type`, `--test-backend`, `--skip-test`, `--add-multi-gpu-test`, From 52174492595e919dcb1919b41ec7b477f40fe946 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Sat, 2 May 2026 20:02:25 +0800 Subject: [PATCH 55/65] [None][chore] revert /bot run --disable-cbts flag Bot service argparser is maintained outside this repo, so an unregistered flag would be rejected by /bot. Drop the half-wired Jenkins side until the bot can recognize it. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 8 -------- jenkins/scripts/cbts/README.md | 6 +----- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c193dd372ebd..6dbeb4912922 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -122,8 +122,6 @@ def DEBUG_MODE = "debug" def DETAILED_LOG = "detailed_log" @Field def CBTS_RESULT = "cbts_result" -@Field -def DISABLE_CBTS = "disable_cbts" // /bot run --disable-cbts → skip CBTS entirely; default off (CBTS on). def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), @@ -143,7 +141,6 @@ def testFilter = [ (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): gitlabParamsFromBot.get(DETAILED_LOG, false), (CBTS_RESULT): null, - (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -717,11 +714,6 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { def getCbtsResult(pipeline, testFilter, globalVars) { - if (testFilter[(DISABLE_CBTS)]) { - pipeline.echo("CBTS: deferring — user passed --disable-cbts") - return null - } - def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { pipeline.echo("CBTS: deferring — post-merge job or alternativeTRT set") diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 59e956decd29..002ae247b2ce 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -77,17 +77,13 @@ The `-k` keyword guard is applied **twice** by design: ## When CBTS activates -CBTS is **on by default** and narrows test selection in **two usages only**: +CBTS narrows test selection in **two usages only**: - `/bot run` — full pre-merge with CBTS narrowing. - `/bot run --post-merge` — post-merge with CBTS narrowing. Layer 2 keeps only post-merge hits; no post-merge hit → no-op (no fallback to full post-merge baseline). -Pass `/bot run --disable-cbts` (or combine with `--post-merge`) to opt out -and run the full baseline filter chain — useful when you suspect CBTS -mis-selection or want a sanity full run. - Any other **stage-selection** flag makes `getCbtsResult` return `null` and the existing filter chain takes over: `--stage-list`, `--extra-stage`, `--gpu-type`, `--test-backend`, `--skip-test`, `--add-multi-gpu-test`, From f476716d979b0fd1602868917b4b942bbcf986de Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 10:49:06 +0800 Subject: [PATCH 56/65] fix conflicts Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 176 ------------------------ 1 file changed, 176 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 9854e59935d7..2772c080b8f0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -264,9 +264,6 @@ full:H20/accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp8 full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,2000-reqs:10-ep:4-tp:8-gpus:8] SKIP (https://nvbugs/5150255) full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) full:NVIDIA_B200/perf/test_perf.py::test_perf[deepseek_r1_fp8-bench-pytorch-float8-maxbs:512-input_output_len:128,128-ep:8-tp:8-gpus:8] SKIP (https://nvbugs/5150255) -triton_server/test_triton.py::test_gpt_2b_ib_lora[gpt-2b-ib-lora] SKIP (https://nvbugs/5470830) -unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781) -triton_server/test_triton.py::test_llava[llava] SKIP (https://nvbugs/5547414) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5948435) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/5961814) @@ -453,176 +450,3 @@ verl/test_verl_cases.py::test_async_server SKIP (https://nvbugs/5981833) verl/test_verl_cases.py::test_rollout_utils SKIP (https://nvbugs/5981833) visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark SKIP (https://nvbugs/6050483) visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos] SKIP (https://nvbugs/6050483) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[4gpus] SKIP (https://nvbugs/6069790) -accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5981293) -accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/5981293) -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6074784) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6071081) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6071081) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6071081) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6071081) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6071081) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6050489) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6050489) -unittest/llmapi/test_llm_pytorch.py::test_llm_disagg_streaming_gen_cancelled SKIP (https://nvbugs/6078431) -unittest/auto_deploy/singlegpu/transformations/library/test_mrope_delta_cache.py::test_qwen_registry_configs_explicitly_enable_mrope_delta_cache SKIP (https://nvbugs/6078421) -accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6080024) -llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_eagle3 SKIP (https://nvbugs/6075431) -llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp SKIP (https://nvbugs/6079440) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6079919) -disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6087632) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput] SKIP (https://nvbugs/6084764) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6084824) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=0] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp] SKIP (https://nvbugs/6084568) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6088149) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6088149) -accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 SKIP (https://nvbugs/6070857) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6094071) -accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6094070) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dummy_load_format SKIP (https://nvbugs/6094072) -cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] SKIP (https://nvbugs/6093820) -cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] SKIP (https://nvbugs/6093820) -accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm] SKIP (https://nvbugs/6094208) -accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-1-trtllm] SKIP (https://nvbugs/6094208) -accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] SKIP (https://nvbugs/6093713) -accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6093713) -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_on] SKIP (https://nvbugs/6094068) -accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] SKIP (https://nvbugs/6093715) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM] SKIP (https://nvbugs/6095421) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6095421) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3] SKIP (https://nvbugs/6095700) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6095851) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6098790) -disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True] SKIP (https://nvbugs/6094066) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler SKIP (https://nvbugs/6105765) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse SKIP (https://nvbugs/6105765) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse SKIP (https://nvbugs/6105765) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler SKIP (https://nvbugs/6105765) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse SKIP (https://nvbugs/6105765) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available SKIP (https://nvbugs/6106174) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse SKIP (https://nvbugs/6106174) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse SKIP (https://nvbugs/6106174) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse SKIP (https://nvbugs/6106174) -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse SKIP (https://nvbugs/6106174) -accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] SKIP (https://nvbugs/5921674) -accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/5955803) -full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16 SKIP (https://nvbugs/6069543) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_8k1k] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_1k1k] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6110326) -perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6110326) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] SKIP (https://nvbugs/6113016) -disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6011317) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto] SKIP (https://nvbugs/5596343) -unittest/_torch/speculative/test_eagle3.py::test_llama_eagle3_dynamic_tree[True-False] SKIP (https://nvbugs/6113021) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-no_overlap_scheduler] SKIP (https://nvbugs/6114821) -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4] SKIP (https://nvbugs/6110074) -test_doc.py::test_url_validity SKIP (https://nvbugs/6109719) -perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6115832) -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] SKIP (https://nvbugs/6112508) -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] SKIP (https://nvbugs/6112508) -accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-trtllm] SKIP (https://nvbugs/6112500) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6112497) -disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114140) -accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6094102) -disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114142) -disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114139) -accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype SKIP (https://nvbugs/6114464) -disaggregated/test_disaggregated.py::test_disaggregated_trtllm_sampler[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114141) -disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_gentp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114612) -test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6114608) -test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6114608) -test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] SKIP (https://nvbugs/6114608) -test_e2e.py::test_multi_nodes_eval[nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-tp16-mmlu] SKIP (https://nvbugs/6114608) -test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https://nvbugs/6115560) -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] SKIP (https://nvbugs/6116088) -test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] SKIP (https://nvbugs/6115562) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False] SKIP (https://nvbugs/5981122) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6117811) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] SKIP (https://nvbugs/6117811) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] SKIP (https://nvbugs/6117811) -accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True] SKIP (https://nvbugs/6117816) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler] SKIP (https://nvbugs/6120553) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] SKIP (https://nvbugs/6120553) -accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer] SKIP (https://nvbugs/6117814) -accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm] SKIP (https://nvbugs/6117814) -accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True] SKIP (https://nvbugs/6117816) -accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True] SKIP (https://nvbugs/6117816) -unittest/tools/test_layer_wise_benchmarks.py::test_performance_alignment[1] SKIP (https://nvbugs/6127669) -full:H100_PCIe/unittest/auto_deploy/standalone SKIP (https://nvbugs/6129630) -full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6128420) -full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] SKIP (https://nvbugs/6128419) -unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py::test_cp_tp_broadcast_object[tp_cp_broadcast-list] SKIP (https://nvbugs/6132301) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k] SKIP (https://nvbugs/6133067) From c243c0fd3fc88d1467cdd3f5ef21d61af532fccd Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 16:32:25 +0800 Subject: [PATCH 57/65] [None][fix] CBTS: filter affected_stages by /bot run trigger mode Layer 2 in L0_Test.groovy was asymmetric: it filtered to Post-Merge stages when --post-merge was set, but the default /bot run case had no symmetric filter. A CBTS narrow that included Post-Merge stages would therefore run them on /bot run too -- running CI the user did not ask for, and on stages whose mako conditions don't hold pre-merge. Push the trigger-mode filter from Groovy into Python: - rules/base.py: PRInputs gains an optional post_merge bool. - L0_MergeRequest.groovy: getCbtsResult passes IS_POST_MERGE through cbts_input.json so Python knows the trigger mode. - main.py: after Selector.run + Layer 3 setup, filter affected_stages by Post-Merge stage-name presence vs pr.post_merge. Recompute affected_cpu_arch and affected_stage_test_counts against the filtered set. Stderr diagnostic gains a [trigger mode: X] header and lists stages dropped by the filter. - L0_Test.groovy Layer 2: simplified to a single mode-agnostic block. Always assigns parallelJobsFiltered from the CBTS subset (already pre-filtered by Python). Empty subset -> empty parallelJobsFiltered = no-op (does NOT fall through to baseline), symmetrically for both /bot run and /bot run --post-merge. Behavior matrix (CBTS narrow contents shown; result is what runs): Trigger Narrow Old New ---------- ---------------- -------------- -------------- /bot run all pre-merge pre-merge same /bot run mixed pre + post runs both (bug) runs only pre /bot run all post-merge runs post (bug) no-op --post-merge all post-merge post-merge same --post-merge mixed pre + post runs only post same --post-merge all pre-merge no-op same scope=null n/a baseline baseline Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 +++ jenkins/L0_Test.groovy | 27 ++++++++++++------ jenkins/scripts/cbts/main.py | 45 ++++++++++++++++++++++++++++-- jenkins/scripts/cbts/rules/base.py | 11 +++++++- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 6dbeb4912922..46cf2f306927 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -756,9 +756,13 @@ def getCbtsResult(pipeline, testFilter, globalVars) } // 3. Write INPUT_JSON (PR data only; Python reads stages/yaml itself). + // `post_merge` lets Python apply the trigger-mode filter on + // affected_stages before returning the JSON, so Layer 2 in Groovy + // (L0_Test.groovy::launchTestJobs) stays scope-/mode-agnostic. def inputJson = groovy.json.JsonOutput.toJson([ changed_files: changedFiles, diffs: diffs, + post_merge: testFilter[(IS_POST_MERGE)] ?: false, ]) def inputPath = "${LLM_ROOT}/cbts_input.json" writeFile file: inputPath, text: inputJson diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 244f69383d10..5d1c31b9f232 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4495,19 +4495,30 @@ def launchTestJobs(pipeline, testFilter) // existing filter rules so unknown / no-decision paths fall through // naturally. Perf stages are excluded — they have their own trigger // model and need full test lists. See jenkins/scripts/cbts/README.md. + // + // Pre-merge vs Post-Merge filtering already happened in Python + // (main.py applies it based on the post_merge flag plumbed through + // cbts_input.json), so `cbts.affected_stages` here is already + // restricted to the user's trigger mode. No-op if empty: matches + // README's "/bot run [--post-merge] with no relevant hit -> no-op" + // semantic, symmetrically for both modes. def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null && cbts.affected_stages) { - def affectedSet = cbts.affected_stages as Set + if (cbts != null) { + // Always assign parallelJobsFiltered from CBTS result (rather than + // falling through to the prior filter chain) — even when the CBTS- + // narrowed set is empty after Python's trigger-mode filter. Empty + // = "CBTS narrowed something, but nothing matches the user's trigger + // mode" → no-op (don't run anything). Falling through to baseline + // here would defeat CBTS's whole point. scope=null cases never + // reach this block: getCbtsResult returns null and cbts is null. + def affectedSet = (cbts.affected_stages ?: []) as Set parallelJobsFiltered = parallelJobs.findAll { key, _ -> affectedSet.contains(key) && !(key =~ /Perf/) } - // Under `/bot run --post-merge`, keep only post-merge hits; if none, - // no-op (no fallback to full post-merge). - if (testFilter[(IS_POST_MERGE)]) { - parallelJobsFiltered = parallelJobsFiltered.findAll { it.key.contains("Post-Merge") } - echo "CBTS [${cbts.scope}] (--post-merge): keeping ${parallelJobsFiltered.size()} affected post-merge stages" - } else { + if (parallelJobsFiltered) { echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} affected stages" + } else { + echo "CBTS [${cbts.scope}]: no stages match trigger mode → no-op (no fallback to baseline)" } } diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index ccd324851e99..72d41ad4df89 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -200,6 +200,7 @@ def _load_pr_inputs(input_json_path: Path) -> PRInputs: return PRInputs( changed_files=list(data.get("changed_files", [])), diffs=dict(data.get("diffs", {})), + post_merge=bool(data.get("post_merge", False)), ) @@ -304,25 +305,63 @@ def main(argv: Optional[list[str]] = None) -> int: block_filters=result.block_filters, ) - _log_decision_to_stderr(stages, result) + # Trigger-mode filter: drop affected_stages incompatible with the user's + # /bot run [--post-merge] flag. Done in Python (not Groovy) so the JSON + # contract Groovy reads already reflects "stages that should run NOW", + # and Layer 2 in Groovy stays scope/mode-agnostic. Narrowed YAML output + # (cbts_test_db) is unaffected — runtime mako matching there picks the + # right block per stage. + pre_filter_stages = set(result.affected_stages) + if pr.post_merge: + result.affected_stages = {s for s in pre_filter_stages if "Post-Merge" in s} + else: + result.affected_stages = {s for s in pre_filter_stages if "Post-Merge" not in s} + # Recompute derived fields against the filtered set. + result.affected_cpu_arch = { + stages[name].cpu_arch for name in result.affected_stages if name in stages + } + result.affected_stage_test_counts = { + k: v for k, v in result.affected_stage_test_counts.items() if k in result.affected_stages + } + + _log_decision_to_stderr(stages, result, pr, pre_filter_stages) sys.stdout.write(result.to_json()) return 0 -def _log_decision_to_stderr(stages: dict[str, Stage], result: SelectionResult) -> None: +def _log_decision_to_stderr( + stages: dict[str, Stage], + result: SelectionResult, + pr: PRInputs, + pre_filter_stages: set[str], +) -> None: """Dump the full CBTS decision to stderr for Jenkins console diagnostics. stdout carries the JSON consumed by Groovy `getCbtsResult`, so all human-readable detail goes to stderr to avoid corrupting that contract. Each affected stage is annotated with its yaml_stem so blocks-vs-stages mismatches (e.g. a stage matched by an unexpected YAML) are obvious. + + `pre_filter_stages` is the affected_stages set BEFORE the trigger-mode + filter (pre-merge vs post-merge). Stages dropped by the filter are + listed separately so reviewers can see why a stage CBTS narrowed to + isn't actually running. """ out = sys.stderr + mode = "post_merge" if pr.post_merge else "pre_merge" + dropped = sorted(pre_filter_stages - set(result.affected_stages)) print("=" * 64, file=out) - print("CBTS decision (diagnostic; stderr only):", file=out) + print(f"CBTS decision (diagnostic; stderr only) [trigger mode: {mode}]:", file=out) print(f" scope: {result.scope}", file=out) print(f" test_db_dir_override: {result.test_db_dir_override}", file=out) print(f" affected_cpu_arch: {sorted(result.affected_cpu_arch)}", file=out) + if dropped: + print( + f" stages dropped by trigger-mode filter ({len(dropped)}, would run if mode flipped):", + file=out, + ) + for s in dropped: + print(f" - {s}", file=out) if result.reasons: print(" reasons:", file=out) for r in result.reasons: diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 47221a64cd7b..b9f53e553b5e 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -22,10 +22,19 @@ @dataclass class PRInputs: - """Inputs about the PR that rules can query.""" + """Inputs about the PR that rules can query. + + `post_merge` reflects the user's `/bot run [--post-merge]` flag. Rules + themselves don't consult it (their narrowing is mode-agnostic); main.py + uses it after `Selector.run` to drop `affected_stages` entries that + don't match the trigger mode (pre-merge vs Post-Merge by stage-name + convention). See `main.py::main` for the filter; default False keeps + backward compat with older Groovy that didn't pass the field. + """ changed_files: list[str] diffs: dict[str, str] + post_merge: bool = False @dataclass From d9edb8d5eee2859539e0b65794041c35b46901a2 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 17:14:50 +0800 Subject: [PATCH 58/65] [None][refactor] CBTS: scope to test cases only; preserve Build and PackageSanityCheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Earlier CBTS L1 (arch-track skip in L0_MergeRequest.groovy) collapsed the entire x86 / SBSA track — including Build — when no stage on that arch survived Python's trigger-mode filter. That overreached: a post-merge-only waive on /bot run produced a wheel-less no-op, and the runtime-named *-PackageSanityCheck-* stages (which the CBTS Python parser cannot see) were silently dropped any time CBTS narrowed. Tighten the scope: CBTS now narrows test cases only. - L0_MergeRequest.groovy: drop the CBTS arch-track short-circuit on both x86_64-Linux and SBSA-Linux. Build always runs so a wheel exists for sanity checks and post-merge consumers. - L0_Test.groovy Layer 2: keep the existing affected-stage filter, but force-include `*-PackageSanityCheck-*` (Perf still excluded). Sanity is a wheel/image gate that should always run after Build. - main.py: SelectionResult gains `trigger_mode_mismatch: bool`, set true when rules resolved stages but the trigger-mode filter dropped them all. Surfaced through cbts JSON and parsed in _cbtsParseSelectionResult so Layer 2 can log the "build + sanity only, no test cases" path explicitly instead of inferring it from `affected_stages.empty`. - README + comments: drop the Layer 1 row, document the Build-always invariant and the trigger-mode-mismatch fallback (analogous to `/bot run --stage-list ""`). New behavior matrix: Trigger Narrow Result ---------- ---------------- ------------------------------------ /bot run all pre-merge Build + affected pre + Sanity /bot run mixed pre + post Build + affected pre + Sanity /bot run all post-merge Build + Sanity (trigger_mode_mismatch) --post-merge all post-merge Build + affected post + Sanity --post-merge mixed pre + post Build + affected post + Sanity --post-merge all pre-merge Build + Sanity (trigger_mode_mismatch) scope=null n/a baseline (CBTS not consulted) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 34 ++++++++++++++--------------- jenkins/L0_Test.groovy | 39 +++++++++++++++++++--------------- jenkins/scripts/cbts/README.md | 19 +++++++++++++++-- jenkins/scripts/cbts/main.py | 19 ++++++++++++++--- 4 files changed, 71 insertions(+), 40 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 46cf2f306927..4b1f3c8781b3 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -709,7 +709,9 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { // affected_stages, affected_tests, reasons} or null (= no decision / fall // back to the existing filter chain). // -// See jenkins/scripts/cbts/README.md for the two-layer consumption model. +// See jenkins/scripts/cbts/README.md for the consumption model. CBTS only +// narrows test cases (Layer 2 stage filter + Layer 3 within-stage filter); +// it never affects Build, so the wheel always exists. // ============================================================================ def getCbtsResult(pipeline, testFilter, globalVars) @@ -773,7 +775,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) returnStdout: true, ) - // 5. Parse stdout into the map shape consumed by Layer 1/2/3. + // 5. Parse stdout into the map shape consumed by Layer 2/2.5/3. def result = _cbtsParseSelectionResult(output) if (result.scope == null) { pipeline.echo("CBTS: deferring — Python returned scope=null. " + @@ -865,7 +867,7 @@ def _cbtsTriggeredUserFlags(testFilter) .collect { "${it}=${testFilter[it]}" } } -// Parse CBTS JSON stdout into the shape consumed by Layer 1/2/3. Always +// Parse CBTS JSON stdout into the shape consumed by Layer 2/2.5/3. Always // returns a map; `scope == null` means "no decision" (caller should log the // reasons and treat as defer). def _cbtsParseSelectionResult(String text) @@ -882,6 +884,10 @@ def _cbtsParseSelectionResult(String text) // launchTestJobs reads this to drop excess pytest-split groups when // the affected stage's narrowed count falls below the 20-test threshold. affected_stage_test_counts: data.affected_stage_test_counts ?: [:], + // True when Python's trigger-mode filter dropped every resolved stage. + // Layer 2 reads this to log "build only, no test cases" explicitly, + // instead of inferring it from `affected_stages.empty`. + trigger_mode_mismatch: data.trigger_mode_mismatch ?: false, ] } @@ -1272,14 +1278,11 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) }, "x86_64-Linux": { script { - // CBTS Layer 1: skip entire x86 track when no x86 stages are affected. - // Scope-agnostic: any non-null cbts result means CBTS produced a decision; - // we trust its affected_cpu_arch regardless of which rule fired. - def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null && !("x86" in cbts.affected_cpu_arch)) { - echo "CBTS [${cbts.scope}]: no x86 stages affected, skipping x86_64-Linux track" - return - } + // CBTS deliberately does NOT short-circuit at the arch / Build + // layer. Build always runs so a wheel exists for sanity checks + // and post-merge consumers; case-level narrowing happens later + // in L0_Test.groovy::launchTestJobs (Layer 2) and renderTestDB + // (Layer 3). def testStageName = "[Build-x86_64] Remote Run" stage(testStageName) { def additionalParameters = [ @@ -1391,13 +1394,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "SBSA build job is skipped due to Jenkins configuration or conditional pipeline run" return } - // CBTS Layer 1: skip entire SBSA track when no sbsa stages are affected. - // Scope-agnostic — see x86 track above for the rationale. - def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null && !("sbsa" in cbts.affected_cpu_arch)) { - echo "CBTS [${cbts.scope}]: no sbsa stages affected, skipping SBSA-Linux track" - return - } + // CBTS deliberately does NOT short-circuit the SBSA Build — + // see x86 track above for the rationale. def testStageName = "[Build-SBSA] Remote Run" stage(testStageName) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5d1c31b9f232..9bc65e76eb4c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4491,34 +4491,39 @@ def launchTestJobs(pipeline, testFilter) checkStageNameSet(testFilter[(EXTRA_STAGE_LIST)], fullSet, EXTRA_STAGE_LIST) } - // CBTS Layer 2: stage-level short-circuit override. Runs AFTER all - // existing filter rules so unknown / no-decision paths fall through - // naturally. Perf stages are excluded — they have their own trigger - // model and need full test lists. See jenkins/scripts/cbts/README.md. + // CBTS Layer 2: stage-level filter. Runs AFTER all existing filter rules + // so unknown / no-decision paths fall through naturally. CBTS narrows + // *test stages* only — Build always runs (the L0_MergeRequest arch-track + // skip was removed deliberately so the wheel and PackageSanityCheck path + // are never disturbed by CBTS). See jenkins/scripts/cbts/README.md. + // + // - Perf stages: excluded. They have their own trigger model and need + // full test lists. + // - PackageSanityCheck stages: force-kept. Their stage names are built + // at runtime so the CBTS Python parser cannot see them, and they are + // wheel/image gates that should always run after Build. + // - Trigger-mode mismatch (cbts.trigger_mode_mismatch): rules resolved + // stages but none match the user's /bot run [--post-merge] mode. The + // filter naturally collapses to PackageSanityCheck only — equivalent + // to "build + sanity, no test cases", similar to /bot run --stage-list "". // // Pre-merge vs Post-Merge filtering already happened in Python // (main.py applies it based on the post_merge flag plumbed through // cbts_input.json), so `cbts.affected_stages` here is already - // restricted to the user's trigger mode. No-op if empty: matches - // README's "/bot run [--post-merge] with no relevant hit -> no-op" - // semantic, symmetrically for both modes. + // restricted to the user's trigger mode. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { - // Always assign parallelJobsFiltered from CBTS result (rather than - // falling through to the prior filter chain) — even when the CBTS- - // narrowed set is empty after Python's trigger-mode filter. Empty - // = "CBTS narrowed something, but nothing matches the user's trigger - // mode" → no-op (don't run anything). Falling through to baseline - // here would defeat CBTS's whole point. scope=null cases never - // reach this block: getCbtsResult returns null and cbts is null. def affectedSet = (cbts.affected_stages ?: []) as Set parallelJobsFiltered = parallelJobs.findAll { key, _ -> - affectedSet.contains(key) && !(key =~ /Perf/) + (affectedSet.contains(key) || key =~ /PackageSanityCheck/) && !(key =~ /Perf/) } - if (parallelJobsFiltered) { + if (cbts.trigger_mode_mismatch) { + echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running only " + + "${parallelJobsFiltered.size()} PackageSanityCheck stage(s); no test cases" + } else if (parallelJobsFiltered) { echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} affected stages" } else { - echo "CBTS [${cbts.scope}]: no stages match trigger mode → no-op (no fallback to baseline)" + echo "CBTS [${cbts.scope}]: empty stage set after filtering" } } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 002ae247b2ce..eb7e76089807 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -9,13 +9,28 @@ consumes the data directly.** ## Consumption layers +CBTS narrows **test cases only**. Build always runs (`L0_MergeRequest.groovy` +arch-track skipping was removed deliberately) so the wheel exists for sanity +checks and post-merge consumers. + | Layer | Where | Action | |---|---|---| -| **1. Arch track** | `L0_MergeRequest.groovy::launchStages` | Skip x86 / SBSA track when no stage on that arch is affected | -| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset (Perf stages excluded — they have their own trigger model and need full lists) | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset. Perf stages are excluded (they have their own trigger model and need full lists); `*-PackageSanityCheck-*` stages are force-kept (their names are runtime-built and invisible to the CBTS Python parser, and they are wheel/image gates that should always run after Build). | | **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOnSlurm` and `runLLMTestlistOnPlatform` entries | When the affected stage's narrowed test count is < 20, collapse pytest-split's splits to 1 — only group 1 runs everything; groups 2..N skip without allocating a machine. At/above 20 the stage's default splits stand and pytest-split parallelizes normally. | | **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the CBTS-narrowed tmp test-db. Each affected block's `tests:` array is filtered to entries in the per-block filter prefix subtree, **and unaffected blocks are dropped entirely** so a `/bot run --post-merge` can't accidentally activate post-merge blocks the PR never touched. | +(Layer 1 in earlier revisions skipped the entire arch track / Build when no +stage on that arch was affected. Removed: see +`L0_MergeRequest.groovy::launchStages`. CBTS no longer touches Build.) + +### Trigger-mode mismatch + +If `/bot run` is issued but every CBTS-resolved stage is post-merge (or the +symmetric case with `--post-merge`), `main.py` records this as +`trigger_mode_mismatch: true` in its JSON output. Layer 2 then narrows to +the PackageSanityCheck stages only — equivalent to a build-and-sanity-only +run, in spirit similar to `/bot run --stage-list ""`. + CBTS only **subtracts** stages and tests, never adds. Anything it can't narrow → full fallback to the existing filter chain. diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 72d41ad4df89..71f7b29eaa9d 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -92,6 +92,12 @@ class SelectionResult: # matches, post-keep-filter). Groovy launchTestJobs uses this to # collapse splits to 1 when the count is below the 20-test threshold. affected_stage_test_counts: dict[str, int] = field(default_factory=dict) + # True iff CBTS resolved at least one stage but the trigger-mode filter + # (pre-merge vs post-merge) dropped them all. Distinguishes "this PR's + # narrow has nothing in the user's trigger mode" from "no narrow at all". + # Layer 2 in Groovy uses this to gate the diagnostic message and to + # express the "build only, no cases" no-op explicitly. + trigger_mode_mismatch: bool = False def to_json(self) -> str: data = { @@ -102,6 +108,7 @@ def to_json(self) -> str: "reasons": list(self.reasons), "test_db_dir_override": self.test_db_dir_override, "affected_stage_test_counts": dict(self.affected_stage_test_counts), + "trigger_mode_mismatch": self.trigger_mode_mismatch, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -156,9 +163,10 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: # Safety net: if rules fired but no Jenkins stages resolved (waive ids # missed both exact and parent-chain lookups in the YAML index), fall - # back to baseline. Returning a non-None scope with empty stages would - # let Layer 1 in Groovy mistake `affected_cpu_arch=∅` for "no arch - # needed" and silently skip both x86 and SBSA tracks. + # back to baseline. Returning a non-None scope with empty stages here + # is distinct from a trigger-mode mismatch: it means CBTS has nothing + # actionable, so we want the existing filter chain to run normally + # rather than CBTS-narrowing to PackageSanityCheck only. if not affected_stages: return SelectionResult( scope=None, @@ -323,6 +331,11 @@ def main(argv: Optional[list[str]] = None) -> int: result.affected_stage_test_counts = { k: v for k, v in result.affected_stage_test_counts.items() if k in result.affected_stages } + # Flag the "build only, no test cases" no-op: rules resolved stages, but + # none survive the user's trigger-mode filter. Surfaced to Groovy so the + # Layer 2 log can say why — and so future consumers can branch on it + # instead of inferring from `not affected_stages`. + result.trigger_mode_mismatch = bool(pre_filter_stages and not result.affected_stages) _log_decision_to_stderr(stages, result, pr, pre_filter_stages) sys.stdout.write(result.to_json()) From aa659df67796372391bd891932ebc531c7449e4c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 17:32:20 +0800 Subject: [PATCH 59/65] [None][refactor] CBTS: trim redundant JSON fields (affected_cpu_arch, tests, trigger_mode_mismatch) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of the cbts/main.py JSON contract found three fields with no load-bearing consumer after the previous "scope to test cases only" refactor: - `affected_cpu_arch`: was load-bearing only for the L1 arch-track skip in L0_MergeRequest.groovy, which got removed when CBTS stopped affecting Build. The remaining single echo "archs=..." was diagnostic fluff. - `tests`: only ever consumed as `result.affected_tests.size()` in one echo line — never used for any decision. RuleResult.tests was symmetric dead weight. - `trigger_mode_mismatch`: fully derivable from `affected_stages.empty` on the Groovy side (the Selector safety net guarantees pre-filter affected_stages is non-empty whenever scope!=null, so post-filter emptiness unambiguously means trigger-mode mismatch). Drop all three. JSON keys go from 8 to 5; SelectionResult dataclass sheds two fields; RuleResult sheds one; the WaivesRule constructors drop the now-unused `tests=` argument; Selector.run skips the aggregation/derivation; main() skips the post-filter recompute; stderr diagnostic loses two now-trivial blocks; the L0_MergeRequest summary echo collapses to `scope=..., stages=N`; Layer 2 in L0_Test reads `affectedSet.isEmpty()` directly. README example JSON and rule-author guide updated accordingly. scope and reasons stay — they live at different abstraction levels (coarse machine label + None-as-baseline gate vs free-form why-string, especially for scope=null defer messages) and are not redundant. Final JSON contract (5 keys): { "scope": "waiveonly" | null, "affected_stages": [...], "reasons": [...], "test_db_dir_override": "cbts_test_db" | null, "affected_stage_test_counts": {...} } Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 16 ++----- jenkins/L0_Test.groovy | 13 +++--- jenkins/scripts/cbts/README.md | 19 +++++---- jenkins/scripts/cbts/main.py | 51 +++++++---------------- jenkins/scripts/cbts/rules/base.py | 1 - jenkins/scripts/cbts/rules/waives_rule.py | 3 -- 6 files changed, 36 insertions(+), 67 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 4b1f3c8781b3..a34e5beaa858 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -705,9 +705,9 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { // // Upstream decision point. Calls jenkins/scripts/cbts/main.py with the PR's // changed_files + diffs; Python self-sources stage configs from L0_Test.groovy -// and YAML blocks from test-db. Returns a dict {scope, affected_cpu_arch, -// affected_stages, affected_tests, reasons} or null (= no decision / fall -// back to the existing filter chain). +// and YAML blocks from test-db. Returns a dict {scope, affected_stages, +// reasons, test_db_dir_override, affected_stage_test_counts} or null +// (= no decision / fall back to the existing filter chain). // // See jenkins/scripts/cbts/README.md for the consumption model. CBTS only // narrows test cases (Layer 2 stage filter + Layer 3 within-stage filter); @@ -805,9 +805,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) "(Layer 2 stage filtering still applies)") } pipeline.echo("CBTS: scope=${result.scope}, " + - "archs=${result.affected_cpu_arch}, " + - "stages=${result.affected_stages.size()}, " + - "tests=${result.affected_tests.size()}") + "stages=${result.affected_stages.size()}") return result } catch (Exception e) { pipeline.echo("CBTS failed, falling back to full run: ${e}") @@ -875,19 +873,13 @@ def _cbtsParseSelectionResult(String text) def data = new groovy.json.JsonSlurper().parseText(text) return [ scope: data.scope, - affected_cpu_arch: data.affected_cpu_arch ?: [], affected_stages: data.affected_stages ?: [], - affected_tests: data.tests ?: [], reasons: data.reasons ?: [], test_db_dir_override: data.test_db_dir_override, // Layer 3: tmp test-db path // Layer 3 split-collapse heuristic: per-stage narrowed test count. // launchTestJobs reads this to drop excess pytest-split groups when // the affected stage's narrowed count falls below the 20-test threshold. affected_stage_test_counts: data.affected_stage_test_counts ?: [:], - // True when Python's trigger-mode filter dropped every resolved stage. - // Layer 2 reads this to log "build only, no test cases" explicitly, - // instead of inferring it from `affected_stages.empty`. - trigger_mode_mismatch: data.trigger_mode_mismatch ?: false, ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9bc65e76eb4c..0eb5ac03c70f 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4502,10 +4502,13 @@ def launchTestJobs(pipeline, testFilter) // - PackageSanityCheck stages: force-kept. Their stage names are built // at runtime so the CBTS Python parser cannot see them, and they are // wheel/image gates that should always run after Build. - // - Trigger-mode mismatch (cbts.trigger_mode_mismatch): rules resolved - // stages but none match the user's /bot run [--post-merge] mode. The - // filter naturally collapses to PackageSanityCheck only — equivalent - // to "build + sanity, no test cases", similar to /bot run --stage-list "". + // - Trigger-mode mismatch (affectedSet empty after Python's pre/post + // filter): rules resolved stages but none match the user's + // /bot run [--post-merge] mode. The filter naturally collapses to + // PackageSanityCheck only — equivalent to "build + sanity, no test + // cases", in spirit similar to /bot run --stage-list "". The Python + // safety net guarantees pre-filter `affected_stages` is non-empty + // whenever `cbts != null`, so empty here unambiguously means mismatch. // // Pre-merge vs Post-Merge filtering already happened in Python // (main.py applies it based on the post_merge flag plumbed through @@ -4517,7 +4520,7 @@ def launchTestJobs(pipeline, testFilter) parallelJobsFiltered = parallelJobs.findAll { key, _ -> (affectedSet.contains(key) || key =~ /PackageSanityCheck/) && !(key =~ /Perf/) } - if (cbts.trigger_mode_mismatch) { + if (affectedSet.isEmpty()) { echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running only " + "${parallelJobsFiltered.size()} PackageSanityCheck stage(s); no test cases" } else if (parallelJobsFiltered) { diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index eb7e76089807..93705ccb298a 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -26,10 +26,13 @@ stage on that arch was affected. Removed: see ### Trigger-mode mismatch If `/bot run` is issued but every CBTS-resolved stage is post-merge (or the -symmetric case with `--post-merge`), `main.py` records this as -`trigger_mode_mismatch: true` in its JSON output. Layer 2 then narrows to -the PackageSanityCheck stages only — equivalent to a build-and-sanity-only -run, in spirit similar to `/bot run --stage-list ""`. +symmetric case with `--post-merge`), Python's trigger-mode filter empties +`affected_stages` while leaving `scope != null`. The Selector safety-net +guarantees `affected_stages` is never empty BEFORE the filter when +`scope != null`, so Layer 2 detects the mismatch unambiguously with +`affectedSet.isEmpty()` and narrows to the PackageSanityCheck stages only +— equivalent to a build-and-sanity-only run, in spirit similar to +`/bot run --stage-list ""`. CBTS only **subtracts** stages and tests, never adds. Anything it can't narrow → full fallback to the existing filter chain. @@ -127,9 +130,7 @@ Decision JSON: ```json { "scope": "waiveonly", - "affected_cpu_arch": ["x86"], "affected_stages": ["A10-PyTorch-1", "A10-PyTorch-2"], - "tests": ["unittest/utils/test_util.py"], "reasons": ["[waives] waives.txt: +1 / -0 → 1 blocks, 2 stages"], "test_db_dir_override": "cbts_test_db", "affected_stage_test_counts": {"A10-PyTorch-1": 5, "A10-PyTorch-2": 5} @@ -223,9 +224,9 @@ what trt-test-db will eventually render. 2. **Register in `main.py`**: add to `RULE_CLASSES` and `build_rules()`. -3. **No Groovy edits needed.** Layers 1 / 2 / 2.5 / 3 are scope-agnostic and - consume `affected_cpu_arch` / `affected_stages` / `block_filters` / - `affected_stage_test_counts` directly. +3. **No Groovy edits needed.** Layers 2 / 2.5 / 3 are scope-agnostic and + consume `affected_stages` / `block_filters` / `affected_stage_test_counts` + directly. Rule order is irrelevant. `Selector` unions `affected_stages` and `block_filters`; scopes are combined via `_combine_scopes` (all-agree → that diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 71f7b29eaa9d..ffd71eb8f32e 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -27,8 +27,9 @@ Python self-sources everything else from the repo: - stage configs: parsed from jenkins/L0_Test.groovy - test-db YAMLs: loaded from tests/integration/test_lists/test-db/ - Output is a JSON blob on stdout with fields `scope`, `affected_cpu_arch`, - `affected_stages`, `tests`, `reasons`. Consumed by `_cbtsParseSelectionResult` + Output is a JSON blob on stdout with fields `scope`, `affected_stages`, + `reasons`, `test_db_dir_override`, `affected_stage_test_counts`. Consumed + by `_cbtsParseSelectionResult` on the Groovy side. Invocation assumes the current working directory is the TRT-LLM repo root, @@ -83,8 +84,6 @@ class SelectionResult: scope: Optional[str] affected_stages: set[str] = field(default_factory=set) - affected_cpu_arch: set[str] = field(default_factory=set) - tests: set[str] = field(default_factory=set) reasons: list[str] = field(default_factory=list) block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) test_db_dir_override: Optional[str] = None @@ -92,23 +91,14 @@ class SelectionResult: # matches, post-keep-filter). Groovy launchTestJobs uses this to # collapse splits to 1 when the count is below the 20-test threshold. affected_stage_test_counts: dict[str, int] = field(default_factory=dict) - # True iff CBTS resolved at least one stage but the trigger-mode filter - # (pre-merge vs post-merge) dropped them all. Distinguishes "this PR's - # narrow has nothing in the user's trigger mode" from "no narrow at all". - # Layer 2 in Groovy uses this to gate the diagnostic message and to - # express the "build only, no cases" no-op explicitly. - trigger_mode_mismatch: bool = False def to_json(self) -> str: data = { "scope": self.scope, - "affected_cpu_arch": sorted(self.affected_cpu_arch), "affected_stages": sorted(self.affected_stages), - "tests": sorted(self.tests), "reasons": list(self.reasons), "test_db_dir_override": self.test_db_dir_override, "affected_stage_test_counts": dict(self.affected_stage_test_counts), - "trigger_mode_mismatch": self.trigger_mode_mismatch, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -156,17 +146,16 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: return SelectionResult(scope=None, reasons=reasons + ["Scopes cannot be combined"]) affected_stages: set[str] = set() - tests: set[str] = set() for _, r in pairs: affected_stages |= r.affected_stages - tests |= r.tests # Safety net: if rules fired but no Jenkins stages resolved (waive ids # missed both exact and parent-chain lookups in the YAML index), fall - # back to baseline. Returning a non-None scope with empty stages here - # is distinct from a trigger-mode mismatch: it means CBTS has nothing - # actionable, so we want the existing filter chain to run normally - # rather than CBTS-narrowing to PackageSanityCheck only. + # back to baseline by returning scope=None. This guarantees an + # invariant the Groovy side relies on: any cbts result with + # scope!=None has a non-empty pre-filter `affected_stages` set, so + # post-filter emptiness unambiguously means trigger-mode mismatch + # (Layer 2 falls back to PackageSanityCheck only). if not affected_stages: return SelectionResult( scope=None, @@ -177,10 +166,6 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: ], ) - affected_cpu_arch = { - self.stages[name].cpu_arch for name in affected_stages if name in self.stages - } - # Aggregate per-block prefix->{waive_ids} maps across rules. Same # block keyed by multiple rules: union the waive_ids per prefix. block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} @@ -193,8 +178,6 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: return SelectionResult( scope=scope, affected_stages=affected_stages, - affected_cpu_arch=affected_cpu_arch, - tests=tests, reasons=reasons, block_filters=block_filters, ) @@ -325,17 +308,15 @@ def main(argv: Optional[list[str]] = None) -> int: else: result.affected_stages = {s for s in pre_filter_stages if "Post-Merge" not in s} # Recompute derived fields against the filtered set. - result.affected_cpu_arch = { - stages[name].cpu_arch for name in result.affected_stages if name in stages - } result.affected_stage_test_counts = { k: v for k, v in result.affected_stage_test_counts.items() if k in result.affected_stages } - # Flag the "build only, no test cases" no-op: rules resolved stages, but - # none survive the user's trigger-mode filter. Surfaced to Groovy so the - # Layer 2 log can say why — and so future consumers can branch on it - # instead of inferring from `not affected_stages`. - result.trigger_mode_mismatch = bool(pre_filter_stages and not result.affected_stages) + # Note: empty `affected_stages` here (after the trigger-mode filter, with + # `scope != None`) means "rules resolved stages but none match the user's + # trigger mode" — a.k.a. trigger-mode mismatch. Layer 2 in Groovy detects + # this with `affected_stages.isEmpty()` and falls back to PackageSanityCheck + # only. The Selector safety-net above ensures we never reach this point + # with `affected_stages` empty BEFORE the filter. _log_decision_to_stderr(stages, result, pr, pre_filter_stages) sys.stdout.write(result.to_json()) @@ -367,7 +348,6 @@ def _log_decision_to_stderr( print(f"CBTS decision (diagnostic; stderr only) [trigger mode: {mode}]:", file=out) print(f" scope: {result.scope}", file=out) print(f" test_db_dir_override: {result.test_db_dir_override}", file=out) - print(f" affected_cpu_arch: {sorted(result.affected_cpu_arch)}", file=out) if dropped: print( f" stages dropped by trigger-mode filter ({len(dropped)}, would run if mode flipped):", @@ -379,9 +359,6 @@ def _log_decision_to_stderr( print(" reasons:", file=out) for r in result.reasons: print(f" - {r}", file=out) - print(f" affected_tests ({len(result.tests)}):", file=out) - for t in sorted(result.tests): - print(f" - {t}", file=out) print(f" block_filters ({len(result.block_filters)} blocks):", file=out) for (yaml_stem, idx), prefix_to_waives in sorted(result.block_filters.items()): print(f" - {yaml_stem}#{idx}:", file=out) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index b9f53e553b5e..19e86a653aed 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -51,7 +51,6 @@ class RuleResult: """ handled_files: set[str] - tests: set[str] affected_stages: set[str] scope: Optional[str] reason: str diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 3d4fa4e287ad..d83e8ad4a57c 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -82,7 +82,6 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: if not changed_test_ids: return RuleResult( handled_files={WAIVES_FILE}, - tests=set(), affected_stages=set(), scope="waiveonly", reason="waives.txt: no actionable test ids in diff", @@ -118,7 +117,6 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: more = f" (+{len(misses) - 3} more)" if len(misses) > 3 else "" return RuleResult( handled_files={WAIVES_FILE}, - tests=changed_test_ids, affected_stages=set(), scope=None, # Selector treats this as "no decision" → fallback reason=f"waives.txt: {len(misses)} unmatchable waive(s): {preview}{more}", @@ -132,7 +130,6 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: return RuleResult( handled_files={WAIVES_FILE}, - tests=changed_test_ids, affected_stages=affected_stage_names, scope="waiveonly", block_filters=block_filters, From 48d1837bf048a164161ce84ecd8ace46ceb36382 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 19:10:10 +0800 Subject: [PATCH 60/65] [None][docs] CBTS: trim verbose comments and README Drop rationale, history, and thinking-process commentary from the CBTS files. Keep one-line descriptions of what the current code does; move deeper context to commit messages and the README's relevant section. No behavior change. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 82 ++------ jenkins/L0_Test.groovy | 43 +---- jenkins/scripts/cbts/README.md | 224 ++++++++-------------- jenkins/scripts/cbts/main.py | 101 +++------- jenkins/scripts/cbts/rules/base.py | 33 +--- jenkins/scripts/cbts/rules/waives_rule.py | 27 +-- 6 files changed, 139 insertions(+), 371 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index a34e5beaa858..0dc437785d79 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -703,15 +703,10 @@ def getAutoTriggerTagList(pipeline, testFilter, globalVars) { // ============================================================================ // CBTS (Change-Based Testing Selection) // -// Upstream decision point. Calls jenkins/scripts/cbts/main.py with the PR's -// changed_files + diffs; Python self-sources stage configs from L0_Test.groovy -// and YAML blocks from test-db. Returns a dict {scope, affected_stages, -// reasons, test_db_dir_override, affected_stage_test_counts} or null -// (= no decision / fall back to the existing filter chain). -// -// See jenkins/scripts/cbts/README.md for the consumption model. CBTS only -// narrows test cases (Layer 2 stage filter + Layer 3 within-stage filter); -// it never affects Build, so the wheel always exists. +// Calls jenkins/scripts/cbts/main.py with PR changed_files + diffs and returns +// a result map (or null = defer to existing filter chain). Result keys: +// scope, affected_stages, reasons, test_db_dir_override, affected_stage_test_counts. +// CBTS narrows test cases only — Build always runs. See cbts/README.md. // ============================================================================ def getCbtsResult(pipeline, testFilter, globalVars) @@ -737,19 +732,15 @@ def getCbtsResult(pipeline, testFilter, globalVars) } try { - // 0. Ensure pyyaml is available on the Jenkins agent (blocks.py needs it - // to parse test-db YAMLs). buildpack-deps has no pip3 by default, - // so install the Debian python3-yaml package directly. + // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // 1. Ask Python for the union of needs_diff_for patterns across all rules. + // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", returnStdout: true, ).trim() def needsDiffFor = patternsOut ? patternsOut.readLines().collect { it.trim() }.findAll { it } : [] - - // 2. For each changed file matching a needs_diff_for pattern, pull the diff. def diffs = [:] for (f in changedFiles) { if (_cbtsMatchesAnyPattern(f, needsDiffFor)) { @@ -757,10 +748,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } - // 3. Write INPUT_JSON (PR data only; Python reads stages/yaml itself). - // `post_merge` lets Python apply the trigger-mode filter on - // affected_stages before returning the JSON, so Layer 2 in Groovy - // (L0_Test.groovy::launchTestJobs) stays scope-/mode-agnostic. + // Write INPUT_JSON; Python reads stages/yaml itself. def inputJson = groovy.json.JsonOutput.toJson([ changed_files: changedFiles, diffs: diffs, @@ -769,30 +757,20 @@ def getCbtsResult(pipeline, testFilter, globalVars) def inputPath = "${LLM_ROOT}/cbts_input.json" writeFile file: inputPath, text: inputJson - // 4. Run Python; capture stdout. def output = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json", returnStdout: true, ) - // 5. Parse stdout into the map shape consumed by Layer 2/2.5/3. def result = _cbtsParseSelectionResult(output) if (result.scope == null) { pipeline.echo("CBTS: deferring — Python returned scope=null. " + "Reasons: ${result.reasons.join('; ')}") return null } - // Layer 3 cross-job seed: piggyback the input JSON on testFilter so - // each L0_Test stage agent can re-run main.py locally and regenerate - // its own copy of cbts_test_db/. The directory written here lives on - // the L0_MergeRequest agent and never reaches downstream pods. - // - // Hard cap to keep us well below ARG_MAX (~2 MB on most kernels) and - // Jenkins's per-parameter handling. CACHED_CHANGED_FILE_LIST already - // hits "Argument list too long" at smaller sizes (see comment on - // launchJob), so 256 KB is conservative. If we exceed it, drop the - // piggyback — Layer 2 stage filtering still applies, and renderTestDB - // falls back to the source test-db automatically. + // Piggyback input JSON on testFilter so each L0_Test stage agent can + // re-run main.py and regenerate cbts_test_db/ locally. Capped at + // 256 KB; oversize → drop piggyback, Layer 3 falls back to source. final int CBTS_INPUT_PIGGYBACK_MAX_BYTES = 256000 def inputJsonSize = inputJson.length() if (inputJsonSize <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { @@ -813,25 +791,17 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Translate an Ant-style glob to a regex. -// **/ zero or more path segments -// ** any chars (including /) -// * any chars except / -// ? single char except / -// Implemented in pure Groovy — `hudson.util.AntPathMatcher` is not visible -// to the Jenkins script sandbox classpath. Exact paths (no glob meta) round- -// trip to a literal regex, so existing rules with literal needs_diff_for keep -// working without changes. +// Translate an Ant-style glob to a regex: +// **/ zero or more path segments +// ** any chars (including /) +// * any chars except / +// ? single char except / def _cbtsGlobToRegex(String glob) { - // 1. Escape regex specials, except glob metas (* and ?) which we handle below. def escaped = glob.collect { c -> (c == '*' || c == '?') ? c : ('.+()[]{}|^$\\'.contains(c) ? '\\' + c : c) }.join('') - // 2. Translate glob metas. Use unambiguous text sentinels so cascading - // replaces don't double-match. Avoid unicode-escape placeholders - // because the Groovy lexer expands those even inside string literals. return '^' + escaped .replace('**/', '__CBTSDOUBLESLASH__') .replace('**', '__CBTSDOUBLESTAR__') @@ -846,15 +816,9 @@ def _cbtsMatchesAnyPattern(String filePath, List patterns) return patterns.any { filePath ==~ _cbtsGlobToRegex(it) } } -// CBTS only activates on `/bot run` and `/bot run --post-merge`. Any other -// stage-selection flag makes it defer to the user's explicit choice. -// Orthogonal flags (REUSE_*, DEBUG_MODE, DETAILED_LOG) and IS_POST_MERGE are -// intentionally not in this list — they either don't affect stage selection -// or are handled specially in Layer 2. Adding a new stage-selection flag in -// the future means adding one entry here; nothing else changes. -// -// All defer flags default to falsy (false / null), so a single truthy check -// captures both boolean and list-typed flags. +// Returns user-set stage-selection flags that should force CBTS to defer. +// IS_POST_MERGE and orthogonal flags (REUSE_*, DEBUG_MODE, DETAILED_LOG, ...) +// are intentionally absent. def _cbtsTriggeredUserFlags(testFilter) { def deferFlags = [ @@ -865,9 +829,8 @@ def _cbtsTriggeredUserFlags(testFilter) .collect { "${it}=${testFilter[it]}" } } -// Parse CBTS JSON stdout into the shape consumed by Layer 2/2.5/3. Always -// returns a map; `scope == null` means "no decision" (caller should log the -// reasons and treat as defer). +// Parse CBTS JSON stdout into a map. `scope == null` → no decision; caller +// logs reasons and defers. def _cbtsParseSelectionResult(String text) { def data = new groovy.json.JsonSlurper().parseText(text) @@ -875,10 +838,7 @@ def _cbtsParseSelectionResult(String text) scope: data.scope, affected_stages: data.affected_stages ?: [], reasons: data.reasons ?: [], - test_db_dir_override: data.test_db_dir_override, // Layer 3: tmp test-db path - // Layer 3 split-collapse heuristic: per-stage narrowed test count. - // launchTestJobs reads this to drop excess pytest-split groups when - // the affected stage's narrowed count falls below the 20-test threshold. + test_db_dir_override: data.test_db_dir_override, affected_stage_test_counts: data.affected_stage_test_counts ?: [:], ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0eb5ac03c70f..4dce9e05ef27 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2607,24 +2607,13 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" - // CBTS Layer 3: use the narrowed test-db when CBTS provides one. - // Perf stages are excluded at Layer 2 (launchTestJobs) and never - // reach this path with cbts != null, so no perfMode guard is needed here. + // CBTS Layer 3: regenerate cbts_test_db/ on this stage agent from the + // piggybacked input JSON if not already present. def cbts = testFilter[(CBTS_RESULT)] - // Regenerate cbts_test_db/ on this stage's agent. L0_MergeRequest's - // getCbtsResult ran main.py on its own pod and produced the dir there, - // but L0_Test stages run in separate Kubernetes pods that never receive - // that dir. Re-running main.py here with the piggybacked input JSON is - // deterministic — output matches what L0_MergeRequest produced. - // Idempotent: only runs when cbts_test_db/ doesn't already exist. if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json) { def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() if (dirExists != "yes") { - // Write input JSON to a JNLP-writable temp location instead of - // ${llmSrc}/, which the build container created as root and - // would reject writeFile() with AccessDeniedException. Matches - // the scriptLaunch*PathLocal pattern used in the SLURM path. def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") pipeline.writeFile(file: cbtsInputLocal, text: cbts.cbts_input_json) sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" @@ -2657,8 +2646,6 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu ].join(" ") sh(label: "Render test list from test-db", script: testDBQueryCmd) - // CBTS diagnostics: show how many tests survived the trt-test-db query - // and which test-db source was used, to make empty-list bugs visible. def testCount = sh(returnStdout: true, script: "wc -l < ${testList} | tr -d ' '").trim() def testDBLabel = (cbts != null && cbts.test_db_dir_override) ? "CBTS-narrowed [${cbts.scope}]" : "source" echo "renderTestDB: stage=${stageName} context=${testContext} test-db=${testDBLabel} dir=${testDBPath} -> ${testCount} tests" @@ -4491,29 +4478,9 @@ def launchTestJobs(pipeline, testFilter) checkStageNameSet(testFilter[(EXTRA_STAGE_LIST)], fullSet, EXTRA_STAGE_LIST) } - // CBTS Layer 2: stage-level filter. Runs AFTER all existing filter rules - // so unknown / no-decision paths fall through naturally. CBTS narrows - // *test stages* only — Build always runs (the L0_MergeRequest arch-track - // skip was removed deliberately so the wheel and PackageSanityCheck path - // are never disturbed by CBTS). See jenkins/scripts/cbts/README.md. - // - // - Perf stages: excluded. They have their own trigger model and need - // full test lists. - // - PackageSanityCheck stages: force-kept. Their stage names are built - // at runtime so the CBTS Python parser cannot see them, and they are - // wheel/image gates that should always run after Build. - // - Trigger-mode mismatch (affectedSet empty after Python's pre/post - // filter): rules resolved stages but none match the user's - // /bot run [--post-merge] mode. The filter naturally collapses to - // PackageSanityCheck only — equivalent to "build + sanity, no test - // cases", in spirit similar to /bot run --stage-list "". The Python - // safety net guarantees pre-filter `affected_stages` is non-empty - // whenever `cbts != null`, so empty here unambiguously means mismatch. - // - // Pre-merge vs Post-Merge filtering already happened in Python - // (main.py applies it based on the post_merge flag plumbed through - // cbts_input.json), so `cbts.affected_stages` here is already - // restricted to the user's trigger mode. + // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus + // PackageSanityCheck (force-kept); Perf is excluded. Empty affectedSet + // means trigger-mode mismatch → PackageSanityCheck only. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { def affectedSet = (cbts.affected_stages ?: []) as Set diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 93705ccb298a..6e21c9805a52 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -1,47 +1,27 @@ # CBTS — Change-Based Testing Selection -Pre-merge CI test-selection tool. Looks at what the PR changed and narrows the -set of Jenkins stages — and the tests inside each stage — that actually need -to run. **Adding new rules is Python-only — Groovy is scope-agnostic and -consumes the data directly.** +CI test-selection tool. Narrows the Jenkins stages and per-stage tests that +run, based on what the PR changed. New rules are added in Python only. --- ## Consumption layers -CBTS narrows **test cases only**. Build always runs (`L0_MergeRequest.groovy` -arch-track skipping was removed deliberately) so the wheel exists for sanity -checks and post-merge consumers. +CBTS narrows test cases only; Build always runs. | Layer | Where | Action | |---|---|---| -| **2. Stage** | `L0_Test.groovy::launchTestJobs` (end of filter chain) | Replace `parallelJobsFiltered` with the CBTS-selected subset. Perf stages are excluded (they have their own trigger model and need full lists); `*-PackageSanityCheck-*` stages are force-kept (their names are runtime-built and invisible to the CBTS Python parser, and they are wheel/image gates that should always run after Build). | -| **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOnSlurm` and `runLLMTestlistOnPlatform` entries | When the affected stage's narrowed test count is < 20, collapse pytest-split's splits to 1 — only group 1 runs everything; groups 2..N skip without allocating a machine. At/above 20 the stage's default splits stand and pytest-split parallelizes normally. | -| **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the CBTS-narrowed tmp test-db. Each affected block's `tests:` array is filtered to entries in the per-block filter prefix subtree, **and unaffected blocks are dropped entirely** so a `/bot run --post-merge` can't accidentally activate post-merge blocks the PR never touched. | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (force-kept). Perf stages are excluded. Empty affectedSet → PackageSanityCheck only (trigger-mode mismatch). | +| **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOn*` entries | Narrowed test count < 20 → collapse pytest-split to splits=1 (only group 1 runs); else default splits stand. | +| **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the narrowed `cbts_test_db/`. Each affected block's `tests:` is restricted to entries in the filter prefix subtree; unaffected blocks are dropped. | -(Layer 1 in earlier revisions skipped the entire arch track / Build when no -stage on that arch was affected. Removed: see -`L0_MergeRequest.groovy::launchStages`. CBTS no longer touches Build.) - -### Trigger-mode mismatch - -If `/bot run` is issued but every CBTS-resolved stage is post-merge (or the -symmetric case with `--post-merge`), Python's trigger-mode filter empties -`affected_stages` while leaving `scope != null`. The Selector safety-net -guarantees `affected_stages` is never empty BEFORE the filter when -`scope != null`, so Layer 2 detects the mismatch unambiguously with -`affectedSet.isEmpty()` and narrows to the PackageSanityCheck stages only -— equivalent to a build-and-sanity-only run, in spirit similar to -`/bot run --stage-list ""`. - -CBTS only **subtracts** stages and tests, never adds. Anything it can't -narrow → full fallback to the existing filter chain. +CBTS only subtracts; anything it can't narrow → fallback to the existing +filter chain. ## v0 scope -- **Only handles** `tests/integration/test_lists/waives.txt` changes (`scope: waiveonly`). +- Only handles `tests/integration/test_lists/waives.txt` changes (`scope: waiveonly`). - Anything else → `scope: none` → full run. -- **v1+ rules can be added in Python alone** (no Groovy edits). ## File map @@ -56,74 +36,53 @@ jenkins/scripts/cbts/ └── waives_rule.py v0's only rule ``` -## Lookup algorithm: parent chain with first-match wins +## Lookup algorithm -Per waive id, `YAMLIndex.find_match_for_waive` walks the pytest tree from the -waive towards the root. The first level whose YAML has a matching entry wins; -that level becomes the **filter prefix** the block uses for Layer 3. Each -prefix remembers the originating waive id(s) so `write_filtered_test_db` can -re-apply the `-k` keyword guard when narrowing. +`YAMLIndex.find_match_for_waive` walks the pytest tree from the waive id +toward the root; the first level whose YAML has a matching entry becomes the +filter prefix for that block. Prefixes remember their originating waive +id(s) so `write_filtered_test_db` can apply the `-k` keyword guard. ``` waive id (raw) ↓ normalize strip SKIP/TIMEOUT/full:gpu/comments ↓ strip [params] if present -target_lookup (function-level when waive was parametrized; otherwise - class/file/dir level — waive's own granularity) +target_lookup (function/class/file/dir level) ↓ try YAML at this level - hit → matched: filter prefix = level (recorded with the originating waive id) - miss → strip one level up (::method → ::class → /file → /dir → ...) - and retry - ↓ all levels miss → fallback: rule emits scope=None, baseline runs + hit → filter prefix = level (with originating waive ids) + miss → strip one level up and retry + ↓ all levels miss → rule emits scope=None ``` -An entry "matches" at a level when its **canonical target** (entry with -`SKIP`/`TIMEOUT`/`full:gpu`, pytest options `-k "..."` / `-m "..."`, and -`[params]` all stripped) equals the level **and** any `-k` keyword filter the -entry carries actually contains an identifier present in the waive id. -`-m` markers are unverifiable from a string and always pass (over-include -when in doubt). +An entry matches a level when its canonical target (with `SKIP`/`TIMEOUT`/ +`full:gpu`/`-k`/`-m`/`[params]` stripped) equals the level AND any `-k` +keyword filter on the entry contains an identifier from the waive id. `-m` +markers always pass (unverifiable from string). -The `-k` keyword guard is applied **twice** by design: - -1. **At lookup** (`find_match_for_waive`) — to decide whether the entry - contributes to a block's filter prefix. -2. **At write** (`write_filtered_test_db`) — to drop sibling `-k "..."` - entries that survive prefix-subtree match but whose keyword can't pick - up the waived test (e.g., waive `func[CUTLASS-fp8-tp4]` keeps - `-k "CUTLASS"` but not `-k "TRTLLM"`). +The `-k` keyword guard runs twice: once at lookup, once when writing +`cbts_test_db/` (drops sibling entries whose `-k` doesn't match the waived +test). ## When CBTS activates -CBTS narrows test selection in **two usages only**: - -- `/bot run` — full pre-merge with CBTS narrowing. -- `/bot run --post-merge` — post-merge with CBTS narrowing. Layer 2 keeps - only post-merge hits; no post-merge hit → no-op (no fallback to full - post-merge baseline). +CBTS activates on bare `/bot run` and `/bot run --post-merge`. Any +stage-selection flag (`--stage-list`, `--extra-stage`, `--gpu-type`, +`--test-backend`, `--skip-test`, `--add-multi-gpu-test`, `--only-multi-gpu-test`, +`--disable-multi-gpu-test`) makes `getCbtsResult` return null. -Any other **stage-selection** flag makes `getCbtsResult` return `null` and -the existing filter chain takes over: `--stage-list`, `--extra-stage`, -`--gpu-type`, `--test-backend`, `--skip-test`, `--add-multi-gpu-test`, -`--only-multi-gpu-test`, `--disable-multi-gpu-test`. - -**Orthogonal** flags don't change stage selection and don't affect CBTS: -`--reuse-test`, `--disable-reuse-test`, `--debug`, `--detailed-log`, -`--disable-fail-fast`, `--high-priority`. +Orthogonal flags (`--reuse-test`, `--disable-reuse-test`, `--debug`, +`--detailed-log`, `--disable-fail-fast`, `--high-priority`) do not affect CBTS. ## How it's invoked (CI) `getCbtsResult` calls `main.py` twice on the L0_MergeRequest agent: -1. `main.py --list-needed-diffs` → patterns whose diffs Groovy fetches. - Patterns are **Ant-style globs** (`tests/**/*.py`, `cpp/kernels/**`, exact - paths), matched via `hudson.util.AntPathMatcher`. -2. `main.py cbts_input.json` → decision JSON on stdout. If any block was - narrowed, also writes `${LLM_ROOT}/cbts_test_db/` containing only the - affected YAMLs with only their affected blocks (others dropped). Each - kept entry preserves `TIMEOUT (n)`, `ISOLATION`, `-k "..."`, `-m "..."` - verbatim (YAML-level `# comments` are dropped by PyYAML round-trip but - no functional info is lost). +1. `main.py --list-needed-diffs` → file patterns whose diffs Groovy must fetch + (Ant-style globs). +2. `main.py cbts_input.json` → decision JSON on stdout. When any block was + narrowed, writes `${LLM_ROOT}/cbts_test_db/` with the affected YAMLs and + only their affected blocks (kept entries preserve `TIMEOUT (n)`, + `ISOLATION`, `-k`, `-m` verbatim). Decision JSON: @@ -137,52 +96,35 @@ Decision JSON: } ``` -- `scope: null` → no decision, full fallback. Groovy doesn't gate on the - scope value — it's metadata for logs and multi-rule combining only. +- `scope: null` → no decision; Groovy defers to baseline. - `test_db_dir_override: null` → no Layer 3 narrowing; trt-test-db reads - the source `tests/integration/test_lists/test-db/` as before. -- `affected_stage_test_counts` → per-stage post-keep-filter test count. - Drives Layer 2.5 split-collapse below. + the source test-db. +- `affected_stage_test_counts` → per-stage post-keep-filter test count for + Layer 2.5 split-collapse. ## Cross-job seed for stage agents -The `cbts_test_db/` written above lives on the L0_MergeRequest pipeline pod -and never reaches downstream `L0_Test-*` jobs (separate Kubernetes pods / -SLURM nodes). To make the narrowed test-db available to each stage agent -without a cross-job stash: - -1. `getCbtsResult` puts the **input JSON itself** (`changed_files` + diffs) - into `result.cbts_input_json`, which rides along inside `testFilter` as - a normal build parameter. -2. `renderTestDB` on the stage agent receives it, writes a temp - `cbts_input.json` (via `Utils.createTempLocation` → JNLP-writable - path), and re-runs `python3 jenkins/scripts/cbts/main.py ` so - the narrowed `cbts_test_db/` materializes locally alongside the - source. main.py is deterministic, so each agent ends up with a - byte-identical copy of what L0_MergeRequest produced. -3. trt-test-db then queries `cbts_test_db/` as usual. - -**Size cap.** `cbts_input_json` is dropped from the piggyback when its -size exceeds 256 KB (well below ARG_MAX). Layer 2 stage filtering still -applies, but Layer 3 narrowing on each stage agent silently degrades to -"no override" and `renderTestDB` falls back to the source test-db. +`cbts_test_db/` is written on the L0_MergeRequest agent and is not +available to downstream `L0_Test-*` pods. To regenerate it per stage: -## Split-collapse heuristic (Layer 2.5) +1. `getCbtsResult` stores the input JSON in `result.cbts_input_json`, + which rides along inside `testFilter`. +2. `renderTestDB` on the stage agent writes it to a temp file and re-runs + `main.py`. Output is deterministic, so each agent gets the same + `cbts_test_db/` as L0_MergeRequest produced. -When the affected stage's narrowed test count is below the hard-coded -threshold of **20** (in `_cbtsMaybeCollapseSplits`): +If `cbts_input_json` exceeds 256 KB the piggyback is dropped; Layer 3 falls +back to the source test-db on each stage agent. Layer 2 still applies. -- `splitId == 1` → keep, override `splits = 1` so this single agent runs - the full narrowed list. -- `splitId > 1` → early `return`; no agent allocated. +## Split-collapse heuristic (Layer 2.5) -At/above the threshold, the stage's default splits stand and pytest-split -parallelizes normally. +In `_cbtsMaybeCollapseSplits`, when the stage's narrowed count < 20: +- `splitId == 1` → run as splits=1 (single agent runs the full list). +- `splitId > 1` → early return; no agent allocated. -The per-stage count is computed by `blocks.compute_stage_test_counts`, -which sums kept entries across blocks the stage's mako matches. The same -keep filter as `write_filtered_test_db` is applied so the count matches -what trt-test-db will eventually render. +At/above 20, default splits stand. The count is computed by +`blocks.compute_stage_test_counts` using the same keep filter as +`write_filtered_test_db`. ## Adding a new rule @@ -205,14 +147,11 @@ what trt-test-db will eventually render. ... return RuleResult( handled_files={...}, - tests={...}, affected_stages={...}, scope="myscope", reason="why this fired", - # Optional Layer 3 contribution: per-block prefix → set of - # waive ids that resolved to it. Selector unions across - # rules; write_filtered_test_db uses both the prefix - # (subtree match) AND the waive ids (-k keyword guard). + # Optional Layer 3 contribution: per-block prefix → + # originating waive ids. Selector unions across rules. block_filters={ (yaml_stem, block_index): { filter_prefix: {originating_waive_id, ...}, @@ -222,41 +161,32 @@ what trt-test-db will eventually render. ) ``` -2. **Register in `main.py`**: add to `RULE_CLASSES` and `build_rules()`. +2. Register in `main.py` (`RULE_CLASSES` and `build_rules()`). -3. **No Groovy edits needed.** Layers 2 / 2.5 / 3 are scope-agnostic and - consume `affected_stages` / `block_filters` / `affected_stage_test_counts` - directly. +3. No Groovy edits needed. -Rule order is irrelevant. `Selector` unions `affected_stages` and -`block_filters`; scopes are combined via `_combine_scopes` (all-agree → that -scope; disagreement → `None`). +`Selector` unions `affected_stages` and `block_filters`; scopes are combined +via `_combine_scopes` (all-agree → that scope; otherwise None). ## Fallback paths -CBTS falls back to the existing filter chain when: +CBTS defers to the existing filter chain when: - PostMerge job / `alternativeTRT` set - `changed_files` is empty -- `main.py` throws / stdout is unparsable -- Python returns `scope: null` ("no decision") -- A waive id misses every level up to the root in `find_match_for_waive` - (likely typo'd or out-of-tree id) — the rule emits `scope: null` -- `affected_stages` is empty (Layer 2 no-op) -- Layer 3 filter would empty a block's `tests:` array — that block keeps - its original tests instead (per-block safety net) -- `cbts_input_json` exceeds the 256 KB piggyback cap — Layer 3 narrowing - is dropped per stage; renderTestDB falls back to source test-db -- The narrowed YAML for this stage's testContext is missing or empty on - the stage agent (e.g., main.py regen failed) — renderTestDB falls back - to source test-db - -Every fallback logs an `echo` line — no silent failures. +- `main.py` throws or stdout is unparsable +- Python returns `scope: null` +- A waive id misses every level in `find_match_for_waive` — rule emits + `scope: null` +- Layer 3 narrowing would empty a block — block keeps original tests +- `cbts_input_json` exceeds 256 KB — Layer 3 falls back per stage +- Narrowed YAML missing/empty on a stage agent — renderTestDB falls back + +Every fallback emits an `echo` log line. ## Keep-in-sync notes -`blocks.py::derive_mako_from_stage` mirrors the Groovy -`getMakoArgsFromStageName` (`L0_Test.groovy` ~line 2079) and -`parseTaskConfigFromStageName` (~line 2066). New backends / orchestrators / -stage-name conventions on the Groovy side need a matching Python update — -file comments flag this. +`blocks.py::derive_mako_from_stage` mirrors Groovy +`getMakoArgsFromStageName` (~`L0_Test.groovy:2079`) and +`parseTaskConfigFromStageName` (~`:2066`). Update both when adding new +backends / orchestrators / stage-name conventions. diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index ffd71eb8f32e..bc9bcf54adb2 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -12,28 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""CBTS entry point — consumed by Jenkins Groovy helper `getCbtsResult`. - -Two invocation modes (see README.md for full context): +"""CBTS entry point. +Modes: python3 main.py --list-needed-diffs - Print the union of all rules' `needs_diff_for` patterns, one per line. - Groovy uses this to decide which changed files to fetch diffs for. - + Print the union of rules' `needs_diff_for` patterns, one per line. python3 main.py INPUT_JSON - Run decision logic. Groovy passes a JSON file containing only PR data: - - changed_files: list[str] - - diffs: {path: diff_content} - Python self-sources everything else from the repo: - - stage configs: parsed from jenkins/L0_Test.groovy - - test-db YAMLs: loaded from tests/integration/test_lists/test-db/ - Output is a JSON blob on stdout with fields `scope`, `affected_stages`, - `reasons`, `test_db_dir_override`, `affected_stage_test_counts`. Consumed - by `_cbtsParseSelectionResult` - on the Groovy side. - -Invocation assumes the current working directory is the TRT-LLM repo root, -or that --repo-root is passed explicitly. + INPUT_JSON: {changed_files: [...], diffs: {path: diff}, post_merge: bool}. + Stages are parsed from jenkins/L0_Test.groovy; YAMLs from + tests/integration/test_lists/test-db/. Decision JSON goes to stdout + with keys: scope, affected_stages, reasons, test_db_dir_override, + affected_stage_test_counts. + +Run from the TRT-LLM repo root or pass --repo-root. """ from __future__ import annotations @@ -73,23 +64,14 @@ def build_rules(yaml_index: YAMLIndex, stages: dict[str, Stage]) -> list[Rule]: @dataclass class SelectionResult: - """Final aggregated decision. - - `block_filters` and `test_db_dir_override` drive CBTS Layer 3 (within-stage - test filtering). After the Selector aggregates per-rule `block_filters`, - `main.py` writes a tmp test-db dir with each affected block's `tests:` - array narrowed to entries in the per-block filter prefix subtree, then - sets `test_db_dir_override` so Groovy points trt-test-db at it. - """ + """Aggregated CBTS decision serialized to JSON for Groovy.""" scope: Optional[str] affected_stages: set[str] = field(default_factory=set) reasons: list[str] = field(default_factory=list) block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) test_db_dir_override: Optional[str] = None - # Per-stage narrowed test count (sum across blocks the stage's mako - # matches, post-keep-filter). Groovy launchTestJobs uses this to - # collapse splits to 1 when the count is below the 20-test threshold. + # Per-stage narrowed test count, used by Layer 2.5 split-collapse. affected_stage_test_counts: dict[str, int] = field(default_factory=dict) def to_json(self) -> str: @@ -104,12 +86,7 @@ def to_json(self) -> str: def _combine_scopes(scopes: list[str]) -> Optional[str]: - """Combine scope labels from multiple rules. - - v0: single rule => passthrough. - Multi-rule future: when all scopes agree, use that scope; otherwise return - None (no-decision / full run) until an explicit priority table is added. - """ + """Return the common scope if all agree, else None.""" if not scopes: return None if len(set(scopes)) == 1: @@ -149,13 +126,9 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: for _, r in pairs: affected_stages |= r.affected_stages - # Safety net: if rules fired but no Jenkins stages resolved (waive ids - # missed both exact and parent-chain lookups in the YAML index), fall - # back to baseline by returning scope=None. This guarantees an - # invariant the Groovy side relies on: any cbts result with - # scope!=None has a non-empty pre-filter `affected_stages` set, so - # post-filter emptiness unambiguously means trigger-mode mismatch - # (Layer 2 falls back to PackageSanityCheck only). + # If rules fired but no stages resolved, return scope=None so + # downstream falls back to baseline. Maintains the invariant that any + # scope!=None result has a non-empty pre-filter affected_stages set. if not affected_stages: return SelectionResult( scope=None, @@ -166,8 +139,7 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: ], ) - # Aggregate per-block prefix->{waive_ids} maps across rules. Same - # block keyed by multiple rules: union the waive_ids per prefix. + # Aggregate per-block prefix->{waive_ids} across rules. block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} for _, r in pairs: for key, prefix_to_waives in r.block_filters.items(): @@ -267,18 +239,14 @@ def main(argv: Optional[list[str]] = None) -> int: return 2 yaml_index = YAMLIndex.load(test_db_dir) - # Include post-merge stages so waives on post-merge-only tests resolve. - # Layer 2 in L0_Test.groovy decides what to run with the post-merge - # subset based on the user's --post-merge flag. + # Include post-merge stages; the trigger-mode filter below selects the + # subset matching the user's flag. stages = parse_stages_from_groovy(groovy_path, include_post_merge=True) pr = _load_pr_inputs(input_path) rules = build_rules(yaml_index, stages) result = Selector(stages).run(pr, rules) - # Layer 3: if any block has filter prefixes, write a tmp test-db so - # trt-test-db downstream renders a narrower testDBList for the affected - # stages. The path is relative to repo_root so Groovy can resolve it as - # `${LLM_ROOT}/cbts_test_db`. + # Layer 3: write narrowed test-db when any block was filtered. if result.scope is not None and result.block_filters: out_dir_name = "cbts_test_db" write_filtered_test_db( @@ -287,8 +255,6 @@ def main(argv: Optional[list[str]] = None) -> int: block_filters=result.block_filters, ) result.test_db_dir_override = out_dir_name - # Per-stage narrowed test count for the launchTestJobs split-collapse - # heuristic (collapse pytest-split to splits=1 when count < 20). result.affected_stage_test_counts = compute_stage_test_counts( yaml_index=yaml_index, stages=stages, @@ -296,27 +262,15 @@ def main(argv: Optional[list[str]] = None) -> int: block_filters=result.block_filters, ) - # Trigger-mode filter: drop affected_stages incompatible with the user's - # /bot run [--post-merge] flag. Done in Python (not Groovy) so the JSON - # contract Groovy reads already reflects "stages that should run NOW", - # and Layer 2 in Groovy stays scope/mode-agnostic. Narrowed YAML output - # (cbts_test_db) is unaffected — runtime mako matching there picks the - # right block per stage. + # Filter affected_stages by trigger mode; recompute derived counts. pre_filter_stages = set(result.affected_stages) if pr.post_merge: result.affected_stages = {s for s in pre_filter_stages if "Post-Merge" in s} else: result.affected_stages = {s for s in pre_filter_stages if "Post-Merge" not in s} - # Recompute derived fields against the filtered set. result.affected_stage_test_counts = { k: v for k, v in result.affected_stage_test_counts.items() if k in result.affected_stages } - # Note: empty `affected_stages` here (after the trigger-mode filter, with - # `scope != None`) means "rules resolved stages but none match the user's - # trigger mode" — a.k.a. trigger-mode mismatch. Layer 2 in Groovy detects - # this with `affected_stages.isEmpty()` and falls back to PackageSanityCheck - # only. The Selector safety-net above ensures we never reach this point - # with `affected_stages` empty BEFORE the filter. _log_decision_to_stderr(stages, result, pr, pre_filter_stages) sys.stdout.write(result.to_json()) @@ -329,18 +283,7 @@ def _log_decision_to_stderr( pr: PRInputs, pre_filter_stages: set[str], ) -> None: - """Dump the full CBTS decision to stderr for Jenkins console diagnostics. - - stdout carries the JSON consumed by Groovy `getCbtsResult`, so all - human-readable detail goes to stderr to avoid corrupting that contract. - Each affected stage is annotated with its yaml_stem so blocks-vs-stages - mismatches (e.g. a stage matched by an unexpected YAML) are obvious. - - `pre_filter_stages` is the affected_stages set BEFORE the trigger-mode - filter (pre-merge vs post-merge). Stages dropped by the filter are - listed separately so reviewers can see why a stage CBTS narrowed to - isn't actually running. - """ + """Print the CBTS decision to stderr for Jenkins console diagnostics.""" out = sys.stderr mode = "post_merge" if pr.post_merge else "pre_merge" dropped = sorted(pre_filter_stages - set(result.affected_stages)) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 19e86a653aed..f760517d58e4 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -22,15 +22,7 @@ @dataclass class PRInputs: - """Inputs about the PR that rules can query. - - `post_merge` reflects the user's `/bot run [--post-merge]` flag. Rules - themselves don't consult it (their narrowing is mode-agnostic); main.py - uses it after `Selector.run` to drop `affected_stages` entries that - don't match the trigger mode (pre-merge vs Post-Merge by stage-name - convention). See `main.py::main` for the filter; default False keeps - backward compat with older Groovy that didn't pass the field. - """ + """PR data rules query. `post_merge` reflects /bot run --post-merge.""" changed_files: list[str] diffs: dict[str, str] @@ -39,15 +31,10 @@ class PRInputs: @dataclass class RuleResult: - """What a single rule contributes when it applies to a PR. + """One rule's contribution. - `block_filters` (CBTS Layer 3): per-block map of filter prefix -> set of - waive ids that resolved to that prefix. Each affected block (keyed by - `(yaml_stem, block_index)`) maps to {prefix: {waive_id, ...}}. The - Selector aggregates this across rules and `write_filtered_test_db` - uses both the prefix (subtree match) AND the waive ids (to skip YAML - entries whose `-k ""` filter doesn't match the waived test). - Empty when the rule doesn't produce Layer 3 narrowing. + `block_filters` is per-block `{filter_prefix: {originating_waive_id, ...}}` + for Layer 3 narrowing. """ handled_files: set[str] @@ -58,15 +45,11 @@ class RuleResult: class Rule(ABC): - """Base class for all CBTS rules. - - A rule declares: - - `name`: identifier used in logs/reasons - - `needs_diff_for`: file paths / glob patterns whose diffs this rule consumes - (Groovy uses this to decide which files to fetch diffs for) + """Base class for CBTS rules. - Subclasses implement `apply(pr)` returning either None (not applicable) or - a RuleResult. + `name`: log/reason identifier. + `needs_diff_for`: file paths/globs whose diffs the rule consumes. + `apply(pr)`: return RuleResult or None (not applicable). """ name: str = "" diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index d83e8ad4a57c..ce9f14ed9761 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -25,13 +25,7 @@ def _extract_test_id(line: str) -> Optional[str]: - """Extract the normalized test identifier from a waives.txt line. - - Returns None if the line doesn't look like a waive entry (empty / pure - comment line). Trailing `SKIP`/`TIMEOUT` annotations, `# comment`s, and - leading `full:/` prefix are stripped via `normalize_test_id` so the - result matches the same key used by `YAMLIndex`. - """ + """Return the normalized test id from a waives.txt line, or None.""" s = line.strip() if not s or s.startswith("#"): return None @@ -40,11 +34,7 @@ def _extract_test_id(line: str) -> Optional[str]: def parse_waives_diff(diff: str) -> tuple[set[str], set[str]]: - """Parse a unified diff of waives.txt. - - Returns (added, removed) sets of normalized test identifiers ready to look - up against `YAMLIndex.blocks_containing_test`. - """ + """Return (added, removed) normalized test ids from a waives.txt diff.""" added: set[str] = set() removed: set[str] = set() for line in diff.splitlines(): @@ -66,8 +56,7 @@ class WaivesRule(Rule): def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]) -> None: self.yaml_index = yaml_index - # Group stages by YAML stem so block->stage lookup is O(stages_in_yaml) - # instead of O(total_stages) per block. + # Group stages by YAML stem for O(stages_in_yaml) per-block lookup. self._stages_by_yaml: dict[str, list[tuple[str, Stage]]] = {} for name, stage in stages.items(): self._stages_by_yaml.setdefault(stage.yaml_stem, []).append((name, stage)) @@ -87,13 +76,9 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: reason="waives.txt: no actionable test ids in diff", ) - # For each waive, walk the parent chain looking for the first level - # where a YAML entry actually applies (-k keyword check included). - # Any unmatchable waive triggers full fallback — better safe than to - # silently drop CI for a typo'd or out-of-tree waive id. - # Record (prefix -> {waive_ids that resolved to it}) per block so - # write_filtered_test_db can re-check `-k` keywords against the - # original waive ids when narrowing entries. + # Walk each waive's parent chain to the first matching YAML level; + # any miss → scope=None (fallback). Record per-block prefix → + # {originating waive ids} for write_filtered_test_db's `-k` re-check. block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} affected_blocks: list[Block] = [] seen_block_keys: set[tuple[str, int]] = set() From 7c61b7208730de5abac853f5ea1b7a51e28eeea5 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 20:04:34 +0800 Subject: [PATCH 61/65] =?UTF-8?q?[None][chore]=20TESTING=20CBTS=20scenario?= =?UTF-8?q?=20A=20=E2=80=94=20REVERT=20before=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two TESTING-only changes to exercise CBTS Layer 2 / Layer 3 in CI on this infra PR: 1. jenkins/scripts/cbts/main.py — comment out the unhandled-files defer in Selector.run so CBTS doesn't bail on this PR's jenkins/ and cbts/ infra changes. 2. waives.txt — change the placeholder bug id on unittest/llmapi/test_memory_profiling.py::test_profile_kvcache to 9999000 so the diff is parseable by parse_waives_diff. That waive maps to one cheap pre-merge stage (A100X-PyTorch-1). Expected behavior on /bot run: - Now we will run stages: A100X-PyTorch-1 + *-PackageSanityCheck-* - A100X-PyTorch-1 renders the narrowed test list (count=1) - Layer 2.5 collapses pytest-split to splits=1 Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 12 +++++++----- tests/integration/test_lists/waives.txt | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index bc9bcf54adb2..a365a92bfe8b 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -108,11 +108,13 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: handled: set[str] = set() for _, r in pairs: handled |= r.handled_files - unhandled = sorted(set(pr.changed_files) - handled) - if unhandled: - preview = unhandled[:5] - more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" - return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) + # TESTING: temporarily skip unhandled-files defer so CBTS can run on + # this infra PR. REVERT before merge. + # unhandled = sorted(set(pr.changed_files) - handled) + # if unhandled: + # preview = unhandled[:5] + # more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" + # return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) if not pairs: return SelectionResult(scope=None, reasons=["No rule contributed"]) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2772c080b8f0..35f5532afd98 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -431,6 +431,7 @@ unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_gptoss_style_nvfp4[lim unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingDSv3-swiglu-1024-1024-1] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_qwen_next-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_topk_4-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) +unittest/api_stability SKIP (https://nvbugs/9999001) unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py::test_build_ad[deepseek-ai/DeepSeek-V3-llm_extra_args10] SKIP (https://nvbugs/5888827) unittest/auto_deploy/singlegpu/transformations/library/test_mrope_delta_cache.py::test_qwen_registry_configs_explicitly_enable_mrope_delta_cache SKIP (https://nvbugs/6078421) unittest/disaggregated/test_agent_multi_backends.py::test_run_with_different_env[1] SKIP (https://nvbugs/5979673) From 3337f0bf454fc0ddedff064f551b0f2a62f5d58a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 22:36:45 +0800 Subject: [PATCH 62/65] [None][feat] CBTS: skip PackageSanityCheck when waive doesn't touch sanity yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For scope=waiveonly PRs, PackageSanityCheck (3-4 GPU pods, ~30 GPU·min) was running unconditionally even though waives.txt is test-infra and the wheel is byte-identical to main. The exception is when a waive targets a test that itself lives in l0_sanity_check.yml — sanity needs to run there to verify the SKIP behavior takes effect (10 of 18 sanity tests are cross-listed in l0_l40s/l0_gb203/l0_h100, so this case is the majority for sanity-test waives). Express the policy per-rule, not in Groovy: - RuleResult.sanity_relevant (default True = safe): rules opt out when their handled changes have nothing the wheel-sanity check would verify. - WaivesRule.apply sets sanity_relevant=any(b.yaml_stem == "l0_sanity_check" for b in affected_blocks). Most waives don't match sanity blocks → False → sanity skipped. - Selector.run aggregates sanity_required = any(r.sanity_relevant) across fired rules. - JSON gains sanity_required: bool. _cbtsParseSelectionResult parses with explicit null-check so `false` survives Groovy coercion. - Layer 2 in L0_Test.groovy gates the PackageSanityCheck force-keep on cbts.sanity_required; trigger mode is irrelevant. Trigger mode (--post-merge) is orthogonal — sanity stages have no "Post-Merge" in their name and aren't filtered by the trigger-mode logic. sanity_required is the sole gate. Behavior matrix (CBTS path; baseline unchanged): scope=waiveonly, waive on non-sanity test → sanity SKIPPED (saves ~30 GPU·min) scope=waiveonly, waive on sanity-only test → safety net fires (affected_stages={}), scope=None, baseline runs sanity scope=waiveonly, waive cross-listed → sanity_required=True, sanity runs with narrowed list scope=waiveonly, trigger-mode mismatch + sanity not required → true no-op scope=null → CBTS not consulted, baseline behavior Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 ++ jenkins/L0_Test.groovy | 18 ++++++++++++------ jenkins/scripts/cbts/README.md | 2 +- jenkins/scripts/cbts/main.py | 7 +++++++ jenkins/scripts/cbts/rules/base.py | 5 +++++ jenkins/scripts/cbts/rules/waives_rule.py | 1 + 6 files changed, 28 insertions(+), 7 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 0dc437785d79..c79fb67ade48 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -840,6 +840,8 @@ def _cbtsParseSelectionResult(String text) reasons: data.reasons ?: [], test_db_dir_override: data.test_db_dir_override, affected_stage_test_counts: data.affected_stage_test_counts ?: [:], + // Explicit null check preserves `false`; default True is safe. + sanity_required: data.sanity_required != null ? data.sanity_required : true, ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 4dce9e05ef27..52ab8c41eba3 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4479,19 +4479,25 @@ def launchTestJobs(pipeline, testFilter) } // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus - // PackageSanityCheck (force-kept); Perf is excluded. Empty affectedSet - // means trigger-mode mismatch → PackageSanityCheck only. + // PackageSanityCheck (kept iff sanity_required); Perf is excluded. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { def affectedSet = (cbts.affected_stages ?: []) as Set + def needsSanity = cbts.sanity_required parallelJobsFiltered = parallelJobs.findAll { key, _ -> - (affectedSet.contains(key) || key =~ /PackageSanityCheck/) && !(key =~ /Perf/) + (affectedSet.contains(key) || (needsSanity && key =~ /PackageSanityCheck/)) + && !(key =~ /Perf/) } if (affectedSet.isEmpty()) { - echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running only " + - "${parallelJobsFiltered.size()} PackageSanityCheck stage(s); no test cases" + if (parallelJobsFiltered.isEmpty()) { + echo "CBTS [${cbts.scope}]: trigger-mode mismatch + sanity not required → no-op" + } else { + echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running " + + "${parallelJobsFiltered.size()} sanity stage(s) only" + } } else if (parallelJobsFiltered) { - echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} affected stages" + echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} stages " + + "(sanity_required=${needsSanity})" } else { echo "CBTS [${cbts.scope}]: empty stage set after filtering" } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 6e21c9805a52..e887f248cc0e 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -11,7 +11,7 @@ CBTS narrows test cases only; Build always runs. | Layer | Where | Action | |---|---|---| -| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (force-kept). Perf stages are excluded. Empty affectedSet → PackageSanityCheck only (trigger-mode mismatch). | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (kept iff `sanity_required`). Perf stages are excluded. Empty affectedSet + `sanity_required=False` → no-op; empty + `sanity_required=True` → sanity-only. | | **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOn*` entries | Narrowed test count < 20 → collapse pytest-split to splits=1 (only group 1 runs); else default splits stand. | | **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the narrowed `cbts_test_db/`. Each affected block's `tests:` is restricted to entries in the filter prefix subtree; unaffected blocks are dropped. | diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index a365a92bfe8b..564afb3a570e 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -73,6 +73,9 @@ class SelectionResult: test_db_dir_override: Optional[str] = None # Per-stage narrowed test count, used by Layer 2.5 split-collapse. affected_stage_test_counts: dict[str, int] = field(default_factory=dict) + # Aggregated `any(rule.sanity_relevant)` across fired rules. Default + # True is safe; Groovy Layer 2 keeps PackageSanityCheck only when True. + sanity_required: bool = True def to_json(self) -> str: data = { @@ -81,6 +84,7 @@ def to_json(self) -> str: "reasons": list(self.reasons), "test_db_dir_override": self.test_db_dir_override, "affected_stage_test_counts": dict(self.affected_stage_test_counts), + "sanity_required": self.sanity_required, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -149,11 +153,14 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: for prefix, waives in prefix_to_waives.items(): dst.setdefault(prefix, set()).update(waives) + sanity_required = any(r.sanity_relevant for _, r in pairs) + return SelectionResult( scope=scope, affected_stages=affected_stages, reasons=reasons, block_filters=block_filters, + sanity_required=sanity_required, ) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index f760517d58e4..2e3650ba05dd 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -35,6 +35,10 @@ class RuleResult: `block_filters` is per-block `{filter_prefix: {originating_waive_id, ...}}` for Layer 3 narrowing. + + `sanity_relevant` (default True = safe): set False when this rule's + matched changes have nothing the wheel-sanity check would verify, so + PackageSanityCheck can be skipped. """ handled_files: set[str] @@ -42,6 +46,7 @@ class RuleResult: scope: Optional[str] reason: str block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) + sanity_relevant: bool = True class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index ce9f14ed9761..c657cb15625c 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -118,6 +118,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: affected_stages=affected_stage_names, scope="waiveonly", block_filters=block_filters, + sanity_relevant=any(b.yaml_stem == "l0_sanity_check" for b in affected_blocks), reason=( f"waives.txt: +{len(added)} / -{len(removed)} → " f"{len(affected_blocks)} blocks, {len(affected_stage_names)} stages" From 824fff1f0dbd75ca29192f802da25255a6a229cf Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 May 2026 22:54:57 +0800 Subject: [PATCH 63/65] [None][feat] CBTS: add perfsanity_required gate symmetric to sanity_required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the PerfSanity exclusion explicit per-rule rather than a blanket regex drop in Layer 2. Mirrors the sanity_required design. - RuleResult.perfsanity_relevant (default True = safe). - WaivesRule sets it False (waives.txt doesn't affect perf benchmarks). - Selector aggregates any() → SelectionResult.perfsanity_required. - JSON gains perfsanity_required: bool. _cbtsParseSelectionResult parses with explicit null-check. - Layer 2 regex changes from /Perf/ to /-Perf-/ so PerfSanity is no longer auto-dropped; force-keep PerfSanity when perfsanity_required. Behavior for waiveonly is unchanged: perfsanity_required=False, so PerfSanity stays excluded under both /bot run and /bot run --post-merge. The contract is now ready for v1+ rules (e.g., CoreCodeRule) that need perf coverage to opt in by leaving perfsanity_relevant at default True. Trigger-mode gating for force-kept *-PerfSanity-Post-Merge-* under plain /bot run is deferred to the first v1 rule that sets the flag True; not needed today since waiveonly always sets it False. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 1 + jenkins/L0_Test.groovy | 17 +++++++++++------ jenkins/scripts/cbts/README.md | 2 +- jenkins/scripts/cbts/main.py | 6 ++++++ jenkins/scripts/cbts/rules/base.py | 3 +++ jenkins/scripts/cbts/rules/waives_rule.py | 1 + 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c79fb67ade48..87fe0a2d0455 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -842,6 +842,7 @@ def _cbtsParseSelectionResult(String text) affected_stage_test_counts: data.affected_stage_test_counts ?: [:], // Explicit null check preserves `false`; default True is safe. sanity_required: data.sanity_required != null ? data.sanity_required : true, + perfsanity_required: data.perfsanity_required != null ? data.perfsanity_required : true, ] } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 52ab8c41eba3..75d04875ce2e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4479,25 +4479,30 @@ def launchTestJobs(pipeline, testFilter) } // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus - // PackageSanityCheck (kept iff sanity_required); Perf is excluded. + // PackageSanityCheck (kept iff sanity_required) and PerfSanity (kept iff + // perfsanity_required). Pure -Perf- stages always excluded (own trigger + // model, full-list benchmarks). def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { def affectedSet = (cbts.affected_stages ?: []) as Set def needsSanity = cbts.sanity_required + def needsPerfSanity = cbts.perfsanity_required parallelJobsFiltered = parallelJobs.findAll { key, _ -> - (affectedSet.contains(key) || (needsSanity && key =~ /PackageSanityCheck/)) - && !(key =~ /Perf/) + if (key =~ /-Perf-/) return false + return affectedSet.contains(key) || + (needsSanity && key =~ /PackageSanityCheck/) || + (needsPerfSanity && key =~ /PerfSanity/) } if (affectedSet.isEmpty()) { if (parallelJobsFiltered.isEmpty()) { - echo "CBTS [${cbts.scope}]: trigger-mode mismatch + sanity not required → no-op" + echo "CBTS [${cbts.scope}]: trigger-mode mismatch + nothing force-kept → no-op" } else { echo "CBTS [${cbts.scope}]: trigger-mode mismatch — running " + - "${parallelJobsFiltered.size()} sanity stage(s) only" + "${parallelJobsFiltered.size()} force-kept stage(s) only" } } else if (parallelJobsFiltered) { echo "CBTS [${cbts.scope}]: limiting to ${parallelJobsFiltered.size()} stages " + - "(sanity_required=${needsSanity})" + "(sanity_required=${needsSanity}, perfsanity_required=${needsPerfSanity})" } else { echo "CBTS [${cbts.scope}]: empty stage set after filtering" } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index e887f248cc0e..ec65558e53e7 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -11,7 +11,7 @@ CBTS narrows test cases only; Build always runs. | Layer | Where | Action | |---|---|---| -| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (kept iff `sanity_required`). Perf stages are excluded. Empty affectedSet + `sanity_required=False` → no-op; empty + `sanity_required=True` → sanity-only. | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (kept iff `sanity_required`) and PerfSanity (kept iff `perfsanity_required`). Pure `-Perf-` stages always excluded. Empty affectedSet + nothing force-kept → no-op. | | **2.5. Split-collapse** | `L0_Test.groovy::runLLMTestlistOn*` entries | Narrowed test count < 20 → collapse pytest-split to splits=1 (only group 1 runs); else default splits stand. | | **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the narrowed `cbts_test_db/`. Each affected block's `tests:` is restricted to entries in the filter prefix subtree; unaffected blocks are dropped. | diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 564afb3a570e..0dfb25b807f9 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -76,6 +76,9 @@ class SelectionResult: # Aggregated `any(rule.sanity_relevant)` across fired rules. Default # True is safe; Groovy Layer 2 keeps PackageSanityCheck only when True. sanity_required: bool = True + # Aggregated `any(rule.perfsanity_relevant)`. Groovy Layer 2 keeps + # *-PerfSanity-* stages only when True. + perfsanity_required: bool = True def to_json(self) -> str: data = { @@ -85,6 +88,7 @@ def to_json(self) -> str: "test_db_dir_override": self.test_db_dir_override, "affected_stage_test_counts": dict(self.affected_stage_test_counts), "sanity_required": self.sanity_required, + "perfsanity_required": self.perfsanity_required, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -154,6 +158,7 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: dst.setdefault(prefix, set()).update(waives) sanity_required = any(r.sanity_relevant for _, r in pairs) + perfsanity_required = any(r.perfsanity_relevant for _, r in pairs) return SelectionResult( scope=scope, @@ -161,6 +166,7 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: reasons=reasons, block_filters=block_filters, sanity_required=sanity_required, + perfsanity_required=perfsanity_required, ) diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index 2e3650ba05dd..018a05438584 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -47,6 +47,9 @@ class RuleResult: reason: str block_filters: dict[tuple[str, int], dict[str, set[str]]] = field(default_factory=dict) sanity_relevant: bool = True + # True (safe default) iff this rule's matched changes might affect perf + # benchmarks. Set False when matched changes are pure test infra. + perfsanity_relevant: bool = True class Rule(ABC): diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index c657cb15625c..d2582a6e4822 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -119,6 +119,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: scope="waiveonly", block_filters=block_filters, sanity_relevant=any(b.yaml_stem == "l0_sanity_check" for b in affected_blocks), + perfsanity_relevant=False, reason=( f"waives.txt: +{len(added)} / -{len(removed)} → " f"{len(affected_blocks)} blocks, {len(affected_stage_names)} stages" From 1fb7255f0c6a94e361aec627d3a6f43a3d567c7c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 6 May 2026 11:42:48 +0800 Subject: [PATCH 64/65] [None][chore] revert TESTING entries from CBTS scenario validation Restore the production state: - jenkins/scripts/cbts/main.py: re-enable the unhandled-files defer in Selector.run that the TESTING commit had commented out. - tests/integration/test_lists/waives.txt: drop the two TESTING-only waives marked with placeholder bug IDs #9999001 and #9999002. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/main.py | 12 +++++------- tests/integration/test_lists/waives.txt | 1 - 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 0dfb25b807f9..5747fccf9bd3 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -116,13 +116,11 @@ def run(self, pr: PRInputs, rules: list[Rule]) -> SelectionResult: handled: set[str] = set() for _, r in pairs: handled |= r.handled_files - # TESTING: temporarily skip unhandled-files defer so CBTS can run on - # this infra PR. REVERT before merge. - # unhandled = sorted(set(pr.changed_files) - handled) - # if unhandled: - # preview = unhandled[:5] - # more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" - # return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) + unhandled = sorted(set(pr.changed_files) - handled) + if unhandled: + preview = unhandled[:5] + more = f" (+{len(unhandled) - 5} more)" if len(unhandled) > 5 else "" + return SelectionResult(scope=None, reasons=[f"Unhandled files: {preview}{more}"]) if not pairs: return SelectionResult(scope=None, reasons=["No rule contributed"]) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 35f5532afd98..2772c080b8f0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -431,7 +431,6 @@ unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_gptoss_style_nvfp4[lim unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingDSv3-swiglu-1024-1024-1] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_qwen_next-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_topk_4-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) -unittest/api_stability SKIP (https://nvbugs/9999001) unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py::test_build_ad[deepseek-ai/DeepSeek-V3-llm_extra_args10] SKIP (https://nvbugs/5888827) unittest/auto_deploy/singlegpu/transformations/library/test_mrope_delta_cache.py::test_qwen_registry_configs_explicitly_enable_mrope_delta_cache SKIP (https://nvbugs/6078421) unittest/disaggregated/test_agent_multi_backends.py::test_run_with_different_env[1] SKIP (https://nvbugs/5979673) From 446d96602b880f90bcef6cde186cb2065e0ef62f Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 8 May 2026 15:13:24 +0800 Subject: [PATCH 65/65] [None][fix] CBTS: rethrow InterruptedException so aborts propagate The generic catch (Exception e) in getCbtsResult swallowed Jenkins abort signals, preventing the pipeline from terminating. Catch InterruptedException first and rethrow, matching the pattern used elsewhere in this file. Other failures still fall back to a full run. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 87fe0a2d0455..2fa9d540d328 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -785,6 +785,8 @@ def getCbtsResult(pipeline, testFilter, globalVars) pipeline.echo("CBTS: scope=${result.scope}, " + "stages=${result.affected_stages.size()}") return result + } catch (InterruptedException e) { + throw e } catch (Exception e) { pipeline.echo("CBTS failed, falling back to full run: ${e}") return null