diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index e9d45f49daa9..dda93fbfc076 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1,6 +1,7 @@ @Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException +import java.nio.charset.StandardCharsets import groovy.transform.Field import groovy.json.JsonOutput import groovy.json.JsonSlurper @@ -770,15 +771,23 @@ def getCbtsResult(pipeline, testFilter, globalVars) return null } // Piggyback input JSON on testFilter so each L0_Test stage agent can - // re-run main.py and regenerate cbts_test_db/ locally. Capped at - // 256 KB; oversize → drop piggyback, Layer 3 falls back to source. + // re-run main.py and regenerate cbts_test_db/ locally. The payload is + // base64-encoded because the raw JSON contains PR diffs and may include + // ${...} or {...} sequences that the Jenkins tokenmacro plugin would + // try to evaluate when the parent serializes globalVars for the + // Parameterized-Remote-Trigger plugin, raising MacroEvaluationException + // and blocking test dispatch. Capped at 256 KB (post-encoding, since + // that is what travels on the wire); oversize → drop piggyback, + // Layer 3 falls back to source. final int CBTS_INPUT_PIGGYBACK_MAX_BYTES = 256000 - def inputJsonSize = inputJson.length() - if (inputJsonSize <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { - result.cbts_input_json = inputJson - pipeline.echo("CBTS Layer 3: cbts_input_json piggyback enabled (${inputJsonSize} bytes)") + def inputJsonB64 = inputJson.getBytes(StandardCharsets.UTF_8).encodeBase64().toString() + def inputJsonB64Size = inputJsonB64.length() + if (inputJsonB64Size <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { + result.cbts_input_json_b64 = inputJsonB64 + pipeline.echo("CBTS Layer 3: cbts_input_json_b64 piggyback enabled " + + "(${inputJsonB64Size} bytes encoded, ${inputJson.length()} bytes raw)") } else { - pipeline.echo("CBTS Layer 3: cbts_input_json is ${inputJsonSize} bytes, " + + pipeline.echo("CBTS Layer 3: cbts_input_json_b64 is ${inputJsonB64Size} bytes, " + "exceeds ${CBTS_INPUT_PIGGYBACK_MAX_BYTES}-byte piggyback limit; " + "downstream stages will fall back to source test-db " + "(Layer 2 stage filtering still applies)") diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0751daffcb99..4c8297debbf6 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1,6 +1,7 @@ @Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException +import java.nio.charset.StandardCharsets import groovy.transform.Field import groovy.json.JsonOutput import com.nvidia.bloom.KubernetesManager @@ -2456,16 +2457,29 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" // CBTS Layer 3: regenerate cbts_test_db/ on this stage agent from the - // piggybacked input JSON if not already present. + // piggybacked input JSON if not already present. The piggyback payload is + // base64-encoded on the orchestrator (see getCbtsResult in + // L0_MergeRequest.groovy) to keep tokenmacro from interpreting ${...} or + // {...} fragments inside the PR diff when globalVars is serialized. If + // decoding or regeneration throws (truncated/malformed payload), we + // swallow the error: the override directory will be absent below, the + // overrideYaml check will fail, and renderTestDB falls back to the + // source test-db. def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json) { + if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json_b64) { def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() if (dirExists != "yes") { - def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") - pipeline.writeFile(file: cbtsInputLocal, text: cbts.cbts_input_json) - sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" - sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py ${cbtsInputLocal} > /dev/null 2>&1 || true" + try { + def cbtsInputJson = new String(cbts.cbts_input_json_b64.decodeBase64(), StandardCharsets.UTF_8) + def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") + pipeline.writeFile(file: cbtsInputLocal, text: cbtsInputJson) + sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" + sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py ${cbtsInputLocal} > /dev/null 2>&1 || true" + } catch (Exception e) { + echo "CBTS Layer 3: failed to materialize piggyback payload " + + "(${e.class.simpleName}: ${e.message}); falling back to source test-db" + } } } def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index aa445265f9d1..9dbb1cdc5f36 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -197,13 +197,19 @@ Decision JSON: `cbts_test_db/` is written on the L0_MergeRequest agent and is not available to downstream `L0_Test-*` pods. To regenerate it per stage: -1. `getCbtsResult` stores the input JSON in `result.cbts_input_json`, - which rides along inside `testFilter`. -2. `renderTestDB` on the stage agent writes it to a temp file and re-runs - `main.py`. Output is deterministic, so each agent gets the same - `cbts_test_db/` as L0_MergeRequest produced. - -If `cbts_input_json` exceeds 256 KB the piggyback is dropped; Layer 3 falls +1. `getCbtsResult` base64-encodes the input JSON and stores it in + `result.cbts_input_json_b64`, which rides along inside `testFilter`. + Encoding is mandatory: the raw payload contains PR diff text which can + include `${...}` or `{...}` fragments (Python f-strings, shell vars, etc.) + that the Jenkins tokenmacro plugin tries to evaluate when the parent + serializes `globalVars` for `Parameterized-Remote-Trigger`, raising + `MacroEvaluationException` and blocking test dispatch. +2. `renderTestDB` on the stage agent decodes `cbts_input_json_b64`, writes + it to a temp file, and re-runs `main.py`. Output is deterministic, so + each agent gets the same `cbts_test_db/` as L0_MergeRequest produced. + +If `cbts_input_json_b64` exceeds 256 KB (post-encoding, the size that +actually travels over the wire) the piggyback is dropped; Layer 3 falls back to the source test-db on each stage agent. Layer 2 still applies. ## Split-collapse heuristic (Layer 2.5) @@ -279,7 +285,7 @@ CBTS defers to the existing filter chain when: YAML edit) - Combined scope is `None` (incompatible mix) - Layer 3 narrowing would empty a block — block keeps original tests -- `cbts_input_json` exceeds 256 KB — Layer 3 falls back per stage +- `cbts_input_json_b64` (post-encoding) exceeds 256 KB — Layer 3 falls back per stage - Narrowed YAML missing/empty on a stage agent — renderTestDB falls back Every fallback emits an `echo` log line.