diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e13d976db38f..c942fff785cb 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4035,6 +4035,24 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu } } + // Log the resolved mako match on every render, tagged with stage+context. + // transformMakoArgsToJson already echoes the bare JSON ("Test DB Mako opts:"), + // but unlabeled and far upstream of the render; the preDefinedMakoOpts path + // skips it entirely. This co-locates the match with the stage/context and the + // "-> N tests" summary below under one greppable renderTestDB: prefix, so a + // wrong-but-non-empty render (a stale/unexpected sysinfo value selecting the + // wrong block) is diagnosable per stage, not just the "na"/empty cases. + echo "renderTestDB: stage=${stageName} context=${testContext} mako match: ${makoOpts}" + + if (makoOpts.contains('"na"')) { + // "na" is a sysinfo probe failure sentinel (see get_sysinfo.py). Blocks + // conditioned on the failed property silently drop out of the render, so + // even a non-empty list may be missing tests. Warn here, where every + // sysinfo-based stage passes through, not just when the list ends up empty. + echo "WARNING: renderTestDB: some sysinfo probes returned \"na\": ${makoOpts}. " + + "Test-db blocks conditioned on those properties (e.g. linux_distribution_name: ubuntu*) " + + "will NOT be selected." + } sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" // CBTS Layer 3: download the pre-built cbts_test_db/ tarball that the // orchestrator uploaded to Artifactory (see getCbtsResult in @@ -4081,10 +4099,30 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu ].join(" ") sh(label: "Render test list from test-db", script: testDBQueryCmd) - def testCount = sh(returnStdout: true, script: "wc -l < ${testList} | tr -d ' '").trim() + // Count non-empty lines, not newlines: trt-test-db writes the test names + // with no trailing newline, so `wc -l` undercounts by one -- it reports 0 + // for a single-test render, which would trip the empty-list guard below + // (and mis-report every stage's count by one). `grep -c .` is agnostic to + // the missing terminator. It exits 1 for no matches (a legitimately empty + // render -> count "0"); accept only that, so a read failure (exit 2: + // missing file, unreadable, etc.) still aborts the step instead of being + // silently masked. + def testCount = sh(returnStdout: true, script: "grep -c . -- ${testList} || test \$? -eq 1").trim() def testDBLabel = (cbts != null && cbts.test_db_dir_override) ? "CBTS-narrowed [${cbts.scope}]" : "source" echo "renderTestDB: stage=${stageName} context=${testContext} test-db=${testDBLabel} dir=${testDBPath} -> ${testCount} tests" sh(script: "cat ${testList}") + if (testCount == "0") { + // An empty render is never legitimate here: every launched stage must + // have tests (CBTS drops stages with an empty selection before launch). + // Fail now with the match query rather than letting pytest --collect-only + // exit 5 later with an unattributable "Test collection failed" message. + def hint = makoOpts.contains('"na"') ? + " Some sysinfo probes returned \"na\" (see the match JSON above); a broken probe" + + " (e.g. the python 'distro' module missing) makes conditions like" + + " linux_distribution_name: ubuntu* match nothing." : "" + error("renderTestDB: rendered EMPTY test list for stage=${stageName} " + + "context=${testContext} test-db=${testDBLabel}. Match query: ${makoOpts}.${hint}") + } recordRenderedStageAttemptEstimate(pipeline, llmSrc, testList, stageName, testCount, clusterName) return testList diff --git a/requirements-dev.txt b/requirements-dev.txt index c44c56324057..6a6706a414d8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,9 @@ # before pytest collection so already-imported Diffusers modules match the # package files. 0.39.0 matches the runtime floor in requirements.txt. diffusers==0.39.0 +# get_sysinfo.py's Linux-distribution probe. Must be explicit: it used to arrive +# transitively via openai, and openai 3.3.1 dropped it, which silently emptied +# every sysinfo-rendered CI test list (linux_distribution_name became 'na'). distro==1.9.0 boto3 einops diff --git a/tests/integration/defs/sysinfo/get_sysinfo.py b/tests/integration/defs/sysinfo/get_sysinfo.py index 02e1a68eaef5..3643433c3e9f 100644 --- a/tests/integration/defs/sysinfo/get_sysinfo.py +++ b/tests/integration/defs/sysinfo/get_sysinfo.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -112,10 +112,28 @@ def get_linux_distribution(): try: import distro return (distro.id(), distro.version(), distro.codename()) - except: - logger.warning( - "Unable to use distro module, defaulting operating system to ('na', 'na', 'na')" - ) + except ImportError: + # distro is not a direct test dependency historically; it used to arrive + # transitively (e.g. via openai<3.3.1) and silently disappeared when that + # dependency was dropped. Fall back to the stdlib rather than to 'na'. + distro_reason = "distro module not installed" + except Exception as e: + # Best-effort probe: distro imported but id()/version()/codename() raised. + # Any failure here should degrade to os-release, not crash the render, so + # the broad catch is deliberate; the reason is preserved for the log below. + distro_reason = f"distro probe raised {type(e).__name__}: {e}" + logger.warning(f"{distro_reason}; falling back to os-release") + try: + # Python 3.10+; reads /etc/os-release, same source distro uses. + os_release = platform.freedesktop_os_release() + return (os_release.get("ID", "na"), os_release.get("VERSION_ID", "na"), + os_release.get("VERSION_CODENAME", "na")) + except OSError: + logger.error( + f"Cannot determine the Linux distribution ({distro_reason}; " + "/etc/os-release also unreadable); reporting ('na', 'na', 'na'). " + "Test-db conditions matching linux_distribution_name (e.g. ubuntu*) " + "will select ZERO tests and the rendered test list will be empty.") return ("na", "na", "na")