From 0087efb362b494e4cdf1fc2c80a318b55b53d9d8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 22:42:24 +0000 Subject: [PATCH 1/4] [AMD][CI] Don't let a dead apt index fail the MORI dependency install The MORI reinstall added an `apt-get update` inside a `bash -c` running under `set -euo pipefail`. The ROCm base image carries an AMD-internal artifactory source (`rocm-osdb-22.04-deb`) that 404s when a ROCm build is rotated out, and `apt-get update` exits 100 for a single unreachable index even though it keeps every index it did fetch. That took out the "Install dependencies" step on ~25 of 27 jobs in one pr-test-amd run plus a nightly-test-amd job, all five AMD workflows sharing this script. The only consumer of that update is the `libgrpc++-dev` install on the next line, which is already best-effort -- rocm.Dockerfile builds MORI without the package -- so make both apt steps non-fatal and say in the log which one degraded. Retries are already configured image-wide via Acquire::Retries and do not help against a 404. Co-authored-by: quitenode --- scripts/ci/amd/amd_ci_install_dependency.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index b5ad5c231c81..b52d7459410e 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -261,6 +261,14 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then fi echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})" + # Neither apt step may be fatal. libgrpc++-dev is optional -- rocm.Dockerfile + # builds MORI without it -- and the ROCm base image carries an AMD-internal + # artifactory source (rocm-osdb) whose index 404s whenever a ROCm build is + # rotated out. apt-get update exits 100 on that while still keeping every + # index it did fetch, so under set -e one dead third-party source we do not + # even install from takes out the dependency install on every AMD runner at + # once. Retries are already configured image-wide (Acquire::Retries) and do + # not help against a 404. docker exec ci_sglang bash -c " set -euo pipefail export MORI_GPU_ARCHS='${MORI_GPU_ARCHS}' @@ -269,8 +277,8 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then cd /sgl-workspace/mori git checkout '${MORI_COMMIT}' git submodule update --init --recursive - apt-get update - apt-get install -y --no-install-recommends libgrpc++-dev 2>/dev/null || true + apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch' + apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it' python3 setup.py develop python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf ldconfig From 95faa08b0a7c5c90804ee67d18e0b733a3d121c6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 22:55:54 +0000 Subject: [PATCH 2/4] [AMD][CI] Add regression test for the MORI apt steps Guards the fix in the previous commit. Replays the R387 failure -- a stub apt-get that 404s one index and exits 100, like AMD's rocm-osdb artifactory did -- against the apt lines extracted from the install script itself, run in the same `set -euo pipefail` context the container gives them, and asserts the MORI build that follows is still reached. Covers the dead-index case, the case where the optional package is also unresolvable, and the healthy path (which must still install). A static check rejects reintroducing an unguarded apt step. Runs on base-a-test-cpu: no docker, no ROCm, no GPU. Against the pre-fix script three of the four cases fail; against the fix all four pass. Co-authored-by: quitenode --- .../tools/test_amd_ci_install_dependency.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 test/registered/unit/tools/test_amd_ci_install_dependency.py diff --git a/test/registered/unit/tools/test_amd_ci_install_dependency.py b/test/registered/unit/tools/test_amd_ci_install_dependency.py new file mode 100644 index 000000000000..256b546ccf94 --- /dev/null +++ b/test/registered/unit/tools/test_amd_ci_install_dependency.py @@ -0,0 +1,136 @@ +"""Regression tests for scripts/ci/amd/amd_ci_install_dependency.sh. + +The MORI reinstall runs its apt steps inside `docker exec ... bash -c` under +`set -euo pipefail`, where a bare `apt-get` is fatal. `apt-get update` exits +100 when any single index is unreachable, even though it keeps every index it +did fetch -- so one dead source in the ROCm base image fails the whole +"Install dependencies" step on every AMD runner at once. That is what took +out ~25 of 27 jobs in pr-test-amd run 32399046576 when AMD's internal +rocm-osdb artifactory started 404ing. + +The packages those steps install are optional (rocm.Dockerfile builds MORI +without them), so none of them may abort the run. +""" + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +REPO_ROOT = Path(__file__).resolve().parents[4] +INSTALL_SCRIPT = REPO_ROOT / "scripts" / "ci" / "amd" / "amd_ci_install_dependency.sh" + +BUILD_MARKER = "MORI_BUILD_REACHED" +INSTALL_MARKER = "APT_INSTALL_RAN" + +# Replays the observed failure: the rocm-osdb index 404s, every other index is +# fetched fine, and apt-get update still exits 100. APT_UPDATE_OK / +# APT_INSTALL_OK select which half of the world is healthy. +APT_STUB = """#!/bin/bash +case "$1" in + update) + echo "Get:1 https://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]" + echo "Fetched 48.0 MB in 3s (18.2 MB/s)" + [ "${APT_UPDATE_OK}" = 1 ] && exit 0 + echo "Err:13 http://compute-artifactory.amd.com/artifactory/list/rocm-osdb-22.04-deb compute-rocm-rel-7.0/38 amd64 Packages" >&2 + echo " 404 Not Found [IP: 10.216.51.87 80]" >&2 + echo "E: Some index files failed to download. They have been ignored, or old ones used instead." >&2 + exit 100 + ;; + install) + [ "${APT_INSTALL_OK}" = 1 ] || { echo "E: Unable to locate package" >&2; exit 100; } + echo "APT_INSTALL_RAN" + ;; +esac +""" + + +def _mori_shell_body(script: str) -> str: + """Body of the `docker exec ... bash -c "..."` that reinstalls MORI.""" + marker = 'docker exec ci_sglang bash -c "' + start = script.index(marker, script.index("[MORI] Reinstalling MORI")) + len(marker) + body = script[start:] + return body[: body.index('\n "')] + + +class TestAmdCiInstallDependencyApt(CustomTestCase): + @classmethod + def setUpClass(cls): + body = _mori_shell_body(INSTALL_SCRIPT.read_text()) + cls.apt_lines = [ + line for line in body.splitlines() if line.strip().startswith("apt-get") + ] + # Without this the tests below would pass vacuously if the block were + # ever restructured out from under them. + assert cls.apt_lines, "found no apt-get lines in the MORI reinstall block" + # The block is a double-quoted host string, so a `$` in these lines + # would be expanded before the container ever sees them; replaying them + # verbatim would then not reflect what really runs. + assert not any("$" in line for line in cls.apt_lines), cls.apt_lines + + def run_apt_lines(self, *, update_ok: str, install_ok: str): + """Run the real apt lines in the context the container gives them.""" + with tempfile.TemporaryDirectory() as stub_dir: + stub = Path(stub_dir) / "apt-get" + stub.write_text(APT_STUB) + stub.chmod(0o755) + return subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail\n" + + "\n".join(self.apt_lines) + + f"\necho {BUILD_MARKER}\n", + ], + env={ + **os.environ, + "PATH": stub_dir + os.pathsep + os.environ["PATH"], + "APT_UPDATE_OK": update_ok, + "APT_INSTALL_OK": install_ok, + }, + capture_output=True, + text=True, + ) + + def test_dead_apt_index_does_not_abort_the_dependency_install(self): + result = self.run_apt_lines(update_ok="0", install_ok="1") + self.assertIn( + BUILD_MARKER, + result.stdout, + f"a 404 on one apt index aborted the MORI build:\n{result.stderr}", + ) + self.assertEqual(result.returncode, 0) + + def test_unavailable_optional_package_does_not_abort_the_dependency_install(self): + result = self.run_apt_lines(update_ok="0", install_ok="0") + self.assertIn( + BUILD_MARKER, + result.stdout, + f"an unavailable optional package aborted the MORI build:\n{result.stderr}", + ) + self.assertEqual(result.returncode, 0) + + def test_healthy_apt_still_installs_the_optional_packages(self): + result = self.run_apt_lines(update_ok="1", install_ok="1") + self.assertIn(INSTALL_MARKER, result.stdout) + self.assertIn(BUILD_MARKER, result.stdout) + self.assertEqual(result.returncode, 0) + + def test_every_mori_apt_line_stays_guarded(self): + for line in self.apt_lines: + self.assertIn( + "||", + line, + "unguarded apt step under `set -e` will fail every AMD job " + f"whenever a single apt source is unreachable: {line.strip()}", + ) + + +if __name__ == "__main__": + unittest.main() From 503d615b223696a320832291fb627581a3e8da67 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 23:12:30 +0000 Subject: [PATCH 3/4] [AMD][CI] Slim the apt guard down to the invariant it protects The subprocess harness replayed the 404 through a stub apt-get to prove the build step is still reached. That was worth running while writing the fix, but as a permanent fixture it mostly asserts that `||` suppresses `set -e`, and it paid for that by string-parsing the `docker exec ... bash -c` block out of the script -- so unrelated edits to the container name, echo text, or closing-quote indentation would break it, and parameterizing the package list would trip its no-`$` precondition. What actually needs guarding is the invariant: no apt call in this script may be able to abort the run. Checking that directly drops 91 lines, removes the coupling to the block's shape, extends the guard to any apt call added elsewhere in the script, and reports the offending line instead of bash stderr. Still fails on the pre-fix script, naming `apt-get update`. Co-authored-by: quitenode --- .../tools/test_amd_ci_install_dependency.py | 141 ++++-------------- 1 file changed, 25 insertions(+), 116 deletions(-) diff --git a/test/registered/unit/tools/test_amd_ci_install_dependency.py b/test/registered/unit/tools/test_amd_ci_install_dependency.py index 256b546ccf94..5e599ef7a224 100644 --- a/test/registered/unit/tools/test_amd_ci_install_dependency.py +++ b/test/registered/unit/tools/test_amd_ci_install_dependency.py @@ -1,135 +1,44 @@ -"""Regression tests for scripts/ci/amd/amd_ci_install_dependency.sh. +"""Guard on the apt calls in scripts/ci/amd/amd_ci_install_dependency.sh. -The MORI reinstall runs its apt steps inside `docker exec ... bash -c` under -`set -euo pipefail`, where a bare `apt-get` is fatal. `apt-get update` exits -100 when any single index is unreachable, even though it keeps every index it -did fetch -- so one dead source in the ROCm base image fails the whole -"Install dependencies" step on every AMD runner at once. That is what took -out ~25 of 27 jobs in pr-test-amd run 32399046576 when AMD's internal -rocm-osdb artifactory started 404ing. +Those calls run under `set -euo pipefail`, and `apt-get update` exits 100 when +any single index is unreachable -- even though it keeps every index it did +fetch. An unguarded call therefore fails the whole "Install dependencies" step +on every AMD runner at once, which is what took out ~25 of 27 jobs in +pr-test-amd run 32399046576 when AMD's internal rocm-osdb artifactory started +404ing on an index this repo never installs from. -The packages those steps install are optional (rocm.Dockerfile builds MORI -without them), so none of them may abort the run. +The packages involved are optional -- rocm.Dockerfile builds MORI without them +-- so no apt call here may be able to abort the run. """ -import os -import subprocess -import tempfile +import re import unittest from pathlib import Path from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=5, suite="base-a-test-cpu") +register_cpu_ci(est_time=1, suite="base-a-test-cpu") -REPO_ROOT = Path(__file__).resolve().parents[4] -INSTALL_SCRIPT = REPO_ROOT / "scripts" / "ci" / "amd" / "amd_ci_install_dependency.sh" - -BUILD_MARKER = "MORI_BUILD_REACHED" -INSTALL_MARKER = "APT_INSTALL_RAN" - -# Replays the observed failure: the rocm-osdb index 404s, every other index is -# fetched fine, and apt-get update still exits 100. APT_UPDATE_OK / -# APT_INSTALL_OK select which half of the world is healthy. -APT_STUB = """#!/bin/bash -case "$1" in - update) - echo "Get:1 https://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]" - echo "Fetched 48.0 MB in 3s (18.2 MB/s)" - [ "${APT_UPDATE_OK}" = 1 ] && exit 0 - echo "Err:13 http://compute-artifactory.amd.com/artifactory/list/rocm-osdb-22.04-deb compute-rocm-rel-7.0/38 amd64 Packages" >&2 - echo " 404 Not Found [IP: 10.216.51.87 80]" >&2 - echo "E: Some index files failed to download. They have been ignored, or old ones used instead." >&2 - exit 100 - ;; - install) - [ "${APT_INSTALL_OK}" = 1 ] || { echo "E: Unable to locate package" >&2; exit 100; } - echo "APT_INSTALL_RAN" - ;; -esac -""" - - -def _mori_shell_body(script: str) -> str: - """Body of the `docker exec ... bash -c "..."` that reinstalls MORI.""" - marker = 'docker exec ci_sglang bash -c "' - start = script.index(marker, script.index("[MORI] Reinstalling MORI")) + len(marker) - body = script[start:] - return body[: body.index('\n "')] +INSTALL_SCRIPT = ( + Path(__file__).resolve().parents[4] / "scripts/ci/amd/amd_ci_install_dependency.sh" +) class TestAmdCiInstallDependencyApt(CustomTestCase): - @classmethod - def setUpClass(cls): - body = _mori_shell_body(INSTALL_SCRIPT.read_text()) - cls.apt_lines = [ - line for line in body.splitlines() if line.strip().startswith("apt-get") + def test_apt_calls_cannot_abort_the_dependency_install(self): + unguarded = [ + line.strip() + for line in INSTALL_SCRIPT.read_text().splitlines() + if re.match(r"\s*(sudo\s+)?apt-get\b", line) and "||" not in line ] - # Without this the tests below would pass vacuously if the block were - # ever restructured out from under them. - assert cls.apt_lines, "found no apt-get lines in the MORI reinstall block" - # The block is a double-quoted host string, so a `$` in these lines - # would be expanded before the container ever sees them; replaying them - # verbatim would then not reflect what really runs. - assert not any("$" in line for line in cls.apt_lines), cls.apt_lines - - def run_apt_lines(self, *, update_ok: str, install_ok: str): - """Run the real apt lines in the context the container gives them.""" - with tempfile.TemporaryDirectory() as stub_dir: - stub = Path(stub_dir) / "apt-get" - stub.write_text(APT_STUB) - stub.chmod(0o755) - return subprocess.run( - [ - "bash", - "-c", - "set -euo pipefail\n" - + "\n".join(self.apt_lines) - + f"\necho {BUILD_MARKER}\n", - ], - env={ - **os.environ, - "PATH": stub_dir + os.pathsep + os.environ["PATH"], - "APT_UPDATE_OK": update_ok, - "APT_INSTALL_OK": install_ok, - }, - capture_output=True, - text=True, - ) - - def test_dead_apt_index_does_not_abort_the_dependency_install(self): - result = self.run_apt_lines(update_ok="0", install_ok="1") - self.assertIn( - BUILD_MARKER, - result.stdout, - f"a 404 on one apt index aborted the MORI build:\n{result.stderr}", - ) - self.assertEqual(result.returncode, 0) - - def test_unavailable_optional_package_does_not_abort_the_dependency_install(self): - result = self.run_apt_lines(update_ok="0", install_ok="0") - self.assertIn( - BUILD_MARKER, - result.stdout, - f"an unavailable optional package aborted the MORI build:\n{result.stderr}", + self.assertEqual( + unguarded, + [], + "an unguarded apt-get under `set -e` fails the dependency install on " + "every AMD runner whenever one apt source is unreachable; give it an " + "`|| echo ...` fallback", ) - self.assertEqual(result.returncode, 0) - - def test_healthy_apt_still_installs_the_optional_packages(self): - result = self.run_apt_lines(update_ok="1", install_ok="1") - self.assertIn(INSTALL_MARKER, result.stdout) - self.assertIn(BUILD_MARKER, result.stdout) - self.assertEqual(result.returncode, 0) - - def test_every_mori_apt_line_stays_guarded(self): - for line in self.apt_lines: - self.assertIn( - "||", - line, - "unguarded apt step under `set -e` will fail every AMD job " - f"whenever a single apt source is unreachable: {line.strip()}", - ) if __name__ == "__main__": From eba588d6a415bd500b3dd6919ccf8edcda16be11 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 21 Aug 2026 03:15:42 +0000 Subject: [PATCH 4/4] [AMD][CI] Run the MORI apt step only on rocm724 Only the rocm724 (noble) base actually lacks libgrpc++-dev: its job log shows the package "previously unselected", downloaded from noble/universe and unpacked. ROCm 7.0 and 7.2.0 built MORI without it for months before the step was added, so on those flavors the apt round trip is pure risk -- and 7.0 is the one image whose apt list carries the dead AMD-internal rocm-osdb source that zeroed the workflow. Gating restores pre-existing behaviour on 7.0/7.2.0 rather than only tolerating the failure there. The `||` guards stay: six external apt hosts are reachable from that step, so rocm-osdb is one of several ways it can die, and 7.2.4 still runs apt for real. Verified by expanding the gate for each flavor against a stub apt-get that exits 100: 7.0 and 7.2.0 skip apt entirely, 7.2.4 runs it and still reaches the MORI build. Co-authored-by: quitenode --- scripts/ci/amd/amd_ci_install_dependency.sh | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index 3a799fa244aa..93e6d4e41cd0 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -265,14 +265,15 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then fi echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})" - # Neither apt step may be fatal. libgrpc++-dev is optional -- rocm.Dockerfile - # builds MORI without it -- and the ROCm base image carries an AMD-internal - # artifactory source (rocm-osdb) whose index 404s whenever a ROCm build is - # rotated out. apt-get update exits 100 on that while still keeping every - # index it did fetch, so under set -e one dead third-party source we do not - # even install from takes out the dependency install on every AMD runner at - # once. Retries are already configured image-wide (Acquire::Retries) and do - # not help against a 404. + # Only the rocm724 (noble) base is missing libgrpc++-dev; 7.0 and 7.2.0 built + # MORI without it for months before this step existed, so skip the apt round + # trip there. Where it does run, neither step may be fatal: apt-get update + # exits 100 for a single unreachable index while still keeping every index it + # did fetch, which under set -e is enough to take out the dependency install + # on every AMD runner at once. Six external apt hosts are in play, so the + # guard is not specific to the rocm-osdb source that first triggered this. + # Retries are already configured image-wide (Acquire::Retries) and do not + # help against a 404. docker exec ci_sglang bash -c " set -euo pipefail export MORI_GPU_ARCHS='${MORI_GPU_ARCHS}' @@ -281,8 +282,10 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then cd /sgl-workspace/mori git checkout '${MORI_COMMIT}' git submodule update --init --recursive - apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch' - apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it' + if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ]; then + apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch' + apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it' + fi python3 setup.py develop python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf ldconfig