diff --git a/.github/workflows/build-sglang-rdna4.yaml b/.github/workflows/build-sglang-rdna4.yaml deleted file mode 100644 index a6f6ed553..000000000 --- a/.github/workflows/build-sglang-rdna4.yaml +++ /dev/null @@ -1,93 +0,0 @@ ---- -# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json -name: Build sglang-rdna4 - -# GPU-free build on a hosted runner — rationale in docker/sglang-rdna4/README.md. -# Fires on merged build-input changes (Renovate FORK_REF / base-digest bumps) or manual -# dispatch. Docs are excluded: the build isn't bit-reproducible, so every rebuild pushes a -# new digest under the in-place tag and Renovate then rolls the serving pod — a README -# edit must not do that. -on: - workflow_dispatch: - push: - branches: ["main"] - paths: - - "docker/sglang-rdna4/**" - - "!docker/sglang-rdna4/*.md" - - ".github/workflows/build-sglang-rdna4.yaml" - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: true # hosted runner — aborting a superseded build costs nothing - -permissions: - contents: read - packages: write - -env: - IMAGE: ghcr.io/tanguille/sglang-rdna4 - TAG: v0.5.16-gfx1201 # stable tag, rebuilt in place; Renovate digest-pins it on the HelmRelease - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # / does get used after all: run 30277992056 died at 43min with "No space left on device" - # writing the runner's own diag log under /home/runner. Reclaiming the preinstalled - # toolchains is ~30GB and costs seconds; the df lines are the evidence for next time. - - name: Reclaim / space - run: | - df -h / - sudo rm -rf /opt/hostedtoolcache /usr/share/dotnet /usr/local/lib/android /opt/ghc - df -h / - - # The `builder` stage (ROCm "complete" ~20GB + conda env ~22GB) outgrows /'s free space, so - # point docker's data root at the larger ~75GB /mnt volume. - - name: Move docker storage to /mnt - run: | - sudo systemctl stop docker.service docker.socket - sudo mkdir -p /mnt/docker - # Merge, don't clobber: preserve any other daemon settings the runner image ships. - echo "$(sudo jq '. + {"data-root":"/mnt/docker"}' /etc/docker/daemon.json 2>/dev/null || echo '{"data-root":"/mnt/docker"}')" \ - | sudo tee /etc/docker/daemon.json - sudo systemctl start docker.service - docker info --format 'docker data root: {{.DockerRootDir}}' - df -h /mnt - - - name: Set up Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Image metadata (OCI labels) - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ env.IMAGE }} - tags: type=raw,value=${{ env.TAG }} - - # provenance/sbom left off: this image is pulled only by our own cluster, pinned by digest — - # attestations would turn the tag into an OCI index and complicate that pin for no gain here. - - name: Build and push - id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: docker/sglang-rdna4 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - provenance: false - sbom: false - - - name: Summary - run: echo "Pushed \`${IMAGE}:${TAG}@${{ steps.build.outputs.digest }}\`" >> "${GITHUB_STEP_SUMMARY}" diff --git a/.renovaterc.json5 b/.renovaterc.json5 index 18f9730aa..b1ae52a2f 100644 --- a/.renovaterc.json5 +++ b/.renovaterc.json5 @@ -38,16 +38,6 @@ currentValueTemplate: "main", datasourceTemplate: "git-refs" }, - { - description: "sglang-rdna4 FORK_REF pinned to the fork's main-branch head (the preset's annotated manager can't do git-refs digest pins); a bump PR merge fires the image rebuild workflow", - customType: "regex", - managerFilePatterns: ["/docker/sglang-rdna4/Dockerfile$/"], - matchStrings: ["ARG FORK_REF=(?[a-f0-9]{40})"], - depNameTemplate: "mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference", - packageNameTemplate: "https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference", - currentValueTemplate: "main", - datasourceTemplate: "git-refs" - } ], packageRules: [ // Auto-merge rules @@ -61,13 +51,7 @@ }, // Grouping rules { - description: "sglang-rdna4 build inputs (fork ref + ROCm base digest + miniforge) — one review PR, no automerge; merging fires the GPU-free build workflow and mints a new image digest", - groupName: "sglang-rdna4", - matchFileNames: ["docker/sglang-rdna4/Dockerfile"], - automerge: false - }, - { - description: "sglang-rdna4 deployed image digest — review before the HelmRelease rolls the serving pod (Recreate on the single GPU)", + description: "sglang-rdna4 deployed image digest — built by our fork of the upstream RDNA4 image, not a pipeline in this repo; review before the HelmRelease rolls the serving pod (Recreate on the single GPU)", matchDatasources: ["docker"], matchPackageNames: ["ghcr.io/tanguille/sglang-rdna4"], automerge: false diff --git a/docker/sglang-rdna4/Dockerfile b/docker/sglang-rdna4/Dockerfile deleted file mode 100644 index edc31b77e..000000000 --- a/docker/sglang-rdna4/Dockerfile +++ /dev/null @@ -1,179 +0,0 @@ -# SGLang for AMD R9700 (RDNA4 / gfx1201), serving Qwen3.6-27B int4-AWQ. -# Vendors github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference, runs its own -# scripts/setup.sh (the only build path the fork supports), then applies the TP=1 fixes -# setup.sh omits (the fork targets a dual-card TP=2 box). Re-pinning FORK_REF — and -# SGLANG_TAG on a version bump — is the whole maintenance surface. This image is the -# git-reproducible replacement for the old runtime-from-PVC env (see ../../kubernetes/ -# apps/ai/sglang/README.md). -# -# Builds WITHOUT a GPU — the HIP kernels cross-compile; see the setup.sh step below and README.md. -# -# Two-stage build: the `builder` compiles the kernels + conda env with the full ROCm "complete" -# SDK; the runtime stage ships from the slim non-complete base and copies only the built env -# (~18 GB smaller). See README.md "Two-stage build" for the why (torch bundles its own ROCm -# runtime, so the dev SDK is build-only) and the Triton-JIT cold-boot validation it requires. - -# ===== Stage 1: builder — full ROCm SDK, compiles the kernels + conda env ===== -# Base pinned by digest — community tags get rebuilt in place (memo project_ik_llama_image_tags). -FROM docker.io/rocm/dev-ubuntu-24.04:7.2.4-complete@sha256:92f309c51b52cef8762867848f1529dee821624f23cd5df38455e819538f762f AS builder - -# gfx1201 is native on ROCm 7.x — do NOT set HSA_OVERRIDE_GFX_VERSION. -# ENV_NAME / REPO_DIR / SGLANG_DIR pin the paths the fork's common.sh now defaults to the -# author's ephemeral /data/* (ENV_NAME=sglang-triton36-v0514). launch.sh sources common.sh -# and `conda activate $ENV_NAME` at runtime, so the env name baked here must match. -ENV DEBIAN_FRONTEND=noninteractive \ - PIP_NO_CACHE_DIR=1 \ - ROCM_PATH=/opt/rocm \ - PYTORCH_ROCM_ARCH=gfx1201 \ - CONDA_BASE=/opt/conda \ - ENV_NAME=sglang-triton36-v0514 \ - REPO_DIR=/opt/rdna4-inference \ - SGLANG_DIR=/opt/rdna4-inference/components/sglang - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - git curl ca-certificates build-essential python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Rust via rustup stable, not apt (Ubuntu 24.04's cargo is too old for the fork's grpc crate build). -ENV RUSTUP_HOME=/opt/rust/rustup \ - CARGO_HOME=/opt/rust/cargo \ - PATH=/opt/rust/cargo/bin:${PATH} -RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable \ - && rustc --version && cargo --version - -# Miniforge: the fork's scripts are conda-coupled (conda activate $ENV_NAME). -# renovate: datasource=github-releases depName=conda-forge/miniforge -ARG MINIFORGE_VERSION=26.3.2-3 -RUN curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" -o /tmp/miniforge.sh \ - && bash /tmp/miniforge.sh -b -p "${CONDA_BASE}" \ - && rm /tmp/miniforge.sh - -# Vendor the fork at a pinned commit (no runtime fetch). -ARG FORK_REPO=https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference.git -ARG FORK_REF=c548341f00ff7c6b6851688e4030939844d770f7 -RUN git clone "${FORK_REPO}" "${REPO_DIR}" \ - && git -C "${REPO_DIR}" checkout "${FORK_REF}" - -# No local patch overrides: the fork's v0.5.16 rebase covers both 073 and 003, including the -# `infllm_v2` guard our 003 override existed for. - -# setup.sh: clone stock SGLang @ SGLANG_TAG -> RDNA4 patch series -> conda env $ENV_NAME -> -# torch 2.11.0+rocm7.2 stable (setup.sh's default; the old 2.12 nightly became unfetchable and -# faulted with expandable_segments) -> triton 3.6 -> the native gfx1201 HIP kernels. -# SGLANG_TAG must match the tree FORK_REF's series was rebased onto; mismatch fails 14 patches. -# SGLANG_COMMIT makes setup.sh verify the tag still resolves to that commit, so a retag cannot -# swap the tree underneath the series. -ARG SGLANG_TAG=v0.5.16 -ARG SGLANG_COMMIT=fdebc938f7f4d16fe6b9f55dcd9a767cf0899ea1 -WORKDIR ${REPO_DIR} -# GPU-free build (explicit PYTORCH_ROCM_ARCH, no device probe) — neuters setup.sh's one fatal -# GPU touch, `assert torch.cuda.is_available()` (~line 139); grep guards catch upstream moving -# it. Full rationale: README.md "Building — GPU-free". -RUN sed -i "s|assert torch.cuda.is_available(), 'CUDA not available'; ||" scripts/setup.sh \ - && ! grep -q "assert torch.cuda.is_available" scripts/setup.sh \ - && grep -qF "device_count()} GPUs" scripts/setup.sh -# conda clean in the SAME layer as the multi-GB download/solve so the pkg cache never lands in a -# committed layer — keeps the builder small on the runner's build disk (the multi-stage COPY needs -# builder + runtime bytes resident at once). -RUN SGLANG_TAG="${SGLANG_TAG}" SGLANG_COMMIT="${SGLANG_COMMIT}" ./scripts/setup.sh \ - && "${CONDA_BASE}/bin/conda" clean -afy - -# --- TP=1 fixes the fork's setup.sh omits (it targets dual-card TP=2). Mirrors --- -# --- scripts/sglang-env-rebuild.sh. Each grep -q fails the build if the sed missed --- -# --- (e.g. an upstream rename at a new SGLANG_TAG) instead of shipping a broken image. --- -# 1. store_cache: the JIT store_cache kernel aborts at TP=1 (kvcache.cuh:204) -> force the naive torch KV store. -RUN sed -i '/^def can_use_store_cache(size: int) -> bool:$/a\ return False # RDNA4 TP=1: JIT store_cache crashes (kvcache.cuh:204) -> naive torch KV store' \ - "${SGLANG_DIR}/python/sglang/jit_kernel/kvcache.py" \ - && grep -q 'RDNA4 TP=1' "${SGLANG_DIR}/python/sglang/jit_kernel/kvcache.py" -# 2. token-id all-reduce: gate the cross-TP sync on world>1 so NCCL never lazily inits at TP=1 -# (its first ~256MB calloc OOMs hours in once VRAM is committed -> grammar/tool-calling restarts). -RUN sed -i 's|^ if SYNC_TOKEN_IDS_ACROSS_TP or sampling_info.grammars:$| if (SYNC_TOKEN_IDS_ACROSS_TP or sampling_info.grammars) and dist.get_world_size(group=self.tp_sync_group) > 1:|' \ - "${SGLANG_DIR}/python/sglang/srt/layers/sampler.py" \ - && grep -q 'get_world_size(group=self.tp_sync_group) > 1' "${SGLANG_DIR}/python/sglang/srt/layers/sampler.py" -# 3. transformers-5.x ships a `kernels` pkg whose hub loader crashes `import sglang`; SGLang uses its own compiled kernels. -RUN "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" pip uninstall kernels -y 2>/dev/null || true -# 4. launch.sh's --disable-cuda-graph is deprecated (ServerArgs now wants the split -# --cuda-graph-backend-{decode,prefill}=disabled flags) -> silences the startup warning, -# same disabled-graph behavior. -RUN sed -i 's/--disable-cuda-graph/--cuda-graph-backend-decode=disabled --cuda-graph-backend-prefill=disabled/g' \ - "${REPO_DIR}/scripts/launch.sh" \ - && ! grep -q -- '--disable-cuda-graph' "${REPO_DIR}/scripts/launch.sh" - -# amdsmi: ROCm GPU-management bindings. Without it SGLang logs "Failed to import amdsmi" and falls back -# to torch for VRAM-capacity detection (works, but coarser mem-fraction sizing on this VRAM-bound card). -# Prefer the ROCm-bundled bindings matching the 7.2.4 runtime; fall back to PyPI. Non-fatal. Mirrors env-rebuild.sh. -RUN "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" pip install /opt/rocm/share/amd_smi 2>/dev/null \ - || "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" pip install amdsmi 2>/dev/null \ - || echo "(amdsmi install failed — non-fatal; SGLang falls back to torch VRAM detection)" - -# Hard gate: fail the build unless sglang actually imports from the patched env — a silently-missed -# patch or a swallowed setup.sh failure dies HERE. -RUN "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" python -c "import sglang; print('sglang', sglang.__version__)" - -# ===== Stage 2: runtime — slim non-complete ROCm base, only the built env copied in ===== -# The non-complete base is the coherent ROCm runtime without the ~18 GB dev SDK (headers, static -# libs, docs, every math component). Pinned by digest like the builder base. -FROM docker.io/rocm/dev-ubuntu-24.04:7.2.4@sha256:bdc8e61026cbb844ede93d44d2c50055f51ebb2041906b60182bf3bee3139054 - -# LLMKube invokes `python3 -m sglang.launch_server` directly, bypassing this -# image's conda-activating ENTRYPOINT. Keep the patched env first in PATH. -ENV ROCM_PATH=/opt/rocm \ - CONDA_BASE=/opt/conda \ - ENV_NAME=sglang-triton36-v0514 \ - REPO_DIR=/opt/rdna4-inference \ - SGLANG_DIR=/opt/rdna4-inference/components/sglang \ - PATH=/opt/conda/envs/sglang-triton36-v0514/bin:/opt/conda/bin:${PATH} - -# HiCache JIT-compiles hash_binding.cpp (#include ) on the first prefill, so the -# headers are a RUNTIME need, not a build one. g++ and ninja already survive in the slim base; -# only the OpenSSL headers were missing. Without them --hicache-storage-backend passes startup -# and health checks, then kills the scheduler mid-request. -# upgrade picks up base-image CVE fixes between digest bumps; it costs reproducibility against the -# pinned base, so the import and hicache gates below are what catch a runtime lib moving underneath. -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -# The built conda env (torch+rocm, triton, sglang, native gfx1201 kernels) and the fork repo -# (launch.sh, common.sh, chat template, editable sglang). Paths match the builder so the env's -# baked RPATHs/shebangs (/opt/conda/...) resolve unchanged. Build-only tooling (rust, gcc, git, -# miniforge installer, the /opt/rocm dev SDK) is left behind in the builder stage. -COPY --from=builder /opt/conda /opt/conda -COPY --from=builder /opt/rdna4-inference /opt/rdna4-inference - -# Gate on the slim base: importing torch, sglang AND the compiled sgl_kernel dlopens their native -# libs, so this fails HERE if a runtime lib lived only in the dev SDK left behind in the builder. -# It does NOT exercise Triton's runtime JIT — that risk is validated at deploy (cold boot, Triton -# cache cleared). -RUN "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" python -c "import torch, sglang, sgl_kernel; print('torch', torch.__version__, 'hip', torch.version.hip); print('sglang', sglang.__version__)" - -# HiCache's hash extension is JIT-compiled on the first prefill, so a missing toolchain surfaces -# as a mid-request scheduler crash rather than a boot failure. Build it here instead. -RUN "${CONDA_BASE}/bin/conda" run -n "${ENV_NAME}" python -c "from sglang.srt.mem_cache.cpp_utils.native_hash import _load_native_hash_module; _load_native_hash_module(); print('hicache_hash_cpp OK')" - -# Correctness-critical gfx1201 env from the fork's common.sh::setup_rdna4_env. launch.sh also -# sets these at runtime (it sources common.sh) — baked here so the image is correct under any -# entrypoint. They select working code paths (AITER is CDNA-only and miscomputes on RDNA4), not tuning. -# Device-count / RCCL / TRITON_CACHE_DIR are single-vs-dual-GPU and PVC concerns -> set by the HelmRelease. -ENV SGLANG_USE_AITER=0 \ - SGLANG_USE_AITER_AR=0 \ - HIP_FORCE_DEV_KERNARG=1 \ - HSA_FORCE_FINE_GRAIN_PCIE=1 \ - GPU_MAX_HW_QUEUES=8 \ - PYTORCH_HIP_ALLOC_CONF=expandable_segments:True \ - VLLM_USE_TRITON_AWQ=1 \ - VLLM_USE_TRITON_FLASH_ATTN=1 \ - FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE \ - PYTORCH_TUNABLEOP_ENABLED=0 \ - TOKENIZERS_PARALLELISM=false \ - SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 \ - PYTHONUNBUFFERED=1 - -COPY --chmod=0755 entrypoint.sh /usr/local/bin/entrypoint.sh - -EXPOSE 8000 -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] -CMD ["python", "-m", "sglang.launch_server", "--help"] diff --git a/docker/sglang-rdna4/README.md b/docker/sglang-rdna4/README.md deleted file mode 100644 index 17d7d17eb..000000000 --- a/docker/sglang-rdna4/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# sglang-rdna4 — custom SGLang image for AMD R9700 (gfx1201) - -Serves **Qwen3.6-27B int4-AWQ** on a single **AMD Radeon AI PRO R9700** (RDNA4 / gfx1201) -via SGLang + the RDNA4 patch fork. This is the **git-reproducible replacement** for the -old runtime-from-PVC env, and is now the consolidated reference for building, pinning and -rolling back the image. The cutover plan it replaced was removed once complete; recover it -with `git log --diff-filter=D -- docs/llm-hosting/sglang-oci-cutover.md`. - -The image vendors [`mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference`](https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference) -at a pinned commit, runs its own `scripts/setup.sh` (the only build path the fork supports — -conda-only, no upstream Dockerfile), then applies the three TP=1 fixes `setup.sh` omits. -**Re-pinning `FORK_REF` (and `SGLANG_TAG` on a version bump) is the whole maintenance surface.** - -> The `Dockerfile` was derived from the retired `sglang-env-rebuild.sh` (same `FORK_REF`, -> `SGLANG_TAG`, and the three patches). That script went with the `kubernetes/apps/ai/sglang` -> directory in b8f12ae75; if the registry is ever unreachable and the PVC-rebuild path is -> needed, recover it with -> `git show b8f12ae75^:kubernetes/apps/ai/sglang/app/scripts/sglang-env-rebuild.sh`. -> There is no longer a second copy to keep in sync — this Dockerfile is the only build path. - -## Pins (verified against `env-rebuild.sh`, 2026-07-12) - -| Component | Pin | -|---|---| -| Base image | `rocm/dev-ubuntu-24.04:7.2.4-complete` @ `sha256:92f309c5…` | -| Fork ref | `9fa3df4080de7abba93a75e520f2359dd74db15c` | -| SGLang | `v0.5.16` @ `fdebc938…` (stock + the fork's own RDNA4 patch series + 3 TP=1 fixes, see below) | -| PyTorch | `2.11.0+rocm7.2` (setup.sh default; 2.12 nightly is unfetchable + faults `expandable_segments`) | -| Triton | `3.6.0` | -| conda env | `sglang-triton36-v0514` (must match the fork's `common.sh` default `launch.sh` activates) | - -### The 3 TP=1 fixes baked after `setup.sh` - -The fork targets a dual-card TP=2 box; we serve single-card TP=1. Without these the server -crashes on the first request or OOM-restarts hours in on the first grammar/tool-calling request: - -1. `jit_kernel/kvcache.py` `can_use_store_cache() -> False` — JIT store_cache aborts at TP=1 (`kvcache.cuh:204`). -2. `srt/layers/sampler.py` — gate the cross-TP token-id all-reduce on `world > 1` (NCCL lazy-init OOM). -3. `pip uninstall kernels` — transformers-5.x's hub-kernels loader crashes `import sglang`. - -### No local patch overrides remain (as of fork ref `9fa3df4`, v0.5.16) - -Through fork ref `a0445f59e` we carried local overrides for patches 064/073/003. The v0.5.15 -rebase (`6dfa7542f`) resolved 064 and 073; 003 outlived them because the fork's rebased copy -only restored the `common_ops` guard and never picked up the hunk our version added for the -unconditional `infllm_v2` import, which crashed `import sglang` with -`ModuleNotFoundError: No module named 'sgl_kernel.infllm_v2'`. - -**The v0.5.16 rebase resolves 003 too.** The fork's own -`003-rdna4-sgl-kernel-fallbacks.patch` now wraps the `infllm_v2` import in `try/except -ImportError`, which is what our override existed to add, so the override and the local patch -file are gone. Upstream issue -([#3](https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference/issues/3)) can be -closed. - -`SGLANG_TAG` and `FORK_REF` must move together: the series is rebased onto one specific upstream -tree, and applying the v0.5.16 series to a v0.5.15 clone fails 14 patches (run 30299718239). -`SGLANG_COMMIT` pins the tree the tag resolved to, so a retag cannot swap it silently. - -Re-check the override list on the next bump — a rebase can regress any of them. - -## Building — GPU-free, no build host requirement - -The kernel C++ cross-compiles (`PYTORCH_ROCM_ARCH=gfx1201` — an explicit target, no device -probe). `setup.sh`'s GPU touches are verification-only, but it runs under `set -eo pipefail` -and one of them is a fatal `assert torch.cuda.is_available()` right after the torch install — -the Dockerfile seds that assert out (grep-guarded, like the TP=1 patches); the final device -verify already passes GPU-free (`device_count()` is 0, so the per-device loop never runs). -The build still hard-gates on `import sglang`. The real smoke test moves to deploy: the pod's -model load exercises the compiled kernels, and rollback is the previous digest. - -Known GPU-less caveat: `build_skinny_gemms_int4.sh` imports its freshly-compiled `.so` — if that -import needs a device it fails with setup.sh's non-fatal WARNING and the wvSplitK **MoE** kernel -is skipped. Our `qwen36-27b` is dense and never calls it, so this is harmless for this image. - -### Two-stage build (slim runtime, ~18 GB smaller) - -The Dockerfile is multi-stage: the `builder` uses the ROCm **`-complete`** SDK to compile, then the -final stage starts from the slim **non-complete** ROCm base (1.2 GB vs 7.4 GB compressed) and copies -only the built conda env + fork repo. This works because torch+rocm bundles its own ROCm runtime -(~13 GB) and the compiled kernels RPATH to `torch/lib`, not `/opt/rocm` — so the ~18 GB dev SDK is -build-only. **Validation caveat:** Triton JIT-compiles at runtime and needs its ROCm backend tools -(vendored under `triton/backends/amd`). The build-time gates (`import torch, sglang`) do **not** -exercise the JIT path — so a new build must be validated -by a **cold boot with the Triton cache cleared** (`/cache/sglang/triton`) before its digest is -pinned into the HelmRelease, to prove runtime compilation still works on the slim base. - -CI (`.github/workflows/build-sglang-rdna4.yaml`) builds on **`ubuntu-latest`** — zero contact -with control-1 / live serving, no maintenance window — and pushes the `v0.5.16-gfx1201` tag. -It fires on a merged change to the build inputs (a Renovate `FORK_REF` / base-digest bump; -docs excluded so a README edit can't repush the image) or on manual dispatch -(`gh workflow run build-sglang-rdna4.yaml`). Build locally with plain -`docker build docker/sglang-rdna4`. Expect ~15-30 min: the ROCm "complete" base is large and -the HIP kernels compile. - -## Pin as tag@digest — the digest is authoritative - -The tag gets rebuilt in place (memo `project_ik_llama_image_tags`), so the digest is what pins -the deployment — but the HelmRelease uses the repo-wide `tag: @sha256:` form, NOT a -bare `@digest`: Renovate's flux manager needs the tag present to issue digest-bump PRs. - -```bash -skopeo inspect docker://ghcr.io/tanguille/sglang-rdna4:v0.5.16-gfx1201 --format '{{.Digest}}' -``` - -## What's baked vs. supplied at runtime - -- **Baked:** the conda env + native gfx1201 kernels, the fork repo (`launch.sh`, `common.sh`, - the qwen3.6 chat template), and the correctness-critical gfx1201 `ENV` from - `common.sh::setup_rdna4_env` (`SGLANG_USE_AITER=0`, Triton AWQ/flash-attn enables, …). -- **Supplied by the HelmRelease:** the `qwen36-27b` launch.sh preset + flags, `TP=1`, the - single-GPU device vars, `CONDA_BASE=/opt/conda`, and the PVC mounts for the **model** - (`/cache/hf`) and the **persisted Triton cache** (`/cache/sglang/triton`). The model and - Triton cache stay on the PVC — only the engine moves into the image. - -The sglang HelmRelease keeps running `scripts/launch.sh qwen36-27b …` (now from -`/opt/rdna4-inference` instead of the PVC); `launch.sh` sources `common.sh`, which activates -the conda env — so the launch config is byte-for-byte what was validated on the PVC. - -## Multimodal: images work, video doesn't (no ffmpeg in the runtime image) - -Boot logs a benign ~60-line `libtorchcodec` traceback (missing `libavutil`) from 3 unused -multimodal processors (`mimo_v2`/`mimo_v2_asr`/`mimo_audio`); sglang catches it, non-fatal. - -`qwen36-27b` (`has_image_understanding: true`) is used for image-detection in prod — images go -through `torchvision`/PIL, unaffected by torchcodec. Video (`VideoDecoderWrapper`) is broken: -torchcodec's native FFmpeg libs aren't installed and `decord` isn't either. Not fixed since -nothing sends video; `decord` is the lighter fix if that changes. diff --git a/docker/sglang-rdna4/entrypoint.sh b/docker/sglang-rdna4/entrypoint.sh deleted file mode 100644 index a28503d82..000000000 --- a/docker/sglang-rdna4/entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# Activate the conda env (which holds the SGLang install + native gfx1201 kernels), -# then hand off. Used when the image runs its default ENTRYPOINT; the sglang HelmRelease -# overrides `command` to run scripts/launch.sh, which sources common.sh and self-activates -# the same env — so both paths end up in sglang-triton36-v0514. -# No `set -u`: conda activate references unbound vars internally. -set -eo pipefail - -source "${CONDA_BASE:-/opt/conda}/etc/profile.d/conda.sh" -conda activate "${ENV_NAME:-sglang-triton36-v0514}" - -# exec so the server is PID 1's child and gets SIGTERM on pod stop. -exec "$@" diff --git a/docs/llm-hosting/engine-benchmarks-gfx1201.md b/docs/llm-hosting/engine-benchmarks-gfx1201.md index 0fa2eda39..d777d4756 100644 --- a/docs/llm-hosting/engine-benchmarks-gfx1201.md +++ b/docs/llm-hosting/engine-benchmarks-gfx1201.md @@ -4,9 +4,11 @@ Multi-engine measurement record for Qwen3.6-27B on RDNA4 — vLLM, SGLang and ll Measurements are dated and kept as a historical series; the open experiments at the bottom are still outstanding. -**Current production engine: SGLang**, baked `ghcr.io/tanguille/sglang-rdna4`. Build, -pin and rollback mechanics are in `docker/sglang-rdna4/README.md`; outstanding upstream -gaps are tracked in `docs/llm-hosting/sglang-blockers.md`. The 2026-06-21 round below +**Current production engine: SGLang**, `ghcr.io/tanguille/sglang-rdna4`, built by our fork +of the upstream RDNA4 image rather than a pipeline in this repo — see +`docs/llm-hosting/sglang-blockers.md` "Current approach" for the build source and the +upstream cutover plan; outstanding upstream gaps are tracked in the same doc. The +2026-06-21 round below concluded in favour of vLLM and has since been superseded — read its numbers as a snapshot of that date, not as current guidance. diff --git a/docs/llm-hosting/sglang-blockers.md b/docs/llm-hosting/sglang-blockers.md index 2a4efc08a..1d80dcf7b 100644 --- a/docs/llm-hosting/sglang-blockers.md +++ b/docs/llm-hosting/sglang-blockers.md @@ -2,7 +2,7 @@ Tracking what needs to land upstream before SGLang can replace vLLM in production without depending on the mattbucci RDNA4 fork. -**Current approach:** `mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference` fork, **v0.5.15** (torch 2.11+rocm7.2, Triton 3.6), baked into the `ghcr.io/tanguille/sglang-rdna4` image by `.github/workflows/build-sglang-rdna4.yaml`. Build/pin/rollback mechanics are in `docker/sglang-rdna4/README.md`. The retired PVC-rebuild recipe is gone with the `sglang` app directory; recover it from git history if ever needed. +**Current approach:** running `ghcr.io/tanguille/sglang-rdna4:sha-cb7b7605...` (torch 2.11+rocm7.2, Triton 3.6, SGLang v0.5.16), built by our fork's own `build-image.yaml` `publish` job (`Tanguille/2x-R9700-RDNA4-GFX1201-sglang-inference`, `main`), not by a pipeline in this repo — `docker/sglang-rdna4/` and `.github/workflows/build-sglang-rdna4.yaml` are retired. That fork branch carries mattbucci's full patch series plus our HiCache OpenSSL-headers fix (`fix/hicache-openssl-headers`, upstream PR mattbucci#6, still open/unmerged there — this is our own fork build, not the upstream package). `SGLANG_RDNA4_DISABLE_STORE_CACHE=1` is required in the HelmRelease env because LLMKube bypasses the image's entrypoint (see the InferenceService manifest). Once mattbucci#6 merges upstream, re-point at `ghcr.io/mattbucci/sglang-rdna4` directly and drop the fork build. The retired PVC-rebuild recipe is gone with the `sglang` app directory; recover it from git history if ever needed. --- @@ -112,7 +112,7 @@ replace the old "quarterly / ROCm CI" trigger — subscribe to #28511 and #30263 ## Blocker 5 — No official SGLang Docker image for gfx1201 -**Impact:** Operational. Requires maintaining a custom Dockerfile and image (defined in `docker/sglang-rdna4/`). +**Impact:** Operational. Previously required maintaining our own Dockerfile and build pipeline; the fork now publishes `ghcr.io/mattbucci/sglang-rdna4` itself, retiring ours (see "Current approach" above). **Upstream references:** - sgl-project/sglang Discussion #12600 — same tracking thread as Blocker 4 diff --git a/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml b/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml index aa7626ffe..8645cf805 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml @@ -77,7 +77,7 @@ spec: runtime: sglang # Operator default binds :: (IPv6-only on this IPv4 cluster); override to v4. bindAddress: 0.0.0.0 - image: ghcr.io/tanguille/sglang-rdna4:v0.5.16-gfx1201@sha256:615cb4265400f5468da4c32221f0086df075b126ce8dbfa78d998e844a5e424d + image: ghcr.io/tanguille/sglang-rdna4:sha-cb7b76050cbf8dda9e5a78c07b24df1a783caeb3@sha256:f961fc274eaad4af065097e0921b88ae5a97a478baa0e384e6c5ee440c355b29 containerPort: 30000 endpoint: # v0.9.8 maps endpoint.port to both Service port and targetPort; use the @@ -121,10 +121,15 @@ spec: value: "0" - name: TRITON_CACHE_DIR value: /cache/sglang/triton + # Required at TP=1: LLMKube runs launch_server directly, bypassing the image's + # entrypoint.sh/gpu-selection.sh, the only place this is normally set. Without it the + # JIT store_cache kernel crashes on first request (kvcache.cuh:204). + - name: SGLANG_RDNA4_DISABLE_STORE_CACHE + value: "1" # Page names encode neither the weights revision nor the engine, so this suffix MUST be # bumped with the image digest above or the AWQ revision, or stale KV pages get served. - name: SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR - value: /hicache/sglang-v0.5.16_awq-f541031d + value: /hicache/sglang-v0.5.16_awq-f541031d_fork-cb7b760 # Unset defaults to unbounded; a full L2 measured 14G on 2026-07-27. - name: SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE value: 64Gi