diff --git a/prod/GLM-5.3-Flash-SGL-TP4-W4AFP8-LongContext.yaml b/prod/GLM-5.3-Flash-SGL-TP4-W4AFP8-LongContext.yaml index c49e0a21..8318bce4 100644 --- a/prod/GLM-5.3-Flash-SGL-TP4-W4AFP8-LongContext.yaml +++ b/prod/GLM-5.3-Flash-SGL-TP4-W4AFP8-LongContext.yaml @@ -8,23 +8,31 @@ # r1: gpu31 campaign-2 arm L2 (2026-09-23), chunk 8192, --prefill-decode-interval 1 # docker.io/nearaidev/sglang@sha256:fde25985aea3ebabf1eb581ae21d53be8540e32933eef942ee8b962a1bfbea20 # published by workflow run 35903077821 from recipe merge commit f8106f096e9c83. -# r2: chunk 16384, --prefill-decode-interval 2, SGLANG_DSA_INDEXER_QSPLIT=1 +# r2: chunk 8192, --prefill-decode-interval 2, SGLANG_DSA_INDEXER_QSPLIT=1 # docker.io/nearaidev/sglang@sha256:8ff1a487b98a52fe08b781715bebd7c8c445d4fe068f312f03f527d5a3c77e84 # published by workflow run 36073196105 from recipe merge commit 556482c78cd3, # which adds the opt-in DSA indexer query split (cvm-compose-files#300). -# On r2 the 16384 chunk and the split are ONE change: without the split a concurrent -# long burst left 0.04-0.65 GB free per GPU, the condition that preceded the gpu02 -# crash; with it, 9.3-9.5 GB. validate_glm53_prod_config.rb rejects either one alone. -# On stored long-tier traffic (gpu31, 2026-09-24) the r2 arm ran TTFT 0.61-0.78x and -# TPOT 0.53-0.85x of the r1 arm, with quality at the noise floor. -# Neither the HCC/PPCIe host-memory path nor the indexer split's all-gather has run in -# a CVM before this file, so gpu02 is their first soak. Roll out with +# +# r2 IS AN ISOLATION ARM, NOT A SHIPPING CONFIG. The 2026-09-25 canary ran the split WITH +# chunk 16384 and regressed the tail: TTFT p95 76.9 s against r1's 47.3 s (1.62x) over a +# 4 h steady-state window, though the decode win did reproduce (ITL p90 0.87x). Traffic +# volume, cache warmth and queue depth did not explain it. The chunk and the split both +# bite hardest on the largest prefills, and the canary changed both at once, so this arm +# holds the split at 8192 to tell them apart: if p95 recovers the chunk was at fault, if +# it stays bad the all-gather under CC/PPCIe is. Do not read this file as an endorsement +# of either setting until that run reports. +# +# chunk 16384 must never be set without the split: it left 0.04-0.65 GB free per GPU on a +# concurrent long burst, the condition that preceded the gpu02 crash. The reverse is +# allowed, which is what makes this arm possible. validate_glm53_prod_config.rb enforces it. +# The HCC/PPCIe host-memory path is confirmed working in a CVM as of 2026-09-25; the +# split's all-gather under CC is what this arm measures. Roll out with # docs/glm53-w4afp8-long-context-rollout.md, one replica at a time. # Do not hand-edit this file. # This is the DEDICATED LONG-CONTEXT TIER variant of prod/GLM-5.3-Flash-SGL-TP4.yaml. # Generated from the long-context file. Routing remains the dedicated long-context -# contract below; both replicas run a W4AFP8 + HiCache engine, r2 as a canary at -# --prefill-decode-interval 2 with chunk 16384 and the DSA indexer split. +# contract below; both replicas run a W4AFP8 + HiCache engine, r2 as an isolation arm +# at --prefill-decode-interval 2 with the DSA indexer split and chunk 8192. # # *** LONG-CONTEXT TIER (dedicated host, both TP4/EP4 replicas) *** # Deploy this file on exactly the host(s) chosen as the GLM-5.3 Flash long-context @@ -66,8 +74,8 @@ # - Request-level tiering trades the tail for the median: p50 TTFT -24%, # p95 TTFT 10.0 s -> 27.4 s on identical replicas. Judge it on the SLO you mean. # The replicas' engine flags differ in --dist-init-addr, --prefill-decode-interval -# (1 on r1, 2 on r2) and --chunked-prefill-size (8192 on r1, 16384 on r2); r2 also -# sets SGLANG_DSA_INDEXER_QSPLIT=1 and runs a different image. r1's 8192 is the +# (1 on r1, 2 on r2); both now run --chunked-prefill-size 8192. r2 additionally sets +# SGLANG_DSA_INDEXER_QSPLIT=1 and runs a different image. The 8192 chunk is the # W4AFP8 setting: FP8 at 8192 OOMed in the DSA indexer under concurrent 400K+ # contexts, while W4AFP8's 2.4x larger device KV pool kept 3.7 GiB free through 8 # concurrent 647K-756K-token cold prefills on gpu31. 16384 was rejected then because @@ -117,8 +125,7 @@ x-logging-conf: &logging-conf # SGLang image is the # pinned build base. The serving envelope is # intentionally conservative: BF16 KV, 0.80 static memory, 32 running requests, a -# bounded 8-request queue, prefill chunks of 8192 (r1) / 16384 (r2) with -# --max-prefill-tokens 32768, +# bounded 8-request queue, 8192-token prefill chunks with --max-prefill-tokens 32768, # decode graphs capped at batch 32, TileLang DSA, the CUTLASS W4A8 MoE path (hence no # --moe-runner-backend and no FP8 --revision), and adaptive EAGLE 5/1/6. This retains # the full 1,048,576-token model context. @@ -498,7 +505,7 @@ services: --max-queued-requests 8 --enable-priority-scheduling --disable-priority-preemption - --chunked-prefill-size 16384 + --chunked-prefill-size 8192 --max-prefill-tokens 32768 --prefill-decode-interval 2 --cuda-graph-max-bs-decode 32 @@ -576,7 +583,7 @@ services: device_ids: ["4","5","6","7"] capabilities: [gpu] labels: - com.datadoghq.ad.logs: '[{"source":"sglang","service":"sglang","tags":["model:z-ai/glm-5.3-flash","model_path:graphistry/GLM-5.3-Flash-W4AFP8","served_model:z-ai/glm-5.3-flash","precision:int4-weights-fp8-activations-bf16-kv","deployment:glm53-flash-sgl-tp4","config_variant:fc91d24-long-context-w4afp8-c16384-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192","request_logging:disabled","engine_image:8ff1a487b98a","env:${ENV}","host:${CVM_HOST}","ip:${HOST_IP}","port:8000","instance:2"]}]' + com.datadoghq.ad.logs: '[{"source":"sglang","service":"sglang","tags":["model:z-ai/glm-5.3-flash","model_path:graphistry/GLM-5.3-Flash-W4AFP8","served_model:z-ai/glm-5.3-flash","precision:int4-weights-fp8-activations-bf16-kv","deployment:glm53-flash-sgl-tp4","config_variant:fc91d24-long-context-w4afp8-c8192-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192","request_logging:disabled","engine_image:8ff1a487b98a","env:${ENV}","host:${CVM_HOST}","ip:${HOST_IP}","port:8000","instance:2"]}]' nearai.otel.scrape: "true" nearai.otel.job: "sglang" nearai.otel.service: "sglang" @@ -588,7 +595,7 @@ services: nearai.otel.model_path: "graphistry/GLM-5.3-Flash-W4AFP8" nearai.otel.served_model: "z-ai/glm-5.3-flash" nearai.otel.deployment: "glm53-flash-sgl-tp4" - nearai.otel.config_variant: "fc91d24-long-context-w4afp8-c16384-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192" + nearai.otel.config_variant: "fc91d24-long-context-w4afp8-c8192-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192" nearai.otel.thinking_budget_policy: "default8192-public-to-native" nearai.otel.request_logging: "disabled" nearai.otel.engine_image: "8ff1a487b98a" @@ -1014,7 +1021,7 @@ configs: ip: "${HOST_IP}" port: "8000" instance: "2" - config_variant: "fc91d24-long-context-w4afp8-c16384-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192" + config_variant: "fc91d24-long-context-w4afp8-c8192-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192" thinking_budget_policy: "default8192-public-to-native" request_logging: "disabled" engine_image: "8ff1a487b98a" diff --git a/scripts/__pycache__/prepare_glm53_shared_kv.cpython-314.pyc b/scripts/__pycache__/prepare_glm53_shared_kv.cpython-314.pyc new file mode 100644 index 00000000..28163bc7 Binary files /dev/null and b/scripts/__pycache__/prepare_glm53_shared_kv.cpython-314.pyc differ diff --git a/scripts/__pycache__/prepare_glm53_w4afp8_long_context.cpython-314.pyc b/scripts/__pycache__/prepare_glm53_w4afp8_long_context.cpython-314.pyc index 6da2c833..e9458f79 100644 Binary files a/scripts/__pycache__/prepare_glm53_w4afp8_long_context.cpython-314.pyc and b/scripts/__pycache__/prepare_glm53_w4afp8_long_context.cpython-314.pyc differ diff --git a/scripts/__pycache__/test_glm53_shared_kv.cpython-314.pyc b/scripts/__pycache__/test_glm53_shared_kv.cpython-314.pyc new file mode 100644 index 00000000..120fef21 Binary files /dev/null and b/scripts/__pycache__/test_glm53_shared_kv.cpython-314.pyc differ diff --git a/scripts/__pycache__/test_glm53_w4afp8_long_context.cpython-314.pyc b/scripts/__pycache__/test_glm53_w4afp8_long_context.cpython-314.pyc index 2ea0216e..3a66b72f 100644 Binary files a/scripts/__pycache__/test_glm53_w4afp8_long_context.cpython-314.pyc and b/scripts/__pycache__/test_glm53_w4afp8_long_context.cpython-314.pyc differ diff --git a/scripts/prepare_glm53_w4afp8_long_context.py b/scripts/prepare_glm53_w4afp8_long_context.py index 79eac947..ffc9a4ee 100755 --- a/scripts/prepare_glm53_w4afp8_long_context.py +++ b/scripts/prepare_glm53_w4afp8_long_context.py @@ -37,10 +37,26 @@ R2_ENGINE_IMAGE_LABEL: Final = "8ff1a487b98a" REPLICA_IMAGE: Final = {1: IMAGE, 2: R2_IMAGE} REPLICA_IMAGE_LABEL: Final = {1: ENGINE_IMAGE_LABEL, 2: R2_ENGINE_IMAGE_LABEL} -# c16384 is only safe WITH the split: without it the indexer scratch left 0.04-0.65 GB free on a -# concurrent long burst, the condition that preceded the gpu02 crash. With it, 9.3-9.5 GB. -# The validator enforces the pairing; never set one without the other. -CHUNKED_PREFILL_SIZE: Final = {1: 8192, 2: 16384} +# r2 is back to 8192 while keeping the split. This is an ISOLATION ARM, not the shipping config. +# +# The 2026-09-25 gpu02 canary ran split + c16384 + pdi 2 and produced the opposite of the lab +# result at the tail: over a 4 h steady-state window TTFT p95 was 76.9 s against r1's 47.3 s +# (1.62x), while the decode win did reproduce (ITL p90 0.87x, p50 0.97x). It was not explained by +# traffic volume (681 vs 688 req/h), cache warmth (79.7% vs 81.6% hit) or queueing (r2's average +# queue depth was LOWER, 1.23 vs 1.56). r2 did draw ~23% more uncached prefill tokens per request, +# which it absorbed for +3% median TTFT but +62% at p95 -- worse only at the tail, i.e. only on the +# largest prefills. +# +# Two candidates bite exactly there, and the canary changed both at once: +# a) the 16384 chunk, which doubles how long one prefill chunk occupies the scheduler; +# b) the indexer split's all-gather, which trades per-rank compute for a collective. gpu31 is +# bare metal with CC off, so its all-gather is cheap; under CC/PPCIe memory encryption a +# collective costs more and the trade can invert. This was flagged untested in #300. +# +# Holding the split at 8192 separates them: if p95 recovers, the chunk was at fault; if it stays +# bad, the all-gather under CC is. Note the validator gate is one-directional -- c16384 REQUIRES +# the split, but the split does not require c16384 -- so this combination is permitted by design. +CHUNKED_PREFILL_SIZE: Final = {1: 8192, 2: 8192} R2_EXTRA_ENV: Final = (" - SGLANG_DSA_INDEXER_QSPLIT=1\n",) SOURCE_SERVICE_PREFIX: Final = "model-sg-glm53-fp8-tp4-r" SERVICE_PREFIX: Final = "model-sg-glm53-w4afp8-tp4-r" @@ -54,7 +70,7 @@ VARIANTS: Final = { 1: "fc91d24-long-context-w4afp8-c8192-hicache-cuda-host-pooled-v1-admission-reserve-disabled" "-pool-clamp-pdi1-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192", - 2: "fc91d24-long-context-w4afp8-c16384-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled" + 2: "fc91d24-long-context-w4afp8-c8192-qsplit-hicache-cuda-host-pooled-v1-admission-reserve-disabled" "-pool-clamp-pdi2-h200-tp4-ep4-eagle-adaptive-5-1-6-strict-budget8192", } PREFILL_DECODE_INTERVAL: Final = {1: 1, 2: 2} @@ -83,17 +99,25 @@ "# r1: gpu31 campaign-2 arm L2 (2026-09-23), chunk 8192, --prefill-decode-interval 1\n" f"# {IMAGE}\n" "# published by workflow run 35903077821 from recipe merge commit f8106f096e9c83.\n" - "# r2: chunk 16384, --prefill-decode-interval 2, SGLANG_DSA_INDEXER_QSPLIT=1\n" + "# r2: chunk 8192, --prefill-decode-interval 2, SGLANG_DSA_INDEXER_QSPLIT=1\n" f"# {R2_IMAGE}\n" "# published by workflow run 36073196105 from recipe merge commit 556482c78cd3,\n" "# which adds the opt-in DSA indexer query split (cvm-compose-files#300).\n" - "# On r2 the 16384 chunk and the split are ONE change: without the split a concurrent\n" - "# long burst left 0.04-0.65 GB free per GPU, the condition that preceded the gpu02\n" - "# crash; with it, 9.3-9.5 GB. validate_glm53_prod_config.rb rejects either one alone.\n" - "# On stored long-tier traffic (gpu31, 2026-09-24) the r2 arm ran TTFT 0.61-0.78x and\n" - "# TPOT 0.53-0.85x of the r1 arm, with quality at the noise floor.\n" - "# Neither the HCC/PPCIe host-memory path nor the indexer split's all-gather has run in\n" - "# a CVM before this file, so gpu02 is their first soak. Roll out with\n" + "#\n" + "# r2 IS AN ISOLATION ARM, NOT A SHIPPING CONFIG. The 2026-09-25 canary ran the split WITH\n" + "# chunk 16384 and regressed the tail: TTFT p95 76.9 s against r1's 47.3 s (1.62x) over a\n" + "# 4 h steady-state window, though the decode win did reproduce (ITL p90 0.87x). Traffic\n" + "# volume, cache warmth and queue depth did not explain it. The chunk and the split both\n" + "# bite hardest on the largest prefills, and the canary changed both at once, so this arm\n" + "# holds the split at 8192 to tell them apart: if p95 recovers the chunk was at fault, if\n" + "# it stays bad the all-gather under CC/PPCIe is. Do not read this file as an endorsement\n" + "# of either setting until that run reports.\n" + "#\n" + "# chunk 16384 must never be set without the split: it left 0.04-0.65 GB free per GPU on a\n" + "# concurrent long burst, the condition that preceded the gpu02 crash. The reverse is\n" + "# allowed, which is what makes this arm possible. validate_glm53_prod_config.rb enforces it.\n" + "# The HCC/PPCIe host-memory path is confirmed working in a CVM as of 2026-09-25; the\n" + "# split's all-gather under CC is what this arm measures. Roll out with\n" "# docs/glm53-w4afp8-long-context-rollout.md, one replica at a time.\n" "# Do not hand-edit this file.\n" ) @@ -104,8 +128,8 @@ "# long-context contract below, while the engines intentionally form an r1 control / r2\n" "# HiCache experiment and therefore are not byte-identical to the canonical file.\n", "# Generated from the long-context file. Routing remains the dedicated long-context\n" - "# contract below; both replicas run a W4AFP8 + HiCache engine, r2 as a canary at\n" - "# --prefill-decode-interval 2 with chunk 16384 and the DSA indexer split.\n", + "# contract below; both replicas run a W4AFP8 + HiCache engine, r2 as an isolation arm\n" + "# at --prefill-decode-interval 2 with the DSA indexer split and chunk 8192.\n", ), ( "# tier; every other host keeps the canonical file. The inference-proxy and routing\n" @@ -121,8 +145,8 @@ "# halves prefill speed, and no other per-replica flag moved the tail. Conversation\n" "# affinity stays on because the prefix cache is worth ~3x in request capacity.\n", "# The replicas' engine flags differ in --dist-init-addr, --prefill-decode-interval\n" - "# (1 on r1, 2 on r2) and --chunked-prefill-size (8192 on r1, 16384 on r2); r2 also\n" - "# sets SGLANG_DSA_INDEXER_QSPLIT=1 and runs a different image. r1's 8192 is the\n" + "# (1 on r1, 2 on r2); both now run --chunked-prefill-size 8192. r2 additionally sets\n" + "# SGLANG_DSA_INDEXER_QSPLIT=1 and runs a different image. The 8192 chunk is the\n" "# W4AFP8 setting: FP8 at 8192 OOMed in the DSA indexer under concurrent 400K+\n" "# contexts, while W4AFP8's 2.4x larger device KV pool kept 3.7 GiB free through 8\n" "# concurrent 647K-756K-token cold prefills on gpu31. 16384 was rejected then because\n" @@ -178,8 +202,7 @@ "# 1,048,576-token model context while avoiding the late K-pool workspace exhaustion\n" "# observed with larger prefill chunks.\n", "# intentionally conservative: BF16 KV, 0.80 static memory, 32 running requests, a\n" - "# bounded 8-request queue, prefill chunks of 8192 (r1) / 16384 (r2) with\n" - "# --max-prefill-tokens 32768,\n" + "# bounded 8-request queue, 8192-token prefill chunks with --max-prefill-tokens 32768,\n" "# decode graphs capped at batch 32, TileLang DSA, the CUTLASS W4A8 MoE path (hence no\n" "# --moe-runner-backend and no FP8 --revision), and adaptive EAGLE 5/1/6. This retains\n" "# the full 1,048,576-token model context.\n", diff --git a/scripts/test_glm53_w4afp8_long_context.py b/scripts/test_glm53_w4afp8_long_context.py index 418ca49c..de737f9d 100644 --- a/scripts/test_glm53_w4afp8_long_context.py +++ b/scripts/test_glm53_w4afp8_long_context.py @@ -122,24 +122,30 @@ def test_rejects_engine_argv_drift(self) -> None: # r1 is the pdi 1 control and r2 the pdi 2 canary; neither may take the other's value. ("\n --prefill-decode-interval 1\n", "\n --prefill-decode-interval 2\n", "--prefill-decode-interval 1"), ("\n --prefill-decode-interval 2\n", "\n --prefill-decode-interval 1\n", "--prefill-decode-interval 2"), - # r1 is the c8192 control and r2 the c16384 canary; neither may take the other's chunk. - ("\n --chunked-prefill-size 8192\n", "\n --chunked-prefill-size 16384\n", "model-sg-glm53-w4afp8-tp4-r1 argv must be"), - ("\n --chunked-prefill-size 16384\n", "\n --chunked-prefill-size 8192\n", "model-sg-glm53-w4afp8-tp4-r2 argv must be"), + # Both replicas run 8192 in this arm, so neither may drift off it. + ("\n --chunked-prefill-size 8192\n", "\n --chunked-prefill-size 4096\n", "model-sg-glm53-w4afp8-tp4-r1 argv must be"), + ("\n --chunked-prefill-size 8192\n", "\n --chunked-prefill-size 4096\n", "model-sg-glm53-w4afp8-tp4-r2 argv must be"), ) for before, after, message in cases: with self.subTest(mutation=after.strip()[:60]): self.assert_fails(self.replace_once(before, after), message) def test_rejects_c16384_without_the_indexer_split(self) -> None: - """The 16384 chunk is only memory-safe with SGLANG_DSA_INDEXER_QSPLIT=1. + """Raising r2 to the 16384 chunk without the split must be rejected by the pairing gate. - Without the split a concurrent long burst left 0.04-0.65 GB free, the condition that - preceded the gpu02 crash; with it, 9.3-9.5 GB. Removing the variable while leaving the - chunk at 16384 must be rejected, and the message must name the pairing rather than - surfacing only as generic environment drift. + This arm runs 8192, so the forbidden pairing has to be constructed: raise the chunk AND + drop the split. Without the split that chunk left 0.04-0.65 GB free per GPU on a + concurrent long burst, the condition that preceded the gpu02 crash. The assertion names + the pairing error specifically, so the test cannot pass merely because some unrelated + equality check fired first. """ - mutated = self.replace_once(" - SGLANG_DSA_INDEXER_QSPLIT=1\n", "") - self.assert_fails(mutated, "SGLANG_DSA_INDEXER_QSPLIT") + mutated = self.replace_once("\n --chunked-prefill-size 8192\n", "\n --chunked-prefill-size 16384\n") + mutated = mutated.replace(" - SGLANG_DSA_INDEXER_QSPLIT=1\n", "", 1) + self.assert_fails(mutated, "without SGLANG_DSA_INDEXER_QSPLIT=1") + + def test_rejects_dropping_the_split_from_r2(self) -> None: + """r2 carries the split in this arm; removing it is the whole variable under test.""" + self.assert_fails(self.replace_once(" - SGLANG_DSA_INDEXER_QSPLIT=1\n", ""), "SGLANG_DSA_INDEXER_QSPLIT") def test_rejects_the_split_on_an_image_without_the_patch(self) -> None: """Only the v2 image carries the split patch; the flag is inert and misleading elsewhere.""" diff --git a/scripts/validate_glm53_prod_config.rb b/scripts/validate_glm53_prod_config.rb index 9eac883e..d60a8e2b 100644 --- a/scripts/validate_glm53_prod_config.rb +++ b/scripts/validate_glm53_prod_config.rb @@ -645,7 +645,7 @@ def validate_long_context(errors, compose, replica_services) "model-sg-glm53-w4afp8-tp4-r1" => { "devices" => %w[0 1 2 3], "dist_init" => "127.0.0.1:29510", "instance" => "1", "pdi" => "1", "image" => W4AFP8_LONG_CONTEXT_IMAGE, "chunk" => "8192", "qsplit" => nil }, "model-sg-glm53-w4afp8-tp4-r2" => { "devices" => %w[4 5 6 7], "dist_init" => "127.0.0.1:29511", "instance" => "2", "pdi" => "2", - "image" => W4AFP8_LONG_CONTEXT_R2_IMAGE, "chunk" => "16384", "qsplit" => "1" }, + "image" => W4AFP8_LONG_CONTEXT_R2_IMAGE, "chunk" => "8192", "qsplit" => "1" }, }.freeze W4AFP8_LONG_CONTEXT_ARGV = Shellwords.split(<<~'ARGV').freeze sglang serve