diff --git a/dag/extdeps/vllm/engine_args.dag b/dag/extdeps/vllm/engine_args.dag index 59849bfdb90..349da7c9cdd 100644 --- a/dag/extdeps/vllm/engine_args.dag +++ b/dag/extdeps/vllm/engine_args.dag @@ -76,10 +76,25 @@ fn vllm_distributed_executor_backend_wire(backend: VllmDistributedExecutorBacken } } -// THE ARGUMENTS A LAUNCH ACTUALLY CARRIES. `max_num_batched_tokens` is deliberately ABSENT from this -// record rather than present-and-optional: see the do-not-set row below, which is a typed refusal -// subject and not a comment. +// THE PREFILL SCHEDULING AXIS, WHICH IS A REQUEST AND NOT AN EFFECTIVE VALUE. +// +// A launch that omits `--max-num-batched-tokens` has NOT thereby chosen a value. This build resolves +// the default from device memory, usage context, parallel configuration and performance mode, and +// then adjusts it again when chunked prefill is off -- so the absence of the flag determines nothing +// a consumer may read. `none` below means UNREQUESTED, and the effective budget is knowable only by +// reading it back off the running engine. +type ChunkedPrefillRequest + = ChunkedPrefillRequested + | ChunkedPrefillNotRequested + +type VllmPrefillSchedulingPolicy { + max_num_batched_tokens: TokenCount + chunked_prefill: ChunkedPrefillRequest +} + +// THE ARGUMENTS A LAUNCH ACTUALLY CARRIES. type VllmEngineArgs { + prefill_scheduling: VllmPrefillSchedulingPolicy? tensor_parallel_size: Nat gpu_memory_utilization: VllmMemoryUtilization max_model_len: TokenCount @@ -127,10 +142,23 @@ type VllmDiscouragedArg { ground: NonEmptyStr } -data vllm_discouraged_max_num_batched_tokens: VllmDiscouragedArg = VllmDiscouragedArg { - arg: "--max-num-batched-tokens" as NonEmptyStr, - ground: "on a unified-memory GB10 this only reduces the KV pool; prefill is compute-bound so a larger batch buys no throughput" as NonEmptyStr -} +// WITHDRAWN 2026-09-07. This row said `--max-num-batched-tokens` "only reduces the KV pool; prefill +// is compute-bound so a larger batch buys no throughput", and both halves are now unsupported. +// +// The batch budget is the per-iteration token budget the scheduler divides between decode and +// prefill, so it participates in a LATENCY/FAIRNESS tradeoff whatever it does to aggregate +// throughput: a co-tenant's next decode token cannot appear until the current mixed step finishes. +// A measured comparison on this fleet -- the same decode taking 3.8s alone and 63.5s beside a cold +// prefill -- is a serving result the old ground did not account for. +// +// And "prefill is compute-bound" was never established here. It was inferred from average link +// utilisation (~7.9 Gb/s of 50 Gb/s during a cold prefill), and low AVERAGE bandwidth is equally +// consistent with latency-bound collectives, one-rank stragglers, or host gaps between kernels. +// Bulk transfer time is a lower bound on communication cost, not a share of the critical path. +// +// It is deliberately NOT replaced with a recommended value. The honest state is an unresolved policy +// axis whose selection needs a step-level trace this repository does not yet capture, and encoding a +// number here would be the same mistake in the other direction. data vllm_verified_negatives: List = [ vllm_negative_nvfp4_kv_for_mla, @@ -138,5 +166,4 @@ data vllm_verified_negatives: List = [ ] data vllm_discouraged_args: List = [ - vllm_discouraged_max_num_batched_tokens, ] diff --git a/dag/extdeps/vllm/kv_layout.dag b/dag/extdeps/vllm/kv_layout.dag new file mode 100644 index 00000000000..03811cf4a81 --- /dev/null +++ b/dag/extdeps/vllm/kv_layout.dag @@ -0,0 +1,409 @@ +module extdeps.vllm.kv_layout + +import std.types { Bool, String, NonEmptyStr, List } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.measure { TokenCount, token_count, token_count_value } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } +import extdeps.vllm.kv_cache { VllmObservedKvPool } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/vllm-project/vllm/blob/main/vllm/v1/core/kv_cache_utils.py" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.vllm.kv_layout", + decl_name: "ResolvedKvLayout", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// ── THE LAYOUT IS THE STRUCTURE; EVERY CAPACITY NUMBER IS A PROJECTION OF IT ─────────────────── +// +// WHAT THIS MODULE IS FOR. Three different quantities in this deployment have been called "the +// pool", and each is a real number the engine reports: +// +// ALLOCATOR INVENTORY num_gpu_blocks -- how many blocks the block pool contains. This is what +// the scheduler allocates from and what kv_cache_usage_perc divides by. +// FULL-CONTEXT EQUIVALENT the "GPU KV cache size: N tokens" line -- max_concurrency x +// max_model_len, i.e. how many max-length requests fit, re-expressed in +// tokens. It is a projection, not an inventory. +// SCHEDULER GRANULARITY block_size -- the MINIMUM across resolved cache groups, which is not +// "the number of tokens every allocator block holds". +// +// Multiplying the first by the third and calling it the second is the error extdeps.vllm.kv_cache +// already refuses at vllm_pool_reconstruction_is_authoritative. This module exists so the three +// stop being three spellings of one field and become projections of one declared structure -- which +// is the only form in which they can be checked against each other. +// +// WHY THE RATIO IS NOT A CONSTANT. Under a hybrid layout the per-request cost is not +// length / block_size: full-attention groups grow with the sequence, sliding-window groups stop +// growing at their window, and a mamba group is a fixed state per sequence. Two engines with the +// same num_gpu_blocks and the same global block_size can therefore admit very different request +// counts. That is precisely why no function here converts an inventory into tokens. +// +// WHAT IS NOT OBSERVABLE YET, stated so the refusal below is not read as a defect in the caller. +// The engine's own get_kv_cache_group_metadata emits kind, block size and sliding window, and does +// NOT emit layer count, page size, the admission cap, or its calculated blocks-per-max-request. So +// `groups` is empty for every incarnation this repository can currently observe, and every function +// that needs per-group structure refuses. The trigger is one JSON launch line carrying those fields. + +// A CACHE GROUP WITH A ZERO BLOCK SIZE IS NOT A DEGENERATE GROUP, IT IS NOT A GROUP. Guarding for +// it downstream would put a division-by-zero branch in every cost function and force each one to +// invent an answer for a layout that cannot exist; refining the field removes the state instead +// (DESIGN section 4b: construction over validation). +type KvBlockSize = Int where range(min: 1, max: 1048576) +type KvLayerCount = Int where range(min: 1, max: 4096) + +// THE WINDOW LIVES IN THE VARIANT THAT HAS ONE. It was a separate optional field beside the kind, +// which made two states writable that the engine cannot produce: a windowed group with no window, +// and a full-attention group carrying one. The first was the dangerous one -- the cost function met +// it and silently charged full-attention cost, so a malformed layout produced a plausible number +// instead of a refusal, and that number would then have been reproduced against the printed lines +// and possibly agreed. Moving the window into the arm deletes both states rather than checking for +// them (DESIGN section 4b: construction over validation). +type KvGroupKind + = FullAttention + | SlidingWindowAttention { window: TokenCount } + | MambaState + +fn kv_group_kind_wire(k: KvGroupKind) -> String { + match k { + FullAttention => "full-attention" + SlidingWindowAttention { window: _ } => "sliding-window" + MambaState => "mamba-state" + } +} + +// One resolved cache group. `block_size` is THIS GROUP's, which the global scheduler block_size is +// only the minimum of; `sliding_window` is present exactly on the windowed kind and is what caps +// that group's per-request cost. +// A GROUP WITH NO LAYERS IS NOT A GROUP EITHER, so the count is refined the way the block size is: +// a zero-layer group contributes zero blocks to every cost, which is a layout that silently makes +// requests look free. +// +// page_size_bytes IS DELIBERATELY ABSENT. It was carried here and read by nothing -- the recorded +// field that no operation consumes, which reads as coverage of a byte-level accounting that does +// not exist. It returns when the unified-memory admission needs bytes rather than blocks, and it +// returns with the consumer that reads it. +type KvGroupLayout { + kind: KvGroupKind + layer_count: KvLayerCount + block_size: KvBlockSize +} + +// THE RESOLVED LAYOUT OF ONE ENGINE. It carries no incarnation: this is the upstream SHAPE, and +// binding a shape to the launch that exhibited it is a receipt, which is an observation this +// repository produces and therefore lives in the observing layer, not in extdeps (DESIGN section 3, +// external upstream decomposition). Putting the incarnation here would author it in two places, +// because the receipt has to name it anyway. +type ResolvedKvLayout sole_constructor { + allocator_blocks: Nat + scheduler_block_size: TokenCount + hash_block_size: TokenCount + max_model_len: TokenCount + max_num_batched_tokens: TokenCount + groups: List +} + +fn resolved_kv_layout( + allocator_blocks: Nat, + scheduler_block_size: TokenCount, + hash_block_size: TokenCount, + max_model_len: TokenCount, + max_num_batched_tokens: TokenCount, + groups: List, +) -> ResolvedKvLayout { + ResolvedKvLayout { + allocator_blocks: allocator_blocks, + scheduler_block_size: scheduler_block_size, + hash_block_size: hash_block_size, + max_model_len: max_model_len, + max_num_batched_tokens: max_num_batched_tokens, + groups: groups, + } +} + +// ── WHAT THE ENGINE REPORTED, AS TWO NUMBERS THAT MUST AGREE WITH THE LAYOUT ─────────────────── +// +// The pool carrier from extdeps.vllm.kv_cache is the KV-cache-size line, structurally bound to that +// line class. The concurrency figure is its sibling on the next line, and the two are carried +// together here because the recovery below needs both and neither is meaningful alone. +type VllmReportedCapacity sole_constructor { + pool: VllmObservedKvPool + max_concurrency_hundredths: Nat + observed_at_revision: VllmSourceRevision +} + +fn vllm_reported_capacity(pool: VllmObservedKvPool, max_concurrency_hundredths: Nat) -> VllmReportedCapacity { + VllmReportedCapacity { + pool: pool, + max_concurrency_hundredths: max_concurrency_hundredths, + observed_at_revision: pool.observed_at_revision, + } +} + +// ── THE ONE DERIVATION AVAILABLE FROM TODAY'S EVIDENCE ───────────────────────────────────────── +// +// The engine computes, in this order: +// +// blocks_per_max_request = (group-aware bytes for one max-length request) / (bytes per block) +// maximum_concurrency = num_gpu_blocks / blocks_per_max_request +// GPU KV cache size = maximum_concurrency * max_model_len +// +// so the per-request block cost it used is RECOVERABLE by inverting the last two: +// +// blocks_per_max_request = allocator_blocks * max_model_len / kv_cache_size_tokens +// +// This is the honest derivation available before per-group metadata exists, and it is the ONLY one +// this module offers, because it reads the engine's own group-aware arithmetic back out rather than +// re-deriving it from fields whose reductions differ. +// +// IT IS A RECOVERY, NOT AN IDENTITY. The reported figures are rounded for printing, so the +// recovered cost is exact only to that precision; the recovery therefore returns a refusal when the +// inputs cannot produce a positive integer, instead of a zero that would read as a free request. +type BlocksPerRequest + = BlocksPerRequestRecovered { blocks: Nat } + | BlocksPerRequestUnrecoverable { cause: NonEmptyStr } + +fn recovered_blocks_per_max_length_request( + layout: ResolvedKvLayout, + reported: VllmReportedCapacity, +) -> BlocksPerRequest { + let pool_tokens = token_count_value(t: reported.pool.tokens) + let context = token_count_value(t: layout.max_model_len) + if pool_tokens == 0 { + BlocksPerRequestUnrecoverable { cause: "the reported KV cache size is zero, so the engine's per-request block cost cannot be inverted out of it" as NonEmptyStr } + } else if layout.allocator_blocks == 0 { + BlocksPerRequestUnrecoverable { cause: "the allocator inventory is zero, so there is no pool to divide" as NonEmptyStr } + } else if context == 0 { + BlocksPerRequestUnrecoverable { cause: "the context cap is zero, so the full-context request the reported figures describe does not exist" as NonEmptyStr } + } else { + let recovered = (layout.allocator_blocks * context) / pool_tokens + if recovered == 0 { + BlocksPerRequestUnrecoverable { cause: "the inversion produced no whole block, so the reported pool and inventory do not describe the same engine" as NonEmptyStr } + } else { + BlocksPerRequestRecovered { blocks: recovered } + } + } +} + +// WHAT THE TWO PRINTED LINES CAN AND CANNOT CHECK, worked out rather than assumed. +// +// The tempting check is "recover the per-request cost, then replay it against the printed +// concurrency". Expand it and the inventory cancels: +// +// recovered_cost = A * M / P +// replayed_concurrency = A / recovered_cost = A * P / (A * M) = P / M +// +// so the replay tests the two PRINTED LINES against each other and never touches the inventory A. +// It is therefore green for any inventory whatsoever -- including one lifted from a different +// engine -- and shipping it as an inventory check would be a permanently-green wall cited as +// coverage (DESIGN section 4b: ask whether the RED is authorable before writing the check). It was +// written that way first, and the witness that was supposed to prove it discriminating is what +// caught it. +// +// So it is named for what it actually decides. The two lines are mutually consistent when the +// reported pool equals the reported concurrency times the context cap, and that is a real -- if +// narrow -- check: a transcription error in either line breaks it. THE INVENTORY REMAINS UNCHECKED +// BY TODAY'S EVIDENCE, and closing that gap is exactly what the per-group layout is needed for. +type PrintedLinesAgreement + = PrintedLinesAgree + | PrintedLinesDisagree { implied_hundredths: Nat, reported_hundredths: Nat } + | PrintedLinesUncheckable { cause: NonEmptyStr } + +// THE IMPLIED CONCURRENCY IS ROUNDED, NOT TRUNCATED, because the engine PRINTS a rounded figure. +// Truncating disagreed by one hundredth with any launch whose true concurrency rounds up: group A +// (11,767,856 tokens) truncates and rounds alike and passed, while group B (11,952,124) implied 1139 +// against a printed 1140 and was reported as garbled startup lines. The defect was mine, and running +// the instrument against the SECOND engine is what exposed it -- the first masked it by coincidence. +fn printed_lines_agree(layout: ResolvedKvLayout, reported: VllmReportedCapacity) -> PrintedLinesAgreement { + let context = token_count_value(t: layout.max_model_len) + let pool_tokens = token_count_value(t: reported.pool.tokens) + if context == 0 { + PrintedLinesUncheckable { cause: "the context cap is zero, so the concurrency the pool line encodes cannot be recovered" as NonEmptyStr } + } else { + let implied = ((pool_tokens * 100) + (context / 2)) / context + if implied == reported.max_concurrency_hundredths { + PrintedLinesAgree + } else { + PrintedLinesDisagree { implied_hundredths: implied, reported_hundredths: reported.max_concurrency_hundredths } + } + } +} + +// ── THE COST OF ONE ADMITTED SEQUENCE, WHICH NEEDS THE GROUPS AND SO REFUSES TODAY ───────────── +// +// A lease must charge what the request will actually hold, and that is a function of the WHOLE +// layout: each group contributes ceil(min(sequence, its window) / its block size) * its layer count, +// and only a full-attention group scales with the sequence all the way up. With `groups` empty +// there is no such function to evaluate, and the honest answer is a refusal naming what is missing. +// +// WHY NOT SCALE THE RECOVERED FULL-CONTEXT COST BY length / max_model_len. Because the windowed +// groups do not scale: halving the sequence does not halve their cost, so the scaled figure +// UNDERSTATES what a shorter request holds and would over-admit. An over-approximation that charged +// every request the full-context cost would be safe in the other direction, but it is not offered +// here either -- it belongs to the policy that decides how much conservatism to buy, not to the +// module that reports what the layout says. +type AdmissionCost + = AdmissionCostBlocks { blocks: Nat } + | AdmissionCostUnestablished { cause: NonEmptyStr } + +fn cold_admission_blocks(layout: ResolvedKvLayout, sequence_budget: TokenCount) -> AdmissionCost { + if length(layout.groups) == 0 { + AdmissionCostUnestablished { + cause: "the resolved cache groups are unobserved, so the per-request block cost cannot be computed from the layout; the engine's group metadata does not currently emit layer count, page size or its calculated blocks-per-max-request" as NonEmptyStr, + } + } else if token_count_value(t: sequence_budget) == 0 { + AdmissionCostUnestablished { cause: "an empty sequence budget names no admission" as NonEmptyStr } + } else if token_count_value(t: sequence_budget) > token_count_value(t: layout.max_model_len) { + AdmissionCostUnestablished { cause: "the sequence budget exceeds the engine's context cap, so no admission of it is representable" as NonEmptyStr } + } else { + AdmissionCostBlocks { blocks: fold(layout.groups, init: 0, f: (acc, g) => acc + group_blocks_for(group: g, sequence: sequence_budget)) } + } +} + +// PER-GROUP COST. The windowed kind is capped at its window, which is the whole reason a hybrid +// layout admits more than the full-attention arithmetic predicts; a mamba group is one state per +// sequence regardless of length. +fn group_blocks_for(group: KvGroupLayout, sequence: TokenCount) -> Nat { + match group.kind { + MambaState => group.layer_count + FullAttention => + ceil_div(numerator: token_count_value(t: sequence), denominator: group.block_size) * group.layer_count + SlidingWindowAttention { window: w } => + ceil_div(numerator: min_nat(a: token_count_value(t: sequence), b: token_count_value(t: w)), denominator: group.block_size) * group.layer_count + } +} + +// The denominator is a KvBlockSize, which cannot be zero, so this is total without a guard. +fn ceil_div(numerator: Nat, denominator: KvBlockSize) -> Nat { + (numerator + denominator - 1) / denominator +} + +fn min_nat(a: Nat, b: Nat) -> Nat { + if a < b { a } else { b } +} + +// ── THE ONE NON-CIRCULAR CHECK: THE GROUPS MUST REPRODUCE THE PRINTED CONCURRENCY ───────────── +// +// printed_lines_agree cannot see the inventory, because the inventory cancels out of it. THIS check +// can, and it is the reason the group layout is worth instrumenting at all: the per-group sum knows +// nothing about the printed lines, and the printed concurrency knows nothing about the groups, so +// agreement between them is evidence and disagreement is located -- either the layout was read from +// a different launch than the lines, or the group metadata is wrong, or the engine's arithmetic is +// not what this module models. +// +// IT IS UNAVAILABLE TODAY, and says so rather than passing. With `groups` empty there is nothing to +// push forward, so the check refuses; it does not read an absent second opinion as agreement. +type LayoutReproduction + = LayoutReproducesReported { blocks_per_max_request: Nat } + | LayoutContradictsReported { predicted_hundredths: Nat, reported_hundredths: Nat } + | LayoutReproductionUnavailable { cause: NonEmptyStr } + +// THE CHECK RUNS FORWARD, NOT AS AN EQUALITY BETWEEN TWO INVERSIONS. +// +// The first version compared the group-derived cost to a cost inverted out of the printed pool line +// and demanded they be EQUAL AS INTEGERS. That is a knife edge on rounded inputs: the pool figure is +// printed as a whole number of tokens after a float multiply, so the inversion lands a block or two +// away from the engine's real divisor and a perfectly correct layout FAILS. A wall a correct input +// cannot pass is not a strong wall; it is a broken one, and it would have been read as evidence that +// the group metadata was wrong. +// +// So the derived cost is pushed FORWARD through the engine's own last step and compared against the +// figure the engine printed, at the precision it printed it: concurrency = inventory / cost, to two +// decimal places. Both sides are integers the engine committed to, the inventory participates (so +// this is not the tautology printed_lines_agree turned out to be), and the tolerance is not invented +// — it is exactly the precision of the published figure. +// +// AND IT IS BOUND TO THE REVISION IT WAS ESTABLISHED AGAINST. The arithmetic modelled here is one +// vLLM revision's; applying it to a reading taken from another asserts a law about code nobody read, +// and the pool and concurrency lines are exactly the kind of derived output an upstream refactor +// moves. The reported capacity carries the revision of its own reading, so the mismatch is decidable +// and refuses rather than silently comparing across versions. +fn layout_reproduces_reported(layout: ResolvedKvLayout, reported: VllmReportedCapacity) -> LayoutReproduction { + if (reported.observed_at_revision.full_revision as String) != (vllm_source_revision_read.full_revision as String) { + LayoutReproductionUnavailable { + cause: join([ + "the reading was taken at vLLM revision ", reported.observed_at_revision.full_revision as String, + " but the capacity arithmetic modelled here was established against ", vllm_source_revision_read.full_revision as String, + "; the reproduction is not valid across revisions", + ], "") as NonEmptyStr, + } + } else { + match cold_admission_blocks(layout: layout, sequence_budget: layout.max_model_len) { + AdmissionCostUnestablished { cause: c } => LayoutReproductionUnavailable { cause: c } + AdmissionCostBlocks { blocks: derived } => + if derived == 0 { + LayoutReproductionUnavailable { cause: "the groups sum to no blocks for a full-context request, so no concurrency can be predicted from them" as NonEmptyStr } + } else { + let predicted = (layout.allocator_blocks * 100) / derived + if predicted == reported.max_concurrency_hundredths { + LayoutReproducesReported { blocks_per_max_request: derived } + } else { + LayoutContradictsReported { predicted_hundredths: predicted, reported_hundredths: reported.max_concurrency_hundredths } + } + } + } + } +} + +// ── THE USAGE GAUGE IS OVER THE INVENTORY, AND IT IS NOT OCCUPANCY ───────────────────────────── +// +// kv_cache_usage_perc is 1 - free_blocks / allocator_blocks, and a block holding a RETAINED +// PREFIX-CACHE entry is not free. So the gauge rises with cache retention on an engine serving +// nothing: gunbc.spark.pair_serving_observed recorded 96.5% at zero requests running, and the group +// B head read 53% with zero running on 2026-09-06. Any reading of this number as saturation, or any +// attempt to divide it into a per-request cost, is measuring the cache and calling it the load. +type AllocatorUsage sole_constructor { + referenced_or_retained_blocks: Nat + allocator_blocks: Nat +} + +// A FREE COUNT LARGER THAN THE INVENTORY IS NOT A FULL POOL, IT IS TWO ENGINES. The previous shape +// clamped it with a min, so an impossible reading reported "nothing free" -- the most alarming +// possible answer, arrived at by fabrication, and indistinguishable from a genuinely saturated +// engine. That is the absorbing fallback section 5 forbids: the widen destroys the only signal that +// the inventory and the free count came from different launches. +type UsageReading + = UsageRead { usage: AllocatorUsage } + | FreeCountImpossible { free_blocks: Nat, allocator_blocks: Nat } + +// There is deliberately NO function from AllocatorUsage to a request count or a token count. The +// numerator mixes live references with evictable retention, and nothing here can separate them. +// +// AND THAT RULING IS THE ABSENCE ITSELF, NOT A PREDICATE ANSWERING "no". This module carried +// `allocator_usage_is_occupancy(usage) -> Bool { false }`, which ignored its argument and returned a +// constant, and its only consumer asserted `!allocator_usage_is_occupancy(...)` -- `!false`, for +// every input, forever. DESIGN 4b: a check whose forbidden state cannot be expressed anywhere the +// check could run "is not a weak wall but a decoration -- permanently green by construction, +// carrying no information, and worse than absent because it will be cited as coverage." +// +// It was also the weaker of two available statements of one fact. "There is no such function" is +// enforced by construction: a caller that wants an occupancy ratio finds nothing to call and stops +// at resolve. A function that exists and answers `false` invites the caller to ask, then hands back +// an answer that reads as a measurement. The absence is the wall; the predicate was validation +// standing where construction was already in place. +fn allocator_usage(layout: ResolvedKvLayout, free_blocks: Nat) -> UsageReading { + if free_blocks > layout.allocator_blocks { + FreeCountImpossible { free_blocks: free_blocks, allocator_blocks: layout.allocator_blocks } + } else { + UsageRead { + usage: AllocatorUsage { + referenced_or_retained_blocks: layout.allocator_blocks - free_blocks, + allocator_blocks: layout.allocator_blocks, + }, + } + } +} diff --git a/dag/extdeps/vllm/log_lines.dag b/dag/extdeps/vllm/log_lines.dag index bc3002722ed..40e8892f543 100644 --- a/dag/extdeps/vllm/log_lines.dag +++ b/dag/extdeps/vllm/log_lines.dag @@ -71,13 +71,23 @@ type VllmStartupReading sole_constructor { observed_at_revision: VllmSourceRevision } +// THE REVISION IS OBSERVED, NOT STAMPED. This constructor used to fill observed_at_revision from +// vllm_source_revision_read, the constant naming the revision the LAWS were read from -- so every +// reading asserted it had been taken at the modelled revision no matter which build produced it, +// and the field the carrier's own note calls load-bearing could not disagree with the model. Every +// downstream revision comparison was therefore permanently green: not a weak wall, a decoration. +// +// A reading is evidence about the build that emitted it, so the caller that captured the text +// supplies the revision it captured it from. Agreeing with the modelled revision is then a real +// question a consumer can ask and get "no" to. fn vllm_startup_reading( line: VllmStartupLine, captured_text: NonEmptyStr, + observed_at_revision: VllmSourceRevision, ) -> VllmStartupReading { VllmStartupReading { line: line, captured_text: captured_text, - observed_at_revision: vllm_source_revision_read + observed_at_revision: observed_at_revision } } diff --git a/dag/extdeps/vllm/metrics.dag b/dag/extdeps/vllm/metrics.dag new file mode 100644 index 00000000000..09130df3ff1 --- /dev/null +++ b/dag/extdeps/vllm/metrics.dag @@ -0,0 +1,254 @@ +module extdeps.vllm.metrics + +import std.types { Bool, String, NonEmptyStr, List } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/vllm-project/vllm/blob/main/vllm/v1/metrics/stats.py" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.vllm.metrics", + decl_name: "VllmServingMetric", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// ── WHAT EACH SERVED METRIC ACTUALLY OBSERVES, AND WHICH QUESTIONS IT REFUSES ───────────────── +// +// WHY THIS MODULE EXISTS. These gauges are the only cheap window into a closed upstream, and a +// wrong reading of one is not catchable by the substrate: the arithmetic succeeds, the units look +// right, and the answer is about a different quantity. It surfaces weeks later as a prediction +// that failed in production against a number that looked authoritative on a dashboard. +// +// It has now cost this deployment three times in two days, and the third time TWO independent +// investigators reached the SAME wrong root cause from the same histogram in one afternoon. That +// is the tell that the metric's NAME was doing the reasoning. The failure mode is rostered at +// gunbc.recurring_failure_mode cumulative_metric_read_as_per_event. +// +// SO THE QUESTION IS THE SUBJECT, NOT THE METRIC. A metric alone cannot be right or wrong; a +// METRIC PAIRED WITH A QUESTION can. This module models that pairing and refuses the pairings the +// upstream source does not support, with the reason attached rather than left to a reader. +// +// AND IT IS REVISION-BOUND, because these observations are implementation details that upstream +// moves. The stronger caveat is that a self-reported version string is a CLAIM THE PROCESS MAKES +// ABOUT ITSELF and not executable provenance: the Spark image is a third-party build, and the +// readings below were confirmed by reading the source INSTALLED IN THAT IMAGE, not the upstream +// repository at the revision the binary names. + +type VllmServingMetric + = KvCacheUsagePerc + | IterationTokensTotal + | NumRequestsRunning + | NumRequestsWaiting + | PrefixCacheQueries + | PrefixCacheHits + | PromptTokensTotal + | GenerationTokensTotal + | NumPreemptionsTotal + +fn vllm_metric_name(m: VllmServingMetric) -> NonEmptyStr { + match m { + KvCacheUsagePerc => "vllm:kv_cache_usage_perc" as NonEmptyStr + IterationTokensTotal => "vllm:iteration_tokens_total" as NonEmptyStr + NumRequestsRunning => "vllm:num_requests_running" as NonEmptyStr + NumRequestsWaiting => "vllm:num_requests_waiting" as NonEmptyStr + PrefixCacheQueries => "vllm:prefix_cache_queries_total" as NonEmptyStr + PrefixCacheHits => "vllm:prefix_cache_hits_total" as NonEmptyStr + PromptTokensTotal => "vllm:prompt_tokens_total" as NonEmptyStr + GenerationTokensTotal => "vllm:generation_tokens_total" as NonEmptyStr + NumPreemptionsTotal => "vllm:num_preemptions_total" as NonEmptyStr + } +} + +// The questions a consumer in this repository has actually wanted to ask. Each is phrased as the +// DECISION it feeds, because that is what makes a wrong pairing expensive. +type ServingQuestion + = HowBusyIsThisReplica + | IsThisReplicaSaturated + | TokensComputedInOneEngineStep + | HowManyEngineStepsHaveRun + | HowLargeAreArrivingPrompts + | IsThePrefixCacheEffective + | IsTheEngineThrashing + +type MetricVerdict + = MetricAnswers { note: NonEmptyStr } + | MetricRefuses { because: NonEmptyStr } + +// ── THE PAIRINGS ────────────────────────────────────────────────────────────────────────────── +// +// A refusal here is not "we have not checked". It is a cited statement that the producer observes +// something else, and every refusal below was paid for by an incident. + +fn kv_usage_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + IsThePrefixCacheEffective => + MetricAnswers { note: "a warm engine sitting near 1.0 is the cache holding every otherwise-free block, which is the intended steady state" as NonEmptyStr } + HowBusyIsThisReplica => + MetricRefuses { because: "it counts every block that is NOT FREE, and a block holding an evictable cached prefix is not free; a completely idle replica reads ~0.99 once warm (measured 2026-09-07: 0.987 with zero running, zero waiting and byte-identical token counters across 30s)" as NonEmptyStr } + IsThisReplicaSaturated => + MetricRefuses { because: "same reading, and the consequence was measured: a load-aware router thresholding on it classified every replica as saturated and silently degraded to the load-blind path it was written to replace, while one head took all traffic and the other took none" as NonEmptyStr } + IsTheEngineThrashing => + MetricRefuses { because: "a full pool is the designed state, not eviction pressure; the interval prefix-cache hit rate answers this and the usage gauge does not" as NonEmptyStr } + TokensComputedInOneEngineStep => + MetricRefuses { because: "it is a ratio of blocks, not a token count" as NonEmptyStr } + HowManyEngineStepsHaveRun => + MetricRefuses { because: "it is an instantaneous ratio and carries no step history" as NonEmptyStr } + HowLargeAreArrivingPrompts => + MetricRefuses { because: "it is a ratio of blocks held, with no per-request grain" as NonEmptyStr } + } +} + +fn iteration_tokens_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + HowManyEngineStepsHaveRun => + MetricAnswers { note: "the histogram COUNT increments once per recorded engine iteration, and polling it live is how chunking was finally established" as NonEmptyStr } + HowLargeAreArrivingPrompts => + MetricAnswers { note: "each observation books one prefill's FULL prompt length, so the large buckets count large-prompt REQUESTS" as NonEmptyStr } + TokensComputedInOneEngineStep => + MetricRefuses { because: "the observation is prompt_token_stats.total plus generation tokens, and total accumulates each prefill's FULL prompt length booked in a single iteration -- not the chunk computed in that step. A 247,582-token observation was read as one monolithic engine step and produced a confident wrong root cause (a missing --max-num-batched-tokens); a live poll of the COUNT refuted it, climbing steadily through a cold prefill because chunking was working the whole time" as NonEmptyStr } + IsThisReplicaSaturated => + MetricRefuses { because: "large observations mean large prompts arrived, not that the engine is full" as NonEmptyStr } + HowBusyIsThisReplica => + MetricRefuses { because: "it is cumulative over the launch and says nothing about the present instant" as NonEmptyStr } + IsThePrefixCacheEffective => + MetricRefuses { because: "cached prompt tokens are inside the booked total, so a fully cached prompt still books its whole length here; the prefix-cache counters answer this" as NonEmptyStr } + IsTheEngineThrashing => + MetricRefuses { because: "it carries no eviction or retention information" as NonEmptyStr } + } +} + +fn running_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + HowBusyIsThisReplica => + MetricAnswers { note: "admitted and being served right now; this is the gauge that distinguishes an idle replica from a working one, and the one the router should have been ranking on" as NonEmptyStr } + IsThisReplicaSaturated => + MetricRefuses { because: "a busy replica is not a refusing one; num_requests_waiting is the fact that the engine is admitting nothing" as NonEmptyStr } + TokensComputedInOneEngineStep => MetricRefuses { because: "it counts requests, not tokens" as NonEmptyStr } + HowManyEngineStepsHaveRun => MetricRefuses { because: "it is an instantaneous count with no step history" as NonEmptyStr } + HowLargeAreArrivingPrompts => MetricRefuses { because: "it carries no prompt grain" as NonEmptyStr } + IsThePrefixCacheEffective => MetricRefuses { because: "it carries no cache information" as NonEmptyStr } + IsTheEngineThrashing => MetricRefuses { because: "it carries no eviction information" as NonEmptyStr } + } +} + +fn waiting_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + IsThisReplicaSaturated => + MetricAnswers { note: "a queue is a fact, not an estimate: the engine is admitting nothing further at this instant" as NonEmptyStr } + HowBusyIsThisReplica => + MetricRefuses { because: "zero waiting is consistent with both an idle replica and a fully loaded one that is keeping up; num_requests_running is the busyness gauge" as NonEmptyStr } + TokensComputedInOneEngineStep => MetricRefuses { because: "it counts requests, not tokens" as NonEmptyStr } + HowManyEngineStepsHaveRun => MetricRefuses { because: "it is instantaneous and carries no step history" as NonEmptyStr } + HowLargeAreArrivingPrompts => MetricRefuses { because: "it carries no prompt grain" as NonEmptyStr } + IsThePrefixCacheEffective => MetricRefuses { because: "it carries no cache information" as NonEmptyStr } + IsTheEngineThrashing => MetricRefuses { because: "a queue has many causes and eviction is not distinguishable here" as NonEmptyStr } + } +} + +fn cumulative_token_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + HowLargeAreArrivingPrompts => + MetricAnswers { note: "cumulative over the launch, so only its DELTA over a bounded interval is a statement about recent work" as NonEmptyStr } + TokensComputedInOneEngineStep => + MetricRefuses { because: "cumulative counters carry no per-step grain, and dividing a delta by a sampling window attributes a step's whole cost to the window in which it happened to complete -- which is how a ~30-second step was reported as 82,527 tokens per second" as NonEmptyStr } + HowBusyIsThisReplica => + MetricRefuses { because: "cumulative over the launch; the instantaneous gauges answer this" as NonEmptyStr } + IsThisReplicaSaturated => MetricRefuses { because: "cumulative over the launch" as NonEmptyStr } + HowManyEngineStepsHaveRun => MetricRefuses { because: "it counts tokens, not steps" as NonEmptyStr } + IsThePrefixCacheEffective => + MetricRefuses { because: "prompt_tokens_total includes cached tokens, so it overstates computed work; the prefix-cache counters answer this" as NonEmptyStr } + IsTheEngineThrashing => MetricRefuses { because: "it carries no eviction information" as NonEmptyStr } + } +} + +fn prefix_cache_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + IsThePrefixCacheEffective => + MetricAnswers { note: "hits over queries, and only over a bounded INTERVAL: the lifetime ratio is dominated by history and moves in hours" as NonEmptyStr } + IsTheEngineThrashing => + MetricAnswers { note: "a falling interval hit rate is the eviction-pressure signal the usage gauge cannot give" as NonEmptyStr } + TokensComputedInOneEngineStep => MetricRefuses { because: "these are token counters with no step grain" as NonEmptyStr } + HowManyEngineStepsHaveRun => MetricRefuses { because: "they count tokens, not steps" as NonEmptyStr } + HowBusyIsThisReplica => MetricRefuses { because: "cumulative over the launch" as NonEmptyStr } + IsThisReplicaSaturated => MetricRefuses { because: "cache effectiveness is orthogonal to admission" as NonEmptyStr } + HowLargeAreArrivingPrompts => MetricRefuses { because: "queries count tokens looked up, not prompts" as NonEmptyStr } + } +} + +fn preemptions_verdict(q: ServingQuestion) -> MetricVerdict { + match q { + IsTheEngineThrashing => + MetricAnswers { note: "a preemption is the engine evicting a RUNNING sequence, which is the unambiguous thrash signal; it has read zero for the lifetime of both Spark heads" as NonEmptyStr } + IsThisReplicaSaturated => + MetricAnswers { note: "cumulative, so only a rising delta is a present-tense claim" as NonEmptyStr } + TokensComputedInOneEngineStep => MetricRefuses { because: "it counts preemptions, not tokens" as NonEmptyStr } + HowManyEngineStepsHaveRun => MetricRefuses { because: "it counts preemptions, not steps" as NonEmptyStr } + HowBusyIsThisReplica => MetricRefuses { because: "an engine can be fully busy and never preempt" as NonEmptyStr } + HowLargeAreArrivingPrompts => MetricRefuses { because: "it carries no prompt grain" as NonEmptyStr } + IsThePrefixCacheEffective => MetricRefuses { because: "preemption is about running sequences, not cached prefixes" as NonEmptyStr } + } +} + +// THE ONE ENTRY POINT. A consumer names the metric AND the question, and gets an answer or a cited +// refusal. There is deliberately no function that takes a metric alone and returns "what it means": +// that shape is what let a name do the reasoning. +fn vllm_metric_answers(metric: VllmServingMetric, question: ServingQuestion) -> MetricVerdict { + match metric { + KvCacheUsagePerc => kv_usage_verdict(q: question) + IterationTokensTotal => iteration_tokens_verdict(q: question) + NumRequestsRunning => running_verdict(q: question) + NumRequestsWaiting => waiting_verdict(q: question) + PromptTokensTotal => cumulative_token_verdict(q: question) + GenerationTokensTotal => cumulative_token_verdict(q: question) + PrefixCacheQueries => prefix_cache_verdict(q: question) + PrefixCacheHits => prefix_cache_verdict(q: question) + NumPreemptionsTotal => preemptions_verdict(q: question) + } +} + +fn vllm_metric_is_admissible(metric: VllmServingMetric, question: ServingQuestion) -> Bool { + match vllm_metric_answers(metric: metric, question: question) { + MetricAnswers { note: _ } => true + MetricRefuses { because: _ } => false + } +} + +// ── THE OFF-METRIC READING THAT ALSO COST A DAY ─────────────────────────────────────────────── +// +// Not a vLLM metric, but the same mode and it belongs beside them: the ranks speak RoCE, and RoCE +// bytes never reach the Ethernet netdev counters. A 176-second prefill measured as ~0 GB of +// inter-node traffic on rx_bytes/tx_bytes; the same prefill moved ~175 GB per rank on the RoCE port +// counters. Reading the netdev counters concluded "the interconnect carries nothing", which is the +// opposite of true. +type InterconnectCounter + = EthernetNetdevBytes + | RocePortDataCounters + +fn interconnect_counter_sees_rdma(c: InterconnectCounter) -> Bool { + match c { + EthernetNetdevBytes => false + RocePortDataCounters => true + } +} + +fn interconnect_counter_note(c: InterconnectCounter) -> NonEmptyStr { + match c { + EthernetNetdevBytes => + "RDMA bypasses the netdev path, so these read ~0 while the link is carrying gigabytes; they are not an interconnect-utilisation source on a RoCE fabric" as NonEmptyStr + RocePortDataCounters => + "port_xmit_data and port_rcv_data, in 4-byte words, are the RoCE traffic; measured 2026-09-07 at ~7.9 Gb/s per rank on a 50 Gb/s link during a cold prefill. That is a BULK-TRANSFER reading and bounds bandwidth saturation only: a low average is equally consistent with latency-bound collectives, a straggler rank, or host gaps between kernels, so it does not establish what share of the critical path was communication" as NonEmptyStr + } +} diff --git a/dag/extdeps/vllm/scheduler_extension.dag b/dag/extdeps/vllm/scheduler_extension.dag new file mode 100644 index 00000000000..ef7207e9e90 --- /dev/null +++ b/dag/extdeps/vllm/scheduler_extension.dag @@ -0,0 +1,278 @@ +module extdeps.vllm.scheduler_extension + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } +import extdeps.vllm.engine_args { VllmDistributedExecutorBackend, ExecutorMp, ExecutorRay, ExecutorExternalLauncher } + +// VLLM'S SUPPORTED SCHEDULER EXTENSION POINT, AND THE OUTPUT FIELDS A WRAPPER MAY READ. +// +// WHY THIS MODULE IS SEPARATE FROM ITS THREE SIBLINGS. extdeps.vllm.server owns the served ROUTES, +// extdeps.vllm.engine_args owns what the engine is LAUNCHED with, extdeps.vllm.metrics owns what the +// engine REPORTS. None of them owns what upstream lets a third party SUBSTITUTE, which is a fourth +// kind of fact about the same product: an interface upstream publishes for replacement rather than a +// surface it serves. Folding it into engine_args would fuse an argument's spelling with the ABC that +// argument resolves to, and those move independently across revisions. +// +// WHAT THIS MODULE DELIBERATELY DOES NOT CARRY. It names no receipt, no allocation, no observation +// gap and no instrument. gunbc.spark.serving_step_receipt and gunbc.spark.serving_launch_instrumentation +// already own that vocabulary, and a second spelling of it here -- even a faithful one -- would be the +// meaning fork this programme is being sequenced to avoid. This module answers only: what does upstream +// publish, at which revision, and what breaks when that revision moves. + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/vllm-project/vllm/blob/1967a5627bc3710b680bbec24ecb99aaddedf22b/vllm/v1/core/sched/interface.py" + } +} + +data extdeps_scheduler_output_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/vllm-project/vllm/blob/1967a5627bc3710b680bbec24ecb99aaddedf22b/vllm/v1/core/sched/output.py" + } +} + +data extdeps_scheduler_config_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/vllm-project/vllm/blob/1967a5627bc3710b680bbec24ecb99aaddedf22b/vllm/config/scheduler.py" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.vllm.scheduler_extension", + decl_name: "VllmSchedulerExtensionPoint", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [extdeps_scheduler_output_authority, extdeps_scheduler_config_authority] +} + +// -- THE EXTENSION POINT IS A CONFIG FIELD, WHICH IS WHY THE BINDING IS FIXED AT START ---------- +// +// gunbc.spark.serving_launch_instrumentation rules that FirstPartyStepObserver binds only at process +// start, and reached that from an OBSERVATION: nothing loads a scheduler into a running engine. The +// upstream source says the same thing structurally, and the two agreeing is worth recording because +// they are independent -- one is a probe of a live host, the other is a read of the pinned source. +// +// SchedulerConfig.scheduler_cls is a plain configuration field, and SchedulerConfig.get_scheduler_cls +// resolves it -- by resolve_obj_by_qualname when it is a string, or by using it directly when it is +// already a type. The engine calls that resolver ONCE, when it constructs its scheduler. There is no +// setter, no route, and no reload path, so a launch that started without the field set has no +// mechanism by which a later request could install one. +type VllmSchedulerSelection + = SelectedByQualname { qualname: NonEmptyStr } + | SelectedByTypeObject { because: NonEmptyStr } + | SelectedByUpstreamDefault + +// -- THE DEFAULT IS CONDITIONAL, AND ASSUMING IT IS NOT IS A BEHAVIOUR-CHANGING BUG ----------- +// +// The observer's entire correctness claim is that it delegates to what would otherwise have run and +// returns that answer unchanged. Which class that IS depends on a second field: with scheduler_cls +// unset, get_scheduler_cls returns AsyncScheduler when async_scheduling is truthy and Scheduler +// otherwise. Setting scheduler_cls takes the branch away -- upstream resolves the named class and +// never consults async_scheduling again. +// +// So a wrapper that hardcodes Scheduler as its delegate does not merely observe an async-scheduled +// engine; it SILENTLY CONVERTS IT to a synchronously-scheduled one, changes throughput and latency, +// and reports the result as a behaviour-preserving observation. That is the exact shape the canary's +// matched control would be least likely to catch, because both arms would be equally converted if +// the wrapper were used on both. +type VllmAsyncSchedulingRequest = AsyncSchedulingOn | AsyncSchedulingOff | AsyncSchedulingUnset + +// UNSET DOES NOT MEAN OFF. IT MEANS "ON UNLESS SOMETHING INCOMPATIBLE", AND THE EXECUTOR DECIDES. +// +// This module previously mapped AsyncSchedulingUnset straight to Scheduler, which read the absent +// flag as a disabled feature. That is not what the pinned source does. VllmConfig post-init takes an +// `elif self.scheduler_config.async_scheduling is None` branch whose own comment reads "Enable async +// scheduling unless there is an incompatible option", and the final else sets it TRUE. +// +// One of the four disabling conditions is `not executor_supports_async_sched`, so which class an +// unset flag resolves to is a property OF THE DISTRIBUTED EXECUTOR BACKEND, not of the flag. Read +// from the running installation on the group A head: Executor.supports_async_scheduling defaults to +// False in v1/executor/abstract.py, MultiprocExecutor and UniProcExecutor override it to True, and +// NEITHER ray_executor.py NOR ray_executor_v2.py overrides it at all. +// +// WHY THE CORRECTION MATTERS MORE THAN THE ANSWER. On this deployment the two errors cancel: the +// backend is ray, so async resolves off and the delegate is Scheduler either way. Move the same +// launch to the mp backend -- which is what replacing Ray with a first-party rank protocol means -- +// and an unset flag silently becomes async ON, the delegate becomes AsyncScheduler, and a wrapper +// built on the old mapping would convert the engine while reporting that it preserved it. +type VllmExecutorAsyncSupport + = ExecutorSupportsAsyncScheduling { by: NonEmptyStr } + | ExecutorDoesNotSupportAsyncScheduling { because: NonEmptyStr } + +fn executor_async_scheduling_support(backend: VllmDistributedExecutorBackend) -> VllmExecutorAsyncSupport { + match backend { + ExecutorMp => + ExecutorSupportsAsyncScheduling { by: "vllm.v1.executor.multiproc_executor.MultiprocExecutor overrides supports_async_scheduling to return True" as NonEmptyStr } + ExecutorExternalLauncher => + ExecutorSupportsAsyncScheduling { by: "vllm.v1.executor.uniproc_executor.ExecutorWithExternalLauncher extends UniProcExecutor, which overrides supports_async_scheduling to return True" as NonEmptyStr } + ExecutorRay => + ExecutorDoesNotSupportAsyncScheduling { because: "neither ray_executor.py nor ray_executor_v2.py declares supports_async_scheduling, so the ray executors inherit Executor.supports_async_scheduling from v1/executor/abstract.py, which returns False" as NonEmptyStr } + } +} + +// The value the engine actually ends up with, which is what selects the delegate. +type ResolvedAsyncScheduling + = AsyncResolvedOn { because: NonEmptyStr } + | AsyncResolvedOff { because: NonEmptyStr } + +fn resolve_async_scheduling( + request: VllmAsyncSchedulingRequest, + backend: VllmDistributedExecutorBackend, +) -> ResolvedAsyncScheduling { + match request { + AsyncSchedulingOff => + AsyncResolvedOff { because: "the launch passed the flag off explicitly" as NonEmptyStr } + AsyncSchedulingOn => + AsyncResolvedOn { because: "the launch enabled async scheduling explicitly, which post-init hard-fails rather than downgrades when the executor cannot support it" as NonEmptyStr } + AsyncSchedulingUnset => + match executor_async_scheduling_support(backend: backend) { + ExecutorSupportsAsyncScheduling { by: b } => + AsyncResolvedOn { because: join(["the flag is unset, and post-init enables async scheduling unless an incompatible option applies -- ", b as String], "") as NonEmptyStr } + ExecutorDoesNotSupportAsyncScheduling { because: b } => + AsyncResolvedOff { because: join(["the flag is unset and post-init would have enabled async scheduling, but the executor cannot support it -- ", b as String], "") as NonEmptyStr } + } + } +} + +// THE DELEGATE IS A FUNCTION OF THE RESOLVED VALUE, never of the requested one. Taking the request +// here is what produced the defect this type now prevents. +fn vllm_default_scheduler_qualname(resolved: ResolvedAsyncScheduling) -> NonEmptyStr { + match resolved { + AsyncResolvedOn { because: _ } => "vllm.v1.core.sched.async_scheduler.AsyncScheduler" as NonEmptyStr + AsyncResolvedOff { because: _ } => "vllm.v1.core.sched.scheduler.Scheduler" as NonEmptyStr + } +} + +// -- UPSTREAM DISCLAIMS THIS INTERFACE, IN ITS OWN WORDS --------------------------------------- +// +// get_scheduler_cls logs, on every custom selection: "Using custom scheduler class %s. This +// scheduler interface is not public and compatibility may not be maintained." +// +// This is not a caveat to note and move past. It is what makes the revision pin the ONLY thing +// standing between this programme and silent drift: for a public interface a version range would be +// a reasonable compatibility statement, and here it is not, because upstream has reserved the right +// to change the shape without changing anything a range could observe. It is also why the extension +// point is nevertheless the right choice -- it is a published seam that fails loudly at construction, +// and the alternative is monkey-patching, which fails silently at any time. +type UpstreamStabilityPosture + = UpstreamPublicInterface { by: NonEmptyStr } + | UpstreamDisclaimsCompatibility { in_words: NonEmptyStr, consequence: NonEmptyStr } + +data vllm_scheduler_interface_posture: UpstreamStabilityPosture = UpstreamDisclaimsCompatibility { + in_words: "This scheduler interface is not public and compatibility may not be maintained" as NonEmptyStr, + consequence: "no version range is an admissible compatibility statement for this surface, so a binding is admitted against one exact revision and refuses against every other" as NonEmptyStr +} + +// The ABC a substituted scheduler must satisfy. Instantiating a class that misses an abstract method +// raises at construction, so an incomplete wrapper fails the launch rather than serving degraded -- +// which is the fail-closed shape we want and did not have to build. +data vllm_scheduler_interface_qualname: NonEmptyStr = "vllm.v1.core.sched.interface.SchedulerInterface" as NonEmptyStr + +type VllmSchedulerExtensionPoint sole_constructor { + config_field: NonEmptyStr + resolver: NonEmptyStr + interface: NonEmptyStr + established_against: VllmSourceRevision +} + +data vllm_scheduler_extension_point: VllmSchedulerExtensionPoint = VllmSchedulerExtensionPoint { + config_field: "vllm.config.scheduler.SchedulerConfig.scheduler_cls" as NonEmptyStr, + resolver: "vllm.config.scheduler.SchedulerConfig.get_scheduler_cls" as NonEmptyStr, + interface: vllm_scheduler_interface_qualname, + established_against: vllm_source_revision_read +} + +// WHETHER A SELECTION CAN BE MADE AFTER THE ENGINE IS RUNNING. This function exists to be the +// upstream-sourced half of a claim the live-host probe already made, so that a future revision which +// added a reload path would red HERE rather than silently making the instrumentation law stale. +type SchedulerSelectionTiming + = SelectableWhileRunning { by: NonEmptyStr } + | SelectableOnlyAtEngineConstruction { because: NonEmptyStr } + +fn scheduler_selection_timing(p: VllmSchedulerExtensionPoint) -> SchedulerSelectionTiming { + SelectableOnlyAtEngineConstruction { + because: join(["the class is resolved by ", p.resolver as String, " when the engine core constructs its scheduler, and the pinned revision publishes no setter, route or reload path that would resolve it a second time"], "") as NonEmptyStr + } +} + +// -- WHAT ONE SCHEDULE CALL RETURNS, LIMITED TO THE FIELDS AN OBSERVER READS ------------------- +// +// This is NOT an inventory of SchedulerOutput. It is the closed set of fields the first-party +// observer depends on, which is the set whose disappearance must break a build rather than quietly +// produce an emptier receipt. A field vLLM carries that we do not read is upstream's business. +// +// THE LOAD-BEARING ONE IS num_scheduled_tokens. It is a dict from request id to token count -- the +// per-request-per-step fact gunbc.spark.serving_step_receipt was authored around, and the fact +// gunbc.recurring_failure_mode cumulative_metric_read_as_per_event says no engine-wide counter can +// reconstruct. It is not derived by the observer; the scheduler already computed it to do its job. +type VllmSchedulerOutputField + = NumScheduledTokens + | TotalNumScheduledTokens + | ScheduledNewReqs + | ScheduledCachedReqs + +fn scheduler_output_field_attribute(f: VllmSchedulerOutputField) -> NonEmptyStr { + match f { + NumScheduledTokens => "num_scheduled_tokens" as NonEmptyStr + TotalNumScheduledTokens => "total_num_scheduled_tokens" as NonEmptyStr + ScheduledNewReqs => "scheduled_new_reqs" as NonEmptyStr + ScheduledCachedReqs => "scheduled_cached_reqs" as NonEmptyStr + } +} + +// The declared Python type at the pinned revision. A wrapper that reads a field whose SHAPE changed +// is a worse failure than one that reads a field which vanished, because the vanished one raises and +// the reshaped one silently produces a wrong receipt. So the shape is recorded and not only the name. +fn scheduler_output_field_shape(f: VllmSchedulerOutputField) -> NonEmptyStr { + match f { + NumScheduledTokens => "dict[str, int] keyed by request id" as NonEmptyStr + TotalNumScheduledTokens => "int, documented upstream as equal to the sum of num_scheduled_tokens values" as NonEmptyStr + ScheduledNewReqs => "list[NewRequestData], each carrying req_id and num_computed_tokens" as NonEmptyStr + ScheduledCachedReqs => "CachedRequestData carrying parallel lists req_ids and num_computed_tokens" as NonEmptyStr + } +} + +fn scheduler_output_fields_read() -> List { + [NumScheduledTokens, TotalNumScheduledTokens, ScheduledNewReqs, ScheduledCachedReqs] +} + +// -- THE COUNTS THAT CLOSE A STEP -------------------------------------------------------------- +// +// get_request_counts returns (num_running_reqs, num_waiting_reqs) and is on the ABC, so it is +// available to a wrapper through its delegate rather than by reaching into the concrete scheduler's +// private queues. Reaching into those queues instead is the shape that makes a wrapper revision +// fragile, so the published method is named here to make the fragile alternative a visible departure. +data vllm_scheduler_request_counts_method: NonEmptyStr = "get_request_counts" as NonEmptyStr + +// -- BINDING OUTCOME --------------------------------------------------------------------------- +// +// A wrapper built against this module either finds every field it declared, or refuses by naming the +// one it did not. There is no arm that proceeds with a smaller set: an observer that silently omitted +// cached-token accounting would emit receipts that fold to a confident wrong number, which is exactly +// the class the receipt contract exists to close. +type SchedulerSurfaceBinding + = SurfaceBound { revision: VllmSourceRevision, fields: Nat } + | SurfaceRefusedForMissingField { attribute: NonEmptyStr, because: NonEmptyStr } + | SurfaceRefusedForRevisionDrift { observed: NonEmptyStr, because: NonEmptyStr } + +fn surface_binding_for_revision(observed: VllmSourceRevision) -> SchedulerSurfaceBinding { + if observed.full_revision == vllm_source_revision_read.full_revision { + SurfaceBound { revision: vllm_source_revision_read, fields: length(scheduler_output_fields_read()) } + } else { + SurfaceRefusedForRevisionDrift { + observed: observed.full_revision, + because: "every field name and shape in this module was read from one pinned revision, and a different revision may carry the same attribute names with different meanings, which is unobservable at runtime and produces a receipt that folds to a wrong answer" as NonEmptyStr + } + } +} diff --git a/dag/gunbc/guarantee_stall/probe_program_assembled_from_string_literals_stall.dag b/dag/gunbc/guarantee_stall/probe_program_assembled_from_string_literals_stall.dag new file mode 100644 index 00000000000..b660b85c9b3 --- /dev/null +++ b/dag/gunbc/guarantee_stall/probe_program_assembled_from_string_literals_stall.dag @@ -0,0 +1,36 @@ +module gunbc.guarantee_stall.probe_program_assembled_from_string_literals_stall + +import gunbc.guarantee_rung { Mitigatable, StructurallyGuaranteed } +import gunbc.guarantee_stall { GuaranteeStall, AwaitsOneGrounding, BoundedPopulation, climbs_when } + +// gunbc.instruments.fabric_capacity_standing capacity_probe_script builds an eleven-field bash +// program by joining String literals -- command substitutions, pipelines, sed and grep -oE scripts, +// and a per-field default -- and group_report ships it to `ssh ... bash -s` as a stdin payload. +// +// THE DEFECT IS NOT THAT IT IS SHELL. docs/plans/shell-to-dag-residual-census-and-arc-completion.md +// splits the remaining arc in two, and a probe executed on a foreign host over SSH falls in its +// LEGITIMATE SHELL bucket: foreign executors stay shell. But that same finding says legitimate shell +// is "emitted through the v2 bash rows (grammar-owned), and is bounded", and this program is neither +// -- it is assembled by string concatenation, so no grammar row describes it and nothing in the +// substrate can read it. ROADMAP's translator row names exactly this population as what it deletes: +// "the scripts assembled by gluing strings together". +// +// WHY IT IS A LADDER ROW AND NOT ONLY UNTIDINESS. Every field this probe returns is an input to a +// capacity standing, and the program that produces them is opaque to the compiler: a quoting defect, +// a pipeline that silently yields empty, or a sed expression that stops matching after an upstream +// format change all reduce to the field's `-` default. The instrument then reports an unestablished +// standing rather than a wrong number, which is the correct failure -- so this sits at mitigatable +// rather than lower -- but the class of defect is invisible to every check the substrate can run, +// and it stays invisible until the program has a grammar. +// +// It is declared here because the census that owns the class does not name this instance: the +// document is dated 2026-07-12 against #6507 and predates this module, so without a row the +// instance is an untracked member of a known population. +data probe_program_assembled_from_string_literals_stall: GuaranteeStall = GuaranteeStall { + subject: "gunbc.instruments.fabric_capacity_standing capacity_probe_script assembles an eleven-field bash program from String literals and ships it to ssh bash -s, so no grammar row describes the program and no substrate check can read it", + current: Mitigatable, + ceiling: StructurallyGuaranteed, + blocker: AwaitsOneGrounding { grounding: "docs/plans/shell-to-dag-residual-census-and-arc-completion.md" }, + population: BoundedPopulation { members: ["gunbc.instruments.fabric_capacity_standing"] }, + next_rung_trigger: climbs_when(capability: "one translator that turns a described plan into a shell program while preserving order, captured output and failure behaviour, sufficient to express this probe's command substitutions, pipelines, per-field default and remote stdin delivery, so that the program is grammar-owned and no consumer assembles one by joining String literals") +} diff --git a/dag/gunbc/guarantee_stall/roster.dag b/dag/gunbc/guarantee_stall/roster.dag index 14e57e6bfe8..063a1a02991 100644 --- a/dag/gunbc/guarantee_stall/roster.dag +++ b/dag/gunbc/guarantee_stall/roster.dag @@ -59,6 +59,8 @@ import gunbc.guarantee_stall.testgen_anchor_generator_totality_stall { testgen_a import gunbc.guarantee_stall.synthesis_established_bound_patch_stall { synthesis_established_bound_patch_stall } import gunbc.guarantee_stall.fn_index_depth_agreement_floor_enrollment_stall { fn_index_depth_agreement_floor_enrollment_stall } import gunbc.guarantee_stall.eval_steps_outside_the_reading_coproduct_stall { eval_steps_outside_the_reading_coproduct_stall } +import gunbc.guarantee_stall.signed_and_ratio_measure_carrier_stall { signed_and_ratio_measure_carrier_stall } +import gunbc.guarantee_stall.probe_program_assembled_from_string_literals_stall { probe_program_assembled_from_string_literals_stall } // THE COHORT PROVENANCE NOTES BELOW WERE AUTHORED WHEN EVERY ROW SAT IN ONE FILE, and they are // carried here verbatim rather than split across the row modules they describe. Their membership @@ -213,6 +215,8 @@ data all_guarantee_stalls: List = [ synthesis_established_bound_patch_stall, fn_index_depth_agreement_floor_enrollment_stall, eval_steps_outside_the_reading_coproduct_stall, + signed_and_ratio_measure_carrier_stall, + probe_program_assembled_from_string_literals_stall, ] // THE FOUR SUBJECTS RESTORED TO THE ROSTER, PINNED BY IDENTITY SO THEIR LOSS REFUSES. diff --git a/dag/gunbc/guarantee_stall/signed_and_ratio_measure_carrier_stall.dag b/dag/gunbc/guarantee_stall/signed_and_ratio_measure_carrier_stall.dag new file mode 100644 index 00000000000..2cd20de7a86 --- /dev/null +++ b/dag/gunbc/guarantee_stall/signed_and_ratio_measure_carrier_stall.dag @@ -0,0 +1,53 @@ +module gunbc.guarantee_stall.signed_and_ratio_measure_carrier_stall + +import gunbc.guarantee_rung { Mitigatable, StructurallyGuaranteed } +import gunbc.guarantee_stall { GuaranteeStall, AwaitsOneGrounding, BoundedPopulation, climbs_when } + +// WHY THIS ROW EXISTS RATHER THAN A FOURTH REVIEW ROUND. Three reviews in succession flagged the +// unit-modeling class across gunbc.spark.serving_critical_path. Most of it was a real fork and is +// now migrated: started_micros, elapsed_micros, four inter-iteration millis, wall_micros across +// twenty-four sample rows, and two watt fields all consume std.measure. +// +// Three fields did not migrate, and leaving them as bare Int with no row is precisely what the +// third review objected to -- not the Int, but the SILENCE. DESIGN 4b(2) says a class below its +// ceiling must name its next-rung trigger, and 4b(3) says a rung may be lowered only by declaring +// it. An undeclared Int beside a migrated Microsecond in the same record is an unstated divergence. +// +// WHAT ACTUALLY BLOCKS EACH ONE. Three different carrier gaps, not one: +// +// intercept_micros a fit intercept, computed by division, may be negative +// intercept_difference_micros a difference of two intercepts, signed by construction +// micros_per_thousand_tokens a RATE: microseconds per token +// milli_tokens_per_second a rate at MILLI scale, in gunbc.spark.serving_step_cost +// gigabits_per_second_tenths a link throughput reading +// link_gigabits the link's rated throughput +// +// SIGNED. Microsecond is Measure, Nat-backed, so it cannot hold either of the two +// signed values, and std.measure declares no signed duration alias. +// +// QUOTIENT. micros_per_thousand_tokens is a dimensional quotient. std.measure publishes +// measure_add, measure_le, measure_count and two scale-fraction folds -- no product, no quotient, +// and no composition on the Quantity type -- so a microseconds-per-token carrier cannot be spelled. +// +// SCALE. milli_tokens_per_second is a rate at milli scale. TokensPerSecond is Measure, and its module states the reason the milli scaling exists: rounding a sub-unit rate to +// an integer would report zero. Migrating to the One-scaled alias would erase the distinction the +// field was created to preserve, so this needs Measure and no alias exists. +// +// AND ONE IS BLOCKED BY A FORK RATHER THAN AN ABSENCE, which is the worst of the four. The two RoCE +// fields would take std.measure Bandwidth = Measure, and std.measure's own +// comment beside MegatransfersPerSecond declares DataRate to be bits/s. Its live consumers do not +// agree with that or with each other: extdeps.systems.nvidia records the GB10's 273 GB/s memory +// bandwidth as bandwidth(count: 273000000000), which is bytes per second, while extdeps.colo +// natcoweb records colo transit ports as bandwidth(count: 1000000) and bandwidth(count: 10000000), +// which are neither. Three consumers, at least two mutually inconsistent readings of one carrier's +// base unit. Joining that carrier would make these rows a THIRD reading and cement the fork, so they +// stay declared here until the base unit has one answer. +data signed_and_ratio_measure_carrier_stall: GuaranteeStall = GuaranteeStall { + subject: "unit fields across gunbc.spark.serving_critical_path and gunbc.spark.serving_step_cost carry bare Nat or Int beside migrated Watt, Hertz, Percent, Milliwatt and Microsecond fields in the same records, because std.measure has no signed duration alias, no dimensional quotient, no milli-scaled frequency alias, and no agreed base unit for Bandwidth", + current: Mitigatable, + ceiling: StructurallyGuaranteed, + blocker: AwaitsOneGrounding { grounding: "std.measure" }, + population: BoundedPopulation { members: ["gunbc.spark.serving_critical_path", "gunbc.spark.serving_step_cost"] }, + next_rung_trigger: climbs_when(capability: "std.measure carries a signed duration magnitude, a dimensional quotient with the arithmetic to inhabit it, a milli-scaled frequency alias, and one adjudicated base unit for DataRate that every live Bandwidth consumer is migrated onto, sufficient to express a fit intercept computed by division, a signed difference of two intercepts, a microseconds-per-token rate, a sub-unit token rate, and a link throughput without any consumer unwrapping to a bare scalar to do so") +} diff --git a/dag/gunbc/instruments/fabric_capacity_standing.dag b/dag/gunbc/instruments/fabric_capacity_standing.dag new file mode 100644 index 00000000000..960fab4e8c1 --- /dev/null +++ b/dag/gunbc/instruments/fabric_capacity_standing.dag @@ -0,0 +1,372 @@ +module gunbc.instruments.fabric_capacity_standing + +import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.algebra { trim } +import std.measure { TokenCount, token_count } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.content_hash { ContentHash, Sha256Hash, Sha256Digest, Sha256DigestHex, sha256_hex_digest, Fnv1a64, content_hash_atom } +import extdeps.ssh.session +import product.placement_supply { HostIdentity } +import gunbc.fleet_intent_network { host_lan_address } +import gunbc.spark.fabric_switch_observed { FabricGroup, fabric_group_wire } +import gunbc.spark.pair_serving_desired { spark_pair_serving_groups, spark_pair_serving_head } +import gunbc.spark.managed_access_apply { spark_managed_login } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch, vllm_serving_launch } +import extdeps.vllm.server { VllmSourceRevision } +import extdeps.vllm.log_lines { vllm_startup_reading, GpuKvCacheSize } +import extdeps.vllm.kv_cache { VllmObservedKvPool, VllmKvPoolAdmission, KvPoolAdmitted, KvPoolRefused, vllm_observed_kv_pool_from_reading } +import extdeps.vllm.kv_layout { ResolvedKvLayout, resolved_kv_layout, VllmReportedCapacity, vllm_reported_capacity } +import gunbc.spark.vllm_kv_layout_observe { + VllmKvLayoutReceipt, KvLayoutObserved, KvLayoutUnreadable, KvLayoutInconsistent, admit_kv_layout, receipt_wire, +} +import gunbc.spark.pair_serving_capacity_floor { + spark_pair_serving_minimum_cold_sessions, + CapacityFloorStanding, MeetsFloor, BelowFloor, FloorUnestablished, capacity_floor_standing, + AdmissionPosture, Admitting, Withdrawn, admission_posture, +} + +// ── WHAT THIS DEPLOYMENT'S CAPACITY ACTUALLY IS, ASKED OF THE LIVE ENGINES ──────────────────── +// +// Every capacity claim in this repository has so far been green by typecheck. This asks the fleet. +// +// gunbc run --source-root dag --source-root src/v2 \ +// --entry dag/gunbc/instruments/fabric_capacity_standing.dag --function fabric_capacity_standing +// +// WHAT IT IS FOR. The expected answer today is that capacity is UNESTABLISHED on every group, +// because the engine emits no per-group cache metadata and the receipt therefore cannot reproduce +// its own printed arithmetic. That is the point rather than a disappointment: knowing mechanically +// that we cannot establish capacity is worth more than reading a dashboard number that says 99% and +// means something else. The instrument earns its keep the day the answer changes without anyone +// intending it to. +// +// ONE ROUND TRIP PER GROUP, and the far side does the text handling. The head curls its OWN metrics +// endpoint, greps its own launch log and normalises everything into fixed-position lines, so this +// module parses integers at known offsets instead of carrying a Prometheus parser and a log parser +// it would then have to keep in step with two upstream formats. A field the far side cannot produce +// is a literal "-", which fails to parse here and refuses -- it never arrives as a zero. + +data capacity_probe_unit: String = "gunbc-spark-pair-head" +data capacity_probe_log: String = "/var/log/gunbc/spark-pair-head.log" +data capacity_probe_identity_file: String = "/home/briansrls/.ssh/fleet-automation" +// THE PRINCIPAL IS THE DECLARED ONE, READ FROM ITS AUTHORITY. +// +// This said "briansrls" — a human login that happens to exist on the group A head and not on the +// group B head, so the first run of this instrument reported group B as unreachable and I wrote it +// up as a fleet key-distribution gap. It was not. gunbc.spark.managed_access_apply already declares +// gunbc-automation as the convergence principal, managed_access_apply provisions it, and both heads +// answer to it on the fleet key. The gap was in this file: a second spelling of "who automation is", +// which read as evidence about the fleet. +// +// So it is not spelled here at all. A probe that names its own principal can disagree with the one +// convergence uses, and the disagreement looks exactly like a broken host. + +// ELEVEN LINES, FIXED ORDER, "-" FOR ANYTHING THIS HOST CANNOT ANSWER. +// +// The two launch figures are taken from the LAST matching line rather than the first: the log +// accumulates across restarts, and the engine serving now is the one that printed last. +// +// NO BRACE APPEARS IN THIS SCRIPT, AND THE REASON RECORDED HERE FOR THAT WAS FALSE. +// +// This block used to state that the heads-only pass indexing modules counts braces without knowing +// it is inside a string literal, so a `--format` template ended the enclosing declaration early and +// the module refused to parse. That is not what the reader does. `heads_skip_block_tokens` walks +// TOKENS and matches on `is_lbrace_shape(t.shape)`; a brace inside a string literal belongs to a +// String token whose shape is not LBrace, so it is never counted. The corpus agrees by execution: +// 691 .dag files carry a brace inside a string literal, `std/types.dag` among them -- a module +// every closure imports and the heads census therefore always parses. +// +// The shape of the script is left as it is rather than restored to `--format` templates, because +// whether this probe survives at all is an open question and rewriting its shell would be the wrong +// investment while that is undecided. What is corrected is the CLAIM: no author should route around +// a language-layer deficit that does not exist, and DESIGN 4c forbids an annotation standing as +// evidence for a machine fact -- which is exactly what this one did, wrongly, for four commits. +fn emit_field(command: String) -> String { + join(["v=$(", command, " 2>/dev/null); [ -n \"$v\" ] || v=-; printf '%s\\n' \"$v\"\n"], "") +} + +fn capacity_probe_script() -> String { + join(concat([ + join(["set -u\nu=", capacity_probe_unit, "\nlog=", capacity_probe_log, "\n"], ""), + "m=$(curl -s --max-time 8 http://127.0.0.1:30000/metrics)\n", + ], map([ + "systemctl show $u -p InvocationID --value", + "systemctl cat $u | sha256sum | cut -d' ' -f1", + "systemctl cat $u | grep -oE 'sha256:[0-9a-f]{64}' | head -1 | cut -d: -f2", + "date -d \"$(systemctl show $u -p ExecMainStartTimestamp --value)\" +%s", + "grep -a 'GPU KV cache size' \"$log\" | tail -1 | sed 's/.*size: //; s/ tokens.*//; s/,//g'", + "grep -a 'Maximum concurrency' \"$log\" | tail -1 | sed 's/.*request: //; s/x.*//; s/[.]//'", + "printf '%s' \"$m\" | grep -o 'num_gpu_blocks=\"[0-9]*\"' | head -1 | tr -dc '0-9'", + "printf '%s' \"$m\" | grep -o 'block_size=\"[0-9]*\"' | head -1 | tr -dc '0-9'", + "curl -s --max-time 8 http://127.0.0.1:30000/v1/models | grep -o 'max_model_len\":[0-9]*' | tr -dc '0-9'", + "printf '%s' \"$m\" | grep '^vllm:num_requests_running' | head -1 | sed 's/.*. //; s/[.].*//'", + "printf '%s' \"$m\" | grep '^vllm:num_requests_waiting.engine' | head -1 | sed 's/.*. //; s/[.].*//'", + ], c => emit_field(command: c))), "") +} + +type CapacityProbeFields { + invocation: String + unit_sha: String + image_sha: String + started_at: Int + pool_tokens: Int + concurrency_hundredths: Int + allocator_blocks: Int + block_size: Int + max_model_len: Int + running: Int + waiting: Int +} + +type CapacityProbeRead + = ProbeFields { fields: CapacityProbeFields } + | ProbeUnreadable { detail: String } + +fn field_at(lines: List, n: Nat) -> String { + trim(s: lines.skip(n: n).first()) +} + +fn int_at(lines: List, n: Nat) -> Int? { + parse_int(s: field_at(lines: lines, n: n)) +} + +fn decode_probe(stdout: String) -> CapacityProbeRead { + let lines = split(s: trim(s: stdout), delimiter: "\n") + if list_length(items: lines) < 11 { + ProbeUnreadable { detail: join(["the probe returned ", to_string(list_length(items: lines)), " lines, not the 11 the script emits"], "") } + } else { + match int_at(lines: lines, n: 3) { + Absent => ProbeUnreadable { detail: "the unit start instant did not parse as epoch seconds" } + Present { value: started } => + match int_at(lines: lines, n: 4) { + Absent => ProbeUnreadable { detail: "the engine's GPU KV cache size line was not found in the launch log" } + Present { value: pool } => + match int_at(lines: lines, n: 5) { + Absent => ProbeUnreadable { detail: "the engine's maximum concurrency line was not found in the launch log" } + Present { value: conc } => + match int_at(lines: lines, n: 6) { + Absent => ProbeUnreadable { detail: "num_gpu_blocks was not present in the engine's cache-config metric" } + Present { value: blocks } => + match int_at(lines: lines, n: 7) { + Absent => ProbeUnreadable { detail: "block_size was not present in the engine's cache-config metric" } + Present { value: bs } => + match int_at(lines: lines, n: 8) { + Absent => ProbeUnreadable { detail: "max_model_len was not readable from /v1/models" } + Present { value: ctx } => + match int_at(lines: lines, n: 9) { + Absent => ProbeUnreadable { detail: "vllm:num_requests_running was not present in the engine's metrics" } + Present { value: run } => + match int_at(lines: lines, n: 10) { + Absent => ProbeUnreadable { detail: "vllm:num_requests_waiting was not present in the engine's metrics" } + Present { value: wait } => + ProbeFields { fields: CapacityProbeFields { + invocation: field_at(lines: lines, n: 0), + unit_sha: field_at(lines: lines, n: 1), + image_sha: field_at(lines: lines, n: 2), + started_at: started, + pool_tokens: pool, + concurrency_hundredths: conc, + allocator_blocks: blocks, + block_size: bs, + max_model_len: ctx, + running: run, + waiting: wait, + } } + } + } + } + } + } + } + } + } + } +} + +// ── THE LAUNCH IDENTITY, WHICH REFUSES RATHER THAN SUBSTITUTING ─────────────────────────────── +// +// A digest that does not parse as 64 lower-hex is not a digest, and an incarnation assembled around +// a placeholder would join to anything. There is no fallback arm here on purpose. +type LaunchRead + = LaunchIdentified { launch: VllmServingLaunch } + | LaunchUnidentified { detail: String } + +fn launch_from_fields(group: FabricGroup, head: HostIdentity, f: CapacityProbeFields) -> LaunchRead { + match sha256_hex_digest(hex: f.image_sha) { + Absent => LaunchUnidentified { detail: join(["the unit-pinned image digest did not read as a sha256: ", f.image_sha], "") } + Present { value: image } => + match sha256_hex_digest(hex: f.unit_sha) { + Absent => LaunchUnidentified { detail: join(["the unit text digest did not read as a sha256: ", f.unit_sha], "") } + Present { value: unit } => + if f.invocation == "-" { + LaunchUnidentified { detail: "the systemd invocation was unavailable, so this launch cannot be named" } + } else { + LaunchIdentified { launch: vllm_serving_launch( + group: group, + head: head, + systemd_invocation: f.invocation as NonEmptyStr, + unit_pinned_base_image_digest: Sha256Hash(image), + unit_digest: Sha256Hash(unit), + started_at: f.started_at, + ) } + } + } + } +} + +// THE REVISION IS THE PROBE'S, NOT THE MODEL'S, AND THIS PROBE DOES NOT CAPTURE ONE. +// +// vllm_startup_reading stopped stamping vllm_source_revision_read for a reason stated at its own +// definition: a reading that asserts it was taken at the modelled revision makes every downstream +// revision comparison permanently green. This instrument was that constructor's first caller, and +// passing the same constant back in reinstated the decoration one module over -- with more +// consequence, not less, because this pool reaches admit_kv_layout, whose refusal on +// reported.observed_at_revision is the wall, and the wall then reaches `admitting`. +// +// So the revision arrives as a parameter and its absence is a REFUSAL rather than a substitution. +// The eleven-field probe reads systemd and two localhost endpoints; none of them answer which vLLM +// SOURCE the engine was built from, and the engine's own metadata does not carry it. That is a +// missing observation, not a value to invent -- exactly the case the module header already rules on +// for every other field. Capturing it is a twelfth field, and until the probe has one this arm is +// the honest answer. +type PoolRead + = PoolObserved { pool: VllmObservedKvPool } + | PoolUnobserved { detail: String } + +fn pool_from_fields(f: CapacityProbeFields, observed_revision: VllmSourceRevision?) -> PoolRead { + match observed_revision { + Absent => PoolUnobserved { detail: "the probe does not capture the vLLM source revision the engine was built from, so a KV-cache reading taken from it cannot be admitted against the modelled revision" } + Present { value: revision } => + match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: f.pool_tokens), + reading: vllm_startup_reading( + line: GpuKvCacheSize, + captured_text: join(["GPU KV cache size: ", to_string(f.pool_tokens), " tokens"], "") as NonEmptyStr, + observed_at_revision: revision, + ), + ) { + KvPoolAdmitted { pool: p } => PoolObserved { pool: p } + KvPoolRefused { cause: _ } => PoolUnobserved { detail: "the captured KV-cache-size reading was refused by its own carrier" } + } + } +} + +// GROUPS ARE EMPTY, AND THAT IS THE OBSERVATION. The engine's metadata does not carry layer count +// or page size, so there is nothing to put here and the receipt will refuse for exactly that +// reason. Filling it with a guess is the one move that would make this instrument lie. +fn layout_from_fields(f: CapacityProbeFields) -> ResolvedKvLayout { + resolved_kv_layout( + allocator_blocks: f.allocator_blocks, + scheduler_block_size: token_count(count: f.block_size), + hash_block_size: token_count(count: f.block_size), + max_model_len: token_count(count: f.max_model_len), + max_num_batched_tokens: token_count(count: 0), + groups: [], + ) +} + +fn standing_wire(s: CapacityFloorStanding) -> String { + match s { + MeetsFloor { required_blocks: r, available_blocks: a } => + join(["meets floor (", to_string(r), " required of ", to_string(a), " blocks)"], "") + BelowFloor { required_blocks: r, available_blocks: a } => + join(["BELOW FLOOR (", to_string(r), " required, ", to_string(a), " available)"], "") + FloorUnestablished { cause: c } => join(["UNESTABLISHED: ", c as String], "") + } +} + +fn posture_wire(p: AdmissionPosture) -> String { + match p { + Admitting { available_blocks: a } => join(["admitting (", to_string(a), " blocks)"], "") + Withdrawn { cause: c } => join(["WITHDRAWN: ", c as String], "") + } +} + +// ONE PROBE PER GROUP, AND THE VERDICT TRAVELS WITH THE TEXT. +// +// The first cut returned a String and decided admittance by searching it for "posture: admitting". +// That was wrong twice over: it re-ran group_report to ask the question, so every group was probed +// over ssh TWICE per invocation — a duplicated side effect, visible in the run as two identical +// remote executions — and it recovered a decision from prose that the fold already knew. The +// standing carries both, so the effect happens once and the verdict is read rather than parsed. +type GroupStanding { + text: String + admitting: Bool +} + +fn refused_standing(text: String) -> GroupStanding { + GroupStanding { text: text, admitting: false } +} + +fn group_report(group: FabricGroup) -> GroupStanding { + let name = fabric_group_wire(g: group) as String + let head = spark_pair_serving_head(group: group) + match host_lan_address(host: head) { + Absent => refused_standing(text: join([name, ": no LAN endpoint row for its head, so it was not probed"], "")) + Present { value: address } => { + let result = ssh.Session.ExecPortableWordsWithStdin( + client_args: [ + "-i", capacity_probe_identity_file, + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + join([spark_managed_login, "@", address as String], ""), + "bash", "-s", + ], + stdin_payload: capacity_probe_script(), + ) + if !result.success { + refused_standing(text: join([name, ": the probe did not run on ", address as String, ": ", trim(s: result.stderr)], "")) + } else { + match decode_probe(stdout: result.stdout) { + ProbeUnreadable { detail: d } => refused_standing(text: join([name, ": ", d], "")) + ProbeFields { fields: f } => + match launch_from_fields(group: group, head: head, f: f) { + LaunchUnidentified { detail: d } => refused_standing(text: join([name, ": the launch could not be identified, so nothing is joined to it -- ", d], "")) + LaunchIdentified { launch: launch } => + match pool_from_fields(f: f, observed_revision: none) { + PoolUnobserved { detail: d } => refused_standing(text: join([name, ": ", d], "")) + PoolObserved { pool: pool } => { + let receipt = admit_kv_layout( + incarnation: launch, + layout: layout_from_fields(f: f), + reported: vllm_reported_capacity(pool: pool, max_concurrency_hundredths: f.concurrency_hundredths), + source_capture_digest: Fnv1a64(content_hash_atom(value: result.stdout as NonEmptyStr)), + ) + let floor = capacity_floor_standing(receipt: receipt, floor: spark_pair_serving_minimum_cold_sessions) + let posture = admission_posture(standing: floor) + GroupStanding { + admitting: match posture { Admitting { available_blocks: _ } => true Withdrawn { cause: _ } => false }, + text: join([ + name, " @ ", address as String, " invocation=", f.invocation, "\n", + " engine: ", to_string(f.running), " running, ", to_string(f.waiting), " waiting; inventory ", + to_string(f.allocator_blocks), " blocks of ", to_string(f.block_size), "; printed pool ", + to_string(f.pool_tokens), " tokens at ", to_string(f.concurrency_hundredths), " hundredths\n", + " receipt: ", receipt_wire(r: receipt), "\n", + " floor: ", standing_wire(s: floor), "\n", + " posture: ", posture_wire(p: posture), + ], ""), + } + } + } + } + } + } + } + } +} + + +// THE EXIT CODE IS THE VERDICT. Today this exits non-zero on both groups, and the reason carries +// the full report: the engine publishes no per-group cache metadata, so the receipt cannot reproduce +// its own printed arithmetic and no capacity is established. That is a true statement about the +// deployment produced by executing against it, which is the thing none of the surrounding model +// could claim on its own. +fn fabric_capacity_standing() -> ProcessExit { + let standings = map(spark_pair_serving_groups, g => group_report(group: g)) + if all(standings, st => st.admitting) { + ExitSuccess + } else { + exit_failure(reason: join(concat(["fabric capacity standing"], map(standings, st => st.text)), "\n")) + } +} diff --git a/dag/gunbc/recurring_failure_mode/cumulative_metric_read_as_per_event.dag b/dag/gunbc/recurring_failure_mode/cumulative_metric_read_as_per_event.dag new file mode 100644 index 00000000000..07870a7bd9a --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/cumulative_metric_read_as_per_event.dag @@ -0,0 +1,24 @@ +module gunbc.recurring_failure_mode.cumulative_metric_read_as_per_event + +import std.types { NonEmptyStr } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data cumulative_metric_read_as_per_event: RecurringFailureMode = RecurringFailureMode { + identity: "cumulative_metric_read_as_per_event" as NonEmptyStr, + + receipts: [ + "a metric that ACCOUNTS is read as a metric that MEASURES (an upstream counter or histogram whose observation is booked once per request, per retention decision or per accounting event, consumed as though it described one event, one step or one instant; the arithmetic succeeds, the units look right, and the answer is about a different quantity). ", + "RECOGNITION: the reading is plausible, dimensionally consistent, and nobody can name the line of upstream source that defines the observation. Ask what the producer OBSERVES, not what the name suggests, and ask it of the installed binary rather than the upstream repository — a self-reported version string is a claim the process makes about itself, not executable provenance. ", + "WHY IT RECURS: these metrics are the only cheap window into a closed upstream, so a wrong reading is not caught by the substrate at all — it is caught by a prediction failing in production, weeks later, against a number that looked authoritative on a dashboard. Two independent investigators reached the SAME wrong root cause from the same histogram in one afternoon, which is the tell that the metric's name was doing the reasoning. ", + "RECEIPTS 2026-09-06/07, DeepSeek V4 Flash on the Spark pairs. (1) kv_cache_usage_perc read as occupancy: it counts every block that is not FREE and an evictable cached prefix is not free, so a warm idle engine reads ~99%; the dashboard rendered it red and captioned it 'referenced by running requests' beside REQ 0, and the router's load-aware landing classified every replica as saturated and silently degraded to the load-blind path it was written to replace — live and inert for a day while one head took all traffic and the other took none. ", + "(2) iteration_tokens_total read as tokens-computed-per-engine-step: the build observes `prompt_token_stats.total`, the FULL prompt length of each prefill booked in one iteration, plus generation tokens. Its large buckets therefore count large-PROMPT REQUESTS, not stop-the-world steps. A single observation of 247,582 tokens was read as one monolithic engine step, producing a confident root cause — a missing --max-num-batched-tokens — that a live poll of the same counter refuted: the count climbs steadily through a cold prefill, so chunking was working the whole time and the proposed flag was a no-op. ", + "(3) Ethernet netdev rx_bytes/tx_bytes read as interconnect traffic: the ranks speak RoCE, whose bytes never reach those counters, so a 176-second prefill measured as ~0 GB of inter-node traffic. The same prefill moved 175 GB per rank on the RoCE port counters. An earlier reading of that port taken across a CACHE HIT was then reported as prefill utilisation. ", + "(4) THE SHARPEST INSTANCE, AND THE ONE THAT SURVIVED THREE REVIEWS. iteration_tokens_total_count is ENGINE-WIDE: every request's steps increment it, so a co-tenant's decode steps land in any window measured around one probe. Dividing a request's computed tokens by that count produced 158,764 / 83 = ~1,913 tokens per step, which was read as the effective scheduler budget and matched to vLLM 2048 fallback. The coincidence is what made it survive: 1,913 sits within 7% of a real default, so a fabricated quantity arrived looking like confirmation of an independent fact. Three further conclusions were then built on the 2048 -- that the budget was already at its floor, that a step costs 2.3s, and that an 88ms excess across a 2048 boundary bounded a per-step cost -- and a fourth, a three-term fit carrying a 126ms per-step term, was authored to refute the third while regressing against the same imaginary step count. ", + "IT WAS SETTLED BY MAKING THE COUNTER MEAN WHAT IT APPEARS TO MEAN. On an IDLE engine no other request increments it, so the engine-wide count IS the per-request step count. Measured there, every window from 512 to 20,082 computed tokens completed in exactly ONE iteration: the effective budget is at least 20,082, an order of magnitude above the 2048 that four conclusions rested on, and no boundary exists anywhere in the range those experiments probed. Under co-tenancy the same ~2,050-token request recorded 4, 5 and 17 iterations. ", + "THE STANDING REMEDY THIS RECEIPT ADDS IS TWO WALLS, BECAUSE EXCLUSIVITY ALONE IS NECESSARY AND NOT SUFFICIENT. FIRST, ATTRIBUTION: an aggregate delta belongs to one request only under a complete attribution boundary -- same launch throughout, exact start and end, no counter reset, complete label population, no foreign contributor, and no booking that lands outside the window. Sparse polling earns only NO FOREIGN REQUEST OBSERVED AT THIS POLLING GRAIN, which is weaker than exclusivity and must be named as such. ", + "SECOND, GRAIN: even under perfect attribution, dividing a delta by an event count yields a POPULATION MEAN and nothing else -- not any particular event's size, not the maximum, not the distribution, not a configured limit. An exact per-event quantity requires either a ONE-EVENT population or per-event accounting emitted by the producer. Both halves of this incident follow from that single rule: 158,764 tokens over 83 iterations is a mean and was read as a budget, while 20,082 tokens over exactly ONE attributed iteration is a behavioural lower bound on what one iteration carried -- and still not a configuration readback, since the configured value may be higher and may not even be the governing quantity. ", + "SHAPE, AND WHY IT IS ONE MODE RATHER THAN THREE: each is a cumulative accounting quantity consumed as an instantaneous or per-event one. The remedy is the same in all three — bind the reading to what the producer observes, at the revision and binary that produced it, and make the wrong question refuse rather than return a number. ", + ], + + evidence: [], +} diff --git a/dag/gunbc/recurring_failure_mode/roster.dag b/dag/gunbc/recurring_failure_mode/roster.dag index 32d65d05e03..d4d57cc18ae 100644 --- a/dag/gunbc/recurring_failure_mode/roster.dag +++ b/dag/gunbc/recurring_failure_mode/roster.dag @@ -18,6 +18,7 @@ import gunbc.recurring_failure_mode.unlanded_citation_indistinguishable_at_the_c import gunbc.recurring_failure_mode.hollow_alias { hollow_alias } import gunbc.recurring_failure_mode.state_space_conflation { state_space_conflation } import gunbc.recurring_failure_mode.absorbing_fallback { absorbing_fallback } +import gunbc.recurring_failure_mode.cumulative_metric_read_as_per_event { cumulative_metric_read_as_per_event } import gunbc.recurring_failure_mode.empty_observation_narrow { empty_observation_narrow } import gunbc.recurring_failure_mode.edit_pass_that_matched_nothing_reports_success { edit_pass_that_matched_nothing_reports_success } import gunbc.recurring_failure_mode.claim_forced_and_unbracketed { claim_forced_and_unbracketed } @@ -189,6 +190,7 @@ data recurring_failure_mode_roster: List = [ hollow_alias, state_space_conflation, absorbing_fallback, + cumulative_metric_read_as_per_event, empty_observation_narrow, edit_pass_that_matched_nothing_reports_success, claim_forced_and_unbracketed, diff --git a/dag/gunbc/spark/pair_serving_capacity_floor.dag b/dag/gunbc/spark/pair_serving_capacity_floor.dag new file mode 100644 index 00000000000..846cb40a501 --- /dev/null +++ b/dag/gunbc/spark/pair_serving_capacity_floor.dag @@ -0,0 +1,205 @@ +module gunbc.spark.pair_serving_capacity_floor + +import std.types { Bool, String, Int, NonEmptyStr, List } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.measure { TokenCount, token_count, token_count_value } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch, launch_wire } +import gunbc.spark.vllm_kv_layout_observe { + VllmKvLayoutReceipt, KvLayoutObserved, KvLayoutUnreadable, KvLayoutInconsistent, +} +import gunbc.spark.pair_serving_launch_receipt { + ServingStanding, ServingOnFirstAttempt, ServingRecoveredAfterFailures, NotServing, ServingStandingUnobserved, +} +import extdeps.vllm.kv_layout { + ResolvedKvLayout, AdmissionCost, AdmissionCostBlocks, AdmissionCostUnestablished, cold_admission_blocks, +} +import gunbc.harness.harness_seat { harness_seat_operator_policy_seats } + +// ── THE FLOOR IS DECLARED IN THE UNIT THE OPERATOR ACTUALLY WANTS ────────────────────────────── +// +// WHAT NOT TO DECLARE. "54 GiB of KV" and "three concurrent sessions" are the same requirement said +// twice, and once both are written down they drift: a change to the sequence budget moves the second +// and leaves the first standing, and nothing detects that the pair no longer describes one intent. +// So exactly one is declared -- the operational one, because it is the one anybody can argue about -- +// and the blocks are DERIVED from it against the engine's own layout. +// +// AND THE SESSION COUNT IS NOT A SECOND NUMBER EITHER. It is the same policy constant the harness +// hands out seats against (gunbc.harness.harness_seat harness_seat_operator_policy_seats). A fabric +// whose floor said four while the harness seated three would be holding capacity nobody could use, +// and one whose floor said two would be admitting a seat the fabric never promised. They are one +// fact, so they are read from one place; when that constant's drop retires and the count becomes +// derived, the floor follows without an edit here. +type MinimumColdSessions { + maximum_sequence_tokens: TokenCount + sessions: Nat +} + +// COLD, meaning no prefix-cache credit. A prospective cache hit is not capacity: it belongs to a +// prefix whose blocks another request may release at any moment, and reserving against it would be +// spending a discount that has not been granted. +data spark_pair_serving_minimum_cold_sessions: MinimumColdSessions = MinimumColdSessions { + maximum_sequence_tokens: token_count(count: 500000), + sessions: harness_seat_operator_policy_seats, +} + +// ── CAPACITY IS A READINESS PROPERTY OF ONE INCARNATION ──────────────────────────────────────── +// +// The blocks the floor requires are the per-request cost at the declared budget, times the declared +// session count. Both come from the receipt's own layout, so a floor assessment is a statement about +// THE ENGINE THE RECEIPT DESCRIBES and nothing else. +type CapacityFloorStanding + = MeetsFloor { required_blocks: Nat, available_blocks: Nat } + | BelowFloor { required_blocks: Nat, available_blocks: Nat } + | FloorUnestablished { cause: NonEmptyStr } + +fn capacity_floor_standing(receipt: VllmKvLayoutReceipt, floor: MinimumColdSessions) -> CapacityFloorStanding { + match receipt { + KvLayoutUnreadable { incarnation: _, cause: c } => + FloorUnestablished { cause: join(["the layout capture was unreadable: ", c as String], "") as NonEmptyStr } + KvLayoutInconsistent { incarnation: _, cause: c } => + FloorUnestablished { cause: join(["the layout is inconsistent: ", c as String], "") as NonEmptyStr } + KvLayoutObserved { incarnation: _, layout: layout, reported: _, blocks_per_max_request: _, source_capture_digest: _ } => + match cold_admission_blocks(layout: layout, sequence_budget: floor.maximum_sequence_tokens) { + AdmissionCostUnestablished { cause: c } => FloorUnestablished { cause: c } + AdmissionCostBlocks { blocks: per_request } => { + let required = per_request * floor.sessions + if required > layout.allocator_blocks { + BelowFloor { required_blocks: required, available_blocks: layout.allocator_blocks } + } else { + MeetsFloor { required_blocks: required, available_blocks: layout.allocator_blocks } + } + } + } + } +} + +// ── WITHDRAWAL IS THE FIRST MOVE, AND IT IS NOT A RESTART ────────────────────────────────────── +// +// "Below floor, therefore restart" is one step where there are two decisions, and collapsing them is +// how a convergence run bounces a busy engine. Free memory on a unified-memory host is a moving +// target: a restart taken because the pool came up small can come up smaller, and a loop that +// restarts on every unfavourable reading will keep restarting while requests are in flight. +// +// So the first move is always the same and is always safe: STOP SENDING NEW WORK. Withdrawal costs +// nothing, is instantly reversible, and is correct under every failing standing including the ones +// where the capacity is merely unknown. Whether to then restart is a separate question with its own +// preconditions, asked below. +type AdmissionPosture + = Admitting { available_blocks: Nat } + | Withdrawn { cause: NonEmptyStr } + +fn admission_posture(standing: CapacityFloorStanding) -> AdmissionPosture { + match standing { + MeetsFloor { required_blocks: _, available_blocks: a } => Admitting { available_blocks: a } + BelowFloor { required_blocks: r, available_blocks: a } => + Withdrawn { cause: join(["the engine is below the declared floor: ", to_string(r), " blocks required, ", to_string(a), " available"], "") as NonEmptyStr } + FloorUnestablished { cause: c } => + Withdrawn { cause: join(["the engine's capacity is unestablished, so it is withdrawn rather than assumed adequate: ", c as String], "") as NonEmptyStr } + } +} + +// ── DRAIN ────────────────────────────────────────────────────────────────────────────────────── +// +// Withdrawal stops new work; it does not end the work already running. A restart taken over live +// grants destroys them, and under the harness's quiescence release law the fabric cannot even tell +// afterwards which seats were freed. Quiescence is therefore OBSERVED, never assumed from elapsed +// time -- a deadline that passed says the lease may be reclaimed, not that the request stopped. +type DrainStanding + = Quiesced + | GrantsOutstanding { count: Nat } + | DrainUnobserved { cause: NonEmptyStr } + +// ── RESTART IS ADMITTED, NOT TRIGGERED ───────────────────────────────────────────────────────── +// +// Five preconditions, and the fold names the FIRST unmet one so an operator reading a refusal learns +// what to change rather than a list. Four of the five are decidable from facts this repository +// already models. The fifth is not, and it refuses. +// +// THE UNIFIED-MEMORY ADMISSION IS THE MISSING ONE, and it is not an oversight. On a GB10 the KV pool +// is a residual of a supply the host shares with the OS page cache, the model weights, the NCCL and +// CUDA contexts and this deployment's own Ray object store; restarting without a plan over that +// supply is what produced 23.6 GiB once and 54.01 GiB later from one unit text. Admitting a restart +// while that plan is unmodelled would be promising an outcome nothing here can predict, so the +// admission refuses and names the capability. That is the M4/M6 slice; when it lands, this arm is +// replaced by a real precondition rather than deleted. +type RestartPreconditionGap + = EngineStillAdmitting + | GrantsStillHeld { count: Nat } + | QuiescenceUnobserved { cause: NonEmptyStr } + | RetryAllowanceExhausted { used: Nat, allowance: Nat } + | UnifiedMemoryPlanUnmodelled + +fn restart_precondition_wire(g: RestartPreconditionGap) -> String { + match g { + EngineStillAdmitting => "the engine has not been withdrawn from new admission" + GrantsStillHeld { count: n } => join([to_string(n), " grant(s) are still outstanding against this incarnation"], "") + QuiescenceUnobserved { cause: c } => join(["quiescence was not observed: ", c as String], "") + RetryAllowanceExhausted { used: u, allowance: a } => + join(["the bounded retry allowance is spent: ", to_string(u), " of ", to_string(a), " attempts used"], "") + UnifiedMemoryPlanUnmodelled => + "no admitted unified-memory plan exists for this host, so the pool the restart would come up with is unpredictable; restarting on an unfavourable reading without one is what produced 23.6 GiB and 54.01 GiB from one unit text" + } +} + +type RestartAdmission + = RestartAdmitted { next_attempt_number: Nat } + | RestartRefused { gap: RestartPreconditionGap } + +// The retry allowance is per CONVERGENCE RUN against one desired digest, not per unit lifetime: a +// unit that is restarted once a week forever is healthy, and one restarted eight times in six +// minutes is not, and only a run-scoped counter tells them apart. +type RetryAllowance { + attempts_used: Nat + allowance: Nat +} + +fn restart_admission( + posture: AdmissionPosture, + drain: DrainStanding, + retries: RetryAllowance, +) -> RestartAdmission { + match posture { + Admitting { available_blocks: _ } => RestartRefused { gap: EngineStillAdmitting } + Withdrawn { cause: _ } => + match drain { + GrantsOutstanding { count: n } => RestartRefused { gap: GrantsStillHeld { count: n } } + DrainUnobserved { cause: c } => RestartRefused { gap: QuiescenceUnobserved { cause: c } } + Quiesced => + if retries.attempts_used >= retries.allowance { + RestartRefused { gap: RetryAllowanceExhausted { used: retries.attempts_used, allowance: retries.allowance } } + } else { + RestartRefused { gap: UnifiedMemoryPlanUnmodelled } + } + } + } +} + +// ── RE-ADMISSION IS EARNED BY A FRESH RECEIPT, NOT BY THE RESTART HAVING HAPPENED ────────────── +// +// The loop closes here. A restart that completed proves a process started; it proves nothing about +// the pool it came up with, and on this hardware that is exactly the thing that varies. So the new +// incarnation is re-admitted only against a receipt of its OWN, assessed against the same floor -- +// and a receipt describing the incarnation that was just replaced does not count, which is why the +// fresh receipt is joined rather than assumed. +type ReAdmission + = ReAdmitted { incarnation: VllmServingLaunch, available_blocks: Nat } + | ReAdmissionRefused { cause: NonEmptyStr } + +fn re_admit(fresh: VllmKvLayoutReceipt, floor: MinimumColdSessions) -> ReAdmission { + match capacity_floor_standing(receipt: fresh, floor: floor) { + MeetsFloor { required_blocks: _, available_blocks: a } => + match fresh { + KvLayoutObserved { incarnation: i, layout: _, reported: _, blocks_per_max_request: _, source_capture_digest: _ } => + ReAdmitted { incarnation: i, available_blocks: a } + KvLayoutUnreadable { incarnation: _, cause: c } => ReAdmissionRefused { cause: c } + KvLayoutInconsistent { incarnation: _, cause: c } => ReAdmissionRefused { cause: c } + } + BelowFloor { required_blocks: r, available_blocks: a } => + ReAdmissionRefused { + cause: join(["the restarted engine is still below the floor: ", to_string(r), " blocks required, ", to_string(a), " available"], "") as NonEmptyStr, + } + FloorUnestablished { cause: c } => + ReAdmissionRefused { cause: join(["the restarted engine's capacity is unestablished, so it is not re-admitted: ", c as String], "") as NonEmptyStr } + } +} diff --git a/dag/gunbc/spark/pair_serving_desired.dag b/dag/gunbc/spark/pair_serving_desired.dag index 597caf7e766..dd8d0f5083d 100644 --- a/dag/gunbc/spark/pair_serving_desired.dag +++ b/dag/gunbc/spark/pair_serving_desired.dag @@ -102,6 +102,7 @@ fn spark_pair_serving_head(group: FabricGroup) -> HostIdentity { fn spark_pair_serving_engine_args(tensor_parallel: Nat, max_num_seqs: Nat) -> VllmEngineArgs { VllmEngineArgs { + prefill_scheduling: none, tensor_parallel_size: tensor_parallel, gpu_memory_utilization: vllm_memory_utilization(fraction: percent(count: 82)), max_model_len: token_count(count: 1048576), diff --git a/dag/gunbc/spark/pair_serving_launch_receipt.dag b/dag/gunbc/spark/pair_serving_launch_receipt.dag new file mode 100644 index 00000000000..a7babeac1d5 --- /dev/null +++ b/dag/gunbc/spark/pair_serving_launch_receipt.dag @@ -0,0 +1,199 @@ +module gunbc.spark.pair_serving_launch_receipt + +import std.types { Bool, String, Int, NonEmptyStr, List, EpochSecs } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.content_hash { ContentHash } +import gunbc.spark.vllm_serving_launch { + VllmServingLaunch, LaunchJoin, SameLaunch, same_launch, join_is_admissible, + launch_join_wire, launch_wire, +} +import gunbc.spark.vllm_kv_layout_observe { VllmKvLayoutReceipt, KvLayoutObserved, KvLayoutUnreadable, KvLayoutInconsistent } + +// ── A LAUNCH THAT FAILED EIGHT TIMES AND THEN WORKED IS NOT A LAUNCH THAT WORKED ─────────────── +// +// WHAT THE APPLY PATH RECORDS TODAY. gunbc.spark.pair_serving_apply renders a unit, compares its +// text, restarts when the text changed OR the unit is not active, and records one line per host: +// applied, with a free-text detail. Everything between "started" and "the unit is active now" is +// discarded. On 2026-09-05 the group A head reached restart counter 8, with status=137, before a +// start that stuck -- and to both systemd and convergence the end state is indistinguishable from a +// unit that came up first time. The failures were not merely unreported; they were unrepresentable. +// +// WHY THAT MATTERS BEYOND REPORTING. The three launches that head produced had 23.6, 21.29 and +// 54.01 GiB of KV. A flapping bring-up is precisely the condition under which the surviving +// incarnation is likely to be a degraded one, because each attempt starts against whatever memory +// the previous attempts left behind. A convergence run that cannot see the flap cannot know it is +// looking at the unlucky draw. + +// THE TERMINAL IS WHAT WAS OBSERVED OF THE PROCESS, and nothing more. Every arm here is a shape a +// process-exit observation can actually have. +type LaunchTerminal + = LaunchServing + | LaunchExited { code: Int } + | LaunchKilled { signal: Nat } + | LaunchReadinessTimedOut + | LaunchTerminalUnobserved { reason: NonEmptyStr } + +// EXIT 137 IS A SIGKILL SHAPE, AND SIGKILL IS NOT OOM. This is the inference the incident invites +// and it is wrong: 128+n is the shell and systemd convention for "terminated by signal n", and +// signal 9 is delivered by the kernel's OOM killer, by `docker rm -f`, by a stop timeout escalating, +// and by an operator. Reading the exit code alone as OOM would attribute every forced teardown in +// this deployment -- and the unit's own ExecStartPre runs `docker rm -f` -- to memory pressure, then +// send someone to tune memory that was never the cause. +// +// So the classification stops at the signal, and OOM is a SEPARATE attribution that requires an +// observation of its own, joined to the same incarnation. +fn launch_terminal_from_exit(code: Int) -> LaunchTerminal { + if code == 0 { + LaunchServing + } else if code > 128 && code < 193 { + LaunchKilled { signal: code - 128 } + } else { + LaunchExited { code: code } + } +} + +fn launch_terminal_wire(t: LaunchTerminal) -> String { + match t { + LaunchServing => "serving" + LaunchExited { code: c } => join(["exited(", to_string(c), ")"], "") + LaunchKilled { signal: s } => join(["killed(signal ", to_string(s), ")"], "") + LaunchReadinessTimedOut => "readiness-timed-out" + LaunchTerminalUnobserved { reason: r } => join(["terminal-unobserved(", r as String, ")"], "") + } +} + +// ── ONE ATTEMPT ──────────────────────────────────────────────────────────────────────────────── +// +// The desired digest is the rendered unit this attempt was made FROM, which is what makes a run of +// attempts attributable to one convergence decision rather than to the unit's name. The layout +// receipt is optional because a launch that never reached readiness has no layout to read, and an +// absent receipt is a different fact from an inconsistent one. +type PairServingLaunchAttempt { + incarnation: VllmServingLaunch + desired_digest: ContentHash + attempt_number: Nat + started_at: EpochSecs + terminal: LaunchTerminal + layout_receipt: VllmKvLayoutReceipt? +} + +// ── OOM IS AN ATTRIBUTION, NOT A READING OF THE EXIT CODE ────────────────────────────────────── +// +// The evidence that promotes a SIGKILL to an OOM kill lives outside the process's own exit status: +// a kernel ring-buffer line, a cgroup memory event counter, or systemd's own OOM result. Each names +// a process; joining it to this attempt is what makes it about THIS launch, and the join goes +// through same_launch for the same reason every other join does. +type OomEvidenceSource + = KernelRingBuffer + | CgroupMemoryEvent + | SystemdOomResult + +fn oom_evidence_source_wire(s: OomEvidenceSource) -> String { + match s { + KernelRingBuffer => "kernel-ring-buffer" + CgroupMemoryEvent => "cgroup-memory-event" + SystemdOomResult => "systemd-oom-result" + } +} + +type OomObservation { + incarnation: VllmServingLaunch + source: OomEvidenceSource + detail: NonEmptyStr +} + +type OomAttribution + = KilledByOom { source: OomEvidenceSource, detail: NonEmptyStr } + | OomUnattributed { cause: NonEmptyStr } + +// The attribution refuses on three separate grounds, and they are not interchangeable: the attempt +// was not signal-killed at all; no OOM observation was offered; or one was offered about a different +// launch. The last is the one a host-keyed join would have accepted. +fn attribute_oom(attempt: PairServingLaunchAttempt, observation: OomObservation?) -> OomAttribution { + match attempt.terminal { + LaunchKilled { signal: _ } => + match observation { + Absent => + OomUnattributed { cause: "the attempt was killed by a signal, but no kernel, cgroup or systemd OOM observation was joined to it; a signal kill is equally a forced teardown, a stop-timeout escalation or an operator" as NonEmptyStr } + Present { value: o } => { + let verdict = same_launch(a: o.incarnation, b: attempt.incarnation) + if join_is_admissible(j: verdict) { + KilledByOom { source: o.source, detail: o.detail } + } else { + OomUnattributed { + cause: join(["an OOM observation was offered but it describes another launch: ", launch_join_wire(j: verdict)], "") as NonEmptyStr, + } + } + } + } + LaunchServing => OomUnattributed { cause: "the attempt is serving, so there is no kill to attribute" as NonEmptyStr } + LaunchExited { code: _ } => OomUnattributed { cause: "the attempt exited with a status rather than being signal-killed, so an OOM kill is not the shape of what happened" as NonEmptyStr } + LaunchReadinessTimedOut => OomUnattributed { cause: "the attempt never reached readiness and was not observed to be killed" as NonEmptyStr } + LaunchTerminalUnobserved { reason: r } => OomUnattributed { cause: join(["the attempt's terminal was not observed, so nothing can be attributed to it: ", r as String], "") as NonEmptyStr } + } +} + +// ── WHAT A RUN OF ATTEMPTS ESTABLISHES ───────────────────────────────────────────────────────── +// +// The standing a convergence run should report, folded over the attempts it made. "Serving" is not +// one state: a unit that came up first time and a unit that came up on the ninth try are different +// facts about the same host, and only the second one says the bring-up is unstable. +type ServingStanding + = ServingOnFirstAttempt { incarnation: VllmServingLaunch } + | ServingRecoveredAfterFailures { incarnation: VllmServingLaunch, failed_attempts: Nat, terminals: List } + | NotServing { failed_attempts: Nat, terminals: List } + | ServingStandingUnobserved { cause: NonEmptyStr } + +fn attempt_is_serving(a: PairServingLaunchAttempt) -> Bool { + match a.terminal { + LaunchServing => true + LaunchExited { code: _ } => false + LaunchKilled { signal: _ } => false + LaunchReadinessTimedOut => false + LaunchTerminalUnobserved { reason: _ } => false + } +} + +// ORDER IS THE CALLER'S, oldest first, because attempt_number is the convergence run's own counter +// and a fold that re-sorted by it would be inventing an order the observations did not carry. +fn serving_standing(attempts: List) -> ServingStanding { + let failures = filter(attempts, a => !attempt_is_serving(a: a)) + let terminals = map(failures, a => launch_terminal_wire(t: a.terminal)) + if length(attempts) == 0 { + ServingStandingUnobserved { cause: "no launch attempt was recorded for this unit, so whether it is serving is unobserved rather than false" as NonEmptyStr } + } else { + match first(filter(attempts, a => attempt_is_serving(a: a))) { + Absent => NotServing { failed_attempts: length(failures), terminals: terminals } + Present { value: winner } => + if length(failures) == 0 { + ServingOnFirstAttempt { incarnation: winner.incarnation } + } else { + ServingRecoveredAfterFailures { incarnation: winner.incarnation, failed_attempts: length(failures), terminals: terminals } + } + } + } +} + +fn serving_standing_wire(s: ServingStanding) -> String { + match s { + ServingOnFirstAttempt { incarnation: i } => join(["serving on first attempt: ", launch_wire(i: i)], "") + ServingRecoveredAfterFailures { incarnation: i, failed_attempts: n, terminals: ts } => + join(["serving after ", to_string(n), " failed attempt(s): ", launch_wire(i: i), " [", join(ts, ", "), "]"], "") + NotServing { failed_attempts: n, terminals: ts } => + join(["not serving after ", to_string(n), " attempt(s): [", join(ts, ", "), "]"], "") + ServingStandingUnobserved { cause: c } => join(["serving standing unobserved: ", c as String], "") + } +} + +// A RECOVERED BRING-UP IS NOT A CLEAN ONE, and this is the predicate a convergence receipt should +// consult rather than "is the unit active". It is deliberately not called `is_healthy`: the question +// it answers is whether the run has anything to report, not whether the engine is good. +fn bring_up_was_clean(s: ServingStanding) -> Bool { + match s { + ServingOnFirstAttempt { incarnation: _ } => true + ServingRecoveredAfterFailures { incarnation: _, failed_attempts: _, terminals: _ } => false + NotServing { failed_attempts: _, terminals: _ } => false + ServingStandingUnobserved { cause: _ } => false + } +} diff --git a/dag/gunbc/spark/serving_critical_path.dag b/dag/gunbc/spark/serving_critical_path.dag new file mode 100644 index 00000000000..19da2fc96ad --- /dev/null +++ b/dag/gunbc/spark/serving_critical_path.dag @@ -0,0 +1,608 @@ +module gunbc.spark.serving_critical_path + +import std.types { Bool, String, Int, NonEmptyStr, List } +import std.nat { Nat } +import std.measure { Watt, watt, watt_count, Milliwatt, milliwatt, milliwatt_count, Hertz, hertz, hertz_count, Percent, percent, percent_count, Microsecond, microsecond, microsecond_count } +import v2.std.optional { Present, Absent } +import extdeps.systems.nvidia { nvidia_dgx_spark_1tb_catalog, nvidia_gb10_gpu_catalog } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch } +import gunbc.spark.serving_launch_instrumentation { + LaunchInstrumentation, spark_group_a_pair_head_instrumentation, + AttributionAvailability, AttributableBy, NotAttributableOnThisLaunch, + launch_attribution_availability, +} + +// -- THE POWER FACTS ARE NVIDIA'S, SO THIS MODULE READS THEM AND DECLARES NONE ---------------- +// +// This module previously minted its own ExternalAuthority for docs.nvidia.com/dgx/dgx-spark and an +// ExternalModelScope whose subject was a GUNBC declaration, then re-stated 240W and 140W as local +// literals. Both facts already live at extdeps.systems.nvidia under the same citation, so there +// were two independently editable copies of one measured number, and the scope row asserted that a +// gunbc module is an authority for NVIDIA hardware. DESIGN 3b names extdeps/ as the home for +// external facts; the fix is to consume the catalog rather than to cite the same page twice. +data spark_system_power_supply: Watt = nvidia_dgx_spark_1tb_catalog.external_power_supply +data spark_soc_thermal_envelope: Watt = nvidia_dgx_spark_1tb_catalog.soc_thermal_design_power + +// AND THE GPU-ONLY CEILING IS NOT A FACT WE HAVE. Two further rows stood here asserting a 120W GPU +// envelope and a 100W peripheral envelope under that same citation. The cited pages state neither. +// extdeps.systems.nvidia leaves nvidia_gb10_gpu_catalog.tdp_watts as `none` deliberately, and its +// annotation gives the reason in the same breath: the 140W figure is the WHOLE SoC's TDP, "not the +// GPU alone or the CPU alone, so it is carried at the system row, never misattributed onto the +// nested GPU catalog row's own tdp_watts field". This module performed exactly that +// misattribution -- its own prose said "roughly 120W" and "about 100W is available to the rest", +// which are an unofficial figure and an arithmetic remainder, and then promoted both to +// authority-backed data. +// +// The denominator was load-bearing: it was what turned an observed 36-44W into "roughly 30-37% of +// the GPU ceiling". So removing the fabricated rows is not enough -- the reading they supported has +// to refuse rather than quietly lose its basis. It is now DERIVED from the catalog's optional field, +// so it refuses today and starts answering by itself if NVIDIA ever publishes a GPU-only TDP and +// that field is filled from the citation. +type GpuPowerFractionStanding + = GpuPowerFraction { percent: Percent } + | GpuDenominatorUnestablished { because: NonEmptyStr } + +fn spark_gpu_power_fraction(observed: Watt) -> GpuPowerFractionStanding { + match nvidia_gb10_gpu_catalog.tdp_watts { + Absent => + GpuDenominatorUnestablished { + because: "the cited docs.nvidia.com pages state a 240W external supply and a 140W whole-SoC TDP and no GPU-only figure, so extdeps.systems.nvidia carries tdp_watts as none; nvidia-smi reports GPU power alone, and dividing it by either published number answers about a different denominator than the one the reading belongs to" as NonEmptyStr + } + Present { value: ceiling } => + if watt_count(w: ceiling) == 0 { + GpuDenominatorUnestablished { because: "the catalog carries a zero GPU ceiling, which no reading can be a fraction of" as NonEmptyStr } + } else { + GpuPowerFraction { percent: percent(count: (watt_count(w: observed) * 100) / watt_count(w: ceiling)) } + } + } +} + +type CriticalPathObservation + = GpuKernelBusyFraction { busy: Percent } + | GpuReportedPowerDraw { median_watts: Watt, max_watts: Watt } + | GpuClockObservation { median_clock: Hertz } + | RoceByteRate { gigabits_per_second_tenths: Nat, link_gigabits: Nat } + | RankCoarseSymmetry { power_spread: Milliwatt, util_spread: Percent } + +data spark_group_a_cold_prefill_observations: List = [ + GpuKernelBusyFraction { busy: percent(count: 96) }, + GpuReportedPowerDraw { median_watts: watt(count: 36), max_watts: watt(count: 44) }, + GpuClockObservation { median_clock: hertz(count: 2408000000) }, + RoceByteRate { gigabits_per_second_tenths: 78, link_gigabits: 50 }, + RankCoarseSymmetry { power_spread: milliwatt(count: 1700), util_spread: percent(count: 0) }, +] + +// ── THE CONSTRUCTION RULES THAT MAKE LAST WEEK'S INFERENCE UNWRITABLE ───────────────────────── +// +// nvidia-smi utilisation is the percentage of the sample period during which one or more kernels +// were executing. It is not FLOPs, not Tensor Core throughput, not SM occupancy, not memory +// bandwidth, not issue bandwidth and not power. So it establishes that kernels were active and +// nothing about whether the GPU was used well. +// +// An earlier version of this module wrote that kernels are present and not computing, and excluded +// compute saturation because power did not rise with load. Both were wrong. The kernels ARE +// executing, and low whole-GPU power does not exclude saturation of a NARROW bottleneck: a kernel +// can fully occupy the unit that limits it while leaving most of the chip idle and the power draw +// modest. The two rules below make that inference unwritable rather than merely discouraged. +type SaturationClaim + = GpuComputeSaturated + | GpuComputeUnderutilised + +type ClaimConstruction + = ClaimConstructible { from: NonEmptyStr } + | ClaimNotConstructible { because: NonEmptyStr } + +fn busy_fraction_constructs(claim: SaturationClaim) -> ClaimConstruction { + match claim { + GpuComputeSaturated => + ClaimNotConstructible { because: "kernel busy fraction counts residency, not work: 96% means kernels were almost always executing and says nothing about FLOPs, occupancy, memory bandwidth or stalls" as NonEmptyStr } + GpuComputeUnderutilised => + ClaimNotConstructible { because: "the same reading is equally consistent with a fully saturated narrow bottleneck; residency cannot distinguish a busy chip from a busy corner of one" as NonEmptyStr } + } +} + +fn power_below_ceiling_excludes_saturation() -> ClaimConstruction { + ClaimNotConstructible { + because: "36-44W against a ~120W GPU ceiling is low, but a kernel bound on one unit -- memory, issue, dependency latency, or a collective wait -- saturates that unit while the package draws modestly; the power reading bounds whole-chip utilisation and not the critical path" as NonEmptyStr + } +} + +// ── THE EFFECTIVE BATCH BUDGET, READ BEHAVIOURALLY BECAUSE IT COULD NOT BE READ ANY OTHER WAY ── +// +// Two earlier versions of this module reasoned from a scheduler budget of 2048 tokens. It was never +// observed: no served metric carries it, the process environment is unreadable unprivileged, and +// 2048 is what the fallback WOULD select. The first version treated it as fact; the second correctly +// downgraded it to nominal. Both were wrong to reason from it at all, and it is now measured. +// +// THE INSTRUMENT IS THE ENGINE ITSELF. On an IDLE engine the engine-wide iteration counter is the +// per-request step count, because no other request increments it. So the budget is the token count +// at which one iteration becomes two -- an observation of the EFFECTIVE value, needing no config +// read, no privileged access and no restart. +// +// computed tokens iterations +// 4,126 1 +// 5,969 1 +// 8,117 1 +// 12,023 1 +// 20,082 1 +// +// No chunking occurs anywhere below 20,082 computed tokens. The budget is therefore at least that, +// which is an order of magnitude above the 2048 that three separate conclusions rested on, and well +// above the 8192 that the larger-memory default would have selected. +// TWO FACTS, NOT ONE. What was measured is a floor on the work one iteration CARRIED; the configured +// value may be higher, and max_num_batched_tokens may not even be the immediately governing quantity +// -- a scheduled-token law or another runtime rule could bind first. Collapsing the observation into +// a configuration readback would launder behaviour into config, which is the same move that turned +// 158,764/83 into a scheduler budget. +type EffectiveBatchBudget + = BehaviouralPerIterationCapacityAtLeast { computed_tokens: Nat, by: NonEmptyStr } + | ConfiguredSchedulerBudgetRead { tokens: Nat, source: NonEmptyStr } + | ConfiguredSchedulerBudgetUnread { because: NonEmptyStr } + +data spark_group_a_effective_batch_budget: EffectiveBatchBudget = BehaviouralPerIterationCapacityAtLeast { + computed_tokens: 20082, + by: "on an idle engine, where the engine-wide iteration counter is the per-request step count, a single 20,082-computed-token request completed in one iteration; no chunking was observed at any size probed from 512 upward" as NonEmptyStr, +} + +// ── HOW PREFILL COST GROWS, ON AN IDLE ENGINE, IN A SINGLE STEP THROUGHOUT ──────────────────── +// +// The earlier cost model here was two handwritten numbers fitted by hand to six points. It is +// replaced by the samples, because a handwritten rate cannot be falsified by adding a measurement +// and a derived one can. +// +// THE STEP-BOUNDARY PREMISE IS GONE, NOT WEAKENED. An earlier version built an experiment around +// crossing 1,927 to 2,052 computed tokens, read a 0.088s excess as a per-step fixed cost, and +// claimed it bounded above by the fit's residual. A later version withdrew the bound by fitting a +// three-term model with a 126ms per-step term that described the same rows better. Both are void: +// every window in this population, from 512 to 20,082 computed tokens, ran in ONE step. There was +// never a boundary to cross, so the excess measured nothing and the three-term model regressed +// against a step count that does not exist. What remains is one fixed per-request term and one +// proportional term, and no per-step term is separable from the fixed one in this range at all. +type ConcurrencyWitness { + max_running: Nat + max_waiting: Nat +} + +// max_running COUNTS THE PROBE. A window with max_running of 1 therefore had NO co-tenant, and an +// earlier version read that field as though 1 meant one other request. +fn co_tenant_present(c: ConcurrencyWitness) -> Bool { + c.max_running > 1 || c.max_waiting > 0 +} + +// PRE-WINDOW CONCURRENCY IS THE ARM SELECTOR, and it is recorded separately from the maximum during +// the window because they answer different questions: whether the engine was quiet when the request +// ARRIVED, and whether anything joined while it ran. +type PrefillScalingSample { + sequence: Nat + computed_tokens: Nat + cached_tokens: Nat + iteration_delta: Nat + wall_micros: Microsecond + arrived_with: ConcurrencyWitness + concurrency: ConcurrencyWitness +} + +type SampleStanding + = ProbeAloneAtObservedConcurrencyGrain + | NoForeignCachedBookingObserved + | ForeignCachedBookingObserved { cached_tokens: Nat } + +fn sample_standing(s: PrefillScalingSample) -> SampleStanding { + if s.cached_tokens > 0 { + ForeignCachedBookingObserved { cached_tokens: s.cached_tokens } + } else { + if s.arrived_with.max_running == 0 && s.arrived_with.max_waiting == 0 { + ProbeAloneAtObservedConcurrencyGrain + } else { + NoForeignCachedBookingObserved + } + } +} + +type EffectiveSchedulerConfigStanding + = EffectiveSchedulerConfigRead { max_num_batched_tokens: Nat, source: NonEmptyStr } + | EffectiveSchedulerConfigUnread { because: NonEmptyStr } + +// THE LAUNCH BINDING IS A STANDING, NOT A STRING. gunbc.spark.vllm_serving_launch exists precisely so +// two observations can be compared by identity rather than by co-location, but these sweeps re-read +// only the systemd invocation and never captured the unit and base-image digests a VllmServingLaunch +// requires. Fabricating them to satisfy the type would be worse than not binding. +type ExperimentLaunchBinding + = ExperimentLaunchBound { launch: VllmServingLaunch } + | ExperimentLaunchUnbound { observed_invocation: NonEmptyStr, missing: NonEmptyStr } + +type PrefillScalingExperiment { + launch: ExperimentLaunchBinding + acquisition: NonEmptyStr + scheduler_config: EffectiveSchedulerConfigStanding + ordered_samples: List +} + +type ExperimentJoinability + = JoinableByLaunch { launch: VllmServingLaunch } + | NotJoinable { because: NonEmptyStr } + +fn experiment_joinability(e: PrefillScalingExperiment) -> ExperimentJoinability { + match e.launch { + ExperimentLaunchBound { launch: l } => JoinableByLaunch { launch: l } + ExperimentLaunchUnbound { observed_invocation: _, missing: m } => NotJoinable { because: m } + } +} + +// One sweep of interleaved repeated sizes, then a boundary probe walking upward. Sequence is +// EXECUTION order: interleaving asserted only in prose cannot be distinguished from a sorted run on +// a drifting machine. A co-tenant arrived at sequence 7 and stayed, which is what makes the +// idle/co-tenant contrast below a within-experiment comparison rather than a cross-day one. +data spark_group_a_prefill_scaling_experiment: PrefillScalingExperiment = PrefillScalingExperiment { + launch: ExperimentLaunchUnbound { + observed_invocation: "group A head 192.168.1.225, unit gunbc-spark-pair-head.service, health 200 before and after" as NonEmptyStr, + missing: "the unit digest and the unit-pinned base image digest were not captured during these sweeps, so no VllmServingLaunch can be constructed and these samples may not be joined by launch identity to any other observation" as NonEmptyStr, + }, + acquisition: "one chat completion per window with a 32-character random nonce and random six-letter words so nothing is served from cache, max_tokens 1; computed and cached token deltas and the iteration count read from the served counters either side; running and waiting read once before the request and polled every 200ms throughout" as NonEmptyStr, + scheduler_config: EffectiveSchedulerConfigUnread { + because: "max_num_batched_tokens is carried by no served metric and the process environment is unreadable unprivileged; the effective value is instead bounded behaviourally by spark_group_a_effective_batch_budget" as NonEmptyStr + }, + ordered_samples: [ + PrefillScalingSample { sequence: 1, computed_tokens: 1910, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 2050000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 2, computed_tokens: 512, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 649000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 3, computed_tokens: 4072, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 4288000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 1, max_waiting: 0 } }, + PrefillScalingSample { sequence: 4, computed_tokens: 2067, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 2323000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 1, max_waiting: 0 } }, + PrefillScalingSample { sequence: 5, computed_tokens: 1046, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 1176000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 6, computed_tokens: 9154, cached_tokens: 233728, iteration_delta: 4, wall_micros: microsecond(count: 10050000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 7, computed_tokens: 2052, cached_tokens: 0, iteration_delta: 5, wall_micros: microsecond(count: 2321000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 8, computed_tokens: 3910, cached_tokens: 0, iteration_delta: 5, wall_micros: microsecond(count: 4180000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 9, computed_tokens: 513, cached_tokens: 0, iteration_delta: 4, wall_micros: microsecond(count: 710000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 10, computed_tokens: 2082, cached_tokens: 0, iteration_delta: 5, wall_micros: microsecond(count: 2440000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 11, computed_tokens: 1564, cached_tokens: 0, iteration_delta: 17, wall_micros: microsecond(count: 2173000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 12, computed_tokens: 4165, cached_tokens: 0, iteration_delta: 7, wall_micros: microsecond(count: 4653000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 13, computed_tokens: 1905, cached_tokens: 0, iteration_delta: 4, wall_micros: microsecond(count: 2082000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 14, computed_tokens: 2065, cached_tokens: 0, iteration_delta: 5, wall_micros: microsecond(count: 2384000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 15, computed_tokens: 2049, cached_tokens: 0, iteration_delta: 5, wall_micros: microsecond(count: 2299000), arrived_with: ConcurrencyWitness { max_running: 1, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 2, max_waiting: 0 } }, + PrefillScalingSample { sequence: 16, computed_tokens: 4126, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 4520000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 17, computed_tokens: 5969, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 6310000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 18, computed_tokens: 8117, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 8560000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 19, computed_tokens: 8281, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 8900000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 20, computed_tokens: 12023, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 12700000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 21, computed_tokens: 16397, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 17420000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + PrefillScalingSample { sequence: 22, computed_tokens: 20082, cached_tokens: 0, iteration_delta: 1, wall_micros: microsecond(count: 21260000), arrived_with: ConcurrencyWitness { max_running: 0, max_waiting: 0 }, concurrency: ConcurrencyWitness { max_running: 0, max_waiting: 0 } }, + ], +} + +// THE 290-SECOND POINT IS RETAINED RATHER THAN DELETED. It is inadmissible to any fit and remains the +// only observation of a delay that large. It is NOT a causal estimate: one event beside one co-tenant +// attributes nothing, and the contrast below is what actually measures co-tenant cost. +type ScalingSampleExcludedFromFit { + computed_tokens: Nat + wall_micros: Microsecond + cause: NonEmptyStr +} + +data spark_group_a_co_tenant_outlier: ScalingSampleExcludedFromFit = ScalingSampleExcludedFromFit { + computed_tokens: 4840, + wall_micros: microsecond(count: 290960000), + cause: "a co-tenant was running; 290.96s against a described 5.1s, roughly 57x, while larger windows at the same co-tenant count sat on the curve -- an extreme event observed beside a co-tenant, far outside the +130ms that the controlled contrast attributes to co-tenancy, and unexplained" as NonEmptyStr, +} + +// ── THE FIT IS DERIVED FROM THE SAMPLES, PER ARM ────────────────────────────────────────────── +// +// Ordinary least squares in microseconds. Nobody can edit the marginal rate: it is a projection of +// the sample population, and adding a measurement that disagrees moves it. +type FitArm + = ArrivedIdle + | ArrivedWithCoTenant + +fn sample_in_arm(s: PrefillScalingSample, arm: FitArm) -> Bool { + if s.cached_tokens > 0 { + false + } else { + match arm { + ArrivedIdle => s.arrived_with.max_running == 0 && s.arrived_with.max_waiting == 0 + ArrivedWithCoTenant => s.arrived_with.max_running > 0 || s.arrived_with.max_waiting > 0 + } + } +} + +fn arm_samples(arm: FitArm) -> List { + filter(spark_group_a_prefill_scaling_experiment.ordered_samples, s => sample_in_arm(s: s, arm: arm)) +} + +type AffineFit { + intercept_micros: Int + micros_per_thousand_tokens: Int + sample_count: Nat +} + +type PrefillScalingStanding + = AffineDescription { fit: AffineFit } + | ScalingUnderObserved { cause: NonEmptyStr } + +fn arm_fit(arm: FitArm) -> PrefillScalingStanding { + let xs = arm_samples(arm) + let n = length(xs) + if n < 4 { + ScalingUnderObserved { cause: "fewer than four admitted samples cannot separate a fixed term from a slope" as NonEmptyStr } + } else { + let sx = fold(xs, init: 0, f: (a, s) => a + s.computed_tokens) + let sy = fold(xs, init: 0, f: (a, s) => a + microsecond_count(m: s.wall_micros)) + let sxx = fold(xs, init: 0, f: (a, s) => a + (s.computed_tokens * s.computed_tokens)) + let sxy = fold(xs, init: 0, f: (a, s) => a + (s.computed_tokens * microsecond_count(m: s.wall_micros))) + let denom = (n * sxx) - (sx * sx) + if denom == 0 { + ScalingUnderObserved { cause: "every admitted sample carries the same token count, so no slope is determined" as NonEmptyStr } + } else { + let slope_milli = (((n * sxy) - (sx * sy)) * 1000) / denom + AffineDescription { + fit: AffineFit { + intercept_micros: (sy - ((slope_milli * sx) / 1000)) / n, + micros_per_thousand_tokens: slope_milli, + sample_count: n, + }, + } + } + } +} + +fn predicted_micros(fit: AffineFit, tokens: Nat) -> Int { + fit.intercept_micros + ((fit.micros_per_thousand_tokens * tokens) / 1000) +} + +fn abs_int(x: Int) -> Int { + if x < 0 { 0 - x } else { x } +} + +fn arm_worst_residual_micros(arm: FitArm, fit: AffineFit) -> Int { + fold(arm_samples(arm), init: 0, f: (worst, s) => { + let e = abs_int(x: predicted_micros(fit: fit, tokens: s.computed_tokens) - microsecond_count(m: s.wall_micros)) + if e > worst { e } else { worst } + }) +} + +// ── WHAT A CO-TENANT COSTS, MEASURED RATHER THAN OBSERVED ───────────────────────────────────── +// +// Both arms come from ONE sweep minutes apart on one launch with one instrument. A co-tenant arrived +// at sequence 7 and stayed, so the contrast is within-experiment: the same sizes, interleaved, either +// side of a change in engine occupancy nobody arranged. +// +// arm n fixed marginal worst residual +// arrived idle 12 ~0.08s ~1.055 ms/token ~0.09s +// co-tenant 9 ~0.26s ~1.032 ms/token ~0.30s +// +// THE MARGINAL RATE AGREES ACROSS THE ARMS to within about three percent, and the difference is not +// resolved even in SIGN: the idle slope estimate moves between 1.026 and 1.057 microseconds per token +// depending on which idle subset is fitted, spanning the co-tenant arm's 1.032. So the honest claim +// is that moderate co-tenancy did not materially change the per-token cost, not that it changed it by +// a measured amount in a measured direction. +// +// The intercept difference is larger and robust to range matching. What it is NOT is a measurement of +// what a co-tenant costs. +// +// ── TWO DIFFERENT SUBJECTS, AND CONFLATING THEM POINTS THE OPERATOR THE WRONG WAY ───────────── +// +// SELF-COST is what this experiment measures: what a cold prefill costs ITSELF, idle versus beside a +// co-tenant. On that subject the proportional term dominates and outer policy cannot reduce it -- no +// routing decision makes 20,082 tokens of execution cheaper than about 21 seconds. +// +// EXTERNALITY is a different subject entirely: what that execution costs OTHER requests. It is not +// bounded by the self-cost contrast and is far larger -- a controlled decode-alone versus +// decode-beside-prefill comparison measured 3.8s against 63.5s. That is the number a user feels, and +// it is exactly what admission and routing can address: send a cold prefill to an idle replica, +// reserve a replica for interactive decode, defer a new cold prefill while both heads carry decoders, +// separate the workload classes. +// +// An earlier reading of this module's own result said outer layers could recover the fixed term "and +// nothing else". That conflated the two subjects and would have argued AGAINST the most immediately +// useful relief available. The correct statement is narrower and points the other way: outer layers +// cannot make the expensive step cheap, and they can decide who is exposed to it. +type ServingCostSubject + = PrefillSelfCost + | CoTenantExternality + +type PolicyLeverage + = OuterPolicyCanAddress { by: NonEmptyStr } + | RequiresInnerExecutionChange { by: NonEmptyStr } + +fn policy_leverage(subject: ServingCostSubject) -> PolicyLeverage { + match subject { + PrefillSelfCost => + RequiresInnerExecutionChange { by: "the cost is proportional to computed tokens at about 1.03 microseconds per token and holds on an idle engine, so no admission, routing or batching decision outside the engine reduces it; only the execution layers can" as NonEmptyStr } + CoTenantExternality => + OuterPolicyCanAddress { by: "a controlled decode-alone versus decode-beside-prefill comparison measured 3.8s against 63.5s, so the latency a co-tenant suffers is far larger than anything in the self-cost contrast and is addressable by placement, reservation, deferral and workload-class separation" as NonEmptyStr } + } +} + +type SharedMinusIdleFitContrast { + intercept_difference_micros: Int + marginal_percent_difference: Int + at_observed_concurrency: Nat +} + +fn shared_minus_idle_contrast() -> SharedMinusIdleFitContrast? { + match arm_fit(ArrivedIdle) { + ScalingUnderObserved { cause: _ } => none + AffineDescription { fit: i } => + match arm_fit(ArrivedWithCoTenant) { + ScalingUnderObserved { cause: _ } => none + AffineDescription { fit: b } => + Present { value: SharedMinusIdleFitContrast { + intercept_difference_micros: b.intercept_micros - i.intercept_micros, + marginal_percent_difference: ((b.micros_per_thousand_tokens - i.micros_per_thousand_tokens) * 100) / i.micros_per_thousand_tokens, + at_observed_concurrency: 2, + } } + } + } +} + +// ── THE PER-STEP QUESTION IS DISSOLVED IN RANGE, NOT ANSWERED ───────────────────────────────── +// +// Every idle window ran in one step, so there is no second step anywhere in the measured range from +// which a per-step term could be distinguished from the per-request term. This is not a bound and not +// an estimate; the question does not arise below the budget, and above it nothing has been measured. +// ONE ARM, BECAUSE ONE ARM HAS A PRODUCER. A PerStepCostBounded { upper_micros: Nat } arm stood +// here and was constructed by nothing -- the only reference outside the declaration was a witness +// match returning false. gunbc.spark.serving_engine has now made this ruling twice against itself, +// deleting WeightsExceedUsablePool and counts_device_reservation on the ground that an arm no +// function emits is a fact the model asserts and never verifies. Its bare-scalar unit field was the +// tell that noticed it; the defect was the arm, so the arm goes rather than its field's carrier. +// It returns with its producer, and with a Microsecond, not before. +type PerStepCostStanding + = PerStepCostNotSeparable { because: NonEmptyStr, range_ceiling_tokens: Nat } + +data spark_group_a_per_step_cost_standing: PerStepCostStanding = PerStepCostNotSeparable { + because: "every idle window from 512 to 20,082 computed tokens completed in exactly one engine iteration, so no per-step term is distinguishable from the fixed per-request term anywhere in the measured range" as NonEmptyStr, + range_ceiling_tokens: 20082, +} + +// ── THE BUDGET COUNTERFACTUAL STAYS WITHDRAWN, FOR A NEW REASON ─────────────────────────────── +// +// This module previously derived a budget trade from the fit and a withdrawn per-step bound, with +// arithmetic that overcounted steps at exact multiples and billed padded tokens in the final partial +// step -- so its zero-per-step-cost lower bound moved by half a second purely from changing the +// budget, contradicting its own assumption. +// +// It stays withdrawn, and the measurement has replaced the reason. The engine does not chunk in the +// range anyone would configure it toward, so a smaller budget is not a knob that is currently doing +// anything; and the marginal rate has not been measured AT any other budget, so extrapolating one +// slope across chunk shapes remains unfounded. Kernel and collective efficiency both change with +// chunk size. +type BudgetTradeStanding + = BudgetTradeUnestablished { missing: List } + +data spark_group_a_budget_trade: BudgetTradeStanding = BudgetTradeUnestablished { + missing: [ + "the marginal rate measured AT each candidate budget, since invariance across chunk shapes is not established and one slope cannot be extrapolated across them" as NonEmptyStr, + "a launch that actually chunks in the range of interest, since the effective budget is at least 20,082 tokens and nothing below it is chunked" as NonEmptyStr, + "per-rank kernel spans, without which a smaller chunk cannot be predicted to cost more or less per token" as NonEmptyStr, + ], +} + +// ── MECHANISM STANDINGS, AT THE GRAIN THE EVIDENCE ACTUALLY DECIDES ─────────────────────────── +// +// The previous version carried one lumped collective-latency candidate and demoted it because cost +// scales with tokens. That was too strong, and the error is worth naming: a collective's payload is +// an activation buffer whose size grows with the scheduled token population, so a token-proportional +// critical path is exactly what per-token collective work ALSO looks like. Proportionality separates +// nothing. +// +// What it does NOT do, either, is weaken the token-independent term. That was the second version's +// error: it read an affine fit's residual as a bound on the term the affine fit omitted. With the +// counterexample above standing, every mechanism here is either unobserved at this grain or +// unresolved, and one is observed. +type CriticalPathMechanism + = GrossRankStraggler + | HostDispatchGaps + | BulkInterconnectBandwidth + | FixedPerRequestCost + | TokenProportionalCompute + | TokenProportionalCollectivePayload + | CollectiveSynchronisationWait + | MemoryOrKernelEfficiency + | PowerOrThermalThrottle + | CoTenantDelay + +type MechanismStanding + = NotObservedAtThisGrain { by: NonEmptyStr } + | Observed { by: NonEmptyStr } + | Unresolved { needs: NonEmptyStr } + +fn cold_prefill_mechanism_standing(m: CriticalPathMechanism) -> MechanismStanding { + match m { + GrossRankStraggler => + NotObservedAtThisGrain { by: "median power within 1.7W and median utilisation within 0% across four ranks; a per-step rank timing could still show one rank extending the critical path" as NonEmptyStr } + HostDispatchGaps => + NotObservedAtThisGrain { by: "kernels active in 98-99% of 250ms samples, so long gaps between kernels are unlikely; sub-sample gaps are invisible at this grain" as NonEmptyStr } + BulkInterconnectBandwidth => + NotObservedAtThisGrain { by: "7.8 Gb/s sustained on a 50 Gb/s link, and about 9.8 Gb/s within a step; bulk saturation is not present, which does not speak to latency" as NonEmptyStr } + FixedPerRequestCost => + Unresolved { needs: "a direct measurement of the fixed-only path; about 0.08s idle and 0.26s beside a co-tenant are affine INTERCEPT ESTIMATES, which absorb request setup, tokenizer and API overhead, fixed model-runner work, measurement-boundary bias and any curvature the affine form cannot carry. It is also not a per-STEP cost: every idle window in range ran in one step, so the two are not separable here" as NonEmptyStr } + TokenProportionalCompute => + Unresolved { needs: "compute-kernel span attribution; token proportionality alone does not distinguish model compute from collective payload, and package power does not either" as NonEmptyStr } + TokenProportionalCollectivePayload => + Unresolved { needs: "collective-kernel span attribution; an all-reduce over an activation buffer grows with the scheduled token population exactly as compute does, so it fits the observed proportionality equally well" as NonEmptyStr } + CollectiveSynchronisationWait => + Unresolved { needs: "per-rank critical-path timing; waiting on a peer is invisible to the aggregate fit, to kernel residency and to package power alike" as NonEmptyStr } + MemoryOrKernelEfficiency => + Unresolved { needs: "achieved-throughput or occupancy counters; a marginal rate near 950 tok/s at a third of the power ceiling is what a memory-bound or dequantisation-heavy path looks like, and equally what per-token collective work looks like" as NonEmptyStr } + PowerOrThermalThrottle => + Unresolved { needs: "throttle-reason and temperature observations, neither of which this trace captured" as NonEmptyStr } + CoTenantDelay => + Observed { by: "the controlled decode-alone versus decode-beside-prefill comparison, 3.8s against 63.5s, which is a matched treatment and control differing in the co-tenancy. The separate idle-versus-shared FIT CONTRAST in this module is descriptive evidence that moderate co-tenancy did not materially change the prefill's own per-token cost, and is not itself a measurement of what a co-tenant costs. The 290.96s outlier stays unexplained by either" as NonEmptyStr } + } +} + +// ── THE PARTITION REMAINS UNRESOLVED, AND MAY NOT CALIBRATE ─────────────────────────────────── +type PartitionRequirement + = ComputeVersusCollectiveKernelSpans + | PerRankCriticalPathTiming + | OccupancyOrAchievedThroughput + | ThrottleAndMemoryStallEvidence + | HigherConcurrencyCoTenantParameterisation + | ChunkedRegimeMeasurement + +type CriticalPathPartition + = PartitionIncomplete { missing: List } + | PartitionEstablished { critical: CriticalPathMechanism, evidence: NonEmptyStr } + +data spark_group_a_cold_prefill_partition: CriticalPathPartition = PartitionIncomplete { + missing: [ + ComputeVersusCollectiveKernelSpans, + PerRankCriticalPathTiming, + OccupancyOrAchievedThroughput, + ThrottleAndMemoryStallEvidence, + HigherConcurrencyCoTenantParameterisation, + ChunkedRegimeMeasurement, + ], +} + +// ── WHY THE MISSING EVIDENCE IS MISSING, WHICH IS NOT THE SAME AS UNDONE ────────────────────── +// +// Four of the six missing requirements need kernel-level or per-rank attribution, and none of them is +// waiting on effort or access. gunbc.spark.serving_launch_instrumentation measured that every +// instrument capable of producing them binds at process start, and that this launch bound none -- so +// on THIS launch they are not obtainable at all, and the remedy is a differently-configured launch. +// +// The distinction matters because "not yet measured" invites another attempt with the same tools, +// which is what the last attempt was. Naming it unobtainable points at the launch profile instead. +type MissingEvidenceDisposition + = ObtainableOnThisLaunch { by: NonEmptyStr } + | RequiresADifferentLaunch { because: NonEmptyStr, remedy: NonEmptyStr } + +fn attribution_disposition() -> MissingEvidenceDisposition { + match launch_attribution_availability(l: spark_group_a_pair_head_instrumentation) { + AttributableBy { instrument: _ } => + ObtainableOnThisLaunch { by: "an attribution instrument is bound to the serving launch" as NonEmptyStr } + NotAttributableOnThisLaunch { because: b, remedy: r } => + RequiresADifferentLaunch { because: b, remedy: r } + } +} + +fn partition_may_calibrate(p: CriticalPathPartition) -> Bool { + match p { + PartitionEstablished { critical: _, evidence: _ } => true + PartitionIncomplete { missing: _ } => false + } +} + +// ── THE LINK QUESTION, DERIVED FROM THE PARTITION RATHER THAN ANSWERED ──────────────────────── +// +// This was previously a function that ignored its argument and returned false, so "a faster link +// cannot help" was written as the answer rather than read from the evidence -- it would have kept +// saying no even for a partition established on bulk bandwidth. What the trace supports is narrower +// and is now what the function says: bulk saturation was not observed, so no link purchase is +// justified by it, and a partition that did establish bulk bandwidth would change the verdict. +type LinkUpgradeStanding + = LinkUpgradeJustified { by: NonEmptyStr } + | LinkUpgradeUnsupportedByCurrentEvidence { by: NonEmptyStr } + +fn link_upgrade_standing(p: CriticalPathPartition) -> LinkUpgradeStanding { + match p { + PartitionEstablished { critical: c, evidence: e } => + match c { + BulkInterconnectBandwidth => LinkUpgradeJustified { by: e } + _ => LinkUpgradeUnsupportedByCurrentEvidence { by: "the established critical path is not bulk transfer, so link speed is not the lever" as NonEmptyStr } + } + PartitionIncomplete { missing: _ } => + LinkUpgradeUnsupportedByCurrentEvidence { + by: "bulk saturation was not observed at any grain measured -- 7.8 Gb/s sustained and about 9.8 Gb/s within a step, on a 50 Gb/s link -- so a 100G or 200G breakout is not justified by this trace; that is weaker than a proof of no benefit under every eventual partition" as NonEmptyStr + } + } +} diff --git a/dag/gunbc/spark/serving_launch_instrumentation.dag b/dag/gunbc/spark/serving_launch_instrumentation.dag new file mode 100644 index 00000000000..75c69a59dc1 --- /dev/null +++ b/dag/gunbc/spark/serving_launch_instrumentation.dag @@ -0,0 +1,160 @@ +module gunbc.spark.serving_launch_instrumentation + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } + +// ── WHETHER A LAUNCH CAN EVER BE ATTRIBUTED IS FIXED WHEN IT STARTS ─────────────────────────── +// +// gunbc.spark.serving_critical_path spent this investigation unable to say where prefill wall time +// goes. The final blocker turned out not to be privilege, effort or access. It is that EVERY +// instrument capable of attributing kernel time binds to a process AT ITS START, so a launch that +// began without one can never be attributed no matter what is granted afterwards. +// +// That makes instrumentation a property OF THE LAUNCH, on the same footing as its unit digest -- +// decided once, unchangeable while it runs, and determining what questions the launch can answer. +// gunbc.spark.vllm_serving_launch rules that runtime facts belong in a receipt ABOUT a launch rather +// than as fields of its identity, so this module is that receipt rather than an edit to the identity. +// +// THE MEASUREMENTS THIS RESTS ON, taken on the group A head on 2026-09-07 against the live launch: +// +// nsys 2025.3.2.474 subcommands launch, start, shutdown, sessions list -- and no attach +// nsys sessions list empty; there is no session to join +// POST /start_profile 404, so the endpoint was never registered +// +// Neither instrument is missing from the host. nsys is installed at /usr/local/bin/nsys. What is +// missing is a BINDING that only a start could have created. +type AttributionInstrument + = NsightSystems + | TorchProfilerEndpoint + | FirstPartyStepObserver + +// THE LAW. Asking whether an instrument can be attached to an already-running engine is the question +// whose wrong answer cost this investigation its last measurement, so it is answered per instrument +// with the reason, and every answer is the same. +type PostHocBindability + = BindsToRunningProcess { by: NonEmptyStr } + | BindsOnlyAtProcessStart { because: NonEmptyStr } + +fn instrument_post_hoc_bindability(i: AttributionInstrument) -> PostHocBindability { + match i { + NsightSystems => + BindsOnlyAtProcessStart { because: "nsys profiles a process it launched into a session it owns; its subcommands are launch, start, shutdown and sessions list, with no attach to an arbitrary pid, and on this host sessions list was empty so there was no existing session to join" as NonEmptyStr } + TorchProfilerEndpoint => + BindsOnlyAtProcessStart { because: "vLLM registers the profile routes only when VLLM_TORCH_PROFILER_DIR is set in the server process environment at startup; unset, POST /start_profile answers 404 and no request can create the route afterwards" as NonEmptyStr } + FirstPartyStepObserver => + BindsOnlyAtProcessStart { because: "a scheduler wrapper is selected when the engine constructs its scheduler, so it must be installed and named before the process starts; nothing loads one into a running engine" as NonEmptyStr } + } +} + +// AND THEREFORE NO GRANT REPAIRS IT. This is the claim worth making unwritable rather than merely +// stating, because the tempting next move after a refused measurement is always to ask for more +// access. Privilege is not what is missing. +type AttributionRemedy + = RemedyIsAdditionalPrivilege { by: NonEmptyStr } + | RemedyIsANewLaunch { because: NonEmptyStr } + +fn attribution_remedy(i: AttributionInstrument) -> AttributionRemedy { + match instrument_post_hoc_bindability(i: i) { + BindsToRunningProcess { by: b } => RemedyIsAdditionalPrivilege { by: b } + BindsOnlyAtProcessStart { because: b } => + RemedyIsANewLaunch { + because: join(["the binding is created at process start and cannot be created later, so no capability granted to an observer reaches it -- ", b as String], "") as NonEmptyStr + } + } +} + +// ── WHAT A LAUNCH DECLARES ABOUT ITSELF ─────────────────────────────────────────────────────── +// +// A launch carries the instruments bound to it. The empty list is the ordinary production case and +// is not a defect: instrumentation costs perturbation, and a serving launch that carries none is +// correctly configured for serving. What is a defect is being unable to SAY which case a launch is +// in, which is how "we have not attributed it yet" survived as a description of something that was +// never obtainable. +type LaunchInstrumentation { + bound_instruments: List + observed_by: NonEmptyStr + revision: VllmSourceRevision +} + +data spark_group_a_pair_head_instrumentation: LaunchInstrumentation = LaunchInstrumentation { + bound_instruments: [], + observed_by: "nsys sessions list empty and POST /start_profile 404 on the group A head, 2026-09-07, against the launch serving at the time" as NonEmptyStr, + revision: vllm_source_revision_read, +} + +type AttributionAvailability + = AttributableBy { instrument: AttributionInstrument } + | NotAttributableOnThisLaunch { because: NonEmptyStr, remedy: NonEmptyStr } + +fn launch_attribution_availability(l: LaunchInstrumentation) -> AttributionAvailability { + match first(l.bound_instruments) { + Present { value: i } => AttributableBy { instrument: i } + Absent => + NotAttributableOnThisLaunch { + because: "no attribution instrument was bound when this launch started" as NonEmptyStr, + remedy: "a separate launch that declares its instrumentation, since every instrument binds only at process start" as NonEmptyStr, + } + } +} + +// ── THE CANARY IS A DIFFERENT LAUNCH IDENTITY, NOT A MODE OF THIS ONE ───────────────────────── +// +// A profiler-enabled engine is not the serving engine with a flag. It perturbs what it measures, so +// its numbers describe ITSELF and may be joined to production numbers only through a matched control +// that measures the perturbation. Modelling it as a mode would let a perturbed measurement inherit +// the production launch's identity and be compared to unperturbed ones without anybody noticing. +// +// So a canary is a launch, and a measurement taken on it carries the control that prices its own +// instrumentation. A canary WITHOUT a matched control cannot be compared to production at all -- +// which is the refusal below, and the reason the control is a field rather than a recommendation. +type PerturbationControl + = MatchedUnprofiledControl { same_workload: NonEmptyStr } + | NoControlTaken + +type CanaryLaunchProfile { + instruments: List + workload: NonEmptyStr + control: PerturbationControl + rank_coverage_required: Nat +} + +type CanaryComparability + = ComparableToProduction { via: NonEmptyStr } + | NotComparableToProduction { because: NonEmptyStr } + +fn canary_comparability(c: CanaryLaunchProfile) -> CanaryComparability { + match c.control { + NoControlTaken => + NotComparableToProduction { because: "a profiled launch perturbs the timing it reports, and with no matched unprofiled control that perturbation is unmeasured, so its numbers describe the profiled launch only" as NonEmptyStr } + MatchedUnprofiledControl { same_workload: w } => + ComparableToProduction { via: join(["the matched unprofiled control over ", w as String, " prices the instrumentation, so the difference between profiled and unprofiled is measured rather than assumed negligible"], "") as NonEmptyStr } + } +} + +// PARTIAL RANK COVERAGE IS NOT PARTIAL EVIDENCE ON A TENSOR-PARALLEL DEPLOYMENT. Progress is bounded +// by the slowest rank, so a trace missing one rank cannot distinguish a fast rank waiting from a +// slow rank computing -- exactly the distinction the whole trace exists to make. +type RankCoverage + = AllRanksCaptured { count: Nat } + | PartialRankCoverage { captured: Nat, required: Nat } + +fn rank_coverage_admits_attribution(c: RankCoverage) -> Bool { + match c { + AllRanksCaptured { count: _ } => true + PartialRankCoverage { captured: _, required: _ } => false + } +} + +// The profile this fabric would need. It is a DESIRED shape and no launch has been made from it: the +// data below is the specification a canary must satisfy, and canary_comparability refuses for any +// construction that drops the control. +data spark_group_a_attribution_canary: CanaryLaunchProfile = CanaryLaunchProfile { + instruments: [NsightSystems, FirstPartyStepObserver], + workload: "one bounded cold prefill near the observed per-iteration capacity floor and one well below it, on an engine with no other request admitted" as NonEmptyStr, + control: MatchedUnprofiledControl { + same_workload: "the identical prompt population replayed on an unprofiled launch of the same runtime image" as NonEmptyStr, + }, + rank_coverage_required: 4, +} diff --git a/dag/gunbc/spark/serving_observer_compatibility.dag b/dag/gunbc/spark/serving_observer_compatibility.dag new file mode 100644 index 00000000000..1938c8e5f0e --- /dev/null +++ b/dag/gunbc/spark/serving_observer_compatibility.dag @@ -0,0 +1,238 @@ +module gunbc.spark.serving_observer_compatibility + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } +import extdeps.vllm.scheduler_extension { + VllmSchedulerSelection, SelectedByQualname, SelectedByTypeObject, SelectedByUpstreamDefault, + VllmAsyncSchedulingRequest, AsyncSchedulingOn, AsyncSchedulingOff, AsyncSchedulingUnset, + ResolvedAsyncScheduling, AsyncResolvedOn, AsyncResolvedOff, resolve_async_scheduling, + VllmSchedulerOutputField, NumScheduledTokens, TotalNumScheduledTokens, ScheduledNewReqs, ScheduledCachedReqs, + scheduler_output_field_attribute, scheduler_output_fields_read, + SchedulerSurfaceBinding, SurfaceBound, SurfaceRefusedForMissingField, SurfaceRefusedForRevisionDrift, + surface_binding_for_revision, vllm_default_scheduler_qualname, + vllm_scheduler_request_counts_method +} +import extdeps.vllm.engine_args { VllmDistributedExecutorBackend, ExecutorMp, ExecutorRay, ExecutorExternalLauncher } +import gunbc.spark.serving_step_receipt { + ServingStepReceipt, RequestStepAllocation, RequestPhase, Prefill, Decode, PrefillAndDecodeInSameStep +} + +// -- THE MANIFEST: WHICH UPSTREAM FIELD ANSWERS EACH RECEIPT FIELD, AT ONE PINNED REVISION ----- +// +// WHY THIS MODULE IS THE FIRST STAGE OF THE EXTRACTION AND NOT PART OF THE OBSERVER. The observer is +// a Python wheel; nothing in it is checkable by this substrate. What IS checkable is the JOIN it must +// implement -- which upstream field feeds which receipt field, and under which conditions that join +// is invalid. Authoring the join here first means the wheel is written against a declared contract +// rather than becoming its own authority by being the only thing that knows the mapping. That is the +// same reasoning gunbc.spark.serving_step_receipt gave for authoring the receipt before the observer, +// applied one layer out. +// +// WHAT THIS MODULE DOES NOT DO. It declares no receipt type, no gap, no instrument and no launch +// identity: those are gunbc.spark.serving_step_receipt's and gunbc.spark.serving_launch_instrumentation's, +// and are imported. It declares no upstream field name: those are extdeps.vllm.scheduler_extension's. +// It owns exactly one fact -- the correspondence -- which belonged to neither side. + +// -- RULE ONE: READ THE OUTPUT, NEVER THE REQUEST ---------------------------------------------- +// +// THE MEASUREMENT THAT FIXES THIS, read from the pinned scheduler source rather than inferred: +// Scheduler.schedule constructs its SchedulerOutput from new_reqs_data and cached_reqs_data, and +// THEN, as its last act before returning, calls _update_after_schedule, which walks +// num_scheduled_tokens and does `request.num_computed_tokens += num_scheduled_token`. +// +// So there are two different values of num_computed_tokens for one step, and which one an observer +// sees is decided purely by WHERE it reads: +// +// SchedulerOutput.scheduled_new_reqs[i].num_computed_tokens the prefix BEFORE this step +// SchedulerOutput.scheduled_cached_reqs.num_computed_tokens[i] the prefix BEFORE this step +// self.requests[req_id].num_computed_tokens, after schedule() the prefix INCLUDING this step +// +// A wrapper that calls the delegate and then reads the live Request objects -- the obvious way to +// write it, since it has the scheduler right there -- records a cached_tokens that already contains +// the step's own computed_tokens. Every receipt is then internally inconsistent by exactly the amount +// the step did, the error is largest on the biggest prefill chunks, and nothing at runtime can catch +// it because both numbers are plausible and the arithmetic succeeds. It is the cumulative-read-as- +// per-event failure mode wearing a per-event shape, which is worse than the original because it would +// arrive carrying this programme's own credibility. +type ReceiptReadSite + = ReadFromSchedulerOutput { attribute: NonEmptyStr } + | ReadFromLiveRequest { because: NonEmptyStr } + +type ReadSiteAdmission + = ReadSiteAdmitted { attribute: NonEmptyStr } + | ReadSiteRefused { because: NonEmptyStr } + +fn read_site_admits(site: ReceiptReadSite) -> ReadSiteAdmission { + match site { + ReadFromSchedulerOutput { attribute: a } => ReadSiteAdmitted { attribute: a } + ReadFromLiveRequest { because: b } => + ReadSiteRefused { + because: join(["Scheduler._update_after_schedule advances request.num_computed_tokens by this step's own scheduled tokens before schedule() returns, so a value read from the live request double-counts the step being recorded -- ", b as String], "") as NonEmptyStr + } + } +} + +// -- THE CORRESPONDENCE ------------------------------------------------------------------------ +// +// One row per receipt field the observer must fill. The receipt field is named by the thing that owns +// it and the upstream field by the thing that owns that, so this module can hold neither authority +// hostage: renaming either side reds here, which is the whole point of writing the join down. +type ReceiptFieldSource + = FromSchedulerOutputField { field: VllmSchedulerOutputField } + | FromSchedulerMethod { method: NonEmptyStr } + | FromObserverClock { because: NonEmptyStr } + +type ReceiptFieldBinding sole_constructor { + receipt_field: NonEmptyStr + source: ReceiptFieldSource + note: NonEmptyStr +} + +fn receipt_field_bindings() -> List { + [ + ReceiptFieldBinding { + receipt_field: "RequestStepAllocation.request_id" as NonEmptyStr, + source: FromSchedulerOutputField { field: NumScheduledTokens }, + note: "the dict key; the allocation population IS this dict's key set, so a request absent from it was not scheduled this step and contributes no allocation" as NonEmptyStr + }, + ReceiptFieldBinding { + receipt_field: "RequestStepAllocation.computed_tokens" as NonEmptyStr, + source: FromSchedulerOutputField { field: NumScheduledTokens }, + note: "the dict value; this is the scheduler's own decision for this request in this step and is not derived, reconstructed or apportioned by the observer" as NonEmptyStr + }, + ReceiptFieldBinding { + receipt_field: "RequestStepAllocation.cached_tokens" as NonEmptyStr, + source: FromSchedulerOutputField { field: ScheduledNewReqs }, + note: "num_computed_tokens on the request's entry in scheduled_new_reqs, or at its index in scheduled_cached_reqs; read from the output and never from the live request, per read_site_admits" as NonEmptyStr + }, + ReceiptFieldBinding { + receipt_field: "ServingStepReceipt.running_after" as NonEmptyStr, + source: FromSchedulerMethod { method: vllm_scheduler_request_counts_method }, + note: "the first element of the published (num_running_reqs, num_waiting_reqs) pair, taken through the delegate rather than by reaching into its private queues" as NonEmptyStr + }, + ReceiptFieldBinding { + receipt_field: "ServingStepReceipt.waiting_after" as NonEmptyStr, + source: FromSchedulerMethod { method: vllm_scheduler_request_counts_method }, + note: "the second element of that same pair, taken in the same call so the two counts describe one instant" as NonEmptyStr + }, + ReceiptFieldBinding { + receipt_field: "ServingStepReceipt.elapsed_micros" as NonEmptyStr, + source: FromObserverClock { because: "no upstream field carries the wall time of one schedule call, so this is the observer's own measurement and is honestly attributed to the observer rather than to the engine" as NonEmptyStr }, + note: "measured around the delegate call only, so it excludes the observer's own recording cost and does not silently charge the engine for instrumentation" as NonEmptyStr + } + ] +} + +// -- THE ONE FIELD THAT IS NOT A COPY ---------------------------------------------------------- +// +// RequestPhase is the only receipt field with no upstream field to read. The scheduler computes a +// closely-related fact -- request.is_prefill_chunk, set as `num_computed_tokens < num_tokens + +// num_output_placeholders` -- but it sets it in _update_after_schedule, on the POST-advance value, +// and it answers "is this a NON-FINAL chunk", which is not the same question. A request whose final +// prefill chunk lands in this step has is_prefill_chunk false while still being a prefill. +// +// So the phase is DERIVED from the two numbers the receipt already carries, and it is derived rather +// than read precisely so that it cannot disagree with them. +type PhaseDerivation + = PhaseDerivedFromAllocation { rule: NonEmptyStr } + | PhaseReadFromUpstreamFlag { attribute: NonEmptyStr } + +data request_phase_derivation: PhaseDerivation = PhaseDerivedFromAllocation { + rule: "a step that advances a request's prompt prefix is a prefill for that request and a step that emits from a complete prefix is a decode; is_prefill_chunk is deliberately not read because it is computed after the advance and answers whether a chunk is non-final rather than whether the step was a prefill" as NonEmptyStr +} + +// -- PROFILE ADMISSION ------------------------------------------------------------------------- +// +// EVERY ARM BELOW HAS A PRODUCER IN observer_profile_admits. gunbc.spark.serving_engine ruled twice +// against carrying a refusal arm no function emits -- once deleting WeightsExceedUsablePool, once +// deleting counts_device_reservation -- on the ground that a required field or arm with no consumer +// is a fact the model asserts and never verifies. An arm here for, say, an unsupported tensor-parallel +// degree would be exactly that: this module has no measurement that makes any degree unsupported. +// THE EXECUTOR BACKEND IS PART OF THE PROFILE BECAUSE IT DECIDES THE DELEGATE. It was absent while +// this type read an unset async flag as "off", which made the profile look complete while carrying +// no way to answer the question it was actually deciding. +type ObserverProfile sole_constructor { + revision: VllmSourceRevision + selection: VllmSchedulerSelection + async_scheduling: VllmAsyncSchedulingRequest + executor: VllmDistributedExecutorBackend +} + +fn observer_profile( + revision: VllmSourceRevision, + selection: VllmSchedulerSelection, + async_scheduling: VllmAsyncSchedulingRequest, + executor: VllmDistributedExecutorBackend, +) -> ObserverProfile { + ObserverProfile { revision: revision, selection: selection, async_scheduling: async_scheduling, executor: executor } +} + +type ObserverProfileRefusal + = ProfileRevisionDrift { observed: NonEmptyStr, because: NonEmptyStr } + | ProfileAlreadyHasForeignScheduler { qualname: NonEmptyStr, because: NonEmptyStr } + | ProfileAsyncSchedulingUnmodelled { delegate: NonEmptyStr, because: NonEmptyStr } + +type ObserverProfileAdmission + = ObserverProfileAdmitted { delegate: NonEmptyStr, fields: Nat } + | ObserverProfileRefused { cause: ObserverProfileRefusal } + +fn observer_profile_admits(p: ObserverProfile) -> ObserverProfileAdmission { + match surface_binding_for_revision(observed: p.revision) { + SurfaceRefusedForRevisionDrift { observed: o, because: b } => + ObserverProfileRefused { cause: ProfileRevisionDrift { observed: o, because: b } } + SurfaceRefusedForMissingField { attribute: a, because: b } => + ObserverProfileRefused { cause: ProfileRevisionDrift { observed: a, because: b } } + SurfaceBound { revision: _, fields: n } => + match p.selection { + SelectedByQualname { qualname: q } => + ObserverProfileRefused { + cause: ProfileAlreadyHasForeignScheduler { + qualname: q, + because: "this launch already substitutes a scheduler, so wrapping it would observe that class rather than the upstream default this contract's laws were established against, and the observer cannot establish what its delegate does" as NonEmptyStr + } + } + SelectedByTypeObject { because: r } => + ObserverProfileRefused { + cause: ProfileAlreadyHasForeignScheduler { + qualname: r, + because: "this launch passes a scheduler class object rather than a qualname, so the delegate cannot even be named in a receipt, let alone established" as NonEmptyStr + } + } + SelectedByUpstreamDefault => + match resolve_async_scheduling(request: p.async_scheduling, backend: p.executor) { + AsyncResolvedOn { because: r } => + ObserverProfileRefused { + cause: ProfileAsyncSchedulingUnmodelled { + delegate: vllm_default_scheduler_qualname(resolved: AsyncResolvedOn { because: r }), + because: join(["AsyncScheduler advances num_computed_tokens against num_output_placeholders for tokens not yet returned, so the prefix a receipt would record is optimistic by an amount this manifest has not established -- and this launch reaches it because ", r as String], "") as NonEmptyStr + } + } + AsyncResolvedOff { because: r } => + ObserverProfileAdmitted { delegate: vllm_default_scheduler_qualname(resolved: AsyncResolvedOff { because: r }), fields: n } + } + } + } +} + +fn observer_profile_refusal_wire(cause: ObserverProfileRefusal) -> String { + match cause { + ProfileRevisionDrift { observed: o, because: _ } => join(["revision-drift(", o as String, ")"], "") + ProfileAlreadyHasForeignScheduler { qualname: q, because: _ } => join(["foreign-scheduler(", q as String, ")"], "") + ProfileAsyncSchedulingUnmodelled { delegate: d, because: _ } => join(["async-scheduling-unmodelled(", d as String, ")"], "") + } +} + +// -- THE FRONTIER THIS MANIFEST OPENS ---------------------------------------------------------- +// +// gunbc.spark.serving_step_receipt declares spark_step_receipt_producer as ProducerUnbuilt with a +// capability-shaped trigger. This module does not discharge that trigger and must not be read as +// doing so: it establishes WHAT the producer must read and WHEN it must refuse, which is the input to +// building one and not a substitute for it. The remaining work is a wheel, and no wheel exists. +type ManifestStanding + = ManifestConsumedByWheel { wheel: NonEmptyStr } + | ManifestAwaitingWheel { trigger: NonEmptyStr } + +data serving_observer_manifest_standing: ManifestStanding = ManifestAwaitingWheel { + trigger: "a gunbai-vllm observer wheel, built at this exact revision, whose scheduler class delegates to the admitted class named by observer_profile_admits, fills every row of receipt_field_bindings from the SchedulerOutput read site and no other, refuses to import under any profile observer_profile_admits refuses, and is proven behaviour-preserving against an unwrapped matched control on a four-rank canary" as NonEmptyStr, +} diff --git a/dag/gunbc/spark/serving_step_cost.dag b/dag/gunbc/spark/serving_step_cost.dag new file mode 100644 index 00000000000..d59867f53ab --- /dev/null +++ b/dag/gunbc/spark/serving_step_cost.dag @@ -0,0 +1,151 @@ +module gunbc.spark.serving_step_cost + +import std.types { Bool, String, Int, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import std.measure { TokenCount, token_count, token_count_value, Millisecond, millisecond, millisecond_count } + +// ── WHAT ONE COLD PREFILL RUN COST, AND WHAT THAT DOES NOT ESTABLISH ───────────────────────── +// +// A cold prefill on the group A head, polled every 200ms so that each completed engine iteration is +// bracketed several times over: +// +// 158,764 computed tokens, 83 engine iterations, 185.5s wall +// inter-iteration interval min 0.94s median 2.30s max 2.68s +// 99% of wall time sits in gaps longer than one second +// +// AN EARLIER VERSION OF THIS SECTION DREW TWO CONCLUSIONS FROM THAT, AND BOTH ARE WITHDRAWN. +// +// It read 158,764 / 83 as ~1,913 computed tokens per step and matched that to the 2048 fallback, +// declaring the scheduler budget observed and already at its floor. Both halves are refuted by +// direct measurement, not merely unsupported. On an idle engine a single request of 20,082 computed +// tokens completes in ONE engine iteration, so the effective batch budget is at least 20,082 and +// nothing chunks below it -- see gunbc.spark.serving_critical_path. And the engine-wide iteration +// counter is not a per-request step count: on a shared engine the same ~2,050-token request recorded +// 4, 5 and 17 iterations across one sweep, because co-tenant decode steps increment it too. +// +// It then said a step takes 2.3 seconds to compute 2,048 tokens, and that this is not a scheduling +// problem. That paired a RUN-WIDE AVERAGE token count with a MEDIAN interval, two numbers never +// observed together, which the next section already identifies as a fabricated observation. Neither +// number describes any step that was measured. +// +// What the run establishes is the aggregate below and nothing finer. +// +// WHY THE STEP GRAIN HAD TO BE POLLED RATHER THAN READ. No served metric carries per-step computed +// tokens -- extdeps.vllm.metrics states that as a refusal, and this measurement confirmed it the +// hard way: prompt_tokens_by_source{local_compute} books a request's ENTIRE computed total in the +// single step where its prefill completes (158,764 in one observation here), exactly like +// prompt_tokens_total does for the booked total. The step COUNT is what carries usable per-step +// information, and only when polled faster than the steps complete. +type StepCostObservation sole_constructor { + profile: NonEmptyStr + shape: ServingStepShape + elapsed_millis: Millisecond + provenance: NonEmptyStr +} + +type ServingStepShape { + computed_prefill: TokenCount + decode_sequences: Nat +} + +// WHAT THE POLL ACTUALLY EARNED, WHICH IS A RUN AGGREGATE AND NOT A STEP CELL. +// +// An earlier shape of this module recorded a single StepCostObservation whose shape was 1,913 +// computed tokens at a 2,300ms median. Those two numbers were never observed together: 1,913 is the +// RUN-WIDE AVERAGE (158,764 tokens over 83 iterations) and 2,300ms is the MEDIAN of the observed +// inter-iteration intervals. Pairing an average token count with a median duration and calling it +// one step's cost is a fabricated observation -- nobody measured a 1,913-token step taking 2.30s, +// and still less that every 2,048-token step costs that. +// +// So the evidence is carried at the grain it was taken: one cold-prefill run, its totals, and the +// distribution of intervals between completed iterations. Every derived quantity below comes from +// run-level terms only. +type ColdPrefillRunAggregate sole_constructor { + profile: NonEmptyStr + total_computed_prefill: TokenCount + completed_iterations: Nat + elapsed_millis: Millisecond + inter_iteration_min_millis: Millisecond + inter_iteration_median_millis: Millisecond + inter_iteration_max_millis: Millisecond + provenance: NonEmptyStr +} + +data spark_group_a_profile: NonEmptyStr = "spark group A, head spark-3bd5, TP=4 over the 50G RoCE rail, deepseek-v4-flash:iq3s-split, fp8_ds_mla, launch invocation d140fa39c1174403bf2ce4d7ed8c2d74" as NonEmptyStr + +data spark_group_a_cold_prefill_run: ColdPrefillRunAggregate = ColdPrefillRunAggregate { + profile: spark_group_a_profile, + total_computed_prefill: token_count(count: 158764), + completed_iterations: 83, + elapsed_millis: millisecond(count: 185500), + inter_iteration_min_millis: millisecond(count: 940), + inter_iteration_median_millis: millisecond(count: 2300), + inter_iteration_max_millis: millisecond(count: 2680), + provenance: "200ms poll of vllm:iteration_tokens_total_count across one cold 158,764-token prefill on an otherwise idle engine, 2026-09-07; intervals taken between consecutive count advances, never between polls" as NonEmptyStr, +} + +// THE EXACT PER-STEP POPULATION IS EMPTY, AND THAT IS THE HONEST STATE. A cell here requires knowing, +// for one completed scheduler step, which requests received how many COMPUTED tokens and how long +// that step took. No served metric carries it (extdeps.vllm.metrics refuses the pairing) and the +// poll above cannot recover it. The authorised privileged step trace is what promotes individual +// steps into this list; until then it stays empty and every lookup refuses. +data spark_serving_step_costs: List = [] + +// ── THE ONE PREDICTION THIS CELL SUPPORTS ───────────────────────────────────────────────────── +// +// A decoding request cannot emit a token before the current engine step completes. So while any +// co-tenant is prefilling, a decoder's inter-token latency IS the step duration, and its observed +// rate is the reciprocal. That is a falsifiable consequence of one measured cell, not a story about +// starvation. +// +// It is stated in thousandths of a token per second because the substrate carries no rationals here +// and rounding a sub-unit rate to an integer would report zero. +type CoTenantDecodeRate + = DecodeRatePredicted { milli_tokens_per_second: Nat, from: NonEmptyStr } + | DecodeRateOutsideCalibration { cause: NonEmptyStr } + +fn co_tenant_decode_rate(run: ColdPrefillRunAggregate) -> CoTenantDecodeRate { + if millisecond_count(m: run.inter_iteration_median_millis) == 0 { + DecodeRateOutsideCalibration { cause: "a zero-duration step predicts an unbounded rate" as NonEmptyStr } + } else { + DecodeRatePredicted { + milli_tokens_per_second: 1000000 / millisecond_count(m: run.inter_iteration_median_millis), + from: run.profile, + } + } +} + +// COMPUTED PREFILL THROUGHPUT IS DERIVED, NOT DECLARED. Nobody should be able to edit "890 tok/s" +// as a literal: it is tokens per step over step duration, and it moves when either does. +fn computed_prefill_tokens_per_second(run: ColdPrefillRunAggregate) -> Nat? { + if millisecond_count(m: run.elapsed_millis) == 0 { + none + } else { + Present { value: (token_count_value(t: run.total_computed_prefill) * 1000) / millisecond_count(m: run.elapsed_millis) } + } +} + +// ── WHAT THIS CELL DOES NOT COVER ───────────────────────────────────────────────────────────── +// +// One cell is one shape. A request for a shape nobody measured refuses rather than interpolating +// toward it -- the neighbouring cell being close is not evidence about the cell that is missing. +type StepCostLookup + = StepCostKnown { observation: StepCostObservation } + | StepCostUnmeasured { cause: NonEmptyStr } + +fn step_cost_for(shape: ServingStepShape) -> StepCostLookup { + match first(filter(spark_serving_step_costs, o => + token_count_value(t: o.shape.computed_prefill) == token_count_value(t: shape.computed_prefill) + && o.shape.decode_sequences == shape.decode_sequences)) { + Present { value: o } => StepCostKnown { observation: o } + Absent => + StepCostUnmeasured { + cause: join([ + "no step-cost cell for ", to_string(token_count_value(t: shape.computed_prefill)), + " computed prefill tokens beside ", to_string(shape.decode_sequences), + " decode sequence(s): the exact per-step population is EMPTY until the privileged step trace runs, and the cold-prefill run aggregate is not a step cell", + ], "") as NonEmptyStr, + } + } +} diff --git a/dag/gunbc/spark/serving_step_receipt.dag b/dag/gunbc/spark/serving_step_receipt.dag new file mode 100644 index 00000000000..b4659baf038 --- /dev/null +++ b/dag/gunbc/spark/serving_step_receipt.dag @@ -0,0 +1,173 @@ +module gunbc.spark.serving_step_receipt + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import std.measure { Microsecond, microsecond } +import v2.std.optional { Present, Absent } +import gunbc.spark.serving_launch_instrumentation { LaunchInstrumentation, AttributionInstrument, FirstPartyStepObserver } + +// ── THE FACT NO SERVED METRIC CAN CARRY ─────────────────────────────────────────────────────── +// +// Every wrong conclusion in the prefill investigation came from deriving a per-request quantity from +// an engine-wide counter. gunbc.recurring_failure_mode.cumulative_metric_read_as_per_event states the +// two walls that follow: an aggregate delta is attributable only under a complete boundary, and even +// then dividing by an event count yields a population MEAN rather than any event's size. +// +// Both walls have the same escape, and it is the only one: per-event accounting emitted by the +// producer. The engine knows exactly how many tokens it scheduled for each request in each step. It +// simply never says so, because no Prometheus series is shaped to carry a per-request-per-step fact. +// +// This module is the shape of what a first-party observer would emit. It is deliberately authored +// BEFORE the observer, because the observer's whole value is conforming to this and a wrapper written +// first would define its own vocabulary and become the authority by accident. +// +// WHAT THE OBSERVER MUST NOT DO is decide anything. It wraps the scheduler, returns upstream's answer +// unchanged, and records what upstream decided. A wrapper that returns a different allocation is a +// different change with a different risk, and mixing the two would make a scheduling defect +// indistinguishable from an observation defect. +type RequestPhase + = Prefill + | Decode + | PrefillAndDecodeInSameStep + +// ONE REQUEST'S SHARE OF ONE STEP. This is the atom the served metrics cannot express: the scheduler +// allocates per request per step, and every aggregate downstream is a fold over these. +type RequestStepAllocation { + request_id: NonEmptyStr + phase: RequestPhase + computed_tokens: Nat + cached_tokens: Nat +} + +// ONE STEP. The allocation list IS the step's scheduled population, so the per-step total is derived +// rather than reported -- a reported total that disagreed with its parts would be a second authority +// for the same quantity. +type ServingStepReceipt { + step_sequence: Nat + started_micros: Microsecond + elapsed_micros: Microsecond + allocations: List + running_after: Nat + waiting_after: Nat +} + +fn step_computed_tokens(r: ServingStepReceipt) -> Nat { + fold(r.allocations, init: 0, f: (acc, a) => acc + a.computed_tokens) +} + +fn step_request_count(r: ServingStepReceipt) -> Nat { + length(r.allocations) +} + +// ── A GAP IS A REFUSAL, NOT A SMALLER POPULATION ────────────────────────────────────────────── +// +// The observer must never block the engine, so its sink is bounded and can drop. A dropped receipt is +// invisible in the surviving population -- the remaining receipts look like a complete run -- which is +// precisely the shape that lets a fold produce a confident wrong answer. +// +// So a gap is recorded as a first-class fact and the population carries it. Serving continues through +// a gap, because performance observation is not a serving-correctness requirement; CALIBRATION does +// not, because a fold over a population with an unrecorded hole is the absorbing fallback this +// programme exists to refuse. +type ObservationGap { + first_missing_sequence: Nat + missing_count: Nat + because: NonEmptyStr +} + +type StepReceiptPopulation { + launch: LaunchInstrumentation + receipts: List + gaps: List +} + +type PopulationCompleteness + = ContiguousAndComplete { steps: Nat } + | HasObservationGaps { gaps: Nat, first_missing: Nat } + +fn population_completeness(p: StepReceiptPopulation) -> PopulationCompleteness { + match first(p.gaps) { + Absent => ContiguousAndComplete { steps: length(p.receipts) } + Present { value: g } => HasObservationGaps { gaps: length(p.gaps), first_missing: g.first_missing_sequence } + } +} + +// THE WALL. A population with a hole may be read, browsed and reported; it may not be folded into a +// calibration. This is the one rule that makes the observer worth building rather than a second +// source of confident wrong numbers. +type CalibrationAdmission + = AdmittedToCalibration { steps: Nat } + | RefusedForObservationGap { because: NonEmptyStr } + | RefusedForUninstrumentedLaunch { because: NonEmptyStr } + +fn population_admits_calibration(p: StepReceiptPopulation) -> CalibrationAdmission { + match first(p.launch.bound_instruments) { + Absent => + RefusedForUninstrumentedLaunch { because: "this population claims receipts from a launch that bound no observer, so its provenance is unaccounted" as NonEmptyStr } + Present { value: _ } => + match population_completeness(p: p) { + HasObservationGaps { gaps: n, first_missing: f } => + RefusedForObservationGap { because: "the receipt population has a recorded hole, and a fold over it would report a complete-looking answer over an incomplete population" as NonEmptyStr } + ContiguousAndComplete { steps: s } => AdmittedToCalibration { steps: s } + } + } +} + +// ── WHAT THE RECEIPT ANSWERS THAT THE COUNTER COULD NOT ─────────────────────────────────────── +// +// The effective batch budget is currently a BEHAVIOURAL FLOOR, obtained by making the engine idle so +// that an engine-wide counter would mean something per request. That trick works only on an idle +// engine, which is the one condition production is never in. A receipt population answers the same +// question directly and under load, because each step reports the population it actually scheduled. +type BudgetEvidence + = LargestScheduledStep { computed_tokens: Nat, at_step: Nat } + | NoStepObserved { because: NonEmptyStr } + +fn largest_scheduled_step(p: StepReceiptPopulation) -> BudgetEvidence { + match population_admits_calibration(p: p) { + RefusedForObservationGap { because: b } => NoStepObserved { because: b } + RefusedForUninstrumentedLaunch { because: b } => NoStepObserved { because: b } + AdmittedToCalibration { steps: _ } => + fold(p.receipts, init: NoStepObserved { because: "the admitted population contains no steps" as NonEmptyStr }, f: (best, r) => { + let t = step_computed_tokens(r: r) + match best { + NoStepObserved { because: _ } => LargestScheduledStep { computed_tokens: t, at_step: r.step_sequence } + LargestScheduledStep { computed_tokens: bt, at_step: bs } => + if t > bt { LargestScheduledStep { computed_tokens: t, at_step: r.step_sequence } } + else { LargestScheduledStep { computed_tokens: bt, at_step: bs } } + } + }) + } +} + +// AND IT SEPARATES SELF-COST FROM CO-TENANCY WITHOUT NEEDING AN IDLE ENGINE. A step carrying one +// request is that request's step; a step carrying several is shared, and the receipt says which +// without anyone having to arrange exclusivity. That is the difference between measuring production +// and measuring a laboratory. +type StepExclusivity + = StepServedOneRequest { request_id: NonEmptyStr } + | StepServedSeveral { request_count: Nat } + | StepServedNothing + +fn step_exclusivity(r: ServingStepReceipt) -> StepExclusivity { + match first(r.allocations) { + Absent => StepServedNothing + Present { value: a } => + if length(r.allocations) == 1 { StepServedOneRequest { request_id: a.request_id } } + else { StepServedSeveral { request_count: length(r.allocations) } } + } +} + +// ── THIS CONTRACT HAS NO PRODUCER YET, AND SAYS SO ──────────────────────────────────────────── +// +// No receipt population exists: producing one needs an observer bound to a launch at start, which is +// the change this contract is authored to constrain. The frontier is declared rather than left +// implicit so that a reader can tell an unbuilt producer from a forgotten one, and so the trigger is +// a capability rather than an artifact. +type ProducerStanding + = ProducerBound { instrument: AttributionInstrument } + | ProducerUnbuilt { trigger: NonEmptyStr } + +data spark_step_receipt_producer: ProducerStanding = ProducerUnbuilt { + trigger: "a first-party scheduler-wrapping observer, installed in the runtime image and named in a launch that starts with it bound, emitting one receipt per engine step with per-request allocations and recording a typed gap whenever its sink drops -- sufficient to answer the effective budget and the exclusivity of any step under production load, which no served metric can" as NonEmptyStr, +} diff --git a/dag/gunbc/spark/vllm_kv_layout_observe.dag b/dag/gunbc/spark/vllm_kv_layout_observe.dag new file mode 100644 index 00000000000..beef8c6b38a --- /dev/null +++ b/dag/gunbc/spark/vllm_kv_layout_observe.dag @@ -0,0 +1,169 @@ +module gunbc.spark.vllm_kv_layout_observe + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import std.content_hash { ContentHash } +import std.measure { TokenCount, token_count_value } +import gunbc.spark.vllm_serving_launch { + VllmServingLaunch, LaunchJoin, SameLaunch, DifferentLaunch, LaunchContradiction, + same_launch, join_is_admissible, launch_join_wire, launch_wire, +} +import extdeps.vllm.kv_layout { + ResolvedKvLayout, VllmReportedCapacity, + LayoutReproduction, LayoutReproducesReported, LayoutContradictsReported, LayoutReproductionUnavailable, + layout_reproduces_reported, + PrintedLinesAgreement, PrintedLinesAgree, PrintedLinesDisagree, PrintedLinesUncheckable, printed_lines_agree, + AdmissionCost, AdmissionCostBlocks, AdmissionCostUnestablished, cold_admission_blocks, +} + +// ── THE ACQUISITION BOUNDARY: A LAYOUT IS EVIDENCE ONLY WHEN IT IS BOUND TO A LAUNCH ─────────── +// +// WHY THIS IS NOT A ROW IN gunbc.spark.vllm_observed. That module is a historical, route-local +// investigation from 2026-09-03 whose cross-route joins were never established; its rows are a +// record of what was seen once. Making it the live seating authority would promote a historical +// fixture into current state, and the first stale row would then be authorizing admissions. Current +// admission consumes a receipt bound to the CURRENTLY ACTIVE incarnation, which is a different +// subject with a different lifetime, so it gets a different module. +// +// WHAT A RECEIPT HAS TO CARRY BEYOND THE NUMBERS. The incarnation, because a reading of one launch +// is not evidence about its successor; and the digest of the captured text it was parsed from, +// because a number authored beside a claim about where it came from is not the same as a number +// traceable to bytes someone can re-read. +// +// THE ADMISSION IS NOT A CHOICE THE CALLER MAKES. KvLayoutObserved has no free constructor: it is +// reachable only through the fold below, and only when the group layout independently reproduces the +// per-request cost the engine printed. Every other outcome is a named refusal. This is the +// difference between a receipt and a record: a record says what was seen, a receipt says what was +// established. + +type VllmKvLayoutReceipt + = KvLayoutObserved { + incarnation: VllmServingLaunch, + layout: ResolvedKvLayout, + reported: VllmReportedCapacity, + blocks_per_max_request: Nat, + source_capture_digest: ContentHash, + } + | KvLayoutUnreadable { incarnation: VllmServingLaunch, cause: NonEmptyStr } + | KvLayoutInconsistent { incarnation: VllmServingLaunch, cause: NonEmptyStr } + +// THE REPORTED FIGURES ARE CARRIED AS THE UPSTREAM CARRIER, not re-spelled as two fields beside it. +// An earlier shape of this type listed reported_equivalent_tokens and reported_max_concurrency +// alongside the layout, which is the same pair of numbers in a second place -- and the pair is +// exactly what the reproduction check reads, so a receipt whose copies drifted from its carrier +// would check one and report the other. + +// ── ADMISSION ────────────────────────────────────────────────────────────────────────────────── +// +// The order is deliberate: the cheapest refusal that is definitely fatal comes first, so a receipt +// never reports a downstream disagreement that an upstream one already explains. +// +// 1. the printed lines must be mutually consistent -- if the pool line and the concurrency line +// disagree, the capture is garbled and nothing derived from it means anything; +// 2. the group layout must reproduce the printed per-request cost -- the one non-circular check, +// and the one that is unavailable until the engine emits group metadata. +// +// TODAY EVERY REAL CAPTURE LANDS ON (2) AS INCONSISTENT, because `groups` is empty and the derived +// route does not exist. That is the honest state and not a bug in this fold: an absent second +// opinion is not agreement. +fn admit_kv_layout( + incarnation: VllmServingLaunch, + layout: ResolvedKvLayout, + reported: VllmReportedCapacity, + source_capture_digest: ContentHash, +) -> VllmKvLayoutReceipt { + match printed_lines_agree(layout: layout, reported: reported) { + PrintedLinesUncheckable { cause: c } => KvLayoutUnreadable { incarnation: incarnation, cause: c } + PrintedLinesDisagree { implied_hundredths: i, reported_hundredths: r } => + KvLayoutInconsistent { + incarnation: incarnation, + cause: join([ + "the captured startup lines disagree with each other: the pool line implies a concurrency of ", + to_string(i), " hundredths while the concurrency line reports ", to_string(r), + ], "") as NonEmptyStr, + } + PrintedLinesAgree => + match layout_reproduces_reported(layout: layout, reported: reported) { + LayoutReproductionUnavailable { cause: c } => + KvLayoutInconsistent { + incarnation: incarnation, + cause: join(["the layout cannot be checked against the engine's own arithmetic: ", c as String], "") as NonEmptyStr, + } + LayoutContradictsReported { predicted_hundredths: p, reported_hundredths: r } => + KvLayoutInconsistent { + incarnation: incarnation, + cause: join([ + "the group layout and the printed lines describe different engines: the groups predict a concurrency of ", + to_string(p), " hundredths while the engine printed ", to_string(r), + ], "") as NonEmptyStr, + } + LayoutReproducesReported { blocks_per_max_request: b } => + KvLayoutObserved { + incarnation: incarnation, + layout: layout, + reported: reported, + blocks_per_max_request: b, + source_capture_digest: source_capture_digest, + } + } + } +} + +// ── WHAT A RECEIPT AUTHORIZES, AND FOR WHICH LAUNCH ──────────────────────────────────────────── +// +// A receipt is not a standing fact about a group. It is a fact about ONE incarnation, and the only +// question a seat lease may ask of it is: does this receipt describe the engine I am about to send +// a request to? Anything that answers that question by comparing groups or hosts has reintroduced +// the co-location test gunbc.spark.vllm_serving_launch exists to displace. +type SeatingAuthority + = SeatingAuthorized { blocks_available: Nat, blocks_per_request: Nat } + | SeatingRefused { cause: NonEmptyStr } + +fn seating_authority_for( + receipt: VllmKvLayoutReceipt, + serving: VllmServingLaunch, + sequence_budget: TokenCount, +) -> SeatingAuthority { + match receipt { + KvLayoutUnreadable { incarnation: _, cause: c } => + SeatingRefused { cause: join(["the layout capture was unreadable, so no capacity is established: ", c as String], "") as NonEmptyStr } + KvLayoutInconsistent { incarnation: _, cause: c } => + SeatingRefused { cause: join(["the layout is inconsistent, so no capacity is established: ", c as String], "") as NonEmptyStr } + KvLayoutObserved { incarnation: observed, layout: layout, reported: _, blocks_per_max_request: _, source_capture_digest: _ } => { + let join_verdict = same_launch(a: observed, b: serving) + if !join_is_admissible(j: join_verdict) { + SeatingRefused { + cause: join([ + "the receipt describes ", launch_wire(i: observed), " but the request would be served by ", + launch_wire(i: serving), ": ", launch_join_wire(j: join_verdict), + ], "") as NonEmptyStr, + } + } else { + match cold_admission_blocks(layout: layout, sequence_budget: sequence_budget) { + AdmissionCostUnestablished { cause: c } => SeatingRefused { cause: c } + AdmissionCostBlocks { blocks: cost } => + SeatingAuthorized { blocks_available: layout.allocator_blocks, blocks_per_request: cost } + } + } + } + } +} + +fn receipt_incarnation(r: VllmKvLayoutReceipt) -> VllmServingLaunch { + match r { + KvLayoutObserved { incarnation: i, layout: _, reported: _, blocks_per_max_request: _, source_capture_digest: _ } => i + KvLayoutUnreadable { incarnation: i, cause: _ } => i + KvLayoutInconsistent { incarnation: i, cause: _ } => i + } +} + +fn receipt_wire(r: VllmKvLayoutReceipt) -> String { + match r { + KvLayoutObserved { incarnation: i, layout: _, reported: _, blocks_per_max_request: b, source_capture_digest: _ } => + join(["observed ", launch_wire(i: i), ": ", to_string(b), " blocks per full-context request"], "") + KvLayoutUnreadable { incarnation: i, cause: c } => + join(["unreadable ", launch_wire(i: i), ": ", c as String], "") + KvLayoutInconsistent { incarnation: i, cause: c } => + join(["inconsistent ", launch_wire(i: i), ": ", c as String], "") + } +} diff --git a/dag/gunbc/spark/vllm_serving_launch.dag b/dag/gunbc/spark/vllm_serving_launch.dag new file mode 100644 index 00000000000..e50e777332d --- /dev/null +++ b/dag/gunbc/spark/vllm_serving_launch.dag @@ -0,0 +1,215 @@ +module gunbc.spark.vllm_serving_launch + +import std.types { Bool, String, NonEmptyStr, List, EpochSecs } +import v2.std.optional { Present, Absent } +import std.content_hash { ContentHash, compare_content_hash, ContentHashComparison, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable } +import product.placement_supply { HostIdentity } +import gunbc.spark.fabric_switch_observed { FabricGroup, fabric_group_wire } + +// ── ONE LAUNCH IS ONE SUBJECT, AND THE UNIT IS NOT IT ────────────────────────────────────────── +// +// WHY THIS TYPE EXISTS. Every capacity, usage, restart and routing observation this repository +// takes about a serving engine has been joined on some combination of host, group, model name and +// self-reported version string. None of those identify a LAUNCH, and the difference is not +// theoretical: on 2026-09-05 the group A head produced engine cores reporting 23.6 GiB, then +// 21.29 GiB, then 54.01 GiB of available KV -- three incarnations from ONE rendered unit whose text +// never changed. Joined on host and unit, those are three readings of one object and the fleet +// concludes its capacity is unstable. Joined on incarnation, they are three objects with one +// reading each, and the live question becomes which one is currently serving. +// +// THE CONFUSION IS A SAFETY DEFECT AND NOT ONLY A REPORTING ONE. A seat granted against the engine +// that answered a probe must not authorize a request against a different engine that has since +// replaced it, and "FabricGroupA" cannot tell them apart. Restart is precisely the event that +// invalidates every outstanding grant, so a grant that cannot name its incarnation cannot be fenced. +// +// WHAT WAS REMOVED, AND WHY IT IMPROVED THE IDENTITY. This type carried `container_id`. Reading it +// requires docker access, which on these hosts means membership of the `docker` group -- effectively +// root, since a member can bind-mount the host root into a container. Convergence already has a +// SEPARATE privileged leg for privileged work, so granting the everyday principal a root-equivalent +// capability to read one field was the wrong trade twice over. +// +// And the field bought nothing. The unit recreates its container on every start (`docker rm -f` +// followed by `docker create` in ExecStartPre), so the container id is one-to-one with the systemd +// invocation and discriminates no pair the invocation does not already separate. It was a redundant +// field with a root-shaped price, which is how it came to block an unprivileged instrument. +// +// WHAT REPLACED THE IMAGE DIGEST, AND HOW LITTLE IT CLAIMS. An earlier spelling of this field said +// the unit "pulls ...@sha256: and creates from it". It does not. The realization is: +// +// docker image pull aidendle94/sparkrun-vllm-ds4-gb10@sha256: <- the pinned BASE +// docker build --tag gunbc-spark-pair-runtime:ray-2.58.0 /var/lib/... <- a LOCAL image +// docker create --name gunbc-spark-pair ... (from the local TAG) <- what actually runs +// +// So the container is created from a locally built image whose digest is nowhere in the unit text. +// The field is named for the only thing it establishes: the unit text names this pinned base-image +// digest as an INPUT TO ITS BUILD. +// +// It does NOT establish the locally built runtime image's digest, the image the active container was +// created from, the executable bytes, the build context, or that no out-of-band replacement +// occurred. Those are privileged runtime facts and they belong in a separate provenance receipt +// ABOUT this launch -- never as a prerequisite for constructing the launch identity, which is the +// dependency that made the identity unreadable to the ordinary observer in the first place. +// +// WHAT IS DELIBERATELY NOT A FIELD. The engine's self-reported vLLM version string is not identity. +// extdeps.vllm.server already carries a VllmSourceRevision for what the LAWS were established +// against, and a version a process prints about itself is a claim rather than provenance: a patched +// or locally built image reports whatever its metadata says. Executable provenance is the digest, +// and an observer that cannot obtain one produces no incarnation rather than substituting a version. + +// The identity of one engine launch. Every field is an OBSERVED fact about that launch; nothing is +// derived from the desired unit, because the desired unit is exactly what fails to distinguish two +// launches of itself. +type VllmServingLaunch sole_constructor { + group: FabricGroup + head: HostIdentity + systemd_invocation: NonEmptyStr + unit_pinned_base_image_digest: ContentHash + unit_digest: ContentHash + started_at: EpochSecs +} + +fn vllm_serving_launch( + group: FabricGroup, + head: HostIdentity, + systemd_invocation: NonEmptyStr, + unit_pinned_base_image_digest: ContentHash, + unit_digest: ContentHash, + started_at: EpochSecs, +) -> VllmServingLaunch { + VllmServingLaunch { + group: group, + head: head, + systemd_invocation: systemd_invocation, + unit_pinned_base_image_digest: unit_pinned_base_image_digest, + unit_digest: unit_digest, + started_at: started_at, + } +} + +// ── THE JOIN IS AN IDENTITY COMPARISON, NOT A CO-LOCATION TEST ───────────────────────────────── +// +// Every join between two observations goes through this. The expression it exists to displace is +// `a.group == b.group`, which reads as an identity test and is a co-location test. +type LaunchJoin + = SameLaunch + | DifferentLaunch { left: NonEmptyStr, right: NonEmptyStr } + | LaunchContradiction { invocation: NonEmptyStr, field: NonEmptyStr } + +// THE CROSS-FAMILY ARM IS A CONTRADICTION, NOT A DISAGREEMENT. std.content_hash refuses to collapse +// a cross-family pair into a silent false precisely so callers cannot read "I could not compare +// these" as "these differ". Two observers that digested one launch under different hash families +// have not established that it changed; they have established that they cannot be joined, which is +// an observer defect. Mapping it to `false` here would reintroduce the collapse one level up. +type DigestAgreement + = DigestsAgree + | DigestsDiffer + | DigestsIncomparable + +fn digests_agree(left: ContentHash, right: ContentHash) -> DigestAgreement { + match compare_content_hash(left: left, right: right) { + ContentHashEqual => DigestsAgree + ContentHashDifferent => DigestsDiffer + ContentHashCrossFamilyIncomparable => DigestsIncomparable + } +} + +// THE AGREEMENT CHECKS ARE ONE ROW LIST FOLDED ONCE, not a conditional chain that recomputes a +// comparison in order to ask whether it fired. Each row names its field and carries its verdict, so +// the reported field is the FIRST that disagreed rather than whichever the nesting reached first. +// +// EVERY IDENTITY FIELD IS COMPARED, AND `head` IS THE ONE THAT WAS MISSED. It sat in the type and +// not in this list, so two launches on DIFFERENT HOSTS that happened to share an invocation id +// joined as the same launch -- and a systemd invocation id is unique per host, not across the +// fleet, so nothing prevents that coincidence between two of the eight Sparks. A carrier field the +// join ignores is worse than no field: it reads as though the host were part of the identity while +// contributing nothing to it. The rule this row list now holds is that the type has no field the +// fold does not consult. +// +// EVERY IDENTITY FIELD IS COMPARED, AND `head` IS THE ONE THAT WAS MISSED. It sat in the type and +// not in this list, so two launches on DIFFERENT HOSTS that happened to share an invocation id +// joined as the same launch -- and a systemd invocation id is unique per host, not across the +// fleet, so nothing prevents that coincidence between two of the eight Sparks. A carrier field the +// join ignores is worse than no field: it reads as though the host were part of the identity while +// contributing nothing to it. The rule this row list now holds is that the type has no field the +// fold does not consult. +type FieldAgreement { + field: NonEmptyStr + agrees: Bool + incomparable: Bool +} + +fn field_agreement(field: NonEmptyStr, agrees: Bool) -> FieldAgreement { + FieldAgreement { field: field, agrees: agrees, incomparable: false } +} + +fn digest_agreement_row(field: NonEmptyStr, agreement: DigestAgreement) -> FieldAgreement { + match agreement { + DigestsAgree => FieldAgreement { field: field, agrees: true, incomparable: false } + DigestsDiffer => FieldAgreement { field: field, agrees: false, incomparable: false } + DigestsIncomparable => FieldAgreement { field: field, agrees: false, incomparable: true } + } +} + +fn agreement_rows(a: VllmServingLaunch, b: VllmServingLaunch) -> List { + [ + field_agreement(field: "head" as NonEmptyStr, agrees: (a.head as String) == (b.head as String)), + digest_agreement_row(field: "unit_pinned_base_image_digest" as NonEmptyStr, agreement: digests_agree(left: a.unit_pinned_base_image_digest, right: b.unit_pinned_base_image_digest)), + digest_agreement_row(field: "unit_digest" as NonEmptyStr, agreement: digests_agree(left: a.unit_digest, right: b.unit_digest)), + field_agreement(field: "group" as NonEmptyStr, agrees: (fabric_group_wire(g: a.group) as String) == (fabric_group_wire(g: b.group) as String)), + field_agreement(field: "started_at" as NonEmptyStr, agrees: a.started_at == b.started_at), + ] +} + +fn contradiction_of_row(invocation: NonEmptyStr, row: FieldAgreement) -> LaunchJoin { + if row.incomparable { + LaunchContradiction { + invocation: invocation, + field: join([row.field as String, " (the digest families are incomparable, so the two observers cannot be joined at all)"], "") as NonEmptyStr, + } + } else { + LaunchContradiction { invocation: invocation, field: row.field } + } +} + +// THE DISCRIMINATOR IS THE INVOCATION. systemd mints a fresh invocation id per start of a unit, so +// two launches differ there and nowhere else that is GUARANTEED to differ -- a restart inside one +// second shares started_at, a rebuild-free restart shares both digests, and a container recreated +// under the same name shares the name. A same-invocation pair therefore still checks the rest: if +// the invocation agrees and a field does not, the two observations disagree about what they watched, +// and that is a contradiction to surface rather than a match to report. +fn same_launch(a: VllmServingLaunch, b: VllmServingLaunch) -> LaunchJoin { + if (a.systemd_invocation as String) != (b.systemd_invocation as String) { + DifferentLaunch { left: a.systemd_invocation, right: b.systemd_invocation } + } else { + match first(filter(agreement_rows(a: a, b: b), r => !r.agrees)) { + Absent => SameLaunch + Present { value: row } => contradiction_of_row(invocation: a.systemd_invocation, row: row) + } + } +} + +// A join is admissible ONLY on SameLaunch. The contradiction arm is deliberately NOT folded +// into the different arm: "these are two launches" and "these two readings disagree about one +// launch" call for different action -- the first is ordinary turnover, the second means an observer +// is wrong and no reading from either side should be trusted until that is resolved. +fn join_is_admissible(j: LaunchJoin) -> Bool { + match j { + SameLaunch => true + DifferentLaunch { left: _, right: _ } => false + LaunchContradiction { invocation: _, field: _ } => false + } +} + +fn launch_join_wire(j: LaunchJoin) -> String { + match j { + SameLaunch => "same-incarnation" + DifferentLaunch { left: l, right: r } => + join(["different-incarnation: ", l as String, " vs ", r as String], "") + LaunchContradiction { invocation: i, field: f } => + join(["incarnation-contradiction: two observations of ", i as String, " disagree on ", f as String], "") + } +} + +fn launch_wire(i: VllmServingLaunch) -> String { + join([fabric_group_wire(g: i.group) as String, "@", i.systemd_invocation as String], "") +} diff --git a/dag/test/claim/spark/fabric_capacity_probe_decode_witness_test.dag b/dag/test/claim/spark/fabric_capacity_probe_decode_witness_test.dag new file mode 100644 index 00000000000..2df3722eceb --- /dev/null +++ b/dag/test/claim/spark/fabric_capacity_probe_decode_witness_test.dag @@ -0,0 +1,100 @@ +module test.claim.spark.fabric_capacity_probe_decode_witness + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision, vllm_source_revision_read } +import gunbc.instruments.fabric_capacity_standing { + CapacityProbeRead, ProbeFields, ProbeUnreadable, decode_probe, + PoolRead, PoolObserved, PoolUnobserved, pool_from_fields, +} + +// THE FIRST WITNESS THIS INSTRUMENT HAS HAD, and its absence is why two fail-open arms survived +// four commits. The module's only entry is an actuator that probes live hosts over ssh, so nothing +// could go red without a fleet; decode_probe and pool_from_fields are ordinary total functions over +// captured text, so the discriminating REDs are authorable here as fixtures and are enrolled. + +data probe_good_lines: List = [ + "abc123", + "1111111111111111111111111111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222222222222222222222222222", + "1757280000", + "163840", + "1042", + "10240", + "16", + "131072", + "3", + "7", +] + +fn probe_stdout(lines: List) -> String { + join(lines, "\n") +} + +fn replace_at(lines: List, n: Nat, v: String) -> List { + concat(lines.take(n: n), concat([v], lines.skip(n: n + 1))) +} + +// POSITIVE CONTROL. Eleven parseable fields decode, and the two occupancy figures arrive as the +// numbers the far side sent rather than as a default. +test fn eleven_readable_fields_decode_with_the_occupancy_the_host_reported() -> Bool { + match decode_probe(stdout: probe_stdout(lines: probe_good_lines)) { + ProbeUnreadable { detail: _ } => false + ProbeFields { fields: f } => f.running == 3 && f.waiting == 7 && f.max_model_len == 131072 + } +} + +// THE DISCRIMINATING RED FOR THE OCCUPANCY ARMS. These two went `Absent => 0` before, so an engine +// whose metrics could not be read reported as idle -- and running/waiting feed the admission +// posture. Each field is refused independently and the refusal names which one. +test fn an_unreadable_running_count_refuses_instead_of_reading_as_an_idle_engine() -> Bool { + match decode_probe(stdout: probe_stdout(lines: replace_at(lines: probe_good_lines, n: 9, v: "-"))) { + ProbeFields { fields: _ } => false + ProbeUnreadable { detail: d } => string_contains(s: d, pattern: "num_requests_running") + } +} + +test fn an_unreadable_waiting_count_refuses_instead_of_reading_as_an_idle_engine() -> Bool { + match decode_probe(stdout: probe_stdout(lines: replace_at(lines: probe_good_lines, n: 10, v: "-"))) { + ProbeFields { fields: _ } => false + ProbeUnreadable { detail: d } => string_contains(s: d, pattern: "num_requests_waiting") + } +} + +// THE DISCRIMINATING RED FOR THE REVISION. pool_from_fields used to stamp vllm_source_revision_read +// into the reading it built, which made admit_kv_layout's refusal on observed_at_revision -- the +// wall that decides `admitting` -- permanently green. With no observed revision it must refuse. +test fn a_pool_built_without_an_observed_revision_refuses_rather_than_stamping_the_model() -> Bool { + match decode_probe(stdout: probe_stdout(lines: probe_good_lines)) { + ProbeUnreadable { detail: _ } => false + ProbeFields { fields: f } => + match pool_from_fields(f: f, observed_revision: none) { + PoolObserved { pool: _ } => false + PoolUnobserved { detail: d } => string_contains(s: d, pattern: "does not capture the vLLM source revision") + } + } +} + +// AND THE POSITIVE CONTROL FOR THE SAME ARM: given a revision, the pool is built from it. This is +// the conjunct that keeps the refusal above from being satisfiable by a function that always +// refuses. +test fn a_pool_built_with_an_observed_revision_is_admitted() -> Bool { + match decode_probe(stdout: probe_stdout(lines: probe_good_lines)) { + ProbeUnreadable { detail: _ } => false + ProbeFields { fields: f } => + match pool_from_fields(f: f, observed_revision: Present { value: vllm_source_revision_read }) { + PoolUnobserved { detail: _ } => false + PoolObserved { pool: _ } => true + } + } +} + +// A SHORT CAPTURE IS NOT A PARTIAL READ. The script emits exactly eleven lines; anything else means +// the far side did not run it, and the arity refusal precedes every field decode. +test fn a_capture_that_is_not_eleven_lines_refuses_before_decoding_any_field() -> Bool { + match decode_probe(stdout: "abc123\n1757280000") { + ProbeFields { fields: _ } => false + ProbeUnreadable { detail: d } => string_contains(s: d, pattern: "not the 11 the script emits") + } +} diff --git a/dag/test/claim/spark/pair_serving_capacity_floor_witness_test.dag b/dag/test/claim/spark/pair_serving_capacity_floor_witness_test.dag new file mode 100644 index 00000000000..c0c013b95a1 --- /dev/null +++ b/dag/test/claim/spark/pair_serving_capacity_floor_witness_test.dag @@ -0,0 +1,193 @@ +module test.claim.spark.pair_serving_capacity_floor_witness + +import std.types { Bool, String, Int, NonEmptyStr, List } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.content_hash { ContentHash, Sha256Hash, Sha256Digest, Sha256DigestHex } +import std.measure { TokenCount, token_count } +import gunbc.spark.fabric_switch_observed { FabricGroupA } +import gunbc.spark.pair_serving_desired { spark_pair_serving_head } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch, vllm_serving_launch } +import extdeps.vllm.log_lines { vllm_startup_reading, GpuKvCacheSize } +import extdeps.vllm.kv_cache { VllmObservedKvPool, KvPoolAdmitted, KvPoolRefused, vllm_observed_kv_pool_from_reading } +import extdeps.vllm.kv_layout { + KvBlockSize, KvLayerCount, KvGroupLayout, FullAttention, ResolvedKvLayout, resolved_kv_layout, + VllmReportedCapacity, vllm_reported_capacity, +} +import gunbc.spark.vllm_kv_layout_observe { VllmKvLayoutReceipt, KvLayoutObserved, KvLayoutInconsistent, admit_kv_layout } +import gunbc.harness.harness_seat { harness_seat_operator_policy_seats } +import gunbc.spark.pair_serving_capacity_floor { + MinimumColdSessions, spark_pair_serving_minimum_cold_sessions, + CapacityFloorStanding, MeetsFloor, BelowFloor, FloorUnestablished, capacity_floor_standing, + AdmissionPosture, Admitting, Withdrawn, admission_posture, + DrainStanding, Quiesced, GrantsOutstanding, DrainUnobserved, + RestartPreconditionGap, EngineStillAdmitting, GrantsStillHeld, QuiescenceUnobserved, + RetryAllowanceExhausted, UnifiedMemoryPlanUnmodelled, restart_precondition_wire, + RestartAdmission, RestartAdmitted, RestartRefused, RetryAllowance, restart_admission, + ReAdmission, ReAdmitted, ReAdmissionRefused, re_admit, +} + +fn w_digest(hex: String) -> ContentHash { + Sha256Hash(Sha256Digest { hex: hex as Sha256DigestHex }) +} + +data w_hex: String = "1111111111111111111111111111111111111111111111111111111111111111" + +fn w_inc(invocation: String) -> VllmServingLaunch { + vllm_serving_launch( + group: FabricGroupA, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: invocation as NonEmptyStr, + unit_pinned_base_image_digest: w_digest(hex: w_hex), + unit_digest: w_digest(hex: w_hex), + started_at: 1788000000, + ) +} + +fn w_pool(tokens: Int) -> VllmObservedKvPool { + match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: tokens), + reading: vllm_startup_reading(line: GpuKvCacheSize, captured_text: "GPU KV cache size: n tokens" as NonEmptyStr, observed_at_revision: vllm_source_revision_read), + ) { + KvPoolAdmitted { pool: p } => p + KvPoolRefused { cause: _ } => w_pool(tokens: tokens) + } +} + +fn w_full(layers: KvLayerCount) -> KvGroupLayout { + KvGroupLayout { kind: FullAttention, layer_count: layers, block_size: 16 } +} + +// A receipt whose groups reproduce its printed lines: one full-attention layer at block 16 costs +// 65,536 blocks for a full-context request, so an inventory of `blocks` against a pool line of +// blocks*16 tokens inverts to exactly that. The inventory is the knob this witness turns. +fn w_receipt(invocation: String, blocks: Nat) -> VllmKvLayoutReceipt { + admit_kv_layout( + incarnation: w_inc(invocation: invocation), + layout: resolved_kv_layout( + allocator_blocks: blocks, + scheduler_block_size: token_count(count: 16), + hash_block_size: token_count(count: 16), + max_model_len: token_count(count: 1048576), + max_num_batched_tokens: token_count(count: 8192), + groups: [w_full(layers: 1)], + ), + reported: vllm_reported_capacity(pool: w_pool(tokens: blocks * 16), max_concurrency_hundredths: (blocks * 16 * 100) / 1048576), + source_capture_digest: w_digest(hex: w_hex), + ) +} + +// THE FLOOR IS ONE DECLARATION AND THE BLOCKS ARE DERIVED FROM IT. Three sessions at a 500,000-token +// budget cost ceil(500000/16) = 31,250 blocks each, so the floor needs 93,750. An engine with 93,750 +// meets it and one with 93,749 does not — and neither number is written down anywhere. +test fn the_block_requirement_is_derived_from_the_declared_session_floor() -> Bool { + spark_pair_serving_minimum_cold_sessions.sessions == harness_seat_operator_policy_seats + && (match capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 93750), floor: spark_pair_serving_minimum_cold_sessions) { + MeetsFloor { required_blocks: r, available_blocks: a } => r == 93750 && a == 93750 + _ => false + }) + && (match capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 93749), floor: spark_pair_serving_minimum_cold_sessions) { + BelowFloor { required_blocks: r, available_blocks: a } => r == 93750 && a == 93749 + _ => false + }) +} + +// THE FLOOR MOVES WITH THE DECLARATION, which is the control on the row above: if the requirement +// were a second literal rather than a derivation, halving the budget would leave it unchanged. +test fn halving_the_declared_budget_halves_the_requirement() -> Bool { + let half = MinimumColdSessions { maximum_sequence_tokens: token_count(count: 250000), sessions: 3 } + match capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 93749), floor: half) { + MeetsFloor { required_blocks: r, available_blocks: _ } => r == 46875 + _ => false + } +} + +// AN UNESTABLISHED CAPACITY WITHDRAWS RATHER THAN ASSUMING ADEQUACY. This is today's real state for +// every engine in the fleet: no group metadata, so the receipt is inconsistent, so the floor is +// unestablished, so the engine stops taking new work. Silence is not a pass. +test fn an_unestablished_capacity_withdraws_the_engine() -> Bool { + let inconsistent = admit_kv_layout( + incarnation: w_inc(invocation: "inv-a"), + layout: resolved_kv_layout( + allocator_blocks: 54475, + scheduler_block_size: token_count(count: 16), + hash_block_size: token_count(count: 16), + max_model_len: token_count(count: 1048576), + max_num_batched_tokens: token_count(count: 8192), + groups: [], + ), + reported: vllm_reported_capacity(pool: w_pool(tokens: 11767856), max_concurrency_hundredths: 1122), + source_capture_digest: w_digest(hex: w_hex), + ) + (match capacity_floor_standing(receipt: inconsistent, floor: spark_pair_serving_minimum_cold_sessions) { + FloorUnestablished { cause: _ } => true + _ => false + }) + && (match admission_posture(standing: capacity_floor_standing(receipt: inconsistent, floor: spark_pair_serving_minimum_cold_sessions)) { + Withdrawn { cause: c } => string_contains(s: c as String, pattern: "rather than assumed adequate") + Admitting { available_blocks: _ } => false + }) +} + +// THE DISCRIMINATING ROW: A BUSY ENGINE BELOW FLOOR IS WITHDRAWN AND NOT RESTARTED. This is the loop +// the ruling exists to prevent — free memory on a unified-memory host is a moving target, so +// "below floor therefore restart" bounces an engine with requests in flight and can come up smaller. +// Withdrawal is instant and reversible; the restart waits behind its own preconditions, and the +// refusal names the FIRST unmet one so an operator learns what to change. +test fn a_busy_engine_below_floor_is_withdrawn_but_not_restarted() -> Bool { + let below = capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 40000), floor: spark_pair_serving_minimum_cold_sessions) + let posture = admission_posture(standing: below) + let allowance = RetryAllowance { attempts_used: 0, allowance: 3 } + (match posture { Withdrawn { cause: _ } => true Admitting { available_blocks: _ } => false }) + && (match restart_admission(posture: posture, drain: GrantsOutstanding { count: 2 }, retries: allowance) { + RestartRefused { gap: GrantsStillHeld { count: n } } => n == 2 + _ => false + }) + && (match restart_admission(posture: posture, drain: DrainUnobserved { cause: "the seat partition head was unreadable" as NonEmptyStr }, retries: allowance) { + RestartRefused { gap: QuiescenceUnobserved { cause: _ } } => true + _ => false + }) +} + +// AN ENGINE STILL TAKING WORK IS NEVER RESTARTED, whatever else is true: withdrawal precedes drain, +// and drain precedes restart. +test fn an_admitting_engine_is_never_restarted() -> Bool { + let meets = capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 93750), floor: spark_pair_serving_minimum_cold_sessions) + match restart_admission(posture: admission_posture(standing: meets), drain: Quiesced, retries: RetryAllowance { attempts_used: 0, allowance: 3 }) { + RestartRefused { gap: EngineStillAdmitting } => true + _ => false + } +} + +// AND WITH EVERY DECIDABLE PRECONDITION MET, THE RESTART STILL REFUSES — naming the unified-memory +// plan. That gap is the honest one: on a GB10 the pool is a residual of a supply shared with the OS, +// the weights, the CUDA and NCCL contexts and this deployment's own Ray object store, and restarting +// without a plan over it is what produced 23.6 GiB once and 54.01 GiB later from one unit text. +// Admitting a restart here would be promising an outcome nothing modelled can predict. +test fn a_drained_withdrawn_engine_still_refuses_restart_naming_the_missing_memory_plan() -> Bool { + let below = capacity_floor_standing(receipt: w_receipt(invocation: "inv-a", blocks: 40000), floor: spark_pair_serving_minimum_cold_sessions) + let posture = admission_posture(standing: below) + (match restart_admission(posture: posture, drain: Quiesced, retries: RetryAllowance { attempts_used: 0, allowance: 3 }) { + RestartRefused { gap: UnifiedMemoryPlanUnmodelled } => true + _ => false + }) + && (match restart_admission(posture: posture, drain: Quiesced, retries: RetryAllowance { attempts_used: 3, allowance: 3 }) { + RestartRefused { gap: RetryAllowanceExhausted { used: u, allowance: a } } => u == 3 && a == 3 + _ => false + }) + && string_contains(s: restart_precondition_wire(g: UnifiedMemoryPlanUnmodelled), pattern: "23.6 GiB") +} + +// RE-ADMISSION IS EARNED BY A FRESH RECEIPT. A restart that completed proves a process started; on +// this hardware it proves nothing about the pool it came up with. A new incarnation that came up +// still short is refused, and one that meets the floor is admitted under its OWN identity. +test fn re_admission_is_decided_by_the_new_receipt_not_by_the_restart_having_happened() -> Bool { + (match re_admit(fresh: w_receipt(invocation: "inv-b", blocks: 40000), floor: spark_pair_serving_minimum_cold_sessions) { + ReAdmissionRefused { cause: c } => string_contains(s: c as String, pattern: "still below the floor") + ReAdmitted { incarnation: _, available_blocks: _ } => false + }) + && (match re_admit(fresh: w_receipt(invocation: "inv-b", blocks: 93750), floor: spark_pair_serving_minimum_cold_sessions) { + ReAdmitted { incarnation: _, available_blocks: a } => a == 93750 + ReAdmissionRefused { cause: _ } => false + }) +} diff --git a/dag/test/claim/spark/pair_serving_launch_receipt_witness_test.dag b/dag/test/claim/spark/pair_serving_launch_receipt_witness_test.dag new file mode 100644 index 00000000000..1dab5f13e9a --- /dev/null +++ b/dag/test/claim/spark/pair_serving_launch_receipt_witness_test.dag @@ -0,0 +1,133 @@ +module test.claim.spark.pair_serving_launch_receipt_witness + +import std.types { Bool, String, Int, NonEmptyStr, List, EpochSecs } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.content_hash { ContentHash, Sha256Hash, Sha256Digest, Sha256DigestHex } +import gunbc.spark.fabric_switch_observed { FabricGroupA } +import gunbc.spark.pair_serving_desired { spark_pair_serving_head } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch, vllm_serving_launch } +import gunbc.spark.pair_serving_launch_receipt { + LaunchTerminal, LaunchServing, LaunchExited, LaunchKilled, LaunchReadinessTimedOut, LaunchTerminalUnobserved, + launch_terminal_from_exit, launch_terminal_wire, + PairServingLaunchAttempt, + OomEvidenceSource, KernelRingBuffer, CgroupMemoryEvent, SystemdOomResult, + OomObservation, OomAttribution, KilledByOom, OomUnattributed, attribute_oom, + ServingStanding, ServingOnFirstAttempt, ServingRecoveredAfterFailures, NotServing, ServingStandingUnobserved, + serving_standing, serving_standing_wire, bring_up_was_clean, +} + +fn w_digest(hex: String) -> ContentHash { + Sha256Hash(Sha256Digest { hex: hex as Sha256DigestHex }) +} + +data w_exe: String = "1111111111111111111111111111111111111111111111111111111111111111" +data w_unit: String = "3333333333333333333333333333333333333333333333333333333333333333" + +fn w_inc(invocation: String) -> VllmServingLaunch { + vllm_serving_launch( + group: FabricGroupA, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: invocation as NonEmptyStr, + unit_pinned_base_image_digest: w_digest(hex: w_exe), + unit_digest: w_digest(hex: w_unit), + started_at: 1788000000, + ) +} + +fn w_attempt(invocation: String, n: Nat, terminal: LaunchTerminal) -> PairServingLaunchAttempt { + PairServingLaunchAttempt { + incarnation: w_inc(invocation: invocation), + desired_digest: w_digest(hex: w_unit), + attempt_number: n, + started_at: 1788000000, + terminal: terminal, + layout_receipt: none, + } +} + +// EXIT 137 IS A SIGKILL SHAPE. 128+n is the convention for "terminated by signal n", so the group A +// head's status=137 classifies as signal 9 and NOT as an ordinary exit status. +test fn exit_codes_above_128_classify_as_the_signal_they_encode() -> Bool { + (match launch_terminal_from_exit(code: 137) { LaunchKilled { signal: s } => s == 9 _ => false }) + && (match launch_terminal_from_exit(code: 143) { LaunchKilled { signal: s } => s == 15 _ => false }) + && (match launch_terminal_from_exit(code: 0) { LaunchServing => true _ => false }) + && (match launch_terminal_from_exit(code: 1) { LaunchExited { code: c } => c == 1 _ => false }) +} + +// THE DISCRIMINATING ROW: exit 137 ALONE DOES NOT ESTABLISH OOM. This is the inference the incident +// invites and it is wrong -- signal 9 is delivered by the kernel's OOM killer, by `docker rm -f` +// (which this very unit runs in its own ExecStartPre), by a stop timeout escalating, and by an +// operator. A model that read the code as OOM would send someone to tune memory that was never the +// cause. The refusal names all three alternatives rather than saying "unknown". +test fn a_signal_kill_without_an_oom_observation_is_not_an_oom_kill() -> Bool { + let killed = w_attempt(invocation: "inv-15-42", n: 8, terminal: launch_terminal_from_exit(code: 137)) + match attribute_oom(attempt: killed, observation: none) { + OomUnattributed { cause: c } => + string_contains(s: c as String, pattern: "forced teardown") + && string_contains(s: c as String, pattern: "operator") + KilledByOom { source: _, detail: _ } => false + } +} + +// THE POSITIVE CONTROL: a joined observation DOES promote it, so the refusal above is about missing +// evidence and not about the attribution being unreachable. +test fn a_joined_oom_observation_promotes_the_kill() -> Bool { + let killed = w_attempt(invocation: "inv-15-42", n: 8, terminal: launch_terminal_from_exit(code: 137)) + let observed = OomObservation { + incarnation: w_inc(invocation: "inv-15-42"), + source: CgroupMemoryEvent, + detail: "memory.events oom_kill incremented" as NonEmptyStr, + } + match attribute_oom(attempt: killed, observation: Present { value: observed }) { + KilledByOom { source: s, detail: _ } => (match s { CgroupMemoryEvent => true _ => false }) + OomUnattributed { cause: _ } => false + } +} + +// AND AN OOM OBSERVATION ABOUT ANOTHER LAUNCH DOES NOT COUNT. This is the case a host-keyed join +// accepts: same host, same unit, same container name, a real OOM — belonging to a different start. +test fn an_oom_observation_from_another_launch_does_not_attribute() -> Bool { + let killed = w_attempt(invocation: "inv-15-42", n: 8, terminal: launch_terminal_from_exit(code: 137)) + let elsewhere = OomObservation { + incarnation: w_inc(invocation: "inv-15-35"), + source: KernelRingBuffer, + detail: "Out of memory: Killed process" as NonEmptyStr, + } + match attribute_oom(attempt: killed, observation: Present { value: elsewhere }) { + OomUnattributed { cause: c } => string_contains(s: c as String, pattern: "another launch") + KilledByOom { source: _, detail: _ } => false + } +} + +// A UNIT THAT CAME UP ON THE NINTH TRY IS NOT A UNIT THAT CAME UP. This is the 2026-09-05 group A +// bring-up: eight signal kills, then a start that stuck. To systemd and to the current apply path +// the end state is indistinguishable from a first-attempt launch; here it is a different standing, +// and the failed terminals travel with it. +test fn a_recovered_bring_up_is_distinguishable_from_a_clean_one() -> Bool { + let flapped = concat( + map([1, 2, 3, 4, 5, 6, 7, 8], n => w_attempt(invocation: "inv-15-3", n: n, terminal: launch_terminal_from_exit(code: 137))), + [w_attempt(invocation: "inv-16-40", n: 9, terminal: LaunchServing)], + ) + let clean = [w_attempt(invocation: "inv-16-40", n: 1, terminal: LaunchServing)] + (match serving_standing(attempts: flapped) { + ServingRecoveredAfterFailures { incarnation: _, failed_attempts: n, terminals: ts } => + n == 8 && length(ts) == 8 + _ => false + }) + && (match serving_standing(attempts: clean) { ServingOnFirstAttempt { incarnation: _ } => true _ => false }) + && !bring_up_was_clean(s: serving_standing(attempts: flapped)) + && bring_up_was_clean(s: serving_standing(attempts: clean)) + && string_contains(s: serving_standing_wire(s: serving_standing(attempts: flapped)), pattern: "8 failed attempt") +} + +// NO ATTEMPTS IS UNOBSERVED, NOT NOT-SERVING. An empty list is the absence of evidence, and reading +// it as a failure would make an unconverged host indistinguishable from a broken one. +test fn no_recorded_attempts_is_unobserved_rather_than_not_serving() -> Bool { + (match serving_standing(attempts: []) { ServingStandingUnobserved { cause: _ } => true _ => false }) + && (match serving_standing(attempts: [w_attempt(invocation: "inv-1", n: 1, terminal: LaunchReadinessTimedOut)]) { + NotServing { failed_attempts: n, terminals: _ } => n == 1 + _ => false + }) + && !bring_up_was_clean(s: serving_standing(attempts: [])) +} diff --git a/dag/test/claim/spark/serving_critical_path_witness_test.dag b/dag/test/claim/spark/serving_critical_path_witness_test.dag new file mode 100644 index 00000000000..f1aefebfc1a --- /dev/null +++ b/dag/test/claim/spark/serving_critical_path_witness_test.dag @@ -0,0 +1,253 @@ +module test.claim.spark.serving_critical_path_witness + +import std.types { Bool, String, Int, NonEmptyStr, List } +import std.nat { Nat } +import std.measure { microsecond_count, watt, watt_count } +import v2.std.optional { Present, Absent } +import gunbc.spark.serving_critical_path { + spark_system_power_supply, spark_soc_thermal_envelope, + GpuPowerFractionStanding, GpuPowerFraction, GpuDenominatorUnestablished, spark_gpu_power_fraction, + SaturationClaim, GpuComputeSaturated, GpuComputeUnderutilised, + ClaimConstruction, ClaimConstructible, ClaimNotConstructible, + busy_fraction_constructs, power_below_ceiling_excludes_saturation, + EffectiveBatchBudget, BehaviouralPerIterationCapacityAtLeast, ConfiguredSchedulerBudgetRead, + ConfiguredSchedulerBudgetUnread, + spark_group_a_effective_batch_budget, + EffectiveSchedulerConfigStanding, EffectiveSchedulerConfigRead, EffectiveSchedulerConfigUnread, + ConcurrencyWitness, co_tenant_present, + PrefillScalingSample, PrefillScalingExperiment, spark_group_a_prefill_scaling_experiment, + SampleStanding, ProbeAloneAtObservedConcurrencyGrain, NoForeignCachedBookingObserved, + ForeignCachedBookingObserved, sample_standing, + ExperimentLaunchBinding, ExperimentLaunchBound, ExperimentLaunchUnbound, + ExperimentJoinability, JoinableByLaunch, NotJoinable, experiment_joinability, + FitArm, ArrivedIdle, ArrivedWithCoTenant, sample_in_arm, arm_samples, + AffineFit, PrefillScalingStanding, AffineDescription, ScalingUnderObserved, + arm_fit, predicted_micros, abs_int, arm_worst_residual_micros, + SharedMinusIdleFitContrast, shared_minus_idle_contrast, + ServingCostSubject, PrefillSelfCost, CoTenantExternality, + PolicyLeverage, OuterPolicyCanAddress, RequiresInnerExecutionChange, policy_leverage, + PerStepCostStanding, PerStepCostNotSeparable, + spark_group_a_per_step_cost_standing, + BudgetTradeStanding, BudgetTradeUnestablished, spark_group_a_budget_trade, + ScalingSampleExcludedFromFit, spark_group_a_co_tenant_outlier, + CriticalPathMechanism, GrossRankStraggler, HostDispatchGaps, BulkInterconnectBandwidth, + FixedPerRequestCost, TokenProportionalCompute, TokenProportionalCollectivePayload, + CollectiveSynchronisationWait, MemoryOrKernelEfficiency, PowerOrThermalThrottle, CoTenantDelay, + MechanismStanding, NotObservedAtThisGrain, Observed, Unresolved, + cold_prefill_mechanism_standing, + PartitionRequirement, ComputeVersusCollectiveKernelSpans, PerRankCriticalPathTiming, + OccupancyOrAchievedThroughput, ThrottleAndMemoryStallEvidence, + HigherConcurrencyCoTenantParameterisation, ChunkedRegimeMeasurement, + CriticalPathPartition, PartitionIncomplete, PartitionEstablished, + spark_group_a_cold_prefill_partition, partition_may_calibrate, + LinkUpgradeStanding, LinkUpgradeJustified, LinkUpgradeUnsupportedByCurrentEvidence, + link_upgrade_standing, +} + +// THE INFERENCE THAT WAS WRONG IS NOW UNWRITABLE. An earlier version excluded compute saturation +// because power did not rise with load. Low whole-GPU power cannot exclude saturation of a narrow +// bottleneck, and kernel residency decides neither direction. +test fn neither_saturation_nor_underutilisation_is_constructible_from_coarse_telemetry() -> Bool { + (match busy_fraction_constructs(claim: GpuComputeSaturated) { + ClaimNotConstructible { because: b } => string_contains(s: b as String, pattern: "residency, not work") + ClaimConstructible { from: _ } => false + }) + && (match busy_fraction_constructs(claim: GpuComputeUnderutilised) { + ClaimNotConstructible { because: b } => string_contains(s: b as String, pattern: "narrow bottleneck") + ClaimConstructible { from: _ } => false + }) + && (match power_below_ceiling_excludes_saturation() { + ClaimNotConstructible { because: b } => string_contains(s: b as String, pattern: "bounds whole-chip utilisation") + ClaimConstructible { from: _ } => false + }) +} + +// THE ENVELOPES NEST, and adding them is the first mistake available. +// +// The first three conjuncts consume the two PUBLISHED facts from extdeps.systems.nvidia rather than +// re-minting them here. The fourth is the discriminating half: an observed 36-44W draw cannot be +// turned into a fraction of a GPU-only ceiling nobody published, and the refusal names the missing +// denominator rather than substituting a plausible one. +test fn the_power_facts_are_read_from_the_catalog_and_the_gpu_ceiling_refuses() -> Bool { + watt_count(w: spark_system_power_supply) == 240 + && watt_count(w: spark_soc_thermal_envelope) == 140 + && watt_count(w: spark_soc_thermal_envelope) < watt_count(w: spark_system_power_supply) + && (match spark_gpu_power_fraction(observed: watt(count: 40)) { + GpuDenominatorUnestablished { because: b } => string_contains(s: b as String, pattern: "no GPU-only figure") + GpuPowerFraction { percent: _ } => false + }) +} + +// THE BUDGET IS OBSERVED BEHAVIOURALLY AND IS AN ORDER OF MAGNITUDE ABOVE THE NOMINAL 2048. Every +// idle window in the population ran in exactly one engine iteration, at every size up to 20,082 +// computed tokens. Three separate conclusions in this branch's history rested on a 2048 boundary +// that does not exist. +test fn no_idle_window_chunks_anywhere_in_the_measured_range() -> Bool { + let idle = arm_samples(ArrivedIdle) + length(idle) == 12 + && fold(idle, init: true, f: (acc, s) => acc && s.iteration_delta == 1) + && length(filter(idle, s => s.computed_tokens == 20082)) == 1 + && length(filter(idle, s => s.computed_tokens > 20082)) == 0 + && (match spark_group_a_effective_batch_budget { + BehaviouralPerIterationCapacityAtLeast { computed_tokens: t, by: _ } => t == 20082 + ConfiguredSchedulerBudgetRead { tokens: _, source: _ } => false + ConfiguredSchedulerBudgetUnread { because: _ } => false + }) +} + +// AND ON A SHARED ENGINE THE SAME COUNTER MEANS NOTHING PER REQUEST. Three windows of about 2,050 +// computed tokens recorded 5, 5 and 5 iterations while one of 1,564 recorded 17 -- because co-tenant +// decode steps increment the engine-wide counter too. This is what made 158,764/83 look like a +// scheduler budget. +test fn the_iteration_counter_is_not_a_per_request_step_count_under_co_tenancy() -> Bool { + let busy = arm_samples(ArrivedWithCoTenant) + length(busy) == 9 + && fold(busy, init: true, f: (acc, s) => acc && s.iteration_delta > 1) + && length(filter(busy, s => s.iteration_delta == 17)) == 1 + && length(filter(busy, s => s.iteration_delta > 17)) == 0 +} + +// max_running COUNTS THE PROBE, so a window with one running request had no co-tenant. Arrival +// concurrency is what selects the arm, and it is recorded separately from the maximum during. +test fn co_tenancy_is_read_from_the_field_rather_than_assumed() -> Bool { + !co_tenant_present(c: ConcurrencyWitness { max_running: 1, max_waiting: 0 }) + && co_tenant_present(c: ConcurrencyWitness { max_running: 2, max_waiting: 0 }) + && co_tenant_present(c: ConcurrencyWitness { max_running: 1, max_waiting: 1 }) + && length(filter(spark_group_a_prefill_scaling_experiment.ordered_samples, s => + match sample_standing(s: s) { ForeignCachedBookingObserved { cached_tokens: c } => c > 100000 _ => false })) == 1 +} + +// THE ARMS AGREE ON THE MARGINAL RATE TO WITHIN A FEW PERCENT, and the difference is not resolved in +// sign -- the idle slope estimate moves across that band depending on which idle subset is fitted. So +// the tolerance here is the claim: moderate co-tenancy did not materially change per-token cost. +// Asserting a tighter band would assert a resolution the samples do not have. +test fn the_arms_agree_on_the_marginal_rate_within_a_few_percent() -> Bool { + match shared_minus_idle_contrast() { + Absent => false + Present { value: c } => + c.marginal_percent_difference <= 4 + && c.marginal_percent_difference >= (0 - 4) + && c.at_observed_concurrency == 2 + } +} + +// THE INTERCEPT DIFFERENCE IS LARGER AND SURVIVES RANGE MATCHING, but it is a contrast between two +// FITTED intercepts and is not named a co-tenant cost. +test fn the_intercept_contrast_is_carried_as_a_fit_contrast_not_a_cost() -> Bool { + match shared_minus_idle_contrast() { + Absent => false + Present { value: c } => c.intercept_difference_micros > 100000 && c.intercept_difference_micros < 260000 + } +} + +// SELF-COST AND EXTERNALITY ARE DIFFERENT SUBJECTS WITH OPPOSITE POLICY ANSWERS. This is the +// discriminator against the reading that outer layers "recover the fixed term and nothing else": +// that conflated the two and argued against the most useful relief available. +test fn policy_leverage_differs_between_self_cost_and_externality() -> Bool { + (match policy_leverage(subject: PrefillSelfCost) { + RequiresInnerExecutionChange { by: b } => string_contains(s: b as String, pattern: "only the execution layers") + OuterPolicyCanAddress { by: _ } => false + }) + && (match policy_leverage(subject: CoTenantExternality) { + OuterPolicyCanAddress { by: b } => string_contains(s: b as String, pattern: "3.8s against 63.5s") + RequiresInnerExecutionChange { by: _ } => false + }) +} + +// THE IDLE FIT IS DERIVED AND TIGHT ACROSS A 39x TOKEN RANGE. The slope is a least squares projection +// of the sample rows, not a literal: ~1.055 ms per token, about 948 tok/s marginal. +test fn the_idle_marginal_rate_is_a_projection_of_the_samples() -> Bool { + match arm_fit(ArrivedIdle) { + ScalingUnderObserved { cause: _ } => false + AffineDescription { fit: f } => + f.sample_count == 12 + && f.micros_per_thousand_tokens >= 1000000 + && f.micros_per_thousand_tokens <= 1120000 + && f.intercept_micros >= 0 + && f.intercept_micros <= 140000 + && arm_worst_residual_micros(arm: ArrivedIdle, fit: f) < 120000 + } +} + +// THE PER-STEP QUESTION IS DISSOLVED, NOT BOUNDED AND NOT ESTIMATED. Two earlier versions answered +// it -- one with a 95ms upper bound read off a residual, one with a 126ms three-term counterexample. +// Both regressed against a step boundary that the measurement shows does not exist in range. +test fn the_per_step_question_is_dissolved_rather_than_answered() -> Bool { + match spark_group_a_per_step_cost_standing { + PerStepCostNotSeparable { because: b, range_ceiling_tokens: t } => + t == 20082 && string_contains(s: b as String, pattern: "exactly one engine iteration") + } +} + +// THE BUDGET COUNTERFACTUAL STAYS WITHDRAWN, and the measurement supplied a new reason: the engine +// does not chunk in the range anyone would tune toward, so a smaller budget is not currently a knob. +test fn the_budget_counterfactual_stays_withdrawn_with_a_measured_reason() -> Bool { + match spark_group_a_budget_trade { + BudgetTradeUnestablished { missing: m } => + length(m) == 3 + && fold(m, init: false, f: (acc, r) => acc || string_contains(s: r as String, pattern: "at least 20,082")) + && fold(m, init: false, f: (acc, r) => acc || string_contains(s: r as String, pattern: "invariance across chunk shapes")) + } +} + +// THE PROPORTIONAL TERM REMAINS UNATTRIBUTED, which is now the whole of the problem. Compute and +// collective payload both scale with tokens and neither is decided. +test fn the_proportional_mechanisms_remain_unresolved() -> Bool { + (match cold_prefill_mechanism_standing(m: TokenProportionalCompute) { Unresolved { needs: _ } => true _ => false }) + && (match cold_prefill_mechanism_standing(m: TokenProportionalCollectivePayload) { Unresolved { needs: _ } => true _ => false }) + && (match cold_prefill_mechanism_standing(m: CollectiveSynchronisationWait) { Unresolved { needs: _ } => true _ => false }) + && (match cold_prefill_mechanism_standing(m: MemoryOrKernelEfficiency) { Unresolved { needs: _ } => true _ => false }) + && (match cold_prefill_mechanism_standing(m: FixedPerRequestCost) { + Unresolved { needs: b } => string_contains(s: b as String, pattern: "INTERCEPT ESTIMATES") + _ => false + }) + && (match cold_prefill_mechanism_standing(m: CoTenantDelay) { + Observed { by: b } => string_contains(s: b as String, pattern: "matched treatment and control") + _ => false + }) +} + +// THE OUTLIER IS RETAINED AND IS NOW VISIBLY OUTSIDE THE MEASURED CO-TENANT COST, which is the point +// of keeping it: 291 seconds is not 0.13 seconds of scheduling latency, so it remains unexplained +// rather than absorbed into the contrast. +test fn the_outlier_is_kept_and_is_not_explained_by_the_measured_co_tenant_cost() -> Bool { + spark_group_a_co_tenant_outlier.computed_tokens == 4840 + && microsecond_count(m: spark_group_a_co_tenant_outlier.wall_micros) == 290960000 + && string_contains(s: spark_group_a_co_tenant_outlier.cause as String, pattern: "unexplained") +} + +// THE UNBOUND LAUNCH STILL REFUSES TO JOIN. No VllmServingLaunch exists for these sweeps, so nothing +// may compare them to another launch by identity. +test fn the_unbound_launch_refuses_to_join_this_experiment_to_anything() -> Bool { + match experiment_joinability(e: spark_group_a_prefill_scaling_experiment) { + JoinableByLaunch { launch: _ } => false + NotJoinable { because: b } => string_contains(s: b as String, pattern: "may not be joined by launch identity") + } +} + +// THE PARTITION IS STILL INCOMPLETE, but three requirements were discharged by measurement and the +// two that replaced them are named by membership rather than counted. +test fn the_partition_is_incomplete_and_names_what_measurement_did_not_close() -> Bool { + !partition_may_calibrate(p: spark_group_a_cold_prefill_partition) + && (match spark_group_a_cold_prefill_partition { + PartitionEstablished { critical: _, evidence: _ } => false + PartitionIncomplete { missing: m } => + length(m) == 6 + && contains(m, ComputeVersusCollectiveKernelSpans) + && contains(m, PerRankCriticalPathTiming) + && contains(m, HigherConcurrencyCoTenantParameterisation) + && contains(m, ChunkedRegimeMeasurement) + }) +} + +// THE LINK VERDICT IS READ FROM THE PARTITION, with an established-bulk-bandwidth positive control. +test fn the_link_verdict_is_derived_and_a_bulk_bandwidth_partition_flips_it() -> Bool { + (match link_upgrade_standing(p: spark_group_a_cold_prefill_partition) { + LinkUpgradeUnsupportedByCurrentEvidence { by: b } => string_contains(s: b as String, pattern: "weaker than a proof of no benefit") + LinkUpgradeJustified { by: _ } => false + }) + && (match link_upgrade_standing(p: PartitionEstablished { critical: BulkInterconnectBandwidth, evidence: "hypothetical established bulk saturation" as NonEmptyStr }) { + LinkUpgradeJustified { by: _ } => true + LinkUpgradeUnsupportedByCurrentEvidence { by: _ } => false + }) +} diff --git a/dag/test/claim/spark/serving_engine_witness_test.dag b/dag/test/claim/spark/serving_engine_witness_test.dag index b51bd826147..defdcd2446b 100644 --- a/dag/test/claim/spark/serving_engine_witness_test.dag +++ b/dag/test/claim/spark/serving_engine_witness_test.dag @@ -31,6 +31,7 @@ import extdeps.ollama.api { ollama_default_port } // THE FIXTURES CARRY NO FLEET FACTS. Every value below is authored here, so a row reds because the // RULE changed and never because a host row moved underneath it. data fx_args: VllmEngineArgs = VllmEngineArgs { + prefill_scheduling: none, tensor_parallel_size: 2, gpu_memory_utilization: vllm_memory_utilization(fraction: percent(count: 82)), max_model_len: token_count(count: 262144), @@ -301,7 +302,7 @@ test fn w_ray_on_a_discrete_host_is_outside_the_counting_rule() -> Bool { // A negative row with no error text is a belief. This asserts the population is non-empty so the // carrier cannot quietly become an empty list that every consumer reads as "nothing is rejected". test fn w_the_verified_negatives_carry_upstream_error_text() -> Bool { - length(vllm_verified_negatives) > 0 && length(vllm_discouraged_args) > 0 + length(vllm_verified_negatives) > 0 } test fn w_all_serving_engine_claims_hold() -> Bool { diff --git a/dag/test/claim/spark/serving_launch_instrumentation_witness_test.dag b/dag/test/claim/spark/serving_launch_instrumentation_witness_test.dag new file mode 100644 index 00000000000..20a1f74f784 --- /dev/null +++ b/dag/test/claim/spark/serving_launch_instrumentation_witness_test.dag @@ -0,0 +1,112 @@ +module test.claim.spark.serving_launch_instrumentation_witness + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import gunbc.spark.serving_launch_instrumentation { + AttributionInstrument, NsightSystems, TorchProfilerEndpoint, FirstPartyStepObserver, + PostHocBindability, BindsToRunningProcess, BindsOnlyAtProcessStart, + instrument_post_hoc_bindability, + AttributionRemedy, RemedyIsAdditionalPrivilege, RemedyIsANewLaunch, attribution_remedy, + LaunchInstrumentation, spark_group_a_pair_head_instrumentation, + AttributionAvailability, AttributableBy, NotAttributableOnThisLaunch, + launch_attribution_availability, + PerturbationControl, MatchedUnprofiledControl, NoControlTaken, + CanaryLaunchProfile, spark_group_a_attribution_canary, + CanaryComparability, ComparableToProduction, NotComparableToProduction, canary_comparability, + RankCoverage, AllRanksCaptured, PartialRankCoverage, rank_coverage_admits_attribution, +} +import gunbc.spark.serving_critical_path { + MissingEvidenceDisposition, ObtainableOnThisLaunch, RequiresADifferentLaunch, + attribution_disposition, +} + +// EVERY ATTRIBUTION INSTRUMENT BINDS AT PROCESS START, and each says which measurement established +// it. This is the whole finding: nsys is installed and the profiler endpoint exists in the codebase, +// so nothing is missing from the host except a binding only a start could have made. +test fn no_attribution_instrument_binds_to_a_running_process() -> Bool { + fold([NsightSystems, TorchProfilerEndpoint, FirstPartyStepObserver], init: true, f: (acc, i) => + acc && (match instrument_post_hoc_bindability(i: i) { + BindsOnlyAtProcessStart { because: _ } => true + BindsToRunningProcess { by: _ } => false + })) + && (match instrument_post_hoc_bindability(i: NsightSystems) { + BindsOnlyAtProcessStart { because: b } => string_contains(s: b as String, pattern: "no attach to an arbitrary pid") + BindsToRunningProcess { by: _ } => false + }) + && (match instrument_post_hoc_bindability(i: TorchProfilerEndpoint) { + BindsOnlyAtProcessStart { because: b } => string_contains(s: b as String, pattern: "404") + BindsToRunningProcess { by: _ } => false + }) +} + +// SO NO GRANT REPAIRS IT. The tempting move after a refused measurement is to ask for more access; +// this makes the answer structural rather than advisory. If any instrument ever did bind to a running +// process, the remedy would flip to privilege and this goes red. +test fn the_remedy_is_never_additional_privilege() -> Bool { + fold([NsightSystems, TorchProfilerEndpoint, FirstPartyStepObserver], init: true, f: (acc, i) => + acc && (match attribution_remedy(i: i) { + RemedyIsANewLaunch { because: b } => string_contains(s: b as String, pattern: "no capability granted to an observer reaches it") + RemedyIsAdditionalPrivilege { by: _ } => false + })) +} + +// THE SERVING LAUNCH CARRIES NONE, which is correct for serving and is why attribution refused. +test fn the_production_launch_is_not_attributable_and_names_its_remedy() -> Bool { + length(spark_group_a_pair_head_instrumentation.bound_instruments) == 0 + && (match launch_attribution_availability(l: spark_group_a_pair_head_instrumentation) { + NotAttributableOnThisLaunch { because: _, remedy: r } => string_contains(s: r as String, pattern: "binds only at process start") + AttributableBy { instrument: _ } => false + }) + && string_contains(s: spark_group_a_pair_head_instrumentation.observed_by as String, pattern: "sessions list empty") +} + +// AND A LAUNCH THAT DID BIND ONE WOULD BE ATTRIBUTABLE. The positive control: this is a property of +// the launch, not a permanent property of the fabric. +test fn a_launch_that_bound_an_instrument_is_attributable() -> Bool { + match launch_attribution_availability(l: LaunchInstrumentation { + bound_instruments: [FirstPartyStepObserver], + observed_by: "hypothetical launch declaring a step observer" as NonEmptyStr, + revision: spark_group_a_pair_head_instrumentation.revision, + }) { + AttributableBy { instrument: _ } => true + NotAttributableOnThisLaunch { because: _, remedy: _ } => false + } +} + +// A CANARY WITHOUT A MATCHED CONTROL CANNOT BE COMPARED TO PRODUCTION AT ALL. The control is a field +// rather than a recommendation precisely so this refusal exists. +test fn a_canary_without_a_control_refuses_comparison() -> Bool { + (match canary_comparability(c: spark_group_a_attribution_canary) { + ComparableToProduction { via: v } => string_contains(s: v as String, pattern: "measured rather than assumed negligible") + NotComparableToProduction { because: _ } => false + }) + && (match canary_comparability(c: CanaryLaunchProfile { + instruments: [NsightSystems], + workload: "same workload" as NonEmptyStr, + control: NoControlTaken, + rank_coverage_required: 4, + }) { + NotComparableToProduction { because: b } => string_contains(s: b as String, pattern: "perturbation is unmeasured") + ComparableToProduction { via: _ } => false + }) +} + +// PARTIAL RANK COVERAGE ADMITS NOTHING on a tensor-parallel deployment, because a missing rank is +// exactly where a straggler would hide. +test fn partial_rank_coverage_admits_no_attribution() -> Bool { + rank_coverage_admits_attribution(c: AllRanksCaptured { count: 4 }) + && !rank_coverage_admits_attribution(c: PartialRankCoverage { captured: 3, required: 4 }) + && spark_group_a_attribution_canary.rank_coverage_required == 4 +} + +// AND THE CRITICAL-PATH PARTITION NOW READS ITS MISSING EVIDENCE AS UNOBTAINABLE-HERE rather than +// undone, which points at the launch profile instead of at another attempt with the same tools. +test fn the_partitions_missing_attribution_requires_a_different_launch() -> Bool { + match attribution_disposition() { + ObtainableOnThisLaunch { by: _ } => false + RequiresADifferentLaunch { because: b, remedy: r } => + string_contains(s: b as String, pattern: "no attribution instrument was bound") + && string_contains(s: r as String, pattern: "declares its instrumentation") + } +} diff --git a/dag/test/claim/spark/serving_observer_compatibility_witness_test.dag b/dag/test/claim/spark/serving_observer_compatibility_witness_test.dag new file mode 100644 index 00000000000..9a570b2ec8c --- /dev/null +++ b/dag/test/claim/spark/serving_observer_compatibility_witness_test.dag @@ -0,0 +1,213 @@ +module test.claim.spark.serving_observer_compatibility_witness + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision, vllm_source_revision_read } +import extdeps.vllm.scheduler_extension { + VllmSchedulerSelection, SelectedByQualname, SelectedByTypeObject, SelectedByUpstreamDefault, + VllmAsyncSchedulingRequest, AsyncSchedulingOn, AsyncSchedulingOff, AsyncSchedulingUnset, + ResolvedAsyncScheduling, AsyncResolvedOn, AsyncResolvedOff, resolve_async_scheduling, + VllmExecutorAsyncSupport, ExecutorSupportsAsyncScheduling, ExecutorDoesNotSupportAsyncScheduling, + executor_async_scheduling_support, + vllm_default_scheduler_qualname, + SchedulerSelectionTiming, SelectableWhileRunning, SelectableOnlyAtEngineConstruction, + scheduler_selection_timing, vllm_scheduler_extension_point, + SchedulerSurfaceBinding, SurfaceBound, SurfaceRefusedForMissingField, SurfaceRefusedForRevisionDrift, + surface_binding_for_revision, scheduler_output_fields_read, + UpstreamStabilityPosture, UpstreamPublicInterface, UpstreamDisclaimsCompatibility, + vllm_scheduler_interface_posture, +} +import extdeps.vllm.engine_args { VllmDistributedExecutorBackend, ExecutorMp, ExecutorRay, ExecutorExternalLauncher } +import gunbc.spark.serving_observer_compatibility { + ReceiptReadSite, ReadFromSchedulerOutput, ReadFromLiveRequest, + ReadSiteAdmission, ReadSiteAdmitted, ReadSiteRefused, read_site_admits, + ReceiptFieldBinding, ReceiptFieldSource, FromSchedulerOutputField, FromSchedulerMethod, FromObserverClock, + receipt_field_bindings, + ObserverProfile, observer_profile, ObserverProfileAdmission, ObserverProfileAdmitted, ObserverProfileRefused, + ObserverProfileRefusal, ProfileRevisionDrift, ProfileAlreadyHasForeignScheduler, ProfileAsyncSchedulingUnmodelled, + observer_profile_admits, + ManifestStanding, ManifestConsumedByWheel, ManifestAwaitingWheel, serving_observer_manifest_standing, +} + +// -- THE READ SITE IS THE WHOLE CONTRACT ------------------------------------------------------- +// +// This is the discriminating pair for the defect that motivated the manifest: reading the same field +// from the live Request instead of from SchedulerOutput yields a cached_tokens that already contains +// this step's own computed_tokens. If read_site_admits ever admitted the live-request site, a wrapper +// written against this contract would produce internally inconsistent receipts and nothing downstream +// could detect it, so this going green on both arms is the failure this test exists to catch. +test fn the_live_request_read_site_is_refused_and_the_output_site_is_admitted() -> Bool { + (match read_site_admits(site: ReadFromSchedulerOutput { attribute: "num_scheduled_tokens" as NonEmptyStr }) { + ReadSiteAdmitted { attribute: a } => a as String == "num_scheduled_tokens" + ReadSiteRefused { because: _ } => false + }) + && (match read_site_admits(site: ReadFromLiveRequest { because: "the wrapper holds the scheduler" as NonEmptyStr }) { + ReadSiteRefused { because: b } => string_contains(s: b as String, pattern: "double-counts the step being recorded") + ReadSiteAdmitted { attribute: _ } => false + }) +} + +// -- THE DEFAULT DELEGATE IS CONDITIONAL ------------------------------------------------------- +// +// A wrapper that hardcodes Scheduler silently converts an async-scheduled engine to a synchronous one +// while reporting a behaviour-preserving observation. The two arms must name DIFFERENT classes; if a +// future edit collapsed them this goes red rather than shipping that conversion. +test fn the_delegate_follows_the_resolved_value_not_the_requested_flag() -> Bool { + vllm_default_scheduler_qualname(resolved: AsyncResolvedOn { because: "x" as NonEmptyStr }) as String + == "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + && vllm_default_scheduler_qualname(resolved: AsyncResolvedOff { because: "x" as NonEmptyStr }) as String + == "vllm.v1.core.sched.scheduler.Scheduler" +} + +// THE DISCRIMINATOR THE LIVE ENGINE REVEALED. One unset flag, two executors, two delegates. This is +// the pair that was unrepresentable while AsyncSchedulingUnset mapped straight to Scheduler, and it +// is the exact transition replacing Ray with a first-party rank protocol would perform. +test fn an_unset_async_flag_resolves_differently_per_executor_backend() -> Bool { + (match resolve_async_scheduling(request: AsyncSchedulingUnset, backend: ExecutorRay) { + AsyncResolvedOff { because: b } => string_contains(s: b as String, pattern: "cannot support it") + AsyncResolvedOn { because: _ } => false + }) + && (match resolve_async_scheduling(request: AsyncSchedulingUnset, backend: ExecutorMp) { + AsyncResolvedOn { because: b } => string_contains(s: b as String, pattern: "unless an incompatible option applies") + AsyncResolvedOff { because: _ } => false + }) + && (vllm_default_scheduler_qualname(resolved: resolve_async_scheduling(request: AsyncSchedulingUnset, backend: ExecutorRay)) as String + != vllm_default_scheduler_qualname(resolved: resolve_async_scheduling(request: AsyncSchedulingUnset, backend: ExecutorMp)) as String) +} + +// WHICH EXECUTORS SUPPORT IT, read from the running installation rather than assumed. Ray is the +// only one of the three that does not, and it is the one this deployment uses. +test fn only_the_ray_backend_lacks_async_scheduling_support() -> Bool { + (match executor_async_scheduling_support(backend: ExecutorRay) { + ExecutorDoesNotSupportAsyncScheduling { because: b } => string_contains(s: b as String, pattern: "abstract.py") + ExecutorSupportsAsyncScheduling { by: _ } => false + }) + && (match executor_async_scheduling_support(backend: ExecutorMp) { + ExecutorSupportsAsyncScheduling { by: _ } => true + ExecutorDoesNotSupportAsyncScheduling { because: _ } => false + }) + && (match executor_async_scheduling_support(backend: ExecutorExternalLauncher) { + ExecutorSupportsAsyncScheduling { by: _ } => true + ExecutorDoesNotSupportAsyncScheduling { because: _ } => false + }) +} + +// -- PROFILE ADMISSION REFUSES WHAT IT CANNOT ESTABLISH ---------------------------------------- +// +// The positive control comes first: an admitted profile must exist, or every refusal below is +// vacuous and the admission function is a permanently-red gate rather than a discriminating one. +test fn the_pinned_default_synchronous_profile_is_admitted() -> Bool { + match observer_profile_admits(p: observer_profile( + revision: vllm_source_revision_read, + selection: SelectedByUpstreamDefault, + async_scheduling: AsyncSchedulingOff, + executor: ExecutorRay, + )) { + ObserverProfileAdmitted { delegate: d, fields: n } => + d as String == "vllm.v1.core.sched.scheduler.Scheduler" + && n == length(scheduler_output_fields_read()) + ObserverProfileRefused { cause: _ } => false + } +} + +test fn a_drifted_revision_is_refused_by_naming_what_was_observed() -> Bool { + match observer_profile_admits(p: observer_profile( + revision: vllm_source_revision( + repository: "github.com/vllm-project/vllm" as NonEmptyStr, + full_revision: "0000000000000000000000000000000000000000" as NonEmptyStr), + selection: SelectedByUpstreamDefault, + async_scheduling: AsyncSchedulingOff, + executor: ExecutorRay, + )) { + ObserverProfileRefused { cause: ProfileRevisionDrift { observed: o, because: _ } } => + o as String == "0000000000000000000000000000000000000000" + ObserverProfileRefused { cause: _ } => false + ObserverProfileAdmitted { delegate: _, fields: _ } => false + } +} + +test fn a_launch_that_already_substitutes_a_scheduler_is_refused() -> Bool { + match observer_profile_admits(p: observer_profile( + revision: vllm_source_revision_read, + selection: SelectedByQualname { qualname: "acme.sched.CustomScheduler" as NonEmptyStr }, + async_scheduling: AsyncSchedulingOff, + executor: ExecutorRay, + )) { + ObserverProfileRefused { cause: ProfileAlreadyHasForeignScheduler { qualname: q, because: _ } } => + q as String == "acme.sched.CustomScheduler" + ObserverProfileRefused { cause: _ } => false + ObserverProfileAdmitted { delegate: _, fields: _ } => false + } +} + +// ASYNC IS REFUSED AND NAMES THE DELEGATE IT WOULD HAVE WRAPPED. The refusal is not "async is +// unsupported" in the abstract: it is that the prefix a receipt would record is optimistic by an +// amount this manifest has not established, so the refusal carries the class and the reason. +test fn async_scheduling_is_refused_until_its_placeholder_semantics_are_modelled() -> Bool { + match observer_profile_admits(p: observer_profile( + revision: vllm_source_revision_read, + selection: SelectedByUpstreamDefault, + async_scheduling: AsyncSchedulingOn, + executor: ExecutorRay, + )) { + ObserverProfileRefused { cause: ProfileAsyncSchedulingUnmodelled { delegate: d, because: b } } => + d as String == "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + && string_contains(s: b as String, pattern: "num_output_placeholders") + ObserverProfileRefused { cause: _ } => false + ObserverProfileAdmitted { delegate: _, fields: _ } => false + } +} + +// -- THE BINDING TABLE COVERS THE RECEIPT AND NAMES ITS READ SITE ------------------------------ +// +// The cached_tokens row is the one the read-site rule exists for, so it must cite the output rather +// than the request in its own note. A row that said "request" would pass every other check here. +test fn every_binding_names_a_source_and_cached_tokens_cites_the_output() -> Bool { + length(receipt_field_bindings()) == 6 + && fold(receipt_field_bindings(), init: true, f: (acc, b) => + acc && (match b.source { + FromSchedulerOutputField { field: _ } => true + FromSchedulerMethod { method: _ } => true + FromObserverClock { because: _ } => true + })) + && fold(receipt_field_bindings(), init: false, f: (acc, b) => + acc || (b.receipt_field as String == "RequestStepAllocation.cached_tokens" + && string_contains(s: b.note as String, pattern: "never from the live request"))) +} + +// -- THE EXTENSION POINT IS START-BOUND, AND UPSTREAM DISCLAIMS IT ----------------------------- +// +// Two independent sources agree that the observer binds at start: a probe of the live host recorded +// in gunbc.spark.serving_launch_instrumentation, and this read of the pinned upstream source. A +// revision that added a reload path would red here, which is the point of writing it twice. +test fn the_scheduler_selection_is_bound_at_engine_construction() -> Bool { + match scheduler_selection_timing(p: vllm_scheduler_extension_point) { + SelectableOnlyAtEngineConstruction { because: b } => + string_contains(s: b as String, pattern: "no setter, route or reload path") + SelectableWhileRunning { by: _ } => false + } +} + +// UPSTREAM'S OWN DISCLAIMER IS WHY THE PIN IS EXACT RATHER THAN A RANGE. If this ever became a +// public interface the consequence recorded here would be wrong and a range would be admissible. +test fn upstream_disclaims_compatibility_so_no_version_range_is_admissible() -> Bool { + match vllm_scheduler_interface_posture { + UpstreamDisclaimsCompatibility { in_words: w, consequence: c } => + string_contains(s: w as String, pattern: "not public and compatibility may not be maintained") + && string_contains(s: c as String, pattern: "one exact revision") + UpstreamPublicInterface { by: _ } => false + } +} + +// -- THE FRONTIER IS DECLARED, NOT DISCHARGED -------------------------------------------------- +// +// This manifest must not be readable as having built the producer. The trigger names a capability -- +// a wheel that delegates, fills every row from the admitted read site, and refuses what this refuses. +test fn the_manifest_declares_an_unbuilt_wheel_with_a_capability_trigger() -> Bool { + match serving_observer_manifest_standing { + ManifestAwaitingWheel { trigger: t } => + string_contains(s: t as String, pattern: "refuses to import under any profile") + && string_contains(s: t as String, pattern: "behaviour-preserving") + ManifestConsumedByWheel { wheel: _ } => false + } +} diff --git a/dag/test/claim/spark/serving_step_cost_witness_test.dag b/dag/test/claim/spark/serving_step_cost_witness_test.dag new file mode 100644 index 00000000000..932b3182bbd --- /dev/null +++ b/dag/test/claim/spark/serving_step_cost_witness_test.dag @@ -0,0 +1,76 @@ +module test.claim.spark.serving_step_cost_witness + +import std.types { Bool, String, Int, NonEmptyStr, List } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import std.measure { TokenCount, token_count, millisecond_count } +import gunbc.spark.serving_step_cost { + ServingStepShape, StepCostObservation, ColdPrefillRunAggregate, + spark_group_a_cold_prefill_run, spark_serving_step_costs, + CoTenantDecodeRate, DecodeRatePredicted, DecodeRateOutsideCalibration, co_tenant_decode_rate, + computed_prefill_tokens_per_second, + StepCostLookup, StepCostKnown, StepCostUnmeasured, step_cost_for, +} + +// THE PREDICTION THE RUN AGGREGATE EARNS. A decoder cannot emit a token before the current engine +// step completes, so while a co-tenant prefills its rate is bounded by the interval between step +// completions. That interval was DIRECTLY measured -- median 2.30s between consecutive count +// advances -- so the reciprocal, 0.43 tok/s, comes from an observed quantity rather than a derived +// pairing. +// +// The operator's session reported 0.4 tok/s on turns 57-59, turns whose own work was ~640 new tokens +// against ~70k cached. A measurement on a different head, on an idle engine, with a synthetic +// prompt, predicts it. +test fn the_measured_inter_step_interval_predicts_the_co_tenant_rate() -> Bool { + match co_tenant_decode_rate(run: spark_group_a_cold_prefill_run) { + DecodeRatePredicted { milli_tokens_per_second: r, from: _ } => r >= 400 && r <= 460 + DecodeRateOutsideCalibration { cause: _ } => false + } +} + +// THROUGHPUT IS RUN-LEVEL AND DERIVED. Total computed tokens over elapsed run time -- both directly +// measured -- so "890 tok/s" is not editable as a literal and moves when either term does. +test fn prefill_throughput_is_derived_from_run_level_terms() -> Bool { + match computed_prefill_tokens_per_second(run: spark_group_a_cold_prefill_run) { + Present { value: derived } => derived >= 780 && derived <= 900 + Absent => false + } +} + +// THE ROW THAT KEEPS THE GRAIN HONEST. The run aggregate carries totals and an interval +// distribution. It does NOT carry, and must not be read as, one step's token count paired with one +// step's duration: 158,764/83 = 1,913 is a run-wide AVERAGE and 2,300ms is a MEDIAN, and no step was +// ever observed to be both. An earlier version of this module recorded exactly that pairing as a +// StepCostObservation, which was a fabricated measurement dressed as a calibration cell. +test fn the_aggregate_carries_totals_and_a_distribution_not_a_step_pairing() -> Bool { + spark_group_a_cold_prefill_run.completed_iterations == 83 + && millisecond_count(m: spark_group_a_cold_prefill_run.elapsed_millis) == 185500 + && millisecond_count(m: spark_group_a_cold_prefill_run.inter_iteration_min_millis) == 940 + && millisecond_count(m: spark_group_a_cold_prefill_run.inter_iteration_median_millis) == 2300 + && millisecond_count(m: spark_group_a_cold_prefill_run.inter_iteration_max_millis) == 2680 +} + +// THE EXACT PER-STEP POPULATION IS EMPTY, AND EVERY LOOKUP REFUSES. This is what stops the run +// aggregate being consumed as though it were a step cell, and what the authorised privileged trace +// exists to change. A simulator asking for a step cost today gets a refusal, not an interpolation +// from a run average. +test fn no_exact_step_cell_exists_yet_and_lookups_refuse() -> Bool { + length(spark_serving_step_costs) == 0 + && (match step_cost_for(shape: ServingStepShape { computed_prefill: token_count(count: 1913), decode_sequences: 0 }) { + StepCostUnmeasured { cause: c } => string_contains(s: c as String, pattern: "EMPTY until the privileged step trace") + StepCostKnown { observation: _ } => false + }) + && (match step_cost_for(shape: ServingStepShape { computed_prefill: token_count(count: 2048), decode_sequences: 1 }) { + StepCostUnmeasured { cause: _ } => true + StepCostKnown { observation: _ } => false + }) +} + +// THE AGGREGATE NAMES THE LAUNCH IT WAS MEASURED ON, so a later image, quantisation or topology +// change cannot silently inherit it. +test fn the_aggregate_is_bound_to_the_launch_that_produced_it() -> Bool { + string_contains(s: spark_group_a_cold_prefill_run.profile as String, pattern: "d140fa39c1174403bf2ce4d7ed8c2d74") + && string_contains(s: spark_group_a_cold_prefill_run.profile as String, pattern: "TP=4") + && string_contains(s: spark_group_a_cold_prefill_run.provenance as String, pattern: "200ms poll") + && string_contains(s: spark_group_a_cold_prefill_run.provenance as String, pattern: "never between polls") +} diff --git a/dag/test/claim/spark/serving_step_receipt_witness_test.dag b/dag/test/claim/spark/serving_step_receipt_witness_test.dag new file mode 100644 index 00000000000..4ee6d89fcc8 --- /dev/null +++ b/dag/test/claim/spark/serving_step_receipt_witness_test.dag @@ -0,0 +1,153 @@ +module test.claim.spark.serving_step_receipt_witness + +import std.types { Bool, String, NonEmptyStr, List } +import std.nat { Nat } +import std.measure { Microsecond, microsecond } +import v2.std.optional { Present, Absent } +import gunbc.spark.serving_launch_instrumentation { + LaunchInstrumentation, AttributionInstrument, FirstPartyStepObserver, + spark_group_a_pair_head_instrumentation, +} +import gunbc.spark.serving_step_receipt { + RequestPhase, Prefill, Decode, PrefillAndDecodeInSameStep, + RequestStepAllocation, ServingStepReceipt, step_computed_tokens, step_request_count, + ObservationGap, StepReceiptPopulation, + PopulationCompleteness, ContiguousAndComplete, HasObservationGaps, population_completeness, + CalibrationAdmission, AdmittedToCalibration, RefusedForObservationGap, RefusedForUninstrumentedLaunch, + population_admits_calibration, + BudgetEvidence, LargestScheduledStep, NoStepObserved, largest_scheduled_step, + StepExclusivity, StepServedOneRequest, StepServedSeveral, StepServedNothing, step_exclusivity, + ProducerStanding, ProducerBound, ProducerUnbuilt, spark_step_receipt_producer, +} + +fn observed_launch() -> LaunchInstrumentation { + LaunchInstrumentation { + bound_instruments: [FirstPartyStepObserver], + observed_by: "fixture launch with a step observer bound at start" as NonEmptyStr, + revision: spark_group_a_pair_head_instrumentation.revision, + } +} + +fn one_request_step(seq: Nat, tokens: Nat) -> ServingStepReceipt { + ServingStepReceipt { + step_sequence: seq, + started_micros: microsecond(count: 0), + elapsed_micros: microsecond(count: 2000000), + allocations: [RequestStepAllocation { request_id: "req-a" as NonEmptyStr, phase: Prefill, computed_tokens: tokens, cached_tokens: 0 }], + running_after: 1, + waiting_after: 0, + } +} + +fn shared_step(seq: Nat) -> ServingStepReceipt { + ServingStepReceipt { + step_sequence: seq, + started_micros: microsecond(count: 0), + elapsed_micros: microsecond(count: 2000000), + allocations: [ + RequestStepAllocation { request_id: "req-a" as NonEmptyStr, phase: Prefill, computed_tokens: 2000, cached_tokens: 0 }, + RequestStepAllocation { request_id: "req-b" as NonEmptyStr, phase: Decode, computed_tokens: 1, cached_tokens: 0 }, + ], + running_after: 2, + waiting_after: 0, + } +} + +// THE PER-STEP TOTAL IS DERIVED FROM THE ALLOCATIONS, never reported alongside them. A reported total +// would be a second authority for one quantity, and the two could disagree. +test fn the_step_total_is_a_fold_over_its_allocations() -> Bool { + step_computed_tokens(r: shared_step(seq: 1)) == 2001 + && step_request_count(r: shared_step(seq: 1)) == 2 + && step_computed_tokens(r: one_request_step(seq: 1, tokens: 4096)) == 4096 +} + +// A GAP REFUSES CALIBRATION. This is the wall the whole contract exists for: a dropped receipt is +// invisible in the surviving population, so a fold over it would report a complete-looking answer +// over an incomplete population -- exactly the absorbing fallback this programme refuses. +test fn an_observation_gap_refuses_calibration_rather_than_shrinking_the_population() -> Bool { + let holed = StepReceiptPopulation { + launch: observed_launch(), + receipts: [one_request_step(seq: 1, tokens: 1000), one_request_step(seq: 3, tokens: 1000)], + gaps: [ObservationGap { first_missing_sequence: 2, missing_count: 1, because: "sink full" as NonEmptyStr }], + } + (match population_completeness(p: holed) { + HasObservationGaps { gaps: n, first_missing: f } => n == 1 && f == 2 + ContiguousAndComplete { steps: _ } => false + }) + && (match population_admits_calibration(p: holed) { + RefusedForObservationGap { because: b } => string_contains(s: b as String, pattern: "complete-looking answer") + _ => false + }) +} + +// AND A POPULATION FROM AN UNINSTRUMENTED LAUNCH REFUSES FOR A DIFFERENT REASON, which is why the two +// refusals are separate arms: one is a hole in the evidence, the other is evidence with no provenance. +test fn receipts_claiming_an_uninstrumented_launch_refuse_separately() -> Bool { + match population_admits_calibration(p: StepReceiptPopulation { + launch: spark_group_a_pair_head_instrumentation, + receipts: [one_request_step(seq: 1, tokens: 1000)], + gaps: [], + }) { + RefusedForUninstrumentedLaunch { because: b } => string_contains(s: b as String, pattern: "provenance is unaccounted") + _ => false + } +} + +// A COMPLETE POPULATION ON AN INSTRUMENTED LAUNCH IS ADMITTED. The positive control -- without it the +// two refusals above would be permanently green and carry no information. +test fn a_complete_population_on_an_instrumented_launch_is_admitted() -> Bool { + match population_admits_calibration(p: StepReceiptPopulation { + launch: observed_launch(), + receipts: [one_request_step(seq: 1, tokens: 1000), one_request_step(seq: 2, tokens: 2000)], + gaps: [], + }) { + AdmittedToCalibration { steps: s } => s == 2 + _ => false + } +} + +// THE BUDGET BECOMES ANSWERABLE UNDER LOAD. Today it is a behavioural floor obtained by making the +// engine idle, which is the one condition production is never in. A receipt population answers it +// directly, and refuses through the same admission wall when the population is holed. +test fn the_budget_is_read_from_receipts_and_inherits_the_refusal() -> Bool { + (match largest_scheduled_step(p: StepReceiptPopulation { + launch: observed_launch(), + receipts: [one_request_step(seq: 1, tokens: 1000), one_request_step(seq: 2, tokens: 8192), one_request_step(seq: 3, tokens: 4096)], + gaps: [], + }) { + LargestScheduledStep { computed_tokens: t, at_step: s } => t == 8192 && s == 2 + NoStepObserved { because: _ } => false + }) + && (match largest_scheduled_step(p: StepReceiptPopulation { + launch: observed_launch(), + receipts: [one_request_step(seq: 1, tokens: 8192)], + gaps: [ObservationGap { first_missing_sequence: 2, missing_count: 4, because: "sink full" as NonEmptyStr }], + }) { + NoStepObserved { because: b } => string_contains(s: b as String, pattern: "complete-looking answer") + LargestScheduledStep { computed_tokens: _, at_step: _ } => false + }) +} + +// EXCLUSIVITY BECOMES A PROPERTY OF EACH STEP rather than something arranged by emptying the engine. +// This is what lets self-cost be separated from co-tenancy in production instead of a laboratory. +test fn exclusivity_is_read_per_step_rather_than_arranged() -> Bool { + (match step_exclusivity(r: one_request_step(seq: 1, tokens: 1000)) { + StepServedOneRequest { request_id: r } => (r as String) == "req-a" + _ => false + }) + && (match step_exclusivity(r: shared_step(seq: 2)) { + StepServedSeveral { request_count: n } => n == 2 + _ => false + }) +} + +// THE CONTRACT DECLARES ITS OWN FRONTIER, and the trigger names a CAPABILITY rather than an artifact +// -- what the observer must be SUFFICIENT FOR, not merely that one exists. +test fn the_producer_is_declared_unbuilt_with_a_capability_trigger() -> Bool { + match spark_step_receipt_producer { + ProducerBound { instrument: _ } => false + ProducerUnbuilt { trigger: t } => + string_contains(s: t as String, pattern: "sufficient to answer") + && string_contains(s: t as String, pattern: "recording a typed gap") + } +} diff --git a/dag/test/claim/spark/vllm_kv_layout_receipt_witness_test.dag b/dag/test/claim/spark/vllm_kv_layout_receipt_witness_test.dag new file mode 100644 index 00000000000..377b6ac1e56 --- /dev/null +++ b/dag/test/claim/spark/vllm_kv_layout_receipt_witness_test.dag @@ -0,0 +1,213 @@ +module test.claim.spark.vllm_kv_layout_receipt_witness + +import std.types { Bool, String, Int, NonEmptyStr, List, EpochSecs } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.content_hash { ContentHash, Sha256Hash, Sha256Digest, Sha256DigestHex } +import std.measure { TokenCount, token_count } +import gunbc.spark.fabric_switch_observed { FabricGroupA } +import gunbc.spark.pair_serving_desired { spark_pair_serving_head } +import gunbc.spark.vllm_serving_launch { VllmServingLaunch, vllm_serving_launch } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision, vllm_source_revision_read } +import extdeps.vllm.log_lines { VllmStartupReading, vllm_startup_reading, GpuKvCacheSize } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision, vllm_source_revision_read } +import extdeps.vllm.kv_cache { VllmObservedKvPool, VllmKvPoolAdmission, KvPoolAdmitted, KvPoolRefused, vllm_observed_kv_pool_from_reading } +import extdeps.vllm.kv_layout { + KvBlockSize, KvLayerCount, KvGroupLayout, FullAttention, SlidingWindowAttention, + ResolvedKvLayout, resolved_kv_layout, + VllmReportedCapacity, vllm_reported_capacity, +} +import gunbc.spark.vllm_kv_layout_observe { + VllmKvLayoutReceipt, KvLayoutObserved, KvLayoutUnreadable, KvLayoutInconsistent, admit_kv_layout, + SeatingAuthority, SeatingAuthorized, SeatingRefused, seating_authority_for, + receipt_wire, +} + +fn w_digest(hex: String) -> ContentHash { + Sha256Hash(Sha256Digest { hex: hex as Sha256DigestHex }) +} + +data w_exe: String = "1111111111111111111111111111111111111111111111111111111111111111" +data w_unit: String = "3333333333333333333333333333333333333333333333333333333333333333" +data w_capture: String = "5555555555555555555555555555555555555555555555555555555555555555" + +fn w_launch_id(invocation: String) -> VllmServingLaunch { + vllm_serving_launch( + group: FabricGroupA, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: invocation as NonEmptyStr, + unit_pinned_base_image_digest: w_digest(hex: w_exe), + unit_digest: w_digest(hex: w_unit), + started_at: 1788000000, + ) +} + +fn w_pool(tokens: Int) -> VllmObservedKvPool { + match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: tokens), + reading: vllm_startup_reading(line: GpuKvCacheSize, captured_text: "GPU KV cache size: n tokens" as NonEmptyStr, observed_at_revision: vllm_source_revision_read), + ) { + KvPoolAdmitted { pool: p } => p + KvPoolRefused { cause: _ } => w_pool(tokens: tokens) + } +} + +fn w_reported(tokens: Int, hundredths: Nat) -> VllmReportedCapacity { + vllm_reported_capacity(pool: w_pool(tokens: tokens), max_concurrency_hundredths: hundredths) +} + +fn w_layout(groups: List) -> ResolvedKvLayout { + w_layout_with(blocks: 54475, groups: groups) +} + +fn w_layout_with(blocks: Nat, groups: List) -> ResolvedKvLayout { + resolved_kv_layout( + allocator_blocks: blocks, + scheduler_block_size: token_count(count: 16), + hash_block_size: token_count(count: 16), + max_model_len: token_count(count: 1048576), + max_num_batched_tokens: token_count(count: 8192), + groups: groups, + ) +} + +fn w_full(layers: KvLayerCount, block: KvBlockSize) -> KvGroupLayout { + KvGroupLayout { kind: FullAttention, layer_count: layers, block_size: block } +} + +// TODAY'S REAL EVIDENCE LANDS INCONSISTENT, WHICH IS THE POINT. The group A head's own two startup +// lines are mutually consistent, its inventory is real, and the receipt still refuses to be +// observed -- because the engine emits no group metadata, so the one non-circular check has no +// second opinion to compare against. An absent second opinion is not agreement. +test fn a_real_capture_with_no_group_metadata_is_inconsistent_not_observed() -> Bool { + let r = admit_kv_layout( + incarnation: w_launch_id(invocation: "inv-20-43"), + layout: w_layout(groups: []), + reported: w_reported(tokens: 11767856, hundredths: 1122), + source_capture_digest: w_digest(hex: w_capture), + ) + match r { + KvLayoutInconsistent { incarnation: _, cause: c } => + string_contains(s: c as String, pattern: "cannot be checked") + && string_contains(s: c as String, pattern: "unobserved") + _ => false + } +} + +// AND IT AUTHORIZES NOTHING. The refusal is not cosmetic: a receipt that is not observed cannot seat +// a request even against the very engine it describes. +test fn an_inconsistent_receipt_seats_nothing() -> Bool { + let r = admit_kv_layout( + incarnation: w_launch_id(invocation: "inv-20-43"), + layout: w_layout(groups: []), + reported: w_reported(tokens: 11767856, hundredths: 1122), + source_capture_digest: w_digest(hex: w_capture), + ) + match seating_authority_for(receipt: r, serving: w_launch_id(invocation: "inv-20-43"), sequence_budget: token_count(count: 500000)) { + SeatingRefused { cause: c } => string_contains(s: c as String, pattern: "no capacity is established") + SeatingAuthorized { blocks_available: _, blocks_per_request: _ } => false + } +} + +// THE POSITIVE CONTROL: a layout whose groups DO reproduce the printed cost is admitted, so the +// refusals above are about the missing metadata and not about the fold being structurally unable to +// say yes. One full-attention layer at block 16 costs 1048576/16 = 65,536 blocks for a full-context +// request; an inventory of 65,536 against a pool line of 1,048,576 tokens inverts to exactly that, +// and 1.00x is the concurrency those two lines imply. +fn w_reproducing_receipt(invocation: String) -> VllmKvLayoutReceipt { + admit_kv_layout( + incarnation: w_launch_id(invocation: invocation), + layout: w_layout_with(blocks: 65536, groups: [w_full(layers: 1, block: 16)]), + reported: w_reported(tokens: 1048576, hundredths: 100), + source_capture_digest: w_digest(hex: w_capture), + ) +} + +test fn a_layout_whose_groups_reproduce_the_printed_cost_is_observed() -> Bool { + match w_reproducing_receipt(invocation: "inv-20-43") { + KvLayoutObserved { incarnation: _, layout: _, reported: _, blocks_per_max_request: b, source_capture_digest: _ } => b == 65536 + _ => false + } +} + +// THE DISCRIMINATING ROW FOR THE WHOLE RECEIPT: an OBSERVED receipt still refuses to seat a request +// that a DIFFERENT launch will serve. This is the case that a group-keyed or host-keyed binding gets +// wrong -- same group, same head, same image, same unit, and the engine has been restarted +// underneath it. A grant taken on the strength of the old receipt would be admitted against an +// engine whose pool it never measured. +test fn an_observed_receipt_refuses_to_seat_a_request_a_later_launch_will_serve() -> Bool { + let r = w_reproducing_receipt(invocation: "inv-20-43") + let budget = token_count(count: 500000) + (match seating_authority_for(receipt: r, serving: w_launch_id(invocation: "inv-22-11"), sequence_budget: budget) { + SeatingRefused { cause: c } => string_contains(s: c as String, pattern: "different-incarnation") + SeatingAuthorized { blocks_available: _, blocks_per_request: _ } => false + }) + && (match seating_authority_for(receipt: r, serving: w_launch_id(invocation: "inv-20-43"), sequence_budget: budget) { + SeatingAuthorized { blocks_available: a, blocks_per_request: p } => a == 65536 && p == 31250 + SeatingRefused { cause: _ } => false + }) +} + +// A GARBLED CAPTURE IS ITS OWN REFUSAL, distinct from a layout that disagrees: the first says the +// bytes cannot be trusted, the second says two trustworthy readings describe different engines. +test fn garbled_lines_and_a_contradicting_layout_are_different_refusals() -> Bool { + let garbled = admit_kv_layout( + incarnation: w_launch_id(invocation: "inv-20-43"), + layout: w_layout(groups: []), + reported: w_reported(tokens: 11767856, hundredths: 440), + source_capture_digest: w_digest(hex: w_capture), + ) + let contradicting = admit_kv_layout( + incarnation: w_launch_id(invocation: "inv-20-43"), + layout: w_layout_with(blocks: 65536, groups: [w_full(layers: 2, block: 16)]), + reported: w_reported(tokens: 1048576, hundredths: 100), + source_capture_digest: w_digest(hex: w_capture), + ) + (match garbled { KvLayoutInconsistent { incarnation: _, cause: c } => string_contains(s: c as String, pattern: "disagree with each other") _ => false }) + && (match contradicting { KvLayoutInconsistent { incarnation: _, cause: c } => string_contains(s: c as String, pattern: "different engines") _ => false }) +} + +// A READING FROM ANOTHER vLLM REVISION REFUSES INSTEAD OF BEING COMPARED, and this row is only +// authorable because the reading now OBSERVES its revision rather than stamping the modelled one. +// While the constructor asserted the constant, the mismatch had no constructor anywhere, so a guard +// against it would have been permanently green — a decoration cited as coverage rather than a wall. +// +// The pool and concurrency lines are derived output, exactly what an upstream refactor moves. A +// layout checked against them under a revision this repository never read is asserting a law about +// code nobody looked at, and it would pass or fail for reasons unrelated to the engine's capacity. +test fn a_reading_from_another_revision_refuses_instead_of_being_compared() -> Bool { + let foreign_pool = match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: 1048576), + reading: vllm_startup_reading( + line: GpuKvCacheSize, + captured_text: "GPU KV cache size: 1,048,576 tokens" as NonEmptyStr, + observed_at_revision: vllm_source_revision( + repository: "github.com/vllm-project/vllm" as NonEmptyStr, + full_revision: "0000000000000000000000000000000000000000" as NonEmptyStr, + ), + ), + ) { + KvPoolAdmitted { pool: p } => p + KvPoolRefused { cause: _ } => w_pool(tokens: 1048576) + } + let r = admit_kv_layout( + incarnation: w_launch_id(invocation: "inv-20-43"), + layout: w_layout_with(blocks: 65536, groups: [w_full(layers: 1, block: 16)]), + reported: vllm_reported_capacity(pool: foreign_pool, max_concurrency_hundredths: 100), + source_capture_digest: w_digest(hex: w_capture), + ) + match r { + KvLayoutInconsistent { incarnation: _, cause: c } => + string_contains(s: c as String, pattern: "not valid across revisions") + _ => false + } +} + +// THE CONTROL: the same layout and the same numbers, read at the revision the laws were established +// against, is observed. So the refusal above is about the revision and nothing else. +test fn the_same_evidence_at_the_modelled_revision_is_observed() -> Bool { + match w_reproducing_receipt(invocation: "inv-20-43") { + KvLayoutObserved { incarnation: _, layout: _, reported: _, blocks_per_max_request: b, source_capture_digest: _ } => b == 65536 + _ => false + } +} diff --git a/dag/test/claim/spark/vllm_serving_launch_witness_test.dag b/dag/test/claim/spark/vllm_serving_launch_witness_test.dag new file mode 100644 index 00000000000..53316a0545d --- /dev/null +++ b/dag/test/claim/spark/vllm_serving_launch_witness_test.dag @@ -0,0 +1,108 @@ +module test.claim.spark.vllm_serving_launch_witness + +import std.types { Bool, String, NonEmptyStr, EpochSecs } +import std.content_hash { ContentHash, Sha256Hash, Sha1Hash, Sha256Digest, Sha1Digest, Sha256DigestHex, Sha1DigestHex } +import gunbc.spark.fabric_switch_observed { FabricGroupA, FabricGroupB } +import gunbc.spark.pair_serving_desired { spark_pair_serving_head } +import gunbc.spark.vllm_serving_launch { + VllmServingLaunch, vllm_serving_launch, + LaunchJoin, SameLaunch, DifferentLaunch, LaunchContradiction, + same_launch, join_is_admissible, launch_join_wire, +} + +fn w_sha256(hex: String) -> ContentHash { + Sha256Hash(Sha256Digest { hex: hex as Sha256DigestHex }) +} + +fn w_sha1(hex: String) -> ContentHash { + Sha1Hash(Sha1Digest { hex: hex as Sha1DigestHex }) +} + +data w_exe_a: String = "1111111111111111111111111111111111111111111111111111111111111111" +data w_exe_b: String = "2222222222222222222222222222222222222222222222222222222222222222" +data w_unit: String = "3333333333333333333333333333333333333333333333333333333333333333" + +// The three launches the group A head actually produced on 2026-09-05 differ only by invocation: +// one rendered unit, one image, three engine cores with different KV. That is the shape this +// witness fixtures. +fn w_launch(invocation: String, container: String, exe: String, started: EpochSecs) -> VllmServingLaunch { + vllm_serving_launch( + group: FabricGroupA, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: invocation as NonEmptyStr, + unit_pinned_base_image_digest: w_sha256(hex: exe), + unit_digest: w_sha256(hex: w_unit), + started_at: started, + ) +} + +fn w_first() -> VllmServingLaunch { + w_launch(invocation: "inv-16-36", container: "gunbc-spark-pair", exe: w_exe_a, started: 1788000000) +} + +// THE DISCRIMINATING ROW, and the whole reason the type exists: two launches of ONE unit on ONE +// host in ONE group, identical in every field a previous join would have compared -- same group, +// same head, same container name, same image digest, same unit digest -- and they are NOT the same +// incarnation. A join on group, host or unit text answers "same" here and authorizes a grant taken +// against a dead engine. +test fn two_launches_of_one_unit_are_not_one_incarnation() -> Bool { + let second = w_launch(invocation: "inv-20-43", container: "gunbc-spark-pair", exe: w_exe_a, started: 1788000000) + (match same_launch(a: w_first(), b: second) { + DifferentLaunch { left: l, right: r } => (l as String) == "inv-16-36" && (r as String) == "inv-20-43" + _ => false + }) + && !join_is_admissible(j: same_launch(a: w_first(), b: second)) + && (match same_launch(a: w_first(), b: w_first()) { SameLaunch => true _ => false }) + && join_is_admissible(j: same_launch(a: w_first(), b: w_first())) +} + +// Same invocation, disagreeing field: that is not turnover, it is two observers contradicting each +// other about one launch, and it gets its own arm so it cannot be actioned as a restart. +test fn one_invocation_with_a_disagreeing_field_is_a_contradiction_not_turnover() -> Bool { + let rebuilt = w_launch(invocation: "inv-16-36", container: "gunbc-spark-pair", exe: w_exe_b, started: 1788000000) + let moved = vllm_serving_launch( + group: FabricGroupB, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: "inv-16-36" as NonEmptyStr, + unit_pinned_base_image_digest: w_sha256(hex: w_exe_a), + unit_digest: w_sha256(hex: w_unit), + started_at: 1788000000, + ) + (match same_launch(a: w_first(), b: rebuilt) { + LaunchContradiction { invocation: i, field: f } => (i as String) == "inv-16-36" && (f as String) == "unit_pinned_base_image_digest" + _ => false + }) + && (match same_launch(a: w_first(), b: moved) { + LaunchContradiction { invocation: _, field: f } => (f as String) == "group" + _ => false + }) + && !join_is_admissible(j: same_launch(a: w_first(), b: rebuilt)) +} + +// AN INCOMPARABLE DIGEST PAIR IS NOT A DIFFERENCE. Two observers that hashed one launch under +// different families have established that they cannot be joined, not that the engine changed -- +// std.content_hash refuses to collapse that into a silent false, and this module must not +// reintroduce the collapse. The refusal names the incomparability so the observer defect is +// legible rather than reading as a rebuild. +test fn cross_family_digests_refuse_as_incomparable_rather_than_reading_as_a_rebuild() -> Bool { + let other_family = vllm_serving_launch( + group: FabricGroupA, + head: spark_pair_serving_head(group: FabricGroupA), + systemd_invocation: "inv-16-36" as NonEmptyStr, + unit_pinned_base_image_digest: w_sha1(hex: "4444444444444444444444444444444444444444"), + unit_digest: w_sha256(hex: w_unit), + started_at: 1788000000, + ) + match same_launch(a: w_first(), b: other_family) { + LaunchContradiction { invocation: _, field: f } => + string_contains(s: f as String, pattern: "unit_pinned_base_image_digest") + && string_contains(s: f as String, pattern: "incomparable") + _ => false + } +} + +test fn the_join_wire_names_both_invocations() -> Bool { + let second = w_launch(invocation: "inv-20-43", container: "gunbc-spark-pair", exe: w_exe_a, started: 1788000000) + string_contains(s: launch_join_wire(j: same_launch(a: w_first(), b: second)), pattern: "inv-16-36") + && string_contains(s: launch_join_wire(j: same_launch(a: w_first(), b: second)), pattern: "inv-20-43") +} diff --git a/dag/test/claim/vllm_kv_layout_witness_test.dag b/dag/test/claim/vllm_kv_layout_witness_test.dag new file mode 100644 index 00000000000..7d4c38876ca --- /dev/null +++ b/dag/test/claim/vllm_kv_layout_witness_test.dag @@ -0,0 +1,188 @@ +module test.claim.vllm_kv_layout_witness + +import std.types { Bool, String, Int, NonEmptyStr, List } +import v2.std.optional { Present, Absent } +import std.nat { Nat } +import std.measure { TokenCount, token_count, token_count_value } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision, vllm_source_revision_read } +import extdeps.vllm.log_lines { VllmStartupReading, vllm_startup_reading, GpuKvCacheSize } +import extdeps.vllm.kv_cache { + VllmObservedKvPool, VllmKvPoolAdmission, KvPoolAdmitted, KvPoolRefused, + vllm_observed_kv_pool_from_reading, + VllmPoolReconstruction, BlockCountTimesBlockSize, UtilizationTimesTotalMemory, + vllm_pool_reconstruction_is_authoritative, +} +import extdeps.vllm.kv_layout { + KvBlockSize, KvLayerCount, KvGroupKind, FullAttention, SlidingWindowAttention, MambaState, + KvGroupLayout, ResolvedKvLayout, resolved_kv_layout, + VllmReportedCapacity, vllm_reported_capacity, + BlocksPerRequest, BlocksPerRequestRecovered, BlocksPerRequestUnrecoverable, + recovered_blocks_per_max_length_request, + PrintedLinesAgreement, PrintedLinesAgree, PrintedLinesDisagree, PrintedLinesUncheckable, printed_lines_agree, + AdmissionCost, AdmissionCostBlocks, AdmissionCostUnestablished, cold_admission_blocks, + AllocatorUsage, UsageReading, UsageRead, FreeCountImpossible, allocator_usage, + group_blocks_for, +} + +// The group A head's own two startup lines, 2026-09-05T20:43:56Z. +fn w_pool() -> VllmObservedKvPool { + match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: 11767856), + reading: vllm_startup_reading( + line: GpuKvCacheSize, + captured_text: "[kv_cache_utils.py:1733] GPU KV cache size: 11,767,856 tokens" as NonEmptyStr, + observed_at_revision: vllm_source_revision_read, + ), + ) { + KvPoolAdmitted { pool: p } => p + KvPoolRefused { cause: _ } => + match vllm_observed_kv_pool_from_reading( + tokens: token_count(count: 0), + reading: vllm_startup_reading(line: GpuKvCacheSize, captured_text: "x" as NonEmptyStr, observed_at_revision: vllm_source_revision_read), + ) { + KvPoolAdmitted { pool: q } => q + KvPoolRefused { cause: _ } => w_pool() + } + } +} + +fn w_reported() -> VllmReportedCapacity { + vllm_reported_capacity(pool: w_pool(), max_concurrency_hundredths: 1122) +} + +// The allocator inventory read off the same engine's cache_config_info. +fn w_layout(groups: List) -> ResolvedKvLayout { + resolved_kv_layout( + allocator_blocks: 54475, + scheduler_block_size: token_count(count: 16), + hash_block_size: token_count(count: 16), + max_model_len: token_count(count: 1048576), + max_num_batched_tokens: token_count(count: 8192), + groups: groups, + ) +} + +// THE ENGINE'S OWN GROUP-AWARE COST, READ BACK OUT OF ITS TWO PRINTED LINES: 54,475 blocks x +// 1,048,576 context / 11,767,856 tokens = 4,854 blocks for one full-context request. That figure is +// the engine's own group-aware arithmetic recovered, not re-derived from fields whose reductions +// differ, and it is the only per-request cost today's evidence yields. +test fn the_engines_per_request_block_cost_is_recoverable_from_its_printed_lines() -> Bool { + match recovered_blocks_per_max_length_request(layout: w_layout(groups: []), reported: w_reported()) { + BlocksPerRequestRecovered { blocks: b } => b == 4854 + BlocksPerRequestUnrecoverable { cause: _ } => false + } +} + +// THE LIMIT OF TODAY'S EVIDENCE, ASSERTED RATHER THAN ASSUMED. The printed-line check tests the pool +// line against the concurrency line, and the inventory CANCELS OUT of it -- so swapping in a +// different engine's inventory leaves it green. This row exists to keep that stated: it is the +// reason the layout receipt is still required, and the reason no wall here may be described as +// checking the inventory. +// +// The recovered per-request cost DOES move with the inventory (4,854 against group A's 54,475 +// blocks, 4,930 against group B's 55,328), which is what makes the cost a function of the launch +// while the agreement check is not. +test fn the_printed_line_check_does_not_see_the_inventory_which_is_why_a_receipt_is_still_needed() -> Bool { + let other = resolved_kv_layout( + allocator_blocks: 55328, + scheduler_block_size: token_count(count: 16), + hash_block_size: token_count(count: 16), + max_model_len: token_count(count: 1048576), + max_num_batched_tokens: token_count(count: 8192), + groups: [], + ) + (match printed_lines_agree(layout: w_layout(groups: []), reported: w_reported()) { PrintedLinesAgree => true _ => false }) + && (match printed_lines_agree(layout: other, reported: w_reported()) { PrintedLinesAgree => true _ => false }) + && (match recovered_blocks_per_max_length_request(layout: other, reported: w_reported()) { + BlocksPerRequestRecovered { blocks: b } => b == 4930 + BlocksPerRequestUnrecoverable { cause: _ } => false + }) +} + +// AND THE CHECK IS NOT VACUOUS: a mistranscribed concurrency line breaks it. Without this row the +// one above could be satisfied by a function that returns Agree unconditionally. +test fn a_mistranscribed_concurrency_line_breaks_the_printed_line_check() -> Bool { + let wrong = vllm_reported_capacity(pool: w_pool(), max_concurrency_hundredths: 440) + match printed_lines_agree(layout: w_layout(groups: []), reported: wrong) { + PrintedLinesDisagree { implied_hundredths: i, reported_hundredths: r } => i == 1122 && r == 440 + _ => false + } +} + +fn w_full(layers: KvLayerCount, block: KvBlockSize) -> KvGroupLayout { + KvGroupLayout { kind: FullAttention, layer_count: layers, block_size: block } +} + +fn w_window(layers: KvLayerCount, block: KvBlockSize, window: Int) -> KvGroupLayout { + KvGroupLayout { kind: SlidingWindowAttention { window: token_count(count: window) }, layer_count: layers, block_size: block } +} + +// THE DISCRIMINATING ROW FOR THE WHOLE MODULE: two layouts with the SAME allocator inventory and the +// SAME global block size admit different costs, because the windows and group sizes differ. Any +// model that reduces capacity to those two fields answers identically for both and is wrong for at +// least one. The windowed group stops growing at 128 tokens while the full-attention group grows +// with the sequence, which is exactly the asymmetry `num_gpu_blocks * block_size` erases. +test fn one_inventory_and_one_block_size_do_not_determine_the_admission_cost() -> Bool { + let mostly_full = w_layout(groups: [w_full(layers: 40, block: 16), w_window(layers: 21, block: 16, window: 128)]) + let mostly_windowed = w_layout(groups: [w_full(layers: 4, block: 16), w_window(layers: 57, block: 16, window: 128)]) + let budget = token_count(count: 500000) + match cold_admission_blocks(layout: mostly_full, sequence_budget: budget) { + AdmissionCostUnestablished { cause: _ } => false + AdmissionCostBlocks { blocks: a } => + match cold_admission_blocks(layout: mostly_windowed, sequence_budget: budget) { + AdmissionCostUnestablished { cause: _ } => false + AdmissionCostBlocks { blocks: b } => + a != b + && a == (31250 * 40) + (8 * 21) + && b == (31250 * 4) + (8 * 57) + } + } +} + +// AND WITH NO GROUPS OBSERVED THERE IS NO COST, WHICH IS TODAY'S REAL STATE. Every incarnation this +// repository can currently observe lands here: the engine does not emit layer count or page size, so +// the layout is inventory-only and the admission cost refuses rather than guessing one. +test fn an_unobserved_group_layout_refuses_the_admission_cost() -> Bool { + (match cold_admission_blocks(layout: w_layout(groups: []), sequence_budget: token_count(count: 500000)) { + AdmissionCostUnestablished { cause: c } => string_contains(s: c as String, pattern: "unobserved") + AdmissionCostBlocks { blocks: _ } => false + }) + && (match cold_admission_blocks(layout: w_layout(groups: [w_full(layers: 61, block: 16)]), sequence_budget: token_count(count: 2000000)) { + AdmissionCostUnestablished { cause: c } => string_contains(s: c as String, pattern: "context cap") + AdmissionCostBlocks { blocks: _ } => false + }) +} + +// THE TWO FORBIDDEN RECONSTRUCTIONS STAY FORBIDDEN, and this module offers no function that performs +// either. The usage gauge is likewise not occupancy: it counts retained prefix-cache blocks, which +// is why the group B head read a majority-used pool with nothing running. +test fn neither_reconstruction_is_authoritative_and_usage_is_not_occupancy() -> Bool { + !vllm_pool_reconstruction_is_authoritative(r: BlockCountTimesBlockSize) + && !vllm_pool_reconstruction_is_authoritative(r: UtilizationTimesTotalMemory) + && (match allocator_usage(layout: w_layout(groups: []), free_blocks: 545) { + UsageRead { usage: u } => u.referenced_or_retained_blocks == 53930 + FreeCountImpossible { free_blocks: _, allocator_blocks: _ } => false + }) +} + +// A FREE COUNT LARGER THAN THE INVENTORY REFUSES. It was clamped to zero-free, which is the most +// alarming answer available and reached by fabrication -- an impossible reading rendered as a +// saturated engine, indistinguishable from a real one. The two numbers came from different launches +// and the only honest output says so. +test fn a_free_count_exceeding_the_inventory_refuses_rather_than_reading_as_saturated() -> Bool { + match allocator_usage(layout: w_layout(groups: []), free_blocks: 99999) { + FreeCountImpossible { free_blocks: f, allocator_blocks: a } => f == 99999 && a == 54475 + UsageRead { usage: _ } => false + } +} + +// A WINDOWED GROUP WITH NO WINDOW IS NOT WRITABLE. The window used to sit beside the kind as an +// optional field, so that state typechecked and the cost function charged it as full attention -- +// a malformed layout yielding a plausible number instead of a refusal. The constructor below is the +// only way to build a windowed group, and it demands the window. +test fn a_windowed_group_carries_its_window_by_construction() -> Bool { + let g = w_window(layers: 1, block: 16, window: 128) + (match g.kind { SlidingWindowAttention { window: w } => token_count_value(t: w) == 128 _ => false }) + && group_blocks_for(group: g, sequence: token_count(count: 1048576)) == 8 + && group_blocks_for(group: w_full(layers: 1, block: 16), sequence: token_count(count: 1048576)) == 65536 +} diff --git a/dag/test/claim/vllm_kv_pool_witness_test.dag b/dag/test/claim/vllm_kv_pool_witness_test.dag index 5739b9fe260..11c0d93c7d6 100644 --- a/dag/test/claim/vllm_kv_pool_witness_test.dag +++ b/dag/test/claim/vllm_kv_pool_witness_test.dag @@ -3,6 +3,7 @@ module test.claim.vllm_kv_pool_witness_test import std.types { Bool, List, NonEmptyStr } import std.nat { Nat } import std.measure { TokenCount, Percent, token_count, percent } +import extdeps.vllm.server { VllmSourceRevision, vllm_source_revision_read } import extdeps.vllm.log_lines { VllmStartupReading, vllm_startup_reading, VllmStartupLine, GpuKvCacheSize, MaximumConcurrency, ResolvedKvCacheFormat, @@ -66,7 +67,8 @@ test fn w_the_three_startup_prefixes_are_distinct() -> Bool { fn kv_size_reading() -> VllmStartupReading { vllm_startup_reading( line: GpuKvCacheSize, - captured_text: "GPU KV cache size: 3,330,000 tokens" as NonEmptyStr) + captured_text: "GPU KV cache size: 3,330,000 tokens" as NonEmptyStr, + observed_at_revision: vllm_source_revision_read) } fn admitted_pool(a: VllmKvPoolAdmission) -> VllmObservedKvPool? { @@ -109,7 +111,8 @@ test fn w_the_concurrency_line_refuses_rather_than_becoming_a_pool() -> Bool { tokens: token_count(count: 3330000), reading: vllm_startup_reading( line: MaximumConcurrency, - captured_text: "Maximum concurrency for 262,144 tokens per request: 9.81x" as NonEmptyStr)), + captured_text: "Maximum concurrency for 262,144 tokens per request: 9.81x" as NonEmptyStr, + observed_at_revision: vllm_source_revision_read)), wanted: vllm_startup_line_prefix(line: MaximumConcurrency)) } @@ -121,7 +124,8 @@ test fn w_the_resolved_format_line_refuses_rather_than_becoming_a_pool() -> Bool tokens: token_count(count: 3330000), reading: vllm_startup_reading( line: ResolvedKvCacheFormat, - captured_text: "Using DeepSeek's fp8_ds_mla KV cache format" as NonEmptyStr)), + captured_text: "Using DeepSeek's fp8_ds_mla KV cache format" as NonEmptyStr, + observed_at_revision: vllm_source_revision_read)), wanted: vllm_startup_line_prefix(line: ResolvedKvCacheFormat)) } diff --git a/dag/test/claim/vllm_metric_semantics_witness_test.dag b/dag/test/claim/vllm_metric_semantics_witness_test.dag new file mode 100644 index 00000000000..7d58d6339b2 --- /dev/null +++ b/dag/test/claim/vllm_metric_semantics_witness_test.dag @@ -0,0 +1,101 @@ +module test.claim.vllm_metric_semantics_witness + +import std.types { Bool, String, NonEmptyStr, List } +import extdeps.vllm.metrics { + VllmServingMetric, KvCacheUsagePerc, IterationTokensTotal, NumRequestsRunning, NumRequestsWaiting, + PrefixCacheQueries, PrefixCacheHits, PromptTokensTotal, GenerationTokensTotal, NumPreemptionsTotal, + ServingQuestion, HowBusyIsThisReplica, IsThisReplicaSaturated, TokensComputedInOneEngineStep, + HowManyEngineStepsHaveRun, HowLargeAreArrivingPrompts, IsThePrefixCacheEffective, IsTheEngineThrashing, + MetricVerdict, MetricAnswers, MetricRefuses, vllm_metric_answers, vllm_metric_is_admissible, + vllm_metric_name, + InterconnectCounter, EthernetNetdevBytes, RocePortDataCounters, + interconnect_counter_sees_rdma, interconnect_counter_note, +} + +fn w_because(m: VllmServingMetric, q: ServingQuestion) -> String { + match vllm_metric_answers(metric: m, question: q) { + MetricRefuses { because: b } => b as String + MetricAnswers { note: _ } => "" + } +} + +// THE ROUTER INCIDENT, AS A WALL. A load-aware landing thresholded on kv_cache_usage_perc, +// classified every replica as saturated because a warm idle engine reads ~0.99, and silently +// degraded to the load-blind path it was written to replace. The pairing that produced that is now +// a refusal, and the gauge that should have been asked instead answers. +test fn kv_usage_cannot_answer_busyness_and_running_can() -> Bool { + !vllm_metric_is_admissible(metric: KvCacheUsagePerc, question: HowBusyIsThisReplica) + && !vllm_metric_is_admissible(metric: KvCacheUsagePerc, question: IsThisReplicaSaturated) + && vllm_metric_is_admissible(metric: NumRequestsRunning, question: HowBusyIsThisReplica) + && vllm_metric_is_admissible(metric: NumRequestsWaiting, question: IsThisReplicaSaturated) + && string_contains(s: w_because(m: KvCacheUsagePerc, q: HowBusyIsThisReplica), pattern: "not free") +} + +// THE ROOT-CAUSE INCIDENT, AS A WALL. iteration_tokens_total books each prefill's FULL prompt +// length in one iteration, so a 247,582-token observation is a large REQUEST, not a monolithic +// engine step. Reading it as per-step compute produced a confident wrong root cause that a live +// poll of the same histogram's COUNT refuted. +test fn iteration_tokens_cannot_answer_per_step_compute_but_counts_steps() -> Bool { + !vllm_metric_is_admissible(metric: IterationTokensTotal, question: TokensComputedInOneEngineStep) + && vllm_metric_is_admissible(metric: IterationTokensTotal, question: HowManyEngineStepsHaveRun) + && vllm_metric_is_admissible(metric: IterationTokensTotal, question: HowLargeAreArrivingPrompts) + && string_contains(s: w_because(m: IterationTokensTotal, q: TokensComputedInOneEngineStep), pattern: "FULL prompt length") +} + +// NO METRIC ANSWERS PER-STEP COMPUTE. This is the row that matters most, because the question is +// the one everybody wants and the honest answer is that the served surface does not carry it. A +// consumer that needs it must instrument the engine, not divide a counter by a sampling window. +test fn nothing_served_answers_tokens_computed_in_one_engine_step() -> Bool { + all([ + KvCacheUsagePerc, IterationTokensTotal, NumRequestsRunning, NumRequestsWaiting, + PrefixCacheQueries, PrefixCacheHits, PromptTokensTotal, GenerationTokensTotal, NumPreemptionsTotal, + ], m => !vllm_metric_is_admissible(metric: m, question: TokensComputedInOneEngineStep)) + && string_contains(s: w_because(m: PromptTokensTotal, q: TokensComputedInOneEngineStep), pattern: "sampling window") +} + +// AND THE MODULE IS NOT A BLANKET REFUSER, which is the control on the three rows above: every +// question has at least one metric that answers it, so a green refusal is a statement about the +// pairing rather than about the module declining to be useful. +test fn every_question_has_an_answering_metric_except_per_step_compute() -> Bool { + let metrics = [ + KvCacheUsagePerc, IterationTokensTotal, NumRequestsRunning, NumRequestsWaiting, + PrefixCacheQueries, PrefixCacheHits, PromptTokensTotal, GenerationTokensTotal, NumPreemptionsTotal, + ] + all([ + HowBusyIsThisReplica, IsThisReplicaSaturated, HowManyEngineStepsHaveRun, + HowLargeAreArrivingPrompts, IsThePrefixCacheEffective, IsTheEngineThrashing, + ], q => any(metrics, m => vllm_metric_is_admissible(metric: m, question: q))) +} + +// THE CACHE QUESTION THE OPERATOR ASKED, ANSWERED BY THE RIGHT INSTRUMENT. A permanently full pool +// is the designed state and says nothing about eviction pressure; the interval hit rate and the +// preemption counter are what carry that. +test fn thrash_is_answered_by_hit_rate_and_preemptions_not_by_a_full_pool() -> Bool { + !vllm_metric_is_admissible(metric: KvCacheUsagePerc, question: IsTheEngineThrashing) + && vllm_metric_is_admissible(metric: PrefixCacheHits, question: IsTheEngineThrashing) + && vllm_metric_is_admissible(metric: NumPreemptionsTotal, question: IsTheEngineThrashing) + && string_contains(s: w_because(m: KvCacheUsagePerc, q: IsTheEngineThrashing), pattern: "designed state") +} + +// THE OFF-METRIC READING, same mode: RDMA bytes never reach the Ethernet netdev counters, so a +// 176-second prefill measured as ~0 GB of inter-node traffic while the RoCE ports carried ~175 GB +// per rank. +// +// AND THE ROCE NOTE MUST NOT CONCLUDE MORE THAN BULK SATURATION. A low average utilisation bounds +// bandwidth and nothing else: it is equally consistent with latency-bound collectives, a straggler +// rank, or host gaps between kernels. This row pins that limit, because the first version of the +// note said "about 16% utilised, so the rail was not the constraint" -- deriving a critical-path +// share from an average, which is the same over-reading one layer up. +test fn ethernet_netdev_counters_do_not_see_rdma_and_roce_bounds_only_bandwidth() -> Bool { + !interconnect_counter_sees_rdma(c: EthernetNetdevBytes) + && interconnect_counter_sees_rdma(c: RocePortDataCounters) + && string_contains(s: interconnect_counter_note(c: EthernetNetdevBytes) as String, pattern: "bypasses the netdev path") + && string_contains(s: interconnect_counter_note(c: RocePortDataCounters) as String, pattern: "BULK-TRANSFER") + && string_contains(s: interconnect_counter_note(c: RocePortDataCounters) as String, pattern: "does not establish what share of the critical path") +} + +test fn the_metric_names_are_the_served_ones() -> Bool { + (vllm_metric_name(m: KvCacheUsagePerc) as String) == "vllm:kv_cache_usage_perc" + && (vllm_metric_name(m: IterationTokensTotal) as String) == "vllm:iteration_tokens_total" + && (vllm_metric_name(m: NumRequestsRunning) as String) == "vllm:num_requests_running" +} diff --git a/docs/design-failure-modes.md b/docs/design-failure-modes.md index 9057f9e2029..dbc218872ea 100644 --- a/docs/design-failure-modes.md +++ b/docs/design-failure-modes.md @@ -11,6 +11,7 @@ The exact `RecurringFailureMode.identity` population, which no other projection - `hollow_alias` - `state_space_conflation` - `absorbing_fallback` +- `cumulative_metric_read_as_per_event` - `empty_observation_narrow` - `edit_pass_that_matched_nothing_reports_success` - `claim_forced_and_unbracketed` @@ -192,6 +193,7 @@ The landing measurement partitions the 31 parser-visible identities into **2 cit **A FOURTH FORM, MEASURED 2026-09-02 (quick-crab-396), AND IT SITS IN THE INTERPRETER RATHER THAN IN ANY MODEL: DECLARED OPTIONAL AND CONSTRUCTED ABSENT ARE TWO STATES WEARING ONE SPELLING.** A record field declared `T?` is read by authors -- and by this document's own §2 reasoning about optional carriers -- as optional-with-a-none-default. It is not. A literal that OMITS the field typechecks, and a later read of that field REFUSES at evaluation with `NoSuchField { type_name, field }`. So `T?` means mandatory-at-construction-and-typed-optional, and the two states the spelling conflates are `the field is absent because the author wrote none` and `the field is absent because the literal predates it`. EVIDENCE, a two-row probe with a positive control, which is what makes it a finding rather than a conjecture about one type: `type Widget { name: String tag: Int? }` with `Widget { name: "w" }` refuses on `.tag` with `NoSuchField { type_name: "Widget", field: "tag" }`, while `Widget { name: "w", tag: Present { value: 3 } }.tag` reads back correctly -- so the mechanism is neither `reads fail` nor `this type is broken`, it is omission-with-no-default, and it is GENERAL to any record type. RUNG: this is FAIL-CLOSED and explicitly NOT silent wrongness -- the interpreter refuses rather than answering the false that a reader would expect -- which is the whole of the good news and it is what sets the rung at mitigatable rather than below the ladder. WHAT THE CORPUS CURRENTLY BELIEVES INSTEAD, and this is the reason the row exists: two carriers filed this on 2026-08-27 as `a nested compile is not evaluable in the interpreter, refusing with NoSuchField Node.ident` -- v1.tests.claim.reference_derived_disposition_census_witness_test and a sibling paragraph in v1.compiler.emit_rust. Nested compiles are one POPULATION that happens to construct Nodes through helpers omitting v1.std.core Node's `ident` and then read it; they are not the class, and the repair that description suggests (route around the nested pipeline with a host builtin) does not touch the cause. **POPULATION: UNENUMERATED, and stated as such rather than guessed.** It is every record type in the corpus carrying an optional field together with a literal that omits it; `v1.std.core Node.ident` is one specimen and the probe above is another. NEXT-RUNG TRIGGER, a capability and not an artifact: construction that supplies every declared field, so that a literal omitting an optional field is either unwritable or carries the absent value -- at which point the distinction between the two absences is made at construction and no read can encounter a field that was never written. NOT REPAIRED HERE BY RULING: a defaulting change alters evaluation semantics corpus-wide and needs its own subject, population and review. **A FIFTH FORM, MEASURED 2026-09-03, AND IT IS THE FIRST ONE WHERE THE DECLARATION ALREADY EXISTED: ONE SENTINEL CARRYING TWO FACTS WHILE A TYPED DECLARATION SAYS THEY ARE DISTINCT.** std.algebra free_monoid_collection_templates declares first and last as [ReceiverSelf] -> OptionalOf { inner: ReceiverElement }, and positional get as [ReceiverSelf, Int] -> the same. So the result is an optional OF THE ELEMENT: where the element type is itself optional the two absences are distinct levels. The v1 interpreter's arm bodies returned items.front().cloned().unwrap_or(Value::Null), collapsing 'the collection had no first element' and 'the first element is itself absent' into one Value::Null, because in the interpreter Null is both the absent-marker and a legal element value. DISCRIMINATING RED, executed on both arms, not asserted: over [none, Present { value: 2 }] -- a list of length TWO whose first element is absent -- the interpreter took the OUTER Absent arm (answering 'the list is empty') while emitted Rust answered Present { Absent }. Positive control [5,6].first() returned 5 in both. WHAT MAKES THIS FORM DIFFERENT FROM THE PRECEDING FOUR, and why it is filed as a receipt on this row rather than as its own class: THE MISSING CONSTRUCTOR WAS NOT MISSING. The declaration existed, was mirrored into the seed as std_algebra.rs, and was READ BY THE TYPE CHECKER -- v1.compiler.infer_method walks algebra_templates_for_profile matching on name, which is why std.target_representation representation_spelling_for typechecks with `-> RepresentationSpelling?` returning `|> first` directly. The EVALUATOR in the same binary referenced std_algebra zero times. One binary, one declaration, the type checker honouring it and the evaluator contradicting it. THE RUNG THIS SAT AT WAS NOT A RUNG: the interpreter did not refuse, it answered confidently and wrongly, which DESIGN section 4b places OUTSIDE the ladder. A REPAIR EXISTS AND IS DELIBERATELY UNLANDED (gunbc#10187, DRAFT, not merged): the two arms WOULD build optional_present/optional_absent, mirroring map_lookup_as_optional, which already performs exactly this for maps -- and the reason it is withheld is the PRIOR FILING recorded below, not an unfinished implementation. NO CLIMB IS CLAIMED BY THIS ROW. The class stands where it stood -- 'decided by call site, not value shape, so a stored V = Optional payload is never mistaken for an already-wrapped result' -- prior art in the same file rather than a new mechanism. THE REALIZATION PICTURE IS THE PART FUTURE READERS SHOULD NOT FLATTEN, AND IT HAS THREE COLUMNS RATHER THAN ONE VERDICT. VERIFIED BY EXECUTION: rust emission nests faithfully; and the python and typescript TYPE CONSTRUCTORS cannot carry the nesting at all -- typing.Optional[typing.Optional[int]] EVALUATES TO typing.Optional[int] and compares equal, and in the typescript realization `{0} | undefined` the absent element and the absent selection are the same value (undefined === undefined). No method template can repair that, because the type it would have to return does not exist in those targets; a boxed encoding would be a different realization of Optional, not a better template. REFUTED BY EXECUTION: the inference that go is faithful BECAUSE its optional row is `*{0}` and **T is a legal Go type. The row permits the nesting and the emitter did not do it -- go emitted *int64 for a declared Optional>. THE ROW PREDICTS CAPABILITY, NOT BEHAVIOUR, and that is the transferable lesson here: the python collapse and the go faithfulness were derived the same way, from the optional row, and only one survived execution. UNOBSERVED: what python and go emission actually ANSWER end-to-end, because for this input neither artifact runs -- go returns two values from single-return signatures and references an undefined Present, python uses Optional and Present with neither imported nor defined. Those sit in accepted_source_emits_uncompilable_target and are UPSTREAM of this class; the denotation question is not reachable there until the artifacts compile. NO REFUSAL IS FILED FOR THOSE PATHS AND THAT IS DELIBERATE. A refusal on unrepresentable nesting would be a wall in front of a hole, and for typescript it would be unreachable in any case: main.rs parses only rust | python | go | dag and RenderTarget has no TypeScript variant, so no fixture can drive that path and any check over it would be permanently green by construction. Recorded here as an OBSERVED BOUNDARY with its evidence rather than as a check that establishes nothing. PRIOR FILING, FOUND AFTER THE REPAIR WAS WRITTEN AND BEFORE IT LANDED, WHICH IS WHY THE REPAIR IS DEFERRED RATHER THAN SHIPPED: this seam is ALREADY FILED as audit-queue item 0 of docs/plans/compiler-guarantee-recovery-gap-analysis.md (2026-08-27, from gunbc#9491), and extdeps.bmc.pid_control_decode carries the minimal reproduction inline plus an admitted workaround -- it renames its own field to magnitude on exactly this ground and claims no compiler guarantee. Item 0 states the class as TWO questions and warns the eventual owner not to collapse them. (1) MEMBER-SELECTION POLICY: v1.compiler.infer_lookup field_summary_for_type opens `if field == "value" && normed_opt` and unwraps otherwise, so FIELD ACCESS THROUGH AN OPTIONAL AUTO-UNWRAPS FOR EVERY SPELLING EXCEPT `value` -- which item 0 calls a language-semantics decision, not obviously a bug, with three candidate answers. (2) TYPE CONFORMANCE, below floor independently of (1): a record-valued expression accepted where an Int is required. THAT PRIOR FILING RECLASSIFIES THIS ROW'S OWN REPAIR AND THE RECLASSIFICATION IS THE POINT. The type path ALREADY treats first() as optional and silently unwraps field access through it; the evaluator returned a bare element, so that unwrap was a no-op. Making the evaluator construct a real Present{...} therefore does not merely expose consumers that depended on a flattening -- it makes the EVALUATOR INCONSISTENT WITH AN AUTO-UNWRAP THE TYPE CHECKER STILL PERFORMS, in the opposite direction from where the two halves started, which is not a climb. Honouring the declaration at the evaluator requires the matching evaluator-side unwrap, and that IS item 0(1)'s open policy question. SO THE INTERPRETER/EMITTED DIVERGENCE AND ITEM 0(1) ARE THE SAME SEAM APPROACHED FROM TWO SIDES, AND WHOEVER SETTLES THE MEMBER-SELECTION POLICY OWNS THE EVALUATOR REPAIR TOO. Filing them as separate subjects would split one decision across two owners. CORROBORATION FOR ITEM 0(2), independently re-derived here with a control rather than taken from the filing: four arms on a binary BUILT FROM MAIN -- `fn f() -> Bool { 5 }` refuses with 1 error (the positive control, proving the return check is not globally dead), while `fn f() -> Bool { xs().first() }` over a plain List compiles with ZERO diagnostics. Same result under a binary containing the repair, so the gap is pre-existing and not created by any change here. That is corroboration of a class filed five days ago and is NOT a second finding; a second row for a filed class is the nicknaming refusal at the ledger layer. NEXT-RUNG TRIGGER, named as a capability: GENERATED ARM BODIES WHOSE RETURN TYPE DERIVES FROM THE ALGEBRA TEMPLATE, so the collapse becomes unwritable rather than merely wrong. Today the arm SELECTION is generated -- v1_interpreter_dispatch_generated.rs -- but from gunbc.v1_interpreter_primitive_surface, a roster that carries no return type and does not import std.algebra. Two rosters name first, neither references the other, and the hand-written body honoured neither; that fork is why the arm and its declared type were separable by hand at all. A POINTER TO ONE SPECIMEN OF THIS SHAPE IN THIS LEDGER'S OWN TOOLING, 2026-09-03: a referential-closure check over these rosters that JOINS BY NAME cannot see two declarations sharing one name whose CONTENT differs. Measured by injection -- main's `recurrence_ledger_scoped_below_the_recurrence` duplicated with a pre-rename spelling of a symbol it cites, 9006 bytes against 8993 -- a name-grain check reports nothing rostered-undeclared and nothing declared-unrostered and PASSES, while the tree carries a citation to a symbol with zero declarations. The guard's state space has no cell for content divergence, so its count is the same either way. What such a check must additionally do is adjudicate any surviving duplicate against main's BYTES and REFUSE when neither copy matches, rather than resolve by position or first-wins, which is silent. **A SIXTH FORM, whose consequence is not a bad message but a SILENT SAFETY HOLE: a `Bool` return carrying a three-valued question, where the collapsed value is the ADMITTING one.** Specimen (gunbc#10337, 2026-09-04): `where_refinement_brand_nominal_mismatch` returned `Bool` over three outcomes -- these are distinct declarations, these are the same declaration, and I CANNOT TELL which declarations these are. The third collapsed onto `false` with the second, and the consumer read `false` as admit, so a cast whose brand identity could not be determined was ACCEPTED. The paired sentinel made it concrete: the declaration-key producer returned the empty string for a type with no declaring span and the consumer guarded on `key != ""`, so absence of identity was spelled as a value and read as permission. **WHAT DISTINGUISHES THIS FORM FROM THE EARLIER ONES IS THE DIRECTION OF THE COLLAPSE.** They produce a wrong or unrankable ANSWER; this one produces NO ANSWER AT ALL on the collapsed arm, because the admitting value is indistinguishable from a genuine pass. A wall with this defect emits no diagnostic, so no red exists for review to find and no population exists to count -- the type's inadequacy is the safety hole rather than a symptom of one. **Recognition rule for this form: for every `Bool` that gates a refusal, ask which value means UNKNOWN. If unknown shares a value with permitted, the predicate cannot refuse what it cannot determine, and the wall has a hole exactly the size of its unknown set.** **AND A NEW DETECTION ROUTE FOR THIS CLASS, which is why the specimen is worth its length: the earlier receipts in this row were found by READING the producer. This one was invisible to reading in the strong sense -- it was read, and approved, by three separate reviews of the diff, and by its own author who had rejected the identical fail-open shape in a competing design an hour earlier.** It was settled by EXECUTING a fixture: `fn f(a: String where brand("x"))` is an inline brand-carrying type with no declaration site, it parses and compiles, and it admitted one brand into a different brand -- the exact violation the wall's own refusing arm exists to catch. Reading establishes that an arm is reachable IN PRINCIPLE; only executing establishes that the state is AUTHORABLE and that the hole is live. Per DESIGN section 4b, the authorability question is decided by what a FIXTURE can hand the compiler, never by what the accepted corpus happens to contain. CLIMB, and it is construction rather than a better check: the key producer now returns `String?` so an undeterminable key is NOT REPRESENTABLE, and the predicate returns a four-variant `BrandNominalVerdict` so no consumer can inherit an answer from a magic value. Fixing only the sentinel would have closed the instance and left the `Bool` free to spawn the collapse again at the next consumer -- the return TYPE was the authority at fault, not the arm. CI CHECK-RENDERING INSTANCE, 2026-09-04, import-cut lane, AND ITS RECEIPT IS THAT ONE COLLAPSED SYMBOL PRODUCED ERRORS IN BOTH DIRECTIONS. `gh pr checks` renders cancelled, failure and timed_out into a single `fail` glyph, and the dashboard digest inherits that rendering. Measured on gunbc run 33875856188: the jobs API reports required-witnesses-build, heal-generated-artifacts and required-witnesses-floor as conclusion=cancelled with a ZERO-LENGTH steps array, and the aggregating witnesses job as conclusion=failure with 3 steps -- while the rollup showed four identical fails and the digest reported the PR as failing. THE TWO STATES HAVE OPPOSITE OWNERS AND OPPOSITE REPAIRS, which is this row's recurring form: a real red means the branch is broken and its author must fix it; a cancellation means NOTHING ABOUT THE BRANCH WAS OBSERVED and there is no signal to act on. Collapsed into one glyph, the second is indistinguishable from the first. THE RECEIPT, and it is why this instance is worth carrying rather than being one more example: the SAME collapse has already produced the opposite error. Cancelled-read-as-PASSED merged gunbc PR 8059. Cancelled-read-as-FAILED, here, manufactures a defect in a branch nobody tested and sends an author to debug against no signal. One symbol, one rendering layer, errors in both directions -- which demonstrates that the harm is not a mis-set threshold but the conflation itself, since no reading of a symbol that means two things can be right in both cases. REPAIR: take the verdict from the jobs API `conclusion` field, never from `gh pr checks` or a digest, and treat an EMPTY steps array or an empty log as UNOBSERVED rather than as a result. The tell that the rendering is lying is shape rather than content -- several independent jobs ending at an identical short duration, because real work does not fail identically.) A SEVENTH FORM, AND THE ONE WORTH THE MOST BECAUSE A REPAIR PRODUCED IT (2026-09-04, import-cut lane, gunbc#10454, found by independent adjudication and confirmed in source). A review correctly refused two hand-rolled variant-to-Bool predicates over a coproduct -- the structural-equality residue DESIGN section 6 names -- and the repair replaced them with a total String projection, `unobservable_reason`, returning the refusal text or `""` for the two OBSERVED verdicts. The refusal was then decided by asking whether that string was empty. So `""` carried TWO meanings, this observation does not refuse and this observation refuses but recorded no reason, and an `Unobservable` with an empty reason was folded away as OBSERVED: the digest stayed empty and the command returned ExitSuccess while the roster could not testify about its subject. That is the EXACT defect the pull request existed to remove, re-entering through its own fix. WHAT MAKES IT A DISTINCT FORM RATHER THAN ANOTHER EXAMPLE: the earlier forms are collapses that were AUTHORED. This one was INTRODUCED BY A CORRECT REPAIR of a different class, in the same motion, and it was invisible for the usual reason plus one more -- no production site constructed the bad value, so no run could exhibit it. The residue removal was right, the projection was right, and the sentinel was the only wrong part; a reviewer checking that the predicates were gone would find them gone. RECOGNITION RULE: when a repair replaces a decision with a PROJECTION, ask what the projection's empty or default value means, and whether the decision now rests on it. A refusal must be driven by the verdict, never by the presence of its prose. AND IT SHARPENS THE AUTHORABILITY RULE THIS ROW ALREADY CARRIES. Not reachable on the corpus is not a defence: DESIGN section 4b asks whether the RED is AUTHORABLE, not whether it fires today, and a fixture constructed the empty-reason refusal in one line. THE FIRST REPAIR WAS ITSELF INSUFFICIENT AND THAT IS PART OF THE RECEIPT: driving the refusal off the canonical verdict projection only traded a reason-text sentinel for a VERDICT-LABEL sentinel, because that projection is also the renderer's word source, so a relabelled or newly added refusing state falls through a String inequality to false. The exhaustive match protected label PRODUCTION while refusal CLASSIFICATION sat on an implicit catch-all over the String domain, in the accepting path. The typed-coproduct fix is a trap -- it returns the classifier to matching variants onto true/false, the very residue whose removal started this -- so the landed shape folds to a LIST of refusing observations by a total, VALUE-RETURNING match, with nonemptiness as the condition. Exhaustiveness then sits over the DECISION rather than only the projection, and no rendered spelling decides success. The discriminating red -- a refusal with no reason must still refuse -- stays enrolled under section 4b(4) and flips alone when the text-driven form is restored, while the stale-citation red and the observed-verdict control both hold. A hole that only stays shut because nobody happens to author the bad input is not a wall, and a sentinel that moved from the prose to the label was still a sentinel.A REPRESENTATION TAG IS PRESENT AND THE READER DISCARDS IT. v1.compiler.infer_types algebra_child_or_placeholder reads a Node child position as a generic argument without requiring the nominal application shape. A NoConnective application uses those positions for arguments; an expanded Disj uses them for variant members. instantiate_algebra_type therefore supplies the Empty variant as ReceiverElement for an expanded FreeMonoid, although Cons.head still carries the actual element type. The wrong type is returned with no diagnostics by that constructor; the later pattern/field consumer produces the cascade. This is not evidence that every surrounding diagnostic census omits the error. The producer that re-derives the specimen is cli_run::resolve_entry_graph on test.claim.emission_provenance_test, selecting emission_module_loss_is_separate_from_declaration_loss and serializing its actual resolved Node, rather than the DAG export's untyped occurrence projection. BuildBuddy invocation 5d99008a-cf50-4994-952f-b6e3e707a188 binds source 55422b847050633e9ae1dc1284200c1d4306319e; retained resolved-control content SHA256 fa463474498b90092ff5a4a70d25d29ea599bc07188f2d8ecb75ffa053e2bbe2. The call-site census names field instantiation and declared method-argument instantiation as independent callers, and apply_type_substitution as an adjacent positional reader requiring its own assessment. Static production reachability is established; a dynamic census of affected nonfixture expressions is not. The existing resolve_node_bounded generic-use producer already keeps actual arguments in a nominal wrapper and structural expansion in inferred. The repair obligation is to preserve or explicitly recover the application evidence at the consumer boundary, refusing when unavailable. Reading a variant as an argument, or special-casing the failing assertion to avoid the callback contract, would conceal the conflation rather than discharge it. The unchanged module-loss control remains the discriminating execution; no repair has yet been claimed. **AN EIGHTH FORM, AND IT IS THE MOST DANGEROUS ONE THIS ROW CARRIES, BECAUSE THE CONFLATION IS DEFENDED BY A PAIR OF GUARDS THAT EACH LOOK CORRECT ALONE.** Specimen, on main at ca1e35e1d1, `v1.compiler.infer_types` at the direct-call site: a lookup returns `FuncSigLookup` = `FuncSigResolved | FuncSigUnresolved | FuncSigAmbiguous`, the consumer narrows it to an `Optional` by mapping BOTH non-resolved constructors to `Absent`, and then computes `formal_authority_available` with `Absent => true` and a refusal reason of `""`. **ABSENCE OF ALL AUTHORITY IS CODED AS AFFIRMATIVE AUTHORITY, WITH AN EMPTY REASON.** The contrast inside the same match is what makes it unmistakable rather than arguable: **EVERY ARM HOLDING A REAL DECLARATION PERFORMS A GENUINE CHECK** -- the declaration-bound and kernel-grounded arms compare the resolved formal count against the call's value-argument count, and the local-formals arm answers `false` -- so the ONLY arm that answers an unconditional `true` is the one that has nothing to check with. Arguments are then typed against an empty formal list rather than the declaration's, so a call whose signature could not be found is inferred in the CALLER's vocabulary and accepted. **THE PAIR IS WHAT MAKES IT SURVIVE REVIEW: BOTH consumers of the Boolean conjoin `sig != none` before reading it** (`04_infer.dag` at the `arg_call` site and at the `formal_authority_diags` site, identically on the merge base 50f1b50f92, so this is latent and pre-existing rather than introduced). The absent case is therefore excluded twice, independently. **A REPAIR THAT TOUCHES ONLY ONE OF THEM PASSES REVIEW, CHANGES NO BEHAVIOUR, AND RETIRES THE FINDING FALSELY** -- flipping `Absent => true` to `false` alone is a provably inert edit, because the guard never lets the value be read in that state. Recognition rule for the pair: when a defective value is protected by a guard, VERIFY THE REPAIR BY EXECUTION AND NOT BY DIFF, because a conjunction-shaped defect needs every conjunct broken to show a difference, and the negative controls must break ONE AT A TIME. **AND THE REASON THE ARM EXISTS IS THE CONFLATION ITSELF, WHICH IS WHY THE OBVIOUS TWO-LINE REPAIR IS WRONG AND WOULD BE WORSE THAN THE DEFECT.** `FuncSigUnresolved` IS NOT A REFUSAL STATE. Verified from the substrate rather than assumed: `v1.compiler.lookup` `resolved_plain_call_target` reads `FuncSigUnresolved => builtin_call_target_or_undetermined(func_name)` -- it is the BUILTIN-OR-MISS state. And its producer at this very site, `body_shadow_aware_func_sig` (whose only caller is the defective site), answers `FuncSigUnresolved` when the callee is a BODY LOCAL, deliberately erasing a distinction it holds in hand; two further producers do the same for descent-context locals and for parameters. **ONE CONSTRUCTOR THEREFORE CARRIES FOUR STATES: builtin callable, locally-bound callee, parameter callee, and genuine missing declaration.** The consumer's `Absent => true` is an author trying to keep the builtin and local paths alive while holding a value that cannot say "builtin". **SO THE BOOLEAN IS A SYMPTOM AND THE CARRIER IS THE DEFECT: A VOCABULARY THAT STOPPED BEING SUFFICIENT FOR THE QUESTION ITS CONSUMER MUST ANSWER.** Flipping the arm AND removing both guards -- the repair that reads as obviously correct from the first two clauses alone -- REFUSES EVERY BUILTIN CALL IN THE CORPUS: it converts a fail-open into an over-refusal and calls it fixed. That is why this receipt carries all four clauses and not the first two: a row naming only the Boolean invites a repair that moves the failure rather than removing it. **CEILING AND REPAIR SHAPE: structurally impossible, and attainable, because the distinction is decidable at the lookup.** The repair is a semantic partition, not a Boolean, and it has FIVE constructors: declared-callable-resolved (carrying identity, signature and resolved formals), builtin-callable-resolved, LOCALLY-BOUND-CALLEE, unresolved, and ambiguous -- with the call decision TOTAL over them. **AVAILABILITY IS THEN REPRESENTED BY WHICH CONSTRUCTOR IS HELD, and no Boolean needs an interpretation for "there is no formal authority because there is no declaration".** **THE PRODUCER AND THE CARRIER ARE ONE CHANGE SEEN FROM ITS TWO ENDS, WHICH IS THE PART A READER IS LIKELY TO GET WRONG: you cannot stop the laundering without somewhere for the distinction to land, and that somewhere IS a constructor.** So `body_shadow_aware_func_sig` DISSOLVES rather than being repaired -- its match becomes the producer of the locally-bound arm instead of a lossy pre-step in front of the lookup -- and the carrier gains the arm it was being erased into. That is a deletion and a climb at once: the state is REPRESENTED rather than inferred from the absence of a signature. **AND THE CONSTRUCTOR MUST CARRY THE BINDING, NOT MERELY ANNOUNCE THE STATE.** The lossy function matches `Present { value: _ }` -- it holds the local's binding and discards it with an underscore before returning a token meaning "no answer" -- and that binding is precisely what a shadowed callee needs in order to be typed. A locally-bound constructor that names the state without carrying the binding gives the consumer A NAME FOR ITS IGNORANCE INSTEAD OF THE INFORMATION, which is this row's own failure mode one turn later. **THE DISCRIMINATING EVIDENCE IS CONJUNCTION-SHAPED AND NEEDS THREE NEGATIVE CONTROLS, ONE PER CONJUNCT** -- the Boolean arm, the `sig != none` guard, and the shadow laundering -- each restoring ONE defect while the others stay repaired, and each of the three must FAIL. A probe that mutates two at once proves the set works and proves no member of it load-bearing. The shadowing case in particular must not be stated as "continues rather than refuses": a broken implementation continues by silently binding to the OUTER DECLARATION and passes such a check. **GIVE THE SHADOWED LOCAL A TYPE THAT DIFFERS FROM THE SAME-NAMED DECLARATION'S AND ASSERT THE CALL IS TYPED FROM THE BINDING**, so leakage is a red rather than a pass and the case discriminates shadowing from ordinary resolution instead of merely from refusal. **EXECUTED RECEIPT THAT THE ARM IS TRAVERSED BY A REAL CORPUS CALL, NOT A FIXTURE** (2026-09-06, quick-heron-85, three cells, one forced predicate, serial under an exclusive lock). Baseline: the deepest structure the compiler builds is 1,800,008 levels at `v2.lens.reference_deps`, at a call whose callee declaration the resolver refuses to admit. Forcing the admission predicate FALSE leaves it bit-identical at 1,800,008; forcing it TRUE collapses it to 85 AND MOVES THE DEEPEST SITE to an ordinary item in an unrelated module -- 85 being exactly the number main and the merge base produce. Four orders of magnitude on one predicate, with the site moving, is the phenomenon disappearing rather than shifting. **THE CALL WHOSE SIGNATURE COULD NOT BE FOUND WAS NOT REFUSED -- IT WAS ELABORATED IN THE CALLER'S VOCABULARY**, which is what the arm does and why a fail-open of this shape is not a cosmetic defect: it built a 1.8-million-level chain out of a call it should have stopped on. **WHAT THE RECEIPT DOES NOT ESTABLISH, recorded so a later reader does not overclaim it:** it says nothing about the repair's shape. The blocking-diagnostic delta between the forced cells is the forced predicate breaking unrelated resolution, and citing it as corroboration for the partition would be reading an artefact of the instrument as evidence about the fix. **RECOGNITION RULE FOR THE FORM: when a consumer narrows a closed variant to an `Optional`, the narrowing is where the states are lost, and the arm that gives the collapsed case a permissive default is where the loss becomes a fail-open.** The tell is a refusal reason that is empty on some path: a refusal that cannot say why was never a refusal. - absorbing fallback (a failure arm that widens — rerun everything, scan all keys, always run — instead of refusing; ⊤-as-answer conflated with ⊤-as-ignorance; the deficit's frequency zeroed by construction, the corpus-denominated cost breaking the budget later) +- a metric that ACCOUNTS is read as a metric that MEASURES (an upstream counter or histogram whose observation is booked once per request, per retention decision or per accounting event, consumed as though it described one event, one step or one instant; the arithmetic succeeds, the units look right, and the answer is about a different quantity). RECOGNITION: the reading is plausible, dimensionally consistent, and nobody can name the line of upstream source that defines the observation. Ask what the producer OBSERVES, not what the name suggests, and ask it of the installed binary rather than the upstream repository — a self-reported version string is a claim the process makes about itself, not executable provenance. WHY IT RECURS: these metrics are the only cheap window into a closed upstream, so a wrong reading is not caught by the substrate at all — it is caught by a prediction failing in production, weeks later, against a number that looked authoritative on a dashboard. Two independent investigators reached the SAME wrong root cause from the same histogram in one afternoon, which is the tell that the metric's name was doing the reasoning. RECEIPTS 2026-09-06/07, DeepSeek V4 Flash on the Spark pairs. (1) kv_cache_usage_perc read as occupancy: it counts every block that is not FREE and an evictable cached prefix is not free, so a warm idle engine reads ~99%; the dashboard rendered it red and captioned it 'referenced by running requests' beside REQ 0, and the router's load-aware landing classified every replica as saturated and silently degraded to the load-blind path it was written to replace — live and inert for a day while one head took all traffic and the other took none. (2) iteration_tokens_total read as tokens-computed-per-engine-step: the build observes `prompt_token_stats.total`, the FULL prompt length of each prefill booked in one iteration, plus generation tokens. Its large buckets therefore count large-PROMPT REQUESTS, not stop-the-world steps. A single observation of 247,582 tokens was read as one monolithic engine step, producing a confident root cause — a missing --max-num-batched-tokens — that a live poll of the same counter refuted: the count climbs steadily through a cold prefill, so chunking was working the whole time and the proposed flag was a no-op. (3) Ethernet netdev rx_bytes/tx_bytes read as interconnect traffic: the ranks speak RoCE, whose bytes never reach those counters, so a 176-second prefill measured as ~0 GB of inter-node traffic. The same prefill moved 175 GB per rank on the RoCE port counters. An earlier reading of that port taken across a CACHE HIT was then reported as prefill utilisation. (4) THE SHARPEST INSTANCE, AND THE ONE THAT SURVIVED THREE REVIEWS. iteration_tokens_total_count is ENGINE-WIDE: every request's steps increment it, so a co-tenant's decode steps land in any window measured around one probe. Dividing a request's computed tokens by that count produced 158,764 / 83 = ~1,913 tokens per step, which was read as the effective scheduler budget and matched to vLLM 2048 fallback. The coincidence is what made it survive: 1,913 sits within 7% of a real default, so a fabricated quantity arrived looking like confirmation of an independent fact. Three further conclusions were then built on the 2048 -- that the budget was already at its floor, that a step costs 2.3s, and that an 88ms excess across a 2048 boundary bounded a per-step cost -- and a fourth, a three-term fit carrying a 126ms per-step term, was authored to refute the third while regressing against the same imaginary step count. IT WAS SETTLED BY MAKING THE COUNTER MEAN WHAT IT APPEARS TO MEAN. On an IDLE engine no other request increments it, so the engine-wide count IS the per-request step count. Measured there, every window from 512 to 20,082 computed tokens completed in exactly ONE iteration: the effective budget is at least 20,082, an order of magnitude above the 2048 that four conclusions rested on, and no boundary exists anywhere in the range those experiments probed. Under co-tenancy the same ~2,050-token request recorded 4, 5 and 17 iterations. THE STANDING REMEDY THIS RECEIPT ADDS IS TWO WALLS, BECAUSE EXCLUSIVITY ALONE IS NECESSARY AND NOT SUFFICIENT. FIRST, ATTRIBUTION: an aggregate delta belongs to one request only under a complete attribution boundary -- same launch throughout, exact start and end, no counter reset, complete label population, no foreign contributor, and no booking that lands outside the window. Sparse polling earns only NO FOREIGN REQUEST OBSERVED AT THIS POLLING GRAIN, which is weaker than exclusivity and must be named as such. SECOND, GRAIN: even under perfect attribution, dividing a delta by an event count yields a POPULATION MEAN and nothing else -- not any particular event's size, not the maximum, not the distribution, not a configured limit. An exact per-event quantity requires either a ONE-EVENT population or per-event accounting emitted by the producer. Both halves of this incident follow from that single rule: 158,764 tokens over 83 iterations is a mean and was read as a budget, while 20,082 tokens over exactly ONE attributed iteration is a behavioural lower bound on what one iteration carried -- and still not a configuration readback, since the configured value may be higher and may not even be the governing quantity. SHAPE, AND WHY IT IS ONE MODE RATHER THAN THREE: each is a cumulative accounting quantity consumed as an instantaneous or per-event one. The remedy is the same in all three — bind the reading to what the producer observes, at the revision and binary that produced it, and make the wrong question refuse rather than return a number. - **empty-observation narrow** (the absorbing fallback's mirror: ⊥-as-answer conflated with ⊥-as-ignorance — an observation that could not express what changed rendered as the verdict "nothing is affected", so the run narrows to nothing instead of widening. Strictly worse than the widen §5 already forbids: a widen is merely expensive, a narrow is silently uncovered. Live specimen: a push whose baseline ref *is* the pushed ref diffs the commit against itself, and the witness floor reported the resulting zero-path observation as "N unaffected (import-closure, skipped without resolve)" — receipt, two main pushes reporting an identical `4118 unaffected` on different commits, a skip count that cannot be a function of the diff. The tell that one observation is wrong rather than three local choices reasonable: its consumers disagree — compile-clean widened to whole-tree, regen gated on the event name, the witness floor skipped) **RECEIPT, AND IT MOVES THE CLASS FROM AN OBSERVATION'S EXPRESSIVENESS TO AN ANALYSIS'S RANGE** (quick-crab-396, 2026-09-02). Authoring a coproduct arm named `Refused`, I ran a corpus-wide collision census over TYPE names, got a clean answer, and authored the name. required-regen then refused: `variant 'Refused' appears in both 'TypeRealizationDecision' and 'EmitterOutcome'`, and re-running the census over VARIANT spellings found `Refused` live in SIX declaring types. THE DEFECT WAS NOT A MISSING ANALYSIS. The compiler already computes exactly this collision -- it is what refused -- but it computes it during v2 self-compile over ONE RESOLVED CLOSURE, and the question being asked was about the CORPUS. Same subject, two ranges. From the author's seat the narrow analysis's silence was indistinguishable from a negative result, which is this row's own conflation with the scope quantifier moved: not an observation that could not EXPRESS what changed, but one that could not REACH where the answer lived. WHY IT IS FILED HERE RATHER THAN AS A NEW CLASS: the mechanism is identical and only the axis differs, and DESIGN section 2 prices a fresh authority for an existing concept as a failed decomposition. RECOGNITION RULE: when a check answers clean, ask what POPULATION it ranged over and whether that is the population the question was about -- an analysis scoped to the wrong range is, from the caller's seat, indistinguishable from an analysis that does not exist. NEXT-RUNG TRIGGER for this instance, named as the capability and not as an artifact: A CORPUS-WIDE NAMESPACE CENSUS OVER EVERY TYPE DECLARATION AND EVERY COPRODUCT ARM, sufficient to answer 'is this spelling already declared anywhere in the corpus' before a name is authored, rather than after emission. ONE census over the one Node tree, not a second check beside the type-name one -- two checks over one namespace is the range split that produced this instance.**RECEIPT, A THIRD AXIS: THE ACQUISITION RETURNED NOTHING AND THE COUNT OVER IT RENDERED AS CLEAN** (bright-ram-778, 2026-09-05). A merge-readiness sweep reported `non-green=0, still-running=0` for a pull request and was read as ready. It was counting conclusions in an EMPTY array: the head had no check-runs at all because its one workflow run was still queued, so every per-conclusion count was faithfully zero. The two earlier receipts on this row are an observation that could not EXPRESS what changed and an analysis that could not REACH where the answer lived; this one is an acquisition that returned NOTHING, and it lands in the same place because the mechanism is the same -- an empty population's answer is indistinguishable from a clean one, and the recognition rule above catches all three by asking what population was ranged over. THE CONSERVATION ASSERT IS THE REPAIR AND IT IS CHEAP: refuse to believe any per-conclusion count until the TOTAL is nonzero. It is the same shape as the roster-name-to-declaration join that caught a nine-row stale census the same night -- a count reads as a plausible near-miss where named unmatched rows are unmistakable. THE NEIGHBOUR PAIR MUST NOT BE COLLAPSED, because they need opposite responses and conflating them is how a lane waits forever: zero check-runs WITH a run of any status is QUEUED and nothing is wrong; zero check-runs with NO run at all means no verdict exists and none is coming. The discriminator is `actions/runs?head_sha=`, never a truncated one.AUDITED AND BOUNDED AGAINST [[required_evidence_absent_reads_as_evidence_of_pass]], WHICH THE THIRD RECEIPT ABOVE COMES CLOSEST TO AND IS NOT. Both are an empty enumeration answering affirmatively; the separation is WHO CONSUMES IT AND WHAT WOULD SUPPLY THE MISSING FACT. Here a READER counts conclusions in an empty array and the repair is the conservation assert already stated -- believe no per-conclusion count until the total is nonzero. There an ADMISSION ARM of a merge gate quantifies over arrived rather than required evidence, and its ceiling is structural in the merge queue, which no assert on a report can reach. The family axis and the full membership are carried once, in [[absent_reads_identically_to_never_looked]]. - **an edit pass that matched nothing reports success** (a scripted deletion, rename or rewrite whose locator binds to the WRONG CONSTRUCT finds zero subjects, changes zero bytes, and exits zero -- and a pass that acted on nothing is byte-for-byte indistinguishable from a pass that correctly had nothing to act on. INVALID STATE: an edit pass whose outcome is consumed as done while nothing establishes that its locator ever bound to the intended subject. IT IS THE EMPTY-CAPTURE CLASS MOVED FROM READING TO WRITING, and the move is what makes it worse. An empty capture yields a false all-clear about someone else's artifact; an empty edit yields a false all-clear about an obligation THE AUTHOR OWES, so the debt stays on the roster while the commit message says it was paid. SPECIMEN, 2026-09-04, gunbc#10324, paying a consumed-admission obligation on NAMESPACE_TRANSITION_ADMISSIONS. The pass anchored the array on the literal `&[`. That substring occurs FIRST in the declaration's TYPE ANNOTATION -- `&[TransitionAdmission]` -- so the parser bound to a 19-character body, walked it, found zero rows, deleted zero rows and would have reported the obligation paid with an EMPTY DIFF. The wall would then have refused the next push for the same two rows, against a branch whose own entry claimed to have deleted them. WHY IT SURVIVES REVIEW: every part reads correct in isolation. The locator is a real substring of the real declaration, the brace walk is correct code, the exit status is honest, and a reviewer reading the diff sees a plausible dissolution entry beside no row changes -- which is exactly what a legitimately-already-empty roster also looks like. THE DISCRIMINATOR IS A DENOMINATOR ASSERTION, NOT A LOUDER LOCATOR: assert `examined == kept + deleted` and require `examined` to equal the population counted by an INDEPENDENT instrument before consuming the result. Here that assertion reported 4 = 2 + 2 and the earlier broken pass reported examined=0, which is what separated the two cases. The independence matters: a grep of `TransitionAdmission {` over the same file answers ONE HIGHER than the roster, because it counts the struct DEFINITION alongside the rows -- so the cross-check must be a different instrument, not the same one re-run. RUNG FOUND AT: mitigatable, and only where the assertion is written. Absent it the class is silent wrongness, because no arm of the pass ever refuses. CEILING: mechanically preventable. An edit pass is an ordinary program, so its subject count is decidable and a harness can require every pass to declare an expected population and refuse when the match count is zero or disagrees. NEXT-RUNG TRIGGER: a shared edit-pass harness that takes the subject population as a REQUIRED input and refuses on a zero or mismatched match count, so the assertion is structural rather than per-author diligence. An author remembering to write the assertion does NOT retire this row -- that is the state the specimen was already in, and it held only because this branch had been burned once already.)**THE MIRROR CASE, AND IT WIDENS THIS ROW'S INVALID STATE RATHER THAN ANNOTATING IT** (calm-eagle-759, 2026-09-05). This row already names a locator binding to the WRONG CONSTRUCT as the cause, but states the outcome only as zero subjects and zero bytes. The wrong-construct binding has a second outcome, and it is the worse one: the locator finds a REAL match that is not the intended subject, and ACTS on it. RECEIPT: a pass meant to remove one arm from a specific function searched the whole file for a variant name that also appears in two other matches over the same coproduct, bound to the first, and edited a different function entirely. The identity was (enclosing function, variant) and the locator carried only the variant. WHY THE MIRROR MATTERS FOR THE TRIGGER STATED ABOVE: a harness that refuses on a zero or mismatched COUNT does not catch it, because the count is right -- one match was expected and exactly one was found. It was simply the wrong one. So the required input is not the population SIZE but the population IDENTITY, and this row's trigger should be read as demanding the latter. In the receipt the only signal was an incidental echo of the removed line, which ended in a span-returning arm where a disposition was expected. The neighbouring row is comparison_keyed_on_part_of_the_identity_it_compares -- the locator keyed on a proper component of a composite identity -- and that is the same defect applied to a WRITE rather than to a comparison. Both are separate from edit_range_bounded_by_a_neighbour_rather_than_the_target, where the locator binds correctly and only the extent overreaches.) - **CLAIM-FORCED AND UNBRACKETED -- a shared memoized host artifact whose identity is not knowable before the fold and which records no fill bracket.** INVALID STATE: the artifact is shared, so a later claim reads it warm, but nothing brackets the fill, so the whole build lands on whichever claim reached it first. HARM, AND THE REASON THIS CLASS HIDES: the number that lands is REAL -- a true measurement of something, charged to a row that does not own it -- SO NOTHING LOOKS WRONG. The row's reported cost becomes a function of DISCOVERY ORDER rather than of the row, and because the required floor's safety limit is a FAIL-STOP, an order-dependent number decides whether the line stops. DISTINGUISHING FACT, AND IT IS THE FORCING CLASS RATHER THAN ANY OBSERVED FIELD: an artifact every identity of which is knowable before the fold is PREPARATION-FORCEABLE -- the floor builds it ahead and bills preparation, no claim window ever contains that build, and it therefore carries NO fill bracket CORRECTLY. ABSENCE OF A BRACKET IS NOT EVIDENCE OF A DEFECT, and reasoning from that absence is the specific misread this class invites. An artifact only a claim can force must record its fill, at which point the claim is charged the marginal half. The third state is the conjunction: claim-forced AND unbracketed. RECOGNITION RULE, AND IT IS NOT AVAILABLE FROM THE COST LOG: a cost log reports what a run OBSERVED, while regime membership is a property of the CONSTRUCTION -- is the artifact memoized, is it shared, is every identity it can be asked for knowable before the fold. Those are decidable by reading the producer and are not recoverable from a measurement, so a row's absence from shared-fill accounting is equally consistent with preparation-forceable, with not-shared-at-all, and with this class -- and the floor output cannot separate the three. SPECIMEN OF THE MISCLASSIFICATION RATHER THAN OF THE CLASS (2026-09-03): one subject was placed in four different regimes in three hours, every placement derived from floor output and each one propagated onward, and it was settled by reading the call site -- v1_interpreter eval_fn_arrow_decl_facts_live has no memo, cache or fill bracket at all, so there is no first toucher and no free later reader, and it reaches the interpreter as a HOST BRIDGE ARM and therefore has no resolved .dag fn node to install in a roster keyed on such nodes. Its sharing tier is unreachable BY CONSTRUCTION rather than by omission, and its cost is its own. THAT ROW IS CORRECTLY BILLED, AND ADMITTING IT TO A MISATTRIBUTION CENSUS WOULD CORRUPT THE POPULATION -- six callers each paying the same unshared walk is a real defect, but it is SHARED WORK NEVER MADE SHAREABLE, whose cost can be made cheap, and not one row carrying cost it does not own. PRIOR INCIDENT, WITH A NUMBER: the census memo sat in this state from gunbc#9428 and refused the required floor at 5812ms against a 5000ms fail-stop while two later claims read the same compiles free. COMPOUNDING RISK, WHICH IS WHY THIS OUTRANKS AN ORDINARY COST DEFECT: gunbc#9517 proposes freezing rows above the line in a shrink-only contract, and a row frozen for cost it does not own can never be made cheap, so it can never leave -- a misattributed measurement becomes PERMANENT. RUNG FOUND AT: mitigatable, on review diligence, which is the authoring change's own honest self-assessment and not a grade assigned here -- the two treatments are two hand-written call sites with no carrier relating them, so this state stays WRITABLE and nothing refuses it. CEILING: STRUCTURALLY IMPOSSIBLE, and it is attainable rather than aspirational because forcing class is DECIDABLE from the artifact's own construction -- the ceiling is not limited by decidability but by a missing carrier. NEXT-RUNG TRIGGER, QUOTED VERBATIM FROM THE AUTHORING CHANGE RATHER THAN RE-COINED, because re-wording an existing well-formed trigger is the nicknaming DESIGN section 3 forbids: a memoized host artifact DECLARES its forcing class and the warm-or-net treatment is DERIVED from that declaration, at which point the third state has no spelling and the bracket cannot be forgotten. It names a CAPABILITY rather than an artifact, which is what section 4b(3) requires. WHY THIS ROW EXISTS AT ALL, GIVEN THAT THE ANALYSIS ABOVE WAS ALREADY AUTHORED: the rung sentence and its trigger were written on 2026-08-27 in gunbc#9477 as a `//` ANNOTATION in gunbc.shared_fill_attribution_seed_growth, and section 4c is explicit that no Accepted program can read an annotation -- the file says so about itself, that nothing refuses this state and the next memo reintroduces it. The TYPED row in that module is a SeedGrowthJustification whose trigger is to DELETE two hand-Rust functions once the memoized compiles are MODELED operations; that is a different subject and would be discharged by a change leaving this state exactly as writable as it is today. The class appeared in NEITHER ledger section 4b names until this row, which is why filing it forks nothing. RELATION TO THE CAPABILITY ROW, CITED ONE-WAY FROM HERE AND DELIBERATELY NOT BACK, AND WITH AN EXPLICIT LIMIT ON WHAT THE CITATION BUYS -- an earlier draft of this sentence claimed the two triggers CANNOT DRIFT because each row named the other, and that was an overclaim of exactly the kind section 4b(1) forbids: `RecurringFailureMode` and `RungDrop` carry NO TYPED RELATION FIELD, so editing either trigger invalidates nothing, refuses nothing and fails no check. Cross-citation makes the relationship REVIEW-VISIBLE and nothing stronger; a typed parent reference would be a different design and is what a mechanical anti-drift claim would require. THE CITATION IS ONE-WAY BECAUSE THE NARROWER ROW OWNS THE RELATIONSHIP and because the parent declares in its own opening that IT NAMES NO CAUSE -- enumerating mechanisms there would start undoing the rename that removed a refuted attribution from its identity. So: gunbc.rung_drop `floor_cost_claim_qualification_unavailable` declares the GENERAL loss -- claim cost is not a stable property of the claim -- and names its closure-level component insensitive to the claim's own assertion work. THIS ROW IS ONE NAMED MECHANISM DEFEATING THAT INVARIANCE, nested inside it rather than disjoint from it, and it is filed separately because it has its own narrower and INDEPENDENTLY SATISFIABLE trigger: buried inside the capability row, discharging this mechanism would read as discharging the capability, which it does not. Satisfying this trigger removes one mechanism and leaves that row standing on its position-sensitive component, whose cause and bound remain unestablished.