diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 7900ff50b53..ada2e1681de 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -53,7 +53,7 @@ on: credential: description: "host_credential_custody_converge only: which rostered custody row to deliver (gunbc.host_credential_custody_converge). A closed choice, never a free-text secret name: each option carries its own SecretRef, path, owner, group, mode and directory, and a row scoped to one host refuses any other" required: false - options: [controller_app_key, approval_ntfy_publisher_token, fabric_state_writer_key, oracle_oci_api_signing_key, claude_code_oauth_harness_token] + options: [controller_app_key, approval_ntfy_publisher_token, fabric_state_writer_key, cursor_worker_turn_api_key, codex_worker_turn_auth, oracle_oci_api_signing_key, claude_code_oauth_harness_token] type: choice d0_consent: description: "pair_serving_d0 only (Cut D, Group A on srv1; expected_revision is required and frozen with the filing): the escalation id the consent is filed under. It names the transaction, and a rerun after a crash MUST name the same id to find its frozen filing and held claim -- a fresh id is a new consent" diff --git a/dag/extdeps/exec/program.dag b/dag/extdeps/exec/program.dag index b3b316d7d6f..df7ab996a97 100644 --- a/dag/extdeps/exec/program.dag +++ b/dag/extdeps/exec/program.dag @@ -185,6 +185,7 @@ fn uncataloged_program(invocation: NonEmptyStr) -> ProgramIdentity decl_ref(module_path: "extdeps.tools.uname", decl_name: "uname_program"), decl_ref(module_path: "extdeps.tools.squashfs_tools", decl_name: "unsquashfs_program"), decl_ref(module_path: "extdeps.tools.util_linux_umount", decl_name: "umount_program"), + decl_ref(module_path: "extdeps.tools.util_linux_flock", decl_name: "flock_program"), decl_ref(module_path: "extdeps.systemd.systemctl", decl_name: "systemctl_path_resolved_program"), decl_ref(module_path: "extdeps.systemd.systemd_run", decl_name: "systemd_run_program"), decl_ref(module_path: "extdeps.ntfy.serve", decl_name: "ntfy_version_command"), diff --git a/dag/extdeps/http/client.dag b/dag/extdeps/http/client.dag index 30d078bd951..227ac12dc3c 100644 --- a/dag/extdeps/http/client.dag +++ b/dag/extdeps/http/client.dag @@ -96,6 +96,13 @@ fn http_client_get_unix_socket_argv(socket: NonEmptyStr, url: NonEmptyStr, max_t // for endpoints that read application/x-www-form-urlencoded (the OAuth token endpoint among // them), one content-type line differs, and the body still rides --data @{file} so a client // secret in the form body is never an argv word (gunbc.credential_argv_exposure). +// PostJsonFromFileWithHeaderFile IS PostJsonFromFile FOR AN ENDPOINT THAT AUTHENTICATES, with the +// credential carried exactly as GetFollowRedirectsBoundedWithHeaderFile carries it: one header line in +// a caller-owned file read by curl -H @file, so the bearer is never an argv word +// (gunbc.credential_argv_exposure) and the body still rides --data @{file}. --fail-with-body keeps an +// error status's body, which is where a hosted API's typed refusal lives. The response headers are dumped +// to a caller-owned file (curl -D) because a refusal's retry hints (Retry-After, X-RateLimit-*) are headers +// and --fail-with-body returns only the body. // PostStdinWithinUnixSocket, THE SAME BOUNDED POST OVER A UNIX SOCKET (curl --unix-socket, curl(1)): // the URL still names the scheme, authority and path the server routes on, but the connection is // made to the socket file and the kernel attests the caller to the server. The exit codes are @@ -309,6 +316,33 @@ service http.Client { } } + operation PostJsonFromFileWithHeaderFile { + requires Network + input { url: NonEmptyStr, request_body_file: NonEmptyStr, header_file: NonEmptyStr, response_headers_file: NonEmptyStr, max_seconds: NonEmptyStr } + output { body: String from "stdout", success: Bool from "exit_success" } + transport shell { + argv: [ + "curl", + "--fail-with-body", + "-sS", + "--max-time", "{max_seconds}", + "-D", "{response_headers_file}", + "-X", "POST", + "{url}", + "-H", "Content-Type: application/json", + "-H", "@{header_file}", + "--data", "@{request_body_file}", + ] + } + exit { + 0 => Unit + nonzero => String "curl POST JSON from file with header file failed" + } + mock_response { + 0 => { body: "", success: false } "hermetic: no live endpoint; a transport that never connects, so the caller answers unreachable rather than fabricating a completion" + } + } + operation PostFormFromFile { input { url: NonEmptyStr, request_body_file: NonEmptyStr, max_seconds: NonEmptyStr } output { body: String from "stdout", success: Bool from "exit_success" } diff --git a/dag/extdeps/llm/codex_auth.dag b/dag/extdeps/llm/codex_auth.dag index b8762766e94..732f8e5b6b6 100644 --- a/dag/extdeps/llm/codex_auth.dag +++ b/dag/extdeps/llm/codex_auth.dag @@ -251,3 +251,33 @@ fn codex_default_organization(token: CodexIdToken) -> CodexOrganizationMembershi fn codex_organization_ids(token: CodexIdToken) -> List { token.openai_auth.organizations |> map(o => o.id) } + +// WHAT THE SOURCE SAYS, BESIDE WHAT ONE HOST SHOWED. codex_observed_refresh_persistence records an +// observation on codex-cli 0.145.0 and stays as observed; this row records a separate fact, read from +// the CLI's source at a pinned revision, so neither one gets rewritten into the other. At +// github.com/openai/codex 7f892275e31002f0422477c6219189284560e689 (2026-10-04), crate codex-rs/login, +// module auth/manager.rs: AuthManager::auth refreshes when should_refresh_proactively holds, meaning +// the access_token JWT exp falls within CHATGPT_ACCESS_TOKEN_REFRESH_WINDOW_MINUTES (5) of now, or +// last_refresh is older than TOKEN_REFRESH_INTERVAL (8 days). refresh_and_persist_chatgpt_token then +// calls persist_tokens, which writes id_token, access_token, the refresh_token the response returned +// and last_refresh back to the auth storage (CODEX_HOME/auth.json under the default +// AuthCredentialsStoreMode::File). So at this revision the file is rewritten about once an hour +// during use, not only at login.\n\nRotation stays a separate fact. The response's refresh_token is +// an Option, and the client maps the issuer's error code refresh_token_reused to +// RefreshTokenFailedReason::Exhausted, so the issuer detects reuse. The source shows the CLIENT is +// ready for rotation; whether the issuer actually rotates on every exchange is server behaviour, and +// codex_observed_refresh_rotation stays CodexRotationUnobserved until it is observed.\n\nThe source +// also has no cross-process lock on the file: the refresh lock is per process, and the guarded +// reload compares account ids, not a lock token. So two processes refreshing one copy can both +// present the same refresh token. +data codex_source_revision: NonEmptyStr = "github.com/openai/codex@7f892275e31002f0422477c6219189284560e689" as NonEmptyStr +data codex_source_refresh_persistence: CodexRefreshPersistence = CodexPersistsOnRefresh +data codex_source_access_token_refresh_window: Second = second(count: 300) + +// THE auth.json KEYS THE HARNESS READS to decide a write-back, as named by codex-rs/login +// token_data.rs TokenData and auth/storage.rs AuthDotJson. They are wire names of the observed file layout above, not new +// vocabulary. +data codex_auth_file_auth_mode_key: NonEmptyStr = "auth_mode" as NonEmptyStr +data codex_auth_file_tokens_key: NonEmptyStr = "tokens" as NonEmptyStr +data codex_auth_file_refresh_token_key: NonEmptyStr = "refresh_token" as NonEmptyStr +data codex_auth_file_account_id_key: NonEmptyStr = "account_id" as NonEmptyStr diff --git a/dag/extdeps/llm/cursor_cli.dag b/dag/extdeps/llm/cursor_cli.dag index 8d1dad9e086..47cbbe602ec 100644 --- a/dag/extdeps/llm/cursor_cli.dag +++ b/dag/extdeps/llm/cursor_cli.dag @@ -108,8 +108,11 @@ data cursor_model_grok_high_fast: CursorModelId = "cursor-grok-4.5-high-fast" as data cursor_observed_model_roster_size: Int = 193 // DELIBERATELY NOT USED: -w/--worktree starts the agent in an isolated git worktree the CLI creates under ~/.cursor/worktrees//. This repository's belt already creates and owns attempt worktrees under the dashboard instance's own attempts root, with the branch name and attempt identity bound to the roadmap node.\n\nAccepting the CLI's worktree would hand that ownership to the provider: the location would move outside the instance root the model treats as owned, the naming would stop deriving from the attempt, and teardown would belong to a directory nobody declared. The flag is recorded so the choice is visible rather than looking like an oversight — a later reader finding two worktree mechanisms should know only one is ours. -type CursorInvocation { - auth: CursorAuth +// THE RUN AND THE CREDENTIAL ARE TWO FACTS. Everything but the key decides the argv; the key decides +// only where the credential travels. Keeping them apart lets a caller whose key is not in hand (a +// spawn that exports it from a custody file in the child, gunbc.roadmap_dispatch_actuator) build the +// same argv this module builds, rather than spelling cursor-agent's flags a second time. +type CursorRunShape { model: CursorModelId mode: CursorExecutionMode sandbox: CursorSandboxChoice @@ -120,6 +123,11 @@ type CursorInvocation { prompt: NonEmptyStr } +type CursorInvocation { + auth: CursorAuth + run: CursorRunShape +} + // THE AUTH FIELD WAS NEVER READ, SO BOTH ARMS PRODUCED THE SAME PROCESS. shape_cursor_agent_argv // built an argv and returned a List of String, and its accompanying note explained at length why // the key is deliberately absent from that argv — which was true, and which meant a @@ -147,20 +155,20 @@ type CursorProcessInvocation { environment: List } -fn cursor_common_argv(inv: CursorInvocation) -> List { +fn cursor_common_argv(run: CursorRunShape) -> List { concat( [cursor_cli_program, "--print"], concat( - ["--output-format", cursor_output_format_wire(f: inv.output_format)], + ["--output-format", cursor_output_format_wire(f: run.output_format)], concat( - ["--model", inv.model as String], + ["--model", run.model as String], concat( - cursor_mode_args(mode: inv.mode), + cursor_mode_args(mode: run.mode), concat( - cursor_sandbox_args(choice: inv.sandbox), + cursor_sandbox_args(choice: run.sandbox), concat( - if inv.force { ["--force"] } else { [] }, - if inv.trust_workspace { ["--trust"] } else { [] }, + if run.force { ["--force"] } else { [] }, + if run.trust_workspace { ["--trust"] } else { [] }, ), ), ), @@ -169,28 +177,34 @@ fn cursor_common_argv(inv: CursorInvocation) -> List { ) } -fn cursor_tail_argv(inv: CursorInvocation) -> List { +fn cursor_tail_argv(run: CursorRunShape) -> List { concat( - ["--workspace", inv.workspace as String], - [inv.prompt as String], + ["--workspace", run.workspace as String], + [run.prompt as String], ) } +// The argv of a run whose key reaches the process through CURSOR_API_KEY set by someone else: the +// environment arm's argv, with nothing about the key in it. +fn cursor_environment_authenticated_argv(run: CursorRunShape) -> List { + concat(cursor_common_argv(run: run), cursor_tail_argv(run: run)) +} + fn shape_cursor_invocation(inv: CursorInvocation) -> CursorProcessInvocation { match inv.auth { CursorApiKeyArgument { key } => CursorProcessInvocation { program: cursor_cli_program, argv: concat( - cursor_common_argv(inv: inv), - concat(["--api-key", key as String], cursor_tail_argv(inv: inv)), + cursor_common_argv(run: inv.run), + concat(["--api-key", key as String], cursor_tail_argv(run: inv.run)), ), environment: [], } CursorApiKeyEnvironment { key } => CursorProcessInvocation { program: cursor_cli_program, - argv: concat(cursor_common_argv(inv: inv), cursor_tail_argv(inv: inv)), + argv: cursor_environment_authenticated_argv(run: inv.run), environment: [ CursorEnvBinding { name: cursor_api_key_env_var, value: key }, ], @@ -212,14 +226,16 @@ fn cursor_automation_invocation( ) -> CursorInvocation { CursorInvocation { auth: CursorApiKeyEnvironment { key: key }, - model: model, - mode: mode, - sandbox: sandbox, - output_format: output_format, - force: force, - trust_workspace: trust_workspace, - workspace: workspace, - prompt: prompt, + run: CursorRunShape { + model: model, + mode: mode, + sandbox: sandbox, + output_format: output_format, + force: force, + trust_workspace: trust_workspace, + workspace: workspace, + prompt: prompt, + }, } } diff --git a/dag/extdeps/nvidia/api_trial_terms.dag b/dag/extdeps/nvidia/api_trial_terms.dag new file mode 100644 index 00000000000..b6a13ddbdd7 --- /dev/null +++ b/dag/extdeps/nvidia/api_trial_terms.dag @@ -0,0 +1,99 @@ +module extdeps.nvidia.api_trial_terms + +import std.types { NonEmptyStr, List } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } + +// THE TERMS THAT GOVERN AN NVIDIA-HOSTED API SERVICE, as their own upstream because they are an +// independently versioned document (DESIGN section 3, external upstream decomposition). They are not +// a property of a model -- the Nemotron 3 Ultra weights are licensed under OpenMDW-1.1, a different +// document with different obligations -- and not of a router that forwards to the service: OpenRouter +// links this PDF as the terms of service of its Nvidia endpoint for the free slug, and the obligations +// below are NVIDIA's whoever relays the request. +// +// READ 2026-10-05 from the PDF itself; the document's own last line is "v. September 19, 2025". Every +// obligation row below cites the section that states it, and the section text was read from that +// version, not transcribed from a summary. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "assets.ngc.nvidia.com/products/api-catalog/legal/NVIDIA%20API%20Trial%20Terms%20of%20Service.pdf", + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.nvidia.api_trial_terms", + decl_name: "nvidia_api_trial_terms_v20250919", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [], +} + +type NvidiaApiTrialTermsVersion20250919 {} + +type NvidiaApiTrialTermsVersion = NvidiaApiTrialTermsVersion20250919 + +// WHAT THE TERMS OBLIGE OF A USER, one arm per obligation a caller's conduct can breach, each named for +// the section that states it. A closed set rather than prose because a consumer deciding whether a use +// is admissible must be able to ask about each obligation separately; a paragraph cannot be asked. +// +// - 1.2 Trial Access Rights: access "for limited trial purposes only and without use of the API +// Service or Generated Content in production"; 1.4 repeats the trial limit for credits. +// - 2.6(a): User Content may not include "any confidential information, controlled or sensitive +// data, including protected health information, personal data"; 4.3 forbids uploading personal, +// financial, health or governmental information. +// - 3.3(iv): NVIDIA collects "User Content and Generated Content to improve NVIDIA products and +// services, including AI models". This is a disclosure rather than a user duty, carried because a +// caller choosing what to send needs it as much as the prohibitions. +// - 4.2: no copying, selling, sublicensing, transferring or distributing the service or Generated +// Content to others. +// - 4.12: no use of the service or Generated Content "to develop or improve products or services +// that compete with the API Service". +type NvidiaApiTrialObligation + = TrialPurposeNoProduction + | NoConfidentialOrPersonalData + | ContentUsedToImproveNvidiaProducts + | NoRedistributionOfServiceOrOutput + | NoCompetingProductDevelopment + +fn nvidia_api_trial_obligation_section(o: NvidiaApiTrialObligation) -> NonEmptyStr { + match o { + TrialPurposeNoProduction => "1.2, 1.4" as NonEmptyStr + NoConfidentialOrPersonalData => "2.6(a), 4.3" as NonEmptyStr + ContentUsedToImproveNvidiaProducts => "3.3(iv)" as NonEmptyStr + NoRedistributionOfServiceOrOutput => "4.2" as NonEmptyStr + NoCompetingProductDevelopment => "4.12" as NonEmptyStr + } +} + +type NvidiaApiTrialTerms { + version: NvidiaApiTrialTermsVersion + version_label: NonEmptyStr + obligations: List + terms: ExternalAuthority +} + +data nvidia_api_trial_terms_v20250919: NvidiaApiTrialTerms = NvidiaApiTrialTerms { + version: NvidiaApiTrialTermsVersion20250919 {}, + version_label: "2025-09-19" as NonEmptyStr, + obligations: [ + TrialPurposeNoProduction, + NoConfidentialOrPersonalData, + ContentUsedToImproveNvidiaProducts, + NoRedistributionOfServiceOrOutput, + NoCompetingProductDevelopment, + ], + terms: extdeps_external_authority_anchor, +} + +fn nvidia_api_trial_terms_wire(t: NvidiaApiTrialTerms) -> NonEmptyStr { + join([ + "NVIDIA API Trial Terms of Service v. ", t.version_label as String, " (", + join(map(t.obligations, o => nvidia_api_trial_obligation_section(o: o) as String), "; "), ")", + ], "") as NonEmptyStr +} diff --git a/dag/extdeps/nvidia/nemotron_3_ultra.dag b/dag/extdeps/nvidia/nemotron_3_ultra.dag new file mode 100644 index 00000000000..c6bc8a62ad5 --- /dev/null +++ b/dag/extdeps/nvidia/nemotron_3_ultra.dag @@ -0,0 +1,46 @@ +module extdeps.nvidia.nemotron_3_ultra + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } + +// NVIDIA NEMOTRON 3 ULTRA, as its own extdeps authority for the reason extdeps.deepseek.deepseek_v4 and +// extdeps.zhipu.glm_5_3_flash are: an independently versioned upstream model is not a property of +// whatever serves it. What a HOSTED endpoint of this model admits -- its window, its output ceiling, the +// reasoning efforts its router accepts -- is the endpoint's fact and lives with the router +// (extdeps.openrouter.openrouter); what the endpoint's operator requires of its users lives with those +// terms (extdeps.nvidia.api_trial_terms). This module carries only what the model card says about the +// model. +// +// READ 2026-10-05 from the model card at the anchor: 550B total / 55B active parameters, a hybrid +// Transformer-Mamba mixture-of-experts, text in and text out, weights under OpenMDW-1.1. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.nvidia.nemotron_3_ultra", + decl_name: "nemotron_ultra_thinking_end_token", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [], +} + +// THE CARD'S REASONING MODES ARE NOT MODELLED HERE, AND THE ABSENCE IS DELIBERATE. The card names three -- +// off and full via chat_template_kwargs enable_thinking, and "Medium-effort reasoning" via medium_effort, +// which "uses significantly fewer reasoning tokens than full thinking mode" -- but those keys address a +// chat template, and the only route this repository has to the model is a router that takes its own +// reasoning parameter and translates it unseen. Nothing here can send them, so a type for them would be +// a declaration with no consumer. A self-hosted Nemotron unit is the consumer that would add it. + +// THE END-OF-THINKING CLOSER. The card's budget client splits a completion on "" and appends it +// when a capped trace never produced one, so it is the model's own delimiter. +data nemotron_ultra_thinking_end_token: NonEmptyStr = "" as NonEmptyStr diff --git a/dag/extdeps/openrouter/nvidia_nemotron_3_ultra_free.dag b/dag/extdeps/openrouter/nvidia_nemotron_3_ultra_free.dag new file mode 100644 index 00000000000..17bec64cc62 --- /dev/null +++ b/dag/extdeps/openrouter/nvidia_nemotron_3_ultra_free.dag @@ -0,0 +1,77 @@ +module extdeps.openrouter.nvidia_nemotron_3_ultra_free + +import std.types { NonEmptyStr, Bool, List } +import std.measure { TokenCount, token_count, MoneyAmountMicro, money_amount_micro } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { + ExternalAuthority, ExternalModelScope, ExternalSubjectRef, +} +import extdeps.uri { Uri, Https } +import extdeps.openrouter.openrouter { OpenRouterReasoningEffort, EffortHigh, EffortMedium } +import extdeps.nvidia.api_trial_terms { NvidiaApiTrialTerms, nvidia_api_trial_terms_v20250919 } + +// OPENROUTER'S LISTING OF NEMOTRON 3 ULTRA'S FREE VARIANT. What a listing admits is OpenRouter's fact +// about its endpoint, not NVIDIA's about the model (extdeps.nvidia.nemotron_3_ultra), so it is a row in +// OpenRouter's module family; a second listing is a second file, never an edit to this one. +// +// READ 2026-10-05 from the endpoints read at the anchor and from GET /api/v1/models: one endpoint, +// provider Nvidia, upstream snapshot nvidia/nemotron-3-ultra-550b-a55b-20260604:free, context_length +// 1000000, max_completion_tokens 65536, pricing prompt 0 / completion 0, supported_parameters reasoning, +// include_reasoning, temperature, max_tokens, seed, top_p, tools, tool_choice, reasoning_effort; and on +// the models listing, reasoning {mandatory: false, default_enabled: true, supported_efforts: ["high", +// "medium"], default_effort: "high"}. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "openrouter.ai/api/v1/models/nvidia/nemotron-3-ultra-550b-a55b:free/endpoints", + } +} + +data openrouter_models_listing_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "openrouter.ai/api/v1/models", + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.openrouter.nvidia_nemotron_3_ultra_free", + decl_name: "nemotron_ultra_free_model_id", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [openrouter_models_listing_authority], +} + +data nemotron_ultra_free_model_id: NonEmptyStr = "nvidia/nemotron-3-ultra-550b-a55b:free" as NonEmptyStr + +// THE ENDPOINT'S WINDOW AND OUTPUT CEILING. Both are the endpoint's admission, which is why they are here +// and not on the model: the paid listing of the same model publishes 262144 and 16384. +data nemotron_ultra_free_context_length: TokenCount = token_count(count: 1000000) + +data nemotron_ultra_free_max_completion_tokens: TokenCount = token_count(count: 65536) + +// THE EFFORTS THIS LISTING ACCEPTS, transcribed from supported_efforts in the listing's own order +// (highest first), and the two facts beside them a request needs: reasoning is not mandatory, so +// `reasoning.enabled: false` is a request the listing admits, and the default effort is high. +data nemotron_ultra_free_supported_efforts: List = [EffortHigh, EffortMedium] + +data nemotron_ultra_free_reasoning_mandatory: Bool = false + +// THE PRICE IS ZERO PER TOKEN (prompt and completion both read "0" USD) AND THE LIMIT IS PER REQUEST, which is what makes reasoning depth cost +// neither money nor quota on this listing: a deeper effort spends latency and completion budget only. +data nemotron_ultra_free_price_per_token: MoneyAmountMicro = money_amount_micro(count: 0) + +// WHICH MODEL MODE EACH ACCEPTED EFFORT REACHES IS NOT STATED, AND NOTHING HERE ASSERTS IT. The two +// efforts coincide with the card's two thinking modes (full and medium), and "high reaches full" is the +// obvious reading -- but neither OpenRouter nor NVIDIA publishes the translation, so it is a bet (DESIGN +// section 4d) and no row consumes it. What IS published is the ORDER: supported_efforts is "returned in +// descending effort order (highest first)" (the reasoning-tokens guide), so a consumer ranking depth reads +// the position in this list, which is cited, rather than a mode, which is not. + +// THE TERMS THE ENDPOINT IS GOVERNED BY. OpenRouter links NVIDIA's API Trial Terms as this provider's +// terms of service; the obligations are NVIDIA's (extdeps.nvidia.api_trial_terms). +data nemotron_ultra_free_terms: NvidiaApiTrialTerms = nvidia_api_trial_terms_v20250919 diff --git a/dag/extdeps/openrouter/openrouter.dag b/dag/extdeps/openrouter/openrouter.dag new file mode 100644 index 00000000000..451aef705f4 --- /dev/null +++ b/dag/extdeps/openrouter/openrouter.dag @@ -0,0 +1,629 @@ +module extdeps.openrouter.openrouter + +import std.types { String, Bool, Int, List, NonEmptyStr } +import std.algebra { trim } +import std.nat { Nat, nat_range_inclusive } +import std.checked_arithmetic { checked_int_to_nat } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import extdeps.api_rate_limit { UpstreamRateLimit, PerMinute, PerDay } +import extdeps.languages.json.emit { JsonValue, JsonObject, JsonNumber, JsonNull, JsonBool, JsonString, JsonArray } +import extdeps.languages.json.parse { + JsonDocumentParsed, JsonDocumentUnreadable, parse_json_document, json_document_gap_text, + JsonFieldRead, FieldAbsent, FieldRead, FieldMalformed, json_field, json_field_string, json_field_int, json_field_bool, +} + +// OPENROUTER, as its own extdeps authority: an independently governed router that relays an +// OpenAI-compatible chat-completions request to one of several providers. It is not OpenAI (its +// errors, limits, reasoning parameter and key surface are its own), so it does not live in +// extdeps.llm.openai_rest, and it is not any provider it relays to. A model OpenRouter lists is a row in +// its own module beside this one (extdeps.openrouter.nvidia_nemotron_3_ultra_free), so a second listing +// is a new file rather than an edit here. +// +// READ 2026-10-05: the API reference overview (the anchor), the limits page and the errors page. The +// facts below cite which. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "openrouter.ai/docs/api-reference/overview", + } +} + +data openrouter_limits_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "openrouter.ai/docs/api-reference/limits", + } +} + +data openrouter_reasoning_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "openrouter.ai/docs/guides/best-practices/reasoning-tokens", + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.openrouter.openrouter", + decl_name: "openrouter_chat_completions_url", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [openrouter_limits_authority, openrouter_reasoning_authority], +} + +// THE TWO ENDPOINTS THIS REPOSITORY CALLS. Chat completions is OpenAI-compatible in shape; the key read +// answers what the presented key may still spend. Both authenticate with `Authorization: Bearer `, +// and both are HTTPS -- there is no plain-http door. +data openrouter_chat_completions_url: NonEmptyStr = "https://openrouter.ai/api/v1/chat/completions" as NonEmptyStr + +data openrouter_key_url: NonEmptyStr = "https://openrouter.ai/api/v1/key" as NonEmptyStr + +fn openrouter_authorization_header_line(key: NonEmptyStr) -> String { + join(["Authorization: Bearer ", key as String, "\n"], "") +} + +// ── THE FREE-MODEL LIMITS, FROM THE LIMITS PAGE ─────────────────────────────────────────────── +// +// Free variants (model ids ending in ":free") are limited to 20 requests per minute, and per day to 50 +// when the account has purchased fewer than 10 credits all-time or 1000 when it has purchased at least +// 10. Both limits bind at once, so a caller leases BOTH pools before each request. +// +// THE DAILY TIER IS SELECTED BY A TYPED FACT, NOT CHOSEN AS A LITERAL. Which row applies is a property +// of the account, and the account says which: the key read below returns free_model_daily_requests.limit, +// so the fact is DERIVED from what OpenRouter reports for the presented key rather than asserted beside +// it. Both tiers share one scope and therefore one quota partition, because they are one pool on the +// upstream whose ceiling moves with the account -- two scopes would let a caller spend both. +type OpenRouterCreditsPurchased + = CreditsPurchasedBelowTen + | CreditsPurchasedAtLeastTen + +data openrouter_free_models_per_minute: UpstreamRateLimit = UpstreamRateLimit { + upstream: "openrouter" as NonEmptyStr, + scope: "free-models-minute" as NonEmptyStr, + requests: 20, + period: PerMinute, +} + +fn openrouter_free_models_per_day(credits: OpenRouterCreditsPurchased) -> UpstreamRateLimit { + UpstreamRateLimit { + upstream: "openrouter" as NonEmptyStr, + scope: "free-models-day" as NonEmptyStr, + requests: match credits { + CreditsPurchasedBelowTen => 50 + CreditsPurchasedAtLeastTen => 1000 + }, + period: PerDay, + } +} + +// THE TIER IS READ OFF THE DAILY LIMIT THE KEY REPORTS, and a limit that is neither published tier is a +// refusal rather than the nearer one: a third tier is a change to the limits page this module has not +// read, and guessing it would lease against a ceiling nobody published. +type OpenRouterCreditsReading + = CreditsTierObserved { credits: OpenRouterCreditsPurchased } + | CreditsTierUnrecognized { daily_limit: Int } + +fn openrouter_credits_from_daily_limit(daily_limit: Int) -> OpenRouterCreditsReading { + if daily_limit == 1000 { + CreditsTierObserved { credits: CreditsPurchasedAtLeastTen } + } else if daily_limit == 50 { + CreditsTierObserved { credits: CreditsPurchasedBelowTen } + } else { + CreditsTierUnrecognized { daily_limit: daily_limit } + } +} + +// ── THE KEY READ ───────────────────────────────────────────────────────────────────────────── +// +// GET /api/v1/key answers data.{limit, limit_remaining, usage, is_free_tier, free_model_daily_requests +// {used, limit, remaining}}. Only the members a consumer reads are decoded: whether the key is accepted +// at all (a 200 that decodes), the key's own credit limit, and the free-model daily counters the tier +// is derived from. +// +// THE KEY'S CREDIT LIMIT IS A DIFFERENT FACT FROM THE ACCOUNT'S TIER. data.limit is a per-KEY spending +// cap set on the key (null when the key has none); the daily free-model ceiling is per ACCOUNT, set by +// credits purchased. A key capped at zero credits can spend nothing on a priced model and is still +// metered by the account's free tier, which is exactly the guard a free-only caller wants: a request +// that reaches a priced model answers 402 from openrouter_key_limit instead of charging anything. +// +// ZERO IS READ OFF THE LEXEME, NOT A FLOAT. The wire carries a JSON number ("0", "0.0"); a lexeme whose +// digits are all zero is zero in every notation JSON admits for it, and anything else is a positive cap +// a free-only caller must refuse. +type OpenRouterKeySpendLimit + = KeySpendUncapped + | KeySpendCappedAtZero + | KeySpendCappedAbove { lexeme: String } + +fn openrouter_lexeme_is_zero(lexeme: String) -> Bool { + lexeme != "" && trim_zero_digits(s: lexeme) == "" +} + +fn trim_zero_digits(s: String) -> String { + replace(s: replace(s: replace(s: replace(s: replace(s: replace(s: s, from: "0", to: ""), from: ".", to: ""), from: "-", to: ""), from: "+", to: ""), from: "e", to: ""), from: "E", to: "") +} + +fn openrouter_decode_key_spend_limit(key_data: JsonValue) -> OpenRouterKeySpendLimit? { + match json_field(obj: key_data, key: "limit") { + FieldAbsent => Present { value: KeySpendUncapped } + FieldMalformed { cause: _ } => none + FieldRead { value: v } => + match v { + JsonNumber { lexeme: lx } => + if openrouter_lexeme_is_zero(lexeme: lx) { Present { value: KeySpendCappedAtZero } } else { Present { value: KeySpendCappedAbove { lexeme: lx } } } + JsonNull => Present { value: KeySpendUncapped } + JsonBool { value: _ } => none + JsonString { value: _ } => none + JsonArray { elements: _ } => none + JsonObject { members: _ } => none + } + } +} + +fn openrouter_key_spend_limit_wire(l: OpenRouterKeySpendLimit) -> String { + match l { + KeySpendUncapped => "no key credit limit" + KeySpendCappedAtZero => "key credit limit 0" + KeySpendCappedAbove { lexeme: lx } => join(["key credit limit ", lx], "") + } +} + +type OpenRouterKeyRead { + is_free_tier: Bool + spend_limit: OpenRouterKeySpendLimit + free_daily_used: Int + free_daily_limit: Int + free_daily_remaining: Int +} + +type OpenRouterKeyDecode + = OpenRouterKeyDecoded { key: OpenRouterKeyRead } + | OpenRouterKeyUndecodable { cause: String } + +fn openrouter_int_member(obj: JsonValue, key: String, subject: String) -> JsonFieldRead { + match json_field_int(obj: obj, key: key) { + FieldAbsent => FieldMalformed { cause: join([subject, " carries no ", key], "") } + FieldMalformed { cause: c } => FieldMalformed { cause: c } + FieldRead { value: n } => FieldRead { value: n } + } +} + +fn openrouter_decode_key_daily(key_data: JsonValue, is_free_tier: Bool, spend_limit: OpenRouterKeySpendLimit) -> OpenRouterKeyDecode { + match json_field(obj: key_data, key: "free_model_daily_requests") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "the key read carries no free_model_daily_requests" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: daily } => + match openrouter_int_member(obj: daily, key: "used", subject: "free_model_daily_requests") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "free_model_daily_requests carries no used" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: used } => + match openrouter_int_member(obj: daily, key: "limit", subject: "free_model_daily_requests") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "free_model_daily_requests carries no limit" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: limit } => + match openrouter_int_member(obj: daily, key: "remaining", subject: "free_model_daily_requests") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "free_model_daily_requests carries no remaining" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: remaining } => + OpenRouterKeyDecoded { key: OpenRouterKeyRead { is_free_tier: is_free_tier, spend_limit: spend_limit, free_daily_used: used, free_daily_limit: limit, free_daily_remaining: remaining } } + } + } + } + } +} + +fn openrouter_decode_key_read(body: String) -> OpenRouterKeyDecode { + match parse_json_document(s: body) { + JsonDocumentUnreadable { gap: g } => OpenRouterKeyUndecodable { cause: join(["the key read is not a JSON document: ", json_document_gap_text(gap: g)], "") } + JsonDocumentParsed { value: doc } => + match json_field(obj: doc, key: "data") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "the key read carries no data member" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: key_data } => + match json_field_bool(obj: key_data, key: "is_free_tier") { + FieldAbsent => OpenRouterKeyUndecodable { cause: "the key read carries no is_free_tier" } + FieldMalformed { cause: c } => OpenRouterKeyUndecodable { cause: c } + FieldRead { value: free } => + match openrouter_decode_key_spend_limit(key_data: key_data) { + Absent => OpenRouterKeyUndecodable { cause: "the key read's limit member is not a number or null" } + Present { value: spend } => openrouter_decode_key_daily(key_data: key_data, is_free_tier: free, spend_limit: spend) + } + } + } + } +} + +// ── THE ERROR BODY, FROM THE ERRORS AND LIMITS PAGES ────────────────────────────────────────── +// +// Every error is {error: {code, message, metadata?}}. Two codes carry metadata a caller must branch on: +// +// - 402: metadata.limit_source names WHICH budget refused -- openrouter_in_flight_budget, +// openrouter_key_limit or openrouter_credits -- and metadata.reason refines the in-flight case as +// in_flight_budget_exhausted or weight_exceeds_budget. metadata.remedy_hint is documented as +// log-only, so it is carried as text and never branched on. Free models are exempt from the +// in-flight budget, but a negative balance still answers 402 from openrouter_credits. +// - 429: metadata.error_type is rate_limit_exceeded, with provider_code when the limit was the +// provider's rather than OpenRouter's own. +// +// THE ENUMS ARE CLOSED AND AN UNKNOWN SPELLING REFUSES. A fourth limit_source decoded as the nearest of +// three would send a caller to repair the wrong budget, which is the whole reason the field exists. +type OpenRouterLimitSource + = InFlightBudgetLimit + | KeySpendLimit + | CreditsLimit + +fn openrouter_limit_source_wire(s: OpenRouterLimitSource) -> NonEmptyStr { + match s { + InFlightBudgetLimit => "openrouter_in_flight_budget" as NonEmptyStr + KeySpendLimit => "openrouter_key_limit" as NonEmptyStr + CreditsLimit => "openrouter_credits" as NonEmptyStr + } +} + +type OpenRouterLimitReason + = InFlightBudgetExhausted + | WeightExceedsBudget + +fn openrouter_limit_reason_wire(r: OpenRouterLimitReason) -> NonEmptyStr { + match r { + InFlightBudgetExhausted => "in_flight_budget_exhausted" as NonEmptyStr + WeightExceedsBudget => "weight_exceeds_budget" as NonEmptyStr + } +} + +type OpenRouterError + = OpenRouterPaymentRequired { limit_source: OpenRouterLimitSource, reason: OpenRouterLimitReason?, message: String, remedy_hint: String? } + | OpenRouterRateLimited { message: String, provider_code: String? } + | OpenRouterOtherError { code: Int, message: String } + +type OpenRouterErrorDecode + = OpenRouterErrorDecoded { error: OpenRouterError } + | OpenRouterErrorUndecodable { cause: String } + +type OpenRouterLimitSourceRead + = LimitSourceRead { source: OpenRouterLimitSource } + | LimitSourceUnrecognized { wire: String } + +fn openrouter_limit_source_from_wire(wire: String) -> OpenRouterLimitSourceRead { + if wire == "openrouter_in_flight_budget" { + LimitSourceRead { source: InFlightBudgetLimit } + } else if wire == "openrouter_key_limit" { + LimitSourceRead { source: KeySpendLimit } + } else if wire == "openrouter_credits" { + LimitSourceRead { source: CreditsLimit } + } else { + LimitSourceUnrecognized { wire: wire } + } +} + +type OpenRouterLimitReasonRead + = LimitReasonAbsent + | LimitReasonRead { reason: OpenRouterLimitReason } + | LimitReasonUnrecognized { wire: String } + +fn openrouter_limit_reason_from_field(field: JsonFieldRead) -> OpenRouterLimitReasonRead { + match field { + FieldAbsent => LimitReasonAbsent + FieldMalformed { cause: c } => LimitReasonUnrecognized { wire: c } + FieldRead { value: w } => + if w == "in_flight_budget_exhausted" { + LimitReasonRead { reason: InFlightBudgetExhausted } + } else if w == "weight_exceeds_budget" { + LimitReasonRead { reason: WeightExceedsBudget } + } else { + LimitReasonUnrecognized { wire: w } + } + } +} + +fn openrouter_optional_string(field: JsonFieldRead) -> String? { + match field { + FieldRead { value: s } => Present { value: s } + FieldAbsent => none + FieldMalformed { cause: _ } => none + } +} + +fn openrouter_decode_402(message: String, metadata: JsonValue) -> OpenRouterErrorDecode { + match json_field_string(obj: metadata, key: "limit_source") { + FieldAbsent => OpenRouterErrorUndecodable { cause: "a 402 whose metadata carries no limit_source names no budget to repair" } + FieldMalformed { cause: c } => OpenRouterErrorUndecodable { cause: join(["the 402 limit_source is unreadable: ", c], "") } + FieldRead { value: source_wire } => + match openrouter_limit_source_from_wire(wire: source_wire) { + LimitSourceUnrecognized { wire: w } => OpenRouterErrorUndecodable { cause: join(["the 402 names an unpublished limit_source '", w, "'"], "") } + LimitSourceRead { source: source } => + match openrouter_limit_reason_from_field(field: json_field_string(obj: metadata, key: "reason")) { + LimitReasonUnrecognized { wire: w } => OpenRouterErrorUndecodable { cause: join(["the 402 names an unpublished reason '", w, "'"], "") } + LimitReasonAbsent => + OpenRouterErrorDecoded { error: OpenRouterPaymentRequired { limit_source: source, reason: none, message: message, remedy_hint: openrouter_optional_string(field: json_field_string(obj: metadata, key: "remedy_hint")) } } + LimitReasonRead { reason: r } => + OpenRouterErrorDecoded { error: OpenRouterPaymentRequired { limit_source: source, reason: Present { value: r }, message: message, remedy_hint: openrouter_optional_string(field: json_field_string(obj: metadata, key: "remedy_hint")) } } + } + } + } +} + +fn openrouter_decode_error_fields(error: JsonValue) -> OpenRouterErrorDecode { + match json_field_int(obj: error, key: "code") { + FieldAbsent => OpenRouterErrorUndecodable { cause: "the error object carries no code" } + FieldMalformed { cause: c } => OpenRouterErrorUndecodable { cause: c } + FieldRead { value: code } => { + let message = match json_field_string(obj: error, key: "message") { + FieldRead { value: m } => m + FieldAbsent => "" + FieldMalformed { cause: c } => c + } + let metadata = match json_field(obj: error, key: "metadata") { + FieldRead { value: m } => m + FieldAbsent => JsonObject { members: [] } + FieldMalformed { cause: _ } => JsonObject { members: [] } + } + if code == 402 { + openrouter_decode_402(message: message, metadata: metadata) + } else if code == 429 { + OpenRouterErrorDecoded { error: OpenRouterRateLimited { message: message, provider_code: openrouter_optional_string(field: json_field_string(obj: metadata, key: "provider_code")) } } + } else { + OpenRouterErrorDecoded { error: OpenRouterOtherError { code: code, message: message } } + } + } + } +} + +fn openrouter_decode_error_body(body: String) -> OpenRouterErrorDecode { + match parse_json_document(s: body) { + JsonDocumentUnreadable { gap: g } => OpenRouterErrorUndecodable { cause: join(["the error body is not a JSON document: ", json_document_gap_text(gap: g)], "") } + JsonDocumentParsed { value: doc } => + match json_field(obj: doc, key: "error") { + FieldAbsent => OpenRouterErrorUndecodable { cause: "the body carries no error member" } + FieldMalformed { cause: c } => OpenRouterErrorUndecodable { cause: c } + FieldRead { value: e } => openrouter_decode_error_fields(error: e) + } + } +} + +// WHAT A 402'S SOURCE MEANS FOR A FREE-ONLY KEY. openrouter_key_limit on a key capped at zero credits +// is not a budget to top up: it says the request reached a PRICED model, which a free-only caller must +// refuse and never retry -- retrying sends the same paid request again. The other two sources keep +// their published meaning. +fn openrouter_limit_source_meaning(s: OpenRouterLimitSource) -> String { + match s { + KeySpendLimit => "a priced model was requested through a key whose credit limit forbids spending (never retried); " + InFlightBudgetLimit => "" + CreditsLimit => "" + } +} + +// ── A 429 IS A TYPED REFUSAL THAT CARRIES THE UPSTREAM'S OWN RETRY HINTS ────────────────────────── +// +// The limits page says an error OpenRouter itself returns for a platform limit carries X-RateLimit-Limit, +// X-RateLimit-Remaining and X-RateLimit-Reset, and that Retry-After is added when every attempted provider +// returned a retry hint. The headers are the upstream's word about when it will answer again, so they ride +// the refusal instead of being dropped with the transport. +// +// WHAT IS NOT PUBLISHED IS NOT ASSERTED (DESIGN section 4d). The page gives neither the unit of +// X-RateLimit-Reset nor says Retry-After is seconds rather than an HTTP date (RFC 9110 allows both). So +// Retry-After is read as delta-seconds only when it is all digits and otherwise carried as the text it +// was; and the reset value is carried VERBATIM, never converted to an instant. A consumer that waits +// computes its wait from Retry-After alone; the reset is evidence for the operator, with a live reading +// the thing that would ground its unit. +type OpenRouterRetryAfter + = RetryAfterSeconds { seconds: Nat } + | RetryAfterUnparsed { text: String } + | RetryAfterAbsent + | RetryAfterUnread { cause: String } + +// THE HEADER DUMP IS READ OR IT IS NOT, AND THE TWO ARE DIFFERENT FACTS. A dump that was read and carries no +// Retry-After says the upstream gave no hint; a dump that could not be read says nothing about whether it did. +// Reading a failed read as "no hint" would invent an absence the caller then acts on, so the failure is carried. +type OpenRouterHeaderDump + = HeaderDumpRead { text: String } + | HeaderDumpUnread { cause: String } + +type OpenRouterRateLimitHints { + retry_after: OpenRouterRetryAfter + reset: String? + limit: String? + remaining: String? +} + +// A header dump is the status line and headers of each response curl saw, in order; only the LAST block +// describes the answer the caller got, so a status line starts a fresh reading. +type HeaderScan { + retry_after: String? + reset: String? + limit: String? + remaining: String? +} + +// HTTP FIELD NAMES ARE CASE-INSENSITIVE (RFC 9110 section 5.1): HTTP/2 sends them lowercase, HTTP/1.1 may +// not, and a dump is read the same either way. Folded over ASCII letters only. +data openrouter_ascii_upper: String = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +data openrouter_ascii_lower: String = "abcdefghijklmnopqrstuvwxyz" + +fn openrouter_ascii_fold(s: String) -> String { + fold(nat_range_inclusive(lo: 0, hi: 25), init: s, f: (acc, i) => + replace(acc, substring(s: openrouter_ascii_upper, start: i, end: i + 1), substring(s: openrouter_ascii_lower, start: i, end: i + 1))) +} + +fn openrouter_header_scan_line(acc: HeaderScan, line: String) -> HeaderScan { + let t = trim(s: line) + if openrouter_ascii_fold(s: t).starts_with("http/") { + HeaderScan { retry_after: none, reset: none, limit: none, remaining: none } + } else { + match split(s: t, delimiter: ":").first() { + Absent => acc + Present { value: raw_name } => { + let name = openrouter_ascii_fold(s: trim(s: raw_name)) + let value = if length(t) > length(raw_name) { trim(s: substring(s: t, start: length(raw_name) + 1, end: length(t))) } else { "" } + if length(t) <= length(raw_name) { + acc + } else if name == "retry-after" { + HeaderScan { retry_after: Present { value: value }, reset: acc.reset, limit: acc.limit, remaining: acc.remaining } + } else if name == "x-ratelimit-reset" { + HeaderScan { retry_after: acc.retry_after, reset: Present { value: value }, limit: acc.limit, remaining: acc.remaining } + } else if name == "x-ratelimit-limit" { + HeaderScan { retry_after: acc.retry_after, reset: acc.reset, limit: Present { value: value }, remaining: acc.remaining } + } else if name == "x-ratelimit-remaining" { + HeaderScan { retry_after: acc.retry_after, reset: acc.reset, limit: acc.limit, remaining: Present { value: value } } + } else { + acc + } + } + } + } +} + +// HTTP delay-seconds IS 1*DIGIT (RFC 9110 section 10.2.3): no sign, no fraction, no unit. The integer parser +// below accepts a leading '+' or '-' (so '+30' and '-0' would read as numbers), so the lexical shape is +// checked first and a value that is not all ASCII digits is carried as the text it was. +fn openrouter_is_ascii_digits(text: String) -> Bool { + length(text) > 0 && length(fold(nat_range_inclusive(lo: 0, hi: 9), init: text, f: (acc, i) => replace(acc, to_string(i), ""))) == 0 +} + +fn openrouter_retry_after_from_text(text: String) -> OpenRouterRetryAfter { + if !openrouter_is_ascii_digits(text: text) { + RetryAfterUnparsed { text: text } + } else { + openrouter_retry_after_from_digits(text: text) + } +} + +fn openrouter_retry_after_from_digits(text: String) -> OpenRouterRetryAfter { + match parse_int(s: text) { + Absent => RetryAfterUnparsed { text: text } + Present { value: n } => + match checked_int_to_nat(n: n) { + Absent => RetryAfterUnparsed { text: text } + Present { value: sec } => RetryAfterSeconds { seconds: sec } + } + } +} + +fn openrouter_rate_limit_hints(dump: OpenRouterHeaderDump) -> OpenRouterRateLimitHints { + match dump { + HeaderDumpUnread { cause: c } => OpenRouterRateLimitHints { retry_after: RetryAfterUnread { cause: c }, reset: none, limit: none, remaining: none } + HeaderDumpRead { text: t } => openrouter_rate_limit_hints_of_text(dump: t) + } +} + +fn openrouter_rate_limit_hints_of_text(dump: String) -> OpenRouterRateLimitHints { + let scan = fold(split(s: dump, delimiter: "\n"), init: HeaderScan { retry_after: none, reset: none, limit: none, remaining: none }, f: (acc, line) => + openrouter_header_scan_line(acc: acc, line: line)) + OpenRouterRateLimitHints { + retry_after: match scan.retry_after { Absent => RetryAfterAbsent Present { value: t } => openrouter_retry_after_from_text(text: t) }, + reset: scan.reset, + limit: scan.limit, + remaining: scan.remaining, + } +} + +// THE REFUSAL ITSELF: an OpenRouter 429 with its hints. It is never retried by the code that decodes it; +// carrying the hint is how a caller that CHOOSES to wait learns how long. +type OpenRouterRateLimitRefusal { + message: String + provider_code: String? + hints: OpenRouterRateLimitHints +} + +fn openrouter_decode_rate_limited(body: String, header_dump: OpenRouterHeaderDump) -> OpenRouterRateLimitRefusal? { + match openrouter_decode_error_body(body: body) { + OpenRouterErrorUndecodable { cause: _ } => none + OpenRouterErrorDecoded { error: e } => + match e { + OpenRouterRateLimited { message: m, provider_code: p } => + Present { value: OpenRouterRateLimitRefusal { message: m, provider_code: p, hints: openrouter_rate_limit_hints(dump: header_dump) } } + OpenRouterPaymentRequired { limit_source: _, reason: _, message: _, remedy_hint: _ } => none + OpenRouterOtherError { code: _, message: _ } => none + } + } +} + +fn openrouter_retry_after_wire(r: OpenRouterRetryAfter) -> String { + match r { + RetryAfterSeconds { seconds: n } => join(["Retry-After ", to_string(n), " s"], "") + RetryAfterUnparsed { text: t } => join(["Retry-After '", t, "' (not delta-seconds; not interpreted)"], "") + RetryAfterAbsent => "no Retry-After" + RetryAfterUnread { cause: c } => join(["Retry-After unknown: the response headers could not be read (", c, ")"], "") + } +} + +fn openrouter_rate_limit_wire(r: OpenRouterRateLimitRefusal) -> String { + join([ + "OpenRouter answered 429 rate_limit_exceeded", + match r.provider_code { Present { value: pc } => join([" from provider ", pc], "") Absent => " from OpenRouter" }, + ": ", r.message, "; ", openrouter_retry_after_wire(r: r.hints.retry_after), + match r.hints.reset { Present { value: v } => join(["; X-RateLimit-Reset=", v, " (unit not documented; carried verbatim)"], "") Absent => "" }, + match r.hints.remaining { Present { value: v } => join(["; X-RateLimit-Remaining=", v], "") Absent => "" }, + "; the request was NOT retried", + ], "") +} + +fn openrouter_error_wire(e: OpenRouterError) -> String { + match e { + OpenRouterPaymentRequired { limit_source: s, reason: r, message: m, remedy_hint: _ } => + join([ + openrouter_limit_source_meaning(s: s), + "402 from ", openrouter_limit_source_wire(s: s) as String, + match r { Present { value: rv } => join([" (", openrouter_limit_reason_wire(r: rv) as String, ")"], "") Absent => "" }, + ": ", m, + ], "") + OpenRouterRateLimited { message: m, provider_code: p } => + join(["429 rate_limit_exceeded", match p { Present { value: pc } => join([" from provider ", pc], "") Absent => " from OpenRouter" }, ": ", m], "") + OpenRouterOtherError { code: c, message: m } => join([to_string(c), ": ", m], "") + } +} + +// ── THE REASONING PARAMETER, FROM THE REASONING-TOKENS GUIDE ───────────────────────────────── +// +// reasoning.effort is one of max, xhigh, high, medium, low, minimal or none; reasoning.enabled false +// turns reasoning off where the model does not make it mandatory. Which efforts a given model honours +// is published per model in GET /api/v1/models as reasoning.supported_efforts -- so the vocabulary is +// OpenRouter's and the admitted subset is each listing's row, never this module's guess. +type OpenRouterReasoningEffort + = EffortMax + | EffortXhigh + | EffortHigh + | EffortMedium + | EffortLow + | EffortMinimal + | EffortNone + +fn openrouter_reasoning_effort_wire(e: OpenRouterReasoningEffort) -> NonEmptyStr { + match e { + EffortMax => "max" as NonEmptyStr + EffortXhigh => "xhigh" as NonEmptyStr + EffortHigh => "high" as NonEmptyStr + EffortMedium => "medium" as NonEmptyStr + EffortLow => "low" as NonEmptyStr + EffortMinimal => "minimal" as NonEmptyStr + EffortNone => "none" as NonEmptyStr + } +} + +fn openrouter_effort_eq(a: OpenRouterReasoningEffort, b: OpenRouterReasoningEffort) -> Bool { + (openrouter_reasoning_effort_wire(e: a) as String) == (openrouter_reasoning_effort_wire(e: b) as String) +} + +// AN EFFORT OUTSIDE THE LISTING'S SUPPORTED SET IS REFUSED, NOT SENT. OpenRouter's own guide shows what +// an unsupported effort becomes on another provider -- Gemini maps xhigh down to high -- so sending it +// is not an error the wire reports, it is a different setting the caller never sees. +type OpenRouterEffortAdmission + = EffortAdmitted { effort: OpenRouterReasoningEffort } + | EffortNotSupported { effort: OpenRouterReasoningEffort, supported: List } + +fn openrouter_effort_admitted(effort: OpenRouterReasoningEffort, supported: List) -> OpenRouterEffortAdmission { + if count(filter(supported, s => openrouter_effort_eq(a: s, b: effort))) > 0 { + EffortAdmitted { effort: effort } + } else { + EffortNotSupported { effort: effort, supported: supported } + } +} + +// THE MESSAGE MEMBER REASONING RIDES ON. The guide: "Reasoning tokens will appear in the reasoning field +// of each message", and the same field is how prior reasoning is passed back. +data openrouter_message_reasoning_field: NonEmptyStr = "reasoning" as NonEmptyStr diff --git a/dag/extdeps/tools/util_linux_flock.dag b/dag/extdeps/tools/util_linux_flock.dag new file mode 100644 index 00000000000..3b9aa119195 --- /dev/null +++ b/dag/extdeps/tools/util_linux_flock.dag @@ -0,0 +1,32 @@ +module extdeps.tools.util_linux_flock + +import extdeps.exec.program { ProgramIdentity, uncataloged_program } + +import std.types { NonEmptyStr, String, Int, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +// flock(1), util-linux: "flock [options] | [...]" -- take a lock +// on the file, run the command, and release the lock when the command exits. Read from the +// installed util-linux 2.41.5 --help and man7.org on 2026-10-05. The lock is held by the process for +// exactly the command's lifetime and the kernel drops it when the process dies, so a crashed command +// leaves no lock to recover. Only the exclusive, non-blocking form is modeled: -n/--nonblock ("fail +// rather than wait") with -E/--conflict-exit-code naming the exit status a conflict reports, so a +// caller can tell "someone else holds it" apart from the command's own failures. -F/--no-fork +// ("execute command without forking") execs the command in flock's place with the lock fd inherited, +// so the process a terminal shows in the foreground is the command, not flock. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "man7.org/linux/man-pages/man1/flock.1.html" + } +} + +fn flock_program() -> ProgramIdentity = uncataloged_program(invocation: "flock") + +fn flock_exclusive_nonblocking_words(lock_path: NonEmptyStr, conflict_exit_code: Int, command: List) -> List { + concat( + [flock_program().invocation as String, "--exclusive", "--nonblock", "--no-fork", "--conflict-exit-code", to_string(value: conflict_exit_code), lock_path as String], + command, + ) +} diff --git a/dag/gunbc/auth/fleet_secret_accessor_roster.dag b/dag/gunbc/auth/fleet_secret_accessor_roster.dag index dfd990157b0..1ae087ba61d 100644 --- a/dag/gunbc/auth/fleet_secret_accessor_roster.dag +++ b/dag/gunbc/auth/fleet_secret_accessor_roster.dag @@ -15,13 +15,16 @@ import gunbc.claude_code_credential { claude_code_oauth_harness_secret_ref } // so its consumers (the host convergence census, its witness) are not renamed by the move. import extdeps.cloud.gcp.secret_ref { SecretRef } import gunbc.spark.credential_workflow { spark_administrator_password_secret_ref } -import gunbc.secret_provision { mtcollins1_bmc_gunbc_secret_ref } +import gunbc.secret_provision { mtcollins1_bmc_gunbc_secret_ref, openrouter_free_tier_key_secret_ref } import gunbc.auth.approval_decision_store { approval_mac_key_secret_ref } import gunbc.auth.approval_request_submission { approval_submission_mac_key_secret_ref } import gunbc.auth.approval_store_receipt { approval_store_receipt_mac_key_secret_ref } import gunbc.auth.approval_ntfy_deployment { approval_ntfy_publisher_token_secret_ref } import gunbc.oracle_oci.api { oci_api_signing_key_secret_ref } import gunbc.auth.gcp_secret_access { SecretAccessGrant, secret_accessor_grant, secret_access_ensure_for } +import extdeps.cloud.gcp.iam { role_secretmanager_secret_version_adder } +import gunbc.codex_harness_credential { codex_auth_harness_secret_ref } +import gunbc.cursor_harness_credential { cursor_api_key_harness_secret_ref } import gunbc.auth.access_token_source { AccessTokenSource, OperatorSuppliedToken, read_supplied_access_token, SuppliedTokenReady, SuppliedTokenUnavailable, } @@ -60,6 +63,28 @@ data approval_ntfy_publisher_token_accessor_row: FleetAccessorGrantRow = FleetAc data fabric_state_key_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "fabric state service credential (controller and website custody)", target: fabric_state_key_secret_ref } +// THE WORKER-TURN CREDENTIALS (node adhoc-932f2935-e8e), both read by +// gunbc.host_credential_custody_converge over WIF for placement on their custody host. NOT YET +// APPLIED: until the ensure runs, those reads refuse and name the secret. +data cursor_api_key_harness_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "cursor worker-turn API key", target: cursor_api_key_harness_secret_ref } +data codex_auth_harness_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "codex worker-turn subscription auth.json", target: codex_auth_harness_secret_ref } + +// THE ONE WRITE GRANT, kept as its own roster because it is a different role and a different risk: +// codex rotates its refresh token on its custody host, and the custody converge reads the host file +// back and adds it as a new version (gunbc.codex_harness_credential codex_harness_add_version). secretVersionAdder can add versions and cannot +// read, disable or destroy them, so a converge that writes back cannot remove the version it read. +data codex_auth_harness_version_adder_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "codex worker-turn auth.json write-back", target: codex_auth_harness_secret_ref } + +data fleet_version_adder_grant_roster: List = [ + codex_auth_harness_version_adder_row, +] + +// THE OPENROUTER FREE-TIER KEY, read by the hosted harness route (gunbc.harness.harness_hosted_route) +// on every bind -- a recurring automated read, so it rides the workload's accessor cell rather than an +// operator session (gunbc.auth.authorization_pattern_selection). NOT YET APPLIED: until this roster's +// approval-gated converge runs, the hosted bind's Secret Manager read refuses and names the secret. +data openrouter_free_tier_key_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "OpenRouter free-tier key (hosted harness route)", target: openrouter_free_tier_key_secret_ref } + // The OCI API signing key, read on srv1 by the custody converge (gunbc.host_credential_custody_converge // OracleOciApiSigningKey). data oracle_oci_api_signing_key_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "oracle oci api signing key (srv1 custody)", target: oci_api_signing_key_secret_ref } @@ -72,6 +97,8 @@ data oracle_oci_api_signing_key_accessor_row: FleetAccessorGrantRow = FleetAcces data claude_code_oauth_harness_accessor_row: FleetAccessorGrantRow = FleetAccessorGrantRow { lane: "claude code subscription OAuth token (srv1 belt custody)", target: claude_code_oauth_harness_secret_ref } data fleet_accessor_grant_roster: List = [ + cursor_api_key_harness_accessor_row, + codex_auth_harness_accessor_row, spark_accessor_row, mtcollins1_bmc_accessor_row, approval_capability_mac_key_accessor_row, @@ -79,6 +106,7 @@ data fleet_accessor_grant_roster: List = [ approval_store_receipt_mac_key_accessor_row, approval_ntfy_publisher_token_accessor_row, fabric_state_key_accessor_row, + openrouter_free_tier_key_accessor_row, oracle_oci_api_signing_key_accessor_row, claude_code_oauth_harness_accessor_row, ] @@ -87,16 +115,28 @@ fn fleet_accessor_grant(row: FleetAccessorGrantRow) -> SecretAccessGrant { secret_accessor_grant(target: row.target) } +fn fleet_version_adder_grant(row: FleetAccessorGrantRow) -> SecretAccessGrant { + SecretAccessGrant { target: row.target, role: role_secretmanager_secret_version_adder, condition: none } +} + +// Every cell the convergence principal must hold, accessor rows first, as the one list each entry folds. +fn fleet_secret_grants() -> List { + concat( + map(fleet_accessor_grant_roster, row => fleet_accessor_grant(row: row)), + map(fleet_version_adder_grant_roster, row => fleet_version_adder_grant(row: row)), + ) +} + // THE FOLD STOPS AT THE FIRST REFUSAL: a grant that did not converge is reported with its lane, and // the rows after it are not attempted, so a partial pass never reads as a full one. fn fleet_accessor_grants_ensure(token_source: AccessTokenSource) -> ProcessExit { - fold(fleet_accessor_grant_roster, init: ExitSuccess, f: (acc, row) => + fold(fleet_secret_grants(), init: ExitSuccess, f: (acc, grant) => match acc { ExitSuccess => - match secret_access_ensure_for(grant: fleet_accessor_grant(row: row), token_source: token_source) { + match secret_access_ensure_for(grant: grant, token_source: token_source) { ExitSuccess => ExitSuccess - ExitFailure { code: c, reason: why } => ExitFailure { code: c, reason: join(["accessor grant for ", row.lane, ": ", why], "") } + ExitFailure { code: c, reason: why } => ExitFailure { code: c, reason: join([grant.role, " grant on ", grant.target.secret as String, ": ", why], "") } } failed => failed } @@ -113,9 +153,9 @@ fn fleet_accessor_grants_converge_with_supplied_token() -> ProcessExit // The shared approval workflow binds the exact grants and beneficiary before obtaining its credential. fn fleet_accessor_grants_request_approval() -> ProcessExit { - secret_grants_via_approval(grants: map(fleet_accessor_grant_roster, row => fleet_accessor_grant(row: row)), request_only: true) + secret_grants_via_approval(grants: fleet_secret_grants(), request_only: true) } fn fleet_accessor_grants_converge_via_approval() -> ProcessExit { - secret_grants_via_approval(grants: map(fleet_accessor_grant_roster, row => fleet_accessor_grant(row: row)), request_only: false) + secret_grants_via_approval(grants: fleet_secret_grants(), request_only: false) } diff --git a/dag/gunbc/auth/materialized_secret.dag b/dag/gunbc/auth/materialized_secret.dag index 468a794ccb8..d3132e53b2b 100644 --- a/dag/gunbc/auth/materialized_secret.dag +++ b/dag/gunbc/auth/materialized_secret.dag @@ -43,10 +43,16 @@ import gunbc.host_phase_status { observation_verdict_eq, host_phase_verdict_from // all. Collapsing them to a path would force the two non-file bindings to invent a file, which is // how a secret ends up on disk that never needed to be there. +// BearerHeaderFile is the fourth binding, added by the first consumer that performs it (the hosted harness +// route, gunbc.harness.harness_hosted_route): an HTTP API whose credential rides one +// "Authorization: Bearer" header line in a 0600 file that curl reads with -H @file. It is a file for the +// same reason SshKeyFile is -- the transport reads a path -- and it is not SshKeyFile because the bytes +// are a header line, not a key, and nothing about ssh's permission check applies to it. type SecretBinding = SshKeyFile | EnvVar { name: NonEmptyStr } | StdinPassword + | BearerHeaderFile type MaterializedSecret { ref: SecretRef diff --git a/dag/gunbc/auth/privileged_effect_census.dag b/dag/gunbc/auth/privileged_effect_census.dag index d6e24a2d7a4..1a1e1b4b25c 100644 --- a/dag/gunbc/auth/privileged_effect_census.dag +++ b/dag/gunbc/auth/privileged_effect_census.dag @@ -675,10 +675,16 @@ data privileged_effect_census: List = [ }, PrivilegedEffectSite { site: site(module_path: "gunbc.host_credential_custody_converge", decl_name: "host_credential_custody_converge_ci_wet"), - effect: recurring_credential_read(subject: "read one rostered custody credential over WIF -- the gunbai-ci App key for the microVM lifecycle controller, the approval ntfy publisher token, the fabric state writer key, or the Claude Code subscription OAuth token the roadmap belt's explicit Claude dispatch reads -- and place it on one fleet host at its row's owner, group and mode" as NonEmptyStr), + effect: recurring_credential_read(subject: "read one rostered custody credential over WIF -- the gunbai-ci App key for the microVM lifecycle controller, the approval ntfy publisher token, the fabric state writer key, the Claude Code subscription OAuth token the roadmap belt's explicit Claude dispatch reads, or a worker-turn credential (cursor-api-key-harness, codex-auth-harness) -- and place it on one fleet host at its row's owner, group and mode; for the host-authoritative codex row, read the placed file back and add it to codex-auth-harness as a new version when codex rotated its refresh token" as NonEmptyStr), realized: RealizedFederatedGrant, divergence_reason: none, }, + PrivilegedEffectSite { + site: site(module_path: "gunbc.harness.harness_hosted_route", decl_name: "hosted_secret_text"), + effect: recurring_credential_read(subject: "read the OpenRouter free-tier key on every hosted harness bind, then write it as a 0600 bearer header file removed at the binding's release" as NonEmptyStr), + realized: RealizedRunSelected, + divergence_reason: none, + }, PrivilegedEffectSite { site: site(module_path: "gunbc.tools.bmc_health_reader_converge", decl_name: "bmc_health_reader_converge"), effect: recurring_credential_read(subject: "BMC health reader credential read" as NonEmptyStr), @@ -768,7 +774,7 @@ data privileged_effect_census: List = [ }, PrivilegedEffectSite { site: site(module_path: "gunbc.auth.fleet_secret_accessor_roster", decl_name: "fleet_accessor_grants_converge_with_supplied_token"), - effect: accessor_grant(subject: "the fleet accessor roster: spark administrator password, mtcollins1 BMC credential, both approval MAC keys" as NonEmptyStr), + effect: accessor_grant(subject: "the fleet accessor roster: spark administrator password, mtcollins1 BMC credential, both approval MAC keys, the OpenRouter free-tier key" as NonEmptyStr), realized: RealizedOperatorSession { arm: OperatorTokenFile }, divergence_reason: Present { value: interim_gcp_iam_adapter_unlanded }, }, diff --git a/dag/gunbc/codex_app_server_press.dag b/dag/gunbc/codex_app_server_press.dag index a4916844438..7157325ae78 100644 --- a/dag/gunbc/codex_app_server_press.dag +++ b/dag/gunbc/codex_app_server_press.dag @@ -73,6 +73,7 @@ import extdeps.llm.codex_app_server { import extdeps.shell import extdeps.shell.exec import extdeps.tools.env { env_path_resolved_program, env_prefixed_args } +import extdeps.tools.util_linux_flock { flock_program, flock_exclusive_nonblocking_words } import v2.std.orchestration { EnvSet } import v2.std.compilers.cli_surface { ProcessArgvExpansion, @@ -544,16 +545,45 @@ fn codex_press_account_trip_request_lines() -> String { ) + "\n" } +// A TRIP THAT MUST NOT RACE A TURN runs under the turn's exclusive flock: account/read may refresh +// the token, and a refresh rotates it (gunbc.codex_harness_credential). The lock is one more argv +// layer in front of the same env-prefixed trip, so the program becomes flock and its arguments carry +// the env invocation; with no lock the trip is exactly as before. +type CodexTripLock { + lock_path: NonEmptyStr + conflict_exit_code: Int +} + +fn codex_press_account_trip_program(lock: CodexTripLock?) -> String { + match lock { + Absent => env_path_resolved_program().invocation as String + Present { value: _ } => flock_program().invocation as String + } +} + fn codex_press_account_trip_invocation( executable: FilePath, codex_home: FilePath, + lock: CodexTripLock?, ) -> ProcessArgvExpansion { + let env_args = env_prefixed_args( + bindings: [EnvSet { name: "CODEX_HOME", value: codex_home as String }], + command_argv: [executable as String, "app-server", "--stdio"], + ) process_argv_expansion( surface: cli_surface_of_literal_words( - words: env_prefixed_args( - bindings: [EnvSet { name: "CODEX_HOME", value: codex_home as String }], - command_argv: [executable as String, "app-server", "--stdio"], - ), + words: match lock { + Absent => env_args + Present { value: l } => + skip( + flock_exclusive_nonblocking_words( + lock_path: l.lock_path, + conflict_exit_code: l.conflict_exit_code, + command: concat([env_path_resolved_program().invocation as String], env_args), + ), + 1, + ) + }, ), ) } @@ -1608,6 +1638,7 @@ fn observe_codex_account_preflight( arguments: codex_press_account_trip_invocation( executable: projected.executable.path, codex_home: codex_home, + lock: none, ), stdin_payload: codex_press_account_trip_request_lines(), ) diff --git a/dag/gunbc/codex_harness_credential.dag b/dag/gunbc/codex_harness_credential.dag new file mode 100644 index 00000000000..a5764aab794 --- /dev/null +++ b/dag/gunbc/codex_harness_credential.dag @@ -0,0 +1,325 @@ +module gunbc.codex_harness_credential + +import std.types { String, NonEmptyStr, Secret, FilePath, Int, List } +import extdeps.filesystem.filesystem_io { + FilesystemFileObservation, + FilesystemFileAbsent, + FilesystemFileRead, + FilesystemFileIndeterminate, + FilesystemFileObservationsDisagree, + FilesystemFileSubjectRefused, +} +import extdeps.cloud.gcp.secret_ref { SecretRef, secret_ref_secret_resource } +import extdeps.cloud.gcp.secret_manager +import extdeps.cloud.gcp.secret_manager { encode_sm_access_version_payload_wire } +import extdeps.transports.rest { + classify_rest_refusal, + RestMutationExchange, + RestAnswered, + RestRefused, + RestExchangeStatusRefused, + RestExchangeCommitAmbiguous, + RestExchangeUnreached, + RestExchangeUndecodable, +} +import extdeps.languages.json.emit { JsonValue, JsonNull, JsonBool, JsonNumber, JsonString, JsonArray, JsonObject } +import extdeps.languages.json.parse { + parse_json_document, + JsonDocumentParsed, + JsonDocumentUnreadable, + json_object_unique_member, + JsonMemberFound, + JsonMemberAbsent, + JsonMemberDuplicated, + JsonMemberNotAnObject, +} +import extdeps.llm.codex_auth { + codex_auth_mode_of_wire, + CodexChatGptMode, + CodexApiKeyMode, + codex_auth_file_relative, + codex_auth_file_auth_mode_key, + codex_auth_file_tokens_key, + codex_auth_file_refresh_token_key, + codex_auth_file_account_id_key, +} +import gunbc.secret_provision { fleet_secret_ref, secret_ref_pin_version } +import gunbc.auth.access_token_source { + AccessTokenSource, + ensure_access_token, + AccessTokenReady, + AccessTokenUpsertRequired, + AccessTokenEnsureRefused, +} +import extdeps.tools.util_linux_flock { flock_exclusive_nonblocking_words } +import gunbc.codex_app_server_press { AmbientCodexCredentialContext } +import product.host_identity { HostIdentity } +import gunbc.fleet_host_identity { operator_host_srv2 } + +// ONE CODEX SUBSCRIPTION CREDENTIAL, HELD ON ONE HOST AND WRITTEN BACK WHEN CODEX ROTATES IT. +// The ambient route binds a CODEX_HOME someone logged into on the host; nothing in it says where the +// login came from or who else holds a copy. This credential comes from ONE secret, codex-auth-harness, +// whose payload is the auth.json a ChatGPT login wrote, byte for byte. +// +// Why the host is authoritative after placement: extdeps.llm.codex_auth +// codex_source_refresh_persistence records that codex rewrites auth.json about once an hour while it +// runs, replacing the refresh token with whatever the issuer returned, and that the issuer refuses a +// reused refresh token. So the copy codex last refreshed is the only live one. The custody converge +// (gunbc.host_credential_custody_converge CodexWorkerTurnAuth) places the file once on the one custody +// host, never overwrites it afterwards, and on every run reads it back and adds it to the store as a +// new version when its refresh token moved on. Turns on that host run under the flock below, so one +// codex process refreshes at a time. +// +// Operator rulings this follows (relayed by swift-ibex-601, 2026-10-05): subscription (chatgpt) mode, +// not API-key mode; never codex's upstream-labelled-internal external-token login; credential delivery +// by host custody under the fleet-converge run's federated grant, with no per-turn Secret Manager +// access; exactly one custody host. +// +// RESIDUAL, stated: the store lags the host until the next custody converge. A host lost after a +// rotation and before that converge loses the live token; the next placement on a rebuilt host then +// carries a spent one, codex reports login required, and dispatch refuses the credential until the +// operator logs in again. +// +// DECLARED FRONTIER (operator ruling, 2026-10-05): the narrower window is a workload identity bound +// to the codex turn runner on the custody host, holding only secretAccessor and secretVersionAdder on +// codex-auth-harness, so a turn could add the rotated file as a version the moment codex writes it +// rather than at the next converge. TRIGGER: gunbc.auth.access_token_source select_access_token_source +// resolves a federated token on the custody host outside an Actions run. Until then delivery is the +// custody converge, and nothing in a turn reaches Secret Manager. + +data codex_auth_harness_secret_id: NonEmptyStr = "codex-auth-harness" as NonEmptyStr + +// latest, because the credential moves: every write-back mints the next version. The fetch resolves +// the alias to an exact version and the receipt carries that version. +data codex_auth_harness_secret_ref: SecretRef = secret_ref_pin_version( + ref: fleet_secret_ref(secret_id: codex_auth_harness_secret_id), + version: "latest", +) + +// THE ONE HOST THAT HOLDS THE CODEX SUBSCRIPTION CREDENTIAL, and the CODEX_HOME on it. The store lags +// the host until the next custody converge reads the host file back, so a second host would receive a +// spent token: gunbc.host_credential_custody_converge refuses to place a host-authoritative row on +// more than one host, and codex harness dispatch is pinned here. +data codex_worker_turn_custody_host: HostIdentity = operator_host_srv2 +data codex_worker_turn_codex_home: NonEmptyStr = "/etc/gunbc-codex-harness" as NonEmptyStr +data codex_worker_turn_auth_host_path: NonEmptyStr = "/etc/gunbc-codex-harness/auth.json" as NonEmptyStr + +// WHAT THE HARNESS NEEDS FROM auth.json, AND NOTHING ELSE. The refresh token decides whether a +// write-back is owed; the account id decides whether the file still belongs to the account the secret +// holds. The whole document is what gets written back, so nothing else needs reading. +type CodexHarnessAuthIdentity { + account_id: NonEmptyStr + refresh_token: Secret +} + +type CodexHarnessAuthFileRefusal + = CodexHarnessAuthFileUnparseable + | CodexHarnessAuthModeNotChatgpt { observed: String } + | CodexHarnessAuthModeAbsent + | CodexHarnessAuthMemberMissing { member: NonEmptyStr } + +type CodexHarnessAuthRead + = CodexHarnessAuthAdmitted { identity: CodexHarnessAuthIdentity } + | CodexHarnessAuthRefused { cause: CodexHarnessAuthFileRefusal } + +fn codex_harness_string_member(v: JsonValue, key: NonEmptyStr) -> String? { + match json_object_unique_member(v: v, key: key as String) { + JsonMemberFound { value } => + match value { + JsonString { value: s } => if s == "" { none } else { Present { value: s } } + JsonNull => none + JsonBool { value: _ } => none + JsonNumber { lexeme: _ } => none + JsonArray { elements: _ } => none + JsonObject { members: _ } => none + } + JsonMemberAbsent => none + JsonMemberDuplicated { count: _ } => none + JsonMemberNotAnObject => none + } +} + +fn codex_harness_member_missing(key: NonEmptyStr) -> CodexHarnessAuthRead { + CodexHarnessAuthRefused { cause: CodexHarnessAuthMemberMissing { member: key } } +} + +fn codex_harness_tokens_identity(tokens: JsonValue) -> CodexHarnessAuthRead { + match codex_harness_string_member(v: tokens, key: codex_auth_file_refresh_token_key) { + Absent => codex_harness_member_missing(key: codex_auth_file_refresh_token_key) + Present { value: refresh } => + match codex_harness_string_member(v: tokens, key: codex_auth_file_account_id_key) { + Absent => codex_harness_member_missing(key: codex_auth_file_account_id_key) + Present { value: account } => + CodexHarnessAuthAdmitted { + identity: CodexHarnessAuthIdentity { + account_id: account as NonEmptyStr, + refresh_token: refresh as Secret, + }, + } + } + } +} + +fn codex_harness_chatgpt_identity(doc: JsonValue) -> CodexHarnessAuthRead { + match json_object_unique_member(v: doc, key: codex_auth_file_tokens_key as String) { + JsonMemberFound { value: tokens } => codex_harness_tokens_identity(tokens: tokens) + JsonMemberAbsent => codex_harness_member_missing(key: codex_auth_file_tokens_key) + JsonMemberDuplicated { count: _ } => codex_harness_member_missing(key: codex_auth_file_tokens_key) + JsonMemberNotAnObject => CodexHarnessAuthRefused { cause: CodexHarnessAuthFileUnparseable } + } +} + +// THE PAYLOAD FORMAT IS CHECKED BEFORE ANY TURN RUNS. An API-key auth.json, or anything that is not +// an auth.json, refuses here with the reason, rather than starting codex against a file it will reject. +fn codex_harness_auth_read(content: String) -> CodexHarnessAuthRead { + match parse_json_document(s: content) { + JsonDocumentUnreadable { gap: _ } => CodexHarnessAuthRefused { cause: CodexHarnessAuthFileUnparseable } + JsonDocumentParsed { value: doc } => + match codex_harness_string_member(v: doc, key: codex_auth_file_auth_mode_key) { + Absent => CodexHarnessAuthRefused { cause: CodexHarnessAuthModeAbsent } + Present { value: wire } => + match codex_auth_mode_of_wire(raw: wire) { + Present { value: mode } => + match mode { + CodexChatGptMode => codex_harness_chatgpt_identity(doc: doc) + CodexApiKeyMode => CodexHarnessAuthRefused { cause: CodexHarnessAuthModeNotChatgpt { observed: wire } } + } + Absent => CodexHarnessAuthRefused { cause: CodexHarnessAuthModeNotChatgpt { observed: wire } } + } + } + } +} + +// THE WRITE-BACK DECISION, from the identity that was materialized and the file read back after the +// turn. A changed refresh token owes a new version carrying the whole file. An unchanged one owes +// nothing. A file that now names a different account, or that cannot be read, is refused rather than +// written: storing it would replace the operator's credential with something this run cannot vouch +// for, and NOT storing it is only safe because the refusal is reported. +type CodexHarnessWriteBackRefusal + = CodexHarnessReadBackUnobservable { detail: String } + | CodexHarnessReadBackAbsent + | CodexHarnessReadBackRefused { cause: CodexHarnessAuthFileRefusal } + | CodexHarnessReadBackAccountChanged { materialized: NonEmptyStr, read_back: NonEmptyStr } + +type CodexHarnessWriteBackDecision + = CodexHarnessWriteBackNotOwed + | CodexHarnessWriteBackOwed { payload: Secret } + | CodexHarnessWriteBackRefused { cause: CodexHarnessWriteBackRefusal } + +fn codex_harness_write_back_decision( + materialized: CodexHarnessAuthIdentity, + read_back: FilesystemFileObservation, +) -> CodexHarnessWriteBackDecision { + match read_back { + FilesystemFileAbsent(_) => CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackAbsent } + FilesystemFileIndeterminate { cause } => + CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackUnobservable { detail: cause } } + FilesystemFileObservationsDisagree { path: _, cause } => + CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackUnobservable { detail: cause } } + FilesystemFileSubjectRefused { directory: _, name: _, cause } => + CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackUnobservable { detail: cause } } + FilesystemFileRead { path: _, content } => + match codex_harness_auth_read(content: content) { + CodexHarnessAuthRefused { cause } => + CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackRefused { cause: cause } } + CodexHarnessAuthAdmitted { identity: after } => + if (after.account_id as String) != (materialized.account_id as String) { + CodexHarnessWriteBackRefused { + cause: CodexHarnessReadBackAccountChanged { + materialized: materialized.account_id, + read_back: after.account_id, + }, + } + } else if (after.refresh_token as String) == (materialized.refresh_token as String) { + CodexHarnessWriteBackNotOwed + } else { + CodexHarnessWriteBackOwed { payload: content as Secret } + } + } + } +} + +// WHAT HAPPENED TO THE STORE, as a receipt the outcome carries. VersionAddAmbiguous is the case worth +// naming: the store may or may not hold the rotated token, so the next acquire may read a spent one. +type CodexHarnessWriteBackReceipt + = CodexHarnessStoreUnchanged + | CodexHarnessVersionAdded { version_name: String } + | CodexHarnessVersionAddRefused { detail: String } + | CodexHarnessVersionAddAmbiguous { detail: NonEmptyStr } + | CodexHarnessWriteBackWithheld { cause: CodexHarnessWriteBackRefusal } + +// THE CONVERGE'S WRITE-BACK. gunbc.host_credential_custody_converge calls this when the host's +// auth.json carries a refresh token the store does not, under the fleet-converge run's federated +// token and the secretVersionAdder grant in gunbc.auth.fleet_secret_accessor_roster. +fn codex_harness_add_version(token_source: AccessTokenSource, payload: Secret) -> CodexHarnessWriteBackReceipt { + match ensure_access_token(source: token_source) { + AccessTokenUpsertRequired { plan: _ } => + CodexHarnessVersionAddRefused { detail: "no Secret Manager access token: access upsert required" } + AccessTokenEnsureRefused { reason } => + CodexHarnessVersionAddRefused { detail: reason as String } + AccessTokenReady { token } => { + match gcp.SecretManager.AddVersion( + access_token: token, + secret_name: secret_ref_secret_resource(ref: codex_auth_harness_secret_ref), + payload_b64: encode_sm_access_version_payload_wire(credential: payload), + ) { + RestAnswered { answer: added } => CodexHarnessVersionAdded { version_name: added.name as String } + RestRefused { refusal } => + match classify_rest_refusal(standing: RestMutationExchange { refusal: refusal }) { + RestExchangeStatusRefused { status: _, body } => CodexHarnessVersionAddRefused { detail: body } + RestExchangeCommitAmbiguous { detail } => CodexHarnessVersionAddAmbiguous { detail: detail } + RestExchangeUnreached { cause } => CodexHarnessVersionAddAmbiguous { detail: cause } + RestExchangeUndecodable { status: _, cause } => CodexHarnessVersionAddAmbiguous { detail: cause } + } + } + } + } +} + +fn codex_harness_write_back_refusal_wire(cause: CodexHarnessWriteBackRefusal) -> String { + match cause { + CodexHarnessReadBackUnobservable { detail } => concat("host auth.json unobservable: ", detail) + CodexHarnessReadBackAbsent => "host auth.json absent" + CodexHarnessReadBackRefused { cause: _ } => "host auth.json is not a chatgpt-mode auth.json" + CodexHarnessReadBackAccountChanged { materialized, read_back } => + join(["host auth.json names account ", read_back as String, ", the store names ", materialized as String], "") + } +} + +fn codex_harness_write_back_receipt_wire(r: CodexHarnessWriteBackReceipt) -> String { + match r { + CodexHarnessStoreUnchanged => "unchanged" + CodexHarnessVersionAdded { version_name } => concat("version-added ", version_name) + CodexHarnessVersionAddRefused { detail } => concat("REFUSED ", detail) + CodexHarnessVersionAddAmbiguous { detail } => concat("AMBIGUOUS ", detail as String) + CodexHarnessWriteBackWithheld { cause } => concat("WITHHELD ", codex_harness_write_back_refusal_wire(cause: cause)) + } +} + +// ── The turn side: one turn at a time over the custody CODEX_HOME ─────────────────────────────── +// +// The turn reads nothing from Secret Manager. It runs codex with CODEX_HOME set to the directory the +// custody converge placed on the one custody host, under an exclusive flock(1) on a lock file in that +// directory, because codex has no cross-process lock on auth.json (extdeps.llm.codex_auth +// codex_source_revision note) and two concurrent refreshes would present one refresh token twice. +// The lock is held by the codex process for exactly its lifetime and the kernel drops it if the +// process dies, so a crashed turn leaves nothing to recover. A second turn while one runs does not +// wait: it exits codex_harness_turn_busy_exit, which names the credential as busy rather than as a +// codex failure. + +data codex_harness_turn_lock_path: NonEmptyStr = "/etc/gunbc-codex-harness/turn.lock" as NonEmptyStr + +// EX_TEMPFAIL (sysexits.h 75): the credential is in use by another turn; retrying later is the remedy. +data codex_harness_turn_busy_exit: Int = 75 + +fn codex_harness_credential_context() -> AmbientCodexCredentialContext { + AmbientCodexCredentialContext { codex_home: codex_worker_turn_codex_home as FilePath } +} + +fn codex_harness_turn_argv(command: List) -> List { + flock_exclusive_nonblocking_words( + lock_path: codex_harness_turn_lock_path, + conflict_exit_code: codex_harness_turn_busy_exit, + command: command, + ) +} diff --git a/dag/gunbc/cursor_harness_credential.dag b/dag/gunbc/cursor_harness_credential.dag new file mode 100644 index 00000000000..8d2a789f02a --- /dev/null +++ b/dag/gunbc/cursor_harness_credential.dag @@ -0,0 +1,206 @@ +module gunbc.cursor_harness_credential + +import std.types { String, NonEmptyStr, Bool, List, FilePath } +import std.upsert_decision { ObservationVerdict, Converged, Drifted, UnknownRefused } +import extdeps.cloud.gcp.secret_ref { SecretRef } +import extdeps.cloud.gcp.secret_manager { SmVersionState } +import extdeps.languages.json.emit { JsonValue, JsonNull, JsonBool, JsonNumber, JsonString, JsonArray, JsonObject } +import extdeps.languages.json.parse { + parse_json_document, + JsonDocumentParsed, + JsonDocumentUnreadable, + json_object_unique_member, + JsonMemberFound, + JsonMemberAbsent, + JsonMemberDuplicated, + JsonMemberNotAnObject, +} +import extdeps.llm.cursor_cli { + cursor_api_key_env_var, + CursorRunShape, + cursor_model_auto, + CursorDefaultEditMode, + CursorSandboxEnabled, + CursorStreamJson, + cursor_environment_authenticated_argv, +} +import extdeps.posix.sh_invocation { posix_sh_export_from_file_then_exec_command, PosixShExportExecReady, PosixShExportExecEmitRefused, PosixShRemoveFileAfterRead } +import extdeps.posix.shell_command_language { admit_posix_shell_name, PosixShellNameAdmitted, PosixShellNameRefused } +import extdeps.exec.command { argv_words } +import product.host_identity { HostIdentity } +import gunbc.fleet_host_identity { operator_host_srv2 } +import gunbc.secret_provision { fleet_secret_ref, secret_ref_pin_version } +import gunbc.readback_independence { ReadBackProbe, ProbeInert } +import gunbc.auth.materialized_secret { + EnvVar, + MaterializedSecret, + MaterializationOutcome, + with_materialized_secret, +} + +// ONE CURSOR API KEY, READ FROM gunbai-secrets, BOUND AS CURSOR_API_KEY. extdeps.llm.cursor_cli +// records that the CLI authenticates per invocation with a value, so the fleet credential is a secret +// and nothing else: no directory, no rotation, nothing to write back. The dispatch credential is the +// existing CursorApiKeySecretRef arm (gunbc.dispatch_selection declared_provider_inventory_for_instance +// builds it from this ref), and gunbc.cursor_sdk_local_binding admits it unchanged. +// +// The version is EXACT, not latest: gunbc.cursor_sdk_secret_admission refuses latest for Cursor, and +// that ruling stands. Version 1 is the operator's first upload; replacing the key is a new version +// and an edit to this row. + +data cursor_api_key_harness_secret_id: NonEmptyStr = "cursor-api-key-harness" as NonEmptyStr + +data cursor_api_key_harness_secret_ref: SecretRef = secret_ref_pin_version( + ref: fleet_secret_ref(secret_id: cursor_api_key_harness_secret_id), + version: "1", +) + + +// THE LIVENESS PROBE IS THE CLI'S OWN STATUS COMMAND. Observed on cursor-agent 2026.10.01-e373342 +// (2026-10-05) with a deliberately invalid key in CURSOR_API_KEY: `cursor-agent status --format json` +// exits 0 and prints {"status":"unauthenticated","isAuthenticated":false,...}. It asks Cursor whether +// the key is accepted and cannot make a rejected key accepted, so it is ProbeInert and +// with_materialized_secret's independence gate admits it. A --print turn with the same invalid key +// exits 1 with prose on stderr and no JSON, which is why the probe runs before the turn rather than +// the turn's failure standing in for it. +data cursor_status_probe: ReadBackProbe = ReadBackProbe { + label: "cursor-agent status --format json with CURSOR_API_KEY bound", + subject: "the API key cursor-api-key-harness holds is one Cursor currently accepts", + effect_on_subject: ProbeInert, +} + +data cursor_status_is_authenticated_key: NonEmptyStr = "isAuthenticated" as NonEmptyStr +data cursor_status_status_key: NonEmptyStr = "status" as NonEmptyStr + +type CursorStatusReading + = CursorStatusAuthenticated + | CursorStatusUnauthenticated { status: String } + | CursorStatusUnreadable { detail: String } + +fn cursor_status_status_text(doc: JsonValue) -> String { + match json_object_unique_member(v: doc, key: cursor_status_status_key as String) { + JsonMemberFound { value } => + match value { + JsonString { value: s } => s + JsonNull => "" + JsonBool { value: _ } => "" + JsonNumber { lexeme: _ } => "" + JsonArray { elements: _ } => "" + JsonObject { members: _ } => "" + } + JsonMemberAbsent => "" + JsonMemberDuplicated { count: _ } => "" + JsonMemberNotAnObject => "" + } +} + +fn cursor_status_reading(stdout: String) -> CursorStatusReading { + match parse_json_document(s: stdout) { + JsonDocumentUnreadable { gap: _ } => CursorStatusUnreadable { detail: "status output is not a JSON document" } + JsonDocumentParsed { value: doc } => + match json_object_unique_member(v: doc, key: cursor_status_is_authenticated_key as String) { + JsonMemberFound { value } => + match value { + JsonBool { value: authenticated } => + if authenticated { + CursorStatusAuthenticated + } else { + CursorStatusUnauthenticated { status: cursor_status_status_text(doc: doc) } + } + JsonNull => CursorStatusUnreadable { detail: "isAuthenticated is not a boolean" } + JsonNumber { lexeme: _ } => CursorStatusUnreadable { detail: "isAuthenticated is not a boolean" } + JsonString { value: _ } => CursorStatusUnreadable { detail: "isAuthenticated is not a boolean" } + JsonArray { elements: _ } => CursorStatusUnreadable { detail: "isAuthenticated is not a boolean" } + JsonObject { members: _ } => CursorStatusUnreadable { detail: "isAuthenticated is not a boolean" } + } + JsonMemberAbsent => CursorStatusUnreadable { detail: "isAuthenticated absent" } + JsonMemberDuplicated { count: _ } => CursorStatusUnreadable { detail: "isAuthenticated duplicated" } + JsonMemberNotAnObject => CursorStatusUnreadable { detail: "status output is not a JSON object" } + } + } +} + +// Rejected is Drifted (the store holds a value upstream no longer accepts), not Absent: the secret +// exists, the key in it is dead, and the remedy is a new key, not a new secret. +fn cursor_status_liveness(reading: CursorStatusReading) -> ObservationVerdict { + match reading { + CursorStatusAuthenticated => Converged + CursorStatusUnauthenticated { status: _ } => Drifted + CursorStatusUnreadable { detail: _ } => UnknownRefused + } +} + +// THE BRACKET, with the status probe's stdout as the liveness observation. No probe output means no +// observation, and with_materialized_secret refuses that as UnknownRefused. +fn with_cursor_harness_api_key( + existence: SmVersionState, + status_stdout: String?, + use: fn(MaterializedSecret) -> R, +) -> MaterializationOutcome { + let observed = match status_stdout { + Present { value: out } => Present { value: cursor_status_liveness(reading: cursor_status_reading(stdout: out)) } + Absent => none + } + with_materialized_secret( + ref: cursor_api_key_harness_secret_ref, + binding: EnvVar { name: cursor_api_key_env_var }, + existence: existence, + probe: cursor_status_probe, + liveness_observation: observed, + use: use, + ) +} + +// ── The spawn: the custody file becomes CURSOR_API_KEY in the child, never argv ───────────────── +// +// gunbc.host_credential_custody_converge CursorWorkerTurnApiKey places the key at this path on this +// host, read-only to the operator account the dispatch spawns run as. The spawn exports it from the +// file inside the child through the shared extdeps.posix.sh_invocation wrapper, which exits 78 if the +// file cannot be read, so the key is never on an argv the process table shows. + +data cursor_worker_turn_custody_host: HostIdentity = operator_host_srv2 +data cursor_worker_turn_api_key_host_path: NonEmptyStr = "/etc/gunbc-cursor-harness/api-key" as NonEmptyStr + +// THE UNATTENDED RUN: print mode with stream-json events, the default editing mode, --force so the +// agent may run commands without a human to approve them, the sandbox explicitly enabled so --force +// does not also mean unconfined, and --trust because an untrusted workspace prompts and an +// unattended run would hang there (extdeps.llm.cursor_cli). +fn cursor_worker_turn_run(worktree_path: String, prompt: NonEmptyStr) -> CursorRunShape { + CursorRunShape { + model: cursor_model_auto, + mode: CursorDefaultEditMode, + sandbox: CursorSandboxEnabled, + output_format: CursorStreamJson, + force: true, + trust_workspace: true, + workspace: worktree_path as FilePath, + prompt: prompt, + } +} + +// key_path is the attempt's credential snapshot, removed by the shell before exec (the shared +// one-snapshot rule, gunbc.roadmap_dispatch_actuator dispatch_credential_snapshot_path_for_instance). +// The env-var name is admitted as a POSIX name before it reaches the wrapper (extdeps.posix. +// shell_command_language admit_posix_shell_name), and the wrapper's program is emitted from the bash +// grammar, which may refuse; either refusal is one arm carrying its reason. +type CursorWorkerTurnArgv + = CursorWorkerTurnArgvReady { argv: List } + | CursorWorkerTurnRefused { reason: NonEmptyStr } + +fn cursor_worker_turn_argv(key_path: NonEmptyStr, worktree_path: String, prompt: NonEmptyStr) -> CursorWorkerTurnArgv { + match admit_posix_shell_name(spelling: cursor_api_key_env_var as String) { + PosixShellNameAdmitted { name } => + match posix_sh_export_from_file_then_exec_command( + name: name, + path: key_path, + command: cursor_environment_authenticated_argv(run: cursor_worker_turn_run(worktree_path: worktree_path, prompt: prompt)), + disposition: PosixShRemoveFileAfterRead, + ) { + PosixShExportExecReady { command } => CursorWorkerTurnArgvReady { argv: argv_words(command: command) } + PosixShExportExecEmitRefused { name: n } => + CursorWorkerTurnRefused { reason: join(["the sh emitter refused the export-and-exec program for ", n as String], "") as NonEmptyStr } + } + PosixShellNameRefused { spelling } => + CursorWorkerTurnRefused { reason: join(["the cursor credential variable name ", spelling, " is not a POSIX shell name"], "") as NonEmptyStr } + } +} diff --git a/dag/gunbc/dispatch_selection.dag b/dag/gunbc/dispatch_selection.dag index 42dceb66b2d..59bb6f972bc 100644 --- a/dag/gunbc/dispatch_selection.dag +++ b/dag/gunbc/dispatch_selection.dag @@ -45,6 +45,7 @@ import gunbc.codex_provider_runtime_receipt { codex_process_fingerprint_unobserved_reason, } import extdeps.llm.cursor_cli { CursorModelId, cursor_model_auto } +import gunbc.cursor_harness_credential { cursor_api_key_harness_secret_ref } import extdeps.cloud.gcp.secret_ref { SecretRef } import gunbc.claude_code_credential { claude_code_oauth_harness_secret_ref } import gunbc.roadmap_sizing { @@ -784,6 +785,32 @@ fn inventory_with_observed_standing(inventory: ProviderInventory, kind: Provider } } +// The offer the codex draw is judged on names the custody CODEX_HOME the standing was observed against +// and the spawn binds, never the instance's ambient account state root. This is the root, not the +// standing: the standing join is inventory_with_observed_standing. +fn inventory_with_codex_state_root(inventory: ProviderInventory, account_state_root: FilePath) -> ProviderInventory { + ProviderInventory { + offers: map(inventory.offers, offer => + match offer { + OfferCodex { offer: o } => + OfferCodex { + offer: CodexOffer { + instance: o.instance, + account_state_root: account_state_root, + account_binding: o.account_binding, + standing: o.standing, + default_effort: o.default_effort, + deep: o.deep, + process_fingerprint: o.process_fingerprint, + }, + } + OfferClaude { offer: _ } => offer + OfferCursor { offer: _ } => offer + } + ), + } +} + fn claude_offer_for_instance_with_standing( instance: HostDashboardInstance, standing: ProviderStanding, @@ -945,7 +972,7 @@ fn declared_provider_inventory_for_instance(instance: HostDashboardInstance) -> reason: "inventory row is declared, not observed at selection time" as NonEmptyStr, }, }, - credential: CursorLocalLoginRef, + credential: CursorApiKeySecretRef { secret_ref: cursor_api_key_harness_secret_ref }, standing: provider_standing_inventory_unobserved(), default_model: cursor_model_auto, process_fingerprint: dispatch_cursor_process_fingerprint, @@ -1470,7 +1497,7 @@ fn provider_selection_request_for_kind(kind: ProviderKind) -> ProviderSelectionR CursorCliProvider => SelectCursor { profile: ProfileAutomatic, - credential: CursorLocalLoginRef, + credential: CursorApiKeySecretRef { secret_ref: cursor_api_key_harness_secret_ref }, model: cursor_model_auto, agent_mode: "default-edit" as NonEmptyStr, } diff --git a/dag/gunbc/extdeps_scope_frontier.dag b/dag/gunbc/extdeps_scope_frontier.dag index 94f0dcdc79a..d2068be5b8d 100644 --- a/dag/gunbc/extdeps_scope_frontier.dag +++ b/dag/gunbc/extdeps_scope_frontier.dag @@ -580,6 +580,10 @@ data scope_carrier_paths: List = [ "dag/extdeps/cloudflare/r2.dag", "dag/extdeps/cloudflare/r2_buckets.dag", "dag/extdeps/deepseek/deepseek_v4_1_flash.dag", + "dag/extdeps/nvidia/api_trial_terms.dag", + "dag/extdeps/nvidia/nemotron_3_ultra.dag", + "dag/extdeps/openrouter/openrouter.dag", + "dag/extdeps/openrouter/nvidia_nemotron_3_ultra_free.dag", "dag/extdeps/zhipu/glm_5_3.dag", "dag/extdeps/radixark/glm_5_3_nvfp4.dag", "dag/extdeps/docker/images/gitcommit90_glm_53_one_spark.dag", diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index bd95ae2f46a..48930300359 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -25,7 +25,7 @@ import product.capacity.event_chain { import product.capacity.pool_events { PoolEvent, pool_event_wire_text, pool_event_decode, PoolFold, PoolFolded, PoolFoldRefused, pool_fold, SeatRequest, SeatProposal, SeatProposed, SeatRefused, propose_acquire, grant_from_admission, - PoolReleased, + PoolReleased, pool_apply_payload, } import product.capacity.lease { LeasePolicy, LeaseGrant, release_law_eq, release_law_wire } import product.capacity.redemption { @@ -415,6 +415,96 @@ fn pool_event_append_wire(a: PoolEventAppend) -> String { } } +// A POOL TRANSITION THAT IS CHECKED BEFORE IT IS WRITTEN. fabric_pool_event_append appends the event it +// is handed, and a caller that reports success on that append can report success over a transition no +// fold will ever apply: a settlement appended after its lease lapsed, or a hold whose reference +// duplicates an earlier one. The fold refuses such an event on replay, and pool_fold refuses the WHOLE +// partition at the first refused event -- so one unchecked append makes every later read of the pool +// fail while the caller believed it had succeeded. +// +// SO THIS IS SETTLEMENT AND OBSERVATION WITH THE ACQUISITION'S DISCIPLINE: read the partition, fold it, +// apply the transition with the same function replay uses (product.capacity.pool_events +// pool_apply_payload), and only if that advances, compare-and-set append against the head it was +// decided on; a moved head is re-read and re-decided, never written blind. A refused transition is its +// own arm and nothing is appended. The payload is built from the head because a transition that mints +// its own hold needs an identity no other reading shares. +type PoolTransition + = PoolTransitionAppended { id: EventId } + | PoolTransitionRefused { wire: String } + | PoolTransitionContended { attempts: Nat } + | PoolTransitionStoreRefused { step: String, reason: String } + +type PoolTransitionState { + done: PoolTransition? +} + +fn pool_transition_head_word(head: HeadExpectation) -> NonEmptyStr { + match head { + HeadAbsent => "genesis" as NonEmptyStr + HeadAt { id: h } => h as String as NonEmptyStr + } +} + +fn pool_transition_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, at: EpochSecs, payload: fn(NonEmptyStr) -> PoolEvent, budget: Nat) -> PoolTransition? { + match event_log_read_partition(store: store, partition: partition, budget: budget) { + PartitionReadRefused { cause: c } => Present { value: PoolTransitionStoreRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + PartitionReadOk { head: head, walk: walk } => + match walk { + ChainIncomplete { missing: m, newest_first_so_far: _ } => Present { value: PoolTransitionStoreRefused { step: "chain", reason: join(["partition chain is missing event ", m as String], "") } } + ChainBudgetExhausted { at: a } => Present { value: PoolTransitionStoreRefused { step: "chain", reason: join(["partition chain exceeded the read budget at ", a as String], "") } } + ChainWalked { oldest_first: xs } => + match pool_fold(root: root, oldest_first: xs) { + PoolFoldRefused { at_event: e, wire: w } => Present { value: PoolTransitionStoreRefused { step: "fold", reason: join(["event ", e as String, " refused: ", w], "") } } + PoolFolded { pool: p, generation: _ } => { + let identity = join(["pending:", pool_transition_head_word(head: head) as String], "") as NonEmptyStr + let event_payload = payload(identity) + match pool_apply_payload(pool: p, payload: event_payload, at: at, identity: identity) { + PoolRefused { refusal: r } => Present { value: PoolTransitionRefused { wire: pool_refusal_wire(r: r) } } + PoolAdvanced { pool: _ } => + match event_log_append( + store: store, + partition: partition, + event: ChainEvent { + partition: partition, + parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, + recorded_at: at, + actor: actor, + payload: event_payload, + }, + expected: head, + ) { + EventAppended { id: h } => Present { value: PoolTransitionAppended { id: h } } + EventAppendStale { expected: _, observed: _ } => none + EventAppendRefused { cause: c } => Present { value: PoolTransitionStoreRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + } + } + } + } + } + } +} + +fn fabric_pool_transition(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, at: EpochSecs, payload: fn(NonEmptyStr) -> PoolEvent, attempts: Nat, budget: Nat) -> PoolTransition { + let st = fold(nat_range_inclusive(lo: 1, hi: attempts), init: PoolTransitionState { done: none }, f: (acc, _i) => + match acc.done { + Present { value: _ } => acc + Absent => PoolTransitionState { done: pool_transition_attempt(store: store, partition: partition, root: root, actor: actor, at: at, payload: payload, budget: budget) } + }) + match st.done { + Present { value: d } => d + Absent => PoolTransitionContended { attempts: attempts } + } +} + +fn pool_transition_wire(t: PoolTransition) -> String { + match t { + PoolTransitionAppended { id: h } => join(["appended ", h as String], "") + PoolTransitionRefused { wire: w } => join(["the pool refuses the transition: ", w], "") + PoolTransitionContended { attempts: n } => join(["contended after ", to_string(n), " attempts"], "") + PoolTransitionStoreRefused { step: s, reason: why } => join(["refused at ", s, ": ", why], "") + } +} + // RELEASING A SEAT, AND IT IS CHECKED AGAINST THE FOLDED POOL BEFORE ANYTHING IS APPENDED. // // fabric_pool_event_append is unchecked by construction -- it writes the event a caller hands it -- diff --git a/dag/gunbc/fabric/fabric_quota.dag b/dag/gunbc/fabric/fabric_quota.dag index 870172d2f17..0c1da31444e 100644 --- a/dag/gunbc/fabric/fabric_quota.dag +++ b/dag/gunbc/fabric/fabric_quota.dag @@ -2,17 +2,19 @@ module gunbc.fabric_quota import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat } +import std.checked_arithmetic { checked_int_to_nat } import std.measure { Measure } import extdeps.api_rate_limit { UpstreamRateLimit, upstream_rate_limit_key } import product.capacity.quota { quota_partition, quota_root_pool } import product.capacity.event_chain { PartitionId } -import product.capacity.pool_events { PoolSettled, SeatRequest } +import product.capacity.pool_events { PoolSettled, PoolUpstreamObserved, SeatRequest } +import product.capacity.pool { pool_window_at, WindowResolved, WholeLifetime, InstantBeforeAnchor } import product.capacity.lease { LeasePolicy, LeaseGrant, FencedResource } import gunbc.fabric_event_log_host { HostEventLogStore, HostStoreResolved, HostStoreRefused, event_log_store_for_host, now_epoch_seconds } import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.fabric_event_log { SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, fabric_seat_acquire, seat_acquisition_wire, - PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, + PoolTransition, PoolTransitionAppended, fabric_pool_transition, pool_transition_wire, } // A CALLER LEASES UPSTREAM QUOTA BEFORE IT SPENDS IT. The lease holds the calls it is about to @@ -25,7 +27,7 @@ import gunbc.fabric_event_log { // expiry frees is the unspent remainder. type QuotaLease - = QuotaLeased { grant: LeaseGrant, partition: PartitionId, store: FabricStorageBinding } + = QuotaLeased { grant: LeaseGrant, partition: PartitionId, store: FabricStorageBinding, limit: UpstreamRateLimit } | QuotaFull { wire: String } | QuotaLeaseRefused { detail: String } @@ -33,24 +35,108 @@ fn quota_policy(term_seconds: Nat) -> LeasePolicy { LeasePolicy { maximum_duration_seconds: term_seconds, release_law: FencedResource } } +// A TERM IS EITHER THE CALLER'S DURATION OR "TO THE END OF THIS WINDOW". A lapsing pool refuses a lease +// that would outlive its window (product.capacity.pool LeaseCrossesLapse), so a caller whose spend is +// an instant -- a request counted the moment it is sent -- cannot know a fixed term that always fits: +// at second 59 of a minute window only one second is left. The window end is read from the SAME +// clock reading the lease is stamped with, so the term fits by construction rather than by luck. +type QuotaTerm + = QuotaTermSeconds { seconds: Nat } + | QuotaTermToWindowEnd + +fn quota_term_to_window_end(limit: UpstreamRateLimit, at: EpochSecs) -> Nat? { + match pool_window_at(pool: quota_root_pool(limit: limit), at: at) { + WindowResolved { window: w } => if w.end > at { checked_int_to_nat(n: w.end - at) } else { none } + WholeLifetime => none + InstantBeforeAnchor { anchor: _, at: _ } => none + } +} + +// THE INSTANT THE CURRENT WINDOW BEGAN, read off the same pool the term is: what a caller needs to tell +// which of its outstanding sends were admitted in an earlier window. +fn quota_window_start(limit: UpstreamRateLimit, at: EpochSecs) -> EpochSecs? { + match pool_window_at(pool: quota_root_pool(limit: limit), at: at) { + WindowResolved { window: w } => Present { value: w.start } + WholeLifetime => none + InstantBeforeAnchor { anchor: _, at: _ } => none + } +} + +fn quota_term_seconds(limit: UpstreamRateLimit, term: QuotaTerm, at: EpochSecs) -> Nat? { + match term { + QuotaTermSeconds { seconds: n } => Present { value: n } + QuotaTermToWindowEnd => quota_term_to_window_end(limit: limit, at: at) + } +} + fn fabric_quota_lease(short_hostname: String, limit: UpstreamRateLimit, amount: Nat, actor: NonEmptyStr, term_seconds: Nat) -> QuotaLease { + fabric_quota_lease_for(short_hostname: short_hostname, limit: limit, amount: amount, actor: actor, term: QuotaTermSeconds { seconds: term_seconds }) +} + +fn fabric_quota_lease_for(short_hostname: String, limit: UpstreamRateLimit, amount: Nat, actor: NonEmptyStr, term: QuotaTerm) -> QuotaLease { match event_log_store_for_host(short_hostname: short_hostname) { HostStoreRefused { detail: d } => QuotaLeaseRefused { detail: d } HostStoreResolved { store: store, executor: _ } => match now_epoch_seconds() { Absent => QuotaLeaseRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => { - let partition = quota_partition(limit: limit) - match fabric_seat_acquire( - store: store, partition: partition, root: quota_root_pool(limit: limit), actor: actor, - request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: amount }, at: now, term_seconds: term_seconds }, - policy: quota_policy(term_seconds: term_seconds), attempts: 3, budget: 4096, - ) { - SeatGranted { grant: g, generation: _, attempts_used: _ } => QuotaLeased { grant: g, partition: partition, store: store } - SeatFull { wire: w, attempts_used: _ } => QuotaFull { wire: join([upstream_rate_limit_key(l: limit) as String, ": ", w], "") } - other => QuotaLeaseRefused { detail: seat_acquisition_wire(a: other) } + Present { value: now } => fabric_quota_lease_at(store: store, now: now, limit: limit, amount: amount, actor: actor, term: term) + } + } +} + +// THE LEASE AT A GIVEN STORE AND INSTANT. fabric_quota_lease_for resolves both from the executing host; +// everything after that -- the term derivation, the seat acquisition, the fold -- is this function, so a +// witness that supplies a store in a temporary directory and an instant runs the PRODUCTION lease rather +// than a constructed QuotaLeased. +fn fabric_quota_lease_at(store: FabricStorageBinding, now: EpochSecs, limit: UpstreamRateLimit, amount: Nat, actor: NonEmptyStr, term: QuotaTerm) -> QuotaLease { + match quota_term_seconds(limit: limit, term: term, at: now) { + Absent => QuotaLeaseRefused { detail: join([upstream_rate_limit_key(l: limit) as String, ": no window end is defined at this instant"], "") } + Present { value: term_seconds } => { + let partition = quota_partition(limit: limit) + match fabric_seat_acquire( + store: store, partition: partition, root: quota_root_pool(limit: limit), actor: actor, + request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: amount }, at: now, term_seconds: term_seconds }, + policy: quota_policy(term_seconds: term_seconds), attempts: 3, budget: 4096, + ) { + SeatGranted { grant: g, generation: _, attempts_used: _ } => QuotaLeased { grant: g, partition: partition, store: store, limit: limit } + SeatFull { wire: w, attempts_used: _ } => QuotaFull { wire: join([upstream_rate_limit_key(l: limit) as String, ": ", w], "") } + other => QuotaLeaseRefused { detail: seat_acquisition_wire(a: other) } + } + } + } +} + +// WHAT THE UPSTREAM SAYS REMAINS IS RECORDED ON THE PARTITION, NOT KEPT BESIDE IT. A fresh local ledger +// knows nothing of requests spent before it existed, or by another client of the same credential; an +// upstream that reports how many remain is the authority on that. The reading is appended as the +// existing PoolUpstreamObserved event, which the fold turns into a hold of the shortfall for the rest +// of the window (product.capacity.pool observe_upstream_remaining) -- so a reading of zero leaves no +// headroom, a partial one leaves exactly that much, and a later, larger reading never gives back what +// an earlier one held. There is no second balance model: the pool is the balance. +type QuotaObservation + = QuotaUpstreamRecorded { partition: PartitionId, remaining: Nat } + | QuotaObservationRefused { detail: String } + +fn fabric_quota_observe_upstream(short_hostname: String, limit: UpstreamRateLimit, remaining: Nat, actor: NonEmptyStr) -> QuotaObservation { + match event_log_store_for_host(short_hostname: short_hostname) { + HostStoreRefused { detail: d } => QuotaObservationRefused { detail: d } + HostStoreResolved { store: store, executor: _ } => + match now_epoch_seconds() { + Absent => QuotaObservationRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match quota_term_to_window_end(limit: limit, at: now) { + Absent => QuotaObservationRefused { detail: join([upstream_rate_limit_key(l: limit) as String, ": no window end is defined at this instant"], "") } + Present { value: term } => { + let partition = quota_partition(limit: limit) + match fabric_pool_transition( + store: store, partition: partition, root: quota_root_pool(limit: limit), actor: actor, at: now, + payload: fn(identity) { PoolUpstreamObserved { remaining: remaining, term_seconds: term } }, attempts: 3, budget: 4096, + ) { + PoolTransitionAppended { id: _ } => QuotaUpstreamRecorded { partition: partition, remaining: remaining } + other => QuotaObservationRefused { detail: pool_transition_wire(t: other) } + } + } } - } } } } @@ -59,32 +145,39 @@ type QuotaSettle = QuotaSettled { partition: PartitionId, actual: Nat } | QuotaSettleRefused { detail: String } -// SETTLEMENT IS ONE PARTITION APPEND, and the observe/append/retry loop it needs is the carrier's -// (gunbc.fabric_event_log fabric_pool_event_append), not a second copy here. It used to be a copy: -// this module held the only one until a compute reservation needed to release, at which point the -// loop would have existed twice and been free to disagree with itself about a contended head. +// SETTLEMENT IS ONE CHECKED PARTITION TRANSITION (gunbc.fabric_event_log fabric_pool_transition): the +// partition is folded and the settlement applied to it before anything is written, under the same +// compare-and-set the acquisition used. It was an unchecked append, which reported success for a +// settlement whose lease had already lapsed -- an event the fold refuses on replay, leaving the whole +// partition unfoldable while the caller believed it had settled. The lease carries its own limit so the +// settlement folds against the pool the lease was granted from. fn fabric_quota_settle(lease: QuotaLease, actual: Nat) -> QuotaSettle { + match now_epoch_seconds() { + Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => fabric_quota_settle_at(lease: lease, actual: actual, now: now) + } +} + +// THE SETTLEMENT AT A GIVEN INSTANT, for the same reason the lease has one: the checked transition and +// everything after the clock read is this function. +fn fabric_quota_settle_at(lease: QuotaLease, actual: Nat, now: EpochSecs) -> QuotaSettle { match lease { QuotaFull { wire: w } => QuotaSettleRefused { detail: join(["nothing to settle: ", w], "") } QuotaLeaseRefused { detail: d } => QuotaSettleRefused { detail: join(["nothing to settle: ", d], "") } - QuotaLeased { grant: g, partition: p, store: store } => - match now_epoch_seconds() { - Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => - match fabric_pool_event_append( - store: store, partition: p, actor: g.reference, at: now, - payload: PoolSettled { reference: g.reference, actual: actual }, attempts: 3, - ) { - PoolEventAppended { id: _ } => QuotaSettled { partition: p, actual: actual } - other => QuotaSettleRefused { detail: pool_event_append_wire(a: other) } - } + QuotaLeased { grant: g, partition: p, store: store, limit: limit } => + match fabric_pool_transition( + store: store, partition: p, root: quota_root_pool(limit: limit), actor: g.reference, at: now, + payload: fn(identity) { PoolSettled { reference: g.reference, actual: actual } }, attempts: 3, budget: 4096, + ) { + PoolTransitionAppended { id: _ } => QuotaSettled { partition: p, actual: actual } + other => QuotaSettleRefused { detail: pool_transition_wire(t: other) } } } } fn quota_lease_wire(l: QuotaLease) -> String { match l { - QuotaLeased { grant: g, partition: p, store: _ } => join(["leased ", g.reference as String, " on ", p as String, " fence=", g.fence.grant as String, "@", to_string(g.fence.generation), " expires_at=", to_string(g.expires_at)], "") + QuotaLeased { grant: g, partition: p, store: _, limit: _ } => join(["leased ", g.reference as String, " on ", p as String, " fence=", g.fence.grant as String, "@", to_string(g.fence.generation), " expires_at=", to_string(g.expires_at)], "") QuotaFull { wire: w } => join(["full ", w], "") QuotaLeaseRefused { detail: d } => join(["refused ", d], "") } diff --git a/dag/gunbc/fleet/host_credential_custody_converge.dag b/dag/gunbc/fleet/host_credential_custody_converge.dag index fdcc603ac5b..8337d63e2a3 100644 --- a/dag/gunbc/fleet/host_credential_custody_converge.dag +++ b/dag/gunbc/fleet/host_credential_custody_converge.dag @@ -18,7 +18,32 @@ import gunbc.clock_read { clock_now_probed_at, probed_at_word } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, fleet_converge_expected_host_env_name } import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } import gunbc.fleet_known_hosts_anchor { FleetSshExecutionContext, SshTarget } -import gunbc.fleet_host_identity { operator_host_srv1 } +import gunbc.fleet_host_identity { operator_host_srv1, operator_host_srv2 } +import gunbc.cursor_harness_credential { cursor_api_key_harness_secret_ref, cursor_worker_turn_api_key_host_path, cursor_worker_turn_custody_host } +import gunbc.codex_harness_credential { + codex_auth_harness_secret_ref, + codex_worker_turn_auth_host_path, + codex_worker_turn_custody_host, + codex_harness_auth_read, + CodexHarnessAuthAdmitted, + CodexHarnessAuthRefused, + codex_harness_write_back_decision, + CodexHarnessWriteBackDecision, + CodexHarnessWriteBackNotOwed, + CodexHarnessWriteBackOwed, + CodexHarnessWriteBackRefused, + codex_harness_write_back_refusal_wire, + codex_harness_add_version, + CodexHarnessWriteBackReceipt, + CodexHarnessVersionAdded, + CodexHarnessStoreUnchanged, + CodexHarnessVersionAddRefused, + CodexHarnessVersionAddAmbiguous, + CodexHarnessWriteBackWithheld, + codex_harness_write_back_receipt_wire, +} +import std.types { Secret } +import extdeps.filesystem.filesystem_io { FilesystemFileRead, FilesystemFileIndeterminate, FilesystemFileAbsent, FilesystemFileObservationsDisagree, FilesystemFileSubjectRefused } import gunbc.fleet_posix_accounts { fleet_posix_root_user } import gunbc.file_access { FileTreeScope, path_within_file_tree_scope } import gunbc.auth.github_apps { gunbai_ci_app_pem_secret } @@ -50,6 +75,7 @@ import gunbc.typed_remote_file_write { RemoteFileMode, remote_file_mode_octal, RemoteFileModeOwnerRead, + RemoteFileModeOwnerReadWrite, RemoteFileModeOwnerReadWriteGroupRead, RemoteFileViaNonInteractiveSudo, TypedRemoteFileWrite, @@ -62,12 +88,15 @@ import gunbc.typed_remote_file_write { TypedRemoteFileStepLegRefused, typed_remote_file_write_seal, typed_remote_file_staging_path, + typed_remote_file_stage_argv, + typed_remote_file_write_stdin_payload, typed_remote_file_compare_argv, typed_remote_file_step, classify_typed_remote_file_comparison, remote_file_privileged_argv, converge_typed_remote_file, } +import extdeps.tools.chmod { chmod_path_resolved_program } import gunbc.runner_host_deploy { runner_host_spec_for, RunnerHostSpecFound, RunnerHostSpecAbsent } import gunbc.runner_host_grants { bootstrap_principal_is_not_the_job_user } import gunbc.fleet_bootstrap_principal { executor_bootstrap_principal } @@ -168,21 +197,49 @@ type HostCustodyCredential = ControllerAppKey | ApprovalNtfyPublisherToken | FabricStateWriterKey + | CursorWorkerTurnApiKey + | CodexWorkerTurnAuth | OracleOciApiSigningKey | ClaudeCodeOauthHarnessToken -data host_custody_credential_roster: List = [ControllerAppKey, ApprovalNtfyPublisherToken, FabricStateWriterKey, OracleOciApiSigningKey, ClaudeCodeOauthHarnessToken] +data host_custody_credential_roster: List = [ControllerAppKey, ApprovalNtfyPublisherToken, FabricStateWriterKey, CursorWorkerTurnApiKey, CodexWorkerTurnAuth, OracleOciApiSigningKey, ClaudeCodeOauthHarnessToken] fn host_custody_credential_wire(c: HostCustodyCredential) -> String { match c { ControllerAppKey => "controller_app_key" ApprovalNtfyPublisherToken => "approval_ntfy_publisher_token" FabricStateWriterKey => "fabric_state_writer_key" + CursorWorkerTurnApiKey => "cursor_worker_turn_api_key" + CodexWorkerTurnAuth => "codex_worker_turn_auth" OracleOciApiSigningKey => "oracle_oci_api_signing_key" ClaudeCodeOauthHarnessToken => "claude_code_oauth_harness_token" } } +// WHO HOLDS THE CURRENT VALUE AFTER PLACEMENT. Every row but one is store-authoritative: the secret +// is the truth and the host copy is converged to it. The codex subscription auth.json is not: codex +// rotates its refresh token on the host while it runs (extdeps.llm.codex_auth +// codex_source_refresh_persistence), so after placement the HOST holds the live token and the store +// lags until this converge reads the host file back and adds it as a new version. Overwriting that +// host file from the store would put a spent token back, and so would placing the store's copy on a +// second host. That is why a host-authoritative row must name exactly one host (placement refuses +// otherwise) and why its converge never writes over a file that is already there. +type CustodyAuthority + = CustodyStoreAuthoritative + | CustodyHostAuthoritativeAfterPlacement + +fn host_custody_authority(c: HostCustodyCredential) -> CustodyAuthority { + match c { + ControllerAppKey => CustodyStoreAuthoritative + ApprovalNtfyPublisherToken => CustodyStoreAuthoritative + FabricStateWriterKey => CustodyStoreAuthoritative + CursorWorkerTurnApiKey => CustodyStoreAuthoritative + CodexWorkerTurnAuth => CustodyHostAuthoritativeAfterPlacement + OracleOciApiSigningKey => CustodyStoreAuthoritative + ClaudeCodeOauthHarnessToken => CustodyStoreAuthoritative + } +} + data host_custody_credential_options: List = map(host_custody_credential_roster, c => host_custody_credential_wire(c: c)) type HostCustodyCredentialSelection @@ -204,11 +261,13 @@ fn select_host_custody_credential(supplied: String) -> HostCustodyCredentialSele // and whether it grants the group read -- four facts, one match each, no octal parameter anywhere. type CustodyFileMode = CustodyFileOwnerRead + | CustodyFileOwnerReadWrite | CustodyFileOwnerReadWriteGroupRead fn custody_file_remote_mode(m: CustodyFileMode) -> RemoteFileMode { match m { CustodyFileOwnerRead => RemoteFileModeOwnerRead + CustodyFileOwnerReadWrite => RemoteFileModeOwnerReadWrite CustodyFileOwnerReadWriteGroupRead => RemoteFileModeOwnerReadWriteGroupRead } } @@ -226,6 +285,7 @@ fn custody_file_exact_perm(m: CustodyFileMode) -> String { fn custody_file_forbidden_perm(m: CustodyFileMode) -> String { match m { CustodyFileOwnerRead => "/077" + CustodyFileOwnerReadWrite => "/077" CustodyFileOwnerReadWriteGroupRead => "/037" } } @@ -233,6 +293,7 @@ fn custody_file_forbidden_perm(m: CustodyFileMode) -> String { fn custody_file_grants_group(m: CustodyFileMode) -> Bool { match m { CustodyFileOwnerRead => false + CustodyFileOwnerReadWrite => false CustodyFileOwnerReadWriteGroupRead => true } } @@ -306,6 +367,16 @@ data custody_root_owned_ntfy_group: PosixOwnerNames = PosixOwnerNames { group: approval_ntfy_principal_name, } +data custody_operator_only: PosixOwnerNames = PosixOwnerNames { + user: fleet_posix_operator_user.name, + group: fleet_posix_operator_user.name, +} + +// THE WORKER-TURN CREDENTIALS live on srv2, the host the declared Cursor offer and the codex harness +// turns dispatch on, owned by the operator account the dispatch spawns run as. Cursor's key is +// read-only on the host. The codex directory IS the turn's CODEX_HOME: codex writes auth.json, +// sessions and logs beneath it, so the directory and the file are owner read-write. + // OracleOciApiSigningKey signs every OCI request gunbc.oracle_oci.compartment_ensure makes, through // the signer's own key reference, never re-spelled. It is delivered to srv1 only, root-owned and // owner-read, because srv1 is where the OCI converge runs and nothing else signs with it. @@ -330,6 +401,26 @@ fn host_credential_custody_row(c: HostCustodyCredential) -> HostCredentialCustod dir_ownership: custody_root_only, dir_mode: DirectoryOwnerOnly, } + CursorWorkerTurnApiKey => HostCredentialCustodyRow { + credential: c, + secret: cursor_api_key_harness_secret_ref, + path: cursor_worker_turn_api_key_host_path, + host_scope: CustodyOnlyOnHost { host: cursor_worker_turn_custody_host }, + file_ownership: custody_operator_only, + file_mode: CustodyFileOwnerRead, + dir_ownership: custody_operator_only, + dir_mode: DirectoryOwnerOnly, + } + CodexWorkerTurnAuth => HostCredentialCustodyRow { + credential: c, + secret: codex_auth_harness_secret_ref, + path: codex_worker_turn_auth_host_path, + host_scope: CustodyOnlyOnHost { host: codex_worker_turn_custody_host }, + file_ownership: custody_operator_only, + file_mode: CustodyFileOwnerReadWrite, + dir_ownership: custody_operator_only, + dir_mode: DirectoryOwnerOnly, + } OracleOciApiSigningKey => HostCredentialCustodyRow { credential: c, secret: oci_api_signing_key_secret_ref, @@ -373,6 +464,7 @@ type CustodyPlacementRefusal | CustodyPathNoDedicatedDirectory { path: NonEmptyStr } | CustodyGroupCannotTraverse { path: NonEmptyStr, dir: NonEmptyStr, group: NonEmptyStr, file_perm: String, dir_perm: String } | CustodyDirectoryGroupDiffers { path: NonEmptyStr, file_group: NonEmptyStr, dir_group: NonEmptyStr } + | CustodyHostAuthoritativeNotSingleHost { path: NonEmptyStr } type CustodyPlacement = CustodyPlaced { row: HostCredentialCustodyRow, key_dir: NonEmptyStr, upper_ancestors: List } @@ -390,6 +482,8 @@ fn custody_placement_refusal_wire(r: CustodyPlacementRefusal) -> String { join(["custody row for ", p as String, " grants group ", g as String, " read (", fp, ") but its directory ", d as String, " is declared ", dp, ", which denies that group traversal -- the reader could not open the file the readback would report as held"], "") CustodyDirectoryGroupDiffers { path: p, file_group: fg, dir_group: dg } => join(["custody row for ", p as String, " grants group ", fg as String, " read but its directory is group ", dg as String, ", so the file's group has no traversal through it"], "") + CustodyHostAuthoritativeNotSingleHost { path: p } => + join(["custody row for ", p as String, " is host-authoritative after placement and must name exactly one host; a second host would receive the store's lagging, possibly spent, copy"], "") } } @@ -404,7 +498,17 @@ fn custody_row_placement(row: HostCredentialCustodyRow) -> CustodyPlacement { let key_path = row.path let ancestors = runner_host_path_ancestors(path: key_path as String) let dir = last_or_empty(xs: ancestors) - if path_within_file_tree_scope(scope: custody_forbidden_tree, path: key_path) { + let host_authoritative_on_many = match host_custody_authority(c: row.credential) { + CustodyStoreAuthoritative => false + CustodyHostAuthoritativeAfterPlacement => + match row.host_scope { + CustodyAnyRunnerHost => true + CustodyOnlyOnHost { host: _ } => false + } + } + if host_authoritative_on_many { + CustodyPlacementRefused { cause: CustodyHostAuthoritativeNotSingleHost { path: key_path } } + } else if path_within_file_tree_scope(scope: custody_forbidden_tree, path: key_path) { CustodyPlacementRefused { cause: CustodyPathInForbiddenTree { path: key_path } } } else if !path_within_file_tree_scope(scope: runner_host_file_scope, path: key_path) { CustodyPlacementRefused { cause: CustodyPathOutsideScope { path: key_path } } @@ -668,12 +772,78 @@ fn host_credential_custody(row: HostCredentialCustodyRow, readback: CustodyReadb type CustodyAction = CustodyLeaveInPlace | CustodyWrite + | CustodyWriteBackToStore { payload: Secret } + | CustodyRepairMetadataInPlace + | CustodyCreateOnly | CustodyActionRefused { reason: String } -// The directory and the ancestors are decided first because a write cannot repair them and writing -// into a directory that is not as declared could publish the credential where an undeclared -// principal holds bits on it. An unsafe FILE is repaired by the write: the staged sibling is -// created by root, moded from the arm, renamed over it, and then owned from the row. +// THE HOST-AUTHORITATIVE DECISION, layered on the store-authoritative one rather than beside it: the +// directory, ancestor and metadata decisions are the same, and only a file that is already there and +// DIFFERS from the store is decided differently. It is never overwritten. If its refresh token moved +// on, it is owed to the store; if not, the host copy is merely fresher in its short-lived tokens and +// is left alone; if it now names another account or cannot be read, the converge refuses. +fn host_authoritative_differing_action(decision: CodexHarnessWriteBackDecision?) -> CustodyAction { + match decision { + Absent => CustodyActionRefused { reason: "host-authoritative file differs from the store and was not read back" } + Present { value: d } => + match d { + CodexHarnessWriteBackNotOwed => CustodyLeaveInPlace + CodexHarnessWriteBackOwed { payload } => CustodyWriteBackToStore { payload: payload } + CodexHarnessWriteBackRefused { cause } => + CustodyActionRefused { reason: concat("host file not written back: ", codex_harness_write_back_refusal_wire(cause: cause)) } + } + } +} + +fn host_authoritative_custody_action(base: CustodyAction, content: RunnerHostFileStanding, decision: CodexHarnessWriteBackDecision?) -> CustodyAction { + match base { + CustodyActionRefused { reason: _ } => base + CustodyWriteBackToStore { payload: _ } => base + CustodyRepairMetadataInPlace => base + CustodyCreateOnly => base + CustodyLeaveInPlace => host_authoritative_settled_action(base: base, content: content, decision: decision) + CustodyWrite => host_authoritative_settled_action(base: base, content: content, decision: decision) + } +} + +fn host_authoritative_settled_action(base: CustodyAction, content: RunnerHostFileStanding, decision: CodexHarnessWriteBackDecision?) -> CustodyAction { + match content { + HostFileDiffers => host_authoritative_differing_action(decision: decision) + HostFileIdentical => host_authoritative_metadata_action(base: base) + HostFileAbsent => host_authoritative_first_placement_action(base: base) + HostFileUnobservable { reason: _ } => base + } +} + +// A HOST-AUTHORITATIVE FILE IS NEVER REPLACED FROM THE STORE, whatever the reason the ordinary +// decision gave. Identical content with wrong owner or mode is repaired in place (chown and chmod on +// the path, no content written): the store-to-host content write stages the STORE bytes and renames +// them over the live file, so a refresh that moved the host file on between observation and rename +// would be lost -- the rename is outside the codex turn lock. +fn host_authoritative_metadata_action(base: CustodyAction) -> CustodyAction { + match base { + CustodyWrite => CustodyRepairMetadataInPlace + CustodyLeaveInPlace => base + CustodyRepairMetadataInPlace => base + CustodyCreateOnly => base + CustodyWriteBackToStore { payload: _ } => base + CustodyActionRefused { reason: _ } => base + } +} + +// FIRST PLACEMENT PUBLISHES CREATE-ONLY: a file that appears after the absence was observed (codex +// logged in by hand, or a racing converge) makes the publication fail and is left as found. +fn host_authoritative_first_placement_action(base: CustodyAction) -> CustodyAction { + match base { + CustodyWrite => CustodyCreateOnly + CustodyLeaveInPlace => base + CustodyRepairMetadataInPlace => base + CustodyCreateOnly => base + CustodyWriteBackToStore { payload: _ } => base + CustodyActionRefused { reason: _ } => base + } +} + fn custody_action( dir: CustodyPathMetadata, upper_ancestors: RunnerHostPathMetadata, @@ -714,6 +884,9 @@ fn custody_action_wire(a: CustodyAction) -> String { match a { CustodyLeaveInPlace => "left-in-place" CustodyWrite => "written" + CustodyWriteBackToStore { payload: _ } => "written-back-to-store" + CustodyRepairMetadataInPlace => "metadata-repaired-in-place" + CustodyCreateOnly => "created-only" CustodyActionRefused { reason: r } => concat("REFUSED ", r) } } @@ -735,6 +908,14 @@ fn custody_file_chown_argv(row: HostCredentialCustodyRow) -> List { chown_argv(owner: ChownOwnerNames { names: row.file_ownership }, path: row.path as String) } +fn custody_chown_argv_at(row: HostCredentialCustodyRow, path: String) -> List { + chown_argv(owner: ChownOwnerNames { names: row.file_ownership }, path: path) +} + +fn custody_chmod_argv_at(row: HostCredentialCustodyRow, path: String) -> List { + [chmod_path_resolved_program as String, remote_file_mode_octal(mode: custody_file_remote_mode(m: row.file_mode)), path] +} + fn staging_remove_argv(staging: String) -> List { ["rm", "-f", "--", staging] } @@ -958,6 +1139,7 @@ type CustodyRun { sm_version: SmResolvedVersionIdentity action: CustodyAction write_refusal: String? + store_write_back: CodexHarnessWriteBackReceipt? readback: CustodyReadback } @@ -1005,9 +1187,134 @@ fn write_and_own(subject: CustodyHost, context: FleetSshExecutionContext, row: H } } +type CustodyLeg { + argv: List + stdin_payload: String +} + +fn run_legs_until_refused(subject: CustodyHost, context: FleetSshExecutionContext, legs: List, step: String) -> String? { + fold(legs, init: none, f: (acc, leg) => + match acc { + Present { value: _ } => acc + Absent => + match run_elevated_leg(subject: subject, context: context, argv: leg.argv, stdin_payload: leg.stdin_payload) { + ElevatedLegRefused { reason: r } => Present { value: join(["step=", step, " not run: ", r], "") } + ElevatedLegRan { exit_code: c, stdout: _, stderr: e } => + if c != 0 { Present { value: join(["step=", step, " exit=", to_string(c), " ", trim(s: e)], "") } } else { none } + } + } + ) +} + +// METADATA ONLY: owner then mode on the existing path; the content is never touched. +fn repair_metadata_in_place(subject: CustodyHost, context: FleetSshExecutionContext, row: HostCredentialCustodyRow) -> String? { + run_legs_until_refused( + subject: subject, context: context, step: "metadata", + legs: [ + CustodyLeg { argv: custody_chown_argv_at(row: row, path: row.path as String), stdin_payload: "" }, + CustodyLeg { argv: custody_chmod_argv_at(row: row, path: row.path as String), stdin_payload: "" }, + ], + ) +} + +// CREATE-ONLY PUBLICATION: stage the store bytes beside the path, give the STAGED file the row's final +// owner and mode, then publish with a hard link, which fails when the destination exists rather +// than replacing it. A file that appeared after the absence observation therefore survives and the +// converge refuses; the staging sibling is removed afterwards by the shared staging cleanup. +fn create_only_publish(subject: CustodyHost, context: FleetSshExecutionContext, row: HostCredentialCustodyRow, write: TypedRemoteFileWrite) -> String? { + let staging = typed_remote_file_staging_path(write: write) + run_legs_until_refused( + subject: subject, context: context, step: "create-only", + legs: [ + CustodyLeg { argv: typed_remote_file_stage_argv(write: write), stdin_payload: typed_remote_file_write_stdin_payload(write: write) }, + CustodyLeg { argv: custody_chown_argv_at(row: row, path: staging), stdin_payload: "" }, + CustodyLeg { argv: custody_chmod_argv_at(row: row, path: staging), stdin_payload: "" }, + CustodyLeg { argv: ["ln", "-T", staging, row.path as String], stdin_payload: "" }, + ], + ) +} + // THE ANCESTORS ARE OBSERVED BEFORE ANY MUTATION (gunbc#11902): ensure_custody_dir creates, so it // runs only once the ancestors read root-only; otherwise the directory is only observed and the same // pure custody_action decides. +// THE HOST FILE AS THE HOST HOLDS IT, read only for a host-authoritative row whose content already +// differs from the store. A failed read is an indeterminate observation, never an empty file. +fn read_host_custody_file(subject: CustodyHost, context: FleetSshExecutionContext, path: String) -> extdeps.filesystem.filesystem_io.FilesystemFileObservation +{ + match run_elevated_leg(subject: subject, context: context, argv: ["cat", path], stdin_payload: "") { + ElevatedLegRefused { reason: r } => FilesystemFileIndeterminate { cause: concat("host file read not run: ", r) } + ElevatedLegRan { exit_code: c, stdout: o, stderr: e } => + if c != 0 { + FilesystemFileIndeterminate { cause: join(["host file read exit=", to_string(c), " ", trim(s: e)], "") } + } else { + FilesystemFileRead { path: path, content: o } + } + } +} + +type HostAuthoritativeReading { + decision: CodexHarnessWriteBackDecision? + host_content: String? +} + +fn read_host_authoritative( + subject: CustodyHost, + context: FleetSshExecutionContext, + row: HostCredentialCustodyRow, + store_payload: Secret, + content: RunnerHostFileStanding, +) -> HostAuthoritativeReading +{ + match content { + HostFileDiffers => + match codex_harness_auth_read(content: store_payload as String) { + CodexHarnessAuthRefused { cause: _ } => HostAuthoritativeReading { decision: none, host_content: none } + CodexHarnessAuthAdmitted { identity } => { + let observed = read_host_custody_file(subject: subject, context: context, path: row.path as String) + HostAuthoritativeReading { + decision: Present { value: codex_harness_write_back_decision(materialized: identity, read_back: observed) }, + host_content: host_custody_file_content(observed: observed), + } + } + } + HostFileIdentical => HostAuthoritativeReading { decision: none, host_content: none } + HostFileAbsent => HostAuthoritativeReading { decision: none, host_content: none } + HostFileUnobservable { reason: _ } => HostAuthoritativeReading { decision: none, host_content: none } + } +} + +fn host_custody_file_content(observed: extdeps.filesystem.filesystem_io.FilesystemFileObservation) -> String? { + match observed { + FilesystemFileRead { path: _, content: c } => Present { value: c } + FilesystemFileAbsent(_) => none + FilesystemFileIndeterminate { cause: _ } => none + FilesystemFileObservationsDisagree { path: _, cause: _ } => none + FilesystemFileSubjectRefused { directory: _, name: _, cause: _ } => none + } +} + +// The content readback against the bytes the action left authoritative: the store's for a write or a +// leave-in-place on identical content; the host file's own bytes, read again, for a host-authoritative +// copy that was left or written back. The second is a re-read of the host, not an assumption about it. +fn observe_custody_content_against( + subject: CustodyHost, + context: FleetSshExecutionContext, + row: HostCredentialCustodyRow, + key_dir: NonEmptyStr, + store_write: TypedRemoteFileWrite, + authoritative_content: String?, +) -> RunnerHostFileStanding +{ + match authoritative_content { + Absent => observe_custody_content(subject: subject, context: context, write: store_write) + Present { value: c } => + match typed_remote_file_write_seal(path: row.path as FilePath, content: c, allowed_scope: FileTreeScope { root_path: key_dir }) { + TypedRemoteFileWriteAdmissionRefused { cause: why } => HostFileUnobservable { reason: concat("host-authoritative readback not sealed: ", why as String) } + TypedRemoteFileWriteAdmitted { write: w } => observe_custody_content(subject: subject, context: context, write: w) + } + } +} + fn converge_host_credential_custody( subject: CustodyHost, context: FleetSshExecutionContext, @@ -1015,6 +1322,7 @@ fn converge_host_credential_custody( key_dir: NonEmptyStr, upper_ancestors: List, write: TypedRemoteFileWrite, + store_payload: Secret, sm_version: SmResolvedVersionIdentity, ) -> CustodyRun { @@ -1029,10 +1337,39 @@ fn converge_host_credential_custody( } let content_before = observe_custody_content(subject: subject, context: context, write: write) let file_before = observe_custody_path(subject: subject, context: context, path: row.path as String, legs: custody_file_legs(row: row)) - let action = custody_action(dir: dir_before, upper_ancestors: ancestors_before, content: content_before, file: file_before) + let base_action = custody_action(dir: dir_before, upper_ancestors: ancestors_before, content: content_before, file: file_before) + let host_reading = match host_custody_authority(c: row.credential) { + CustodyStoreAuthoritative => HostAuthoritativeReading { decision: none, host_content: none } + CustodyHostAuthoritativeAfterPlacement => + read_host_authoritative(subject: subject, context: context, row: row, store_payload: store_payload, content: content_before) + } + let action = match host_custody_authority(c: row.credential) { + CustodyStoreAuthoritative => base_action + CustodyHostAuthoritativeAfterPlacement => + host_authoritative_custody_action(base: base_action, content: content_before, decision: host_reading.decision) + } let write_refusal = match action { CustodyWrite => write_and_own(subject: subject, context: context, row: row, write: write) + CustodyRepairMetadataInPlace => repair_metadata_in_place(subject: subject, context: context, row: row) + CustodyCreateOnly => create_only_publish(subject: subject, context: context, row: row, write: write) CustodyLeaveInPlace => none + CustodyWriteBackToStore { payload: _ } => none + CustodyActionRefused { reason: _ } => none + } + let store_write_back = match action { + CustodyWriteBackToStore { payload } => Present { value: codex_harness_add_version(token_source: WorkloadIdentityToken, payload: payload) } + CustodyLeaveInPlace => none + CustodyWrite => none + CustodyRepairMetadataInPlace => none + CustodyCreateOnly => none + CustodyActionRefused { reason: _ } => none + } + let authoritative_content = match action { + CustodyWriteBackToStore { payload: _ } => host_reading.host_content + CustodyLeaveInPlace => host_reading.host_content + CustodyRepairMetadataInPlace => host_reading.host_content + CustodyCreateOnly => none + CustodyWrite => none CustodyActionRefused { reason: _ } => none } let staging_after = match admit_staging_cleanup( @@ -1048,7 +1385,10 @@ fn converge_host_credential_custody( let file_after = observe_custody_path(subject: subject, context: context, path: row.path as String, legs: custody_file_legs(row: row)) let dir_after = observe_custody_path(subject: subject, context: context, path: key_dir as String, legs: custody_dir_legs(row: row)) let upper_ancestors_after = observe_upper_ancestors(subject: subject, context: context, paths: upper_ancestors) - let content_after = observe_custody_content(subject: subject, context: context, write: write) + let content_after = observe_custody_content_against( + subject: subject, context: context, row: row, key_dir: key_dir, + store_write: write, authoritative_content: authoritative_content, + ) let readback = CustodyReadback { file: file_after, dir: dir_after, @@ -1062,6 +1402,7 @@ fn converge_host_credential_custody( sm_version: sm_version, action: action, write_refusal: write_refusal, + store_write_back: store_write_back, readback: readback, } } @@ -1071,16 +1412,30 @@ fn custody_run_settled(run: CustodyRun) -> Bool { CustodyActionRefused { reason: _ } => true CustodyLeaveInPlace => false CustodyWrite => false + CustodyRepairMetadataInPlace => false + CustodyCreateOnly => false + CustodyWriteBackToStore { payload: _ } => false } let write_failed = match run.write_refusal { Present { value: _ } => true Absent => false } + let store_write_failed = match run.store_write_back { + Absent => false + Present { value: r } => + match r { + CodexHarnessVersionAdded { version_name: _ } => false + CodexHarnessStoreUnchanged => true + CodexHarnessVersionAddRefused { detail: _ } => true + CodexHarnessVersionAddAmbiguous { detail: _ } => true + CodexHarnessWriteBackWithheld { cause: _ } => true + } + } let held = match host_credential_custody(row: run.row, readback: run.readback) { CustodyHeld => true CustodyNotHeld { causes: _ } => false } - !refused_before && !write_failed && held + !refused_before && !write_failed && !store_write_failed && held } // NEVER THE BYTES. Every field below is metadata or a judgment; the content is named only by the @@ -1097,6 +1452,10 @@ fn custody_receipt(run: CustodyRun) -> String { Present { value: w } => concat(" write=REFUSED ", w) Absent => "" }, + match run.store_write_back { + Present { value: r } => concat(" store=", codex_harness_write_back_receipt_wire(r: r)) + Absent => "" + }, "\n file=", custody_path_metadata_wire(m: run.readback.file, legs: custody_file_legs(row: run.row)), "\n dir=", custody_path_metadata_wire(m: run.readback.dir, legs: custody_dir_legs(row: run.row)), "\n ancestors=", runner_host_path_metadata_wire(m: run.readback.upper_ancestors), @@ -1207,7 +1566,7 @@ fn host_credential_custody_converge_ci_wet() -> ProcessExit FleetSshContextReady { context: context, receipt: context_receipt } => { let run = converge_host_credential_custody( subject: subject, context: context, placement_row: placed, key_dir: kd, - upper_ancestors: ua, write: w, sm_version: version, + upper_ancestors: ua, write: w, store_payload: material, sm_version: version, ) write_receipt_then_exit( body: join([stamp, "\n", context_receipt, custody_receipt(run: run)], ""), diff --git a/dag/gunbc/harness/harness_cli.dag b/dag/gunbc/harness/harness_cli.dag index b4860965697..defa14d04d7 100644 --- a/dag/gunbc/harness/harness_cli.dag +++ b/dag/gunbc/harness/harness_cli.dag @@ -14,7 +14,7 @@ import std.process { ProcessExit, ExitSuccess, ExitFailure } import gunbc.harness.harness_tool { HarnessToolEnvironment } import gunbc.harness.harness_turn { HarnessTurnConfig, HarnessTurnOutcome, TurnProviderStopped, TurnCompletedByArtifact, TurnAlignmentCheckpointDue, harness_tool_contract_wire, - CompletesOnProviderStop, CompletesWhenEntryPresent, harness_run_turn, harness_outcome_label, + CompletesOnProviderStop, CompletesWhenEntryPresent, harness_run_turn, harness_outcome_label, TurnHostedQuotaFull, TurnHostedQuotaUnleasable, TurnHostedRateLimited, harness_outcome_detail, harness_event_placement_waiting, } import gunbc.harness.harness_guidance { @@ -29,15 +29,16 @@ import gunbc.harness.harness_supervisor_guidance { supervisor_system_prompt_at } import gunbc.harness.harness_goal_audit_guidance { harness_auditor_guidance_at, audit_guidance_prose } import gunbc.roadmap.roadmap_review_role { review_verdict_shape_text } import gunbc.harness.harness_wire { harness_context_budget_for } -import gunbc.harness.harness_reasoning_wire { HarnessServingUnit } +import gunbc.harness.harness_reasoning_wire { HarnessServingUnit, NemotronUltraFreeOnOpenRouter } +import gunbc.harness.harness_hosted_route { HarnessBackendAccess, BackendAdmittedBySeat, harness_hosted_backend_access, harness_hosted_access_wire } import gunbc.serving.turn_admission { ServingCoTenancyClass, InteractiveQualitySensitive, BatchQualityTolerant, serving_co_tenancy_class_wire, } import gunbc.harness.harness_seat_pointer { harness_seat_pointer_path, harness_seat_pointer_json } import gunbc.serving.caller_retry_contract { caller_backlog_wire, serving_route_caller_backlog } import gunbc.harness.harness_seat { harness_turn_request, harness_seat_pointer_of, - HarnessSeatBinding, HarnessSeatBound, HarnessSeatPending, HarnessSeatRefused, harness_bind_seat, - HarnessSeatRelease, HarnessSeatReleased, HarnessSeatReleaseRefused, harness_release_seat, + HarnessSeatBinding, HarnessSeatBound, HarnessSeatPending, HarnessSeatRefused, HarnessHostedBound, harness_bind_seat, harness_bind_hosted, + HarnessSeatRelease, HarnessSeatReleased, HarnessHostedCredentialRemoved, HarnessSeatReleaseRefused, harness_release_seat, harness_seat_release_wire, harness_release_held, harness_release_capability, HarnessReleaseCapability, harness_release_capability_wire, ServingModelRequirement, AnyAdmittedModel, @@ -152,7 +153,9 @@ data harness_default_wire_shape: HarnessWireShape = OpenAiChatCompletionsShape // One longer round would have ended the run through the truncation refusal, correctly. At the // declared floor 16,384 tokens is about eleven minutes of decode, and the deadline below follows // by derivation rather than being re-guessed beside it. -data harness_default_max_tokens: Int = token_count_value(t: token_count(count: 16384)) +data harness_default_max_token_count: TokenCount = token_count(count: 16384) + +data harness_default_max_tokens: Int = token_count_value(t: harness_default_max_token_count) // THE SLOWEST DECODE THIS DEPLOYMENT IS EXPECTED TO SUSTAIN, in tokens per second. It is a declared // policy floor rather than a measurement of any one run: observed rates here span 2.4 tok/s under @@ -206,6 +209,7 @@ fn harness_executor_short_hostname() -> String { fn harness_turn_exit_released(outcome: HarnessTurnOutcome, seat: HarnessSeatRelease) -> ProcessExit { match seat { HarnessSeatReleased { partition: _ } => harness_turn_exit(outcome: outcome) + HarnessHostedCredentialRemoved => harness_turn_exit(outcome: outcome) HarnessSeatReleaseRefused { detail: d } => ExitFailure { code: 1, reason: join([harness_outcome_label(outcome: outcome), "; seat ", harness_seat_release_wire(r: seat)], "") } } @@ -236,22 +240,23 @@ fn harness_macbook_environment() -> HarnessToolEnvironment { // field the way it takes the worktree, the timeout program and its two paths -- so the same loop // runs against a different serving surface by being handed a different number, not by importing a // different authority. -fn harness_macbook_config(backend_url: String, model: NonEmptyStr, unit: HarnessServingUnit) -> HarnessTurnConfig { +fn harness_macbook_config(backend: HarnessBoundBackend) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: backend_url, - model: model as String, + backend_url: backend.url, + model: backend.model as String, max_tokens: harness_default_max_tokens, - serving_unit: unit, - reasoning_mode: harness_reasoning_mode, + serving_unit: backend.unit, + reasoning_mode: backend.reasoning, wire_shape: harness_default_wire_shape, system_prompt: harness_default_system_prompt, alignment_chain_fingerprint: "", request_scratch_path: harness_request_scratch_path, events_path: harness_probe_events_path, - context_budget: harness_context_budget_for(unit: unit), + context_budget: harness_context_budget_for(unit: backend.unit), request_deadline: harness_request_deadline, environment: harness_macbook_environment(), completion: CompletesOnProviderStop, + backend_access: backend.access, } } @@ -262,9 +267,9 @@ fn harness_macbook_config(backend_url: String, model: NonEmptyStr, unit: Harness // posted, received a tool_use, executed it, posted the result back, and received a terminal. data harness_probe_prompt: String = "How many files in the dag/gunbc/harness directory of this repository end in .dag? Use run_command to find out, then state the number." -fn harness_probe_outcome(backend_url: String, model: NonEmptyStr, unit: HarnessServingUnit) -> HarnessTurnOutcome { +fn harness_probe_outcome(backend: HarnessBoundBackend) -> HarnessTurnOutcome { harness_run_turn( - config: harness_macbook_config(backend_url: backend_url, model: model, unit: unit), + config: harness_macbook_config(backend: backend), prompt: harness_probe_prompt, step_budget: harness_default_step_budget, ) @@ -291,6 +296,9 @@ fn harness_turn_exit(outcome: HarnessTurnOutcome) -> ProcessExit { TurnThinkingFormatInvalid { cause: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) TurnUsageUnavailable { cause: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) TurnSubmissionUndecodable { reason: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) + TurnHostedQuotaFull { wire: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) + TurnHostedQuotaUnleasable { detail: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) + TurnHostedRateLimited { refusal: _, steps_taken: _ } => harness_turn_refusal_exit(outcome: outcome) TurnAlignmentCheckpointDue { steps_taken: _ } => ExitFailure { code: 3, reason: harness_outcome_label(outcome: outcome) } } @@ -367,6 +375,7 @@ fn harness_bind_seat_waiting( } HarnessSeatRefused { detail: _ } => acc HarnessSeatBound { url: _, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: _, unit: _, class: _, seat: _ } => acc + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: _ } => acc }, ) waited.binding @@ -500,6 +509,8 @@ fn harness_place_for(request: String, attempt: String, receipt: String) -> Proce SeatPermitMinted { permit: permit } => harness_place_receipt(binding: binding, url: url, group_wire: group_wire, bound_model: bound_model, launch: launch, capability: capability, grant: grant, permit: permit, receipt: receipt) } } + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: _ } => + harness_place_receipt_failed(binding: binding, cause: "an any-model placement answered with a hosted binding, which holds no seat a router can be handed") } } @@ -588,6 +599,7 @@ fn harness_release_admitted(partition: String, reference: String, grant: String, } match harness_release_held(short_hostname: harness_executor_short_hostname(), capability: capability, reason: reason as NonEmptyStr) { HarnessSeatReleased { partition: _ } => ExitSuccess + HarnessHostedCredentialRemoved => ExitFailure { code: 1, reason: join(["harness seat release for ", harness_release_capability_wire(c: capability), " answered as a hosted credential removal; the seat may still be held"], "") } HarnessSeatReleaseRefused { detail: d } => ExitFailure { code: 1, reason: join(["harness seat release refused for ", harness_release_capability_wire(c: capability), ": ", d], "") } } @@ -602,11 +614,11 @@ fn harness_release_admitted(partition: String, reference: String, grant: String, // while something else is in flight. fn harness_probe_cli() -> ProcessExit { let binding = harness_bind_seat(short_hostname: harness_executor_short_hostname(), class: BatchQualityTolerant, requirement: AnyAdmittedModel, shape: harness_default_wire_shape, inputs: harness_model_request_inputs(role: "probe", shape: harness_default_wire_shape, system_prompt: harness_default_system_prompt, user_input: harness_probe_prompt), actor: "probe" as NonEmptyStr, attempt: harness_probe_attempt(label: "probe"), policy: harness_seat_policy) - match binding { - HarnessSeatRefused { detail } => harness_backend_refusal_exit(detail: detail) - HarnessSeatPending { detail } => harness_placement_pending_exit(detail: detail) - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { - let outcome = harness_probe_outcome(backend_url: url, model: bound_model, unit: bound_unit) + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { + let outcome = harness_probe_outcome(backend: backend) let seat_release = harness_release_seat(binding: binding, reason: harness_outcome_label(outcome: outcome) as NonEmptyStr) harness_turn_exit_released(outcome: outcome, seat: seat_release) } @@ -642,24 +654,22 @@ fn harness_worker_config( timeout_program: FilePath, request_scratch_path: FilePath, events_path: FilePath, - backend_url: String, - model: NonEmptyStr, - unit: HarnessServingUnit, + backend: HarnessBoundBackend, system_prompt: String, alignment_chain_fingerprint: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: backend_url, - model: model as String, + backend_url: backend.url, + model: backend.model as String, max_tokens: harness_default_max_tokens, - serving_unit: unit, - reasoning_mode: harness_reasoning_mode, + serving_unit: backend.unit, + reasoning_mode: backend.reasoning, wire_shape: harness_default_wire_shape, system_prompt: system_prompt, alignment_chain_fingerprint: alignment_chain_fingerprint, request_scratch_path: request_scratch_path, events_path: events_path, - context_budget: harness_context_budget_for(unit: unit), + context_budget: harness_context_budget_for(unit: backend.unit), request_deadline: harness_request_deadline, environment: HarnessToolEnvironment { working_directory: worktree, @@ -667,6 +677,7 @@ fn harness_worker_config( duration_limit_seconds: harness_default_duration_limit_seconds, }, completion: CompletesOnProviderStop, + backend_access: backend.access, } } @@ -732,10 +743,10 @@ fn harness_worker_cli( ) -> ProcessExit { let system_prompt = harness_worker_system_prompt_at(worktree: worktree, repo_root: repo_root, gunbc_program: gunbc_program) let binding = harness_bind_seat_waiting(role: "worker", system_prompt: system_prompt, brief: brief, actor: join(["worker:", worktree], "") as NonEmptyStr, attempt: attempt_identity as NonEmptyStr, events_path: events_path) - match binding { - HarnessSeatRefused { detail } => harness_backend_refusal_exit(detail: detail) - HarnessSeatPending { detail } => harness_placement_pending_exit(detail: detail) - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { match harness_seat_pointer_persist(events_path: events_path, binding: binding) { HarnessSeatPointerUnwritten { detail: pointer_detail } => harness_seat_pointer_refused_exit(binding: binding, detail: pointer_detail) HarnessSeatPointerWritten => { @@ -745,9 +756,7 @@ fn harness_worker_cli( timeout_program: timeout_program as FilePath, request_scratch_path: request_scratch_path as FilePath, events_path: events_path as FilePath, - backend_url: url, - model: bound_model, - unit: bound_unit, + backend: backend, system_prompt: system_prompt, alignment_chain_fingerprint: alignment_chain_fingerprint, ), @@ -787,23 +796,21 @@ fn harness_reviewer_config( events_path: FilePath, verdict_dir: String, verdict_name: String, - backend_url: String, - model: NonEmptyStr, - unit: HarnessServingUnit, + backend: HarnessBoundBackend, system_prompt: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: backend_url, - model: model as String, + backend_url: backend.url, + model: backend.model as String, max_tokens: harness_default_max_tokens, - serving_unit: unit, - reasoning_mode: harness_reasoning_mode, + serving_unit: backend.unit, + reasoning_mode: backend.reasoning, wire_shape: harness_default_wire_shape, system_prompt: system_prompt, alignment_chain_fingerprint: "", request_scratch_path: request_scratch_path, events_path: events_path, - context_budget: harness_context_budget_for(unit: unit), + context_budget: harness_context_budget_for(unit: backend.unit), request_deadline: harness_request_deadline, environment: HarnessToolEnvironment { working_directory: worktree, @@ -811,6 +818,7 @@ fn harness_reviewer_config( duration_limit_seconds: harness_default_duration_limit_seconds, }, completion: CompletesWhenEntryPresent { directory: verdict_dir, name: verdict_name }, + backend_access: backend.access, } } @@ -831,10 +839,10 @@ fn harness_reviewer_cli( let verdict_path = join([verdict_dir, "/", verdict_name], "") let system_prompt = harness_reviewer_system_prompt_at(worktree: worktree, verdict_path: verdict_path) let binding = harness_bind_seat_waiting(role: "reviewer", system_prompt: system_prompt, brief: brief, actor: join(["reviewer:", verdict_path], "") as NonEmptyStr, attempt: attempt_identity as NonEmptyStr, events_path: events_path) - match binding { - HarnessSeatRefused { detail } => harness_backend_refusal_exit(detail: detail) - HarnessSeatPending { detail } => harness_placement_pending_exit(detail: detail) - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { match harness_seat_pointer_persist(events_path: events_path, binding: binding) { HarnessSeatPointerUnwritten { detail: pointer_detail } => harness_seat_pointer_refused_exit(binding: binding, detail: pointer_detail) HarnessSeatPointerWritten => { @@ -846,9 +854,7 @@ fn harness_reviewer_cli( events_path: events_path as FilePath, verdict_dir: verdict_dir, verdict_name: verdict_name, - backend_url: url, - model: bound_model, - unit: bound_unit, + backend: backend, system_prompt: system_prompt, ), prompt: brief, @@ -874,23 +880,21 @@ fn harness_auditor_config( events_path: FilePath, verdict_dir: String, verdict_name: String, - backend_url: String, - model: NonEmptyStr, - unit: HarnessServingUnit, + backend: HarnessBoundBackend, system_prompt: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: backend_url, - model: model as String, + backend_url: backend.url, + model: backend.model as String, max_tokens: harness_default_max_tokens, - serving_unit: unit, - reasoning_mode: harness_reasoning_mode, + serving_unit: backend.unit, + reasoning_mode: backend.reasoning, wire_shape: harness_default_wire_shape, system_prompt: system_prompt, alignment_chain_fingerprint: "", request_scratch_path: request_scratch_path, events_path: events_path, - context_budget: harness_context_budget_for(unit: unit), + context_budget: harness_context_budget_for(unit: backend.unit), request_deadline: harness_request_deadline, environment: HarnessToolEnvironment { working_directory: worktree, @@ -898,6 +902,7 @@ fn harness_auditor_config( duration_limit_seconds: harness_default_duration_limit_seconds, }, completion: CompletesWhenEntryPresent { directory: verdict_dir, name: verdict_name }, + backend_access: backend.access, } } @@ -917,10 +922,10 @@ fn harness_auditor_cli( let verdict_path = join([verdict_dir, "/", verdict_name], "") let system_prompt = harness_auditor_system_prompt_at(worktree: worktree, verdict_path: verdict_path) let binding = harness_bind_seat_waiting(role: "auditor", system_prompt: system_prompt, brief: brief, actor: join(["auditor:", verdict_path], "") as NonEmptyStr, attempt: attempt_identity as NonEmptyStr, events_path: events_path) - match binding { - HarnessSeatRefused { detail } => harness_backend_refusal_exit(detail: detail) - HarnessSeatPending { detail } => harness_placement_pending_exit(detail: detail) - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { match harness_seat_pointer_persist(events_path: events_path, binding: binding) { HarnessSeatPointerUnwritten { detail: pointer_detail } => harness_seat_pointer_refused_exit(binding: binding, detail: pointer_detail) HarnessSeatPointerWritten => { @@ -932,9 +937,7 @@ fn harness_auditor_cli( events_path: events_path as FilePath, verdict_dir: verdict_dir, verdict_name: verdict_name, - backend_url: url, - model: bound_model, - unit: bound_unit, + backend: backend, system_prompt: system_prompt, ), prompt: brief, @@ -970,23 +973,21 @@ fn harness_supervisor_config( events_path: FilePath, verdict_dir: String, verdict_name: String, - backend_url: String, - model: NonEmptyStr, - unit: HarnessServingUnit, + backend: HarnessBoundBackend, system_prompt: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: backend_url, - model: model as String, + backend_url: backend.url, + model: backend.model as String, max_tokens: harness_default_max_tokens, - serving_unit: unit, - reasoning_mode: harness_reasoning_mode, + serving_unit: backend.unit, + reasoning_mode: backend.reasoning, wire_shape: harness_default_wire_shape, system_prompt: system_prompt, alignment_chain_fingerprint: "", request_scratch_path: request_scratch_path, events_path: events_path, - context_budget: harness_context_budget_for(unit: unit), + context_budget: harness_context_budget_for(unit: backend.unit), request_deadline: harness_request_deadline, environment: HarnessToolEnvironment { working_directory: worktree, @@ -994,6 +995,7 @@ fn harness_supervisor_config( duration_limit_seconds: harness_default_duration_limit_seconds, }, completion: CompletesWhenEntryPresent { directory: verdict_dir, name: verdict_name }, + backend_access: backend.access, } } @@ -1010,10 +1012,10 @@ fn harness_supervisor_cli( let verdict_path = join([verdict_dir, "/", verdict_name], "") let system_prompt = supervisor_system_prompt_at(worktree: worktree, verdict_path: verdict_path) let binding = harness_bind_seat_waiting(role: "supervisor", system_prompt: system_prompt, brief: brief, actor: join(["supervisor:", verdict_path], "") as NonEmptyStr, attempt: attempt_identity as NonEmptyStr, events_path: events_path) - match binding { - HarnessSeatRefused { detail } => harness_backend_refusal_exit(detail: detail) - HarnessSeatPending { detail } => harness_placement_pending_exit(detail: detail) - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { match harness_seat_pointer_persist(events_path: events_path, binding: binding) { HarnessSeatPointerUnwritten { detail: pointer_detail } => harness_seat_pointer_refused_exit(binding: binding, detail: pointer_detail) HarnessSeatPointerWritten => { @@ -1025,9 +1027,7 @@ fn harness_supervisor_cli( events_path: events_path as FilePath, verdict_dir: verdict_dir, verdict_name: verdict_name, - backend_url: url, - model: bound_model, - unit: bound_unit, + backend: backend, system_prompt: system_prompt, ), prompt: brief, @@ -1045,11 +1045,11 @@ fn harness_supervisor_cli( // with a door beside it. fn harness_probe_report() -> String { let binding = harness_bind_seat(short_hostname: harness_executor_short_hostname(), class: BatchQualityTolerant, requirement: AnyAdmittedModel, shape: harness_default_wire_shape, inputs: harness_model_request_inputs(role: "probe", shape: harness_default_wire_shape, system_prompt: harness_default_system_prompt, user_input: harness_probe_prompt), actor: "probe-report" as NonEmptyStr, attempt: harness_probe_attempt(label: "probe-report"), policy: harness_seat_policy) - match binding { - HarnessSeatRefused { detail } => join(["harness probe refused before the turn: ", detail, "\n"], "") - HarnessSeatPending { detail } => join(["harness probe is pending placement: ", detail, "\n"], "") - HarnessSeatBound { url, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: bound_model, unit: bound_unit, class: _, seat: _ } => { - let outcome = harness_probe_outcome(backend_url: url, model: bound_model, unit: bound_unit) + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => join(["harness probe refused before the turn: ", detail, "\n"], "") + BindingPending { detail } => join(["harness probe is pending placement: ", detail, "\n"], "") + BindingReady { backend } => { + let outcome = harness_probe_outcome(backend: backend) let seat_release = harness_release_seat(binding: binding, reason: harness_outcome_label(outcome: outcome) as NonEmptyStr) join([ "harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n", @@ -1059,3 +1059,90 @@ fn harness_probe_report() -> String { } } } + +// ── ONE BACKEND PER BINDING, WHICHEVER WAY IT WAS ADMITTED ──────────────────────────────────── +// +// Every entry runs the same turn against what its binding names: where requests go, which model and +// unit answer them, at which reasoning level, and how each request is admitted. A fleet seat runs at +// the operator's declared level and posts unauthenticated over the fleet network; a hosted binding runs +// at the level its route SELECTED and leases quota before each request. Deriving that once here is what +// keeps the hosted arm from being a second copy of every entry body. +type HarnessBoundBackend { + url: String + model: NonEmptyStr + unit: HarnessServingUnit + reasoning: HarnessReasoningLevel + access: HarnessBackendAccess +} + +type HarnessBindingAnswer + = BindingRefused { detail: String } + | BindingPending { detail: String } + | BindingReady { backend: HarnessBoundBackend } + +fn harness_binding_answer(binding: HarnessSeatBinding) -> HarnessBindingAnswer { + match binding { + HarnessSeatRefused { detail: d } => BindingRefused { detail: d } + HarnessSeatPending { detail: d } => BindingPending { detail: d } + HarnessSeatBound { url: u, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: m, unit: un, class: _, seat: _ } => + BindingReady { backend: HarnessBoundBackend { url: u, model: m, unit: un, reasoning: harness_reasoning_mode, access: BackendAdmittedBySeat } } + HarnessHostedBound { url: u, model: m, unit: un, level: l, access: a } => + BindingReady { backend: HarnessBoundBackend { url: u, model: m, unit: un, reasoning: l, access: harness_hosted_backend_access(access: a) } } + } +} + +// THE HOSTED PROBE: the probe turn, placed on the hosted Nemotron 3 Ultra free listing by NAME. It is the +// one entry that reaches a hosted unit, because an unnamed requirement never does (gunbc.harness.harness_seat +// harness_requirement_placement): the listing is governed by NVIDIA's trial terms, so reaching it is an +// operator's explicit act, not a placement default. It takes its tool environment as arguments, as the +// worker does, so it runs on whatever host invokes it rather than assuming the MacBook's. +// THE ROUTE'S STANDING IS WRITTEN BESIDE THE EVENTS BEFORE THE HOSTED PROBE'S TURN RUNS: the tier and daily +// counters the key read reported, the quota rows each request is charged to, and the NVIDIA trial terms +// that govern the endpoint. A run whose receipt cannot be written does not run, and gives its credential back. +fn harness_hosted_route_receipt(binding: HarnessSeatBinding) -> String? { + match binding { + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: a } => Present { value: harness_hosted_access_wire(access: a) } + HarnessSeatBound { url: _, grant: _, partition: _, ceiling: _, group_wire: _, launch: _, store: _, model: _, unit: _, class: _, seat: _ } => none + HarnessSeatPending { detail: _ } => none + HarnessSeatRefused { detail: _ } => none + } +} + +fn harness_hosted_probe_cli(worktree: String, timeout_program: String, request_scratch_path: String, events_path: String) -> ProcessExit { + let binding = harness_bind_hosted(short_hostname: harness_executor_short_hostname(), unit: NemotronUltraFreeOnOpenRouter, actor: join(["hosted-probe:", worktree], "") as NonEmptyStr, max_tokens: harness_default_max_token_count) + match harness_binding_answer(binding: binding) { + BindingRefused { detail } => harness_backend_refusal_exit(detail: detail) + BindingPending { detail } => harness_placement_pending_exit(detail: detail) + BindingReady { backend } => { + match harness_hosted_route_receipt(binding: binding) { + Absent => { + let seat_release = harness_release_seat(binding: binding, reason: "the hosted route receipt could not be derived" as NonEmptyStr) + ExitFailure { code: 1, reason: join(["hosted probe: the binding carries no hosted access to describe; ", harness_seat_release_wire(r: seat_release)], "") } + } + Present { value: receipt } => { + let written = Filesystem.Write(path: join([events_path, ".route"], ""), content: concat(receipt, "\n")) + if !written.success { + let seat_release = harness_release_seat(binding: binding, reason: "the hosted route receipt could not be written" as NonEmptyStr) + ExitFailure { code: 1, reason: join(["hosted probe: the route receipt could not be written to ", events_path, ".route: ", written.error, "; ", harness_seat_release_wire(r: seat_release)], "") } + } else { + let outcome = harness_run_turn( + config: harness_worker_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + backend: backend, + system_prompt: harness_default_system_prompt, + alignment_chain_fingerprint: "", + ), + prompt: harness_probe_prompt, + step_budget: harness_default_step_budget, + ) + let seat_release = harness_release_seat(binding: binding, reason: harness_outcome_label(outcome: outcome) as NonEmptyStr) + harness_turn_exit_released(outcome: outcome, seat: seat_release) + } + } + } + } + } +} diff --git a/dag/gunbc/harness/harness_hosted_route.dag b/dag/gunbc/harness/harness_hosted_route.dag new file mode 100644 index 00000000000..84a1d39210e --- /dev/null +++ b/dag/gunbc/harness/harness_hosted_route.dag @@ -0,0 +1,670 @@ +module gunbc.harness.harness_hosted_route + +import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, EpochSecs } +import std.algebra { trim } +import std.nat { Nat } +import std.measure { TokenCount, token_count_value, Second, second, second_count, MoneyAmountMicro, money_amount_micro_count } +import std.checked_arithmetic { checked_int_to_nat } +import std.decl_ref { DeclarationRef, decl_ref, declaration_ref_eq } +import std.interval { degenerate_interval } +import std.pareto { AxisGap, AxisReading, HigherIsBetter, LowerIsBetter, ParetoEntry, SelectionAxis } +import std.decision { + ChoiceDomain, ConstraintSatisfied, CausalModelEstablished, DecisionAssessment, DecisionConstraint, + DecisionRegion, DecisionSubject, DecisionVariable, ContextParameter, HardConstraint, ObjectiveMeasure, + ObjectiveTerm, ParameterBound, PhysicalModelEstablished, RealizationSelectionResult, MonotoneIncreasing, + UniqueSurvivorRequired, VariesAlong, VendorDocumentedCoupling, select_realization, + SelectedWithin, CandidateFieldUnestablished, NoFeasibleRealization, SelectionNeedsEvidence, + SelectionNeedsPolicy, RealizationSelectionRefused, +} +import std.upsert_decision { ObservationVerdict, Converged, Inaccessible } +import extdeps.external_authority { CitedToAuthority } +import extdeps.api_rate_limit { UpstreamRateLimit } +import extdeps.cloud.gcp.secret_ref { SecretRef } +import extdeps.cloud.gcp.secret_manager { SmEnabled } +import extdeps.http.client +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.shell +import extdeps.openrouter.openrouter { + OpenRouterReasoningEffort, openrouter_chat_completions_url, openrouter_key_url, openrouter_authorization_header_line, + openrouter_free_models_per_minute, openrouter_free_models_per_day, openrouter_reasoning_authority, + CreditsTierObserved, CreditsTierUnrecognized, openrouter_credits_from_daily_limit, + OpenRouterKeyRead, OpenRouterKeyDecoded, OpenRouterKeyUndecodable, openrouter_decode_key_read, + KeySpendUncapped, KeySpendCappedAtZero, KeySpendCappedAbove, openrouter_key_spend_limit_wire, + OpenRouterErrorDecoded, OpenRouterErrorUndecodable, openrouter_decode_error_body, openrouter_error_wire, + openrouter_effort_eq, +} +import extdeps.openrouter.nvidia_nemotron_3_ultra_free { + nemotron_ultra_free_max_completion_tokens, nemotron_ultra_free_supported_efforts, + nemotron_ultra_free_price_per_token, nemotron_ultra_free_terms, +} +import extdeps.nvidia.api_trial_terms { NvidiaApiTrialTerms, nvidia_api_trial_terms_wire } +import gunbc.secret_provision { openrouter_free_tier_key_secret_ref } +import gunbc.auth.access_token_source { AccessTokenReady, AccessTokenUpsertRequired, AccessTokenEnsureRefused, resolve_access_token, access_token_upsert_required_reason } +import gunbc.auth.secret_ref_credential { + SecretCredentialReady, SecretCredentialAccessUpsertRequired, SecretCredentialWireDecodeRefused, + SecretCredentialFetchRefused, SecretCredentialResolvedVersionMismatch, fetch_secret_ref_credential_with_token, +} +import gunbc.auth.materialized_secret { + BearerHeaderFile, with_materialized_secret, MaterializationCompleted, MaterializationRefused, + SecretNotLive, SecretVersionUnusable, LivenessProbeNotIndependent, +} +import gunbc.readback_independence { ReadBackProbe, ProbeInert } +import gunbc.fabric_quota { + QuotaLease, QuotaLeased, QuotaFull, QuotaLeaseRefused, fabric_quota_lease_for, QuotaTermToWindowEnd, + fabric_quota_settle, QuotaSettle, QuotaSettled, QuotaSettleRefused, quota_settle_wire, + fabric_quota_observe_upstream, QuotaUpstreamRecorded, QuotaObservationRefused, +} +import gunbc.harness.harness_reasoning_wire { + HarnessServingUnit, DeepseekV4FlashOnVllm, Glm53FlashOnVllm, NemotronUltraFreeOnOpenRouter, + HarnessReasoningLevel, ReasoningOff, ReasoningStandard, ReasoningMaximum, harness_reasoning_level_wire, + harness_openrouter_level_effort, harness_reasoning_kwargs_for, ReasoningKwargsAdmitted, ReasoningLevelUnsupported, + harness_unit_model_id, harness_serving_unit_wire, +} + +// A HOSTED HARNESS ROUTE: a unit the harness reaches through an upstream it does not operate, admitted by +// leasing that upstream's quota rather than by taking a seat on a serving group. The seat model +// (gunbc.harness.harness_seat) is about co-tenancy on an engine with loaded state; a hosted upstream +// meters REQUESTS against a credential, so the honest capacity fact is the per-request quota and its +// carrier is gunbc.fabric_quota over the upstream's published UpstreamRateLimit rows. +// +// FOUR THINGS HAPPEN HERE AND NOWHERE ELSE: the route's facts are joined from their upstream authorities; +// the reasoning level a turn on the route runs at is SELECTED, not written down; the credential is +// materialized and its liveness read before any turn spends it; and each request leases both of the +// upstream's pools before it goes out and settles them after. + +// ── THE ROUTE, JOINED FROM ITS AUTHORITIES ──────────────────────────────────────────────────── + +type HarnessHostedRoute { + unit: HarnessServingUnit + url: NonEmptyStr + key_url: NonEmptyStr + secret: SecretRef + per_minute: UpstreamRateLimit + max_completion: TokenCount + supported_efforts: List + price_per_token: MoneyAmountMicro + terms: NvidiaApiTrialTerms +} + +// A FLEET UNIT HAS NO HOSTED ROUTE, and the answer is absent rather than a default route: a caller that +// reached this function with a fleet unit has mis-dispatched, and the seat binder refuses it by name. +fn harness_hosted_route_for(unit: HarnessServingUnit) -> HarnessHostedRoute? { + match unit { + DeepseekV4FlashOnVllm => none + Glm53FlashOnVllm => none + NemotronUltraFreeOnOpenRouter => Present { value: HarnessHostedRoute { + unit: unit, + url: openrouter_chat_completions_url, + key_url: openrouter_key_url, + secret: openrouter_free_tier_key_secret_ref, + per_minute: openrouter_free_models_per_minute, + max_completion: nemotron_ultra_free_max_completion_tokens, + supported_efforts: nemotron_ultra_free_supported_efforts, + price_per_token: nemotron_ultra_free_price_per_token, + terms: nemotron_ultra_free_terms, + } } + } +} + +// ── THE REASONING LEVEL IS SELECTED, NOT WRITTEN DOWN ───────────────────────────────────────── +// +// The fleet harness runs at a declared level (gunbc.harness.harness_cli harness_reasoning_mode), chosen by +// the operator for an engine whose compute is the fleet's. On a hosted free listing the trade is +// different and stated by the listing itself: the price is zero per token and the limit is per request, +// so a deeper effort costs neither money nor quota -- only latency and the completion budget its thinking +// shares with the answer. That is a selection over a funded field (DESIGN section 3d), so it is asked of +// std.decision rather than answered by a constant. +// +// THE FIELD is the harness's three levels. THE AXES are reasoning depth (higher is better; the rank is the +// effort's position in the listing's supported_efforts, which the reasoning-tokens guide says are +// "returned in descending effort order", with disabled below every effort), money per request and quota +// per request (lower is better; equal across the field by the listing's price and the limits page's +// per-request metering), and admissibility. THE HARD CONSTRAINT is admissibility: a level the unit cannot +// express, or a request whose max_tokens exceeds the listing's completion ceiling, reads 0 and is excluded +// before Pareto, never ranked. +// +// LATENCY IS NOT A FUNDED AXIS, and that is the stated reason the answer is the deepest admitted level: +// nothing in this harness measures per-effort latency on this listing, and a turn's deadline already +// bounds it (gunbc.harness.harness_cli harness_request_deadline). Funding latency without a reading +// would answer SelectionNeedsEvidence, which is the honest arm the day a latency budget binds; until then +// depth dominates on a field where money and quota are equal. +data harness_hosted_selection_module: String = "gunbc.harness.harness_hosted_route" + +fn hosted_ref(name: String) -> DeclarationRef { + decl_ref(module_path: harness_hosted_selection_module, decl_name: name) +} + +data ax_hosted_reasoning_depth: DeclarationRef = hosted_ref(name: "ax_hosted_reasoning_depth") +data ax_hosted_money_per_request: DeclarationRef = hosted_ref(name: "ax_hosted_money_per_request") +data ax_hosted_quota_per_request: DeclarationRef = hosted_ref(name: "ax_hosted_quota_per_request") +data ax_hosted_level_admitted: DeclarationRef = hosted_ref(name: "ax_hosted_level_admitted") + +data harness_hosted_level_axes: List = [ + SelectionAxis { identity: ax_hosted_reasoning_depth, goal: HigherIsBetter }, + SelectionAxis { identity: ax_hosted_money_per_request, goal: LowerIsBetter }, + SelectionAxis { identity: ax_hosted_quota_per_request, goal: LowerIsBetter }, + SelectionAxis { identity: ax_hosted_level_admitted, goal: HigherIsBetter }, +] + +data hosted_level_admitted_constraint: DeclarationRef = hosted_ref(name: "hosted_level_admitted_constraint") + +data harness_hosted_level_constraints: List = [ + HardConstraint { identity: hosted_level_admitted_constraint, axis: ax_hosted_level_admitted, bound_micro: 1000000 }, +] + +fn hosted_level_identity(level: HarnessReasoningLevel) -> DeclarationRef { + hosted_ref(name: join(["hosted_level_", harness_reasoning_level_wire(level: level) as String], "")) +} + +// THE DEPTH RANK, from the listing's own order. Disabled is 0; an effort at index i of an n-long +// descending list ranks n - i; an effort the listing does not carry has no rank, and the level that maps +// to it is inadmissible anyway, so its reading is 0 and the hard constraint is what excludes it. +type HostedRankAcc { seen: Bool, rank: Int } + +fn hosted_effort_rank(efforts: List, effort: OpenRouterReasoningEffort) -> Int { + let acc = fold(efforts, init: HostedRankAcc { seen: false, rank: 0 }, f: (a, e) => { + let seen = a.seen || openrouter_effort_eq(a: e, b: effort) + HostedRankAcc { seen: seen, rank: if seen { a.rank + 1 } else { 0 } } + }) + acc.rank +} + +fn hosted_level_depth_rank(route: HarnessHostedRoute, level: HarnessReasoningLevel) -> Int { + match harness_openrouter_level_effort(level: level) { + Absent => 0 + Present { value: effort } => hosted_effort_rank(efforts: route.supported_efforts, effort: effort) + } +} + +fn hosted_level_admitted(route: HarnessHostedRoute, level: HarnessReasoningLevel, max_tokens: TokenCount) -> Bool { + token_count_value(t: max_tokens) <= token_count_value(t: route.max_completion) + && (match harness_reasoning_kwargs_for(unit: route.unit, level: level) { + ReasoningKwargsAdmitted { kwargs: _ } => true + ReasoningLevelUnsupported { unit: _, level: _, cause: _ } => false + }) +} + +fn hosted_level_entry(route: HarnessHostedRoute, level: HarnessReasoningLevel, max_tokens: TokenCount) -> ParetoEntry { + let no_gaps: List = [] + ParetoEntry { + identity: hosted_level_identity(level: level), + display_label: harness_reasoning_level_wire(level: level), + readings: [ + AxisReading { axis: ax_hosted_reasoning_depth, interval_micro: degenerate_interval(point: hosted_level_depth_rank(route: route, level: level) * 1000000) }, + AxisReading { axis: ax_hosted_money_per_request, interval_micro: degenerate_interval(point: (money_amount_micro_count(m: route.price_per_token) as Int) * token_count_value(t: max_tokens)) }, + AxisReading { axis: ax_hosted_quota_per_request, interval_micro: degenerate_interval(point: 1000000) }, + AxisReading { axis: ax_hosted_level_admitted, interval_micro: degenerate_interval(point: if hosted_level_admitted(route: route, level: level, max_tokens: max_tokens) { 1000000 } else { 0 }) }, + ], + missing_inputs: no_gaps, + } +} + +data harness_hosted_levels: List = [ReasoningOff, ReasoningStandard, ReasoningMaximum] + +fn harness_hosted_level_field(route: HarnessHostedRoute, max_tokens: TokenCount) -> List { + map(harness_hosted_levels, l => hosted_level_entry(route: route, level: l, max_tokens: max_tokens)) +} + +fn harness_hosted_level_assessment() -> DecisionAssessment { + let subject_id = hosted_ref(name: "harness_hosted_level_subject") + let var_id = hosted_ref(name: "harness_hosted_level_variable") + let term_id = hosted_ref(name: "harness_hosted_reasoning_depth_term") + let param_id = hosted_ref(name: "harness_hosted_route_parameter") + DecisionAssessment { + subject: DecisionSubject { + identity: subject_id, + question: "which reasoning level a harness turn on a hosted free listing runs at" as NonEmptyStr, + parameters: [ContextParameter { identity: param_id }], + constraints: [DecisionConstraint { identity: hosted_level_admitted_constraint }], + objective: ObjectiveMeasure { + identity: hosted_ref(name: "harness_hosted_level_objective"), + goal: HigherIsBetter, + terms: [ObjectiveTerm { identity: term_id, ground: CitedToAuthority { authority: openrouter_reasoning_authority } }], + }, + choice_domain: ChoiceDomain { + identity: hosted_ref(name: "harness_hosted_level_choice_domain"), + variables: [DecisionVariable { identity: var_id }], + }, + }, + sensitivities: [ + VariesAlong { + term: term_id, + variable: var_id, + region: DecisionRegion { identity: hosted_ref(name: "harness_hosted_level_region") }, + relation: MonotoneIncreasing, + coupling: CitedToAuthority { authority: openrouter_reasoning_authority }, + }, + ], + constraints: [ + ConstraintSatisfied { constraint: hosted_level_admitted_constraint, evidence: "admissibility applied to the field as the hard constraint: the unit's reasoning seam and the listing's completion ceiling" as NonEmptyStr }, + ], + parameters: [ + ParameterBound { parameter: param_id, evidence: "the hosted route joined from its listing" as NonEmptyStr }, + ], + selected_candidate: Absent, + term_drops: [], + physical_model: PhysicalModelEstablished { terms: [term_id] }, + causal_model: CausalModelEstablished { + couplings: [ + VendorDocumentedCoupling { authority: openrouter_reasoning_authority, term: term_id, variable: var_id, subject: subject_id }, + ], + }, + } +} + +fn harness_hosted_level_selection(route: HarnessHostedRoute, max_tokens: TokenCount) -> RealizationSelectionResult { + select_realization( + assessment: harness_hosted_level_assessment(), + candidate_field_identity: hosted_ref(name: "harness_hosted_level_field"), + field: harness_hosted_level_field(route: route, max_tokens: max_tokens), + axes: harness_hosted_level_axes, + constraints: harness_hosted_level_constraints, + constraint_authority: hosted_ref(name: "harness_hosted_level_constraints"), + policy: UniqueSurvivorRequired { identity: hosted_ref(name: "harness_hosted_level_policy") }, + evidence_snapshot: hosted_ref(name: "harness_hosted_route_for"), + ) +} + +type HostedLevelChoice + = HostedLevelSelected { level: HarnessReasoningLevel } + | HostedLevelUnselected { cause: String } + +fn hosted_level_of_identity(identity: DeclarationRef) -> HarnessReasoningLevel? { + match filter(harness_hosted_levels, l => declaration_ref_eq(a: hosted_level_identity(level: l), b: identity)).first() { + Absent => none + Present { value: l } => Present { value: l } + } +} + +// ANY ARM BUT A SELECTION IS A REFUSAL, carried with the arm's name. A turn does not run at a level the +// selection did not choose: there is no fallback level, because a fallback is exactly the literal this +// selection replaced. +fn harness_hosted_level_choice(route: HarnessHostedRoute, max_tokens: TokenCount) -> HostedLevelChoice { + match harness_hosted_level_selection(route: route, max_tokens: max_tokens) { + SelectedWithin { candidate: c, receipt: _ } => + match hosted_level_of_identity(identity: c.identity) { + Present { value: l } => HostedLevelSelected { level: l } + Absent => HostedLevelUnselected { cause: "the selection returned a candidate that is not one of the harness's levels" } + } + CandidateFieldUnestablished { causes: cs } => HostedLevelUnselected { cause: join(["the level field is unestablished: ", join(map(cs, c => c as String), "; ")], "") } + NoFeasibleRealization { causes: cs, scope: _ } => HostedLevelUnselected { cause: join(["no level is admissible on this route: ", join(map(cs, c => c as String), "; ")], "") } + SelectionNeedsEvidence { needs: _ } => HostedLevelUnselected { cause: "the level selection needs evidence it does not have" } + SelectionNeedsPolicy { survivors: _ } => HostedLevelUnselected { cause: "more than one level survives and no policy chooses between them" } + RealizationSelectionRefused { defects: ds } => HostedLevelUnselected { cause: join(["the level selection refused: ", join(map(ds, d => d as String), "; ")], "") } + } +} + +// ── THE CREDENTIAL: MATERIALIZED, LIVENESS-READ, AND SCOPED TO THE BINDING ────────────────────── +// +// The key is fetched from Secret Manager, written as one header line into a 0600 file in a fresh +// directory, and handed to curl by path (-H @file), so it is never an argv word. Liveness is read by the +// key endpoint itself: GET /api/v1/key with the same header file asks OpenRouter whether this key is +// accepted and cannot make a rejected key accepted, so the probe is inert (gunbc.readback_independence) +// and gunbc.auth.materialized_secret with_materialized_secret is the gate that refuses a key the upstream +// does not accept. The directory lives as long as the binding and is removed by its release. +data openrouter_key_read_probe: ReadBackProbe = ReadBackProbe { + label: "GET https://openrouter.ai/api/v1/key with the materialized bearer header file", + subject: "the key this SecretRef holds is one OpenRouter currently accepts", + effect_on_subject: ProbeInert, +} + +data hosted_key_read_deadline: Second = second(count: 20) + +type HarnessHostedQuota { + short_hostname: String + actor: NonEmptyStr + per_minute: UpstreamRateLimit + per_day: UpstreamRateLimit +} + +// WHAT A HOSTED BINDING HOLDS IN PLACE OF A SEAT: the header file and the directory it lives in, the two +// quota rows each request leases, the selected level, what the key read reported, and the terms that +// govern the route. The terms ride the binding so every turn placed on it renders the standing it ran +// under; they are not a gate this code can evaluate, because what the operator sends is not visible to it. +type HarnessHostedAccess { + header_dir: NonEmptyStr + header_file: NonEmptyStr + quota: HarnessHostedQuota + key: OpenRouterKeyRead + terms: NvidiaApiTrialTerms +} + +type HostedBindOutcome + = HostedBound { url: NonEmptyStr, model: NonEmptyStr, unit: HarnessServingUnit, level: HarnessReasoningLevel, access: HarnessHostedAccess } + | HostedBindRefused { detail: String } + +// EACH WAY THE KEY CAN FAIL TO ARRIVE IS ITS OWN CAUSE, because they have different remedies: no Secret +// Manager access token to fetch with (a federation or upsert problem, nothing to do with the secret), a +// refused read (the secret or its accessor cell), a response that did not decode, and a response that +// answered about another version. Collapsing them to "could not be read" would send the operator to the +// secret when the fault is the token. +type HostedKeyFetch + = HostedKeyFetched { text: String } + | HostedKeyFetchRefused { cause: String } + +fn hosted_secret_text(route: HarnessHostedRoute) -> HostedKeyFetch { + match resolve_access_token() { + AccessTokenReady { token: t } => + match fetch_secret_ref_credential_with_token(secret_ref: route.secret, access_token: t) { + SecretCredentialReady { credential: c, resolved_version: _ } => HostedKeyFetched { text: trim(s: c as String) } + SecretCredentialAccessUpsertRequired { plan: p } => + HostedKeyFetchRefused { cause: join(["the secret read needs a Secret Manager access upsert first: ", access_token_upsert_required_reason(plan: p)], "") } + SecretCredentialWireDecodeRefused { identity_cause: i, payload_cause: pc } => + HostedKeyFetchRefused { cause: join([ + "the Secret Manager access response did not decode (version identity ", match i { Present { value: _ } => "refused" Absent => "decoded" }, + ", payload ", match pc { Present { value: _ } => "refused" Absent => "decoded" }, ")", + ], "") } + SecretCredentialFetchRefused { reason: r } => HostedKeyFetchRefused { cause: join(["the Secret Manager read was refused: ", r as String], "") } + SecretCredentialResolvedVersionMismatch { requested: q, resolved: r } => + HostedKeyFetchRefused { cause: join(["the Secret Manager read asked for version ", q, " and was answered about ", r], "") } + } + AccessTokenUpsertRequired { plan: p } => + HostedKeyFetchRefused { cause: join(["no Secret Manager access token is available and an access upsert is required: ", access_token_upsert_required_reason(plan: p)], "") } + AccessTokenEnsureRefused { reason: r } => + HostedKeyFetchRefused { cause: join(["no Secret Manager access token could be ensured: ", r as String], "") } + } +} + +// THE KEY READ, AS AN OBSERVATION VERDICT FOR THE MATERIALIZATION GATE, plus the decoded body for the +// binding. A transport failure is Inaccessible, never Converged: an unread key is not a live one. +type HostedKeyObservation + = HostedKeyObserved { key: OpenRouterKeyRead } + | HostedKeyUnobserved { cause: String } + +fn hosted_read_key(route: HarnessHostedRoute, header_file: NonEmptyStr) -> HostedKeyObservation { + let read = http.Client.GetFollowRedirectsBoundedWithHeaderFile( + url: route.key_url, + max_seconds: to_string(second_count(s: hosted_key_read_deadline)) as NonEmptyStr, + accept: "Accept: application/json" as NonEmptyStr, + header_file: header_file, + ) + if !read.success { + HostedKeyUnobserved { cause: join(["the key read failed: ", read.body], "") } + } else { + match openrouter_decode_key_read(body: read.body) { + OpenRouterKeyUndecodable { cause: c } => HostedKeyUnobserved { cause: c } + OpenRouterKeyDecoded { key: k } => HostedKeyObserved { key: k } + } + } +} + +fn hosted_key_liveness(observation: HostedKeyObservation) -> ObservationVerdict? { + match observation { + HostedKeyObserved { key: _ } => Present { value: Converged } + HostedKeyUnobserved { cause: _ } => Present { value: Inaccessible } + } +} + +// THE KEY MUST BE ONE THAT CANNOT SPEND MONEY, and the binding refuses one that can. The operator capped +// this key at zero credits so a request that strays onto a priced model answers 402 instead of charging; +// a key read showing no cap, or a positive one, means that guard is gone, and a free-only route that ran +// anyway would be the externalized degradation DESIGN section 5 names -- the same name, a different +// burden. The daily tier is read from the same response: it is per ACCOUNT and the key cap does not +// change it. +fn hosted_access_from_key(route: HarnessHostedRoute, key: OpenRouterKeyRead, level: HarnessReasoningLevel, header_dir: NonEmptyStr, header_file: NonEmptyStr, short_hostname: String, actor: NonEmptyStr) -> HostedBindOutcome? { + match key.spend_limit { + KeySpendUncapped => none + KeySpendCappedAbove { lexeme: _ } => none + KeySpendCappedAtZero => + match openrouter_credits_from_daily_limit(daily_limit: key.free_daily_limit) { + CreditsTierUnrecognized { daily_limit: _ } => none + CreditsTierObserved { credits: credits } => Present { value: HostedBound { + url: route.url, + model: harness_unit_model_id(unit: route.unit), + unit: route.unit, + level: level, + access: HarnessHostedAccess { + header_dir: header_dir, + header_file: header_file, + quota: HarnessHostedQuota { short_hostname: short_hostname, actor: actor, per_minute: route.per_minute, per_day: openrouter_free_models_per_day(credits: credits) }, + key: key, + terms: route.terms, + }, + } } + } + } +} + +// THE DAY POOL STARTS FROM WHAT THE UPSTREAM SAYS REMAINS, NOT FROM ITS CEILING. The tier sets the +// ceiling; free_model_daily_requests.remaining says how much of it is left today, including requests +// another client of this key spent and requests made before this ledger existed. The reading is +// recorded on the day partition (gunbc.fabric_quota fabric_quota_observe_upstream) before the binding is +// handed out, so the first lease already sees it. A reading that cannot be recorded refuses the bind: +// leasing against the bare ceiling would admit requests the upstream has already refused. +fn hosted_day_remaining(key: OpenRouterKeyRead) -> Nat? { + checked_int_to_nat(n: key.free_daily_remaining) +} + +fn hosted_record_day_remaining(bound: HostedBindOutcome, key: OpenRouterKeyRead) -> HostedBindOutcome { + match bound { + HostedBindRefused { detail: d } => HostedBindRefused { detail: d } + HostedBound { url: u, model: m, unit: un, level: l, access: a } => + match hosted_day_remaining(key: key) { + Absent => HostedBindRefused { detail: join(["the key read reports a negative free-model remaining count (", to_string(key.free_daily_remaining), ")"], "") } + Present { value: remaining } => + match fabric_quota_observe_upstream(short_hostname: a.quota.short_hostname, limit: a.quota.per_day, remaining: remaining, actor: a.quota.actor) { + QuotaObservationRefused { detail: d } => HostedBindRefused { detail: join(["the upstream's remaining free-model requests could not be recorded on the day pool: ", d], "") } + QuotaUpstreamRecorded { partition: _, remaining: _ } => HostedBound { url: u, model: m, unit: un, level: l, access: a } + } + } + } +} + +fn hosted_key_refusal(key: OpenRouterKeyRead) -> String { + match key.spend_limit { + KeySpendCappedAtZero => + join(["the key reports a free-model daily limit of ", to_string(key.free_daily_limit), ", which is neither published tier (50 or 1000); refusing to lease against an unpublished ceiling"], "") + KeySpendUncapped => "the key reports no credit limit: it can spend money on a priced model, and this route admits only a key capped at zero credits" + KeySpendCappedAbove { lexeme: lx } => join(["the key reports a credit limit of ", lx, ": it can spend money on a priced model, and this route admits only a key capped at zero credits"], "") + } +} + +fn hosted_bind_with_header(route: HarnessHostedRoute, header_dir: NonEmptyStr, header_file: NonEmptyStr, short_hostname: String, actor: NonEmptyStr, max_tokens: TokenCount) -> HostedBindOutcome { + let observation = hosted_read_key(route: route, header_file: header_file) + match with_materialized_secret( + ref: route.secret, + binding: BearerHeaderFile, + existence: SmEnabled, + probe: openrouter_key_read_probe, + liveness_observation: hosted_key_liveness(observation: observation), + use: m => observation, + ) { + MaterializationRefused { cause: SecretNotLive { verdict: _ } } => + HostedBindRefused { detail: match observation { HostedKeyUnobserved { cause: c } => join(["the OpenRouter key is not live: ", c], "") HostedKeyObserved { key: _ } => "the OpenRouter key is not live" } } + MaterializationRefused { cause: SecretVersionUnusable { state: _ } } => HostedBindRefused { detail: "the OpenRouter key's secret version is not usable" } + MaterializationRefused { cause: LivenessProbeNotIndependent { probe_label: p } } => HostedBindRefused { detail: join(["the key liveness probe is not independent: ", p as String], "") } + MaterializationCompleted { value: HostedKeyUnobserved { cause: c } } => HostedBindRefused { detail: c } + MaterializationCompleted { value: HostedKeyObserved { key: key } } => + match harness_hosted_level_choice(route: route, max_tokens: max_tokens) { + HostedLevelUnselected { cause: c } => HostedBindRefused { detail: c } + HostedLevelSelected { level: l } => + match hosted_access_from_key(route: route, key: key, level: l, header_dir: header_dir, header_file: header_file, short_hostname: short_hostname, actor: actor) { + Absent => HostedBindRefused { detail: hosted_key_refusal(key: key) } + Present { value: bound } => hosted_record_day_remaining(bound: bound, key: key) + } + } + } +} + +// THE BIND. A fetched secret that could not be written leaves nothing behind; any refusal after the +// header file exists removes its directory before answering, so a refused bind holds no credential on +// disk. +fn harness_hosted_bind(unit: HarnessServingUnit, short_hostname: String, actor: NonEmptyStr, max_tokens: TokenCount) -> HostedBindOutcome { + match harness_hosted_route_for(unit: unit) { + Absent => HostedBindRefused { detail: join(["the unit ", harness_serving_unit_wire(unit: unit) as String, " is served on a fleet seat and has no hosted route"], "") } + Present { value: route } => + match hosted_secret_text(route: route) { + HostedKeyFetchRefused { cause: c } => HostedBindRefused { detail: join(["the OpenRouter key (", route.secret.secret as String, ") could not be fetched: ", c], "") } + HostedKeyFetched { text: key_text } => + if key_text == "" { + HostedBindRefused { detail: "the OpenRouter key secret is empty" } + } else { + let dir = shell.Mktemp.Dir() + if !dir.success { + HostedBindRefused { detail: "mktemp -d failed for the bearer header file" } + } else { + let header_file = join([dir.path as String, "/authorization"], "") as NonEmptyStr + let wrote = Filesystem.WriteOwnerOnly(path: header_file as String, content: openrouter_authorization_header_line(key: key_text as NonEmptyStr)) + if !wrote.success { + let cleaned = shell.Remove.RecursiveForce(path: dir.path) + HostedBindRefused { detail: "could not write the bearer header file" } + } else { + match hosted_bind_with_header(route: route, header_dir: dir.path as String as NonEmptyStr, header_file: header_file, short_hostname: short_hostname, actor: actor, max_tokens: max_tokens) { + HostedBindRefused { detail: d } => { + let cleaned = shell.Remove.RecursiveForce(path: dir.path) + HostedBindRefused { detail: d } + } + HostedBound { url: u, model: m, unit: un, level: l, access: a } => HostedBound { url: u, model: m, unit: un, level: l, access: a } + } + } + } + } + } + } +} + +type HostedRelease + = HostedReleased + | HostedReleaseRefused { detail: String } + +fn harness_hosted_release(access: HarnessHostedAccess) -> HostedRelease { + let removed = shell.Remove.RecursiveForce(path: access.header_dir as String as FilePath) + if removed.success { HostedReleased } else { HostedReleaseRefused { detail: join(["the bearer header directory ", access.header_dir as String, " could not be removed"], "") } } +} + +fn harness_hosted_access_wire(access: HarnessHostedAccess) -> String { + join([ + "hosted quota ", to_string(access.quota.per_minute.requests), "/minute and ", to_string(access.quota.per_day.requests), "/day; ", + openrouter_key_spend_limit_wire(l: access.key.spend_limit), "; free-model requests today ", to_string(access.key.free_daily_used), "/", to_string(access.key.free_daily_limit), + "; governed by ", nvidia_api_trial_terms_wire(t: access.terms) as String, + ], "") +} + +// ── EACH REQUEST IS CHARGED TO BOTH POOLS BEFORE IT IS SENT ─────────────────────────────────── +// +// THE LOCAL QUOTA IS A CLOSE, CONSERVATIVE PRE-CHECK, NOT AN EXACT GATE, AND THE UPSTREAM'S 429 IS THE +// BACKSTOP. This route deliberately revises the 2026-09-06 ruling that a caller refuses before the upstream +// would have answered 429 (gunbc.fabric_quota): the pools below make an over-spend rare and keep the +// ledger honest, but nothing here can make the local count equal the upstream's at a window boundary -- +// the upstream counts a request when it RECEIVES it, and nothing on this side bounds when a process's first +// request byte leaves. A 429 that gets through is a typed refusal carrying the upstream's own retry hints +// (extdeps.openrouter.openrouter OpenRouterRateLimitRefusal), reported in the turn outcome and never +// retried (operator decision, escalation msg_46a417a3). Machinery whose only job was to make the count +// exact -- an admission lock, charging every window a request might span -- is deliberately absent. +// +// THE CHARGE AND THE RESPONSE ARE TWO LIFETIMES. The upstream counts a request in the window it ARRIVES +// in, whatever happens to its answer afterwards; the answer may take minutes. An earlier revision held +// the lease across the response and settled it afterwards, with the response deadline as the term -- +// and a 2,348-second term can never fit a 60-second lapsing window, so the minute pool refused +// LeaseCrossesLapse on every request and nothing was ever sent. Shortening the term would not have +// fixed it: a lease held until the answer arrives still outlives a window the request was counted in. +// +// SO A REQUEST IS CHARGED, NOT HELD. Each pool is leased for the remainder of its current window +// (gunbc.fabric_quota QuotaTermToWindowEnd, read off the same clock reading the lease is stamped with), +// and both leases are settled at one BEFORE the request goes out. The charge then stands for the rest of +// the window whatever the response does, which is also the conservative accounting for a request whose +// fate is unknown: a crash after the send, or a transport that failed mid-flight, still counted upstream +// as far as anyone here can tell, so it is never refunded. +// +// BOTH POOLS OR NEITHER. The minute pool is leased first because it refills soonest; if the day pool then +// refuses, the minute lease is settled at zero so its slot returns at once. +type HostedRoundLease + = HostedRoundCharged + | HostedRoundQuotaFull { wire: String } + | HostedRoundUnleasable { detail: String } + +// THE DECISION IS PURE AND THE LEASING IS NOT. What a pair of lease answers MEANS is decided from +// supplied answers; the day pool is asked only when the minute pool leased, so its answer is absent +// otherwise, and a refusal after the minute pool leased names that lease as owed back. +type HostedRoundDecision + = RoundBothLeased { minute: QuotaLease, day: QuotaLease } + | RoundRefusedFull { wire: String, minute_to_return: QuotaLease? } + | RoundRefusedUnleasable { detail: String, minute_to_return: QuotaLease? } + +fn hosted_round_decision(minute: QuotaLease, day: QuotaLease?) -> HostedRoundDecision { + match minute { + QuotaFull { wire: w } => RoundRefusedFull { wire: w, minute_to_return: none } + QuotaLeaseRefused { detail: d } => RoundRefusedUnleasable { detail: d, minute_to_return: none } + QuotaLeased { grant: _, partition: _, store: _, limit: _ } => + match day { + Absent => RoundRefusedUnleasable { detail: "the minute pool leased but the day pool was never asked", minute_to_return: Present { value: minute } } + Present { value: QuotaLeased { grant: g, partition: p, store: st, limit: l } } => RoundBothLeased { minute: minute, day: QuotaLeased { grant: g, partition: p, store: st, limit: l } } + Present { value: QuotaFull { wire: w } } => RoundRefusedFull { wire: w, minute_to_return: Present { value: minute } } + Present { value: QuotaLeaseRefused { detail: d } } => RoundRefusedUnleasable { detail: d, minute_to_return: Present { value: minute } } + } + } +} + +fn hosted_return_minute(minute: QuotaLease?) -> String { + match minute { + Absent => "" + Present { value: m } => join(["; minute lease returned: ", quota_settle_wire(s: fabric_quota_settle(lease: m, actual: 0))], "") + } +} + +fn hosted_settle_refusal(s: QuotaSettle, pool: String) -> List { + match s { + QuotaSettled { partition: _, actual: _ } => [] + QuotaSettleRefused { detail: d } => [join([pool, ": ", d], "")] + } +} + +// A CHARGE THAT DID NOT RECORD IS NOT SENT. The request has not gone out yet, so refusing costs nothing +// upstream; sending it would spend quota the ledger does not show, which is the over-admission the +// lease exists to prevent. A lease left unsettled by the refusal still charges until its window ends, +// so the refusal errs on the side of spending less. +fn hosted_round_charge_both(minute: QuotaLease, day: QuotaLease, minute_amount: Nat, day_amount: Nat) -> HostedRoundLease { + let refused = concat( + hosted_settle_refusal(s: fabric_quota_settle(lease: minute, actual: minute_amount), pool: "minute"), + hosted_settle_refusal(s: fabric_quota_settle(lease: day, actual: day_amount), pool: "day"), + ) + if count(refused) == 0 { + HostedRoundCharged + } else { + HostedRoundUnleasable { detail: join(["the request's charge did not record, so it was not sent: ", join(refused, "; ")], "") } + } +} + +// THE CHARGE COVERS THIS SEND AND EVERY SEND STILL IN FLIGHT FROM AN EARLIER WINDOW +// (gunbc.harness.harness_seat hosted_inflight_carried): each may arrive in this window, and none was +// charged to it. A refusal carries the held seats' release hint, so a pool starved by seats whose holders +// died says which seats and how to release them. +fn harness_hosted_round_charge(quota: HarnessHostedQuota, carried_minute: Nat, carried_day: Nat, release_hint: String) -> HostedRoundLease { + let minute_amount = carried_minute + 1 + let day_amount = carried_day + 1 + let minute = fabric_quota_lease_for(short_hostname: quota.short_hostname, limit: quota.per_minute, amount: minute_amount, actor: quota.actor, term: QuotaTermToWindowEnd) + let day = match minute { + QuotaLeased { grant: _, partition: _, store: _, limit: _ } => + Present { value: fabric_quota_lease_for(short_hostname: quota.short_hostname, limit: quota.per_day, amount: day_amount, actor: quota.actor, term: QuotaTermToWindowEnd) } + QuotaFull { wire: _ } => none + QuotaLeaseRefused { detail: _ } => none + } + match hosted_round_decision(minute: minute, day: day) { + RoundBothLeased { minute: m, day: d } => hosted_round_charge_both(minute: m, day: d, minute_amount: minute_amount, day_amount: day_amount) + RoundRefusedFull { wire: w, minute_to_return: r } => HostedRoundQuotaFull { wire: join([w, hosted_return_minute(minute: r), release_hint], "") } + RoundRefusedUnleasable { detail: d, minute_to_return: r } => HostedRoundUnleasable { detail: join([d, hosted_return_minute(minute: r)], "") } + } +} + +// THE UPSTREAM'S REFUSAL, DECODED. An error body is OpenRouter's typed error when it decodes; a body that +// does not decode is reported as itself, because a transport failure has no error object to read. +fn harness_hosted_refusal_cause(body: String) -> String { + match openrouter_decode_error_body(body: body) { + OpenRouterErrorDecoded { error: e } => openrouter_error_wire(e: e) + OpenRouterErrorUndecodable { cause: _ } => body + } +} + +// ── WHAT A TURN CARRIES ABOUT HOW ITS REQUESTS ARE ADMITTED ─────────────────────────────────── +// +// A fleet turn's requests are admitted by the seat it already holds, unauthenticated over the fleet's +// network. A hosted turn's requests each lease quota and carry the bearer header file. The turn config +// carries which, so the POST is chosen by the binding the turn was placed on, never re-derived from the +// unit inside the loop. +type HarnessBackendAccess + = BackendAdmittedBySeat + | BackendAdmittedByHostedQuota { header_file: NonEmptyStr, quota: HarnessHostedQuota } + +fn harness_hosted_backend_access(access: HarnessHostedAccess) -> HarnessBackendAccess { + BackendAdmittedByHostedQuota { header_file: access.header_file, quota: access.quota } +} diff --git a/dag/gunbc/harness/harness_reasoning_wire.dag b/dag/gunbc/harness/harness_reasoning_wire.dag index f414986029a..d033b0b52dc 100644 --- a/dag/gunbc/harness/harness_reasoning_wire.dag +++ b/dag/gunbc/harness/harness_reasoning_wire.dag @@ -3,6 +3,15 @@ module gunbc.harness.harness_reasoning_wire import std.types { String, Bool, List, NonEmptyStr } import std.measure { TokenCount } import extdeps.languages.json.emit { JsonKeyValue, JsonBool, json_kv, json_string, json_object } +import extdeps.openrouter.openrouter { + OpenRouterReasoningEffort, EffortHigh, EffortMedium, OpenRouterEffortAdmission, EffortAdmitted, EffortNotSupported, + openrouter_effort_admitted, openrouter_reasoning_effort_wire, openrouter_message_reasoning_field, +} +import extdeps.openrouter.nvidia_nemotron_3_ultra_free { + nemotron_ultra_free_model_id, nemotron_ultra_free_context_length, nemotron_ultra_free_supported_efforts, + nemotron_ultra_free_reasoning_mandatory, +} +import extdeps.nvidia.nemotron_3_ultra { nemotron_ultra_thinking_end_token } import gunbc.spark.serving_arm { ReasoningResponseProjection, ParsedReasoningField, InlineReasoning, gitcommit90_sm121_exl3_row, gunbc_sparkrun_ds4_row, @@ -38,14 +47,38 @@ import extdeps.zhipu.glm_5_3_flash { // for that family -- vLLM ships glm45 for GLM and, on this fleet's build, nothing for DeepSeek V4. // Attributing that to the model would be authority substitution; attributing it to the unit is // exactly right. +// +// A HOSTED UNIT IS A MODEL PLUS THE ROUTER IT IS REACHED THROUGH, by the same reasoning: whether reasoning +// comes back split, under which member, and which efforts are accepted are the router's facts about its +// listing (extdeps.openrouter.nvidia_nemotron_3_ultra_free), not the model's. How the harness PLACES a turn +// on each unit -- a fleet seat or a hosted quota lease -- is asked of harness_unit_placement below. type HarnessServingUnit = DeepseekV4FlashOnVllm | Glm53FlashOnVllm + | NemotronUltraFreeOnOpenRouter fn harness_serving_unit_wire(unit: HarnessServingUnit) -> NonEmptyStr { match unit { DeepseekV4FlashOnVllm => "deepseek-v4-flash on vllm" as NonEmptyStr Glm53FlashOnVllm => "glm-5.3-flash on vllm" as NonEmptyStr + NemotronUltraFreeOnOpenRouter => "nemotron-3-ultra:free on openrouter" as NonEmptyStr + } +} + +// HOW A TURN ON THIS UNIT IS ADMITTED. A fleet unit is served by an engine this repository launches, so a +// turn takes a seat on a serving group (gunbc.harness.harness_seat harness_bind_seat). A hosted unit has no +// group, no replica and no loaded state to share: what bounds it is the upstream's per-request quota, so a +// turn leases that quota (gunbc.fabric_quota) before each request instead. The two are different capacity +// facts, which is why this is a closed choice and not a flag. +type HarnessUnitPlacement + = PlacedOnFleetSeat + | LeasedFromHostedQuota + +fn harness_unit_placement(unit: HarnessServingUnit) -> HarnessUnitPlacement { + match unit { + DeepseekV4FlashOnVllm => PlacedOnFleetSeat + Glm53FlashOnVllm => PlacedOnFleetSeat + NemotronUltraFreeOnOpenRouter => LeasedFromHostedQuota } } @@ -133,10 +166,62 @@ fn harness_glm_reasoning_kwargs(level: HarnessReasoningLevel) -> HarnessReasonin } } +// THE OPENROUTER ARM, AND ITS WIRE IS THE ROUTER'S, NOT THE MODEL'S. The request carries OpenRouter's +// top-level `reasoning` object, never Nemotron's chat_template_kwargs: the router owns the translation to +// the provider, and a client spelling the model's template keys past it would be addressing a template +// it cannot see. +// +// EACH LEVEL IS MAPPED TO AN EFFORT AND THE EFFORT IS ADMITTED AGAINST THE LISTING'S PUBLISHED SET, so +// what this arm may send is decided by the listing's row and never by this function. Standard is medium +// and maximum is high, because those are the two efforts the listing accepts and that is their order; +// off is `enabled: false`, which the listing admits because its reasoning is not mandatory. A listing +// whose set lacks the mapped effort, or whose reasoning is mandatory, refuses the level with both names. +fn harness_openrouter_level_effort(level: HarnessReasoningLevel) -> OpenRouterReasoningEffort? { + match level { + ReasoningOff => none + ReasoningStandard => Present { value: EffortMedium } + ReasoningMaximum => Present { value: EffortHigh } + } +} + +fn harness_openrouter_reasoning_kwargs(unit: HarnessServingUnit, level: HarnessReasoningLevel, supported: List, mandatory: Bool) -> HarnessReasoningKwargs { + match harness_openrouter_level_effort(level: level) { + Absent => + if mandatory { + ReasoningLevelUnsupported { + unit: harness_serving_unit_wire(unit: unit), + level: harness_reasoning_level_wire(level: level), + cause: "the listing marks reasoning mandatory, so a request with reasoning disabled is not one it admits" as NonEmptyStr, + } + } else { + ReasoningKwargsAdmitted { kwargs: [json_kv(key: "reasoning", value: json_object([ + json_kv(key: "enabled", value: JsonBool { value: false }), + ]))] } + } + Present { value: effort } => + match openrouter_effort_admitted(effort: effort, supported: supported) { + EffortNotSupported { effort: e, supported: _ } => ReasoningLevelUnsupported { + unit: harness_serving_unit_wire(unit: unit), + level: harness_reasoning_level_wire(level: level), + cause: join([ + "the level maps to reasoning effort ", openrouter_reasoning_effort_wire(e: e) as String, + ", which the listing's supported_efforts do not include; an unsupported effort is translated by the router to a setting the caller never sees, so it is refused rather than sent", + ], "") as NonEmptyStr, + } + EffortAdmitted { effort: e } => + ReasoningKwargsAdmitted { kwargs: [json_kv(key: "reasoning", value: json_object([ + json_kv(key: "effort", value: json_string(s: openrouter_reasoning_effort_wire(e: e) as String)), + ]))] } + } + } +} + fn harness_reasoning_kwargs_for(unit: HarnessServingUnit, level: HarnessReasoningLevel) -> HarnessReasoningKwargs { match unit { DeepseekV4FlashOnVllm => harness_deepseek_reasoning_kwargs(level: level) Glm53FlashOnVllm => harness_glm_reasoning_kwargs(level: level) + NemotronUltraFreeOnOpenRouter => + harness_openrouter_reasoning_kwargs(unit: unit, level: level, supported: nemotron_ultra_free_supported_efforts, mandatory: nemotron_ultra_free_reasoning_mandatory) } } @@ -154,6 +239,12 @@ fn harness_unit_thinking_requested(unit: HarnessServingUnit, level: HarnessReaso ReasoningStandard => true ReasoningMaximum => true } + NemotronUltraFreeOnOpenRouter => + match level { + ReasoningOff => false + ReasoningStandard => true + ReasoningMaximum => true + } } } @@ -163,16 +254,21 @@ fn harness_unit_model_id(unit: HarnessServingUnit) -> NonEmptyStr { match unit { DeepseekV4FlashOnVllm => "deepseek-v4-flash:iq3s-split" as NonEmptyStr Glm53FlashOnVllm => glm_5_3_flash_model_id + NemotronUltraFreeOnOpenRouter => nemotron_ultra_free_model_id } } // THE ENGINE'S TOOL-CALL PARSER FOR THIS UNIT, delegated for the same reason. The DeepSeek arm keeps // the string because extdeps.deepseek.deepseek_v4 carries no parser row; that asymmetry is the // DeepSeek module's gap, named here rather than papered over with a second GLM-shaped row. -fn harness_unit_tool_call_parser(unit: HarnessServingUnit) -> NonEmptyStr { +// A HOSTED UNIT HAS NO ENGINE THIS REPOSITORY LAUNCHES, so there is no parser for it to name: the +// router's provider parses tool calls and returns them already structured. The answer is absent rather +// than a borrowed name, because a launch reading a name here would launch a parser nobody chose. +fn harness_unit_tool_call_parser(unit: HarnessServingUnit) -> NonEmptyStr? { match unit { - DeepseekV4FlashOnVllm => "deepseek_v4" as NonEmptyStr - Glm53FlashOnVllm => glm_5_3_flash_vllm_tool_call_parser + DeepseekV4FlashOnVllm => Present { value: "deepseek_v4" as NonEmptyStr } + Glm53FlashOnVllm => Present { value: glm_5_3_flash_vllm_tool_call_parser } + NemotronUltraFreeOnOpenRouter => none } } @@ -206,6 +302,7 @@ fn harness_unit_reasoning_projection(unit: HarnessServingUnit) -> ReasoningRespo match unit { DeepseekV4FlashOnVllm => gunbc_sparkrun_ds4_row.reasoning_projection Glm53FlashOnVllm => gitcommit90_sm121_exl3_row.reasoning_projection + NemotronUltraFreeOnOpenRouter => Present { value: ParsedReasoningField { field: openrouter_message_reasoning_field } } } } @@ -234,6 +331,7 @@ fn harness_unit_model_reasoning_field(unit: HarnessServingUnit) -> NonEmptyStr { match unit { DeepseekV4FlashOnVllm => deepseek_v4_chat_reasoning_field Glm53FlashOnVllm => glm_5_3_flash_chat_reasoning_field + NemotronUltraFreeOnOpenRouter => openrouter_message_reasoning_field } } @@ -308,6 +406,7 @@ fn harness_unit_server_parses_reasoning(unit: HarnessServingUnit) -> Bool { match unit { DeepseekV4FlashOnVllm => false Glm53FlashOnVllm => true + NemotronUltraFreeOnOpenRouter => true } } @@ -318,6 +417,7 @@ fn harness_unit_thinking_end_token(unit: HarnessServingUnit) -> NonEmptyStr { match unit { DeepseekV4FlashOnVllm => deepseek_v4_thinking_end_token Glm53FlashOnVllm => glm_5_3_flash_thinking_end_token + NemotronUltraFreeOnOpenRouter => nemotron_ultra_thinking_end_token } } @@ -327,5 +427,6 @@ fn harness_unit_declared_context_ceiling(unit: HarnessServingUnit) -> TokenCount match unit { DeepseekV4FlashOnVllm => deepseek_v4_context_length Glm53FlashOnVllm => glm_5_3_flash_context_length + NemotronUltraFreeOnOpenRouter => nemotron_ultra_free_context_length } } diff --git a/dag/gunbc/harness/harness_seat.dag b/dag/gunbc/harness/harness_seat.dag index 19d77dcfa3a..bfd0ffb2d5a 100644 --- a/dag/gunbc/harness/harness_seat.dag +++ b/dag/gunbc/harness/harness_seat.dag @@ -3,7 +3,9 @@ module gunbc.harness.harness_seat import product.host_identity { HostIdentity } import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive, nat_min } -import std.measure { Measure, Count, One, money_amount_micro, second, measure_count } +import std.measure { Measure, Count, One, money_amount_micro, second, measure_count, TokenCount, Second, second_count } +import extdeps.accounting.encumbrance { Held, Expended, Released } +import extdeps.api_rate_limit { upstream_rate_limit_key } import gunbc.spark.fabric_switch_observed { FabricGroup, FabricGroupA, FabricGroupB, fabric_group_wire, fabric_group_hosts } import gunbc.serving.serving_enrollment { ServingRoute, ServingRouteResolved, ServingRouteUnresolved, ServingRouteEndpoint, @@ -27,7 +29,7 @@ import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.fabric_event_log { HeadExpectation, event_log_refusal_wire, SeatStoreRefused, SeatStandingStoreRefused, HeadObserved, HeadObservationRefused, event_log_observe_head, EventAppended, EventAppendStale, EventAppendRefused, event_log_append, SeatGranted, SeatFull, SeatContended, - SeatAcquireRefused, fabric_seat_acquire, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, + SeatAcquireRefused, fabric_seat_acquire, seat_acquisition_wire, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, fabric_seat_observe, PartitionRead, PartitionReadOk, PartitionReadRefused, event_log_read_partition, RedemptionReading, RedemptionReadAt, RedemptionReadRefused, fabric_redemption_read, } @@ -38,7 +40,14 @@ import gunbc.harness.harness_backend { harness_route_availability, HarnessEngineReading, harness_observe_engine_at, } import gunbc.harness.harness_wire { HarnessWireShape, AnthropicMessagesShape, OpenAiChatCompletionsShape, harness_backend_url_for_endpoint } -import gunbc.harness.harness_reasoning_wire { HarnessServingUnit, harness_unit_model_id } +import gunbc.harness.harness_reasoning_wire { + HarnessServingUnit, harness_unit_model_id, HarnessReasoningLevel, harness_unit_placement, HarnessUnitPlacement, PlacedOnFleetSeat, + LeasedFromHostedQuota, harness_serving_unit_wire, +} +import gunbc.harness.harness_hosted_route { + HarnessHostedAccess, HarnessHostedQuota, HostedBound, HostedBindRefused, harness_hosted_bind, HostedReleased, HostedReleaseRefused, + harness_hosted_release, +} import product.fabric.work { ExecutionRequirements, ControlPlaneCapacity, Work, ServiceInterfaceRef, ArtifactManifestRef, EffectContractRef, OutputContractRef, identity_frame } import product.fabric.supply { EstimatedGrantDuration, GrantDuration, AffinityKeyedRendezvous, PlacementAffinityKey } import product.fabric.selection { @@ -291,6 +300,7 @@ type HarnessSeatBinding = HarnessSeatBound { url: String, grant: LeaseGrant, partition: PartitionId, ceiling: Nat, group_wire: String, launch: VllmEndpointProcessLaunch, store: FabricStorageBinding, model: NonEmptyStr, unit: HarnessServingUnit, class: ServingCoTenancyClass, seat: HarnessSeatReference } | HarnessSeatPending { detail: String } | HarnessSeatRefused { detail: String } + | HarnessHostedBound { url: String, model: NonEmptyStr, unit: HarnessServingUnit, level: HarnessReasoningLevel, access: HarnessHostedAccess } fn seat_bind_detail(group: FabricGroup, text: String) -> String { join([fabric_group_wire(g: group) as String, ": ", text], "") @@ -1204,6 +1214,7 @@ fn harness_seat_after_grant_authority(store: FabricStorageBinding, group: Fabric SeatAuthorityRefuses { cause: why } => match release_retry(store: store, partition: partition, ceiling: ceiling, reference: grant.reference, reason: "the pair-serving authority moved between admission and the seat's acquisition" as NonEmptyStr, now: now) { HarnessSeatReleased { partition: _ } => SeatReleasedAuthorityMoved { cause: why } + HarnessHostedCredentialRemoved => SeatUnreleasedAuthorityMoved { cause: why, release_detail: "a fleet seat release answered as a hosted credential removal" } HarnessSeatReleaseRefused { detail: d } => SeatUnreleasedAuthorityMoved { cause: why, release_detail: d } } } @@ -1223,6 +1234,8 @@ fn harness_fence_grant(candidate: HarnessCandidate, class: ServingCoTenancyClass } EndpointProcessWasReplaced { endpoint: _, expected: _, observed: _, observed_at: _, receipt: _ } => match release_retry(store: store, partition: partition, ceiling: candidate.ceiling, reference: grant.reference, reason: "the engine was replaced between selection and acquisition" as NonEmptyStr, now: now) { + HarnessHostedCredentialRemoved => + AcquisitionRefused { detail: seat_bind_detail(group: candidate.group, text: "a fleet seat release answered as a hosted credential removal; the seat may still be held") } HarnessSeatReleased { partition: _ } => AcquisitionLost { key: candidate.key, @@ -1421,6 +1434,37 @@ data harness_placement_rounds: Nat = 3 // that constructed the request itself could hand in a request naming one requirement beside a // screen argument naming another. fn harness_bind_seat(short_hostname: String, class: ServingCoTenancyClass, requirement: ServingModelRequirement, shape: HarnessWireShape, inputs: List, actor: NonEmptyStr, attempt: NonEmptyStr, policy: LeasePolicy) -> HarnessSeatBinding { + match harness_requirement_placement(requirement: requirement) { + LeasedFromHostedQuota => HarnessSeatRefused { detail: "the required unit is reached through a hosted quota lease, not a fleet seat; bind it with harness_bind_hosted" } + PlacedOnFleetSeat => harness_bind_fleet_seat(short_hostname: short_hostname, class: class, requirement: requirement, shape: shape, inputs: inputs, actor: actor, attempt: attempt, policy: policy) + } +} + +// WHERE A REQUIREMENT IS PLACED. Any admitted model is a fleet seat: the hosted listing is governed by +// trial terms (extdeps.nvidia.api_trial_terms), and an unnamed requirement must not quietly route a turn +// to an upstream whose terms the caller never chose. Only a caller naming the hosted unit reaches it. +fn harness_requirement_placement(requirement: ServingModelRequirement) -> HarnessUnitPlacement { + match requirement { + AnyAdmittedModel => PlacedOnFleetSeat + ExactServingUnit { unit: u } => harness_unit_placement(unit: u) + } +} + +// THE HOSTED BIND. A hosted unit takes no seat: the credential is materialized and its liveness read, the +// level is selected, and the binding carries the two quota rows each request will lease. A fleet unit +// handed here refuses by name rather than being placed on a seat behind the caller's back. +fn harness_bind_hosted(short_hostname: String, unit: HarnessServingUnit, actor: NonEmptyStr, max_tokens: TokenCount) -> HarnessSeatBinding { + match harness_unit_placement(unit: unit) { + PlacedOnFleetSeat => HarnessSeatRefused { detail: join(["the unit ", harness_serving_unit_wire(unit: unit) as String, " is served on a fleet seat; bind it with harness_bind_seat"], "") } + LeasedFromHostedQuota => + match harness_hosted_bind(unit: unit, short_hostname: short_hostname, actor: actor, max_tokens: max_tokens) { + HostedBindRefused { detail: d } => HarnessSeatRefused { detail: d } + HostedBound { url: u, model: m, unit: un, level: l, access: a } => HarnessHostedBound { url: u as String, model: m, unit: un, level: l, access: a } + } + } +} + +fn harness_bind_fleet_seat(short_hostname: String, class: ServingCoTenancyClass, requirement: ServingModelRequirement, shape: HarnessWireShape, inputs: List, actor: NonEmptyStr, attempt: NonEmptyStr, policy: LeasePolicy) -> HarnessSeatBinding { let subject = harness_turn_subject(request: harness_turn_request(shape: shape, requirement: requirement, inputs: inputs), attempt: attempt) match event_log_store_for_host(short_hostname: short_hostname) { HostStoreRefused { detail: d } => HarnessSeatRefused { detail: d } @@ -1445,8 +1489,12 @@ fn harness_bind_seat(short_hostname: String, class: ServingCoTenancyClass, requi // RELEASE IS AN EVENT ON THE SAME PARTITION, appended against the head observed at release time // and retried on stale, because the seat's neighbours keep moving the head while this turn ran. +// A HOSTED BINDING HOLDS NO SEAT, so its release is its own arm: each request already settled its quota +// leases, and what remains to give back is the credential's header file. Reporting it as a released +// partition would name a pool the binding never held. type HarnessSeatRelease = HarnessSeatReleased { partition: PartitionId } + | HarnessHostedCredentialRemoved | HarnessSeatReleaseRefused { detail: String } type ReleaseFold { @@ -1543,6 +1591,7 @@ fn harness_seat_pointer_of(binding: HarnessSeatBinding) -> HarnessSeatPointer? { Present { value: HarnessSeatPointer { partition: p, reference: g.reference, owner: r.attempt } } HarnessSeatRefused { detail: _ } => none HarnessSeatPending { detail: _ } => none + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: _ } => none } } @@ -1563,6 +1612,11 @@ fn harness_release_seat(binding: HarnessSeatBinding, reason: NonEmptyStr) -> Har match binding { HarnessSeatRefused { detail: d } => HarnessSeatReleaseRefused { detail: join(["nothing to release: ", d], "") } HarnessSeatPending { detail: d } => HarnessSeatReleaseRefused { detail: join(["nothing to release, the placement is pending: ", d], "") } + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: a } => + match harness_hosted_release(access: a) { + HostedReleased => HarnessHostedCredentialRemoved + HostedReleaseRefused { detail: d } => HarnessSeatReleaseRefused { detail: d } + } HarnessSeatBound { url: _, grant: g, partition: p, ceiling: ceiling, group_wire: _, launch: _, store: store, model: _, unit: _, class: _, seat: _ } => match now_epoch_seconds() { Absent => HarnessSeatReleaseRefused { detail: "the clock could not be read as epoch seconds" } @@ -1719,6 +1773,7 @@ fn harness_release_fenced(store: FabricStorageBinding, capability: HarnessReleas fn harness_seat_release_wire(r: HarnessSeatRelease) -> String { match r { HarnessSeatReleased { partition: p } => join(["released on ", p as String], "") + HarnessHostedCredentialRemoved => "hosted credential removed" HarnessSeatReleaseRefused { detail: d } => join(["release refused: ", d], "") } } @@ -1726,3 +1781,141 @@ fn harness_seat_release_wire(r: HarnessSeatRelease) -> String { fn harness_now_epoch() -> EpochSecs? { now_epoch_seconds() } + +data hosted_inflight_none: Nat = 0 + +// ── IN-FLIGHT HOSTED SENDS: A CHARGE THAT SURVIVES THE ROLLOVER ───────────────────────────────── +// +// A hosted request is charged to the window it is admitted in (gunbc.harness.harness_hosted_route +// harness_hosted_round_charge), but nothing on this side can bound WHEN its first byte leaves: curl's +// connect deadline stops applying once the connection is open, and a process can pause between the +// admission and the write for as long as it likes. Measured: a pause before the first header write put the +// request's first byte 8.0 s after admission, past a 5.98 s deadline, with curl exiting 0. So a send that +// has been admitted and whose transport has not returned may arrive in ANY later window, and it must be +// counted there too. +// +// THE QUOTA POOLS CANNOT CARRY THAT, because every charge in a lapsing pool belongs to one window. So an +// admitted send also holds a SEAT in a per-credential in-flight partition, under the same pool shape every +// harness seat uses (harness_seat_pool_root: no replenishment, quiescence required -- a held seat charges +// until evidence releases it, never by expiry). Each later admission counts the in-flight seats acquired +// BEFORE its window began and charges its quota pools for them as well as for itself; the seat is released +// when the transport returns, whatever it returned. +// +// TWO CONSEQUENCES, BOTH CONSERVATIVE AND STATED RATHER THAN ENGINEERED AWAY. Two admissions in one window +// each count the same carried seats, so a carried send can be charged more than once in a window. And a +// seat held by a process that crashed charges every later window until it is released; the refusal that +// starvation produces names each held seat, its holder, and the release that frees it -- the same +// harness_release_cli route a stuck serving seat takes. +fn hosted_inflight_partition(quota: HarnessHostedQuota) -> PartitionId { + join(["inflight-", upstream_rate_limit_key(l: quota.per_minute) as String], "") as PartitionId +} + +fn hosted_inflight_ceiling(quota: HarnessHostedQuota) -> Nat { + quota.per_minute.requests +} + +type HostedInflightSeat { + reference: NonEmptyStr + acquired_at: EpochSecs +} + +fn hosted_inflight_held(pool: Pool) -> List { + let held = filter(pool.ledger.entries, e => match e.state { + Held { maximum_liability: _ } => true + Expended { actual: _, settled_at: _ } => false + Released { reason: _, released_at: _ } => false + }) + fold(held, init: [] as List, f: (acc, e) => + match filter(pool.terms, t => (t.reference as String) == (e.reference as String)).first() { + Absent => acc + Present { value: t } => concat(acc, [HostedInflightSeat { reference: e.reference, acquired_at: t.acquired_at }]) + }) +} + +// THE SENDS A WINDOW MUST COVER BESIDES ITS OWN: those admitted before it began and not yet returned. +fn hosted_inflight_carried(pool: Pool, window_start: EpochSecs) -> Nat { + fold(filter(hosted_inflight_held(pool: pool), s => s.acquired_at < window_start), init: hosted_inflight_none, f: (n, _s) => n + 1) +} + +type HostedInflightReading + = HostedInflightRead { pool: Pool, generation: Nat, store: FabricStorageBinding, release_hint: String } + | HostedInflightUnread { detail: String } + +fn hosted_inflight_release_hint(quota: HarnessHostedQuota, pool: Pool, generation: Nat, walk: ChainWalk) -> String { + let seats = hosted_inflight_held(pool: pool) + if count(seats) == 0 { "" } else { + join([ + "; in-flight seats held on ", hosted_inflight_partition(quota: quota) as String, " (release each once its holder is known to be gone: gunbc run --entry dag/gunbc/harness/harness_cli.dag --function harness_release_cli --arg partition=", hosted_inflight_partition(quota: quota) as String, + " --arg ceiling=", to_string(hosted_inflight_ceiling(quota: quota)), " --arg generation=", to_string(generation), " --arg reason= --arg reference= --arg grant=): ", + join(map(seats, s => join([ + s.reference as String, " acquired_at=", to_string(s.acquired_at), + " grant=", match admitting_event_for(walk: walk, reference: s.reference) { Present { value: g } => g as String Absent => "unknown" }, + ], "")), "; "), + ], "") + } +} + +fn harness_hosted_inflight_read(quota: HarnessHostedQuota) -> HostedInflightReading { + match event_log_store_for_host(short_hostname: quota.short_hostname) { + HostStoreRefused { detail: d } => HostedInflightUnread { detail: d } + HostStoreResolved { store: store, executor: _ } => { + let partition = hosted_inflight_partition(quota: quota) + match event_log_read_partition(store: store, partition: partition, budget: 4096) { + PartitionReadRefused { cause: c } => HostedInflightUnread { detail: join(["the in-flight partition could not be read: ", event_log_refusal_wire(cause: c)], "") } + PartitionReadOk { head: _, walk: walk } => + match walk { + ChainIncomplete { missing: m, newest_first_so_far: _ } => HostedInflightUnread { detail: join(["the in-flight partition chain is missing event ", m as String], "") } + ChainBudgetExhausted { at: a } => HostedInflightUnread { detail: join(["the in-flight partition exceeded the read budget at ", a as String], "") } + ChainWalked { oldest_first: xs } => + match pool_fold(root: harness_seat_pool_root(partition: partition, ceiling: hosted_inflight_ceiling(quota: quota)), oldest_first: xs) { + PoolFoldRefused { at_event: e, wire: w } => HostedInflightUnread { detail: join(["the in-flight partition does not fold at ", e as String, ": ", w], "") } + PoolFolded { pool: p, generation: g } => + HostedInflightRead { pool: p, generation: g, store: store, release_hint: hosted_inflight_release_hint(quota: quota, pool: p, generation: g, walk: walk) } + } + } + } + } + } +} + +type HostedInflightHold + = HostedInflightHeld { reference: NonEmptyStr, store: FabricStorageBinding } + | HostedInflightRefused { detail: String } + +fn harness_hosted_inflight_acquire(quota: HarnessHostedQuota, store: FabricStorageBinding, reference: NonEmptyStr, term: Second) -> HostedInflightHold { + match now_epoch_seconds() { + Absent => HostedInflightRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => { + let partition = hosted_inflight_partition(quota: quota) + let term_seconds = second_count(s: term) + match fabric_seat_acquire( + store: store, partition: partition, + root: harness_seat_pool_root(partition: partition, ceiling: hosted_inflight_ceiling(quota: quota)), + actor: quota.actor, + request: SeatRequest { reference: reference, amount: Measure { count: 1 }, at: now, term_seconds: term_seconds }, + policy: LeasePolicy { maximum_duration_seconds: term_seconds, release_law: QuiescenceRequired }, + attempts: 3, budget: 4096, + ) { + SeatGranted { grant: _, generation: _, attempts_used: _ } => HostedInflightHeld { reference: reference, store: store } + SeatFull { wire: w, attempts_used: _ } => HostedInflightRefused { detail: join(["every in-flight seat is held (", w, "), so another send cannot be accounted for"], "") } + other => HostedInflightRefused { detail: seat_acquisition_wire(a: other) } + } + } + } +} + +fn harness_hosted_inflight_release(quota: HarnessHostedQuota, hold: HostedInflightHold, reason: NonEmptyStr) -> String? { + match hold { + HostedInflightRefused { detail: _ } => none + HostedInflightHeld { reference: r, store: store } => + match now_epoch_seconds() { + Absent => Present { value: join(["the in-flight seat ", r as String, " could not be released: the clock could not be read"], "") } + Present { value: now } => + match release_retry(store: store, partition: hosted_inflight_partition(quota: quota), ceiling: hosted_inflight_ceiling(quota: quota), reference: r, reason: reason, now: now) { + HarnessSeatReleased { partition: _ } => none + HarnessHostedCredentialRemoved => Present { value: join(["the in-flight seat ", r as String, " release answered as a credential removal"], "") } + HarnessSeatReleaseRefused { detail: d } => Present { value: join(["the in-flight seat ", r as String, " could not be released: ", d], "") } + } + } + } +} diff --git a/dag/gunbc/harness/harness_turn.dag b/dag/gunbc/harness/harness_turn.dag index b96b65119c4..3c56aefa019 100644 --- a/dag/gunbc/harness/harness_turn.dag +++ b/dag/gunbc/harness/harness_turn.dag @@ -13,6 +13,22 @@ import gunbc.harness.harness_reasoning_wire { harness_unit_thinking_end_token, harness_unit_reasoning_field, harness_serving_unit_wire, harness_reasoning_level_wire, } +import gunbc.harness.harness_hosted_route { + HarnessBackendAccess, BackendAdmittedBySeat, BackendAdmittedByHostedQuota, HarnessHostedQuota, HostedRoundCharged, + HostedRoundQuotaFull, HostedRoundUnleasable, harness_hosted_round_charge, harness_hosted_refusal_cause, + HostedRoundLease, +} +import extdeps.openrouter.openrouter { + OpenRouterRateLimitRefusal, OpenRouterHeaderDump, HeaderDumpRead, HeaderDumpUnread, openrouter_decode_rate_limited, openrouter_rate_limit_wire, +} +import gunbc.harness.harness_seat { + HostedInflightRead, HostedInflightUnread, HostedInflightHeld, HostedInflightRefused, harness_hosted_inflight_read, + harness_hosted_inflight_acquire, harness_hosted_inflight_release, hosted_inflight_carried, +} +import gunbc.fabric_quota { quota_window_start } +import gunbc.fabric_event_log_host { now_epoch_seconds } +import gunbc.fabric_storage_client { FabricStorageBinding +} import extdeps.clock { ClockUnixMillisRead, ClockUnixMillisObserved, ClockUnixMillisRefused, clock_unix_millis_read } import extdeps.http.client import extdeps.git.inspect @@ -244,6 +260,7 @@ type HarnessTurnConfig { request_deadline: Second environment: HarnessToolEnvironment completion: HarnessTurnCompletion + backend_access: HarnessBackendAccess } // THE WINDOW THE HARNESS WORKS IN AND THE WINDOW THE ENGINE ADMITS ARE ONE NUMBER, and this field @@ -539,6 +556,9 @@ type HarnessTurnOutcome | TurnUsageUnavailable { cause: String, steps_taken: Int } | TurnSubmissionUndecodable { reason: String, steps_taken: Int } | TurnAlignmentCheckpointDue { steps_taken: Int } + | TurnHostedQuotaFull { wire: String, steps_taken: Int } + | TurnHostedQuotaUnleasable { detail: String, steps_taken: Int } + | TurnHostedRateLimited { refusal: OpenRouterRateLimitRefusal, steps_taken: Int } fn harness_outcome_label(outcome: HarnessTurnOutcome) -> String { match outcome { @@ -557,6 +577,9 @@ fn harness_outcome_label(outcome: HarnessTurnOutcome) -> String { TurnUsageUnavailable { cause: _, steps_taken: _ } => "usage-unavailable" TurnSubmissionUndecodable { reason: _, steps_taken: _ } => "submission-undecodable" TurnAlignmentCheckpointDue { steps_taken: _ } => "alignment-checkpoint-due" + TurnHostedQuotaFull { wire: _, steps_taken: _ } => "hosted-quota-full" + TurnHostedQuotaUnleasable { detail: _, steps_taken: _ } => "hosted-quota-unleasable" + TurnHostedRateLimited { refusal: _, steps_taken: _ } => "hosted-rate-limited" } } @@ -613,6 +636,21 @@ fn harness_outcome_detail(outcome: HarnessTurnOutcome) -> String { " control to the workflow, whose adjudication lane decides whether the same lineage", " resumes, steers, re-plans, routes or stops. This is not a failure and not a completion.", ], "") + TurnHostedQuotaFull { wire: w, steps_taken: n } => + join([ + "after ", to_string(value: n), " tool rounds the local quota ledger refused the request (", w, + "); it was not sent. The ledger is this fleet's own pre-check, not an answer from the upstream, whose count it can differ from at a window boundary", + ], "") + TurnHostedQuotaUnleasable { detail: d, steps_taken: n } => + join([ + "after ", to_string(value: n), " tool rounds the hosted upstream's quota could not be leased (", d, + "); the request was not sent, because an unleased request is one the fleet cannot account for", + ], "") + TurnHostedRateLimited { refusal: r, steps_taken: k } => + join([ + "after ", to_string(value: k), " tool rounds ", openrouter_rate_limit_wire(r: r), + "; the turn stopped rather than retrying, and a caller that waits owns when it asks again", + ], "") TurnTransportRefused { detail: d } => d TurnResponseUnreadable { cause: c } => c TurnRequestUnwritable { path: p, detail: d } => @@ -1042,8 +1080,29 @@ fn harness_thinking_format_cause(unit: HarnessServingUnit, blocks: List String { + serialize_json(v: json_object([ + json_kv(key: "type", value: json_string(s: "turn.inflight_release_unrecorded")), + json_kv(key: "detail", value: json_string(s: detail)), + ])) } +// A HOSTED ROUND THAT COULD NOT BE CHARGED IS NOT POSTED, and the loop is told which way it was refused. +// The charge is recorded before the request goes out (gunbc.harness.harness_hosted_route +// harness_hosted_round_charge), so nothing about the round's quota is left to do after the answer. +type HarnessRoundPost + = RoundPosted { posted: HarnessPostedRound } + | RoundQuotaFull { wire: String } + | RoundQuotaUnleasable { detail: String } + + fn harness_wire_log_path(config: HarnessTurnConfig, round: Int, kind: String) -> String { join([config.events_path as String, ".wire/", to_string(value: round), kind], "") } @@ -1056,10 +1115,10 @@ fn harness_post_round( config: HarnessTurnConfig, messages: List, round: Int, -) -> HarnessPostedRound { +) -> HarnessRoundPost { match harness_reasoning_kwargs_for(unit: config.serving_unit, level: config.reasoning_mode) { ReasoningLevelUnsupported { unit, level, cause } => - HarnessPostedRound { + RoundPosted { posted: HarnessPostedRound { read: HarnessResponseUnreadable { cause: join([ "the serving unit ", unit as String, " cannot express reasoning level ", level as String, @@ -1067,21 +1126,93 @@ fn harness_post_round( ], ""), }, wire_log_detail: none, + inflight_release_detail: none, + rate_limited: none, + } } + ReasoningKwargsAdmitted { kwargs } => + match harness_prepare_round(config: config, messages: messages, reasoning_kwargs: kwargs, round: round) { + RoundUnprepared { posted: p } => RoundPosted { posted: p } + RoundPrepared { wire_request_detail: wd } => + match config.backend_access { + BackendAdmittedBySeat => RoundPosted { posted: harness_send_round(config: config, round: round, header_file: none, wire_request_detail: wd) } + BackendAdmittedByHostedQuota { header_file: hf, quota: q } => harness_hosted_round_post(config: config, round: round, header_file: hf, quota: q, wire_request_detail: wd) + } } - ReasoningKwargsAdmitted { kwargs } => harness_post_round_admitted(config: config, messages: messages, reasoning_kwargs: kwargs, round: round) } } -fn harness_post_round_admitted( +// A HOSTED ROUND IS PREPARED, THEN CHARGED, THEN SENT -- in that order, because the charge is counted in +// the window the request ARRIVES in. Everything that can take time on this side (serializing the body, +// writing the scratch and the wire log) is done before the charge, so the charge is as close to the send +// as this process can put it. Nothing here bounds when the first byte actually leaves, so the send also +// holds an in-flight seat from before it until the transport returns (gunbc.harness.harness_seat, in-flight +// hosted sends): every later window counts the seats still held from earlier ones and charges for them as +// well as for its own send. +// +// THE CHARGE IS A CLOSE PRE-CHECK AND THE UPSTREAM'S 429 IS THE BACKSTOP (operator decision, escalation +// msg_46a417a3; gunbc.harness.harness_hosted_route): a request the pre-check admitted that OpenRouter +// still refuses with 429 ends the turn as TurnHostedRateLimited, carrying the upstream's own retry hints, +// and is never retried here. +fn harness_hosted_round_post(config: HarnessTurnConfig, round: Int, header_file: NonEmptyStr, quota: HarnessHostedQuota, wire_request_detail: String?) -> HarnessRoundPost { + match harness_hosted_inflight_read(quota: quota) { + HostedInflightUnread { detail: d } => RoundQuotaUnleasable { detail: join(["the in-flight sends could not be counted, so the charge cannot cover them: ", d], "") } + HostedInflightRead { pool: p, generation: _, store: store, release_hint: hint } => + match now_epoch_seconds() { + Absent => RoundQuotaUnleasable { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match quota_window_start(limit: quota.per_minute, at: now) { + Absent => RoundQuotaUnleasable { detail: "the minute window has no start at this instant" } + Present { value: minute_start } => + match quota_window_start(limit: quota.per_day, at: now) { + Absent => RoundQuotaUnleasable { detail: "the day window has no start at this instant" } + Present { value: day_start } => + harness_hosted_round_charged_post( + config: config, round: round, header_file: header_file, quota: quota, wire_request_detail: wire_request_detail, store: store, + reference: join([quota.actor as String, "@", to_string(value: now), "#round", to_string(value: round)], "") as NonEmptyStr, + charge: harness_hosted_round_charge( + quota: quota, + carried_minute: hosted_inflight_carried(pool: p, window_start: minute_start), + carried_day: hosted_inflight_carried(pool: p, window_start: day_start), + release_hint: hint, + ), + ) + } + } + } + } +} + +fn harness_hosted_round_charged_post(config: HarnessTurnConfig, round: Int, header_file: NonEmptyStr, quota: HarnessHostedQuota, wire_request_detail: String?, store: FabricStorageBinding, reference: NonEmptyStr, charge: HostedRoundLease) -> HarnessRoundPost { + match charge { + HostedRoundQuotaFull { wire: w } => RoundQuotaFull { wire: w } + HostedRoundUnleasable { detail: d } => RoundQuotaUnleasable { detail: d } + HostedRoundCharged => + match harness_hosted_inflight_acquire(quota: quota, store: store, reference: reference, term: config.request_deadline) { + HostedInflightRefused { detail: d } => RoundQuotaUnleasable { detail: d } + HostedInflightHeld { reference: r, store: st } => { + let hold = HostedInflightHeld { reference: r, store: st } + let posted = harness_send_round(config: config, round: round, header_file: Present { value: header_file }, wire_request_detail: wire_request_detail) + let released = harness_hosted_inflight_release(quota: quota, hold: hold, reason: "the transport returned" as NonEmptyStr) + RoundPosted { posted: HarnessPostedRound { read: posted.read, wire_log_detail: posted.wire_log_detail, inflight_release_detail: released, rate_limited: posted.rate_limited } } + } + } + } +} + +type HarnessRoundPreparation + = RoundPrepared { wire_request_detail: String? } + | RoundUnprepared { posted: HarnessPostedRound } + +fn harness_prepare_round( config: HarnessTurnConfig, messages: List, reasoning_kwargs: List, round: Int, -) -> HarnessPostedRound { +) -> HarnessRoundPreparation { let body = serialize_json(v: harness_request_json(config: config, messages: messages, reasoning_kwargs: reasoning_kwargs)) let written = Filesystem.Write(path: config.request_scratch_path as String, content: body) if !written.success { - HarnessPostedRound { + RoundUnprepared { posted: HarnessPostedRound { read: HarnessResponseUnreadable { cause: join([ "request body could not be written to ", config.request_scratch_path as String, @@ -1089,15 +1220,25 @@ fn harness_post_round_admitted( ], ""), }, wire_log_detail: none, - } + inflight_release_detail: none, + rate_limited: none, + } } } else { let wire_request_path = harness_wire_log_path(config: config, round: round, kind: ".request.json") let wire_request = Filesystem.Write(path: wire_request_path, content: body) - let posted = http.Client.PostJsonFromFile( - url: config.backend_url as NonEmptyStr, - request_body_file: config.request_scratch_path as String as NonEmptyStr, - max_seconds: to_string(value: second_count(s: config.request_deadline)) as NonEmptyStr, - ) + RoundPrepared { + wire_request_detail: if wire_request.success { none } else { Present { value: join(["request wire log ", wire_request_path, " could not be written: ", wire_request.error], "") } }, + } + } +} + +fn harness_send_round( + config: HarnessTurnConfig, + round: Int, + header_file: NonEmptyStr?, + wire_request_detail: String?, +) -> HarnessPostedRound { + let posted = harness_post_json(config: config, header_file: header_file) let wire_response_path = harness_wire_log_path(config: config, round: round, kind: ".response.json") let wire_response = Filesystem.Write(path: wire_response_path, content: posted.body) let kept = Filesystem.Write( @@ -1105,7 +1246,7 @@ fn harness_post_round_admitted( content: posted.body, ) let wire_details = concat( - if wire_request.success { [] } else { [join(["request wire log ", wire_request_path, " could not be written: ", wire_request.error], "")] }, + match wire_request_detail { Present { value: d } => [d] Absent => [] }, if wire_response.success { [] } else { [join(["response wire log ", wire_response_path, " could not be written: ", wire_response.error], "")] }, ) let wire_log_detail = if count(wire_details) == 0 { @@ -1117,10 +1258,12 @@ fn harness_post_round_admitted( HarnessPostedRound { read: HarnessResponseUnreadable { cause: join([ - "the backend refused the request or the transport failed; body was: ", posted.body, + "the backend refused the request or the transport failed; ", harness_refusal_body_text(config: config, body: posted.body), ], ""), }, wire_log_detail: wire_log_detail, + inflight_release_detail: none, + rate_limited: harness_hosted_rate_limited(config: config, posted: posted), } } else if !kept.success { HarnessPostedRound { @@ -1128,16 +1271,84 @@ fn harness_post_round_admitted( cause: join(["the response could not be kept beside the request: ", kept.error, "; refusing to decode a body no one can re-read"], ""), }, wire_log_detail: wire_log_detail, + inflight_release_detail: none, + rate_limited: none, } } else { HarnessPostedRound { read: harness_decode_for_shape(unit: config.serving_unit, shape: config.wire_shape, body: posted.body), wire_log_detail: wire_log_detail, + inflight_release_detail: none, + rate_limited: none, + } + } +} + +// ONE POST, TWO ADMISSIONS. A seat-admitted backend is the fleet's own engine over its network and takes +// no credential; a quota-admitted one is a hosted upstream and takes the bearer header file by path. +type HarnessPostOutcome { + body: String + success: Bool + headers: OpenRouterHeaderDump? +} + +// A 429 FROM A HOSTED UPSTREAM IS A TYPED REFUSAL WITH THE UPSTREAM'S OWN RETRY HINTS, decoded from the +// error body and the response header dump; it is never retried here. A fleet engine has no such refusal. +fn harness_hosted_rate_limited(config: HarnessTurnConfig, posted: HarnessPostOutcome) -> OpenRouterRateLimitRefusal? { + match config.backend_access { + BackendAdmittedBySeat => none + BackendAdmittedByHostedQuota { header_file: _, quota: _ } => + openrouter_decode_rate_limited( + body: posted.body, + header_dump: match posted.headers { + Present { value: d } => d + Absent => HeaderDumpUnread { cause: "no response header dump was taken for this request" } + }, + ) + } +} + +fn harness_response_headers_path(config: HarnessTurnConfig) -> String { + join([config.request_scratch_path as String, ".response.headers"], "") +} + +fn harness_post_json(config: HarnessTurnConfig, header_file: NonEmptyStr?) -> HarnessPostOutcome { + match header_file { + Absent => { + let posted = http.Client.PostJsonFromFile( + url: config.backend_url as NonEmptyStr, + request_body_file: config.request_scratch_path as String as NonEmptyStr, + max_seconds: to_string(value: second_count(s: config.request_deadline)) as NonEmptyStr, + ) + HarnessPostOutcome { body: posted.body, success: posted.success, headers: none } + } + Present { value: hf } => { + let posted = http.Client.PostJsonFromFileWithHeaderFile( + url: config.backend_url as NonEmptyStr, + request_body_file: config.request_scratch_path as String as NonEmptyStr, + header_file: hf, + response_headers_file: harness_response_headers_path(config: config) as NonEmptyStr, + max_seconds: to_string(value: second_count(s: config.request_deadline)) as NonEmptyStr, + ) + let dump = Filesystem.Read(path: harness_response_headers_path(config: config)) + HarnessPostOutcome { + body: posted.body, + success: posted.success, + headers: Present { value: if dump.success { HeaderDumpRead { text: dump.content } } else { HeaderDumpUnread { cause: join(["the curl header dump ", harness_response_headers_path(config: config), " could not be read: ", dump.error], "") } } }, } } } } +// A HOSTED UPSTREAM'S REFUSAL IS DECODED TO ITS TYPED ERROR, so a 402 says which budget refused and a +// 429 says whose limit it was; a fleet engine's body is reported as it came. +fn harness_refusal_body_text(config: HarnessTurnConfig, body: String) -> String { + match config.backend_access { + BackendAdmittedBySeat => join(["body was: ", body], "") + BackendAdmittedByHostedQuota { header_file: _, quota: _ } => join(["upstream answered: ", harness_hosted_refusal_cause(body: body)], "") + } +} + // THE LOOP. Each round either reaches a provider terminal, or executes the tool calls and recurses // with a strictly smaller budget. There is no arm that continues without either consuming a step or // terminating, which is what makes the descent argument readable rather than asserted. @@ -1276,19 +1487,61 @@ fn harness_turn_loop( harness_turn_checkpoint_due(config: config, steps_taken: steps_taken, events: events) } } else { - let posted_round = harness_post_round(config: config, messages: messages, round: steps_taken + 1) - let events_after_wire = match posted_round.wire_log_detail { - Present { value: detail } => concat(events, [harness_event_wire_log_unwritable(detail: detail)]) - Absent => events - } - match posted_round.read { - HarnessResponseUnreadable { cause: c } => { - let failed = concat(events_after_wire, [harness_event_turn_failed(message: c)]) + match harness_post_round(config: config, messages: messages, round: steps_taken + 1) { + RoundQuotaFull { wire: w } => { + let failed = concat(events, [harness_event_turn_failed(message: join(["hosted quota full: ", w], ""))]) + match harness_write_events(config: config, events: failed) { + HarnessEventsUnwritable { detail: d } => harness_event_write_refusal(config: config, detail: d) + HarnessEventsWritten => TurnHostedQuotaFull { wire: w, steps_taken: steps_taken } + } + } + RoundQuotaUnleasable { detail: qd } => { + let failed = concat(events, [harness_event_turn_failed(message: join(["hosted quota unleasable: ", qd], ""))]) match harness_write_events(config: config, events: failed) { HarnessEventsUnwritable { detail: d } => harness_event_write_refusal(config: config, detail: d) - HarnessEventsWritten => TurnResponseUnreadable { cause: c } + HarnessEventsWritten => TurnHostedQuotaUnleasable { detail: qd, steps_taken: steps_taken } } } + RoundPosted { posted: posted_round } => harness_turn_after_post(config: config, messages: messages, steps_remaining: steps_remaining, steps_taken: steps_taken, events: events, repairs_used: repairs_used, posted_round: posted_round) + } + } +} + +fn harness_turn_after_post( + config: HarnessTurnConfig, + messages: List, + steps_remaining: Int, + steps_taken: Int, + events: List, + repairs_used: Int, + posted_round: HarnessPostedRound, +) -> HarnessTurnOutcome { + let events_after_release = match posted_round.inflight_release_detail { + Present { value: detail } => concat(events, [harness_event_inflight_release_unrecorded(detail: detail)]) + Absent => events + } + let events_after_wire = match posted_round.wire_log_detail { + Present { value: detail } => concat(events_after_release, [harness_event_wire_log_unwritable(detail: detail)]) + Absent => events_after_release + } + match posted_round.read { + HarnessResponseUnreadable { cause: c } => + match posted_round.rate_limited { + Present { value: limited } => { + let failed = concat(events_after_wire, [harness_event_turn_failed(message: openrouter_rate_limit_wire(r: limited))]) + match harness_write_events(config: config, events: failed) { + HarnessEventsUnwritable { detail: d } => harness_event_write_refusal(config: config, detail: d) + HarnessEventsWritten => TurnHostedRateLimited { refusal: limited, steps_taken: steps_taken } + } + } + Absent => { + let failed = concat(events_after_wire, [harness_event_turn_failed(message: c)]) + match harness_write_events(config: config, events: failed) { + HarnessEventsUnwritable { detail: d } => harness_event_write_refusal(config: config, detail: d) + HarnessEventsWritten => TurnResponseUnreadable { cause: c } + } + } + } HarnessResponseParsed { blocks: raw_blocks, stop_reason, usage: usage_read } => match usage_read { UsageUnreadable { cause: uc } => { @@ -1315,7 +1568,6 @@ fn harness_turn_loop( ) } } - } } // THE ROUND THAT DECODED AND WAS COUNTED. Factored out of the loop when usage became a typed read: diff --git a/dag/gunbc/instruments/fabric_seat_probe.dag b/dag/gunbc/instruments/fabric_seat_probe.dag index 643e939fffe..1b4e8dbe30c 100644 --- a/dag/gunbc/instruments/fabric_seat_probe.dag +++ b/dag/gunbc/instruments/fabric_seat_probe.dag @@ -16,7 +16,7 @@ import product.capacity.pool_events { import product.capacity.lease { LeasePolicy, QuiescenceRequired } import gunbc.harness.harness_wire { OpenAiChatCompletionsShape } import gunbc.harness.harness_seat { harness_turn_request, - HarnessSeatBound, HarnessSeatPending, HarnessSeatRefused, harness_bind_seat, HarnessSeatReleased, + HarnessSeatBound, HarnessSeatPending, HarnessSeatRefused, HarnessHostedBound, harness_bind_seat, HarnessSeatReleased, HarnessHostedCredentialRemoved, HarnessSeatReleaseRefused, harness_release_seat, harness_seat_release_wire, harness_now_epoch, AnyAdmittedModel, } @@ -62,6 +62,7 @@ fn fabric_seat_probe(receipt: NonEmptyStr) -> ProcessExit { match binding { HarnessSeatRefused { detail: d } => exit_failure(reason: join(["seat probe: bind refused: ", d], "")) HarnessSeatPending { detail: d } => exit_failure(reason: join(["seat probe: placement pending: ", d], "")) + HarnessHostedBound { url: _, model: _, unit: _, level: _, access: _ } => exit_failure(reason: "seat probe: an any-model bind answered with a hosted binding, which harness_requirement_placement never yields") HarnessSeatBound { url: u, grant: g, partition: p, ceiling: _, group_wire: gw, launch: launch, store: _, model: m, unit: _, class: c, seat: _ } => { let line = join([ "host=", short, " group=", gw, " launch=", launch.process_start_field as String, @@ -73,6 +74,7 @@ fn fabric_seat_probe(receipt: NonEmptyStr) -> ProcessExit { let seat_release = harness_release_seat(binding: binding, reason: "seat-probe complete" as NonEmptyStr) let w = Filesystem.Write(path: receipt as String, content: join([line, harness_seat_release_wire(r: seat_release), "\n"], "")) match seat_release { + HarnessHostedCredentialRemoved => exit_failure(reason: "seat probe: a fleet seat release answered as a hosted credential removal") HarnessSeatReleased { partition: _ } => if w.success { ExitSuccess } else { exit_failure(reason: join(["seat probe: receipt write failed: ", w.error], "")) } HarnessSeatReleaseRefused { detail: d } => exit_failure(reason: join(["seat probe: granted but release refused: ", d], "")) } diff --git a/dag/gunbc/product/capacity/pool.dag b/dag/gunbc/product/capacity/pool.dag index ca90795298f..a0197d034cf 100644 --- a/dag/gunbc/product/capacity/pool.dag +++ b/dag/gunbc/product/capacity/pool.dag @@ -384,7 +384,13 @@ fn lapse_pass(pool: Pool, at: EpochSecs) -> LapsePass { // reference that names the reading, for the rest of the window, so the fold shows it and a later // reading can supersede it. A reading that says more remains changes nothing: the ledger's // outstanding leases are the thing the upstream cannot see. -fn observe_upstream_remaining(pool: Pool, remaining: Measure, at: EpochSecs, term_seconds: Nat) -> PoolOutcome { +// +// THE READING IS NAMED BY ITS OWN IDENTITY, NOT BY ITS SECOND. The hold's reference was +// "upstream-observation@", so two decreasing readings in one second minted the same +// reference and the second was a duplicate the fold refused -- making the whole partition +// unfoldable on replay. The caller now supplies the reading's identity: at replay that is the +// event's own id (product.capacity.pool_events pool_apply_event), which is unique by construction. +fn observe_upstream_remaining(pool: Pool, remaining: Measure, at: EpochSecs, term_seconds: Nat, reading: NonEmptyStr) -> PoolOutcome { match pool_reading_at(pool: pool, at: at) { PoolReadingRefused { at: t, anchor: a } => PoolRefused { refusal: PoolInstantBeforeAnchor { anchor: a, at: t } } PoolRead { committed: _, ceiling: _, headroom: h, over_committed: _ } => @@ -393,7 +399,7 @@ fn observe_upstream_remaining(pool: Pool, remaining: Measure EventDecode { } fn pool_apply_event(pool: Pool, env: ChainEnvelope) -> PoolOutcome { - let at = env.event.recorded_at - match env.event.payload { + pool_apply_payload(pool: pool, payload: env.event.payload, at: env.event.recorded_at, identity: env.id as String as NonEmptyStr) +} + +// THE ONE TRANSITION FUNCTION, shared by replay and by validation before an append +// (gunbc.fabric_event_log fabric_pool_transition). An event that would be refused here on replay is +// refused there before it is written, because both ask this function: a carrier that appended without +// asking it could commit a transition no later fold can apply, which makes the whole partition +// unfoldable. The identity names the event -- its id at replay, the head it was decided against +// before append -- and is what a transition that mints its own hold (an upstream reading) is keyed by. +fn pool_apply_payload(pool: Pool, payload: PoolEvent, at: EpochSecs, identity: NonEmptyStr) -> PoolOutcome { + match payload { PoolAcquired { reference: r, amount: a, term_seconds: t } => pool_replay_acquire(pool: pool, reference: r, amount: Measure { count: a }, at: at, term_seconds: t) PoolSettled { reference: r, actual: a } => pool_settle(pool: pool, reference: r, actual: Measure { count: a }, at: at) PoolReleased { reference: r, reason: why } => pool_release(pool: pool, reference: r, reason: why, at: at) PoolLapsed => PoolAdvanced { pool: lapse_pass(pool: pool, at: at).pool } - PoolUpstreamObserved { remaining: n, term_seconds: t } => observe_upstream_remaining(pool: pool, remaining: Measure { count: n }, at: at, term_seconds: t) + PoolUpstreamObserved { remaining: n, term_seconds: t } => observe_upstream_remaining(pool: pool, remaining: Measure { count: n }, at: at, term_seconds: t, reading: identity) PoolResetObserved { reset_at: r } => PoolAdvanced { pool: pool_reanchor(pool: pool, reset_at: r) } } } diff --git a/dag/gunbc/provider_standing_probe_bridge.dag b/dag/gunbc/provider_standing_probe_bridge.dag index a198f42a724..7d6057eac37 100644 --- a/dag/gunbc/provider_standing_probe_bridge.dag +++ b/dag/gunbc/provider_standing_probe_bridge.dag @@ -43,6 +43,7 @@ import gunbc.provider_standing { } import gunbc.codex_app_server_press { CodexAccountStandingReceipt, + CodexAccountStandingEvidence, AmbientCodexCredentialContext, CodexRateLimitBucketVerdict, CodexBucketCapacityAvailable, @@ -366,17 +367,38 @@ data codex_press_standing_bundle_tool_environment_unobserved_reason: NonEmptyStr fn provider_standing_bundle_from_codex_press_receipt( receipt: CodexAccountStandingReceipt, ) -> ProviderStandingBundle { - ProviderStandingBundle { + provider_standing_bundle_from_codex_account_evidence( + evidence: receipt.standing.evidence, + credential: receipt.subject.credential, installation: provider_standing_bundle_unobserved_installation( reason: codex_press_standing_bundle_installation_unobserved_reason, ), - authentication: provider_authentication_from_codex_press_receipt(receipt: receipt), - entitlement: provider_entitlement_from_codex_press_receipt(receipt: receipt), - limits: provider_limit_observations_from_codex_press_receipt(receipt: receipt), - turn: TurnNotStarted, tool_environment: provider_standing_bundle_unobserved_tool_environment( reason: codex_press_standing_bundle_tool_environment_unobserved_reason, ), + ) +} + +// THE ACCOUNT TRIP'S EVIDENCE AS A STANDING, for a caller that ran the trip itself and so knows the +// two axes the receipt bridge cannot: whether the executable ran (installation) and whether the +// spawn's environment was admitted (tool environment). The receipt bridge above is this with both +// unobserved; the authentication, entitlement and limit projections are the same functions. +fn provider_standing_bundle_from_codex_account_evidence( + evidence: CodexAccountStandingEvidence, + credential: AmbientCodexCredentialContext, + installation: ProviderInstallationStanding, + tool_environment: ProviderToolEnvironmentStanding, +) -> ProviderStandingBundle { + ProviderStandingBundle { + installation: installation, + authentication: provider_authentication_from_codex_credential_liveness(liveness: evidence.credential_liveness), + entitlement: provider_entitlement_from_codex_realm(realm: evidence.account_realm), + limits: provider_limit_observations_from_codex_buckets( + credential: codex_press_credential_identity(credential: credential), + buckets: evidence.rate_limit_buckets, + ), + turn: TurnNotStarted, + tool_environment: tool_environment, } } diff --git a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag index 73023923c91..79d18c7d35a 100644 --- a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag +++ b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag @@ -129,6 +129,7 @@ import extdeps.posix.sh_invocation { posix_sh_export_from_file_unreadable_exit, import gunbc.dispatch_selection { ClaudeCredentialRef, ClaudeOAuthTokenSecretRef, ClaudeApiKeySecretRef, ClaudeStateRootRef, claude_dispatch_credential, inventory_with_observed_standing, claude_offer_for_instance, + inventory_with_codex_state_root, OfferCodex, OfferClaude, OfferCursor, ExecutionProfile, ProfileAutomatic, ProviderSelectionRequest, @@ -211,6 +212,9 @@ import gunbc.roadmap_provider_events { codex_provider_event_projection_args, } import gunbc.repo_identity { gunbc_merge_target_ref } +import gunbc.codex_harness_credential { codex_worker_turn_custody_host, codex_worker_turn_codex_home, codex_harness_turn_argv } +import gunbc.worker_turn_standing { codex_observed_standing_for_instance, cursor_observed_standing_from_snapshot } +import gunbc.cursor_harness_credential { cursor_worker_turn_api_key_host_path, cursor_worker_turn_custody_host, cursor_worker_turn_argv, CursorWorkerTurnArgvReady, CursorWorkerTurnRefused } import gunbc.roadmap_dispatch_destination { node_dispatch_destination, DispatchDestinationResolved, @@ -669,6 +673,130 @@ fn dispatch_actuator_claude_selection_observed( } } +// THE CODEX AND CURSOR DRAWS OBSERVE BEFORE THEY SELECT, as the Claude draw does: the declared row's +// standing admits no draw, so each request runs its custody-bound observation +// (gunbc.worker_turn_standing), substitutes it into the declared row and selects over that. A rejected +// credential, an exhausted codex bucket, a busy codex lock or a credential held on another host +// therefore refuses typed before any spawn. +fn codex_offer_provider_instance(inventory: ProviderInventory) -> ProviderInstance? { + fold(inventory.offers, init: none, f: (acc, offer) => + match acc { + Present { value: _ } => acc + Absent => + match offer { + OfferCodex { offer: o } => Present { value: o.instance } + OfferClaude { offer: _ } => acc + OfferCursor { offer: _ } => acc + } + } + ) +} + +// THE CODEX DRAW binds the custody CODEX_HOME directory itself, not a copy: a copied auth.json would +// fork the refresh token, and the issuer refuses a reused one. So the shared one-snapshot rule +// (dispatch_credential_snapshot_path_for_instance) is met by the directory and its flock instead -- +// admission observes it under the lock, the turn runs in it under the lock, and nothing else ever +// writes it but codex under that lock. A turn that runs between admission and this attempt's spawn +// may move the bucket the admission read; the spawn still binds the same credential, and a bucket +// exhausted in that window fails the turn as the CLI reports it. +fn codex_observed_inventory(instance: HostDashboardInstance) -> ProviderInventory { + let inventory = provider_inventory_for_instance(instance: instance) + inventory_with_codex_state_root( + inventory: inventory_with_observed_standing( + inventory: inventory, + kind: CodexCliProvider, + standing: match codex_offer_provider_instance(inventory: inventory) { + Absent => ProviderStandingUnobserved { reason: "this instance's inventory declares no codex offer" as NonEmptyStr } + Present { value: provider_instance } => codex_observed_standing_for_instance(instance: instance, provider: provider_instance) + }, + ), + account_state_root: codex_worker_turn_codex_home as FilePath, + ) +} + +fn worker_turn_selection_of(inventory: ProviderInventory, kind: ProviderKind, sizing_expectation: SizingProfileExpectation) -> DispatchActuatorSelection { + match resolve_dispatch_selection( + inventory: inventory, + request: provider_selection_request_for_kind(kind: kind), + sizing_expectation: sizing_expectation, + ) { + DispatchSelectionResolved { receipt } => + DispatchActuatorSelectionOk { + provider: provider_kind_to_dispatch_cli(kind: receipt.provider_kind), + effort: receipt.applied_effort, + model: resolved_model_selection(resolved: receipt.resolved_selection), + process_fingerprint: resolved_process_fingerprint(resolved: receipt.resolved_selection), + } + DispatchSelectionRefused { sizing_expectation: _, requested_selection: _, reason } => + DispatchActuatorSelectionRefused { reason: reason } + } +} + +// THE CURSOR DRAW takes the attempt's credential snapshot first (shared rule above): the status probe +// reads that snapshot and keeps it, and the spawn loads the same snapshot and removes it before exec. A +// refused draw discards the snapshot it took, as the Claude draw does; after selection the belt owns +// the discard (gunbc.roadmap_belt_actuate belt_snapshot_owned_outcome). +fn cursor_selection_observed( + instance: HostDashboardInstance, + node_id: RoadmapNodeId, + attempt_key: String, + sizing_expectation: SizingProfileExpectation, +) -> DispatchActuatorSelection { + if (instance.host_identity as String) != (cursor_worker_turn_custody_host as String) { + DispatchActuatorSelectionRefused { + reason: provider_off_custody_host_reason(provider_label: "cursor", custody_host: cursor_worker_turn_custody_host, instance_host: instance.host_identity), + } + } else { + let snapshot_path = dispatch_credential_snapshot_path_for_instance( + instance: instance, node_id: node_id, attempt_key: attempt_key, provider: CursorDispatchProvider, + ) + match dispatch_credential_snapshot_take(source: cursor_worker_turn_api_key_host_path, snapshot: snapshot_path) { + CredentialSnapshotRefused { reason } => DispatchActuatorSelectionRefused { reason: reason } + CredentialSnapshotTaken { path: snapshot } => { + let selection = worker_turn_selection_of( + inventory: inventory_with_observed_standing( + inventory: provider_inventory_for_instance(instance: instance), + kind: CursorCliProvider, + standing: cursor_observed_standing_from_snapshot(snapshot: snapshot), + ), + kind: CursorCliProvider, + sizing_expectation: sizing_expectation, + ) + match selection { + DispatchActuatorSelectionOk { provider: _, effort: _, model: _, process_fingerprint: _ } => selection + DispatchActuatorSelectionRefused { reason } => + if dispatch_credential_snapshot_discard(snapshot: snapshot) { + selection + } else { + DispatchActuatorSelectionRefused { + reason: join([reason as String, "; and the credential snapshot ", snapshot as String, " could not be removed"], "") as NonEmptyStr, + } + } + } + } + } + } +} + +fn dispatch_actuator_worker_turn_selection_observed( + instance: HostDashboardInstance, + provider: DispatchCliProvider, + node_id: RoadmapNodeId, + attempt_key: String, + sizing_expectation: SizingProfileExpectation, +) -> DispatchActuatorSelection { + match provider { + CodexDispatchProvider => + worker_turn_selection_of(inventory: codex_observed_inventory(instance: instance), kind: CodexCliProvider, sizing_expectation: sizing_expectation) + CursorDispatchProvider => + cursor_selection_observed(instance: instance, node_id: node_id, attempt_key: attempt_key, sizing_expectation: sizing_expectation) + ClaudeCodeProvider => + DispatchActuatorSelectionRefused { reason: "the claude draw is observed by dispatch_actuator_claude_selection_observed" as NonEmptyStr } + GunbcHarnessProvider => + DispatchActuatorSelectionRefused { reason: "worker-turn selection was asked for the gunbc harness, which has no vendor credential" as NonEmptyStr } + } +} + fn dispatch_actuator_selection_for_request( instance: HostDashboardInstance, request: DispatchExecutorRequest, @@ -684,8 +812,14 @@ fn dispatch_actuator_selection_for_request( dispatch_actuator_claude_selection_observed( instance: instance, node_id: node_id, attempt_key: attempt_key, sizing_expectation: sizing_expectation, ) - CodexDispatchProvider => dispatch_actuator_selection_for_provider_on(instance: instance, provider: provider, sizing_expectation: sizing_expectation) - CursorDispatchProvider => dispatch_actuator_selection_for_provider_on(instance: instance, provider: provider, sizing_expectation: sizing_expectation) + CodexDispatchProvider => + dispatch_actuator_worker_turn_selection_observed( + instance: instance, provider: provider, node_id: node_id, attempt_key: attempt_key, sizing_expectation: sizing_expectation, + ) + CursorDispatchProvider => + dispatch_actuator_worker_turn_selection_observed( + instance: instance, provider: provider, node_id: node_id, attempt_key: attempt_key, sizing_expectation: sizing_expectation, + ) GunbcHarnessProvider => dispatch_actuator_selection(sizing_expectation: sizing_expectation) } } @@ -2235,11 +2369,22 @@ data gunbc_harness_needs_instance_reason: NonEmptyStr = "the gunbc harness worke type DispatchProviderInnerArgv = DispatchProviderInnerArgvReady { argv: List } | CodexProviderExecutableAbsent { reason: NonEmptyStr } - | CursorProviderSpawnUnwired { reason: NonEmptyStr } + | CursorProviderOffCustodyHost { reason: NonEmptyStr } + | CursorProviderCredentialUnbound { reason: NonEmptyStr } + | CodexProviderOffCustodyHost { reason: NonEmptyStr } | GunbcHarnessSpawnNeedsInstance { reason: NonEmptyStr } | ClaudeProviderCredentialUnbound { reason: NonEmptyStr } -data cursor_provider_spawn_unwired_reason: NonEmptyStr = "cursor dispatch inner argv is not wired on this actuator path" as NonEmptyStr +// THE WORKER-TURN CREDENTIALS LIVE ON ONE HOST EACH (gunbc.cursor_harness_credential, +// gunbc.codex_harness_credential), so a Cursor or Codex spawn on any other instance host refuses here +// rather than running against an ambient login. The instance-less Cursor form cannot check the host +// at all and refuses. On the custody host, codex runs with CODEX_HOME set to the custody directory, +// never the instance's ambient login, and the run holds its flock for its whole life. +data provider_custody_host_unknown_reason: NonEmptyStr = "no dashboard instance: the worker-turn credential's custody host cannot be checked" as NonEmptyStr + +fn provider_off_custody_host_reason(provider_label: String, custody_host: HostIdentity, instance_host: NonEmptyStr) -> NonEmptyStr { + join([provider_label, " worker-turn credential is held only on ", custody_host as String, "; this instance runs on ", instance_host as String], "") as NonEmptyStr +} // THE HARNESS SPAWN. The worker is this repository's own binary running a .dag entry, so there is // no runtime to install, no provider home to seed and no version to reconcile against a committed @@ -2351,7 +2496,7 @@ fn dispatch_provider_inner_argv( ), } CursorDispatchProvider => - CursorProviderSpawnUnwired { reason: cursor_provider_spawn_unwired_reason } + CursorProviderOffCustodyHost { reason: provider_custody_host_unknown_reason } GunbcHarnessProvider => GunbcHarnessSpawnNeedsInstance { reason: gunbc_harness_needs_instance_reason } } @@ -2425,30 +2570,53 @@ fn dispatch_provider_inner_argv_for_instance( ), ) CodexDispatchProvider => - match dashboard_instance_provider_executable(instance: instance) { - Absent => - CodexProviderExecutableAbsent { - reason: dashboard_instance_provider_executable_refusal_detail(instance: instance) as NonEmptyStr, - } - Present { value: codex_program } => - DispatchProviderInnerArgvReady { - argv: shape_codex_exec_argv_for_programs( - env_program: dispatch_capability_program( - environment: instance.execution_environment, - cap: EnvironmentBindingCapability, + if (instance.host_identity as String) != (codex_worker_turn_custody_host as String) { + CodexProviderOffCustodyHost { + reason: provider_off_custody_host_reason(provider_label: "codex", custody_host: codex_worker_turn_custody_host, instance_host: instance.host_identity), + } + } else { + match dashboard_instance_provider_executable(instance: instance) { + Absent => + CodexProviderExecutableAbsent { + reason: dashboard_instance_provider_executable_refusal_detail(instance: instance) as NonEmptyStr, + } + Present { value: codex_program } => + DispatchProviderInnerArgvReady { + argv: codex_harness_turn_argv( + command: shape_codex_exec_argv_for_programs( + env_program: dispatch_capability_program( + environment: instance.execution_environment, + cap: EnvironmentBindingCapability, + ), + program: codex_program as String, + codex_home: codex_worker_turn_codex_home as String, + cwd: worktree_path, + prompt: join([brief, "\n\n", dispatch_codex_start_prompt], ""), + reasoning_effort: effort, + model: model, + spec: dispatch_codex_exec_spec, + ), ), - program: codex_program as String, - codex_home: dashboard_instance_provider_state_root(instance: instance) as String, - cwd: worktree_path, - prompt: join([brief, "\n\n", dispatch_codex_start_prompt], ""), - reasoning_effort: effort, - model: model, - spec: dispatch_codex_exec_spec, - ), - } + } + } } CursorDispatchProvider => - CursorProviderSpawnUnwired { reason: cursor_provider_spawn_unwired_reason } + if (instance.host_identity as String) != (cursor_worker_turn_custody_host as String) { + CursorProviderOffCustodyHost { + reason: provider_off_custody_host_reason(provider_label: "cursor", custody_host: cursor_worker_turn_custody_host, instance_host: instance.host_identity), + } + } else { + match cursor_worker_turn_argv( + key_path: dispatch_credential_snapshot_path_for_instance( + instance: instance, node_id: node_id, attempt_key: attempt_key, provider: CursorDispatchProvider, + ), + worktree_path: worktree_path, + prompt: brief as NonEmptyStr, + ) { + CursorWorkerTurnArgvReady { argv } => DispatchProviderInnerArgvReady { argv: argv } + CursorWorkerTurnRefused { reason } => CursorProviderCredentialUnbound { reason: reason } + } + } GunbcHarnessProvider => DispatchProviderInnerArgvReady { argv: gunbc_harness_spawn_argv( @@ -3235,9 +3403,19 @@ fn dispatch_spawn_commands_for_resolved_instance( brief: brief, alignment_chain_fingerprint: alignment_chain_fingerprint(chain: chain), ) { - CursorProviderSpawnUnwired { reason } => + CursorProviderOffCustodyHost { reason } => + DispatchSpawnRefused { + step: "cursor-credential-custody-host" as NonEmptyStr, + detail: reason as String, + } + CodexProviderOffCustodyHost { reason } => + DispatchSpawnRefused { + step: "codex-credential-custody-host" as NonEmptyStr, + detail: reason as String, + } + CursorProviderCredentialUnbound { reason } => DispatchSpawnRefused { - step: "cursor-provider-spawn" as NonEmptyStr, + step: "cursor-credential-binding" as NonEmptyStr, detail: reason as String, } CodexProviderExecutableAbsent { reason } => diff --git a/dag/gunbc/roadmap/roadmap_publish_observe.dag b/dag/gunbc/roadmap/roadmap_publish_observe.dag index df0e098eda0..c436ba0e1b4 100644 --- a/dag/gunbc/roadmap/roadmap_publish_observe.dag +++ b/dag/gunbc/roadmap/roadmap_publish_observe.dag @@ -256,7 +256,7 @@ fn observe_pull_requests( match lease { QuotaFull { wire: w } => PullRequestsUnreadable { cause: PullRequestQuotaWithheld { repository: repo.full_name, detail: w } } QuotaLeaseRefused { detail: d } => PullRequestsUnreadable { cause: PullRequestQuotaWithheld { repository: repo.full_name, detail: d } } - QuotaLeased { grant: _, partition: _, store: _ } => { + QuotaLeased { grant: _, partition: _, store: _, limit: _ } => { let result = github.Pulls.List( auth_token: source, owner: repo.owner, diff --git a/dag/gunbc/secret_provision.dag b/dag/gunbc/secret_provision.dag index cde05b4d270..c4f3195bc4d 100644 --- a/dag/gunbc/secret_provision.dag +++ b/dag/gunbc/secret_provision.dag @@ -252,6 +252,16 @@ fn fleet_secret_ref(secret_id: NonEmptyStr) -> SecretRef { } } +// THE OPENROUTER FREE-TIER KEY THE HOSTED HARNESS ROUTE READS. Created by the operator 2026-10-05 as +// Secret Manager secret openrouter-free-tier-api-key-harness; the alias "latest" is requested and the +// fetch proves the version it resolved. It is a key whose own credit limit is zero -- a guard so it can +// only spend on free variants -- and gunbc.harness.harness_hosted_route reads that limit back at every +// bind rather than trusting it. Its accessor cell is a row in gunbc.auth.fleet_secret_accessor_roster. +data openrouter_free_tier_key_secret_ref: SecretRef = secret_ref_pin_version( + ref: fleet_secret_ref(secret_id: "openrouter-free-tier-api-key-harness"), + version: "latest", +) + // MT. COLLINS UNIT 1 MANAGED BMC CREDENTIAL LOCUS. Minted 2026-09-13 as Secret Manager // secret bmc-mtcollins1-gunbc. The user-3 route artifact records version 2 ENABLED and set // on the controller, version 1 DISABLED and never deployed. Pin the observed version; hash diff --git a/dag/gunbc/serving/admitted_model.dag b/dag/gunbc/serving/admitted_model.dag index ea08f36d799..c9521430d1d 100644 --- a/dag/gunbc/serving/admitted_model.dag +++ b/dag/gunbc/serving/admitted_model.dag @@ -2,7 +2,7 @@ module gunbc.serving.admitted_model import std.types { String, NonEmptyStr, List } import gunbc.harness.harness_reasoning_wire { - HarnessServingUnit, DeepseekV4FlashOnVllm, Glm53FlashOnVllm, + HarnessServingUnit, DeepseekV4FlashOnVllm, Glm53FlashOnVllm, NemotronUltraFreeOnOpenRouter, harness_unit_model_id, harness_serving_unit_wire, } import gunbc.spark.group_b_serving_capacity { group_b_restoration_served_alias } @@ -51,9 +51,14 @@ fn serving_admitted_ids(unit: HarnessServingUnit) -> List { match unit { DeepseekV4FlashOnVllm => [harness_unit_model_id(unit: DeepseekV4FlashOnVllm)] Glm53FlashOnVllm => [harness_unit_model_id(unit: Glm53FlashOnVllm), group_b_restoration_served_alias] + NemotronUltraFreeOnOpenRouter => [] as List } } +// A HOSTED UNIT IS ADVERTISED BY NO FLEET ROUTE, so it answers to no id a fleet probe could read: it is +// reached by name through gunbc.harness.harness_seat harness_bind_hosted, never resolved from what a +// serving group reports, and it is deliberately absent from serving_admitted_units below. + data serving_admitted_units: List = [DeepseekV4FlashOnVllm, Glm53FlashOnVllm] // ── THE RESOLUTION, AND WHAT IT REFUSES ─────────────────────────────────────────────────────── diff --git a/dag/gunbc/serving/serving_front_door.dag b/dag/gunbc/serving/serving_front_door.dag index e5d32d6c44f..372c7e19031 100644 --- a/dag/gunbc/serving/serving_front_door.dag +++ b/dag/gunbc/serving/serving_front_door.dag @@ -37,7 +37,7 @@ import gunbc.spark.vllm_endpoint_process_launch { VllmEndpointProcessLaunch, vll import gunbc.spark.fabric_switch_observed { FabricGroup, fabric_group_wire } import gunbc.harness.harness_seat { HarnessSeatReference, HarnessSeatPoolIdentity, harness_seat_reference_wire, - release_retry, HarnessSeatReleased, HarnessSeatReleaseRefused, + release_retry, HarnessSeatReleased, HarnessHostedCredentialRemoved, HarnessSeatReleaseRefused, } // ── THE SECOND DOOR, AND WHY AUTHENTICATION ALONE DOES NOT CLOSE IT ────────────────────────── @@ -712,6 +712,7 @@ type FrontDoorStreamEnd fn front_door_release_after_end(store: FabricStorageBinding, pool: HarnessSeatPoolIdentity, reference: NonEmptyStr, now: EpochSecs) -> FrontDoorStreamEnd { match release_retry(store: store, partition: pool.partition, ceiling: pool.ceiling, reference: reference, reason: "the front door's proxied stream ended" as NonEmptyStr, now: now) { HarnessSeatReleased { partition: _ } => StreamEndedSeatReleased + HarnessHostedCredentialRemoved => StreamEndedReleaseRefused { detail: "a seat-pool release answered as a hosted credential removal; the seat may still be held" } HarnessSeatReleaseRefused { detail: d } => StreamEndedReleaseRefused { detail: d } } } diff --git a/dag/gunbc/worker_turn_standing.dag b/dag/gunbc/worker_turn_standing.dag new file mode 100644 index 00000000000..9b14549380a --- /dev/null +++ b/dag/gunbc/worker_turn_standing.dag @@ -0,0 +1,172 @@ +module gunbc.worker_turn_standing + +import std.types { String, NonEmptyStr, List, FilePath, Int } +import std.claim_evidence { RecordedFactId } +import std.optional { Present, Absent } +import extdeps.shell +import extdeps.shell.exec +import extdeps.exec.command { LocalExec } +import extdeps.posix.sh_invocation { + posix_sh_export_from_file_then_exec_command, + PosixShExportExecReady, + PosixShExportExecEmitRefused, + PosixShKeepFile, + posix_sh_export_from_file_unreadable_exit, +} +import extdeps.posix.shell_command_language { admit_posix_shell_name, PosixShellNameAdmitted, PosixShellNameRefused } +import extdeps.llm.cursor_cli { cursor_cli_program, cursor_api_key_env_var } +import gunbc.command_runner { run_shell_command_observe } +import extdeps.llm.cli_lifecycle { ProviderInstance } +import gunbc.provider_standing { + ProviderStanding, + ProviderStandingUnobserved, + ProviderInstallationStanding, + InstallationReady, + InstallationStandingUnobserved, + ToolEnvironmentReady, + provider_standing_from_observed_bundle, +} +import gunbc.provider_standing_probe_bridge { provider_standing_bundle_from_codex_account_evidence } +import gunbc.codex_app_server_press { + AmbientCodexCredentialContext, + codex_press_account_trip_request_lines, + codex_press_account_trip_program, + codex_press_account_trip_invocation, + CodexTripLock, + parse_account_trip_session, + AccountTripDigestStructural, +} +import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_provider_executable } +import gunbc.codex_harness_credential { + codex_worker_turn_custody_host, + codex_worker_turn_codex_home, + codex_harness_turn_lock_path, + codex_harness_turn_busy_exit, + codex_harness_credential_context, +} +import gunbc.cursor_harness_credential { + cursor_status_reading, + CursorStatusReading, + CursorStatusAuthenticated, + CursorStatusUnauthenticated, + CursorStatusUnreadable, +} + +// THE CODEX AND CURSOR OFFERS ARE JUDGED ON THE CREDENTIAL THE SPAWN BINDS, the way the Claude draw +// is (gunbc.roadmap_dispatch_actuator claude_observed_standing_for_instance). Before this, both offers +// carried the declared inventory's unobserved standing, so creating the secrets and converging their +// custody could not change selection: a declared row admits no draw. Each observation here runs on +// the custody host against the custody credential and nothing ambient, so a login someone made by hand +// on the host cannot change the verdict. They run only on an operator's explicit executor request. + +data codex_harness_standing_trip_fact_id: RecordedFactId = "codex-harness-standing-trip" as RecordedFactId + +// THE CODEX STANDING TRIP is the existing app-server account trip (initialize, account/read, +// account/rateLimits/read; gunbc.codex_app_server_press codex_press_account_trip_invocation) with +// CODEX_HOME set to the custody directory and run under the same flock as a turn, because account/read may refresh the token and a refresh +// rotates it: a trip racing a turn would present one refresh token twice. A trip that finds the lock +// held reports the credential busy rather than a codex failure. Exhausted rate-limit buckets and a +// dead login reach selection through provider_standing_bundle_from_codex_account_evidence, the same +// projection the press receipt uses, and refuse there before any spawn. +fn codex_harness_trip_lock() -> CodexTripLock { + CodexTripLock { lock_path: codex_harness_turn_lock_path, conflict_exit_code: codex_harness_turn_busy_exit } +} + +fn codex_observed_standing_for_instance(instance: HostDashboardInstance, provider: ProviderInstance) -> ProviderStanding { + if (instance.host_identity as String) != (codex_worker_turn_custody_host as String) { + ProviderStandingUnobserved { + reason: join(["the codex worker-turn credential is held only on ", codex_worker_turn_custody_host as String, "; this instance runs on ", instance.host_identity as String], "") as NonEmptyStr, + } + } else { + match dashboard_instance_provider_executable(instance: instance) { + Absent => ProviderStandingUnobserved { reason: "no codex executable is placed for this instance" as NonEmptyStr } + Present { value: executable } => { + let run = shell.Exec.RunArgvStdin( + program: codex_press_account_trip_program(lock: Present { value: codex_harness_trip_lock() }), + arguments: codex_press_account_trip_invocation( + executable: executable, + codex_home: codex_worker_turn_codex_home as FilePath, + lock: Present { value: codex_harness_trip_lock() }, + ), + stdin_payload: codex_press_account_trip_request_lines(), + ) + if run.exit_code == codex_harness_turn_busy_exit { + ProviderStandingUnobserved { reason: "the codex worker-turn credential is in use by another turn" as NonEmptyStr } + } else { + provider_standing_from_observed_bundle( + fact_id: codex_harness_standing_trip_fact_id, + provider: provider, + bundle: provider_standing_bundle_from_codex_account_evidence( + evidence: parse_account_trip_session( + stdout: run.stdout, + stderr: run.stderr, + evidence_root: instance.instance_root, + mode: AccountTripDigestStructural, + ), + credential: codex_harness_credential_context(), + installation: codex_trip_installation(exit_code: run.exit_code, stderr: run.stderr), + tool_environment: ToolEnvironmentReady, + ), + ) + } + } + } + } +} + +fn codex_trip_installation(exit_code: Int, stderr: String) -> ProviderInstallationStanding { + if exit_code == 0 { + InstallationReady + } else { + InstallationStandingUnobserved { + reason: join(["the codex app-server trip exited ", to_string(value: exit_code), ": ", stderr], "") as NonEmptyStr, + } + } +} + +// THE CURSOR STANDING. cursor-agent status runs under the custody key, exported inside the child as +// the spawn does it, and answers only whether Cursor accepts the key. A rejected key refuses here. An +// accepted key still does not make the offer selectable: entitlement and quota are observed by a +// one-word turn (operator ruling D2, 2026-10-05), whose result shape is modeled from its first live +// call, so until then the standing is unobserved and the draw refuses -- never a fabricated +// "available". +data cursor_entitlement_trip_pending_reason: NonEmptyStr = "cursor accepts the custody key, but entitlement and quota are read by a one-word turn whose result shape is modeled from its first live call (operator ruling D2, 2026-10-05); until then the cursor draw refuses" as NonEmptyStr + +fn cursor_standing_from_status(reading: CursorStatusReading) -> ProviderStanding { + match reading { + CursorStatusAuthenticated => ProviderStandingUnobserved { reason: cursor_entitlement_trip_pending_reason } + CursorStatusUnauthenticated { status } => + ProviderStandingUnobserved { reason: join(["cursor rejects the custody key (status ", status, ")"], "") as NonEmptyStr } + CursorStatusUnreadable { detail } => + ProviderStandingUnobserved { reason: join(["cursor-agent status output unreadable: ", detail], "") as NonEmptyStr } + } +} + +// The status probe reads the attempt's credential snapshot and keeps it for the spawn +// (PosixShKeepFile); the snapshot is taken by the caller (gunbc.roadmap_dispatch_actuator). +fn cursor_observed_standing_from_snapshot(snapshot: NonEmptyStr) -> ProviderStanding { + match admit_posix_shell_name(spelling: cursor_api_key_env_var as String) { + PosixShellNameRefused { spelling } => + ProviderStandingUnobserved { reason: join(["the cursor credential variable name ", spelling, " is not a POSIX shell name"], "") as NonEmptyStr } + PosixShellNameAdmitted { name } => + match posix_sh_export_from_file_then_exec_command( + name: name, + path: snapshot, + command: [cursor_cli_program, "status", "--format", "json"], + disposition: PosixShKeepFile, + ) { + PosixShExportExecEmitRefused { name: n } => + ProviderStandingUnobserved { reason: join(["the sh emitter refused the export-and-exec program for ", n as String], "") as NonEmptyStr } + PosixShExportExecReady { command } => { + let run = run_shell_command_observe(command: command, transport: LocalExec) + if run.exit_code == posix_sh_export_from_file_unreadable_exit { + ProviderStandingUnobserved { + reason: join(["the cursor credential snapshot ", snapshot as String, " is unreadable: ", run.stderr], "") as NonEmptyStr, + } + } else { + cursor_standing_from_status(reading: cursor_status_reading(stdout: run.stdout)) + } + } + } + } +} diff --git a/dag/test/claim/capacity_pool_witness_test.dag b/dag/test/claim/capacity_pool_witness_test.dag index 3ea99b7afa6..619c7b8a468 100644 --- a/dag/test/claim/capacity_pool_witness_test.dag +++ b/dag/test/claim/capacity_pool_witness_test.dag @@ -130,7 +130,7 @@ test fn an_upstream_remaining_below_the_prediction_shrinks_headroom_and_above_it match w_advanced(o: pool_acquire(pool: w_core_pool(), reference: "belt-tick-1" as NonEmptyStr, amount: w_seats(n: 4000), at: 100, term_seconds: 600)) { Absent => false Present { value: p } => - (match w_advanced(o: observe_upstream_remaining(pool: p, remaining: w_seats(n: 500), at: 300, term_seconds: 3300)) { + (match w_advanced(o: observe_upstream_remaining(pool: p, remaining: w_seats(n: 500), at: 300, term_seconds: 3300, reading: "r500" as NonEmptyStr)) { Absent => false Present { value: shrunk } => w_headroom(p: shrunk, at: 300) == 500 @@ -138,7 +138,7 @@ test fn an_upstream_remaining_below_the_prediction_shrinks_headroom_and_above_it && (match w_advanced(o: pool_acquire(pool: shrunk, reference: "belt-tick-2" as NonEmptyStr, amount: w_seats(n: 500), at: 300, term_seconds: 60)) { Present { value: _ } => true Absent => false }) && w_headroom(p: shrunk, at: 3600) == 5000 }) - && (match w_advanced(o: observe_upstream_remaining(pool: p, remaining: w_seats(n: 4500), at: 300, term_seconds: 3300)) { + && (match w_advanced(o: observe_upstream_remaining(pool: p, remaining: w_seats(n: 4500), at: 300, term_seconds: 3300, reading: "r4500" as NonEmptyStr)) { Absent => false Present { value: same } => w_headroom(p: same, at: 300) == 1000 }) diff --git a/dag/test/claim/claude_code_dispatch_witness_test.dag b/dag/test/claim/claude_code_dispatch_witness_test.dag index 77b9d605a4d..e7b780b0886 100644 --- a/dag/test/claim/claude_code_dispatch_witness_test.dag +++ b/dag/test/claim/claude_code_dispatch_witness_test.dag @@ -92,7 +92,9 @@ import gunbc.roadmap_dispatch_actuator { GunbcHarnessProvider, DispatchProviderInnerArgvReady, CodexProviderExecutableAbsent, - CursorProviderSpawnUnwired, + CursorProviderOffCustodyHost, + CursorProviderCredentialUnbound, + CodexProviderOffCustodyHost, GunbcHarnessSpawnNeedsInstance, ClaudeProviderCredentialUnbound, RecordedAttemptFingerprint, @@ -404,7 +406,9 @@ test fn the_setup_token_credential_binds_through_the_env_wrapper_never_argv() -> && !any(argv, a => starts_with(s: a, prefix: "CLAUDE_CODE_OAUTH_TOKEN=") && !string_contains(s: a, pattern: "\"$0\"")) ClaudeProviderCredentialUnbound { reason: _ } => false CodexProviderExecutableAbsent { reason: _ } => false - CursorProviderSpawnUnwired { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false GunbcHarnessSpawnNeedsInstance { reason: _ } => false } } @@ -418,7 +422,9 @@ test fn an_ambient_profile_tree_is_not_a_bindable_credential() -> Bool { ClaudeProviderCredentialUnbound { reason: _ } => true DispatchProviderInnerArgvReady { argv: _ } => false CodexProviderExecutableAbsent { reason: _ } => false - CursorProviderSpawnUnwired { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false GunbcHarnessSpawnNeedsInstance { reason: _ } => false } } @@ -464,7 +470,9 @@ test fn a_snapshot_that_cannot_be_removed_refuses_the_launch() -> Bool { any(argv, a => string_contains(s: a, pattern: "'rm' '-f' '--' \"$0\" || exit 79")) ClaudeProviderCredentialUnbound { reason: _ } => false CodexProviderExecutableAbsent { reason: _ } => false - CursorProviderSpawnUnwired { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false GunbcHarnessSpawnNeedsInstance { reason: _ } => false } } @@ -630,7 +638,9 @@ test fn the_captured_trip_reaches_a_credential_bound_claude_spawn_on_srv1() -> B && any(argv, a => a == "claude") ClaudeProviderCredentialUnbound { reason: _ } => false CodexProviderExecutableAbsent { reason: _ } => false - CursorProviderSpawnUnwired { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false GunbcHarnessSpawnNeedsInstance { reason: _ } => false } } diff --git a/dag/test/claim/codex_app_server_press_witness_test.dag b/dag/test/claim/codex_app_server_press_witness_test.dag index 4d0ca454833..0089646a317 100644 --- a/dag/test/claim/codex_app_server_press_witness_test.dag +++ b/dag/test/claim/codex_app_server_press_witness_test.dag @@ -709,6 +709,7 @@ fn codex_account_trip_transport_surface_text() -> String { let invocation = codex_press_account_trip_invocation( executable: "/usr/local/bin/codex" as FilePath, codex_home: "/home/witness/.codex" as FilePath, + lock: none, ) let argument_words: List = list_flat_map( xs: cli_surface_arguments(surface: process_argv_expansion_surface(expansion: invocation)), @@ -743,6 +744,7 @@ test fn codex_account_trip_typed_invocation_preserves_trip_semantics() -> Bool { let invocation = codex_press_account_trip_invocation( executable: "/usr/local/bin/codex" as FilePath, codex_home: "/home/witness/.codex" as FilePath, + lock: none, ) let words: List = list_flat_map( xs: cli_surface_arguments(surface: process_argv_expansion_surface(expansion: invocation)), diff --git a/dag/test/claim/codex_harness_credential_witness_test.dag b/dag/test/claim/codex_harness_credential_witness_test.dag new file mode 100644 index 00000000000..22ac87667c3 --- /dev/null +++ b/dag/test/claim/codex_harness_credential_witness_test.dag @@ -0,0 +1,149 @@ +module test.claim.codex_harness_credential + +import std.types { String, Bool, NonEmptyStr, Secret } +import extdeps.filesystem.filesystem_io { FilesystemFileRead, FilesystemFileIndeterminate } +import gunbc.codex_harness_credential { + CodexHarnessAuthIdentity, + CodexHarnessAuthAdmitted, + CodexHarnessAuthRefused, + CodexHarnessAuthModeNotChatgpt, + CodexHarnessAuthModeAbsent, + CodexHarnessAuthMemberMissing, + CodexHarnessAuthFileUnparseable, + CodexHarnessWriteBackNotOwed, + CodexHarnessWriteBackOwed, + CodexHarnessWriteBackRefused, + CodexHarnessReadBackAccountChanged, + CodexHarnessReadBackUnobservable, + CodexHarnessReadBackRefused, + codex_harness_auth_read, + codex_harness_write_back_decision, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// Supplied auth.json documents in the layout extdeps.llm.codex_auth records (keys only; the token +// values are inert placeholders, never a real credential). +data chatgpt_auth_v1: String = "{\"auth_mode\":\"chatgpt\",\"OPENAI_API_KEY\":null,\"tokens\":{\"id_token\":\"h.p.s\",\"access_token\":\"h.p.s\",\"refresh_token\":\"rt-one\",\"account_id\":\"acct-a\"},\"last_refresh\":\"2026-10-05T00:00:00Z\"}" +data chatgpt_auth_rotated: String = "{\"auth_mode\":\"chatgpt\",\"OPENAI_API_KEY\":null,\"tokens\":{\"id_token\":\"h.p.s\",\"access_token\":\"h.p.t\",\"refresh_token\":\"rt-two\",\"account_id\":\"acct-a\"},\"last_refresh\":\"2026-10-05T01:00:00Z\"}" +data chatgpt_auth_other_account: String = "{\"auth_mode\":\"chatgpt\",\"tokens\":{\"id_token\":\"h.p.s\",\"access_token\":\"h.p.s\",\"refresh_token\":\"rt-three\",\"account_id\":\"acct-b\"}}" +data apikey_auth: String = "{\"auth_mode\":\"apikey\",\"OPENAI_API_KEY\":\"sk-placeholder\"}" +data chatgpt_auth_no_refresh: String = "{\"auth_mode\":\"chatgpt\",\"tokens\":{\"id_token\":\"h.p.s\",\"access_token\":\"h.p.s\",\"account_id\":\"acct-a\"}}" +data chatgpt_auth_no_mode: String = "{\"tokens\":{\"refresh_token\":\"rt-one\",\"account_id\":\"acct-a\"}}" + +fn materialized_v1() -> CodexHarnessAuthIdentity { + CodexHarnessAuthIdentity { account_id: "acct-a" as NonEmptyStr, refresh_token: "rt-one" as Secret } +} + +fn read_back(content: String) -> extdeps.filesystem.filesystem_io.FilesystemFileObservation { + FilesystemFileRead { path: "/private/codex-home/auth.json", content: content } +} + +// The payload the operator uploads is admitted, and the two facts the write-back needs are the +// ones the file carries. +test fn witness_chatgpt_payload_admitted_with_account_and_refresh_token() -> Bool { + match codex_harness_auth_read(content: chatgpt_auth_v1) { + CodexHarnessAuthAdmitted { identity } => + (identity.account_id as String) == "acct-a" && (identity.refresh_token as String) == "rt-one" + CodexHarnessAuthRefused { cause: _ } => false + } +} + +// An API-key auth.json is the other billing mode; the operator ruled subscription mode, so it +// refuses with the mode it found rather than running codex against it. +test fn witness_apikey_payload_refused_naming_mode() -> Bool { + match codex_harness_auth_read(content: apikey_auth) { + CodexHarnessAuthRefused { cause } => + match cause { + CodexHarnessAuthModeNotChatgpt { observed } => observed == "apikey" + _ => false + } + CodexHarnessAuthAdmitted { identity: _ } => false + } +} + +test fn witness_payload_without_refresh_token_refused_naming_member() -> Bool { + match codex_harness_auth_read(content: chatgpt_auth_no_refresh) { + CodexHarnessAuthRefused { cause } => + match cause { + CodexHarnessAuthMemberMissing { member } => (member as String) == "refresh_token" + _ => false + } + CodexHarnessAuthAdmitted { identity: _ } => false + } +} + +test fn witness_payload_without_mode_and_non_json_refused_distinctly() -> Bool { + let no_mode = match codex_harness_auth_read(content: chatgpt_auth_no_mode) { + CodexHarnessAuthRefused { cause } => + match cause { + CodexHarnessAuthModeAbsent => true + _ => false + } + CodexHarnessAuthAdmitted { identity: _ } => false + } + let not_json = match codex_harness_auth_read(content: "not json") { + CodexHarnessAuthRefused { cause } => + match cause { + CodexHarnessAuthFileUnparseable => true + _ => false + } + CodexHarnessAuthAdmitted { identity: _ } => false + } + no_mode && not_json +} + +// THE DISCRIMINATING PAIR. Codex rotated the refresh token during the turn: a new version is owed and +// it carries the WHOLE file codex wrote. Codex did not refresh: nothing is owed. If the decision +// compared anything other than the refresh token, one of these two would flip. +test fn witness_rotated_refresh_token_owes_whole_file() -> Bool { + match codex_harness_write_back_decision(materialized: materialized_v1(), read_back: read_back(content: chatgpt_auth_rotated)) { + CodexHarnessWriteBackOwed { payload } => (payload as String) == chatgpt_auth_rotated + _ => false + } +} + +test fn witness_unrotated_refresh_token_owes_nothing() -> Bool { + match codex_harness_write_back_decision(materialized: materialized_v1(), read_back: read_back(content: chatgpt_auth_v1)) { + CodexHarnessWriteBackNotOwed => true + _ => false + } +} + +// A file that now names another account is not written back over the operator's credential, even +// though its refresh token differs: the rotation check alone would have owed a write here. +test fn witness_account_change_withholds_write_back() -> Bool { + match codex_harness_write_back_decision(materialized: materialized_v1(), read_back: read_back(content: chatgpt_auth_other_account)) { + CodexHarnessWriteBackRefused { cause } => + match cause { + CodexHarnessReadBackAccountChanged { materialized, read_back } => + (materialized as String) == "acct-a" && (read_back as String) == "acct-b" + _ => false + } + _ => false + } +} + +test fn witness_unreadable_read_back_withholds_write_back() -> Bool { + let indeterminate = match codex_harness_write_back_decision( + materialized: materialized_v1(), + read_back: FilesystemFileIndeterminate { cause: "listing failed" }, + ) { + CodexHarnessWriteBackRefused { cause } => + match cause { + CodexHarnessReadBackUnobservable { detail } => detail == "listing failed" + _ => false + } + _ => false + } + let corrupted = match codex_harness_write_back_decision(materialized: materialized_v1(), read_back: read_back(content: apikey_auth)) { + CodexHarnessWriteBackRefused { cause } => + match cause { + CodexHarnessReadBackRefused { cause: _ } => true + _ => false + } + _ => false + } + indeterminate && corrupted +} diff --git a/dag/test/claim/cursor_cli_invocation_witness_test.dag b/dag/test/claim/cursor_cli_invocation_witness_test.dag index 06291dd82f5..3b15aed9e38 100644 --- a/dag/test/claim/cursor_cli_invocation_witness_test.dag +++ b/dag/test/claim/cursor_cli_invocation_witness_test.dag @@ -6,6 +6,7 @@ import extdeps.llm.cursor_cli { CursorApiKeyArgument, CursorApiKeyEnvironment, CursorInvocation, + CursorRunShape, CursorProcessInvocation, CursorEnvBinding, CursorModelId, @@ -35,14 +36,16 @@ data witness_key: NonEmptyStr = "synthetic-not-a-real-key" as NonEmptyStr fn witness_invocation(auth: CursorAuth) -> CursorInvocation { CursorInvocation { auth: auth, - model: cursor_model_auto, - mode: CursorDefaultEditMode, - sandbox: CursorSandboxEnabled, - output_format: CursorStreamJson, - force: false, - trust_workspace: true, - workspace: "/srv/attempt" as FilePath, - prompt: "do the work" as NonEmptyStr, + run: CursorRunShape { + model: cursor_model_auto, + mode: CursorDefaultEditMode, + sandbox: CursorSandboxEnabled, + output_format: CursorStreamJson, + force: false, + trust_workspace: true, + workspace: "/srv/attempt" as FilePath, + prompt: "do the work" as NonEmptyStr, + }, } } diff --git a/dag/test/claim/cursor_harness_credential_witness_test.dag b/dag/test/claim/cursor_harness_credential_witness_test.dag new file mode 100644 index 00000000000..cbb45c9df0f --- /dev/null +++ b/dag/test/claim/cursor_harness_credential_witness_test.dag @@ -0,0 +1,172 @@ +module test.claim.cursor_harness_credential + +import std.types { String, Bool, Int } +import extdeps.cloud.gcp.secret_manager { SmEnabled, SmDisabled } +import gunbc.auth.materialized_secret { + MaterializedSecret, + MaterializationCompleted, + MaterializationRefused, + SecretNotLive, + SecretVersionUnusable, + LivenessProbeNotIndependent, + EnvVar, + SshKeyFile, + StdinPassword, +} +import gunbc.host_phase_status { observation_verdict_eq } +import gunbc.cursor_harness_credential { + CursorStatusAuthenticated, + CursorStatusUnauthenticated, + CursorStatusUnreadable, + cursor_status_reading, + with_cursor_harness_api_key, + cursor_api_key_harness_secret_ref, +} +import gunbc.dispatch_selection { + CursorApiKeySecretRef, + CursorLocalLoginRef, + OfferCodex, + OfferClaude, + OfferCursor, + declared_provider_inventory_for_instance, +} +import gunbc.roadmap_dashboard_instance { srv2_lab_dashboard_instance } +import extdeps.llm.cursor_sdk { CursorSdkApiKeyFromSecretRef, CursorSdkApiKeyFromEnvironment } +import gunbc.cursor_sdk_local_binding { + cursor_sdk_local_credential_from_dispatch, + CursorSdkLocalCredentialAdmitted, + CursorSdkLocalCredentialRefused, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// The rejected-key output, verbatim from cursor-agent 2026.10.01-e373342 run with an invalid key. +data observed_unauthenticated_status: String = "{\n \"status\": \"unauthenticated\",\n \"isAuthenticated\": false,\n \"hasAccessToken\": false,\n \"hasRefreshToken\": false,\n \"message\": \"Not logged in\"\n}" + +// The accepted-key shape is the same document with the flag true. It is SUPPLIED, not observed: the +// live probe against cursor-api-key-harness is the inhabitance step once the operator creates it. +data supplied_authenticated_status: String = "{\"status\":\"authenticated\",\"isAuthenticated\":true}" + +fn binding_marker(m: MaterializedSecret) -> Int { + match m.binding { + EnvVar { name } => if (name as String) == "CURSOR_API_KEY" { 1 } else { 2 } + SshKeyFile => 3 + StdinPassword => 4 + BearerHeaderFile => 5 + } +} + +test fn witness_observed_rejected_key_reads_unauthenticated() -> Bool { + match cursor_status_reading(stdout: observed_unauthenticated_status) { + CursorStatusUnauthenticated { status } => status == "unauthenticated" + _ => false + } +} + +// THE DISCRIMINATING PAIR THROUGH THE REAL BRACKET. Same store state, same probe; only the status +// output differs. Accepted key: the use runs with CURSOR_API_KEY as the binding. Rejected key: the +// bracket refuses with SecretNotLive, and the use never runs. +test fn witness_accepted_key_materializes_as_cursor_api_key_env() -> Bool { + match with_cursor_harness_api_key( + existence: SmEnabled, + status_stdout: Present { value: supplied_authenticated_status }, + use: fn(m) { binding_marker(m: m) }, + ) { + MaterializationCompleted { value } => value == 1 + MaterializationRefused { cause: _ } => false + } +} + +test fn witness_rejected_key_refuses_secret_not_live() -> Bool { + match with_cursor_harness_api_key( + existence: SmEnabled, + status_stdout: Present { value: observed_unauthenticated_status }, + use: fn(m) { binding_marker(m: m) }, + ) { + MaterializationRefused { cause } => + match cause { + SecretNotLive { verdict } => observation_verdict_eq(a: verdict, b: std.upsert_decision.Drifted) + _ => false + } + MaterializationCompleted { value: _ } => false + } +} + +// No probe ran: refused as unknown, never materialized on the store's word alone. +test fn witness_unprobed_key_refuses_unknown() -> Bool { + match with_cursor_harness_api_key(existence: SmEnabled, status_stdout: none, use: fn(m) { binding_marker(m: m) }) { + MaterializationRefused { cause } => + match cause { + SecretNotLive { verdict } => observation_verdict_eq(a: verdict, b: std.upsert_decision.UnknownRefused) + _ => false + } + MaterializationCompleted { value: _ } => false + } +} + +test fn witness_disabled_version_refuses_on_store_axis_before_probe() -> Bool { + match with_cursor_harness_api_key( + existence: SmDisabled, + status_stdout: Present { value: supplied_authenticated_status }, + use: fn(m) { binding_marker(m: m) }, + ) { + MaterializationRefused { cause } => + match cause { + SecretVersionUnusable { state: _ } => true + _ => false + } + MaterializationCompleted { value: _ } => false + } +} + +test fn witness_unreadable_status_reads_unreadable() -> Bool { + match cursor_status_reading(stdout: "Warning: The provided API key is invalid.") { + CursorStatusUnreadable { detail: _ } => true + _ => false + } +} + +// The harness credential is the dispatch arm the existing SDK-local binding already admits: exact +// version, CursorApiKeySecretRef. +test fn witness_harness_credential_admitted_by_existing_cursor_binding() -> Bool { + match cursor_sdk_local_credential_from_dispatch( + credential: CursorApiKeySecretRef { secret_ref: cursor_api_key_harness_secret_ref }, + ) { + CursorSdkLocalCredentialAdmitted { credential: _, materialization: _ } => true + CursorSdkLocalCredentialRefused { reason: _ } => false + } +} + +// THE ROUTE, not a supplied credential: the Cursor offer the srv2 lab instance actually declares is +// handed to the existing SDK-local binding, and what comes out is a materialization of exactly +// cursor-api-key-harness at its pinned version. Before this change the declared offer carried +// CursorLocalLoginRef and this binding refused it, so no declared Cursor offer could be admitted. +test fn witness_declared_srv2_cursor_offer_materializes_harness_secret() -> Bool { + let inventory = declared_provider_inventory_for_instance(instance: srv2_lab_dashboard_instance()) + let admitted = map(inventory.offers, o => match o { + OfferCursor { offer } => + match cursor_sdk_local_credential_from_dispatch(credential: offer.credential) { + CursorSdkLocalCredentialAdmitted { credential: _, materialization } => + match materialization { + CursorSdkApiKeyFromSecretRef { secret_ref } => + (secret_ref.secret as String) == "cursor-api-key-harness" + && (secret_ref.version as String) == "1" + && (secret_ref.project as String) == "gunbai-secrets" + CursorSdkApiKeyFromEnvironment { env_var: _ } => false + } + CursorSdkLocalCredentialRefused { reason: _ } => false + } + OfferCodex { offer: _ } => false + OfferClaude { offer: _ } => false + }) + any(admitted, b => b) +} + +// The control the route witness is measured against: the arm the offer used to carry still refuses. +test fn witness_local_login_credential_still_refused() -> Bool { + match cursor_sdk_local_credential_from_dispatch(credential: CursorLocalLoginRef) { + CursorSdkLocalCredentialRefused { reason: _ } => true + CursorSdkLocalCredentialAdmitted { credential: _, materialization: _ } => false + } +} diff --git a/dag/test/claim/cursor_sdk_argv_projection_witness_test.dag b/dag/test/claim/cursor_sdk_argv_projection_witness_test.dag index e76864c026f..63fd71500d8 100644 --- a/dag/test/claim/cursor_sdk_argv_projection_witness_test.dag +++ b/dag/test/claim/cursor_sdk_argv_projection_witness_test.dag @@ -13,6 +13,7 @@ import extdeps.llm.cursor_cli { CursorAuth, CursorApiKeyArgument, CursorInvocation, + CursorRunShape, CursorDefaultEditMode, CursorSandboxEnabled, CursorStreamJson, @@ -27,14 +28,16 @@ data witness_key: NonEmptyStr = "synthetic-not-a-real-key" as NonEmptyStr fn witness_invocation(auth: CursorAuth) -> CursorInvocation { CursorInvocation { auth: auth, - model: cursor_model_auto, - mode: CursorDefaultEditMode, - sandbox: CursorSandboxEnabled, - output_format: CursorStreamJson, - force: false, - trust_workspace: true, - workspace: "/srv/attempt" as FilePath, - prompt: "do the work" as NonEmptyStr, + run: CursorRunShape { + model: cursor_model_auto, + mode: CursorDefaultEditMode, + sandbox: CursorSandboxEnabled, + output_format: CursorStreamJson, + force: false, + trust_workspace: true, + workspace: "/srv/attempt" as FilePath, + prompt: "do the work" as NonEmptyStr, + }, } } diff --git a/dag/test/claim/gcp_secret_access_witness_test.dag b/dag/test/claim/gcp_secret_access_witness_test.dag index 2dc8f43f8f3..920821f13f5 100644 --- a/dag/test/claim/gcp_secret_access_witness_test.dag +++ b/dag/test/claim/gcp_secret_access_witness_test.dag @@ -28,6 +28,9 @@ import gunbc.auth.fleet_secret_accessor_roster { claude_code_oauth_harness_accessor_row, approval_capability_mac_key_accessor_row, approval_submission_mac_key_accessor_row, approval_store_receipt_mac_key_accessor_row, approval_ntfy_publisher_token_accessor_row, + cursor_api_key_harness_accessor_row, codex_auth_harness_accessor_row, + codex_auth_harness_version_adder_row, fleet_secret_grants, fabric_state_key_accessor_row, + openrouter_free_tier_key_accessor_row, } import gunbc.spark.secret_access_ensure { spark_secret_access_grant } import std.types { String, List } @@ -479,7 +482,12 @@ fn roster_has_secret(row: FleetAccessorGrantRow) -> Bool { data named_accessor_rows: List = [ spark_accessor_row, mtcollins1_bmc_accessor_row, approval_capability_mac_key_accessor_row, approval_submission_mac_key_accessor_row, approval_store_receipt_mac_key_accessor_row, - approval_ntfy_publisher_token_accessor_row, fabric_state_key_accessor_row, oracle_oci_api_signing_key_accessor_row, claude_code_oauth_harness_accessor_row, + approval_ntfy_publisher_token_accessor_row, + cursor_api_key_harness_accessor_row, codex_auth_harness_accessor_row, + fabric_state_key_accessor_row, + oracle_oci_api_signing_key_accessor_row, + claude_code_oauth_harness_accessor_row, + openrouter_free_tier_key_accessor_row, ] test fn the_roster_is_the_single_authority_for_the_spark_and_mtcollins1_grants() -> Bool { let bmc_cells = secret_access_desired_cells(member: convergence_member, grant: fleet_accessor_grant(row: mtcollins1_bmc_accessor_row)) @@ -617,3 +625,15 @@ test fn only_the_convergence_arms_project_to_a_successful_exit() -> Bool { && !exits_successfully(r: SecretAccessConvergenceClaimUnseen { observed: policy_without_grant() }) && !exits_successfully(r: SecretAccessRefused { cause: "x" }) } + +// THE CODEX WRITE-BACK GRANT IS secretVersionAdder ON codex-auth-harness AND NOTHING WIDER, and it is +// in the list the converge folds. If the version-adder row were folded through the accessor +// constructor, the role check below would read secretAccessor and the write-back would 403 live. +test fn codex_write_back_grant_is_version_adder_on_its_secret_and_converged() -> Bool { + let adder = filter(fleet_secret_grants(), g => g.role == "roles/secretmanager.secretVersionAdder") + count(adder) == 1 + && all(adder, g => (g.target.secret as String) == "codex-auth-harness") + && codex_auth_harness_version_adder_row.target.secret == codex_auth_harness_accessor_row.target.secret + && any(fleet_secret_grants(), g => g.role == "roles/secretmanager.secretAccessor" && (g.target.secret as String) == "cursor-api-key-harness") + && any(fleet_secret_grants(), g => g.role == "roles/secretmanager.secretAccessor" && (g.target.secret as String) == "codex-auth-harness") +} diff --git a/dag/test/claim/host_credential_custody_converge_witness_test.dag b/dag/test/claim/host_credential_custody_converge_witness_test.dag index d37b966d81f..1caf040825b 100644 --- a/dag/test/claim/host_credential_custody_converge_witness_test.dag +++ b/dag/test/claim/host_credential_custody_converge_witness_test.dag @@ -77,6 +77,23 @@ import gunbc.host_credential_custody_converge { admit_staging_cleanup, StagingCleanupAuthorized, StagingCleanupWithheld, + CursorWorkerTurnApiKey, + CodexWorkerTurnAuth, + CustodyAnyRunnerHost, + CustodyOnlyOnHost, + CustodyHostAuthoritativeNotSingleHost, + CustodyWriteBackToStore, + CustodyRepairMetadataInPlace, + CustodyCreateOnly, + host_authoritative_custody_action, +} +import std.types { Secret } +import gunbc.codex_harness_credential { + CodexHarnessWriteBackNotOwed, + CodexHarnessWriteBackOwed, + CodexHarnessWriteBackRefused, + CodexHarnessReadBackAccountChanged, + codex_worker_turn_custody_host, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -481,3 +498,138 @@ test fn protected_state_key_has_operator_group_custody_outside_job_trees() -> Bo && (d.ownership.user as String) == "root" && (d.ownership.group as String) == "briansrls" && placed(row: row) } + +// ── worker-turn credentials (node adhoc-932f2935-e8e) ─────────────────────────────────────────── + +// THE CODEX ROW CANNOT BE SPREAD. The same row with its scope widened to any runner host is refused +// at placement, so a second host can never receive the store's lagging copy of a token codex rotated. +// The control is the row as declared, which places. +test fn a_host_authoritative_row_on_any_runner_host_is_refused_at_placement() -> Bool { + let row = host_credential_custody_row(c: CodexWorkerTurnAuth) + let widened = HostCredentialCustodyRow { + credential: row.credential, secret: row.secret, path: row.path, + host_scope: CustodyAnyRunnerHost, + file_ownership: row.file_ownership, file_mode: row.file_mode, + dir_ownership: row.dir_ownership, dir_mode: row.dir_mode, + } + let refused = match custody_row_placement(row: widened) { + CustodyPlacementRefused { cause } => + match cause { + CustodyHostAuthoritativeNotSingleHost { path: _ } => true + _ => false + } + CustodyPlaced { row: _, key_dir: _, upper_ancestors: _ } => false + } + let declared_places = match custody_row_placement(row: row) { + CustodyPlaced { row: _, key_dir: kd, upper_ancestors: _ } => (kd as String) == "/etc/gunbc-codex-harness" + CustodyPlacementRefused { cause: _ } => false + } + refused && declared_places +} + +// A store-authoritative row widened the same way still places: the refusal is about authority, not +// about the worker-turn rows. +test fn a_store_authoritative_row_on_any_runner_host_still_places() -> Bool { + let row = host_credential_custody_row(c: CursorWorkerTurnApiKey) + let widened = HostCredentialCustodyRow { + credential: row.credential, secret: row.secret, path: row.path, + host_scope: CustodyAnyRunnerHost, + file_ownership: row.file_ownership, file_mode: row.file_mode, + dir_ownership: row.dir_ownership, dir_mode: row.dir_mode, + } + match custody_row_placement(row: widened) { + CustodyPlaced { row: _, key_dir: _, upper_ancestors: _ } => true + CustodyPlacementRefused { cause: _ } => false + } +} + +test fn the_codex_row_is_delivered_only_to_its_custody_host() -> Bool { + let row = host_credential_custody_row(c: CodexWorkerTurnAuth) + let other = match custody_host_admission(row: row, host: "srv1" as HostIdentity) { + CustodyHostRefused { cause: _ } => true + CustodyHostAdmitted => false + } + let own = match custody_host_admission(row: row, host: codex_worker_turn_custody_host) { + CustodyHostAdmitted => true + CustodyHostRefused { cause: _ } => false + } + other && own && (codex_worker_turn_custody_host as String) == (operator_host_srv2 as String) +} + +// THE DISCRIMINATING PAIR FOR A DIFFERING HOST FILE. The store-authoritative decision for differing +// bytes is CustodyWrite, which would put a spent token back. Host-authoritative: a file whose refresh +// token did not move is LEFT (it is only fresher in short-lived tokens), and one whose refresh token +// moved is written BACK to the store, never overwritten. +test fn a_differing_host_file_is_left_or_written_back_never_overwritten() -> Bool { + let base = custody_action(dir: CustodyPathAsDeclared, upper_ancestors: PathsRootOnly, content: HostFileDiffers, file: CustodyPathAsDeclared) + let base_writes = match base { + CustodyWrite => true + _ => false + } + let left = match host_authoritative_custody_action(base: base, content: HostFileDiffers, decision: Present { value: CodexHarnessWriteBackNotOwed }) { + CustodyLeaveInPlace => true + _ => false + } + let written_back = match host_authoritative_custody_action( + base: base, content: HostFileDiffers, + decision: Present { value: CodexHarnessWriteBackOwed { payload: "{\"auth_mode\":\"chatgpt\"}" as Secret } }, + ) { + CustodyWriteBackToStore { payload: _ } => true + _ => false + } + base_writes && left && written_back +} + +test fn a_host_file_naming_another_account_refuses() -> Bool { + let base = custody_action(dir: CustodyPathAsDeclared, upper_ancestors: PathsRootOnly, content: HostFileDiffers, file: CustodyPathAsDeclared) + match host_authoritative_custody_action( + base: base, content: HostFileDiffers, + decision: Present { value: CodexHarnessWriteBackRefused { cause: CodexHarnessReadBackAccountChanged { materialized: "acct-a" as NonEmptyStr, read_back: "acct-b" as NonEmptyStr } } }, + ) { + CustodyActionRefused { reason: _ } => true + _ => false + } +} + +// An absent host file is placed from the store CREATE-ONLY (first placement), and a directory that is not as +// declared still refuses before anything: the host-authoritative layer only changes the differing arm. +test fn absent_host_file_is_placed_and_a_bad_directory_still_refuses() -> Bool { + let absent_base = custody_action(dir: CustodyPathAsDeclared, upper_ancestors: PathsRootOnly, content: gunbc.runner_host_file_converge.HostFileAbsent, file: CustodyPathAbsent) + let placed = match host_authoritative_custody_action(base: absent_base, content: gunbc.runner_host_file_converge.HostFileAbsent, decision: none) { + CustodyCreateOnly => true + _ => false + } + let bad_dir = custody_action(dir: CustodyPathForbiddenBits, upper_ancestors: PathsRootOnly, content: HostFileDiffers, file: CustodyPathAsDeclared) + let still_refused = match host_authoritative_custody_action(base: bad_dir, content: HostFileDiffers, decision: Present { value: CodexHarnessWriteBackNotOwed }) { + CustodyActionRefused { reason: _ } => true + _ => false + } + placed && still_refused +} + +// THE METADATA-REPAIR ROUTE (side-chat P2). Identical bytes with a wrong owner or mode make the +// ordinary decision CustodyWrite, whose store-to-host content write renames the STORE bytes over the +// live file outside the codex turn lock and would lose a refresh that landed in between. A +// host-authoritative row must repair the metadata in place instead, and a store-authoritative row +// keeps the ordinary rewrite. +test fn a_host_authoritative_file_with_wrong_metadata_is_repaired_in_place_never_rewritten() -> Bool { + let base = custody_action(dir: CustodyPathAsDeclared, upper_ancestors: PathsRootOnly, content: HostFileIdentical, file: CustodyPathForbiddenBits) + let ordinary_rewrites = match base { + CustodyWrite => true + _ => false + } + let repaired = match host_authoritative_custody_action(base: base, content: HostFileIdentical, decision: none) { + CustodyRepairMetadataInPlace => true + _ => false + } + ordinary_rewrites && repaired +} + +// A file that is already declared is untouched, host-authoritative or not. +test fn a_declared_host_authoritative_file_is_left_in_place() -> Bool { + let base = custody_action(dir: CustodyPathAsDeclared, upper_ancestors: PathsRootOnly, content: HostFileIdentical, file: CustodyPathAsDeclared) + match host_authoritative_custody_action(base: base, content: HostFileIdentical, decision: none) { + CustodyLeaveInPlace => true + _ => false + } +} diff --git a/dag/test/claim/openrouter_hosted_route_witness_test.dag b/dag/test/claim/openrouter_hosted_route_witness_test.dag new file mode 100644 index 00000000000..af8854cbc84 --- /dev/null +++ b/dag/test/claim/openrouter_hosted_route_witness_test.dag @@ -0,0 +1,703 @@ +module test.claim.openrouter_hosted_route_witness_test + +import std.types { Bool, String, Int, List, NonEmptyStr } +import std.nat { Nat } +import std.optional { Present, Absent } +import extdeps.languages.json.emit { serialize_json, json_object } +import extdeps.openrouter.openrouter { + openrouter_decode_error_body, OpenRouterErrorDecoded, OpenRouterErrorUndecodable, + OpenRouterPaymentRequired, OpenRouterRateLimited, OpenRouterOtherError, + KeySpendLimit, InFlightBudgetLimit, CreditsLimit, WeightExceedsBudget, + openrouter_decode_key_read, OpenRouterKeyDecoded, OpenRouterKeyUndecodable, + KeySpendCappedAtZero, KeySpendUncapped, KeySpendCappedAbove, + openrouter_credits_from_daily_limit, CreditsTierObserved, CreditsTierUnrecognized, + CreditsPurchasedAtLeastTen, CreditsPurchasedBelowTen, openrouter_free_models_per_day, openrouter_free_models_per_minute, + EffortHigh, EffortMedium, + openrouter_decode_rate_limited, OpenRouterRateLimitRefusal, RetryAfterSeconds, RetryAfterUnparsed, RetryAfterAbsent, RetryAfterUnread, + openrouter_rate_limit_wire, openrouter_rate_limit_hints, HeaderDumpRead, HeaderDumpUnread, +} +import gunbc.harness.harness_reasoning_wire { + NemotronUltraFreeOnOpenRouter, ReasoningOff, ReasoningStandard, ReasoningMaximum, + harness_openrouter_reasoning_kwargs, harness_reasoning_kwargs_for, ReasoningKwargsAdmitted, ReasoningLevelUnsupported, + harness_unit_model_id, harness_unit_placement, LeasedFromHostedQuota, PlacedOnFleetSeat, +} +import gunbc.harness.harness_hosted_route { + hosted_round_decision, RoundBothLeased, RoundRefusedFull, RoundRefusedUnleasable, + harness_hosted_route_for, harness_hosted_level_choice, HostedLevelSelected, HostedLevelUnselected, + hosted_key_refusal, hosted_day_remaining, hosted_access_from_key, harness_hosted_round_charge, HostedRoundCharged, HostedRoundQuotaFull, HostedRoundUnleasable, +} +import product.capacity.pool_events { PoolEvent, PoolAcquired, PoolSettled, PoolReleased, PoolUpstreamObserved, pool_apply_payload, pool_fold, PoolFolded, PoolFoldRefused } +import product.capacity.event_chain { ChainEnvelope, ChainEvent, EventId } +import gunbc.fabric_quota { + quota_term_to_window_end, quota_window_start, fabric_quota_lease_for, fabric_quota_settle_at, QuotaTermToWindowEnd, + QuotaSettled, QuotaSettleRefused, +} +import gunbc.harness.harness_seat { harness_seat_pool_root, hosted_inflight_carried, hosted_inflight_release_hint, hosted_inflight_partition } +import gunbc.harness.harness_hosted_route { HarnessHostedQuota } +import product.capacity.event_chain { ChainWalked } +import std.nat { nat_range_inclusive } +import product.capacity.quota { quota_root_pool } +import product.capacity.pool { + Pool, pool_acquire, pool_settle, observe_upstream_remaining, PoolOutcome, PoolAdvanced, PoolRefused, + LeaseCrossesLapse, LeaseAlreadyLapsed, LedgerRefusal, pool_reading_at, PoolRead, PoolReadingRefused, +} +import std.measure { Measure, Count, One, second_count } +import extdeps.api_rate_limit { UpstreamRateLimit } +import gunbc.fabric_quota { QuotaLease, QuotaLeased, QuotaFull, QuotaLeaseRefused } +import product.capacity.lease { LeaseGrant, LeaseFence, GrantIdentity } +import product.capacity.event_chain { PartitionId } +import gunbc.fabric_storage_client { FabricStorageServed } +import gunbc.harness.harness_turn { TurnHostedQuotaFull, TurnHostedRateLimited, harness_outcome_label, harness_outcome_detail } +import gunbc.harness.harness_cli { harness_turn_exit, harness_default_max_token_count, harness_request_deadline } +import std.process { ExitSuccess, ExitFailure } +import std.measure { token_count } + +// ── THE QUOTA REFUSAL: a full pool stops the request before it is sent ────────────────────── +// +// Supplied lease answers at the decision's own interface (DESIGN section 3, a witness discriminates at one +// interface): the leases are gunbc.fabric_quota's subject and are witnessed there. + +fn w_leased(reference: String) -> QuotaLease { + QuotaLeased { + grant: LeaseGrant { + reference: reference as NonEmptyStr, + fence: LeaseFence { grant: join(["g-", reference], "") as GrantIdentity, generation: 1 }, + granted_at: 100, + maximum_duration_seconds: 60, + expires_at: 160, + }, + partition: join(["quota-openrouter-", reference], "") as PartitionId, + store: FabricStorageServed { endpoint: "witness" as NonEmptyStr }, + limit: openrouter_free_models_per_minute, + } +} + +// RED: the day pool is full after the minute pool leased. The round refuses as FULL, and the minute +// lease is named as owed back -- a refusal that kept it would hold a slot for a request never sent. +test fn hosted_round_refuses_full_when_the_day_pool_is_full_and_returns_the_minute_lease() -> Bool { + match hosted_round_decision(minute: w_leased(reference: "m"), day: Present { value: QuotaFull { wire: "openrouter-free-models-day: 1000/1000" } }) { + RoundRefusedFull { wire: w, minute_to_return: Present { value: QuotaLeased { grant: g, partition: _, store: _, limit: _ } } } => + w.contains("free-models-day") && (g.reference as String) == "m" + RoundRefusedFull { wire: _, minute_to_return: _ } => false + RoundBothLeased { minute: _, day: _ } => false + RoundRefusedUnleasable { detail: _, minute_to_return: _ } => false + } +} + +// RED: the minute pool is full, so the day pool is never asked and nothing is owed back. +test fn hosted_round_refuses_full_when_the_minute_pool_is_full_and_asks_nothing_else() -> Bool { + match hosted_round_decision(minute: QuotaFull { wire: "openrouter-free-models-minute: 20/20" }, day: none) { + RoundRefusedFull { wire: w, minute_to_return: Absent } => w.contains("free-models-minute") + RoundRefusedFull { wire: _, minute_to_return: Present { value: _ } } => false + RoundBothLeased { minute: _, day: _ } => false + RoundRefusedUnleasable { detail: _, minute_to_return: _ } => false + } +} + +// CONTROL: both pools leased is the only arm that posts. +test fn hosted_round_posts_only_when_both_pools_leased() -> Bool { + match hosted_round_decision(minute: w_leased(reference: "m"), day: Present { value: w_leased(reference: "d") }) { + RoundBothLeased { minute: _, day: _ } => true + RoundRefusedFull { wire: _, minute_to_return: _ } => false + RoundRefusedUnleasable { detail: _, minute_to_return: _ } => false + } +} + +// A FULL ROUND IS A REFUSAL EXIT, NOT A COMPLETED TURN. +test fn a_hosted_quota_full_turn_exits_as_a_refusal() -> Bool { + let o = TurnHostedQuotaFull { wire: "openrouter-free-models-minute: 20/20", steps_taken: 2 } + harness_outcome_label(outcome: o) == "hosted-quota-full" + && (match harness_turn_exit(outcome: o) { ExitSuccess => false ExitFailure { code, reason: _ } => code == 1 }) +} + +// ── THE 402 DECODE: limit_source is a closed set and names which budget refused ───────────── + +test fn a_402_from_the_key_limit_decodes_to_the_key_spend_limit() -> Bool { + match openrouter_decode_error_body(body: "{\"error\":{\"code\":402,\"message\":\"Key limit exceeded\",\"metadata\":{\"limit_source\":\"openrouter_key_limit\",\"remedy_hint\":\"raise the key limit\"}}}") { + OpenRouterErrorDecoded { error: OpenRouterPaymentRequired { limit_source: KeySpendLimit, reason: Absent, message: m, remedy_hint: Present { value: _ } } } => m == "Key limit exceeded" + OpenRouterErrorDecoded { error: _ } => false + OpenRouterErrorUndecodable { cause: _ } => false + } +} + +test fn a_402_from_the_in_flight_budget_carries_its_reason() -> Bool { + match openrouter_decode_error_body(body: "{\"error\":{\"code\":402,\"message\":\"busy\",\"metadata\":{\"limit_source\":\"openrouter_in_flight_budget\",\"reason\":\"weight_exceeds_budget\"}}}") { + OpenRouterErrorDecoded { error: OpenRouterPaymentRequired { limit_source: InFlightBudgetLimit, reason: Present { value: WeightExceedsBudget }, message: _, remedy_hint: Absent } } => true + OpenRouterErrorDecoded { error: _ } => false + OpenRouterErrorUndecodable { cause: _ } => false + } +} + +// RED: an unpublished limit_source refuses rather than decoding as the nearest of three. +test fn a_402_with_an_unpublished_limit_source_refuses_to_decode() -> Bool { + match openrouter_decode_error_body(body: "{\"error\":{\"code\":402,\"message\":\"x\",\"metadata\":{\"limit_source\":\"openrouter_mystery_budget\"}}}") { + OpenRouterErrorUndecodable { cause: c } => c.contains("openrouter_mystery_budget") + OpenRouterErrorDecoded { error: _ } => false + } +} + +// RED: a 402 naming no budget is not decodable as a payment refusal. +test fn a_402_without_a_limit_source_refuses_to_decode() -> Bool { + match openrouter_decode_error_body(body: "{\"error\":{\"code\":402,\"message\":\"x\"}}") { + OpenRouterErrorUndecodable { cause: c } => c.contains("limit_source") + OpenRouterErrorDecoded { error: _ } => false + } +} + +test fn a_429_names_whose_limit_it_was() -> Bool { + match openrouter_decode_error_body(body: "{\"error\":{\"code\":429,\"message\":\"slow down\",\"metadata\":{\"error_type\":\"rate_limit_exceeded\",\"provider_code\":\"nvidia\"}}}") { + OpenRouterErrorDecoded { error: OpenRouterRateLimited { message: _, provider_code: Present { value: p } } } => p == "nvidia" + OpenRouterErrorDecoded { error: _ } => false + OpenRouterErrorUndecodable { cause: _ } => false + } +} + +// ── THE REASONING LEVEL: mapped to a listed effort or refused ─────────────────────────────── + +// RED: a listing whose supported efforts lack medium refuses the standard level instead of sending it. +test fn a_level_mapping_to_an_unlisted_effort_is_refused() -> Bool { + match harness_openrouter_reasoning_kwargs(unit: NemotronUltraFreeOnOpenRouter, level: ReasoningStandard, supported: [EffortHigh], mandatory: false) { + ReasoningLevelUnsupported { unit: _, level: _, cause: c } => (c as String).contains("medium") + ReasoningKwargsAdmitted { kwargs: _ } => false + } +} + +// RED: a listing that makes reasoning mandatory refuses the off level. +test fn reasoning_off_is_refused_where_the_listing_makes_reasoning_mandatory() -> Bool { + match harness_openrouter_reasoning_kwargs(unit: NemotronUltraFreeOnOpenRouter, level: ReasoningOff, supported: [EffortHigh, EffortMedium], mandatory: true) { + ReasoningLevelUnsupported { unit: _, level: _, cause: _ } => true + ReasoningKwargsAdmitted { kwargs: _ } => false + } +} + +// CONTROL: the same mapping against the real listing's row sends OpenRouter's reasoning object, never the +// model's chat-template keys. +test fn the_standard_level_sends_effort_medium_on_the_real_listing() -> Bool { + match harness_reasoning_kwargs_for(unit: NemotronUltraFreeOnOpenRouter, level: ReasoningStandard) { + ReasoningKwargsAdmitted { kwargs: k } => { + let wire = serialize_json(v: json_object(k)) + wire.contains("\"effort\": \"medium\"") && !wire.contains("chat_template_kwargs") + } + ReasoningLevelUnsupported { unit: _, level: _, cause: _ } => false + } +} + +// ── THE KEY READ: a key that can spend money is refused ───────────────────────────────────── + +test fn a_key_capped_at_zero_on_the_thousand_tier_decodes_both_facts() -> Bool { + match openrouter_decode_key_read(body: "{\"data\":{\"limit\":0,\"limit_remaining\":0,\"usage\":0,\"is_free_tier\":false,\"free_model_daily_requests\":{\"used\":3,\"limit\":1000,\"remaining\":997}}}") { + OpenRouterKeyDecoded { key: k } => + (match k.spend_limit { KeySpendCappedAtZero => true KeySpendUncapped => false KeySpendCappedAbove { lexeme: _ } => false }) + && (match openrouter_credits_from_daily_limit(daily_limit: k.free_daily_limit) { + CreditsTierObserved { credits: CreditsPurchasedAtLeastTen } => true + CreditsTierObserved { credits: CreditsPurchasedBelowTen } => false + CreditsTierUnrecognized { daily_limit: _ } => false + }) + OpenRouterKeyUndecodable { cause: _ } => false + } +} + +// RED: an uncapped key is refused by name. +test fn an_uncapped_key_is_refused_as_able_to_spend() -> Bool { + match openrouter_decode_key_read(body: "{\"data\":{\"limit\":null,\"is_free_tier\":false,\"free_model_daily_requests\":{\"used\":0,\"limit\":1000,\"remaining\":1000}}}") { + OpenRouterKeyDecoded { key: k } => hosted_key_refusal(key: k).contains("no credit limit") + OpenRouterKeyUndecodable { cause: _ } => false + } +} + +// THE TIER SETS THE DAY POOL'S CEILING, and an unpublished ceiling is not a tier. +test fn the_daily_tier_selects_the_day_pool_ceiling_and_an_unpublished_one_refuses() -> Bool { + openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen).requests == 1000 + && openrouter_free_models_per_day(credits: CreditsPurchasedBelowTen).requests == 50 + && (match openrouter_credits_from_daily_limit(daily_limit: 200) { CreditsTierUnrecognized { daily_limit: n } => n == 200 CreditsTierObserved { credits: _ } => false }) +} + +// ── INHABITANCE ON THE REAL PATH ───────────────────────────────────────────────────────────── +// +// The real unit's real route, joined from the real listing rows, reaches the real decisions: it is placed +// by a hosted quota lease rather than a seat; its route is HTTPS and names the listing's model id; the +// selection over the real field chooses the maximum level for the harness's real max_tokens; and that level +// renders effort high. Deleting the hosted arm from any of harness_unit_placement, harness_hosted_route_for, +// the selection or the reasoning seam turns this red. +test fn the_hosted_nemotron_route_is_quota_leased_https_and_selects_effort_high() -> Bool { + match harness_hosted_route_for(unit: NemotronUltraFreeOnOpenRouter) { + Absent => false + Present { value: route } => + (match harness_unit_placement(unit: NemotronUltraFreeOnOpenRouter) { LeasedFromHostedQuota => true PlacedOnFleetSeat => false }) + && (route.url as String).starts_with("https://") + && (harness_unit_model_id(unit: NemotronUltraFreeOnOpenRouter) as String) == "nvidia/nemotron-3-ultra-550b-a55b:free" + && (match harness_hosted_level_choice(route: route, max_tokens: harness_default_max_token_count) { + HostedLevelSelected { level: ReasoningMaximum } => + (match harness_reasoning_kwargs_for(unit: NemotronUltraFreeOnOpenRouter, level: ReasoningMaximum) { + ReasoningKwargsAdmitted { kwargs: k } => serialize_json(v: json_object(k)).contains("\"effort\": \"high\"") + ReasoningLevelUnsupported { unit: _, level: _, cause: _ } => false + }) + HostedLevelSelected { level: _ } => false + HostedLevelUnselected { cause: _ } => false + }) + } +} + +// RED: a max_tokens above the listing's completion ceiling leaves no admissible level, and the selection +// refuses rather than choosing one. +test fn a_request_above_the_completion_ceiling_selects_no_level() -> Bool { + match harness_hosted_route_for(unit: NemotronUltraFreeOnOpenRouter) { + Absent => false + Present { value: route } => + match harness_hosted_level_choice(route: route, max_tokens: token_count(count: 70000)) { + HostedLevelUnselected { cause: _ } => true + HostedLevelSelected { level: _ } => false + } + } +} + +// ── THE CHARGE FITS ITS WINDOW AT EVERY INSTANT, THROUGH THE REAL POOL ADMISSION ────────────── +// +// The pools are the production pools (product.capacity.quota quota_root_pool over the real OpenRouter +// rows the hosted route carries), the term is the production derivation (gunbc.fabric_quota +// quota_term_to_window_end), and admission is product.capacity.pool pool_acquire -- the function a +// partition append adjudicates with. Only the clock is supplied: an instant one second into a UTC day +// that is also one second into its minute, and instants one second before the minute and the day end. +data w_day_start: Int = 1780531200 + +fn w_charge_admitted(limit: UpstreamRateLimit, at: Int) -> Bool { + match quota_term_to_window_end(limit: limit, at: at) { + Absent => false + Present { value: term } => + match pool_acquire(pool: quota_root_pool(limit: limit), reference: "charge" as NonEmptyStr, amount: Measure { count: 1 }, at: at, term_seconds: term) { + PoolAdvanced { pool: _ } => true + PoolRefused { refusal: _ } => false + } + } +} + +test fn a_hosted_charge_fits_both_real_pools_early_and_late_in_their_windows() -> Bool { + match harness_hosted_route_for(unit: NemotronUltraFreeOnOpenRouter) { + Absent => false + Present { value: route } => { + let day = openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) + w_charge_admitted(limit: route.per_minute, at: w_day_start + 1) + && w_charge_admitted(limit: route.per_minute, at: w_day_start + 59) + && w_charge_admitted(limit: day, at: w_day_start + 1) + && w_charge_admitted(limit: day, at: w_day_start + 86399) + } + } +} + +// RED, THE DEFECT THIS REPLACED: a lease held for the response deadline cannot fit the minute window at +// any instant, so the pool refuses it as crossing the lapse. +test fn a_lease_for_the_response_deadline_crosses_the_minute_lapse() -> Bool { + match harness_hosted_route_for(unit: NemotronUltraFreeOnOpenRouter) { + Absent => false + Present { value: route } => + match pool_acquire(pool: quota_root_pool(limit: route.per_minute), reference: "held" as NonEmptyStr, amount: Measure { count: 1 }, at: w_day_start + 1, term_seconds: second_count(s: harness_request_deadline)) { + PoolRefused { refusal: LeaseCrossesLapse { reference: _, expires_at: _, window_end: _ } } => true + PoolRefused { refusal: _ } => false + PoolAdvanced { pool: _ } => false + } + } +} + +// ── THE DAY POOL STARTS FROM WHAT THE UPSTREAM SAYS REMAINS ──────────────────────────────────── +// +// The remaining count is read by the production reader off a decoded key read, recorded through the +// fold's own observe_upstream_remaining, and then asked for leases through pool_acquire. +fn w_key_remaining(used: Int, remaining: Int) -> Nat? { + match openrouter_decode_key_read(body: join(["{\"data\":{\"limit\":0,\"is_free_tier\":false,\"free_model_daily_requests\":{\"used\":", to_string(used), ",\"limit\":1000,\"remaining\":", to_string(remaining), "}}}"], "")) { + OpenRouterKeyDecoded { key: k } => + hosted_day_remaining(key: k) + OpenRouterKeyUndecodable { cause: _ } => none + } +} + +fn w_observed(pool: Pool, remaining: Nat, at: Int) -> Pool? { + let day = openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) + match quota_term_to_window_end(limit: day, at: at) { + Absent => none + Present { value: term } => + match observe_upstream_remaining(pool: pool, remaining: Measure { count: remaining }, at: at, term_seconds: term, reading: join(["witness@", to_string(at)], "") as NonEmptyStr) { + PoolAdvanced { pool: p } => Present { value: p } + PoolRefused { refusal: _ } => none + } + } +} + +fn w_day_headroom(pool: Pool, at: Int) -> Int { + match pool_reading_at(pool: pool, at: at) { + PoolRead { committed: _, ceiling: _, headroom: h, over_committed: _ } => h.count as Int + PoolReadingRefused { at: _, anchor: _ } => -1 + } +} + +// RED: the upstream reports every free request spent (used = limit, remaining = 0) and the local ledger +// is fresh -- the day pool admits nothing, so no request is ever sent. +test fn an_exhausted_upstream_day_leaves_a_fresh_day_pool_no_headroom() -> Bool { + let day = openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) + match w_key_remaining(used: 1000, remaining: 0) { + Absent => false + Present { value: r } => + match w_observed(pool: quota_root_pool(limit: day), remaining: r, at: w_day_start + 100) { + Absent => false + Present { value: p } => w_day_headroom(pool: p, at: w_day_start + 101) == 0 + } + } +} + +// A PARTIAL REMAINDER IS RESPECTED EXACTLY, and a later reading claiming more never gives any back. +test fn a_partial_upstream_remainder_is_the_headroom_and_later_readings_never_replenish() -> Bool { + let day = openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) + match w_key_remaining(used: 997, remaining: 3) { + Absent => false + Present { value: r } => + match w_observed(pool: quota_root_pool(limit: day), remaining: r, at: w_day_start + 100) { + Absent => false + Present { value: p } => + w_day_headroom(pool: p, at: w_day_start + 101) == 3 + && (match w_observed(pool: p, remaining: 1000, at: w_day_start + 102) { + Absent => false + Present { value: p2 } => w_day_headroom(pool: p2, at: w_day_start + 103) == 3 + }) + } + } +} + +// ── A TRANSITION IS CHECKED AGAINST THE FOLDED POOL BEFORE IT IS WRITTEN ───────────────────────── +// +// gunbc.fabric_event_log fabric_pool_transition appends only what product.capacity.pool_events +// pool_apply_payload advances on the folded pool -- the same function replay applies. These supply the +// folded pool and ask that function, which is the decision the carrier makes before any write. +fn w_env(id: String, at: Int, payload: PoolEvent) -> ChainEnvelope { + ChainEnvelope { + id: id as EventId, + event: ChainEvent { partition: "quota-openrouter-free-models-minute" as PartitionId, parent: none, recorded_at: at, actor: "witness" as NonEmptyStr, payload: payload }, + } +} + +// RED: the minute lease is taken one second before rollover (its term runs to the window end) and the +// settlement arrives after it. The transition is refused -- so the checked carrier writes nothing and +// reports no settlement -- and the CONTROL shows why that matters: the same settlement appended blind +// makes the whole partition unfoldable. +test fn a_settlement_after_rollover_is_refused_before_it_could_poison_the_partition() -> Bool { + let minute = openrouter_free_models_per_minute + let at = w_day_start + 59 + match quota_term_to_window_end(limit: minute, at: at) { + Absent => false + Present { value: term } => { + let acquired = w_env(id: "e1", at: at, payload: PoolAcquired { reference: "charge" as NonEmptyStr, amount: 1, term_seconds: term }) + match pool_fold(root: quota_root_pool(limit: minute), oldest_first: [acquired]) { + PoolFoldRefused { at_event: _, wire: _ } => false + PoolFolded { pool: p, generation: _ } => + (match pool_apply_payload(pool: p, payload: PoolSettled { reference: "charge" as NonEmptyStr, actual: 1 }, at: w_day_start + 60, identity: "pending:e1" as NonEmptyStr) { + PoolRefused { refusal: LeaseAlreadyLapsed { reference: _, lapsed_at: _, at: _ } } => true + PoolRefused { refusal: _ } => false + PoolAdvanced { pool: _ } => false + }) + && (match pool_apply_payload(pool: p, payload: PoolSettled { reference: "charge" as NonEmptyStr, actual: 1 }, at: at, identity: "pending:e1" as NonEmptyStr) { + PoolAdvanced { pool: _ } => true + PoolRefused { refusal: _ } => false + }) + && (match pool_fold(root: quota_root_pool(limit: minute), oldest_first: [acquired, w_env(id: "e2", at: w_day_start + 60, payload: PoolSettled { reference: "charge" as NonEmptyStr, actual: 1 })]) { + PoolFoldRefused { at_event: e, wire: _ } => (e as String) == "e2" + PoolFolded { pool: _, generation: _ } => false + }) + } + } + } +} + +// TWO DECREASING UPSTREAM READINGS IN ONE SECOND STAY REPLAYABLE: each hold is named by its own event, +// so the second is not a duplicate of the first, and the pool ends at the lower reading. +test fn two_decreasing_upstream_readings_in_one_second_fold_to_the_lower() -> Bool { + let day = openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) + let at = w_day_start + 100 + match quota_term_to_window_end(limit: day, at: at) { + Absent => false + Present { value: term } => + match pool_fold(root: quota_root_pool(limit: day), oldest_first: [ + w_env(id: "o1", at: at, payload: PoolUpstreamObserved { remaining: 3, term_seconds: term }), + w_env(id: "o2", at: at, payload: PoolUpstreamObserved { remaining: 1, term_seconds: term }), + ]) { + PoolFoldRefused { at_event: _, wire: _ } => false + PoolFolded { pool: p, generation: g } => g == 2 && w_day_headroom(pool: p, at: at) == 1 + } + } +} + +// ── A SEND IN FLIGHT ACROSS THE ROLLOVER IS CHARGED IN THE WINDOW IT MAY ARRIVE IN ─────────────── +// +// The in-flight partition is the production seat-pool shape (gunbc.harness.harness_seat +// harness_seat_pool_root) under the production partition name; the carried count is the production +// hosted_inflight_carried; the charge is the real minute pool's pool_acquire at the amount the round asks +// for (its own send plus the carried ones). Only the clock and the events are supplied. +fn w_quota() -> HarnessHostedQuota { + HarnessHostedQuota { short_hostname: "witness", actor: "hosted-witness" as NonEmptyStr, per_minute: openrouter_free_models_per_minute, per_day: openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) } +} + +fn w_inflight(events: List>) -> Pool? { + match pool_fold(root: harness_seat_pool_root(partition: hosted_inflight_partition(quota: w_quota()), ceiling: 20), oldest_first: events) { + PoolFolded { pool: p, generation: _ } => Present { value: p } + PoolFoldRefused { at_event: _, wire: _ } => none + } +} + +// The minute after the rollover, with `n` sends already charged and settled in it. +fn w_minute_with(n: Nat, at: Int) -> Pool? { + fold(nat_range_inclusive(lo: 1, hi: n), init: Present { value: quota_root_pool(limit: openrouter_free_models_per_minute) }, f: (acc, i) => + match acc { + Absent => none + Present { value: p } => { + let r = join(["charged-", to_string(i)], "") as NonEmptyStr + match pool_acquire(pool: p, reference: r, amount: Measure { count: 1 }, at: at, term_seconds: 1) { + PoolRefused { refusal: _ } => none + PoolAdvanced { pool: q } => + match pool_settle(pool: q, reference: r, actual: Measure { count: 1 }, at: at) { + PoolRefused { refusal: _ } => none + PoolAdvanced { pool: settled } => Present { value: settled } + } + } + } + }) +} + +fn w_round_admitted(minute: Pool, carried: Nat, at: Int) -> Bool { + match quota_term_to_window_end(limit: openrouter_free_models_per_minute, at: at) { + Absent => false + Present { value: term } => + match pool_acquire(pool: minute, reference: "this-round" as NonEmptyStr, amount: Measure { count: carried + 1 }, at: at, term_seconds: term) { + PoolAdvanced { pool: _ } => true + PoolRefused { refusal: _ } => false + } + } +} + +// RED: a send admitted at T+58 pauses across the rollover and is still in flight at T+61. With 19 sends +// already charged in the new minute, the next round must cover 2 -- its own and the carried one -- so it +// refuses: no request goes out in a window whose quota the paused send may already have spent. The same +// round with nothing carried is admitted, so the refusal is the carried send's. +test fn a_send_paused_across_the_rollover_is_charged_in_the_next_window() -> Bool { + match w_inflight(events: [w_env(id: "s1", at: w_day_start + 58, payload: PoolAcquired { reference: "paused" as NonEmptyStr, amount: 1, term_seconds: 2348 })]) { + Absent => false + Present { value: inflight } => + match quota_window_start(limit: openrouter_free_models_per_minute, at: w_day_start + 61) { + Absent => false + Present { value: start } => + match w_minute_with(n: 19, at: w_day_start + 60) { + Absent => false + Present { value: minute } => { + let carried = hosted_inflight_carried(pool: inflight, window_start: start) + carried == 1 + && !w_round_admitted(minute: minute, carried: carried, at: w_day_start + 61) + && w_round_admitted(minute: minute, carried: 0, at: w_day_start + 61) + } + } + } + } +} + +// CONTROL: a send made inside its window whose answer arrives much later carries nothing into later +// windows once its transport returns and its seat is released -- and the send is counted in its own window +// only once, by its own charge, not by the in-flight count. +test fn a_send_released_after_a_late_response_carries_nothing_forward() -> Bool { + match w_inflight(events: [ + w_env(id: "s1", at: w_day_start + 10, payload: PoolAcquired { reference: "slow-answer" as NonEmptyStr, amount: 1, term_seconds: 2348 }), + w_env(id: "s2", at: w_day_start + 300, payload: PoolReleased { reference: "slow-answer" as NonEmptyStr, reason: "the transport returned" as NonEmptyStr }), + ]) { + Absent => false + Present { value: inflight } => + hosted_inflight_carried(pool: inflight, window_start: w_day_start + 300) == 0 + && hosted_inflight_carried(pool: inflight, window_start: w_day_start) == 0 + } +} + +// A SEAT HELD BY A HOLDER THAT DIED IS NAMED WITH ITS RELEASE, so a starved route says what to free. +test fn a_held_inflight_seat_is_named_with_its_release_in_the_refusal_hint() -> Bool { + let events = [w_env(id: "s1", at: w_day_start + 58, payload: PoolAcquired { reference: "worker-7@1780531258#round3" as NonEmptyStr, amount: 1, term_seconds: 2348 })] + match w_inflight(events: events) { + Absent => false + Present { value: inflight } => { + let hint = hosted_inflight_release_hint(quota: w_quota(), pool: inflight, generation: 1, walk: ChainWalked { oldest_first: events }) + hint.contains("worker-7@1780531258#round3") && hint.contains("grant=s1") && hint.contains("harness_release_cli") + && hint.contains(hosted_inflight_partition(quota: w_quota()) as String) + } + } +} + +// ── A 429 FROM THE UPSTREAM IS THE TYPED BACKSTOP, AND IT IS NEVER RETRIED ─────────────────────────── +// +// The decode is the production decode over the shapes the limits page documents: the error body, and the +// header dump curl writes (the status line starts a block; only the last block is the answer). +data w_429_body: String = "{\"error\":{\"code\":429,\"message\":\"Rate limit exceeded: free-models-per-min\",\"metadata\":{\"error_type\":\"rate_limit_exceeded\"}}}" + +data w_429_headers: String = "HTTP/2 429\r\ncontent-type: application/json\r\nretry-after: 30\r\nx-ratelimit-limit: 20\r\nx-ratelimit-remaining: 0\r\nx-ratelimit-reset: 1791265800000\r\n\r\n" + +test fn a_429_decodes_to_the_typed_refusal_with_the_upstreams_retry_hints() -> Bool { + match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: w_429_headers }) { + Absent => false + Present { value: r } => + (match r.hints.retry_after { RetryAfterSeconds { seconds: n } => n == 30 RetryAfterUnparsed { text: _ } => false RetryAfterAbsent => false RetryAfterUnread { cause: _ } => false }) + && (match r.hints.reset { Present { value: v } => v == "1791265800000" Absent => false }) + && (match r.hints.remaining { Present { value: v } => v == "0" Absent => false }) + && (match r.provider_code { Absent => true Present { value: _ } => false }) + && openrouter_rate_limit_wire(r: r).contains("Retry-After 30 s") + && openrouter_rate_limit_wire(r: r).contains("unit not documented") + && openrouter_rate_limit_wire(r: r).contains("NOT retried") + } +} + +// A RETRY-AFTER THAT IS NOT DELTA-SECONDS IS CARRIED AS TEXT, never read as a number it is not; and a +// refusal with no hint headers is still a typed refusal, saying so. The LAST block of a dump is the answer. +test fn a_retry_after_that_is_not_seconds_and_a_missing_hint_are_stated_not_guessed() -> Bool { + (match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: "HTTP/2 429\r\nretry-after: Wed, 21 Oct 2026 07:28:00 GMT\r\n\r\n" }) { + Present { value: r } => (match r.hints.retry_after { RetryAfterUnparsed { text: t } => t.contains("21 Oct 2026") RetryAfterSeconds { seconds: _ } => false RetryAfterAbsent => false RetryAfterUnread { cause: _ } => false }) + Absent => false + }) + && (match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 12\r\nX-RateLimit-Reset: 5\r\n\r\n" }) { + Present { value: r } => (match r.hints.retry_after { RetryAfterSeconds { seconds: n } => n == 12 RetryAfterUnparsed { text: _ } => false RetryAfterAbsent => false RetryAfterUnread { cause: _ } => false }) + && (match r.hints.reset { Present { value: v } => v == "5" Absent => false }) + Absent => false + }) + && (match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: "" }) { + Present { value: r } => (match r.hints.retry_after { RetryAfterAbsent => true RetryAfterSeconds { seconds: _ } => false RetryAfterUnparsed { text: _ } => false RetryAfterUnread { cause: _ } => false }) && openrouter_rate_limit_wire(r: r).contains("no Retry-After") + Absent => false + }) + && (match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: "HTTP/1.1 100 Continue\r\nretry-after: 99\r\n\r\nHTTP/2 429\r\nretry-after: 7\r\n\r\n" }) { + Present { value: r } => (match r.hints.retry_after { RetryAfterSeconds { seconds: n } => n == 7 RetryAfterUnparsed { text: _ } => false RetryAfterAbsent => false RetryAfterUnread { cause: _ } => false }) + Absent => false + }) +} + +// RED: a header dump that could not be read is NOT "no Retry-After". The upstream may have sent a hint that +// this process failed to read, so the refusal says the hint is unknown and why, and carries no reset or counters. +test fn an_unread_header_dump_is_stated_unknown_not_reported_as_no_retry_after() -> Bool { + match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpUnread { cause: "the dump file is absent" }) { + Absent => false + Present { value: r } => + (match r.hints.retry_after { RetryAfterUnread { cause: c } => c.contains("dump file is absent") RetryAfterAbsent => false RetryAfterSeconds { seconds: _ } => false RetryAfterUnparsed { text: _ } => false }) + && (match r.hints.reset { Absent => true Present { value: _ } => false }) + && openrouter_rate_limit_wire(r: r).contains("Retry-After unknown") + && !openrouter_rate_limit_wire(r: r).contains("no Retry-After") + && openrouter_rate_limit_wire(r: r).contains("NOT retried") + } +} + +// RED: delay-seconds is 1*DIGIT, and the integer parser would accept a sign, so a signed, fractional or +// unit-suffixed value is carried as text and never read as a number of seconds. +fn w_retry_after_of(value: String) -> OpenRouterRetryAfter? { + match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: join(["HTTP/2 429\r\nretry-after: ", value, "\r\n\r\n"], "") }) { + Present { value: r } => Present { value: r.hints.retry_after } + Absent => none + } +} + +fn w_is_unparsed(r: OpenRouterRetryAfter?) -> Bool { + match r { + Present { value: RetryAfterUnparsed { text: _ } } => true + Present { value: RetryAfterSeconds { seconds: _ } } => false + Present { value: RetryAfterAbsent } => false + Present { value: RetryAfterUnread { cause: _ } } => false + Absent => false + } +} + +test fn a_signed_fractional_or_suffixed_retry_after_is_not_read_as_seconds() -> Bool { + w_is_unparsed(r: w_retry_after_of(value: "+30")) + && w_is_unparsed(r: w_retry_after_of(value: "-0")) + && w_is_unparsed(r: w_retry_after_of(value: "-30")) + && w_is_unparsed(r: w_retry_after_of(value: "3.5")) + && w_is_unparsed(r: w_retry_after_of(value: "30s")) + && (match w_retry_after_of(value: "30") { Present { value: RetryAfterSeconds { seconds: n } } => n == 30 Present { value: _ } => false Absent => false }) +} + +// RED: a 402, or any body that is not a 429, is NOT a rate-limit refusal -- it keeps its own typed error +// (the 402 limit_source decode above), so a paid-model refusal is never read as a wait-and-retry. +test fn only_a_429_is_a_rate_limit_refusal() -> Bool { + (match openrouter_decode_rate_limited(body: "{\"error\":{\"code\":402,\"message\":\"x\",\"metadata\":{\"limit_source\":\"openrouter_key_limit\"}}}", header_dump: HeaderDumpRead { text: w_429_headers }) { + Absent => true + Present { value: _ } => false + }) + && (match openrouter_decode_rate_limited(body: "not json", header_dump: HeaderDumpRead { text: w_429_headers }) { Absent => true Present { value: _ } => false }) +} + +// THE TURN ENDS ON IT. It is a terminal outcome of the turn -- there is no retry arm in the loop to take -- +// and a refusal exit that says it was not retried and carries the hint into the operator's line. +test fn a_hosted_rate_limit_ends_the_turn_as_a_refusal_naming_the_hint_and_no_retry() -> Bool { + match openrouter_decode_rate_limited(body: w_429_body, header_dump: HeaderDumpRead { text: w_429_headers }) { + Absent => false + Present { value: r } => { + let o = TurnHostedRateLimited { refusal: r, steps_taken: 3 } + harness_outcome_label(outcome: o) == "hosted-rate-limited" + && harness_outcome_detail(outcome: o).contains("Retry-After 30 s") + && harness_outcome_detail(outcome: o).contains("rather than retrying") + && (match harness_turn_exit(outcome: o) { ExitSuccess => false ExitFailure { code, reason: _ } => code == 1 }) + } + } +} + +// ── THE SPENDABLE-KEY WALL, AT THE FUNCTION THAT ENFORCES IT ───────────────────────────────────── +// +// hosted_access_from_key is what refuses to build a binding; a witness over the rendering helper beside it +// would stay green if the wall were removed. These decode real key reads and ask the wall itself. +fn w_bind_from_key(limit_json: String, daily_limit: Int) -> Bool? { + match openrouter_decode_key_read(body: join(["{\"data\":{\"limit\":", limit_json, ",\"is_free_tier\":false,\"free_model_daily_requests\":{\"used\":0,\"limit\":", to_string(daily_limit), ",\"remaining\":", to_string(daily_limit), "}}}"], "")) { + OpenRouterKeyUndecodable { cause: _ } => none + OpenRouterKeyDecoded { key: k } => + match harness_hosted_route_for(unit: NemotronUltraFreeOnOpenRouter) { + Absent => none + Present { value: route } => + match hosted_access_from_key(route: route, key: k, level: ReasoningMaximum, header_dir: "/tmp/witness" as NonEmptyStr, header_file: "/tmp/witness/authorization" as NonEmptyStr, short_hostname: "witness", actor: "witness" as NonEmptyStr) { + Absent => Present { value: false } + Present { value: _ } => Present { value: true } + } + } + } +} + +// RED: a key with no credit limit, and one with a positive limit, can spend money on a priced model, so no +// binding is built; the zero-credit key on a published tier is the only one that is. A tier that is neither +// published value refuses even on the safe key. +test fn only_a_zero_credit_key_on_a_published_tier_builds_a_binding() -> Bool { + (match w_bind_from_key(limit_json: "null", daily_limit: 1000) { Present { value: b } => !b Absent => false }) + && (match w_bind_from_key(limit_json: "5", daily_limit: 1000) { Present { value: b } => !b Absent => false }) + && (match w_bind_from_key(limit_json: "0", daily_limit: 200) { Present { value: b } => !b Absent => false }) + && (match w_bind_from_key(limit_json: "0", daily_limit: 1000) { Present { value: b } => b Absent => false }) + && (match w_bind_from_key(limit_json: "0.0", daily_limit: 50) { Present { value: b } => b Absent => false }) +} + +// ── THE REAL LEASE PRODUCER IS EXECUTED, THROUGH THE PRODUCTION CHARGE ─────────────────────────── +// +// The constructed leases above are for the decision's own boundary. This runs the PRODUCTION charge +// (harness_hosted_round_charge -> gunbc.fabric_quota fabric_quota_lease_for) for a host the fleet declares no +// dashboard instance for: the real producer answers with its typed refusal, the charge turns it into +// HostedRoundUnleasable naming the host, and nothing is leased, charged or sent. Deleting the producer, or +// the charge's call to it, turns this red. +// +// WHAT THIS DOES NOT EXECUTE, STATED: the producer's success arm -- a seat acquired and settled on a real +// event-log store -- needs a store, and the floor refuses a new real-execution witness +// (v2.workflow.floor_changed_witness), so no claim here can stand one up. Its pieces are witnessed at their +// own interfaces above: the term derivation and admission through the production pool_acquire, and the +// checked settlement through the replay's own pool_apply_payload. The joined route runs in the post-merge +// live exercise. +fn w_unplaced_quota() -> HarnessHostedQuota { + HarnessHostedQuota { short_hostname: "no-such-host-witness", actor: "hosted-witness" as NonEmptyStr, per_minute: openrouter_free_models_per_minute, per_day: openrouter_free_models_per_day(credits: CreditsPurchasedAtLeastTen) } +} + +test fn the_production_charge_refuses_through_the_real_lease_producer_for_an_undeclared_host() -> Bool { + (match harness_hosted_round_charge(quota: w_unplaced_quota(), carried_minute: 0, carried_day: 0, release_hint: "") { + HostedRoundUnleasable { detail: d } => d.contains("no dashboard instance is declared for host no-such-host-witness") + HostedRoundCharged => false + HostedRoundQuotaFull { wire: _ } => false + }) + && (match fabric_quota_lease_for(short_hostname: "no-such-host-witness", limit: openrouter_free_models_per_minute, amount: 1, actor: "hosted-witness" as NonEmptyStr, term: QuotaTermToWindowEnd) { + QuotaLeaseRefused { detail: d } => d.contains("no dashboard instance is declared") + QuotaLeased { grant: _, partition: _, store: _, limit: _ } => false + QuotaFull { wire: _ } => false + }) +} + +// A SETTLEMENT OF A LEASE THAT NEVER EXISTED IS A TYPED REFUSAL, not a recorded charge. +test fn settling_a_refused_lease_records_nothing() -> Bool { + match fabric_quota_settle_at(lease: QuotaLeaseRefused { detail: "no dashboard instance is declared for host x" }, actual: 1, now: w_day_start + 59) { + QuotaSettleRefused { detail: d } => d.contains("nothing to settle") + QuotaSettled { partition: _, actual: _ } => false + } +} diff --git a/dag/test/claim/roadmap/roadmap_dispatch_actuator_witness_test.dag b/dag/test/claim/roadmap/roadmap_dispatch_actuator_witness_test.dag index dcccbf697a5..b14a0412fd7 100644 --- a/dag/test/claim/roadmap/roadmap_dispatch_actuator_witness_test.dag +++ b/dag/test/claim/roadmap/roadmap_dispatch_actuator_witness_test.dag @@ -76,11 +76,14 @@ import gunbc.roadmap_dispatch_actuator { dispatch_provider_inner_argv, DispatchProviderInnerArgv, DispatchProviderInnerArgvReady, - CursorProviderSpawnUnwired, + CursorProviderOffCustodyHost, + CursorProviderCredentialUnbound, + CodexProviderOffCustodyHost, CodexProviderExecutableAbsent, GunbcHarnessSpawnNeedsInstance, ClaudeProviderCredentialUnbound, ClaudeCodeProvider, + dispatch_credential_snapshot_path_for_instance, CodexDispatchProvider, dispatch_tmux_session_name, dispatch_tmux_session_prefix, @@ -175,6 +178,7 @@ import gunbc.roadmap_dashboard_instance { HostDashboardInstance, srv1_lab_dashboard_instance, srv1_live_dashboard_instance, + srv2_lab_dashboard_instance, dashboard_instance_provider_executable, } import gunbc.roadmap.roadmap_attempt_continuation { @@ -456,9 +460,9 @@ fn codex_tmux_arm_shape_holds(spawn: HostExecArgv, provider_start: HostExecArgv, test fn witness_selected_provider_shapes_codex_tmux_command() -> Bool { let node = sized_derivable(id: "codex-selected", prs: [1], title: "Codex selected") match dispatch_spawn_commands_for_inventory( - instance: srv1_live_dashboard_instance(), + instance: srv2_lab_dashboard_instance(), inventory: fixture_inventory_codex_only_available( - instance: srv1_live_dashboard_instance(), + instance: srv2_lab_dashboard_instance(), ), node: node, session_uuid: "uuid-unused-by-codex", @@ -477,7 +481,7 @@ test fn witness_selected_provider_shapes_codex_tmux_command() -> Bool { codex_tmux_command_shape_holds( cmds: cmds, codex_executable: match dashboard_instance_provider_executable( - instance: srv1_live_dashboard_instance(), + instance: srv2_lab_dashboard_instance(), ) { Present { value: executable } => executable as String Absent => "" @@ -495,10 +499,6 @@ fn lab_instance_command_shape_holds(commands: DispatchSpawnCommands, lab: HostDa } fn lab_instance_tmux_shape_holds(commands: DispatchSpawnCommands, lab: HostDashboardInstance, spawn: HostExecArgv, event_pipe: HostExecArgv, provider_start: HostExecArgv) -> Bool { - match dashboard_instance_provider_executable(instance: lab) { - Absent => false - Present { value: codex_path } => { - let codex_executable = codex_path as String let git_wire = join(commands.worktree_add.args, separator: "\0") let state_wire = join(commands.attempt_state_prepare.args, separator: "\0") let pointer_wire = join(commands.attempt_current_pointer.args, separator: "\0") @@ -534,37 +534,52 @@ fn lab_instance_tmux_shape_holds(commands: DispatchSpawnCommands, lab: HostDashb ) && string_contains( s: provider_wire, - pattern: concat("\0CODEX_HOME=/home/briansrls/.codex@gunbc-roadmap\0", codex_executable), + pattern: dispatch_credential_snapshot_path_for_instance( + instance: lab, + node_id: "ts-dispatch-provider-events" as NonEmptyStr as RoadmapNodeId, + attempt_key: "feedfacefeedface", + provider: ClaudeCodeProvider, + ) as String, ) - } - } } -test fn witness_lab_instance_moves_repo_tmux_and_codex_state_together() -> Bool { +// THE LAB INSTANCE MOVES ITS REPO, TMUX SERVER AND ATTEMPT STATE TOGETHER, shown with the Claude +// spawn (its credential is held on srv1). It no longer moves a codex state root with it: codex worker +// turns run on the one credential held on its custody host (gunbc.codex_harness_credential, operator +// ruling 2026-10-05), so the same lab refuses a codex spawn at the custody-host step -- the control +// that a codex turn cannot fall back to the lab's ambient codex login. +test fn witness_lab_instance_moves_repo_tmux_and_attempt_state_together() -> Bool { let lab = srv1_lab_dashboard_instance() - match dispatch_spawn_commands_for_inventory( + let node = sized_derivable(id: "ts-dispatch-provider-events", prs: [], title: "Provider execution facts") + let claude_moves = match dispatch_spawn_commands_for_resolved_instance( instance: lab, - inventory: fixture_inventory_codex_only_available(instance: lab), - node: sized_derivable( - id: "ts-dispatch-provider-events", - prs: [], - title: "Provider execution facts", - ), + node: node, session_uuid: "00000000-0000-4000-8000-000000000001", attempt_key: "feedfacefeedface", - request: SelectCodex { - profile: ProfileAutomatic, - account: codex_roadmap_account_selection(), - model: dispatch_codex_default_model, - effort: ReasoningMedium, - operator_asserted_native_limit_bucket: Absent, - }, + provider: ClaudeCodeProvider, + effort: ReasoningMedium, + model: ProviderAccountDefaultModel, + origin: FreshFromBase, resolution: fixture_ready_resolution(node_id: "ts-dispatch-provider-events", project_ids: []), ) { DispatchSpawnRefused { step: _, detail: _ } => false - DispatchSpawnReady { commands } => - lab_instance_command_shape_holds(commands: commands, lab: lab) + DispatchSpawnReady { commands } => lab_instance_command_shape_holds(commands: commands, lab: lab) + } + let codex_refused = match dispatch_spawn_commands_for_resolved_instance( + instance: lab, + node: node, + session_uuid: "00000000-0000-4000-8000-000000000001", + attempt_key: "feedfacefeedface", + provider: CodexDispatchProvider, + effort: ReasoningMedium, + model: dispatch_codex_default_model, + origin: FreshFromBase, + resolution: fixture_ready_resolution(node_id: "ts-dispatch-provider-events", project_ids: []), + ) { + DispatchSpawnRefused { step, detail: _ } => step as String == "codex-credential-custody-host" + DispatchSpawnReady { commands: _ } => false } + claude_moves && codex_refused } test fn witness_tmux_attempt_panes_parse_alive_and_exit() -> Bool { @@ -1110,7 +1125,9 @@ test fn witness_codex_account_default_passes_no_model_flag() -> Bool { fn witness_inner_argv_wire(inner: DispatchProviderInnerArgv) -> String { match inner { DispatchProviderInnerArgvReady { argv } => join(argv, separator: "\0") - CursorProviderSpawnUnwired { reason: r } => r as String + CursorProviderOffCustodyHost { reason: r } => r as String + CursorProviderCredentialUnbound { reason: r } => r as String + CodexProviderOffCustodyHost { reason: r } => r as String CodexProviderExecutableAbsent { reason: r } => r as String GunbcHarnessSpawnNeedsInstance { reason: r } => r as String ClaudeProviderCredentialUnbound { reason: r } => r as String @@ -1217,9 +1234,9 @@ test fn witness_continuation_origin_branches_the_worktree_from_the_prior_attempt node: sized_derivable(id: "ts-continuation-origin", prs: [], title: "Continuation origin wiring"), session_uuid: "00000000-0000-4000-8000-000000000002", attempt_key: "feedfacefeedface", - provider: CodexDispatchProvider, + provider: ClaudeCodeProvider, effort: ReasoningMedium, - model: dispatch_codex_default_model, + model: ProviderAccountDefaultModel, origin: ContinueFromAttempt { parent_attempt_key: "deadbeefdeadbeef", branch: "roadmap/ts-continuation-origin/deadbeefdeadbeef", diff --git a/dag/test/claim/serving_routing_witness_test.dag b/dag/test/claim/serving_routing_witness_test.dag index d9928bb3401..473e44cd312 100644 --- a/dag/test/claim/serving_routing_witness_test.dag +++ b/dag/test/claim/serving_routing_witness_test.dag @@ -13,7 +13,7 @@ import gunbc.serving.admitted_model { serving_admits_id, serving_admit_advertised, ServingModelAdmitted, ServingModelNotAdmitted, } import gunbc.harness.harness_reasoning_wire { - DeepseekV4FlashOnVllm, Glm53FlashOnVllm, harness_unit_model_id, + DeepseekV4FlashOnVllm, Glm53FlashOnVllm, NemotronUltraFreeOnOpenRouter, harness_unit_model_id, harness_unit_declared_context_ceiling, } import gunbc.harness.harness_wire { harness_context_budget_for } @@ -126,7 +126,7 @@ fn w_unit_is_glm(id: String) -> Bool { match serving_admits_id(id: id) { ServingModelNotAdmitted { advertised: _ } => false ServingModelAdmitted { id: _, unit: u } => - match u { Glm53FlashOnVllm => true DeepseekV4FlashOnVllm => false } + match u { Glm53FlashOnVllm => true DeepseekV4FlashOnVllm => false NemotronUltraFreeOnOpenRouter => false } } } @@ -174,7 +174,7 @@ test fn the_admitted_id_is_the_advertised_one_and_not_the_units_canonical_id() - ServingModelAdmitted { id: i, unit: u } => (i as String) == "glm-5.3-flash" && (i as String) != (harness_unit_model_id(unit: Glm53FlashOnVllm) as String) - && (match u { Glm53FlashOnVllm => true DeepseekV4FlashOnVllm => false }) + && (match u { Glm53FlashOnVllm => true DeepseekV4FlashOnVllm => false NemotronUltraFreeOnOpenRouter => false }) ServingModelNotAdmitted { advertised: _ } => false }) && (harness_unit_model_id(unit: Glm53FlashOnVllm) as String) != "glm-5.3-flash" diff --git a/dag/test/claim/shell/shell_dag_codex_app_server_trip_script_witness_test.dag b/dag/test/claim/shell/shell_dag_codex_app_server_trip_script_witness_test.dag index 1153d4b20d1..5552275c336 100644 --- a/dag/test/claim/shell/shell_dag_codex_app_server_trip_script_witness_test.dag +++ b/dag/test/claim/shell/shell_dag_codex_app_server_trip_script_witness_test.dag @@ -50,6 +50,7 @@ fn trip_surface(codex_home: String) -> String { let invocation = codex_press_account_trip_invocation( executable: "/usr/bin/codex" as FilePath, codex_home: codex_home as FilePath, + lock: none, ) let argument_words: List = list_flat_map( xs: cli_surface_arguments(surface: process_argv_expansion_surface(expansion: invocation)), diff --git a/dag/test/claim/worker_turn_dispatch_witness_test.dag b/dag/test/claim/worker_turn_dispatch_witness_test.dag new file mode 100644 index 00000000000..2280b8a2026 --- /dev/null +++ b/dag/test/claim/worker_turn_dispatch_witness_test.dag @@ -0,0 +1,172 @@ +module test.claim.worker_turn_dispatch_witness_test + +import std.types { Bool, List, NonEmptyStr, String } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.optional { Present, Absent } +import gunbc.roadmap_model { RoadmapNodeId } +import gunbc.roadmap_launch_admission { launch_identity_exemplar } +import gunbc.roadmap_belt_actuate { belt_snapshot_disposition, SnapshotConsumedByChild, SnapshotDiscardRequired, SpawnFailed } +import gunbc.roadmap_dashboard_instance { HostDashboardInstance, srv1_live_dashboard_instance, srv2_lab_dashboard_instance } +import gunbc.roadmap_dispatch_actuator { + DispatchProviderInnerArgv, + DispatchProviderInnerArgvReady, + CodexProviderExecutableAbsent, + CursorProviderOffCustodyHost, + CursorProviderCredentialUnbound, + CodexProviderOffCustodyHost, + ClaudeProviderCredentialUnbound, + GunbcHarnessSpawnNeedsInstance, + CursorDispatchProvider, + CodexDispatchProvider, + DispatchCliProvider, + dispatch_provider_inner_argv_for_instance, + dispatch_credential_snapshot_path_for_instance, +} +import extdeps.llm.cli { ReasoningMedium, ProviderAccountDefaultModel } +import gunbc.cursor_harness_credential { cursor_worker_turn_api_key_host_path } +import gunbc.codex_harness_credential { codex_harness_turn_lock_path, codex_worker_turn_codex_home } +import gunbc.host_layout { srv2_roadmap_codex_home } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE PRODUCTION SPAWN FOLD for a dashboard instance, with nothing supplied but the instance: these +// claims exercise dispatch_provider_inner_argv_for_instance over the declared srv1 and srv2 instances, +// so a change that sends a Cursor or Codex turn to an ambient login, or to a host that does not hold +// the credential, reds here. +fn inner_for(instance: HostDashboardInstance, provider: DispatchCliProvider) -> DispatchProviderInnerArgv { + dispatch_provider_inner_argv_for_instance( + instance: instance, + provider: provider, + effort: ReasoningMedium, + model: ProviderAccountDefaultModel, + session_uuid: "00000000-0000-4000-8000-000000000000", + node_id: "node-worker-turn" as NonEmptyStr as RoadmapNodeId, + attempt_key: "0123456789abcdef", + worktree_path: "/tmp/w", + brief: "brief", + alignment_chain_fingerprint: "fp", + ) +} + +fn ready_argv(inner: DispatchProviderInnerArgv) -> List? { + match inner { + DispatchProviderInnerArgvReady { argv } => Present { value: argv } + CodexProviderExecutableAbsent { reason: _ } => none + CursorProviderOffCustodyHost { reason: _ } => none + CursorProviderCredentialUnbound { reason: _ } => none + CodexProviderOffCustodyHost { reason: _ } => none + ClaudeProviderCredentialUnbound { reason: _ } => none + GunbcHarnessSpawnNeedsInstance { reason: _ } => none + } +} + +fn first_word_is(argv: List, word: String) -> Bool { + match argv.first() { + Present { value: w } => w == word + Absent => false + } +} + +fn has_pair(argv: List, flag: String, value: String) -> Bool { + string_contains(s: join(argv, separator: "\0"), pattern: join(["\0", flag, "\0", value], "")) +} + +// THE CURSOR ROUTE on the custody host: the key is exported inside the child from the attempt's +// credential snapshot, never the mutable custody file (the shared sh wrapper is the first word and the +// snapshot path is its argument), the agent runs +// unattended in print mode against the attempt worktree, and no word of the argv is --api-key. +test fn a_cursor_turn_on_srv2_binds_the_custody_key_through_the_env_wrapper() -> Bool { + match ready_argv(inner: inner_for(instance: srv2_lab_dashboard_instance(), provider: CursorDispatchProvider)) { + Absent => false + Present { value: argv } => + first_word_is(argv: argv, word: "sh") + && any(argv, a => a == dispatch_credential_snapshot_path_for_instance( + instance: srv2_lab_dashboard_instance(), + node_id: "node-worker-turn" as NonEmptyStr as RoadmapNodeId, + attempt_key: "0123456789abcdef", + provider: CursorDispatchProvider, + ) as String) + && !any(argv, a => a == cursor_worker_turn_api_key_host_path as String) + && any(argv, a => a == "cursor-agent") + && any(argv, a => a == "--print") + && any(argv, a => a == "--force") + && any(argv, a => a == "--trust") + && has_pair(argv: argv, flag: "--sandbox", value: "enabled") + && has_pair(argv: argv, flag: "--workspace", value: "/tmp/w") + && !any(argv, a => a == "--api-key") + } +} + +test fn a_cursor_turn_off_its_custody_host_refuses() -> Bool { + match inner_for(instance: srv1_live_dashboard_instance(), provider: CursorDispatchProvider) { + CursorProviderOffCustodyHost { reason } => string_contains(s: reason as String, pattern: "srv2") + CursorProviderCredentialUnbound { reason: _ } => false + DispatchProviderInnerArgvReady { argv: _ } => false + CodexProviderExecutableAbsent { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false + ClaudeProviderCredentialUnbound { reason: _ } => false + GunbcHarnessSpawnNeedsInstance { reason: _ } => false + } +} + +// THE CODEX ROUTE on the custody host: the whole command runs under the exclusive non-blocking flock +// on the custody lock, with conflict exit 75, and CODEX_HOME is the custody directory -- never the +// instance's ambient roadmap login, which is the arm this replaced. +test fn a_codex_turn_on_srv2_runs_under_the_flock_on_the_custody_codex_home() -> Bool { + match inner_for(instance: srv2_lab_dashboard_instance(), provider: CodexDispatchProvider) { + DispatchProviderInnerArgvReady { argv } => + first_word_is(argv: argv, word: "flock") + && any(argv, a => a == "--exclusive") + && any(argv, a => a == "--nonblock") + && has_pair(argv: argv, flag: "--conflict-exit-code", value: "75") + && any(argv, a => a == codex_harness_turn_lock_path as String) + && any(argv, a => a == concat("CODEX_HOME=", codex_worker_turn_codex_home as String)) + && !any(argv, a => a == concat("CODEX_HOME=", srv2_roadmap_codex_home as String)) + CodexProviderExecutableAbsent { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + CodexProviderOffCustodyHost { reason: _ } => false + ClaudeProviderCredentialUnbound { reason: _ } => false + GunbcHarnessSpawnNeedsInstance { reason: _ } => false + } +} + +test fn a_codex_turn_off_its_custody_host_refuses() -> Bool { + match inner_for(instance: srv1_live_dashboard_instance(), provider: CodexDispatchProvider) { + CodexProviderOffCustodyHost { reason } => string_contains(s: reason as String, pattern: "srv2") + DispatchProviderInnerArgvReady { argv: _ } => false + CodexProviderExecutableAbsent { reason: _ } => false + CursorProviderOffCustodyHost { reason: _ } => false + CursorProviderCredentialUnbound { reason: _ } => false + ClaudeProviderCredentialUnbound { reason: _ } => false + GunbcHarnessSpawnNeedsInstance { reason: _ } => false + } +} + +// THE CURSOR ABANDONMENT CONTROL. The snapshot the cursor spawn loads is the provider-labelled path +// the belt discards for every post-selection outcome but a launched child +// (gunbc.roadmap_belt_actuate belt_snapshot_owned_outcome), and the spawn's wrapper refuses the exec +// (exit 79) when it cannot remove that snapshot. If the cursor spawn read any other path, the belt's +// discard would miss it and an abandoned attempt would leave the key on disk. +test fn an_abandoned_cursor_attempt_leaves_no_snapshot_the_belt_does_not_discard() -> Bool { + let belt_path = dispatch_credential_snapshot_path_for_instance( + instance: srv2_lab_dashboard_instance(), + node_id: "node-worker-turn" as NonEmptyStr as RoadmapNodeId, + attempt_key: "0123456789abcdef", + provider: CursorDispatchProvider, + ) + let id = launch_identity_exemplar() + let abandoned_discards = match belt_snapshot_disposition( + outcome: SpawnFailed { node_id: "node-worker-turn", step: "cursor-provider-preflight", detail: "refused", provider: "cursor", launch: id }, + ) { + SnapshotDiscardRequired => true + SnapshotConsumedByChild => false + } + match ready_argv(inner: inner_for(instance: srv2_lab_dashboard_instance(), provider: CursorDispatchProvider)) { + Absent => false + Present { value: argv } => + abandoned_discards + && any(argv, a => a == belt_path as String) + && any(argv, a => string_contains(s: a, pattern: "|| exit 79")) + } +} diff --git a/dag/test/claim/worker_turn_standing_witness_test.dag b/dag/test/claim/worker_turn_standing_witness_test.dag new file mode 100644 index 00000000000..56d7e45f269 --- /dev/null +++ b/dag/test/claim/worker_turn_standing_witness_test.dag @@ -0,0 +1,192 @@ +module test.claim.worker_turn_standing_witness_test + +import std.types { Bool, List, NonEmptyStr, String, FilePath } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.optional { Present, Absent } +import std.claim_evidence { RecordedFactId } +import gunbc.roadmap_model { RoadmapNodeId } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv2_lab_dashboard_instance } +import gunbc.codex_app_server_press { parse_account_trip_stdout_for_tests, CodexAccountStandingEvidence } +import gunbc.provider_standing_probe_bridge { provider_standing_bundle_from_codex_account_evidence } +import gunbc.provider_standing { + ProviderStanding, + ProviderStandingUnobserved, + ProviderStandingFromFact, + InstallationReady, + ToolEnvironmentReady, + LimitDispatchDrawAutomatic, + provider_standing_from_observed_bundle, + provider_standing_is_selection_eligible, +} +import gunbc.dispatch_selection { + ProviderInventory, + OfferCodex, + OfferClaude, + OfferCursor, + provider_inventory_for_instance, + inventory_with_observed_standing, + inventory_with_codex_state_root, + resolve_dispatch_selection, + provider_selection_request_for_kind, + DispatchSelectionResolved, + DispatchSelectionRefused, + NoDerivedSizingExpectation, + dispatch_codex_process_fingerprint_unobserved_selection_refusal, +} +import extdeps.llm.cli_lifecycle { CodexCliProvider, ProviderInstance } +import gunbc.codex_harness_credential { codex_harness_credential_context, codex_worker_turn_codex_home } +import gunbc.cursor_harness_credential { cursor_status_reading } +import gunbc.worker_turn_standing { cursor_standing_from_status, cursor_entitlement_trip_pending_reason } +import gunbc.roadmap_dispatch_actuator { + dispatch_actuator_worker_turn_selection_observed, + DispatchActuatorSelectionOk, + DispatchActuatorSelectionRefused, + CodexDispatchProvider, + CursorDispatchProvider, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// SUPPLIED AT THE TRIP BOUNDARY: the account trip's stdout, in the wire shapes the press witnesses +// already pin (test.claim.codex_app_server_press). The healthy trip is a chatgpt/pro account with +// capacity left in the codex bucket; the exhausted one carries the live 2026-08-06 rateLimits +// capture verbatim (usedPercent 100, rate_limit_reached). The inhabitance of the trip itself -- the +// app-server run against the custody CODEX_HOME -- is the live exercise once custody lands. +data initialize_line: String = "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"userAgent\":\"x\"}}\n" +data account_read_line: String = "{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"account\":{\"type\":\"chatgpt\",\"planType\":\"pro\",\"email\":\"REDACTED@example.invalid\"},\"requiresOpenaiAuth\":true}}\n" +data rate_limits_capacity_line: String = "{\"jsonrpc\":\"2.0\",\"id\":3,\"result\":{\"rateLimits\":{\"limitId\":\"codex\",\"primary\":{\"usedPercent\":12},\"secondary\":{\"usedPercent\":3},\"spendControlReached\":false}}}\n" +data rate_limits_exhausted_line: String = "{\"jsonrpc\":\"2.0\",\"id\":3,\"result\":{\"rateLimits\":{\"limitId\":\"codex\",\"limitName\":null,\"primary\":{\"usedPercent\":100,\"windowDurationMins\":10080,\"resetsAt\":1786159940},\"secondary\":null,\"spendControlReached\":false,\"planType\":\"pro\",\"rateLimitReachedType\":\"rate_limit_reached\"}}}\n" + +fn trip_evidence(rate_limits_line: String) -> CodexAccountStandingEvidence { + parse_account_trip_stdout_for_tests(stdout: concat(initialize_line, concat(account_read_line, rate_limits_line))) +} + +fn observed_codex_standing(rate_limits_line: String) -> ProviderStanding { + match first_codex_instance(inventory: provider_inventory_for_instance(instance: srv2_lab_dashboard_instance())) { + Absent => ProviderStandingUnobserved { reason: "no codex offer on srv2 lab" as NonEmptyStr } + Present { value: provider } => + provider_standing_from_observed_bundle( + fact_id: "witness-codex-harness-trip" as RecordedFactId, + provider: provider, + bundle: provider_standing_bundle_from_codex_account_evidence( + evidence: trip_evidence(rate_limits_line: rate_limits_line), + credential: codex_harness_credential_context(), + installation: InstallationReady, + tool_environment: ToolEnvironmentReady, + ), + ) + } +} + +fn first_codex_instance(inventory: ProviderInventory) -> ProviderInstance? { + fold(inventory.offers, init: none, f: (acc, offer) => + match acc { + Present { value: _ } => acc + Absent => + match offer { + OfferCodex { offer: o } => Present { value: o.instance } + OfferClaude { offer: _ } => acc + OfferCursor { offer: _ } => acc + } + } + ) +} + +fn codex_selection_reason(standing: ProviderStanding) -> String { + match resolve_dispatch_selection( + inventory: inventory_with_codex_state_root( + inventory: inventory_with_observed_standing(inventory: provider_inventory_for_instance(instance: srv2_lab_dashboard_instance()), kind: CodexCliProvider, standing: standing), + account_state_root: codex_worker_turn_codex_home as FilePath, + ), + request: provider_selection_request_for_kind(kind: CodexCliProvider), + sizing_expectation: NoDerivedSizingExpectation, + ) { + DispatchSelectionResolved { receipt: _ } => "RESOLVED" + DispatchSelectionRefused { sizing_expectation: _, requested_selection: _, reason } => reason as String + } +} + +// THE OBSERVED STANDING REACHES SELECTION. A healthy custody trip makes the codex offer eligible, so +// selection passes eligibility and stops at the next gate, the unobserved process fingerprint (kept by +// operator ruling D1, 2026-10-05, until the wet tmux observation on srv2 is taken). The exhausted trip +// is refused at eligibility instead, with a different reason. If the observation were not reaching +// selection, the two would refuse for the same reason. +test fn a_healthy_custody_trip_passes_eligibility_and_an_exhausted_one_does_not() -> Bool { + let healthy = codex_selection_reason(standing: observed_codex_standing(rate_limits_line: rate_limits_capacity_line)) + let exhausted = codex_selection_reason(standing: observed_codex_standing(rate_limits_line: rate_limits_exhausted_line)) + healthy == (dispatch_codex_process_fingerprint_unobserved_selection_refusal as String) + && exhausted != healthy + && exhausted != "RESOLVED" +} + +test fn the_exhausted_custody_trip_is_not_selection_eligible_and_the_healthy_one_is() -> Bool { + provider_standing_is_selection_eligible( + standing: observed_codex_standing(rate_limits_line: rate_limits_capacity_line), + draw: LimitDispatchDrawAutomatic, + ) + && !provider_standing_is_selection_eligible( + standing: observed_codex_standing(rate_limits_line: rate_limits_exhausted_line), + draw: LimitDispatchDrawAutomatic, + ) +} + +// THE CODEX OFFER IS JUDGED ON THE CUSTODY CODEX_HOME, not the instance's ambient account state root, +// so changing the ambient login cannot change the verdict or the root the spawn binds. +test fn the_observed_codex_offer_names_the_custody_codex_home() -> Bool { + let inventory = inventory_with_codex_state_root( + inventory: inventory_with_observed_standing(inventory: provider_inventory_for_instance(instance: srv2_lab_dashboard_instance()), kind: CodexCliProvider, standing: observed_codex_standing(rate_limits_line: rate_limits_capacity_line)), + account_state_root: codex_worker_turn_codex_home as FilePath, + ) + any(inventory.offers, offer => + match offer { + OfferCodex { offer: o } => (o.account_state_root as String) == (codex_worker_turn_codex_home as String) + OfferClaude { offer: _ } => false + OfferCursor { offer: _ } => false + }) +} + +// CURSOR: a rejected key refuses naming the rejection; an accepted key is still not selectable, +// because entitlement waits on the one-word trip (D2). Neither arm is "available". +data observed_unauthenticated_status: String = "{\n \"status\": \"unauthenticated\",\n \"isAuthenticated\": false,\n \"hasAccessToken\": false,\n \"hasRefreshToken\": false,\n \"message\": \"Not logged in\"\n}" +data supplied_authenticated_status: String = "{\"status\":\"authenticated\",\"isAuthenticated\":true}" + +test fn cursor_status_never_makes_the_offer_selectable_before_the_entitlement_trip() -> Bool { + let accepted = cursor_standing_from_status(reading: cursor_status_reading(stdout: supplied_authenticated_status)) + let rejected = cursor_standing_from_status(reading: cursor_status_reading(stdout: observed_unauthenticated_status)) + let accepted_pending = match accepted { + ProviderStandingUnobserved { reason } => reason == cursor_entitlement_trip_pending_reason + ProviderStandingFromFact { fact: _ } => false + } + let rejected_named = match rejected { + ProviderStandingUnobserved { reason } => string_contains(s: reason as String, pattern: "rejects") + ProviderStandingFromFact { fact: _ } => false + } + accepted_pending && rejected_named + && !provider_standing_is_selection_eligible(standing: accepted, draw: LimitDispatchDrawAutomatic) +} + +// THE ROUTE, through the production actuator selection: on an instance that does not hold the +// credentials, both observed draws refuse before any observation runs or any spawn is planned. +test fn worker_turn_draws_on_a_non_custody_instance_refuse() -> Bool { + let codex = match dispatch_actuator_worker_turn_selection_observed( + instance: srv1_live_dashboard_instance(), + provider: CodexDispatchProvider, + node_id: "node-worker-turn" as NonEmptyStr as RoadmapNodeId, + attempt_key: "0123456789abcdef", + sizing_expectation: NoDerivedSizingExpectation, + ) { + DispatchActuatorSelectionRefused { reason: _ } => true + DispatchActuatorSelectionOk { provider: _, effort: _, model: _, process_fingerprint: _ } => false + } + let cursor = match dispatch_actuator_worker_turn_selection_observed( + instance: srv1_live_dashboard_instance(), + provider: CursorDispatchProvider, + node_id: "node-worker-turn" as NonEmptyStr as RoadmapNodeId, + attempt_key: "0123456789abcdef", + sizing_expectation: NoDerivedSizingExpectation, + ) { + DispatchActuatorSelectionRefused { reason: _ } => true + DispatchActuatorSelectionOk { provider: _, effort: _, model: _, process_fingerprint: _ } => false + } + codex && cursor +}