From 50cf46c332cfb62e567b075eaba57635ce9f122c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 13:54:22 +0000 Subject: [PATCH 01/10] Provider spine: native request families, persistent local store, typed file-failure channel Extend std.materialization_provider with NativeCompilerArtifactRequest, UniverseArtifactRequest and NativeModuleVerdictBundleRequest (+ artifact variants, manifests, RealizationReceipt carrying provider identity outside the key). Add std.materialization_object (realization-agnostic store object, observations, lookup/commit decisions), std.materialization_store_grant (durable host-volume root vs ephemeral witness scratch), and extdeps.realization.materialization_store_local (first real realization: put-if-absent via WriteCreateNew, exact read-back, no listing, no shadow store). GCS interface rows in extdeps.cloud.gcp.storage (no handler). Prerequisite: Filesystem.Read / WriteCreateNew gain a typed error_kind channel (interpreter + 05_emit/05_emit_rust, mirrors regenerated to fixed point) so a miss is never inferred from error text. Controls: 17 hermetic + 8 wet (enrolled on the local-repo wet lane). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/extdeps/cache.dag | 2 + dag/extdeps/cloud/gcp/storage.dag | 91 +++ dag/extdeps/filesystem/filesystem_io.dag | 137 +++- .../materialization_store_local.dag | 360 ++++++++++ dag/gunbc/ci/ci_layer_roots.dag | 5 + dag/std/cache_identity.dag | 6 + dag/std/materialization_object.dag | 674 ++++++++++++++++++ dag/std/materialization_provider.dag | 339 ++++++++- dag/std/materialization_store_grant.dag | 120 ++++ ...alization_store_local_wet_witness_test.dag | 290 ++++++++ .../materialization_store_witness_test.dag | 346 +++++++++ src/v1/05_emit.dag | 2 + src/v1/05_emit_rust.dag | 26 +- src/v1/stage0/src/v1_compiler_emit.rs | 17 +- src/v1/stage0/src/v1_compiler_emit_rust.rs | 28 +- src/v1/stage0/src/v1_interpreter.rs | 39 +- src/v2/workflow/floor_route_gap.dag | 38 +- src/v2/workflow/local_repo_wet_terminal.dag | 58 ++ 18 files changed, 2538 insertions(+), 40 deletions(-) create mode 100644 dag/extdeps/cloud/gcp/storage.dag create mode 100644 dag/extdeps/realization/materialization_store_local.dag create mode 100644 dag/std/materialization_object.dag create mode 100644 dag/std/materialization_store_grant.dag create mode 100644 dag/test/claim/materialization_store_local_wet_witness_test.dag create mode 100644 dag/test/claim/materialization_store_witness_test.dag diff --git a/dag/extdeps/cache.dag b/dag/extdeps/cache.dag index 51e8c061b7b..b60a6793055 100644 --- a/dag/extdeps/cache.dag +++ b/dag/extdeps/cache.dag @@ -34,6 +34,7 @@ import extdeps.realization.reconcile_in_process { typed_module_cache_id, } import extdeps.realization.resolved_graph { resolved_graph_cache_facts, resolved_graph_cache_id } +import extdeps.realization.materialization_store_local { materialization_store_local_facts } import std.cache_identity { CacheInterfaceId } import std.cache_interface { ArtifactIdentity, @@ -143,6 +144,7 @@ data cache_catalog: List = [ resolved_graph_cache, parse_table_memo, compile_stage_memo_facts, + materialization_store_local_facts, ] data cache_catalog_rows: List = cache_catalog diff --git a/dag/extdeps/cloud/gcp/storage.dag b/dag/extdeps/cloud/gcp/storage.dag new file mode 100644 index 00000000000..1a56c76fa89 --- /dev/null +++ b/dag/extdeps/cloud/gcp/storage.dag @@ -0,0 +1,91 @@ +module extdeps.cloud.gcp.storage + +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import std.types { Int, List, String } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "cloud.google.com/storage/docs/json_api/v1/objects" + } +} + +// GOOGLE CLOUD STORAGE JSON API v1 -- THE INTERFACE FACTS A MATERIALIZATION-STORE REALIZATION +// WOULD BIND TO, AND NOTHING ELSE. This module owns the upstream's shape (DESIGN section 3: the +// interface is extdeps', the transport handler is one realization bound to it, the policy is the +// workflow's). It carries no handler and no dispatch: the handler lands when the operator says +// (brief of the provider-spine lane, 2026-09-11), and until then these rows are a DECLARED FRONTIER +// -- their first consumer is a GCS realization of std.materialization_object's observations, the +// same two observations extdeps.realization.materialization_store_local produces from a +// filesystem. TRIGGER: the operator names the GCS realization lane. +// +// What the rows establish, cited: +// objects.get with alt=media returns the object's bytes; 404 is a typed absent object +// (cloud.google.com/storage/docs/json_api/v1/objects/get) -- the exact-read semantics, with no +// listing and no prefix fallback; +// objects.insert with ifGenerationMatch=0 succeeds only if no live object has that name and +// answers 412 Precondition Failed otherwise (cloud.google.com/storage/docs/request-preconditions) +// -- put-if-absent; an object becomes visible only when the upload completes, so a reader never +// observes a partial object; +// read-after-write and read-after-metadata-update are strongly consistent for objects +// (cloud.google.com/storage/docs/consistency) -- a commit's exact read-back is meaningful; +// 401/403 are authorization refusals and 429/5xx are availability refusals +// (cloud.google.com/storage/docs/json_api/v1/status-codes) -- neither is ever an absent object. +// The object NAME is the only locator; there is no rename and no directory, which is exactly why the +// store's interface names neither. + +type GcsObjectOperation + = GcsObjectsGet + | GcsObjectsInsert + +type GcsObjectOperationFacts { + operation: GcsObjectOperation + method: String + path_template: String + required_query: List + citation: String +} + +data gcs_objects_get: GcsObjectOperationFacts = GcsObjectOperationFacts { + operation: GcsObjectsGet, + method: "GET", + path_template: "https://storage.googleapis.com/storage/v1/b/\{bucket\}/o/\{object\}", + required_query: ["alt=media"], + citation: "cloud.google.com/storage/docs/json_api/v1/objects/get" +} + +data gcs_objects_insert_if_absent: GcsObjectOperationFacts = GcsObjectOperationFacts { + operation: GcsObjectsInsert, + method: "POST", + path_template: "https://storage.googleapis.com/upload/storage/v1/b/\{bucket\}/o", + required_query: ["uploadType=media", "name=\{object\}", "ifGenerationMatch=0"], + citation: "cloud.google.com/storage/docs/request-preconditions" +} + +// The status codes the two operations answer, each with the upstream's own meaning. The mapping +// of these meanings onto a store observation is the HANDLER's, not this module's. +type GcsObjectStatusMeaning + = GcsObjectReturned + | GcsObjectNotFound + | GcsPreconditionFailed + | GcsUnauthenticated + | GcsForbidden + | GcsRateLimited + | GcsServerError + +type GcsObjectStatusFact { + status: Int + meaning: GcsObjectStatusMeaning +} + +data gcs_object_status_facts: List = [ + GcsObjectStatusFact { status: 200, meaning: GcsObjectReturned }, + GcsObjectStatusFact { status: 404, meaning: GcsObjectNotFound }, + GcsObjectStatusFact { status: 412, meaning: GcsPreconditionFailed }, + GcsObjectStatusFact { status: 401, meaning: GcsUnauthenticated }, + GcsObjectStatusFact { status: 403, meaning: GcsForbidden }, + GcsObjectStatusFact { status: 429, meaning: GcsRateLimited }, + GcsObjectStatusFact { status: 500, meaning: GcsServerError }, + GcsObjectStatusFact { status: 503, meaning: GcsServerError }, +] diff --git a/dag/extdeps/filesystem/filesystem_io.dag b/dag/extdeps/filesystem/filesystem_io.dag index 3f1ef1009a5..645669553f8 100644 --- a/dag/extdeps/filesystem/filesystem_io.dag +++ b/dag/extdeps/filesystem/filesystem_io.dag @@ -34,7 +34,123 @@ fn filesystem_read_outcome( } } -data filesystem_read_outcome_adoption_standing: String = "RUNG: mitigatable. filesystem_read_outcome provides the single modeled fold from Filesystem.Read's scalar transport observation into FilesystemReadSucceeded | FilesystemReadRefused, but nothing forces callers through it. gunbc.ci_yaml_validate is the first converted consumer and preserves read refusal separately from YAML parse refusal. The raw operation remains directly consumed elsewhere, so content+success+error nonsense combinations remain writable at those sites. SUBJECT: an unconverted consumption site is a Filesystem Read call whose bound result has any success, error, or content projection outside the three arguments of filesystem_read_outcome. The call and those three projections remain after conversion because they supply the fold; adoption changes where they are consumed, not whether they exist. BASELINE measured on origin/main b21b710d5378387ae0c841f07323292c5d72faba: 92 unconverted consumption sites across 48 .dag files. REMAINDER after the first conversion: 91 unconverted consumption sites across 47 files. RE-DERIVATION: enumerate Filesystem Read assignment calls in tracked .dag source, excluding this adoption-standing data declaration so the instrument cannot count its own prose; for each bound result, classify it converted only when every success, error, and content projection is an argument of filesystem_read_outcome, otherwise classify it unconverted; count unconverted rows and distinct paths. NEXT-RUNG TRIGGER: the unconverted population reaches zero -- every Filesystem Read result projection occurs only as an argument to filesystem_read_outcome. The compiler-only filesystem_read intrinsic is a separate replacement migration and is not part of this population or this fold." +// THE HOST'S OWN FAILURE CLASSIFICATION, AS A CHANNEL RATHER THAN AS TEXT. The file transport +// answers `error_kind` beside `error`: the realization projects the host error's kind +// (std::io::ErrorKind on the Rust realizations) into one of the four closed names below, and +// answers the empty string when the operation succeeded. It is the next-rung trigger the +// WriteCreateNew annotation names -- "a modeled already-exists channel on the file transport" -- +// and it discharges it without the heuristic that annotation refused: nothing here reads the error +// TEXT. The host distinguished NotFound from PermissionDenied when the syscall failed; the channel +// carries that distinction instead of discarding it at the transport. +// +// THE ADMISSION REFUSES AN UNRECOGNIZED NAME rather than mapping it to Other. A realization that +// answers a name outside this roster has a different vocabulary from this interface, and folding +// that into the catch-all would hide the divergence (the absorbing fallback, section 5). +type FilesystemFailureKind + = FilesystemNotFound + | FilesystemAlreadyExists + | FilesystemPermissionDenied + | FilesystemOtherFailure + +type FilesystemFailureKindAdmission + = FilesystemFailureKindAdmitted { kind: FilesystemFailureKind } + | FilesystemFailureKindUnrecognized { observed: String } + +fn admit_filesystem_failure_kind(observed: String) -> FilesystemFailureKindAdmission { + if observed == "not_found" { + FilesystemFailureKindAdmitted { kind: FilesystemNotFound } + } else if observed == "already_exists" { + FilesystemFailureKindAdmitted { kind: FilesystemAlreadyExists } + } else if observed == "permission_denied" { + FilesystemFailureKindAdmitted { kind: FilesystemPermissionDenied } + } else if observed == "other" { + FilesystemFailureKindAdmitted { kind: FilesystemOtherFailure } + } else { + FilesystemFailureKindUnrecognized { observed: observed } + } +} + +fn filesystem_failure_kind_name(kind: FilesystemFailureKind) -> String { + match kind { + FilesystemNotFound => "not_found" + FilesystemAlreadyExists => "already_exists" + FilesystemPermissionDenied => "permission_denied" + FilesystemOtherFailure => "other" + } +} + +// A READ WHOSE FAILURE IS TYPED BY THE HOST. `FilesystemExactPathAbsent` is the one arm that +// concludes absence, and it concludes it only for THE EXACT PATH READ and only because the host +// answered NotFound for that path -- which is true both when the final component is missing and when +// an ANCESTOR directory is. A consumer that needs "the directory exists and this entry does not" +// must establish the directory separately (a successful read of a known member of it does); this arm +// does not claim it. Every other failure is `FilesystemExactPathUnreadable` carrying the host's +// kind: present-but-unreadable, permission, I/O -- the worlds the Bool `success` channel conflated +// with absence, and the reason the listing rule above exists for consumers that have only that Bool. +type FilesystemExactRead + = FilesystemExactPathRead { path: String, content: String } + | FilesystemExactPathAbsent { path: String } + | FilesystemExactPathUnreadable { path: String, kind: FilesystemFailureKind, error: String } + | FilesystemExactPathKindUnrecognized { path: String, observed: String, error: String } + +fn filesystem_exact_read( + path: String, + content: String, + success: Bool, + error: String, + error_kind: String, +) -> FilesystemExactRead { + if success { + FilesystemExactPathRead { path: path, content: content } + } else { + match admit_filesystem_failure_kind(observed: error_kind) { + FilesystemFailureKindUnrecognized { observed: o } => + FilesystemExactPathKindUnrecognized { path: path, observed: o, error: error } + FilesystemFailureKindAdmitted { kind: k } => + match k { + FilesystemNotFound => FilesystemExactPathAbsent { path: path } + FilesystemAlreadyExists => FilesystemExactPathUnreadable { path: path, kind: k, error: error } + FilesystemPermissionDenied => FilesystemExactPathUnreadable { path: path, kind: k, error: error } + FilesystemOtherFailure => FilesystemExactPathUnreadable { path: path, kind: k, error: error } + } + } + } +} + +// A CREATE-NEW WHOSE REFUSAL IS TYPED BY THE HOST. `FilesystemCreateTargetOccupied` is the arm a +// put-if-absent protocol needs: the target existed at the instant of publication, so another writer +// (or an earlier run) holds the name, and the caller's next move is to read what is there -- never +// to overwrite it. Every other refusal is the host's and carries its kind. +type FilesystemCreateNew + = FilesystemCreated { path: String } + | FilesystemCreateTargetOccupied { path: String } + | FilesystemCreateRefused { path: String, kind: FilesystemFailureKind, error: String } + | FilesystemCreateKindUnrecognized { path: String, observed: String, error: String } + +fn filesystem_create_new( + path: String, + success: Bool, + error: String, + error_kind: String, +) -> FilesystemCreateNew { + if success { + FilesystemCreated { path: path } + } else { + match admit_filesystem_failure_kind(observed: error_kind) { + FilesystemFailureKindUnrecognized { observed: o } => + FilesystemCreateKindUnrecognized { path: path, observed: o, error: error } + FilesystemFailureKindAdmitted { kind: k } => + match k { + FilesystemAlreadyExists => FilesystemCreateTargetOccupied { path: path } + FilesystemNotFound => FilesystemCreateRefused { path: path, kind: k, error: error } + FilesystemPermissionDenied => FilesystemCreateRefused { path: path, kind: k, error: error } + FilesystemOtherFailure => FilesystemCreateRefused { path: path, kind: k, error: error } + } + } + } +} + +data filesystem_read_outcome_adoption_standing: String ="RUNG: mitigatable. filesystem_read_outcome provides the single modeled fold from Filesystem.Read's scalar transport observation into FilesystemReadSucceeded | FilesystemReadRefused, but nothing forces callers through it. gunbc.ci_yaml_validate is the first converted consumer and preserves read refusal separately from YAML parse refusal. The raw operation remains directly consumed elsewhere, so content+success+error nonsense combinations remain writable at those sites. SUBJECT: an unconverted consumption site is a Filesystem Read call whose bound result has any success, error, or content projection outside the three arguments of filesystem_read_outcome. The call and those three projections remain after conversion because they supply the fold; adoption changes where they are consumed, not whether they exist. BASELINE measured on origin/main b21b710d5378387ae0c841f07323292c5d72faba: 92 unconverted consumption sites across 48 .dag files. REMAINDER after the first conversion: 91 unconverted consumption sites across 47 files. RE-DERIVATION: enumerate Filesystem Read assignment calls in tracked .dag source, excluding this adoption-standing data declaration so the instrument cannot count its own prose; for each bound result, classify it converted only when every success, error, and content projection is an argument of filesystem_read_outcome, otherwise classify it unconverted; count unconverted rows and distinct paths. NEXT-RUNG TRIGGER: the unconverted population reaches zero -- every Filesystem Read result projection occurs only as an argument to filesystem_read_outcome. The compiler-only filesystem_read intrinsic is a separate replacement migration and is not part of this population or this fold." // ABSENCE IS ESTABLISHED BY A SUCCESSFUL LISTING, NEVER BY A FAILED READ. `Read` answers // success=false both for a path that does not exist and for one that exists and cannot be read, and @@ -534,14 +650,15 @@ data filesystem_absence_establishment_adoption_standing: String = "RUNG: structu // THE CREATION the same syscall, so the race has no window rather than a narrow one. Construction // over validation (section 5) at a boundary where validation cannot win. // -// THE REFUSAL DOES NOT CLASSIFY ITSELF, ON PURPOSE. `error` carries the host's message verbatim and -// the operation does NOT report whether the refusal was "already existed" or "permission denied": -// the channels available here cannot separate them, and deciding it by matching on the error TEXT -// would be a heuristic standing in for an observation -- the smuggled-confidence shape section 5 -// names. A caller learns that the create did not happen and why the host said so, which is what it -// needs in order to refuse; it does not learn a classification nothing measured. -// NEXT-RUNG TRIGGER: a modeled already-exists channel on the file transport, at which point the race -// and the permission failure become separate arms and this paragraph is deleted. +// THE REFUSAL IS CLASSIFIED BY THE HOST, NEVER BY ITS TEXT. `error` carries the host's message +// verbatim; `error_kind` carries the kind the host's error already had (see FilesystemFailureKind), +// so "already existed" and "permission denied" are separate arms of filesystem_create_new without +// anything matching on the message. Deciding it from the TEXT would have been a heuristic standing in +// for an observation -- the smuggled-confidence shape section 5 names -- which is why this annotation +// used to leave the refusal unclassified and named the channel as its next-rung trigger. The channel +// is declared on the two operations that have a consumer for it (Read and WriteCreateNew, both +// consumed by extdeps.realization.materialization_store_local); an operation gains it when a +// consumer needs it, since the transport answers it for every verb. service Filesystem { operation Write { @@ -573,6 +690,7 @@ service Filesystem { bytes_written: Int from "bytes_written" path: String from "path" error: String from "error" + error_kind: String from "error_kind" } transport file { path: "{path}", verb: "write_create_new" } } @@ -583,6 +701,7 @@ service Filesystem { content: String from "content" success: Bool from "read_success" error: String from "error" + error_kind: String from "error_kind" } readonly transport file { path: "{path}" } diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag new file mode 100644 index 00000000000..7c0164520ba --- /dev/null +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -0,0 +1,360 @@ +module extdeps.realization.materialization_store_local + +import extdeps.filesystem.filesystem_io { + FilesystemFailureKind, FilesystemNotFound, FilesystemAlreadyExists, FilesystemPermissionDenied, + FilesystemOtherFailure, + FilesystemExactRead, FilesystemExactPathRead, FilesystemExactPathAbsent, + FilesystemExactPathUnreadable, FilesystemExactPathKindUnrecognized, + FilesystemCreateNew, FilesystemCreated, FilesystemCreateTargetOccupied, FilesystemCreateRefused, + FilesystemCreateKindUnrecognized, + filesystem_exact_read, filesystem_create_new, filesystem_failure_kind_name, +} +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.cache.types { CacheInterfaceCatalogFacts, ContentAddressed, CatalogCasContentEncoding } +import std.types { String, Bool, Int, List, NonEmptyStr } +import std.content_hash { ContentHash, content_hash_of_value } +import std.cache_identity { CacheInterfaceId } +import std.cache_interface { + ReleasedNever, + CapacityUnobserved, + ContentAddressLookup, + WriteOnce, + MissThenCreate, + StructuredArtifact, + PerHostFilesystem, + WriteThenCommit, + FilesystemPerms, + LocalDiskUs, + Strong, + OperatorObserved, + ToolchainChange, + ManualPurge, + Org, +} +import v2.std.optional { Absent } +import std.access { Permit, Deny } +import std.effect_grant { Verb, Read, Write, NamespacePosition } +import std.filesystem_path_grant { absolute_path_position } +import std.materialization_ladder { Frame } +import std.materialization_provider { ArtifactRequest, request_key } +import std.materialization_store_grant { + MaterializationStoreRoot, + MaterializationStoreDurability, + materialization_store_root_path, + materialization_store_durability, + materialization_store_durability_name, + materialization_store_admissible, + materialization_store_frame, +} +import std.materialization_object { + ArtifactPayload, + StoreFault, StoreFaultClass, StoreFaultUnauthorized, StoreFaultUnreachable, StoreFaultTransportVocabulary, + StoreObjectRead, StoreObjectPresent, StoreObjectAbsent, StoreObjectUnavailable, + StoreObjectPublish, StoreObjectPublished, StoreObjectOccupied, StoreObjectPublishRefused, + OpenedMaterializationStore, + StoreLookup, StoreLookupRefused, StoreLookupObjectUnavailable, + StoreCommit, StoreCommitRefused, StoreCommitPublishRefused, + StoreCommitPreparation, StoreCommitPrepared, StoreCommitNotPrepared, + store_object_name, store_lookup_decide, store_commit_prepare, store_commit_settle, +} + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/gunb-ai/gunbc/tree/main/dag/extdeps/realization" + } +} + +// THE FIRST REAL REALIZATION OF THE MATERIALIZATION STORE: a local persistent filesystem. +// +// This module is a TRANSPORT HANDLER and nothing more (DESIGN section 3: interface, realization and +// policy are three facts). The store's semantics -- object format, lookup and commit decisions, +// integrity, completeness, kind, the miss/refuse partition -- live in std.materialization_object +// over std.materialization_provider. What lives here is only what is true of THIS realization: +// +// WHERE -- the store root is a std.materialization_store_grant MaterializationStoreRoot, and every +// effect is admitted by that grant before it runs; an object is /. +// HOW A READ IS OBSERVED -- Filesystem.Read's typed failure channel: NotFound is an absent +// object, every other kind is an unavailable one. No listing is consulted, ever. +// HOW A PUBLISH IS OBSERVED -- Filesystem.WriteCreateNew, whose realization stages the bytes, +// fsyncs them and publishes with a hard link, so the name appears complete or not at all and a +// second writer of the same name gets AlreadyExists: put-if-absent by construction. No rename, +// no overwrite, no temp file visible under the object's name. +// WHICH STORE -- a store is OPENED by reading its identity marker; the marker's content hash is the +// store INSTANCE on every receipt. A root with no marker is not a store: open refuses as +// unavailable, so a missing volume can never be misread as an empty store full of misses, and +// nothing is ever written to a substitute location (no shadow store). +// +// Initialization is the one act that creates a store, and it is create-new too, so two +// initializers converge on one marker. + +data materialization_store_local_id: CacheInterfaceId = "materialization_store_local" + +data materialization_store_marker_name: String = "materialization-store.identity" + +data materialization_store_marker_magic: String = "gunbc-materialization-store v1" + +fn store_path(root_path: String, name: String) -> String { + if ends_with(s: root_path, suffix: "/") { concat(root_path, name) } else { concat(root_path, "/", name) } +} + +fn store_marker_content(root: MaterializationStoreRoot, instance_label: NonEmptyStr) -> String { + join([ + materialization_store_marker_magic, + concat("durability ", materialization_store_durability_name(d: materialization_store_durability(root: root))), + concat("instance ", instance_label as String), + "", + ], "\n") +} + +fn store_marker_recognized(root: MaterializationStoreRoot, content: String) -> Bool { + starts_with( + s: content, + prefix: join([ + materialization_store_marker_magic, + concat("durability ", materialization_store_durability_name(d: materialization_store_durability(root: root))), + "instance ", + ], "\n") + ) +} + +// --------------------------------------------------------------------------------------------- +// Transport observations -> the realization-agnostic observations. + +fn filesystem_fault(kind: FilesystemFailureKind, error: String) -> StoreFault { + match kind { + FilesystemPermissionDenied => StoreFault { class: StoreFaultUnauthorized, detail: error } + FilesystemNotFound => StoreFault { class: StoreFaultUnreachable, detail: error } + FilesystemAlreadyExists => StoreFault { class: StoreFaultUnreachable, detail: error } + FilesystemOtherFailure => StoreFault { class: StoreFaultUnreachable, detail: error } + } +} + +fn store_read_observation(read: FilesystemExactRead) -> StoreObjectRead { + match read { + FilesystemExactPathRead { path: _, content: c } => StoreObjectPresent { content: c } + FilesystemExactPathAbsent { path: _ } => StoreObjectAbsent + FilesystemExactPathUnreadable { path: p, kind: k, error: e } => + StoreObjectUnavailable { fault: filesystem_fault(kind: k, error: concat(p, ": ", e)) } + FilesystemExactPathKindUnrecognized { path: p, observed: o, error: e } => + StoreObjectUnavailable { fault: StoreFault { class: StoreFaultTransportVocabulary, detail: join([p, ": unrecognized error_kind ", o, ": ", e], "") } } + } +} + +fn store_publish_observation(create: FilesystemCreateNew) -> StoreObjectPublish { + match create { + FilesystemCreated { path: _ } => StoreObjectPublished + FilesystemCreateTargetOccupied { path: _ } => StoreObjectOccupied + FilesystemCreateRefused { path: p, kind: k, error: e } => + StoreObjectPublishRefused { fault: filesystem_fault(kind: k, error: concat(p, ": ", e)) } + FilesystemCreateKindUnrecognized { path: p, observed: o, error: e } => + StoreObjectPublishRefused { fault: StoreFault { class: StoreFaultTransportVocabulary, detail: join([p, ": unrecognized error_kind ", o, ": ", e], "") } } + } +} + +// --------------------------------------------------------------------------------------------- +// Open. + +type LocalStoreUnavailableCause + = LocalStoreGrantRefused { verb: Verb, target: NamespacePosition, frame: Frame } + | LocalStoreNotInitialized { marker_path: String } + | LocalStoreMarkerUnrecognized { marker_path: String } + | LocalStoreMarkerUnavailable { fault: StoreFault } + +type LocalStoreOpening + = LocalStoreOpened { root: MaterializationStoreRoot, store: OpenedMaterializationStore, durability: MaterializationStoreDurability } + | LocalStoreUnavailable { root_path: String, cause: LocalStoreUnavailableCause } + +// PURE: the open decision over the marker read. Hermetic controls call this directly. +fn local_store_open_decide(root: MaterializationStoreRoot, marker_read: FilesystemExactRead) -> LocalStoreOpening { + let root_path = materialization_store_root_path(root: root) + let marker_path = store_path(root_path: root_path, name: materialization_store_marker_name) + match marker_read { + FilesystemExactPathAbsent { path: _ } => + LocalStoreUnavailable { root_path: root_path, cause: LocalStoreNotInitialized { marker_path: marker_path } } + FilesystemExactPathUnreadable { path: _, kind: k, error: e } => + LocalStoreUnavailable { root_path: root_path, cause: LocalStoreMarkerUnavailable { fault: filesystem_fault(kind: k, error: e) } } + FilesystemExactPathKindUnrecognized { path: _, observed: o, error: e } => + LocalStoreUnavailable { root_path: root_path, cause: LocalStoreMarkerUnavailable { fault: StoreFault { class: StoreFaultTransportVocabulary, detail: concat(o, ": ", e) } } } + FilesystemExactPathRead { path: _, content: c } => + if store_marker_recognized(root: root, content: c) { + LocalStoreOpened { + root: root, + store: OpenedMaterializationStore { + provider: materialization_store_local_id, + instance: content_hash_of_value(value: c as NonEmptyStr) + }, + durability: materialization_store_durability(root: root) + } + } else { + LocalStoreUnavailable { root_path: root_path, cause: LocalStoreMarkerUnrecognized { marker_path: marker_path } } + } + } +} + +func local_store_read(path: String) -> FilesystemExactRead { + let r = Filesystem.Read(path: path) + filesystem_exact_read(path: path, content: r.content, success: r.success, error: r.error, error_kind: r.error_kind) +} + +func local_store_create_new(path: String, content: String) -> FilesystemCreateNew { + let w = Filesystem.WriteCreateNew(path: path, content: content) + filesystem_create_new(path: path, success: w.success, error: w.error, error_kind: w.error_kind) +} + +func local_store_open(root: MaterializationStoreRoot) -> LocalStoreOpening { + let root_path = materialization_store_root_path(root: root) + match materialization_store_admissible(root: root, verb: Read) { + Deny { refusal: _ } => + LocalStoreUnavailable { + root_path: root_path, + cause: LocalStoreGrantRefused { verb: Read, target: absolute_path_position(path: root_path), frame: materialization_store_frame } + } + Permit => + local_store_open_decide( + root: root, + marker_read: local_store_read(path: store_path(root_path: root_path, name: materialization_store_marker_name)) + ) + } +} + +type LocalStoreInitialization + = LocalStoreInitialized { opening: LocalStoreOpening } + | LocalStoreInitializationRefused { root_path: String, cause: LocalStoreUnavailableCause } + +// Initialization publishes the marker create-new and then OPENS the store it created, so the +// returned opening is the same observation every later open makes -- a marker written by a +// concurrent initializer with a different label opens as THAT store, never as ours. +func local_store_initialize(root: MaterializationStoreRoot, instance_label: NonEmptyStr) -> LocalStoreInitialization { + let root_path = materialization_store_root_path(root: root) + match materialization_store_admissible(root: root, verb: Write) { + Deny { refusal: _ } => + LocalStoreInitializationRefused { + root_path: root_path, + cause: LocalStoreGrantRefused { verb: Write, target: absolute_path_position(path: root_path), frame: materialization_store_frame } + } + Permit => + match store_publish_observation(create: local_store_create_new( + path: store_path(root_path: root_path, name: materialization_store_marker_name), + content: store_marker_content(root: root, instance_label: instance_label) + )) { + StoreObjectPublishRefused { fault: f } => + LocalStoreInitializationRefused { root_path: root_path, cause: LocalStoreMarkerUnavailable { fault: f } } + StoreObjectPublished => LocalStoreInitialized { opening: local_store_open(root: root) } + StoreObjectOccupied => LocalStoreInitialized { opening: local_store_open(root: root) } + } + } +} + +// --------------------------------------------------------------------------------------------- +// Lookup and commit over an opened store. + +func local_store_lookup(root: MaterializationStoreRoot, store: OpenedMaterializationStore, req: ArtifactRequest) -> StoreLookup { + let root_path = materialization_store_root_path(root: root) + match materialization_store_admissible(root: root, verb: Read) { + Deny { refusal: _ } => + StoreLookupRefused { + key: request_key(req: req), + refusal: StoreLookupObjectUnavailable { fault: StoreFault { class: StoreFaultUnauthorized, detail: concat("read grant refused for ", root_path) } } + } + Permit => + store_lookup_decide( + store: store, + req: req, + read: store_read_observation(read: local_store_read(path: store_path(root_path: root_path, name: store_object_name(key: request_key(req: req))))) + ) + } +} + +func local_store_commit( + root: MaterializationStoreRoot, + store: OpenedMaterializationStore, + req: ArtifactRequest, + payloads: List +) -> StoreCommit { + let root_path = materialization_store_root_path(root: root) + match store_commit_prepare(req: req, payloads: payloads) { + StoreCommitNotPrepared { commit: c } => c + StoreCommitPrepared { key: k, object_name: name, object: object, offered_digest: offered } => + match materialization_store_admissible(root: root, verb: Write) { + Deny { refusal: _ } => + StoreCommitRefused { + key: k, + refusal: StoreCommitPublishRefused { fault: StoreFault { class: StoreFaultUnauthorized, detail: concat("write grant refused for ", root_path) } } + } + Permit => { + let publish = store_publish_observation(create: local_store_create_new(path: store_path(root_path: root_path, name: name), content: object)) + store_commit_settle( + store: store, + prepared_key: k, + offered_digest: offered, + publish: publish, + read_back: local_store_lookup(root: root, store: store, req: req) + ) + } + } + } +} + +// BATCH: one open, then one exact read per request. The open is the only store-level observation; +// if it refuses, EVERY request in the batch is refused with the same cause -- none is answered as a +// miss, and none is answered from anywhere else. +type LocalStoreBatchLookup + = LocalStoreBatchLooked { store: OpenedMaterializationStore, lookups: List } + | LocalStoreBatchUnavailable { root_path: String, cause: LocalStoreUnavailableCause } + +func local_store_lookup_batch(root: MaterializationStoreRoot, reqs: List) -> LocalStoreBatchLookup { + match local_store_open(root: root) { + LocalStoreUnavailable { root_path: p, cause: c } => LocalStoreBatchUnavailable { root_path: p, cause: c } + LocalStoreOpened { root: r, store: s, durability: _ } => + LocalStoreBatchLooked { store: s, lookups: reqs |> map(req => local_store_lookup(root: r, store: s, req: req)) } + } +} + +// --------------------------------------------------------------------------------------------- +// The cache-interface row this realization's receipts name. + +data materialization_store_local_facts: CacheInterfaceCatalogFacts = { + identity: materialization_store_local_id + backing_surface: "gunbc-materialization-store-persistent-host-volume" + transport_encoding: CatalogCasContentEncoding + key_derivation: { + key_inputs: ContentAddressed + overwritable: false + prefix_fallback_allowed: false + content_verified_on_read: true + visibility_scope: Org + invalidation_triggers: [ToolchainChange, ManualPurge] + } + io: { + lookup_semantics: ContentAddressLookup + write_semantics: WriteOnce + miss_semantics: MissThenCreate + } + placement: { + value_shape: StructuredArtifact + locality: PerHostFilesystem + retention: { release_policy: ReleasedNever, capacity: CapacityUnobserved } + atomicity: WriteThenCommit + auth: FilesystemPerms + read_latency: LocalDiskUs + consistency: Strong + } + evidence: [ + { + field: "atomicity" + evidence: OperatorObserved { + note: "Filesystem.WriteCreateNew stages, fsyncs, and publishes with hard_link (extdeps.filesystem.rust_realization rust_file_write_create_new_fn_def): the object name appears complete or not at all, and a second writer receives AlreadyExists" + observed_at: Absent + } + }, + { + field: "retention" + evidence: OperatorObserved { + note: "no capacity ceiling or eviction is applied by this realization yet: retention and capacity are owned by std.artifact_store under the provider, and this row stays CapacityUnobserved until that accounting binds to the store" + observed_at: Absent + } + }, + ] +} diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index 87b967ae634..f8144aac846 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -902,6 +902,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "materialization_store_local_wet_witness_test.dag", + classification: LocalRepoWetLane, + reason: excl_local_repo_wet_tempdir_write_reason, + dissolution: excl_local_repo_wet_dissolve}, WitnessExclusionRow { pattern: "commit_writer_heal_admission_real_execution_witness_test.dag", classification: LocalRepoWetLane, diff --git a/dag/std/cache_identity.dag b/dag/std/cache_identity.dag index 4e13f488705..40cf760cf12 100644 --- a/dag/std/cache_identity.dag +++ b/dag/std/cache_identity.dag @@ -30,4 +30,10 @@ data typed_module_artifact_kind: ArtifactKindId = "typed_module" data hermetic_fixture_artifact_kind: ArtifactKindId = "hermetic_fixture" +data native_compiler_artifact_kind: ArtifactKindId = "native_compiler" + +data native_test_universe_artifact_kind: ArtifactKindId = "native_test_universe" + +data native_module_verdict_bundle_artifact_kind: ArtifactKindId = "native_module_verdict_bundle" + data cache_interface_product_dissolve_on: DissolutionCondition = unbound_dissolution(description: "dissolve-on: std.cache_identity.CacheInterfaceProduct — deleted closed enum fork; sole cache-backend identity authority is CacheInterfaceId (extdeps/cache/*.dag cited rows keyed by id).") diff --git a/dag/std/materialization_object.dag b/dag/std/materialization_object.dag new file mode 100644 index 00000000000..ca72f9e14e9 --- /dev/null +++ b/dag/std/materialization_object.dag @@ -0,0 +1,674 @@ +module std.materialization_object + +import std.types { Bool, Int, List, String, NonEmptyStr } +import std.content_hash { + ContentHash, + Fnv1a64Structural, + content_hash_atom, + content_hash_from_structural_digest, + fnv1a64_structural_hex_digest, + serialize_content_hash, +} +import std.cache_identity { ArtifactKindId, CacheInterfaceId } +import std.measure { ByteSize, byte_size, measure_count } +import std.materialization_provider { + ArtifactRequest, + ArtifactOutputId, + CarriedOutput, + MaterializedArtifact, + ArtifactManifest, + ManifestDecode, ManifestDecoded, ManifestKindUndeclared, ManifestPartUndeclared, ManifestPartRepeated, + RequestArtifactBinding, BindingOk, BindingKindMismatch, BindingWrongArtifact, BindingWrongContent, + BindingIncomplete, + ProviderLookup, + ProbeFound, + RealizationReceipt, RealizationDisposition, + RealizationServed, RealizationCommitted, RealizationConvergedOnPriorCommit, + request_key, request_kind, + artifact_manifest, artifact_from_manifest, artifact_content_digest, artifact_kind, + carried_outputs_digest, request_artifact_binding, provider_serve, lookup_fold, + provider_lookup_outcome_tag, +} + +// THE REALIZATION-AGNOSTIC HALF OF A PERSISTENT MATERIALIZATION STORE. +// +// Division of authority (operator rulings 2026-09-11, design review C2/C3): the native route emits +// typed ArtifactRequests; std.materialization_provider owns request<->artifact correctness; a +// realization binding at the host/workflow edge selects WHERE the bytes live (local persistent +// filesystem today; GCS and NFS later); retention, capacity and eviction stay with +// std.artifact_store and the provider machinery under it. This module is the seam between the +// second and third: everything a store must decide that does NOT depend on where it stores -- +// the stored OBJECT's format, what a read or a publish OBSERVED, and the lookup and commit +// DECISIONS over those observations. A realization maps its transport onto the two observation +// coproducts below and calls these folds; it adds no semantics of its own. +// +// That split is what "GCS-ready on day one" means, stated so it can be checked: no type in this +// module or in std.materialization_provider names a path, mount, rename, directory or +// permission. The local realization reaches the store through a file path and a create-new +// publish; a GCS realization would reach the same observations through an object name, a GET that +// answers 404, and an insert under the ifGenerationMatch=0 precondition that answers 412 when the +// name is taken (extdeps.cloud.gcp.storage carries those interface facts). NFS is a third, distinct +// realization with its own cross-host obligations -- fencing, leases, stale-writer and temp +// cleanup, read-after-commit across hosts -- and nothing here assumes a single host: publication is +// put-if-absent against ONE name per request key, and correctness never consults a listing. +// +// THE PORTABLE SEMANTICS, each carried by a construction below rather than by a promise: +// content addressing -- the object is named by the request key alone (store_object_name); +// kinds with manifests -- every object carries its ArtifactManifest, decoded only through +// std.materialization_provider artifact_from_manifest; +// put-if-absent / commit -- StoreObjectPublish has an Occupied arm and no overwrite arm; +// exact read-back -- a commit is not reported until a read of the committed name verifies; +// no prefix or existence fallback -- a lookup reads exactly one name and a present object is +// SERVED only by the contract's door, never because it exists; +// batch -- the batch folds are per-request folds over one opened store; +// no listing -- no observation here is a listing; +// typed unavailable / corrupt / incomplete -- each is its own refusal arm. +// +// AND THE ONE RULE THE BRIEF STATES AS SEMANTICS: a MISS may execute; unavailable, corrupt, +// wrong-kind, incomplete or unverifiable read-back REFUSE and are never reclassified as a miss. +// That is enforced by where StoreLookupMiss can come from: only StoreObjectAbsent reaches it. + +// --------------------------------------------------------------------------------------------- +// Payloads and their parts. +// +// A payload is the bytes of one required output. Its PART -- the (id, digest, size) the manifest +// and the contract carry -- is DERIVED from the payload here, and this is the only place a part is +// minted from bytes, which is the grounding std.materialization_provider's derivation invariant +// names for source (3): observed_digest is computed from the stored parts by this derivation, not +// asserted by a caller. The digest is tagged so an empty payload (an empty diagnostics output is +// ordinary) still hashes a non-empty value. The size is the payload's string_length, the same +// measure the decode re-derives, so a size that disagrees with the bytes changes the content digest. + +type ArtifactPayload { + id: ArtifactOutputId + content: String +} + +fn payload_part(payload: ArtifactPayload) -> CarriedOutput { + CarriedOutput { + id: payload.id, + digest: content_hash_atom(value: concat("materialization_payload:", payload.content) as NonEmptyStr), + bytes: byte_size(count: string_length(payload.content)) + } +} + +fn payload_for(payloads: List, id: ArtifactOutputId) -> ArtifactPayload? { + payloads |> filter(p => p.id == id) |> first() +} + +// --------------------------------------------------------------------------------------------- +// The stored object: one self-describing text object per request key. +// +// ONE object, not a manifest beside payload files, because the publish must be ONE put-if-absent: +// a manifest committed before its payloads is an incomplete artifact visible to readers, and a GCS +// object or a create-new file publishes atomically only as a unit. Format (line-oriented, "\n"): +// +// gunbc-materialization-object v1 +// kind +// request_key +// part (one per manifest part, manifest order) +// payload (one per manifest part, manifest order) +// end +// +// Payload text is percent-encoded for exactly three code points -- "%" as %25, line feed as %0A, +// space as %20 (RFC 3986 percent-encoding, restricted to the delimiters this format uses) -- so +// every line splits into a fixed number of space-separated fields and no payload can forge a line. +// Decoding replaces %0A and %20 before %25; since every encoded "%" is followed by "25", "0A" or +// "20", no decoded escape can be manufactured by an earlier replacement. + +data store_object_magic: String = "gunbc-materialization-object v1" + +fn store_object_name(key: ContentHash) -> String { + concat(serialize_content_hash(hash: key) as String, ".materialization") +} + +fn percent_encode_payload(s: String) -> String { + replace(replace(replace(s, "%", "%25"), "\n", "%0A"), " ", "%20") +} + +fn percent_decode_payload(s: String) -> String { + replace(replace(replace(s, "%0A", "\n"), "%20", " "), "%25", "%") +} + +fn store_object_part_line(o: CarriedOutput) -> String { + join(["part", o.id as String, o.digest.digest as String, to_string(measure_count(m: o.bytes))], " ") +} + +fn store_object_payload_line(id: ArtifactOutputId, payloads: List) -> String { + match payload_for(payloads: payloads, id: id) { + Present { value: p } => join(["payload", id as String, percent_encode_payload(s: p.content)], " ") + Absent => join(["payload", id as String], " ") + } +} + +fn encode_store_object(manifest: ArtifactManifest, payloads: List) -> String { + let header = [ + store_object_magic, + join(["kind", manifest.kind as String], " "), + join(["request_key", serialize_content_hash(hash: manifest.request_key) as String], " "), + ] + let parts = manifest.parts |> map(o => store_object_part_line(o: o)) + let bodies = manifest.parts |> map(o => store_object_payload_line(id: o.id, payloads: payloads)) + join(concat(concat(concat(header, parts), bodies), ["end"]), "\n") +} + +type StoreObjectLine + = StorePartLine { part: CarriedOutput } + | StorePayloadLine { payload: ArtifactPayload } + | StoreEndLine + | StoreMalformedLine { line: String } + +fn store_object_line(line: String) -> StoreObjectLine { + let words = split(s: line, delimiter: " ") + let n = words |> count() + match get(xs: words, index: 0) { + Absent => StoreMalformedLine { line: line } + Present { value: tag } => + if tag == "end" && n == 1 { + StoreEndLine + } else if tag == "payload" && (n == 3 || n == 2) { + match get(xs: words, index: 1) { + Absent => StoreMalformedLine { line: line } + Present { value: id } => + if id == "" { StoreMalformedLine { line: line } } + else { + let body = match get(xs: words, index: 2) { + Present { value: b } => b + Absent => "" + } + StorePayloadLine { + payload: ArtifactPayload { id: id as ArtifactOutputId, content: percent_decode_payload(s: body) } + } + } + } + } else if tag == "part" && n == 4 { + match get(xs: words, index: 1) { + Absent => StoreMalformedLine { line: line } + Present { value: id } => + match get(xs: words, index: 2) { + Absent => StoreMalformedLine { line: line } + Present { value: hex } => + match fnv1a64_structural_hex_digest(hex: hex) { + Absent => StoreMalformedLine { line: line } + Present { value: digest } => + match get(xs: words, index: 3) { + Absent => StoreMalformedLine { line: line } + Present { value: size_word } => + match parse_int(s: size_word) { + Absent => StoreMalformedLine { line: line } + Present { value: size } => + if id == "" || size < 0 { StoreMalformedLine { line: line } } + else { + StorePartLine { + part: CarriedOutput { id: id as ArtifactOutputId, digest: digest, bytes: byte_size(count: size) } + } + } + } + } + } + } + } + } else { + StoreMalformedLine { line: line } + } + } +} + +type StoreObjectDecode + = StoreObjectDecoded { manifest: ArtifactManifest, payloads: List } + | StoreObjectMalformed { cause: String } + +fn store_object_header_value(lines: List, index: Int, tag: String) -> String? { + match get(xs: lines, index: index) { + Absent => none + Present { value: line } => + if starts_with(s: line, prefix: concat(tag, " ")) { + match get(xs: split(s: line, delimiter: " "), index: 1) { + Present { value: v } => if split(s: line, delimiter: " ") |> count() == 2 { Present { value: v } } else { none } + Absent => none + } + } else { + none + } + } +} + +fn store_object_body_lines(lines: List) -> List { + lines |> skip(n: 3) |> map(l => store_object_line(line: l)) +} + +fn store_object_line_malformed_text(l: StoreObjectLine) -> String? { + match l { + StoreMalformedLine { line: t } => Present { value: t } + StorePartLine { part: _ } => none + StorePayloadLine { payload: _ } => none + StoreEndLine => none + } +} + +fn store_object_first_malformed(body: List) -> String? { + match body |> filter(l => match store_object_line_malformed_text(l: l) { Present { value: _ } => true, Absent => false }) |> first() { + Present { value: l } => store_object_line_malformed_text(l: l) + Absent => none + } +} + +fn store_object_end_count(body: List) -> Int { + body |> filter(l => match l { + StoreEndLine => true + StorePartLine { part: _ } => false + StorePayloadLine { payload: _ } => false + StoreMalformedLine { line: _ } => false + }) |> count() +} + +fn store_object_last_is_end(body: List) -> Bool { + let n = body |> count() + if n == 0 { false } else { + match get(xs: body, index: n - 1) { + Present { value: l } => match l { + StoreEndLine => true + StorePartLine { part: _ } => false + StorePayloadLine { payload: _ } => false + StoreMalformedLine { line: _ } => false + } + Absent => false + } + } +} + +fn store_object_parts(body: List) -> List { + body |> flat_map(l => match l { + StorePartLine { part: p } => [p] + StorePayloadLine { payload: _ } => [] + StoreEndLine => [] + StoreMalformedLine { line: _ } => [] + }) +} + +fn store_object_payloads(body: List) -> List { + body |> flat_map(l => match l { + StorePayloadLine { payload: p } => [p] + StorePartLine { part: _ } => [] + StoreEndLine => [] + StoreMalformedLine { line: _ } => [] + }) +} + +// A TRUNCATED OBJECT IS MALFORMED, NOT INCOMPLETE. The trailing `end` line is required exactly once +// and last, so an object cut anywhere is refused here rather than decoded into a shorter manifest +// that would then read as a legitimately incomplete artifact. Incompleteness is a property of a +// WELL-FORMED manifest that names too few parts; a torn object is corruption. +fn decode_store_object(content: String) -> StoreObjectDecode { + let lines = split(s: content, delimiter: "\n") + let body = store_object_body_lines(lines: lines) + match get(xs: lines, index: 0) { + Absent => StoreObjectMalformed { cause: "the object is empty" } + Present { value: magic } => + if (magic == store_object_magic) == false { + StoreObjectMalformed { cause: concat("the object does not begin with ", store_object_magic) } + } else { + match store_object_header_value(lines: lines, index: 1, tag: "kind") { + Absent => StoreObjectMalformed { cause: "line 2 is not a `kind ` header" } + Present { value: kind } => + match store_object_header_value(lines: lines, index: 2, tag: "request_key") { + Absent => StoreObjectMalformed { cause: "line 3 is not a `request_key ` header" } + Present { value: key_text } => + match content_hash_from_structural_digest(digest: key_text) { + Absent => StoreObjectMalformed { cause: concat("the recorded request key is not a structural digest: ", key_text) } + Present { value: key } => + match store_object_first_malformed(body: body) { + Present { value: t } => StoreObjectMalformed { cause: concat("unrecognized object line: ", t) } + Absent => + if store_object_end_count(body: body) == 1 && store_object_last_is_end(body: body) { + StoreObjectDecoded { + manifest: ArtifactManifest { + kind: kind as ArtifactKindId, + request_key: key, + parts: store_object_parts(body: body) + }, + payloads: store_object_payloads(body: body) + } + } else { + StoreObjectMalformed { cause: "the object is truncated: it does not end with exactly one `end` line" } + } + } + } + } + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// What a realization observed. These are the ONLY inputs the decisions take from a transport. + +type StoreFaultClass + = StoreFaultUnauthorized + | StoreFaultUnreachable + | StoreFaultTransportVocabulary + +type StoreFault { + class: StoreFaultClass + detail: String +} + +type StoreObjectRead + = StoreObjectPresent { content: String } + | StoreObjectAbsent + | StoreObjectUnavailable { fault: StoreFault } + +type StoreObjectPublish + = StoreObjectPublished + | StoreObjectOccupied + | StoreObjectPublishRefused { fault: StoreFault } + +// An OPENED store: the realization established that the store exists and is the store it claims to +// be before any per-key read is interpreted. This is what makes StoreObjectAbsent mean "no artifact +// at this key" rather than "no store at all" -- a realization that cannot establish its store +// answers unavailable at open, and no lookup is attempted. +type OpenedMaterializationStore { + provider: CacheInterfaceId + instance: ContentHash +} + +// --------------------------------------------------------------------------------------------- +// Lookup. + +type StoreLookupRefusal + = StoreLookupObjectUnavailable { fault: StoreFault } + | StoreLookupObjectMalformed { cause: String } + | StoreLookupManifestUninterpretable { decode: ManifestDecode } + | StoreLookupPayloadMissing { part: ArtifactOutputId } + | StoreLookupPayloadUndeclared { part: ArtifactOutputId } + | StoreLookupContractRefused { lookup: ProviderLookup } + +type StoreLookup + = StoreLookupHit { artifact: MaterializedArtifact, payloads: List, receipt: RealizationReceipt } + | StoreLookupMiss { key: ContentHash } + | StoreLookupRefused { key: ContentHash, refusal: StoreLookupRefusal } + +fn store_payload_undeclared(manifest: ArtifactManifest, payloads: List) -> ArtifactOutputId? { + match payloads |> filter(p => !(manifest.parts |> any(o => o.id == p.id))) |> first() { + Present { value: p } => Present { value: p.id } + Absent => none + } +} + +fn store_payload_missing(manifest: ArtifactManifest, payloads: List) -> ArtifactOutputId? { + match manifest.parts |> filter(o => !(payloads |> any(p => p.id == o.id))) |> first() { + Present { value: o } => Present { value: o.id } + Absent => none + } +} + +// THE OBSERVED DIGEST IS RE-DERIVED FROM THE PAYLOAD BYTES, in manifest order, and the artifact the +// door judges is decoded from the manifest's CLAIMS. The contract compares the two: a flipped payload +// byte, a payload truncated inside its line, or a part digest or size edited in the manifest all +// land on BindingWrongContent -- the integrity refusal -- and nothing here recomputes the artifact +// as a fallback. +fn store_observed_digest(manifest: ArtifactManifest, payloads: List) -> ContentHash { + carried_outputs_digest(outputs: manifest.parts |> flat_map(o => + match payload_for(payloads: payloads, id: o.id) { + Present { value: p } => [payload_part(payload: p)] + Absent => [] + } + )) +} + +fn store_lookup_decide( + store: OpenedMaterializationStore, + req: ArtifactRequest, + read: StoreObjectRead +) -> StoreLookup { + let key = request_key(req: req) + match read { + StoreObjectAbsent => StoreLookupMiss { key: key } + StoreObjectUnavailable { fault: f } => + StoreLookupRefused { key: key, refusal: StoreLookupObjectUnavailable { fault: f } } + StoreObjectPresent { content: c } => + match decode_store_object(content: c) { + StoreObjectMalformed { cause: cause } => + StoreLookupRefused { key: key, refusal: StoreLookupObjectMalformed { cause: cause } } + StoreObjectDecoded { manifest: m, payloads: ps } => + match store_payload_undeclared(manifest: m, payloads: ps) { + Present { value: p } => StoreLookupRefused { key: key, refusal: StoreLookupPayloadUndeclared { part: p } } + Absent => + match store_payload_missing(manifest: m, payloads: ps) { + Present { value: p } => StoreLookupRefused { key: key, refusal: StoreLookupPayloadMissing { part: p } } + Absent => + match artifact_from_manifest(m: m) { + ManifestDecoded { artifact: a } => { + let served = provider_serve( + req: req, + probe: ProbeFound { artifact: a, observed_digest: store_observed_digest(manifest: m, payloads: ps) } + ) + lookup_fold( + l: served, + hit: (artifact, identity) => StoreLookupHit { + artifact: artifact, + payloads: ps, + receipt: RealizationReceipt { + request_key: key, + kind: artifact_kind(artifact: artifact), + content_digest: artifact_content_digest(artifact: artifact), + provider: store.provider, + store_instance: store.instance, + disposition: RealizationServed + } + }, + miss: k => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } }, + kind_mismatch: (k, r, f) => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } }, + wrong_artifact: (k, fk) => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } }, + wrong_content: (k, r, o) => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } }, + cross_family: (k, f) => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } }, + incomplete: (k, missing) => StoreLookupRefused { key: key, refusal: StoreLookupContractRefused { lookup: served } } + ) + } + ManifestKindUndeclared { kind: _ } => + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + ManifestPartUndeclared { kind: _, part: _ } => + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + ManifestPartRepeated { kind: _, part: _ } => + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + } + } + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// Commit: prepare (pure), publish (the realization's one effect), read back (a lookup), settle. + +type StoreCommitRefusal + = StoreCommitManifestUninterpretable { decode: ManifestDecode } + | StoreCommitContractRefused { binding: RequestArtifactBinding } + | StoreCommitPublishRefused { fault: StoreFault } + | StoreCommitReadBackRefused { read_back: StoreLookup } + | StoreCommitConflict { committed: ContentHash, offered: ContentHash } + +type StoreCommit + = StoreCommitSettled { receipt: RealizationReceipt } + | StoreCommitRefused { key: ContentHash, refusal: StoreCommitRefusal } + +type StoreCommitPreparation + = StoreCommitPrepared { key: ContentHash, object_name: String, object: String, offered_digest: ContentHash } + | StoreCommitNotPrepared { commit: StoreCommit } + +// THE WRITE DOOR DERIVES EVERYTHING FROM THE REQUEST AND THE BYTES. The caller hands over payloads +// only: the kind and the request key come from the request, the parts from the payloads, and the +// artifact from artifact_from_manifest -- so a payload the kind does not declare, a repeated +// payload, or a missing required output is refused here, before any byte is published, by the same +// authorities the read side uses. +fn store_commit_prepare(req: ArtifactRequest, payloads: List) -> StoreCommitPreparation { + let key = request_key(req: req) + let offered = ArtifactManifest { + kind: request_kind(req: req), + request_key: key, + parts: payloads |> map(p => payload_part(payload: p)) + } + match artifact_from_manifest(m: offered) { + ManifestDecoded { artifact: a } => + match request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) { + BindingOk => StoreCommitPrepared { + key: key, + object_name: store_object_name(key: key), + object: encode_store_object(manifest: artifact_manifest(artifact: a), payloads: payloads), + offered_digest: artifact_content_digest(artifact: a) + } + BindingKindMismatch { requested: _, found: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + BindingWrongArtifact { key: _, found_key: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + BindingWrongContent { recorded: _, observed: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + BindingIncomplete { missing: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + } + ManifestKindUndeclared { kind: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + ManifestPartUndeclared { kind: _, part: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + ManifestPartRepeated { kind: _, part: _ } => + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + } +} + +// A COMMIT IS NOT REPORTED UNTIL IT READS BACK. Both publish outcomes that can end in a stored +// artifact -- this writer published, or the name was already taken -- are followed by a lookup of the +// same request, and only a verified HIT settles: +// published + hit with the offered digest => Committed; +// occupied + hit with the offered digest => ConvergedOnPriorCommit (concurrent or earlier writer, +// identical verified content -- the protocol working); +// either + hit with a DIFFERENT digest => Conflict: two productions of one key disagree, a +// determinism defect in the producer, refused loudly; +// either + anything but a hit => ReadBackRefused, including a MISS after a publish +// reported success -- an unverifiable commit is not a +// commit. +// A refused publish settles as its fault; no second store is tried and nothing is retained locally. +fn store_commit_settle( + store: OpenedMaterializationStore, + prepared_key: ContentHash, + offered_digest: ContentHash, + publish: StoreObjectPublish, + read_back: StoreLookup +) -> StoreCommit { + match publish { + StoreObjectPublishRefused { fault: f } => + StoreCommitRefused { key: prepared_key, refusal: StoreCommitPublishRefused { fault: f } } + StoreObjectPublished => store_commit_verify(store: store, key: prepared_key, offered: offered_digest, read_back: read_back, disposition: RealizationCommitted) + StoreObjectOccupied => store_commit_verify(store: store, key: prepared_key, offered: offered_digest, read_back: read_back, disposition: RealizationConvergedOnPriorCommit) + } +} + +fn store_commit_verify( + store: OpenedMaterializationStore, + key: ContentHash, + offered: ContentHash, + read_back: StoreLookup, + disposition: RealizationDisposition +) -> StoreCommit { + match read_back { + StoreLookupHit { artifact: a, payloads: _, receipt: r } => + if artifact_content_digest(artifact: a) == offered { + StoreCommitSettled { + receipt: RealizationReceipt { + request_key: r.request_key, + kind: r.kind, + content_digest: r.content_digest, + provider: store.provider, + store_instance: store.instance, + disposition: disposition + } + } + } else { + StoreCommitRefused { key: key, refusal: StoreCommitConflict { committed: artifact_content_digest(artifact: a), offered: offered } } + } + StoreLookupMiss { key: _ } => + StoreCommitRefused { key: key, refusal: StoreCommitReadBackRefused { read_back: read_back } } + StoreLookupRefused { key: _, refusal: _ } => + StoreCommitRefused { key: key, refusal: StoreCommitReadBackRefused { read_back: read_back } } + } +} + +// --------------------------------------------------------------------------------------------- +// Readers. One fold per coproduct; predicates derive from it. + +fn store_lookup_fold( + l: StoreLookup, + hit: fn(MaterializedArtifact, List, RealizationReceipt) -> T, + miss: fn(ContentHash) -> T, + refused: fn(ContentHash, StoreLookupRefusal) -> T +) -> T { + match l { + StoreLookupHit { artifact: a, payloads: p, receipt: r } => hit(a, p, r) + StoreLookupMiss { key: k } => miss(k) + StoreLookupRefused { key: k, refusal: r } => refused(k, r) + } +} + +fn store_lookup_is_hit(l: StoreLookup) -> Bool { + store_lookup_fold(l: l, hit: (a, p, r) => true, miss: k => false, refused: (k, r) => false) +} + +fn store_lookup_is_miss(l: StoreLookup) -> Bool { + store_lookup_fold(l: l, hit: (a, p, r) => false, miss: k => true, refused: (k, r) => false) +} + +fn store_lookup_payload(l: StoreLookup, id: ArtifactOutputId) -> String? { + store_lookup_fold( + l: l, + hit: (a, ps, r) => match payload_for(payloads: ps, id: id) { + Present { value: p } => Present { value: p.content } + Absent => none + }, + miss: k => none, + refused: (k, r) => none + ) +} + +fn store_lookup_refusal_tag(l: StoreLookup) -> String { + store_lookup_fold( + l: l, + hit: (a, p, r) => "hit", + miss: k => "miss", + refused: (k, r) => match r { + StoreLookupObjectUnavailable { fault: _ } => "unavailable" + StoreLookupObjectMalformed { cause: _ } => "malformed" + StoreLookupManifestUninterpretable { decode: _ } => "manifest_uninterpretable" + StoreLookupPayloadMissing { part: _ } => "payload_missing" + StoreLookupPayloadUndeclared { part: _ } => "payload_undeclared" + StoreLookupContractRefused { lookup: pl } => concat("contract_", provider_lookup_outcome_tag(l: pl)) + } + ) +} + +fn store_commit_fold( + c: StoreCommit, + settled: fn(RealizationReceipt) -> T, + refused: fn(ContentHash, StoreCommitRefusal) -> T +) -> T { + match c { + StoreCommitSettled { receipt: r } => settled(r) + StoreCommitRefused { key: k, refusal: r } => refused(k, r) + } +} + +fn store_commit_refusal_tag(c: StoreCommit) -> String { + store_commit_fold( + c: c, + settled: r => match r.disposition { + RealizationServed => "served" + RealizationCommitted => "committed" + RealizationConvergedOnPriorCommit => "converged_on_prior_commit" + }, + refused: (k, r) => match r { + StoreCommitManifestUninterpretable { decode: _ } => "manifest_uninterpretable" + StoreCommitContractRefused { binding: _ } => "contract_refused" + StoreCommitPublishRefused { fault: _ } => "publish_refused" + StoreCommitReadBackRefused { read_back: _ } => "read_back_refused" + StoreCommitConflict { committed: _, offered: _ } => "conflict" + } + ) +} diff --git a/dag/std/materialization_provider.dag b/dag/std/materialization_provider.dag index 6410a4fa2a0..2ce30a9bcb0 100644 --- a/dag/std/materialization_provider.dag +++ b/dag/std/materialization_provider.dag @@ -20,9 +20,13 @@ import std.content_hash { import std.interface_summary { InterfaceHash, ModuleKey, TypedModuleKey, module_key, typed_module_key } import std.cache_identity { ArtifactKindId, + CacheInterfaceId, parse_source_artifact_kind, typed_module_artifact_kind, - resolved_graph_artifact_kind + resolved_graph_artifact_kind, + native_compiler_artifact_kind, + native_test_universe_artifact_kind, + native_module_verdict_bundle_artifact_kind } import std.computation_identity { ComputationIdentity, StructurallyIdentical, IdentityUnknown, @@ -56,6 +60,13 @@ data materialization_provider_derivation_invariant: String = "THE GOVERNING INVA THE ONE CARVE-OUT, AND ITS HONEST CLASS (parent ruling on codex review 44286): source (3), the read-back, is today an ASSERTION — that the transport computed observed_digest from the stored parts by this same derivation. Classifying that as a permanent residual trust boundary would be the section 5 'never' trap: a permanent-sounding residue carries no trigger, so its frequency is zero by construction and it never ranks for fixing. It is not permanent. It is a WALL AFTER GROUNDING, and the grounding is decidable — just not INSIDE this module, which never receives bytes. It is decidable at the TRANSPORT: if the only way to obtain a ProviderProbe is to have called the realization's modeled read operation, the provenance is structural rather than asserted. Note what that does NOT mean — the grounding is not a brand on the digest value, because a brand is freely constructible here and would be forgeable theater (the same argument the 44265 custody token lost); it is that the PROBE is produced by the read rather than hand-built. What blocks it today is that ProbeFound stays freely constructible from any module — the same constructor-confinement gap DESIGN recorded for TransportScript. That precedent has since RESOLVED in the direction this carve-out needs: TransportScript is a sole_constructor record whose single mint is admit_callers-sealed to named production decls, and the cast form of sole_constructor was closed in 04_infer sole_constructor_construction_diags, so 'freely constructible from any module' is no longer the language's fate for a carrier — it is a property of ProbeFound's own declaration, which this module has not yet changed. DISSOLUTION TRIGGER: this carve-out grounds when ProviderProbe is minted only by the modeled read operation (extdeps.realization.artifact_store_fs is the concrete site — a function taking an ArtifactFsLookup and computing the digest itself makes a hand-built ProbeFound impossible at the seam that matters) and constructor confinement lands; tracked against the node/subtree visibility-grants lane, whose Reference verb gates whether the reference edge may be formed at all. Until then this is counted debt, not accepted permanence. WHY IT IS STATED RATHER THAN PATCHED (parent ruling, DESIGN section 6 root-cause-do-not-patch-the-sites): four review findings — 44175 caller-supplied required_outputs, 44265 caller-supplied ComputationIdentity, 44278 caller-authored carried_outputs beside a caller-authored content_digest, and artifact_bytes, found by the parent by reading the three together — were ONE defect found four times, each an accepted assertion where a derivation was available. The fourth was the sharpest because it was load-bearing on a wall built for it: provider_admit_bound branches the budget check on the size and hands that same size to store_put as the stored size, so an understated value did not merely walk past ProviderBudgetExhausted once, it corrupted store_bytes_used and therefore every subsequent LRU eviction decision. The symmetric fix was chosen over carrying the size in the probe precisely BECAUSE of this invariant: a probe field would have been honest but would have added a second item to the trust boundary, whereas a fold over the parts makes the roster the preimage of BOTH the digest and the size — one construction, one argument, no new trust." +// PROGRESS ON THE CARVE-OUT ABOVE, stated narrowly so it is not read as the dissolution: the +// persistent-store realization (std.materialization_object store_lookup_decide) now mints its +// ProbeFound from the stored BYTES -- payload_part derives every part digest and size from the +// payload, and store_observed_digest folds them -- so on that route observed_digest is computed, +// not asserted. The carve-out still stands for every other caller, because ProbeFound remains +// freely constructible; its trigger (constructor confinement of ProviderProbe) is unchanged. + // Computation identity is CONSUMED, never re-invented (mandate rule; DESIGN duplicate-work thread): // the ComputationIdentity qualification belongs to the materialize spine (v2.std.materialize is the // structural-corner qualifier; std.computation_identity is the lattice authority). This slice @@ -114,6 +125,12 @@ data output_typecheck_diagnostics: ArtifactOutputId = "typecheck_diagnostics" data output_resolved_graph: ArtifactOutputId = "resolved_graph" data output_source_indices: ArtifactOutputId = "source_indices" data output_compile_clean_diagnostic_union: ArtifactOutputId = "compile_clean_diagnostic_union" +data output_native_compiler_executable: ArtifactOutputId = "native_compiler_executable" +data output_native_compiler_build_diagnostics: ArtifactOutputId = "native_compiler_build_diagnostics" +data output_universe_test_identity_population: ArtifactOutputId = "universe_test_identity_population" +data output_universe_module_import_closures: ArtifactOutputId = "universe_module_import_closures" +data output_module_test_verdicts: ArtifactOutputId = "module_test_verdicts" +data output_module_preparation_diagnostics: ArtifactOutputId = "module_preparation_diagnostics" type DeclaredInput { ref: NonEmptyStr @@ -138,27 +155,116 @@ type ArtifactRequest compiler: DeclaredInput } | ResolveClosureRequest { closure_digest: Fnv1a64Structural, compiler: DeclaredInput } + | NativeCompilerArtifactRequest { + emitted_closure_identity: Fnv1a64Structural, + build_toolchain: DeclaredInput, + target_platform: DeclaredInput + } + | UniverseArtifactRequest { tested_tree_identity: Fnv1a64Structural, route_binary: DeclaredInput } + | NativeModuleVerdictBundleRequest(NativeModuleVerdictBundleSubject) + +// THE NATIVE ROUTE'S THREE REQUEST FAMILIES (operator rulings 2026-09-11: the replacement native CI +// job is a materialization). Each is a VARIANT for the reason the header gives: kind, declared +// inputs and required outputs are derived, so none of them can be under-specified by a caller. +// +// Every field is an IDENTITY DIGEST, never a location. A path, mount, object name, bucket, rename +// or permission appears in no request and no artifact -- those are facts of one realization (the +// local persistent store names a file; a GCS realization names an object), and putting one here +// would make the computation key depend on where the bytes live. Provider identity is likewise +// absent from every key: it lives in the RealizationReceipt below, so the same request keys the +// same artifact in every realization, which is what lets a later realization serve what an +// earlier one committed without re-keying. +// +// NativeCompilerArtifactRequest: the executable built from the seed-prepared emitted closure. +// Subject = the emitted closure's content identity (gunbc.witness.v2_native_route +// NativeRouteReceipt emitted_closure_identity); declared inputs = the build toolchain identity +// and the target platform identity, because the same closure built by a different rustc or for a +// different triple is a different executable. Requires the executable AND the build diagnostics: +// a build whose warnings were dropped is not the build that ran. +// +// UniverseArtifactRequest: the derived test universe of one tested tree under one route binary -- +// the ordered test-identity population and the per-module import closures the route ingests. +// Subject = the tested tree identity; declared input = the route binary that derived it. +// +// NativeModuleVerdictBundleRequest: one module's verdicts. Its five facts are carried as one +// subject record because every one of them is load-bearing on the verdicts and none is the single +// "thing transformed": the route binary that adjudicates, the module, its canonical import +// closure (a dependency's change must key apart), the ORDERED test-identity population (a test +// added, removed or reordered is a different bundle -- order is part of the identity, as for +// declared inputs above), and the evaluation-interpretation identity (the evaluator semantics +// the verdicts were produced under). Requires the verdicts AND the preparation diagnostics: a +// bundle that dropped the diagnostics of a module that failed to prepare would read as a module +// with no tests, which is the fabricated-plausible-output arm section 5 forbids. +// +// DECLARED FRONTIER (DESIGN section 3c). These three families are consumed today only by the store +// semantics (std.materialization_object, extdeps.realization.materialization_store_local) and their +// executed controls. Their production consumer is the EMITTED NATIVE ROUTE (the successor of +// gunbc.witness.v2_native_route's operator-invoked instrument), which lands in a later lane. +// TRIGGER: the emitted native route constructs these requests from its own receipt identities and +// serves or commits them through the local store instead of re-preparing every module per run. +type NativeModuleVerdictBundleSubject { + route_binary_identity: Fnv1a64Structural + module_identity: Fnv1a64Structural + canonical_module_import_closure_identity: Fnv1a64Structural + ordered_test_identity_population: Fnv1a64Structural + eval_interpretation_identity: Fnv1a64Structural +} fn request_fold( req: ArtifactRequest, parse_module: fn(Fnv1a64Structural, DeclaredInput) -> T, typecheck_module: fn(Fnv1a64Structural, List, DeclaredInput) -> T, - resolve_closure: fn(Fnv1a64Structural, DeclaredInput) -> T + resolve_closure: fn(Fnv1a64Structural, DeclaredInput) -> T, + native_compiler: fn(Fnv1a64Structural, DeclaredInput, DeclaredInput) -> T, + universe: fn(Fnv1a64Structural, DeclaredInput) -> T, + module_verdict_bundle: fn(NativeModuleVerdictBundleSubject) -> T ) -> T { match req { ParseModuleRequest { source_digest: sd, compiler: c } => parse_module(sd, c) TypecheckModuleRequest { source_digest: sd, direct_import_interfaces: ii, compiler: c } => typecheck_module(sd, ii, c) ResolveClosureRequest { closure_digest: cd, compiler: c } => resolve_closure(cd, c) + NativeCompilerArtifactRequest { + emitted_closure_identity: ec, + build_toolchain: bt, + target_platform: tp + } => native_compiler(ec, bt, tp) + UniverseArtifactRequest { tested_tree_identity: tt, route_binary: rb } => universe(tt, rb) + NativeModuleVerdictBundleRequest(subject) => module_verdict_bundle(subject) } } +// The verdict bundle's subject digest folds its five identities in DECLARED field order under a +// tag of its own, so two bundles agree on the subject only when all five agree; the tag keeps it +// from colliding with any other family's subject over the same digests. +fn module_verdict_bundle_subject_digest(subject: NativeModuleVerdictBundleSubject) -> Fnv1a64Structural { + content_hash_tagged_structural( + tag: "materialization_provider.native_module_verdict_bundle", + payload: content_hash_combine_structural( + left: content_hash_combine_structural( + left: content_hash_combine_structural( + left: content_hash_combine_structural( + left: subject.route_binary_identity, + right: subject.module_identity + ), + right: subject.canonical_module_import_closure_identity + ), + right: subject.ordered_test_identity_population + ), + right: subject.eval_interpretation_identity + ) + ) +} + fn request_kind(req: ArtifactRequest) -> ArtifactKindId { request_fold( req: req, parse_module: (sd, c) => parse_source_artifact_kind, typecheck_module: (sd, ii, c) => typed_module_artifact_kind, - resolve_closure: (cd, c) => resolved_graph_artifact_kind + resolve_closure: (cd, c) => resolved_graph_artifact_kind, + native_compiler: (ec, bt, tp) => native_compiler_artifact_kind, + universe: (tt, rb) => native_test_universe_artifact_kind, + module_verdict_bundle: subject => native_module_verdict_bundle_artifact_kind ) } @@ -167,7 +273,10 @@ fn request_subject_digest(req: ArtifactRequest) -> Fnv1a64Structural { req: req, parse_module: (sd, c) => sd, typecheck_module: (sd, ii, c) => sd, - resolve_closure: (cd, c) => cd + resolve_closure: (cd, c) => cd, + native_compiler: (ec, bt, tp) => ec, + universe: (tt, rb) => tt, + module_verdict_bundle: subject => module_verdict_bundle_subject_digest(subject: subject) ) } @@ -177,7 +286,10 @@ fn request_declared_inputs(req: ArtifactRequest) -> List { parse_module: (sd, c) => [c], typecheck_module: (sd, ii, c) => concat(ii |> map(h => DeclaredInput { ref: "direct_import_interface", digest: h }), [c]), - resolve_closure: (cd, c) => [c] + resolve_closure: (cd, c) => [c], + native_compiler: (ec, bt, tp) => [bt, tp], + universe: (tt, rb) => [rb], + module_verdict_bundle: subject => [] ) } @@ -187,7 +299,13 @@ fn request_required_outputs(req: ArtifactRequest) -> List { parse_module: (sd, c) => [output_parse_tree, output_parse_diagnostics], typecheck_module: (sd, ii, c) => [output_typed_module_interface, output_typecheck_diagnostics], resolve_closure: (cd, c) => - [output_resolved_graph, output_source_indices, output_compile_clean_diagnostic_union] + [output_resolved_graph, output_source_indices, output_compile_clean_diagnostic_union], + native_compiler: (ec, bt, tp) => + [output_native_compiler_executable, output_native_compiler_build_diagnostics], + universe: (tt, rb) => + [output_universe_test_identity_population, output_universe_module_import_closures], + module_verdict_bundle: subject => + [output_module_test_verdicts, output_module_preparation_diagnostics] ) } @@ -253,7 +371,10 @@ fn request_key(req: ArtifactRequest) -> ContentHash { interface_key: module_key(source_hash: sd, direct_import_interface_hashes: ii), compiler_identity: c.digest ), - resolve_closure: (cd, c) => local_request_key(req: req) + resolve_closure: (cd, c) => local_request_key(req: req), + native_compiler: (ec, bt, tp) => local_request_key(req: req), + universe: (tt, rb) => local_request_key(req: req), + module_verdict_bundle: subject => local_request_key(req: req) ) } @@ -371,6 +492,21 @@ type MaterializedArtifact source_indices: ArtifactPart, compile_clean_diagnostic_union: ArtifactPart } + | NativeCompilerArtifact { + request_key: ContentHash, + executable: ArtifactPart, + build_diagnostics: ArtifactPart + } + | UniverseArtifact { + request_key: ContentHash, + test_identity_population: ArtifactPart, + module_import_closures: ArtifactPart + } + | ModuleVerdictBundleArtifact { + request_key: ContentHash, + test_verdicts: ArtifactPart, + preparation_diagnostics: ArtifactPart + } | LegacyIncompleteArtifact { kind: ArtifactKindId, request_key: ContentHash, @@ -382,6 +518,9 @@ fn artifact_fold( parsed: fn(ContentHash, ArtifactPart, ArtifactPart) -> T, typed_module: fn(ContentHash, ArtifactPart, ArtifactPart) -> T, resolved_graph: fn(ContentHash, ArtifactPart, ArtifactPart, ArtifactPart) -> T, + native_compiler: fn(ContentHash, ArtifactPart, ArtifactPart) -> T, + universe: fn(ContentHash, ArtifactPart, ArtifactPart) -> T, + module_verdict_bundle: fn(ContentHash, ArtifactPart, ArtifactPart) -> T, legacy_incomplete: fn(ArtifactKindId, ContentHash, List) -> T ) -> T { match artifact { @@ -394,6 +533,12 @@ fn artifact_fold( source_indices: s, compile_clean_diagnostic_union: u } => resolved_graph(k, g, s, u) + NativeCompilerArtifact { request_key: k, executable: e, build_diagnostics: d } => + native_compiler(k, e, d) + UniverseArtifact { request_key: k, test_identity_population: p, module_import_closures: c } => + universe(k, p, c) + ModuleVerdictBundleArtifact { request_key: k, test_verdicts: v, preparation_diagnostics: d } => + module_verdict_bundle(k, v, d) LegacyIncompleteArtifact { kind: kd, request_key: k, carried_outputs: cs } => legacy_incomplete(kd, k, cs) } @@ -405,6 +550,9 @@ fn artifact_kind(artifact: MaterializedArtifact) -> ArtifactKindId { parsed: (k, t, d) => parse_source_artifact_kind, typed_module: (k, i, d) => typed_module_artifact_kind, resolved_graph: (k, g, s, u) => resolved_graph_artifact_kind, + native_compiler: (k, e, d) => native_compiler_artifact_kind, + universe: (k, p, c) => native_test_universe_artifact_kind, + module_verdict_bundle: (k, v, d) => native_module_verdict_bundle_artifact_kind, legacy_incomplete: (kd, k, cs) => kd ) } @@ -415,6 +563,9 @@ fn artifact_request_key(artifact: MaterializedArtifact) -> ContentHash { parsed: (k, t, d) => k, typed_module: (k, i, d) => k, resolved_graph: (k, g, s, u) => k, + native_compiler: (k, e, d) => k, + universe: (k, p, c) => k, + module_verdict_bundle: (k, v, d) => k, legacy_incomplete: (kd, k, cs) => k ) } @@ -435,6 +586,18 @@ fn artifact_carried_outputs(artifact: MaterializedArtifact) -> List [ + positioned_part(id: output_native_compiler_executable, part: e), + positioned_part(id: output_native_compiler_build_diagnostics, part: d) + ], + universe: (k, p, c) => [ + positioned_part(id: output_universe_test_identity_population, part: p), + positioned_part(id: output_universe_module_import_closures, part: c) + ], + module_verdict_bundle: (k, v, d) => [ + positioned_part(id: output_module_test_verdicts, part: v), + positioned_part(id: output_module_preparation_diagnostics, part: d) + ], legacy_incomplete: (kd, k, cs) => cs ) } @@ -451,6 +614,168 @@ fn artifact_bytes(artifact: MaterializedArtifact) -> ByteSize { carried_outputs_bytes(outputs: artifact_carried_outputs(artifact: artifact)) } +// EVERY ARTIFACT KIND CARRIES A MANIFEST, AND THE MANIFEST IS A PROJECTION, NOT A SECOND RECORD. +// A realization stores bytes somewhere this contract never sees, so it must also store WHAT THOSE +// BYTES CLAIM TO BE: the kind, the request key they were produced for, and every part's (id, +// digest, size) in the variant's positional order. artifact_manifest derives that from the +// artifact, and artifact_from_manifest is its inverse -- the ONE place a stored claim becomes a +// MaterializedArtifact again. The two are the contract's own vocabulary read in both directions, +// so a realization that stores manifests cannot invent a kind mapping of its own (section 3). +type ArtifactManifest { + kind: ArtifactKindId + request_key: ContentHash + parts: List +} + +fn artifact_manifest(artifact: MaterializedArtifact) -> ArtifactManifest { + ArtifactManifest { + kind: artifact_kind(artifact: artifact), + request_key: artifact_request_key(artifact: artifact), + parts: artifact_carried_outputs(artifact: artifact) + } +} + +// THE DECODE IS TOTAL AND REFUSES WHAT IT CANNOT NAME. A manifest whose kind this contract does not +// declare, or that names a part its kind does not position, or names one part twice, is not an +// incomplete artifact and not a miss: it is a stored claim this contract cannot interpret, and it +// refuses as such. A manifest MISSING a part its kind requires decodes to the named incompleteness +// variant, so the door's completeness check -- the one authority for it -- is what refuses it, on +// the same arm a request-derived obligation would. +type ManifestDecode + = ManifestDecoded { artifact: MaterializedArtifact } + | ManifestKindUndeclared { kind: ArtifactKindId } + | ManifestPartUndeclared { kind: ArtifactKindId, part: ArtifactOutputId } + | ManifestPartRepeated { kind: ArtifactKindId, part: ArtifactOutputId } + +fn manifest_part(parts: List, id: ArtifactOutputId) -> ArtifactPart? { + match parts |> filter(o => o.id == id) |> first() { + Present { value: o } => Present { value: ArtifactPart { digest: o.digest, bytes: o.bytes } } + Absent => none + } +} + +fn manifest_part_occurrences(parts: List, id: ArtifactOutputId) -> Int { + parts |> filter(o => o.id == id) |> count() +} + +fn manifest_first_undeclared(parts: List, roster: List) -> ArtifactOutputId? { + match parts |> filter(o => !output_list_contains(xs: roster, wanted: o.id)) |> first() { + Present { value: o } => Present { value: o.id } + Absent => none + } +} + +fn manifest_first_repeated(parts: List) -> ArtifactOutputId? { + match parts |> filter(o => manifest_part_occurrences(parts: parts, id: o.id) > 1) |> first() { + Present { value: o } => Present { value: o.id } + Absent => none + } +} + +fn manifest_two_part( + m: ArtifactManifest, + first: ArtifactOutputId, + second: ArtifactOutputId, + build: fn(ArtifactPart, ArtifactPart) -> MaterializedArtifact +) -> ManifestDecode { + match manifest_first_undeclared(parts: m.parts, roster: [first, second]) { + Present { value: p } => ManifestPartUndeclared { kind: m.kind, part: p } + Absent => + match manifest_first_repeated(parts: m.parts) { + Present { value: p } => ManifestPartRepeated { kind: m.kind, part: p } + Absent => + match manifest_part(parts: m.parts, id: first) { + Absent => ManifestDecoded { artifact: LegacyIncompleteArtifact { kind: m.kind, request_key: m.request_key, carried_outputs: m.parts } } + Present { value: a } => + match manifest_part(parts: m.parts, id: second) { + Absent => ManifestDecoded { artifact: LegacyIncompleteArtifact { kind: m.kind, request_key: m.request_key, carried_outputs: m.parts } } + Present { value: b } => ManifestDecoded { artifact: build(a, b) } + } + } + } + } +} + +fn manifest_three_part( + m: ArtifactManifest, + first: ArtifactOutputId, + second: ArtifactOutputId, + third: ArtifactOutputId, + build: fn(ArtifactPart, ArtifactPart, ArtifactPart) -> MaterializedArtifact +) -> ManifestDecode { + match manifest_first_undeclared(parts: m.parts, roster: [first, second, third]) { + Present { value: p } => ManifestPartUndeclared { kind: m.kind, part: p } + Absent => + match manifest_first_repeated(parts: m.parts) { + Present { value: p } => ManifestPartRepeated { kind: m.kind, part: p } + Absent => + match manifest_part(parts: m.parts, id: first) { + Absent => ManifestDecoded { artifact: LegacyIncompleteArtifact { kind: m.kind, request_key: m.request_key, carried_outputs: m.parts } } + Present { value: a } => + match manifest_part(parts: m.parts, id: second) { + Absent => ManifestDecoded { artifact: LegacyIncompleteArtifact { kind: m.kind, request_key: m.request_key, carried_outputs: m.parts } } + Present { value: b } => + match manifest_part(parts: m.parts, id: third) { + Absent => ManifestDecoded { artifact: LegacyIncompleteArtifact { kind: m.kind, request_key: m.request_key, carried_outputs: m.parts } } + Present { value: c } => ManifestDecoded { artifact: build(a, b, c) } + } + } + } + } + } +} + +fn artifact_from_manifest(m: ArtifactManifest) -> ManifestDecode { + let k = m.request_key + if m.kind == parse_source_artifact_kind { + manifest_two_part(m: m, first: output_parse_tree, second: output_parse_diagnostics, + build: (a, b) => ParsedArtifact { request_key: k, parse_tree: a, parse_diagnostics: b }) + } else if m.kind == typed_module_artifact_kind { + manifest_two_part(m: m, first: output_typed_module_interface, second: output_typecheck_diagnostics, + build: (a, b) => TypedModuleArtifact { request_key: k, typed_module_interface: a, typecheck_diagnostics: b }) + } else if m.kind == resolved_graph_artifact_kind { + manifest_three_part(m: m, first: output_resolved_graph, second: output_source_indices, third: output_compile_clean_diagnostic_union, + build: (a, b, c) => ResolvedGraphArtifact { request_key: k, resolved_graph: a, source_indices: b, compile_clean_diagnostic_union: c }) + } else if m.kind == native_compiler_artifact_kind { + manifest_two_part(m: m, first: output_native_compiler_executable, second: output_native_compiler_build_diagnostics, + build: (a, b) => NativeCompilerArtifact { request_key: k, executable: a, build_diagnostics: b }) + } else if m.kind == native_test_universe_artifact_kind { + manifest_two_part(m: m, first: output_universe_test_identity_population, second: output_universe_module_import_closures, + build: (a, b) => UniverseArtifact { request_key: k, test_identity_population: a, module_import_closures: b }) + } else if m.kind == native_module_verdict_bundle_artifact_kind { + manifest_two_part(m: m, first: output_module_test_verdicts, second: output_module_preparation_diagnostics, + build: (a, b) => ModuleVerdictBundleArtifact { request_key: k, test_verdicts: a, preparation_diagnostics: b }) + } else { + ManifestKindUndeclared { kind: m.kind } + } +} + +// PROVIDER IDENTITY LIVES HERE AND NOWHERE ELSE. The request key names WHAT was computed; the +// receipt names WHO realized it and WHERE: the realization's cache-interface identity (a cited +// std.cache_identity CacheInterfaceId row -- the local persistent store, and later GCS or NFS, each +// its own row in its own extdeps module) plus the identity of the store INSTANCE it used. Two +// realizations answering one request therefore produce two receipts over ONE key and ONE content +// digest, which is exactly the comparison a later cross-realization check needs; were the provider +// in the key, the same artifact would have as many identities as it has homes. +// +// The disposition says how the artifact came to be available: served from an existing commit, +// committed by this writer, or found already committed -- with identical verified content -- by a +// concurrent or earlier writer. The last is a SUCCESS, not a race to report: under put-if-absent, +// losing the publication is the protocol working. +type RealizationDisposition + = RealizationServed + | RealizationCommitted + | RealizationConvergedOnPriorCommit + +type RealizationReceipt { + request_key: ContentHash + kind: ArtifactKindId + content_digest: ContentHash + provider: CacheInterfaceId + store_instance: ContentHash + disposition: RealizationDisposition +} + type ArtifactCompleteness = ArtifactComplete | ArtifactMissingOutputs { missing: List } diff --git a/dag/std/materialization_store_grant.dag b/dag/std/materialization_store_grant.dag new file mode 100644 index 00000000000..3f743c46662 --- /dev/null +++ b/dag/std/materialization_store_grant.dag @@ -0,0 +1,120 @@ +module std.materialization_store_grant + +import std.types { String, Bool, List } +import std.access { AccessDecision } +import std.effect_grant { + Verb, Read, Write, + filesystem_position, + NamespacePosition, + RealTransport, + LifecycleByConvention, + Grant, Envelope, + admit_effect, +} +import std.filesystem_path_grant { absolute_path_position } +import std.materialization_ladder { Frame, SharedStateFrame } +import std.artifact_store_scratch_grant { artifact_store_scratch_root, is_gunbc_tmp_scratch_store_root } + +// THE DURABLE ROOT GRANT FOR THE MATERIALIZATION STORE, AND WHY IT IS A COPRODUCT RATHER THAN A PATH. +// +// std.artifact_store_scratch_grant admits /tmp/gunbc_* roots, which are frame-owned scratch: on a +// Firecracker runner the guest /tmp dies with the attempt, so a store rooted there is a per-attempt +// cache and can never discharge a reuse obligation whose lifetime spans runs (DESIGN section 2: +// retention must span the obligated lifetime). The native CI job needs a store that outlives the +// attempt, so its root is a gunbc-owned PERSISTENT HOST VOLUME reached through the declared guest +// boundary, not guest /tmp. +// +// The caller does not supply that path. `DurableHostVolumeRoot` carries no field: the durable root +// is the one declared datum below, so a consumer that asks for durability cannot be handed an +// arbitrary directory (including /tmp) under that name. `WitnessScratchRoot` is the only arm that +// takes a path, it admits only the existing /tmp/gunbc_* scratch grant, and its durability is +// DERIVED as ephemeral -- so an executed control can exercise the real store realization in a +// throwaway directory without that directory ever being able to answer "durable". +// +// DECLARED FRONTIER, stated so the grant is not read as the whole boundary: this module grants the +// guest-side PATH. Attaching the gunbc-owned host volume as a Firecracker drive and mounting it at +// that path inside the guest is the runner lane's act (gunbc.runner runner_attempt_launch attaches +// drives today; it attaches no materialization-store volume yet). Until that attachment lands, a +// durable-root open on a runner guest finds no initialized store and REFUSES as unavailable +// (extdeps.realization.materialization_store_local), never falls back to scratch -- there is no +// shadow store. TRIGGER: the runner attempt launch attaches the persistent host volume at +// materialization_store_durable_root_path. +// +// LIFECYCLE GRADE: LifecycleByConvention, honestly. A durable store is not released by frame +// teardown -- that is its purpose -- so it cannot claim LifecycleByConstruction; its release is the +// retention and eviction the provider machinery owns (std.artifact_store accounting under the +// provider), which is a cleanup that can fail or be forgotten, i.e. the weaker grade. + +type MaterializationStoreRoot + = DurableHostVolumeRoot + | WitnessScratchRoot { root: String } + +type MaterializationStoreDurability + = DurableAcrossRuns + | EphemeralToFrame + +data materialization_store_frame: Frame = Frame { + name: "materialization-store", + kind: SharedStateFrame +} + +data materialization_store_durable_root_path: String = "/var/lib/gunbc/materialization-store" + +data materialization_store_durable_root: NamespacePosition = filesystem_position( + path: ["var", "lib", "gunbc", "materialization-store"] +) + +fn materialization_store_root_path(root: MaterializationStoreRoot) -> String { + match root { + DurableHostVolumeRoot => materialization_store_durable_root_path + WitnessScratchRoot { root: r } => r + } +} + +fn materialization_store_durability(root: MaterializationStoreRoot) -> MaterializationStoreDurability { + match root { + DurableHostVolumeRoot => DurableAcrossRuns + WitnessScratchRoot { root: _ } => EphemeralToFrame + } +} + +fn materialization_store_durability_name(d: MaterializationStoreDurability) -> String { + match d { + DurableAcrossRuns => "durable_across_runs" + EphemeralToFrame => "ephemeral_to_frame" + } +} + +fn materialization_store_grants(root_position: NamespacePosition) -> List { + [ + Grant { verb: Read, root: root_position, binding: RealTransport, lifecycle: LifecycleByConvention }, + Grant { verb: Write, root: root_position, binding: RealTransport, lifecycle: LifecycleByConvention }, + ] +} + +fn materialization_store_envelope(root: MaterializationStoreRoot) -> Envelope { + match root { + DurableHostVolumeRoot => + Envelope { + frame: materialization_store_frame, + grants: materialization_store_grants(root_position: materialization_store_durable_root) + } + WitnessScratchRoot { root: r } => + if is_gunbc_tmp_scratch_store_root(store_root: r) { + Envelope { + frame: materialization_store_frame, + grants: materialization_store_grants(root_position: artifact_store_scratch_root) + } + } else { + Envelope { frame: materialization_store_frame, grants: [] } + } + } +} + +fn materialization_store_admissible(root: MaterializationStoreRoot, verb: Verb) -> AccessDecision { + admit_effect( + env: materialization_store_envelope(root: root), + verb: verb, + target: absolute_path_position(path: materialization_store_root_path(root: root)) + ) +} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag new file mode 100644 index 00000000000..a747cd6f5fe --- /dev/null +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -0,0 +1,290 @@ +module test.claim.materialization_store_local_wet_witness + +import extdeps.shell +import extdeps.filesystem.filesystem_io +import std.types { Bool, Int, List, String, NonEmptyStr } +import std.content_hash { ContentHash, Fnv1a64Structural, content_hash_atom } +import std.materialization_provider { + DeclaredInput, ArtifactRequest, + UniverseArtifactRequest, NativeModuleVerdictBundleRequest, NativeModuleVerdictBundleSubject, + RealizationServed, RealizationCommitted, RealizationConvergedOnPriorCommit, + request_key, + output_module_test_verdicts, output_module_preparation_diagnostics, + output_universe_test_identity_population, output_universe_module_import_closures, +} +import std.materialization_store_grant { MaterializationStoreRoot, WitnessScratchRoot, DurableHostVolumeRoot } +import std.materialization_object { + ArtifactPayload, + OpenedMaterializationStore, + StoreLookup, StoreCommit, StoreCommitSettled, StoreCommitRefused, + StoreCommitPrepared, StoreCommitNotPrepared, + store_commit_prepare, store_object_name, + store_lookup_is_hit, store_lookup_is_miss, store_lookup_payload, store_lookup_refusal_tag, + store_commit_refusal_tag, store_commit_fold, +} +import extdeps.realization.materialization_store_local { + LocalStoreOpening, LocalStoreOpened, LocalStoreUnavailable, + LocalStoreInitialization, LocalStoreInitialized, LocalStoreInitializationRefused, + LocalStoreNotInitialized, LocalStoreGrantRefused, + LocalStoreBatchLookup, LocalStoreBatchLooked, LocalStoreBatchUnavailable, + local_store_initialize, local_store_open, local_store_lookup, local_store_commit, + local_store_lookup_batch, store_path, materialization_store_marker_name, +} + +// WET CONTROLS FOR THE LOCAL PERSISTENT REALIZATION: every control of the provider-spine brief, +// executed against a REAL filesystem through extdeps.realization.materialization_store_local. Each +// test owns a fresh store under a throwaway /tmp/gunbc_matstore.* directory (the WitnessScratchRoot +// arm, whose durability is derived ephemeral -- this file never touches the durable root) and +// removes it before returning. Effects: shell.Mktemp.DirWithTemplate, Filesystem.Read / +// WriteCreateNew / Write (the Write is the corruption injector, never the store's own path), +// shell.Test.IsFile, shell.Remove.RecursiveForce. No network, no cargo, no remote host. +// +// The hermetic decision evidence is test.claim.materialization_store_witness; this file is the +// transport's evidence: that the real Filesystem observations reach those decisions with the +// meanings the realization claims (NotFound is absent, AlreadyExists is occupied, the marker gates +// the store) and that the bytes a commit publishes are the bytes a later read verifies. +// +// Concurrency: the second writer here is SEQUENTIAL, which exercises exactly the loser's arm -- a +// create-new that answers already_exists, then a read-back -- because WriteCreateNew's publication +// is one link(2), so a racing loser observes the identical AlreadyExists. The race itself is +// executed at the primitive by v1_interpreter write_file_create_new_tests +// concurrent_same_target_creates_produce_one_winner_and_no_residue, which also pins every loser to +// the already_exists channel value. + +fn h(v: String) -> Fnv1a64Structural { + content_hash_atom(value: v as NonEmptyStr) +} + +fn bundle_req(module: String) -> ArtifactRequest { + NativeModuleVerdictBundleRequest(NativeModuleVerdictBundleSubject { + route_binary_identity: h(v: "route-binary-1"), + module_identity: h(v: module), + canonical_module_import_closure_identity: h(v: "closure-1"), + ordered_test_identity_population: h(v: "t1,t2"), + eval_interpretation_identity: h(v: "eval-1") + }) +} + +fn universe_req() -> ArtifactRequest { + UniverseArtifactRequest { + tested_tree_identity: h(v: "tree-1"), + route_binary: DeclaredInput { ref: "route_binary_identity", digest: h(v: "route-binary-1") } + } +} + +fn verdicts(text: String) -> List { + [ + ArtifactPayload { id: output_module_test_verdicts, content: text }, + ArtifactPayload { id: output_module_preparation_diagnostics, content: "" }, + ] +} + +fn universe_payloads() -> List { + [ + ArtifactPayload { id: output_universe_test_identity_population, content: "v2.test.alpha.t1" }, + ArtifactPayload { id: output_universe_module_import_closures, content: "v2.test.alpha: v2.std.logic" }, + ] +} + +fn prepared_object(req: ArtifactRequest, payloads: List) -> String { + match store_commit_prepare(req: req, payloads: payloads) { + StoreCommitPrepared { key: _, object_name: _, object: o, offered_digest: _ } => o + StoreCommitNotPrepared { commit: _ } => "" + } +} + +fn object_path(root_path: String, req: ArtifactRequest) -> String { + store_path(root_path: root_path, name: store_object_name(key: request_key(req: req))) +} + +fn commit_tag(c: StoreCommit) -> String { + store_commit_refusal_tag(c: c) +} + +func fresh_root_path() -> String { + shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_matstore.XXXXXX").path +} + +func opened(root: MaterializationStoreRoot) -> OpenedMaterializationStore? { + match local_store_initialize(root: root, instance_label: "wet-witness") { + LocalStoreInitialized { opening: o } => match o { + LocalStoreOpened { root: _, store: s, durability: _ } => Present { value: s } + LocalStoreUnavailable { root_path: _, cause: _ } => none + } + LocalStoreInitializationRefused { root_path: _, cause: _ } => none + } +} + +// commit + exact read-back; a fresh read of the committed name serves the same bytes. +test fn commit_then_read_back_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let committed = commit_tag(c: local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass\na.t2 fail"))) + let read = local_store_lookup(root: root, store: store, req: bundle_req(module: "a")) + let bytes_ok = match store_lookup_payload(l: read, id: output_module_test_verdicts) { + Present { value: v } => v == "a.t1 pass\na.t2 fail" + Absent => false + } + committed == "committed" && store_lookup_is_hit(l: read) && bytes_ok + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// corruption -> integrity refusal, and the refusal is not a miss (nothing may recompute past it). +test fn corruption_is_an_integrity_refusal_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let committed = commit_tag(c: local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass"))) + let good = Filesystem.Read(path: object_path(root_path: root_path, req: bundle_req(module: "a"))) + let corrupt = Filesystem.Write(path: object_path(root_path: root_path, req: bundle_req(module: "a")), content: replace(good.content, "a.t1%20pass", "a.t1%20PASS")) + let read = local_store_lookup(root: root, store: store, req: bundle_req(module: "a")) + committed == "committed" && good.success && corrupt.success + && store_lookup_refusal_tag(l: read) == "contract_wrong_content" + && store_lookup_is_miss(l: read) == false + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// wrong kind: a universe artifact's object planted at the verdict bundle's name. +test fn wrong_kind_refuses_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let planted = Filesystem.WriteCreateNew(path: object_path(root_path: root_path, req: bundle_req(module: "a")), content: prepared_object(req: universe_req(), payloads: universe_payloads())) + let read = local_store_lookup(root: root, store: store, req: bundle_req(module: "a")) + planted.success && store_lookup_refusal_tag(l: read) == "contract_kind_mismatch" + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// incomplete: the stored manifest omits a part the kind requires. +// drop the diagnostics part line and its payload line +test fn incomplete_refuses_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let full = prepared_object(req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass")) + let lines = split(s: full, delimiter: "\n") + let kept = lines |> filter(l => (string_contains(s: l, pattern: output_module_preparation_diagnostics as String)) == false) + let planted = Filesystem.WriteCreateNew(path: object_path(root_path: root_path, req: bundle_req(module: "a")), content: join(kept, "\n")) + let read = local_store_lookup(root: root, store: store, req: bundle_req(module: "a")) + planted.success && (kept |> count()) == (lines |> count()) - 2 + && store_lookup_refusal_tag(l: read) == "contract_incomplete" + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// concurrent writers of one key: the second converges on the first's commit, and every reader sees +// the one verified content. +test fn a_second_writer_converges_on_the_committed_artifact_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let first = local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass")) + let second = local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass")) + let same_content = match first { + StoreCommitSettled { receipt: r1 } => match second { + StoreCommitSettled { receipt: r2 } => r1.content_digest == r2.content_digest && r1.request_key == r2.request_key + StoreCommitRefused { key: _, refusal: _ } => false + } + StoreCommitRefused { key: _, refusal: _ } => false + } + commit_tag(c: first) == "committed" && commit_tag(c: second) == "converged_on_prior_commit" && same_content + && store_lookup_is_hit(l: local_store_lookup(root: root, store: store, req: bundle_req(module: "a"))) + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// a second writer with DIFFERENT content for the same key is a conflict; the committed artifact is +// untouched and is what every reader gets. +test fn a_divergent_second_writer_conflicts_and_changes_nothing_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let first = local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass")) + let second = local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 fail")) + let read = local_store_lookup(root: root, store: store, req: bundle_req(module: "a")) + let still_first = match store_lookup_payload(l: read, id: output_module_test_verdicts) { + Present { value: v } => v == "a.t1 pass" + Absent => false + } + commit_tag(c: first) == "committed" && commit_tag(c: second) == "conflict" && still_first + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + +// provider unavailable: an uninitialized root is not a store. Open, batch and lookup all refuse, +// nothing is answered as a miss, and nothing is written anywhere to stand in for it. +test fn an_unavailable_store_refuses_and_leaves_no_shadow_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let open_refused = match local_store_open(root: root) { + LocalStoreUnavailable { root_path: _, cause: c } => match c { + LocalStoreNotInitialized { marker_path: _ } => true + _ => false + } + LocalStoreOpened { root: _, store: _, durability: _ } => false + } + let batch_refused = match local_store_lookup_batch(root: root, reqs: [bundle_req(module: "a"), bundle_req(module: "b")]) { + LocalStoreBatchUnavailable { root_path: _, cause: _ } => true + LocalStoreBatchLooked { store: _, lookups: _ } => false + } + let ungranted = match local_store_open(root: WitnessScratchRoot { root: "/tmp/not_a_gunbc_store_root" }) { + LocalStoreUnavailable { root_path: _, cause: c } => match c { + LocalStoreGrantRefused { verb: _, target: _, frame: _ } => true + _ => false + } + LocalStoreOpened { root: _, store: _, durability: _ } => false + } + let no_marker = shell.Test.IsFile(path: store_path(root_path: root_path, name: materialization_store_marker_name)) + let no_object = shell.Test.IsFile(path: object_path(root_path: root_path, req: bundle_req(module: "a"))) + let removed = shell.Remove.RecursiveForce(path: root_path) + open_refused && batch_refused && ungranted && !no_marker.is_file && !no_object.is_file && removed.success +} + +// batch: one open, one exact read per request -- a committed request hits, an uncommitted one misses. +test fn a_batch_answers_each_request_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => { + let committed = commit_tag(c: local_store_commit(root: root, store: store, req: bundle_req(module: "a"), payloads: verdicts(text: "a.t1 pass"))) + match local_store_lookup_batch(root: root, reqs: [bundle_req(module: "a"), bundle_req(module: "b")]) { + LocalStoreBatchUnavailable { root_path: _, cause: _ } => false + LocalStoreBatchLooked { store: _, lookups: ls } => + committed == "committed" && (ls |> count()) == 2 + && match get(xs: ls, index: 0) { Present { value: l } => store_lookup_is_hit(l: l), Absent => false } + && match get(xs: ls, index: 1) { Present { value: l } => store_lookup_is_miss(l: l), Absent => false } + } + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} diff --git a/dag/test/claim/materialization_store_witness_test.dag b/dag/test/claim/materialization_store_witness_test.dag new file mode 100644 index 00000000000..94041c2e029 --- /dev/null +++ b/dag/test/claim/materialization_store_witness_test.dag @@ -0,0 +1,346 @@ +module test.claim.materialization_store_witness + +import std.types { Bool, Int, List, String, NonEmptyStr } +import std.content_hash { ContentHash, Fnv1a64Structural, content_hash_atom, content_hash_of_value } +import std.cache_identity { CacheInterfaceId } +import std.materialization_provider { + DeclaredInput, ArtifactRequest, ArtifactManifest, + NativeCompilerArtifactRequest, UniverseArtifactRequest, NativeModuleVerdictBundleRequest, + NativeModuleVerdictBundleSubject, + RealizationServed, RealizationCommitted, RealizationConvergedOnPriorCommit, + request_key, request_kind, request_required_outputs, + output_module_test_verdicts, output_module_preparation_diagnostics, + output_universe_test_identity_population, output_universe_module_import_closures, + output_native_compiler_executable, +} +import std.materialization_store_grant { WitnessScratchRoot, DurableHostVolumeRoot } +import std.materialization_object { + ArtifactPayload, payload_part, + OpenedMaterializationStore, + StoreFault, StoreFaultUnreachable, + StoreObjectPresent, StoreObjectAbsent, StoreObjectUnavailable, + StoreObjectPublished, StoreObjectOccupied, StoreObjectPublishRefused, + StoreLookup, StoreLookupHit, StoreLookupMiss, StoreLookupRefused, + StoreCommitPreparation, StoreCommitPrepared, StoreCommitNotPrepared, StoreCommitSettled, StoreCommitRefused, + StoreObjectPublish, + store_commit_prepare, store_commit_settle, store_lookup_decide, + store_lookup_is_hit, store_lookup_is_miss, store_lookup_payload, store_lookup_refusal_tag, + store_commit_refusal_tag, store_lookup_fold, + encode_store_object, percent_encode_payload, percent_decode_payload, +} +import extdeps.filesystem.filesystem_io { + FilesystemExactPathAbsent, FilesystemExactPathUnreadable, FilesystemExactPathKindUnrecognized, + FilesystemExactPathRead, FilesystemCreateTargetOccupied, FilesystemCreateRefused, + filesystem_exact_read, filesystem_create_new, +} +import extdeps.realization.materialization_store_local { + LocalStoreOpened, LocalStoreUnavailable, + LocalStoreNotInitialized, LocalStoreMarkerUnrecognized, LocalStoreMarkerUnavailable, + local_store_open_decide, store_marker_content, materialization_store_local_id, +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// HERMETIC CONTROLS FOR THE PROVIDER SPINE: the native route's three request families, and the +// store semantics of std.materialization_object driven through its OBSERVATION coproducts, so every +// lookup and commit arm executes on the floor with no filesystem. The same decisions run against a +// real filesystem in test.claim.materialization_store_local_wet_witness; the two files are the +// decision's evidence and the transport's evidence, and neither stands in for the other. +// +// Each RED direction is paired with the positive control that makes it discriminating: a corrupted, +// wrong-kind, wrong-artifact, incomplete, truncated or unavailable object refuses, and the SAME +// request over the uncorrupted object hits -- so a decision that refused everything would fail the +// hit control, and one that served everything would fail every refusal. + +fn h(v: String) -> Fnv1a64Structural { + content_hash_atom(value: v as NonEmptyStr) +} + +fn subject(module: String) -> NativeModuleVerdictBundleSubject { + NativeModuleVerdictBundleSubject { + route_binary_identity: h(v: "route-binary-1"), + module_identity: h(v: module), + canonical_module_import_closure_identity: h(v: "closure-1"), + ordered_test_identity_population: h(v: "t1,t2"), + eval_interpretation_identity: h(v: "eval-1") + } +} + +fn bundle_req() -> ArtifactRequest { + NativeModuleVerdictBundleRequest(subject(module: "v2.test.alpha")) +} + +fn universe_req() -> ArtifactRequest { + UniverseArtifactRequest { + tested_tree_identity: h(v: "tree-1"), + route_binary: DeclaredInput { ref: "route_binary_identity", digest: h(v: "route-binary-1") } + } +} + +fn compiler_req() -> ArtifactRequest { + NativeCompilerArtifactRequest { + emitted_closure_identity: h(v: "tree-1"), + build_toolchain: DeclaredInput { ref: "build_toolchain_identity", digest: h(v: "route-binary-1") }, + target_platform: DeclaredInput { ref: "target_platform_identity", digest: h(v: "x86_64-unknown-linux-gnu") } + } +} + +fn bundle_payloads() -> List { + [ + ArtifactPayload { id: output_module_test_verdicts, content: "v2.test.alpha.t1 pass\nv2.test.alpha.t2 fail 100% sure" }, + ArtifactPayload { id: output_module_preparation_diagnostics, content: "" }, + ] +} + +fn universe_payloads() -> List { + [ + ArtifactPayload { id: output_universe_test_identity_population, content: "v2.test.alpha.t1\nv2.test.alpha.t2" }, + ArtifactPayload { id: output_universe_module_import_closures, content: "v2.test.alpha: v2.std.logic" }, + ] +} + +fn witness_store() -> OpenedMaterializationStore { + OpenedMaterializationStore { + provider: materialization_store_local_id, + instance: content_hash_of_value(value: "witness-store-instance-1") + } +} + +fn prepared_object(req: ArtifactRequest, payloads: List) -> String { + match store_commit_prepare(req: req, payloads: payloads) { + StoreCommitPrepared { key: _, object_name: _, object: o, offered_digest: _ } => o + StoreCommitNotPrepared { commit: _ } => "" + } +} + +fn lookup_present(req: ArtifactRequest, object: String) -> StoreLookup { + store_lookup_decide(store: witness_store(), req: req, read: StoreObjectPresent { content: object }) +} + +// ---- request families + +test fn every_verdict_bundle_identity_is_load_bearing_on_the_key() -> Bool { + let base = subject(module: "v2.test.alpha") + let k = request_key(req: NativeModuleVerdictBundleRequest(base)) + let variants = [ + NativeModuleVerdictBundleSubject { route_binary_identity: h(v: "route-binary-2"), module_identity: base.module_identity, canonical_module_import_closure_identity: base.canonical_module_import_closure_identity, ordered_test_identity_population: base.ordered_test_identity_population, eval_interpretation_identity: base.eval_interpretation_identity }, + NativeModuleVerdictBundleSubject { route_binary_identity: base.route_binary_identity, module_identity: h(v: "v2.test.beta"), canonical_module_import_closure_identity: base.canonical_module_import_closure_identity, ordered_test_identity_population: base.ordered_test_identity_population, eval_interpretation_identity: base.eval_interpretation_identity }, + NativeModuleVerdictBundleSubject { route_binary_identity: base.route_binary_identity, module_identity: base.module_identity, canonical_module_import_closure_identity: h(v: "closure-2"), ordered_test_identity_population: base.ordered_test_identity_population, eval_interpretation_identity: base.eval_interpretation_identity }, + NativeModuleVerdictBundleSubject { route_binary_identity: base.route_binary_identity, module_identity: base.module_identity, canonical_module_import_closure_identity: base.canonical_module_import_closure_identity, ordered_test_identity_population: h(v: "t2,t1"), eval_interpretation_identity: base.eval_interpretation_identity }, + NativeModuleVerdictBundleSubject { route_binary_identity: base.route_binary_identity, module_identity: base.module_identity, canonical_module_import_closure_identity: base.canonical_module_import_closure_identity, ordered_test_identity_population: base.ordered_test_identity_population, eval_interpretation_identity: h(v: "eval-2") }, + ] + let keys = variants |> map(s => request_key(req: NativeModuleVerdictBundleRequest(s))) + let same_again = request_key(req: NativeModuleVerdictBundleRequest(subject(module: "v2.test.alpha"))) == k + let all_apart = (keys |> any(x => x == k)) == false + let pairwise_apart = (keys |> any(x => (keys |> filter(y => y == x) |> count()) > 1)) == false + same_again && all_apart && pairwise_apart && (variants |> count()) == 5 +} + +// universe_req and compiler_req carry the SAME digests in their subject and first input; the +// kind inside the key is what keeps them apart. +test fn native_families_key_apart_over_identical_digests() -> Bool { + (request_key(req: universe_req()) == request_key(req: compiler_req())) == false + && (request_kind(req: universe_req()) == request_kind(req: compiler_req())) == false + && request_required_outputs(req: compiler_req()) |> any(o => o == output_native_compiler_executable) +} + +// ---- lookup: the positive control and each refusal + +test fn a_committed_object_reads_back_as_a_hit_with_its_payloads() -> Bool { + let l = lookup_present(req: bundle_req(), object: prepared_object(req: bundle_req(), payloads: bundle_payloads())) + let payload_ok = match store_lookup_payload(l: l, id: output_module_test_verdicts) { + Present { value: v } => v == "v2.test.alpha.t1 pass\nv2.test.alpha.t2 fail 100% sure" + Absent => false + } + let empty_ok = match store_lookup_payload(l: l, id: output_module_preparation_diagnostics) { + Present { value: v } => v == "" + Absent => false + } + let receipt_ok = store_lookup_fold( + l: l, + hit: (a, ps, r) => r.request_key == request_key(req: bundle_req()) + && r.provider == materialization_store_local_id + && match r.disposition { RealizationServed => true, RealizationCommitted => false, RealizationConvergedOnPriorCommit => false }, + miss: k => false, + refused: (k, r) => false + ) + store_lookup_is_hit(l: l) && payload_ok && empty_ok && receipt_ok +} + +test fn an_absent_object_is_the_only_miss() -> Bool { + store_lookup_is_miss(l: store_lookup_decide(store: witness_store(), req: bundle_req(), read: StoreObjectAbsent)) +} + +test fn an_unavailable_object_refuses_and_is_not_a_miss() -> Bool { + let l = store_lookup_decide( + store: witness_store(), + req: bundle_req(), + read: StoreObjectUnavailable { fault: StoreFault { class: StoreFaultUnreachable, detail: "I/O error" } } + ) + store_lookup_refusal_tag(l: l) == "unavailable" && store_lookup_is_miss(l: l) == false +} + +test fn a_corrupted_payload_is_an_integrity_refusal() -> Bool { + let good = prepared_object(req: bundle_req(), payloads: bundle_payloads()) + let flipped = replace(good, "t1%20pass", "t1%20PASS") + (flipped == good) == false + && store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: flipped)) == "contract_wrong_content" + && store_lookup_is_hit(l: lookup_present(req: bundle_req(), object: good)) +} + +// the empty diagnostics part is recorded with size 0; claim 1 instead +test fn a_tampered_part_size_is_an_integrity_refusal() -> Bool { + let good = prepared_object(req: bundle_req(), payloads: bundle_payloads()) + let parts_line = join(["part", output_module_preparation_diagnostics as String, payload_part(payload: ArtifactPayload { id: output_module_preparation_diagnostics, content: "" }).digest.digest as String], " ") + let tampered = replace(good, concat(parts_line, " 0"), concat(parts_line, " 1")) + (tampered == good) == false + && store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: tampered)) == "contract_wrong_content" +} + +test fn a_truncated_object_is_malformed_not_incomplete() -> Bool { + let good = prepared_object(req: bundle_req(), payloads: bundle_payloads()) + let torn = replace(good, "\nend", "") + store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: torn)) == "malformed" +} + +// a universe artifact's object read where the verdict bundle's object belongs +test fn an_object_of_another_kind_refuses_as_kind_mismatch() -> Bool { + let foreign = prepared_object(req: universe_req(), payloads: universe_payloads()) + store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: foreign)) == "contract_kind_mismatch" +} + +test fn an_object_for_another_request_of_the_same_kind_refuses_as_wrong_artifact() -> Bool { + let other = prepared_object(req: NativeModuleVerdictBundleRequest(subject(module: "v2.test.beta")), payloads: bundle_payloads()) + store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: other)) == "contract_wrong_artifact" +} + +test fn an_object_missing_a_required_part_refuses_as_incomplete() -> Bool { + let verdicts = ArtifactPayload { id: output_module_test_verdicts, content: "v2.test.alpha.t1 pass" } + let incomplete = encode_store_object( + manifest: ArtifactManifest { + kind: request_kind(req: bundle_req()), + request_key: request_key(req: bundle_req()), + parts: [payload_part(payload: verdicts)] + }, + payloads: [verdicts] + ) + store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: incomplete)) == "contract_incomplete" +} + +// ---- commit: the door, and each settle arm + +test fn the_write_door_refuses_before_publishing() -> Bool { + let missing = store_commit_prepare(req: bundle_req(), payloads: [ArtifactPayload { id: output_module_test_verdicts, content: "x" }]) + let undeclared = store_commit_prepare(req: bundle_req(), payloads: concat(bundle_payloads(), [ArtifactPayload { id: output_native_compiler_executable, content: "x" }])) + let repeated = store_commit_prepare(req: bundle_req(), payloads: concat(bundle_payloads(), [ArtifactPayload { id: output_module_test_verdicts, content: "y" }])) + preparation_tag(p: missing) == "contract_refused" && preparation_tag(p: undeclared) == "manifest_uninterpretable" + && preparation_tag(p: repeated) == "manifest_uninterpretable" + && preparation_tag(p: store_commit_prepare(req: bundle_req(), payloads: bundle_payloads())) == "prepared" +} + +fn preparation_tag(p: StoreCommitPreparation) -> String { + match p { + StoreCommitNotPrepared { commit: c } => store_commit_refusal_tag(c: c) + StoreCommitPrepared { key: _, object_name: _, object: _, offered_digest: _ } => "prepared" + } +} + +fn offered_digest(req: ArtifactRequest, payloads: List) -> ContentHash { + match store_commit_prepare(req: req, payloads: payloads) { + StoreCommitPrepared { key: _, object_name: _, object: _, offered_digest: d } => d + StoreCommitNotPrepared { commit: _ } => request_key(req: req) + } +} + +test fn commit_settles_only_on_a_verified_read_back() -> Bool { + let key = request_key(req: bundle_req()) + let offered = offered_digest(req: bundle_req(), payloads: bundle_payloads()) + let ours = lookup_present(req: bundle_req(), object: prepared_object(req: bundle_req(), payloads: bundle_payloads())) + let theirs = lookup_present(req: bundle_req(), object: prepared_object(req: bundle_req(), payloads: [ + ArtifactPayload { id: output_module_test_verdicts, content: "v2.test.alpha.t1 fail" }, + ArtifactPayload { id: output_module_preparation_diagnostics, content: "" }, + ])) + settle_tag(key: key, offered: offered, publish: StoreObjectPublished, read_back: ours) == "committed" + && settle_tag(key: key, offered: offered, publish: StoreObjectOccupied, read_back: ours) == "converged_on_prior_commit" + && settle_tag(key: key, offered: offered, publish: StoreObjectOccupied, read_back: theirs) == "conflict" + && settle_tag(key: key, offered: offered, publish: StoreObjectPublished, read_back: StoreLookupMiss { key: key }) == "read_back_refused" + && settle_tag(key: key, offered: offered, publish: StoreObjectOccupied, read_back: lookup_present(req: bundle_req(), object: "garbage")) == "read_back_refused" + && settle_tag(key: key, offered: offered, publish: StoreObjectPublishRefused { fault: StoreFault { class: StoreFaultUnreachable, detail: "ENOSPC" } }, read_back: ours) == "publish_refused" +} + +fn settle_tag(key: ContentHash, offered: ContentHash, publish: StoreObjectPublish, read_back: StoreLookup) -> String { + store_commit_refusal_tag(c: store_commit_settle( + store: witness_store(), prepared_key: key, offered_digest: offered, publish: publish, read_back: read_back)) +} + +test fn provider_identity_is_in_the_receipt_and_not_in_the_key() -> Bool { + let object = prepared_object(req: bundle_req(), payloads: bundle_payloads()) + let other_store = OpenedMaterializationStore { provider: "some_other_realization" as CacheInterfaceId, instance: content_hash_of_value(value: "other-instance") } + let a = store_lookup_decide(store: witness_store(), req: bundle_req(), read: StoreObjectPresent { content: object }) + let b = store_lookup_decide(store: other_store, req: bundle_req(), read: StoreObjectPresent { content: object }) + match a { + StoreLookupHit { artifact: _, payloads: _, receipt: ra } => + match b { + StoreLookupHit { artifact: _, payloads: _, receipt: rb } => + ra.request_key == rb.request_key && ra.content_digest == rb.content_digest + && (ra.provider == rb.provider) == false && (ra.store_instance == rb.store_instance) == false + StoreLookupMiss { key: _ } => false + StoreLookupRefused { key: _, refusal: _ } => false + } + StoreLookupMiss { key: _ } => false + StoreLookupRefused { key: _, refusal: _ } => false + } +} + +test fn payload_encoding_round_trips_its_own_delimiters() -> Bool { + let s = "a %0A b\n%25 %20\n\nend" + percent_decode_payload(s: percent_encode_payload(s: s)) == s + && string_contains(s: percent_encode_payload(s: s), pattern: "\n") == false + && string_contains(s: percent_encode_payload(s: s), pattern: " ") == false +} + +// ---- the transport folds and the open decision + +test fn the_error_kind_channel_separates_absent_from_unreadable() -> Bool { + let absent = match filesystem_exact_read(path: "/p", content: "", success: false, error: "No such file", error_kind: "not_found") { + FilesystemExactPathAbsent { path: _ } => true + _ => false + } + let unreadable = match filesystem_exact_read(path: "/p", content: "", success: false, error: "denied", error_kind: "permission_denied") { + FilesystemExactPathUnreadable { path: _, kind: _, error: _ } => true + _ => false + } + let unrecognized = match filesystem_exact_read(path: "/p", content: "", success: false, error: "?", error_kind: "") { + FilesystemExactPathKindUnrecognized { path: _, observed: _, error: _ } => true + _ => false + } + let occupied = match filesystem_create_new(path: "/p", success: false, error: "exists", error_kind: "already_exists") { + FilesystemCreateTargetOccupied { path: _ } => true + _ => false + } + let refused = match filesystem_create_new(path: "/p", success: false, error: "denied", error_kind: "permission_denied") { + FilesystemCreateRefused { path: _, kind: _, error: _ } => true + _ => false + } + absent && unreadable && unrecognized && occupied && refused +} + +// a durable store's marker is not an ephemeral store's marker: a scratch root holding it refuses +test fn a_store_opens_only_on_its_own_marker() -> Bool { + let root = WitnessScratchRoot { root: "/tmp/gunbc_matstore_witness" } + let marker = store_marker_content(root: root, instance_label: "witness") + let opened = match local_store_open_decide(root: root, marker_read: FilesystemExactPathRead { path: "/m", content: marker }) { + LocalStoreOpened { root: _, store: s, durability: _ } => s.provider == materialization_store_local_id + LocalStoreUnavailable { root_path: _, cause: _ } => false + } + let uninitialized = match local_store_open_decide(root: root, marker_read: FilesystemExactPathAbsent { path: "/m" }) { + LocalStoreUnavailable { root_path: _, cause: c } => match c { LocalStoreNotInitialized { marker_path: _ } => true, _ => false } + LocalStoreOpened { root: _, store: _, durability: _ } => false + } + let foreign = match local_store_open_decide(root: root, marker_read: FilesystemExactPathRead { path: "/m", content: store_marker_content(root: DurableHostVolumeRoot, instance_label: "witness") }) { + LocalStoreUnavailable { root_path: _, cause: c } => match c { LocalStoreMarkerUnrecognized { marker_path: _ } => true, _ => false } + LocalStoreOpened { root: _, store: _, durability: _ } => false + } + opened && uninitialized && foreign +} diff --git a/src/v1/05_emit.dag b/src/v1/05_emit.dag index 132bbe69906..23491599c58 100644 --- a/src/v1/05_emit.dag +++ b/src/v1/05_emit.dag @@ -1160,6 +1160,7 @@ type FileResultChannel | FileChanByteCount | FileChanPath | FileChanError + | FileChanErrorKind | FileChanContent type FileResultField { @@ -1203,6 +1204,7 @@ fn file_result_channel_of_key(key: String) -> FileResultChannel? { else if key == "bytes_written" || key == "bytes" || key == "byte_count" { Present { value: FileChanByteCount } } else if key == "path" { Present { value: FileChanPath } } else if key == "error" { Present { value: FileChanError } } + else if key == "error_kind" { Present { value: FileChanErrorKind } } else if key == "content" || key == "entries" { Present { value: FileChanContent } } else { none } } diff --git a/src/v1/05_emit_rust.dag b/src/v1/05_emit_rust.dag index b02d6680dc7..f396b905031 100644 --- a/src/v1/05_emit_rust.dag +++ b/src/v1/05_emit_rust.dag @@ -244,7 +244,7 @@ import v1.compiler.emit { bind_operation_transport, transport_binding_refusal_fact, FileVerb, FileRead, FileWrite, FileWriteOwnerOnly, FileWriteCreateNew, FileDelete, FileList, FileResultField, FileResultChannel, FileChanSuccess, FileChanByteCount, - FileChanPath, FileChanError, FileChanContent + FileChanPath, FileChanError, FileChanErrorKind, FileChanContent } import v1.compiler.languages { wrap_shared_type, @@ -15239,7 +15239,7 @@ fn emit_file_call(verb: FileVerb, path_template: Node, result_fields: List join(separator: "\n") + [path_line, file_empty_path_guard, file_io_error_kind_fn, action, return_line] |> join(separator: "\n") } // One expression per verb, selected on the CLOSED coproduct the binding resolved. @@ -15300,15 +15300,22 @@ fn emit_file_path_line(path_template: Node, source_indices: Map \"already_exists\",\n std::io::ErrorKind::PermissionDenied => \"permission_denied\",\n _ => \"other\",\n }\n .to_string()\n};" -data file_read_match_expr: String = "match std::fs::read_to_string(&file_path) {\n Ok(read_text) => { let read_bytes = read_text.len() as i64; (true, read_text, String::new(), read_bytes) }\n Err(read_err) => (false, String::new(), format!(\"{}\", read_err), 0i64),\n};" +data file_channel_binding_prefix: String = "#[allow(unused_variables)]\nlet (file_success, file_content, file_error, file_byte_count, file_error_kind): (bool, String, String, i64, String) = " -data file_delete_match_expr: String = "match std::fs::remove_file(&file_path) {\n Ok(()) => (true, String::new(), String::new(), 0i64),\n Err(delete_err) => (false, String::new(), format!(\"{}\", delete_err), 0i64),\n};" +data file_read_match_expr: String = "match std::fs::read_to_string(&file_path) {\n Ok(read_text) => { let read_bytes = read_text.len() as i64; (true, read_text, String::new(), read_bytes, String::new()) }\n Err(read_err) => (false, String::new(), format!(\"{}\", read_err), 0i64, file_io_error_kind(&read_err)),\n};" -data file_list_match_expr: String = "match std::fs::read_dir(&file_path) {\n Ok(dir_entries) => {\n let mut entry_names: Vec = Vec::new();\n let mut listing_refusal: Option = None;\n for dir_entry in dir_entries {\n let dir_entry = match dir_entry {\n Ok(admitted) => admitted,\n Err(advance_err) => {\n listing_refusal = Some(format!(\"directory enumeration failed after {} entr(ies) were read, so this listing accounts for no known portion of the directory and establishes no absence: {}\", entry_names.len(), advance_err));\n break;\n }\n };\n let entry_name = match dir_entry.file_name().into_string() {\n Ok(admitted) => admitted,\n Err(native_name) => {\n listing_refusal = Some(format!(\"directory entry name {:?} is not valid UTF-8, and this listing encoding carries Unicode names only, so the entry cannot be represented faithfully. The rendering shown is a diagnostic and is not the operational pathname\", native_name));\n break;\n }\n };\n if entry_name.contains('\\n') {\n listing_refusal = Some(format!(\"directory entry name {:?} contains a line feed, which is the delimiter this listing encoding joins entries with, so one entry would be indistinguishable from two and a membership test could answer yes for an entry that is not there\", entry_name));\n break;\n }\n entry_names.push(entry_name);\n }\n match listing_refusal {\n Some(refused) => (false, String::new(), refused, 0i64),\n None => {\n entry_names.sort();\n let listing = entry_names.join(\"\\n\");\n let listing_bytes = listing.len() as i64;\n (true, listing, String::new(), listing_bytes)\n }\n }\n }\n Err(list_err) => (false, String::new(), format!(\"{}\", list_err), 0i64),\n};" +data file_delete_match_expr: String = "match std::fs::remove_file(&file_path) {\n Ok(()) => (true, String::new(), String::new(), 0i64, String::new()),\n Err(delete_err) => (false, String::new(), format!(\"{}\", delete_err), 0i64, file_io_error_kind(&delete_err)),\n};" -data file_write_expr: String = "{\n let payload_bytes = content.len() as i64;\n match std::fs::write(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(write_err) => (false, String::new(), format!(\"{}\", write_err), 0i64),\n }\n};" +data file_list_match_expr: String = "match std::fs::read_dir(&file_path) {\n Ok(dir_entries) => {\n let mut entry_names: Vec = Vec::new();\n let mut listing_refusal: Option = None;\n for dir_entry in dir_entries {\n let dir_entry = match dir_entry {\n Ok(admitted) => admitted,\n Err(advance_err) => {\n listing_refusal = Some(format!(\"directory enumeration failed after {} entr(ies) were read, so this listing accounts for no known portion of the directory and establishes no absence: {}\", entry_names.len(), advance_err));\n break;\n }\n };\n let entry_name = match dir_entry.file_name().into_string() {\n Ok(admitted) => admitted,\n Err(native_name) => {\n listing_refusal = Some(format!(\"directory entry name {:?} is not valid UTF-8, and this listing encoding carries Unicode names only, so the entry cannot be represented faithfully. The rendering shown is a diagnostic and is not the operational pathname\", native_name));\n break;\n }\n };\n if entry_name.contains('\\n') {\n listing_refusal = Some(format!(\"directory entry name {:?} contains a line feed, which is the delimiter this listing encoding joins entries with, so one entry would be indistinguishable from two and a membership test could answer yes for an entry that is not there\", entry_name));\n break;\n }\n entry_names.push(entry_name);\n }\n match listing_refusal {\n Some(refused) => (false, String::new(), refused, 0i64, \"other\".to_string()),\n None => {\n entry_names.sort();\n let listing = entry_names.join(\"\\n\");\n let listing_bytes = listing.len() as i64;\n (true, listing, String::new(), listing_bytes, String::new())\n }\n }\n }\n Err(list_err) => (false, String::new(), format!(\"{}\", list_err), 0i64, file_io_error_kind(&list_err)),\n};" + +data file_write_expr: String = "{\n let payload_bytes = content.len() as i64;\n match std::fs::write(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(write_err) => (false, String::new(), format!(\"{}\", write_err), 0i64, file_io_error_kind(&write_err)),\n }\n};" // THE EMITTED REALIZATION PUBLISHES ONLY COMPLETE CONTENT, exactly as the interpreter helper does, // and this row exists because review 5089156132 on gunbc#10069 found the two had DIVERGED. The @@ -15339,9 +15346,9 @@ data file_write_expr: String = "{\n let payload_bytes = content.len() as i64; // policy -- a stringified error and a byte count -- and the realization deliberately does not // return it, because doing so would stringify std::io::Error inside the realization and discard // the host's structured failure before any consumer asked for that loss. -data file_write_create_new_expr: String = "{\n let payload_bytes = content.len() as i64;\n match v1_rt::gunbc_file_write_create_new(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(create_new_err) => (false, String::new(), format!(\"{}\", create_new_err), 0i64),\n }\n};" +data file_write_create_new_expr: String = "{\n let payload_bytes = content.len() as i64;\n match v1_rt::gunbc_file_write_create_new(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(create_new_err) => (false, String::new(), format!(\"{}\", create_new_err), 0i64, file_io_error_kind(&create_new_err)),\n }\n};" -data file_write_owner_only_expr: String = "{\n let payload_bytes = content.len() as i64;\n let owner_only_result = (|| -> std::io::Result<()> {\n #[cfg(unix)]\n {\n use std::io::Write;\n use std::os::unix::fs::OpenOptionsExt;\n let mut owner_only_file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&file_path)?;\n owner_only_file.write_all(content.as_bytes())?;\n return Ok(());\n }\n #[cfg(not(unix))]\n {\n return Err(std::io::Error::new(std::io::ErrorKind::Unsupported, \"write_owner_only refused: owner-only mode-at-creation is unavailable on this platform\"));\n }\n })();\n match owner_only_result {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(owner_only_err) => (false, String::new(), format!(\"{}\", owner_only_err), 0i64),\n }\n};" +data file_write_owner_only_expr: String = "{\n let payload_bytes = content.len() as i64;\n let owner_only_result = (|| -> std::io::Result<()> {\n #[cfg(unix)]\n {\n use std::io::Write;\n use std::os::unix::fs::OpenOptionsExt;\n let mut owner_only_file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&file_path)?;\n owner_only_file.write_all(content.as_bytes())?;\n return Ok(());\n }\n #[cfg(not(unix))]\n {\n return Err(std::io::Error::new(std::io::ErrorKind::Unsupported, \"write_owner_only refused: owner-only mode-at-creation is unavailable on this platform\"));\n }\n })();\n match owner_only_result {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(owner_only_err) => (false, String::new(), format!(\"{}\", owner_only_err), 0i64, file_io_error_kind(&owner_only_err)),\n }\n};" // TOTAL BY CONSTRUCTION, and this is where the climb is visible. #8929 took the // channel as a String and needed a final arm for a key that reached the renderer @@ -15361,6 +15368,7 @@ fn emit_file_channel_expr(channel: FileResultChannel, is_optional: Bool) -> Stri FileChanByteCount => "file_byte_count" FileChanPath => "file_path.clone()" FileChanError => "file_error.clone()" + FileChanErrorKind => "file_error_kind.clone()" FileChanContent => "file_content.clone()" } if is_optional { concat("Some(", base, ")") } else { base } diff --git a/src/v1/stage0/src/v1_compiler_emit.rs b/src/v1/stage0/src/v1_compiler_emit.rs index 90ddfd14eea..1cf3340cef7 100644 --- a/src/v1/stage0/src/v1_compiler_emit.rs +++ b/src/v1/stage0/src/v1_compiler_emit.rs @@ -2610,6 +2610,7 @@ pub enum FileResultChannel { FileChanByteCount, FileChanPath, FileChanError, + FileChanErrorKind, FileChanContent, } @@ -2676,12 +2677,16 @@ pub fn file_result_channel_of_key(key: String) -> Option { if (key.clone() == "error".to_string()) { Some(FileResultChannel::FileChanError) } else { - if ((key.clone() == "content".to_string()) - || (key.clone() == "entries".to_string())) - { - Some(FileResultChannel::FileChanContent) + if (key.clone() == "error_kind".to_string()) { + Some(FileResultChannel::FileChanErrorKind) } else { - std::option::Option::None + if ((key.clone() == "content".to_string()) + || (key.clone() == "entries".to_string())) + { + Some(FileResultChannel::FileChanContent) + } else { + std::option::Option::None + } } } } @@ -8305,6 +8310,8 @@ pub struct FileChanPath; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct FileChanError; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct FileChanErrorKind; +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct FileChanContent; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ShellChanStdout; diff --git a/src/v1/stage0/src/v1_compiler_emit_rust.rs b/src/v1/stage0/src/v1_compiler_emit_rust.rs index 50a87e82090..66137867e3c 100644 --- a/src/v1/stage0/src/v1_compiler_emit_rust.rs +++ b/src/v1/stage0/src/v1_compiler_emit_rust.rs @@ -129,7 +129,8 @@ use crate::v1_compiler_emit::BoundOperation::{ }; use crate::v1_compiler_emit::EmitterOutcome::{Emitted, Refused}; use crate::v1_compiler_emit::FileResultChannel::{ - FileChanByteCount, FileChanContent, FileChanError, FileChanPath, FileChanSuccess, + FileChanByteCount, FileChanContent, FileChanError, FileChanErrorKind, FileChanPath, + FileChanSuccess, }; use crate::v1_compiler_emit::FileVerb::{ FileDelete, FileList, FileRead, FileWrite, FileWriteCreateNew, FileWriteOwnerOnly, @@ -36168,6 +36169,7 @@ pub fn emit_file_call( Rc::new(vec![ path_line.clone(), file_empty_path_guard(), + file_io_error_kind_fn(), action.clone(), return_line.clone(), ]) @@ -36319,10 +36321,19 @@ pub fn file_empty_path_guard() -> String { CACHED.with(|c: &String| c.clone()) } +pub fn file_io_error_kind_fn() -> String { + thread_local! { + static CACHED: String = { + "let file_io_error_kind = |host_err: &std::io::Error| -> String {\n match host_err.kind() {\n std::io::ErrorKind::NotFound => \"not_found\",\n std::io::ErrorKind::AlreadyExists => \"already_exists\",\n std::io::ErrorKind::PermissionDenied => \"permission_denied\",\n _ => \"other\",\n }\n .to_string()\n};".to_string() + }; + } + CACHED.with(|c: &String| c.clone()) +} + pub fn file_channel_binding_prefix() -> String { thread_local! { static CACHED: String = { - "#[allow(unused_variables)]\nlet (file_success, file_content, file_error, file_byte_count): (bool, String, String, i64) = ".to_string() + "#[allow(unused_variables)]\nlet (file_success, file_content, file_error, file_byte_count, file_error_kind): (bool, String, String, i64, String) = ".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36331,7 +36342,7 @@ pub fn file_channel_binding_prefix() -> String { pub fn file_read_match_expr() -> String { thread_local! { static CACHED: String = { - "match std::fs::read_to_string(&file_path) {\n Ok(read_text) => { let read_bytes = read_text.len() as i64; (true, read_text, String::new(), read_bytes) }\n Err(read_err) => (false, String::new(), format!(\"{}\", read_err), 0i64),\n};".to_string() + "match std::fs::read_to_string(&file_path) {\n Ok(read_text) => { let read_bytes = read_text.len() as i64; (true, read_text, String::new(), read_bytes, String::new()) }\n Err(read_err) => (false, String::new(), format!(\"{}\", read_err), 0i64, file_io_error_kind(&read_err)),\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36340,7 +36351,7 @@ pub fn file_read_match_expr() -> String { pub fn file_delete_match_expr() -> String { thread_local! { static CACHED: String = { - "match std::fs::remove_file(&file_path) {\n Ok(()) => (true, String::new(), String::new(), 0i64),\n Err(delete_err) => (false, String::new(), format!(\"{}\", delete_err), 0i64),\n};".to_string() + "match std::fs::remove_file(&file_path) {\n Ok(()) => (true, String::new(), String::new(), 0i64, String::new()),\n Err(delete_err) => (false, String::new(), format!(\"{}\", delete_err), 0i64, file_io_error_kind(&delete_err)),\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36349,7 +36360,7 @@ pub fn file_delete_match_expr() -> String { pub fn file_list_match_expr() -> String { thread_local! { static CACHED: String = { - "match std::fs::read_dir(&file_path) {\n Ok(dir_entries) => {\n let mut entry_names: Vec = Vec::new();\n let mut listing_refusal: Option = None;\n for dir_entry in dir_entries {\n let dir_entry = match dir_entry {\n Ok(admitted) => admitted,\n Err(advance_err) => {\n listing_refusal = Some(format!(\"directory enumeration failed after {} entr(ies) were read, so this listing accounts for no known portion of the directory and establishes no absence: {}\", entry_names.len(), advance_err));\n break;\n }\n };\n let entry_name = match dir_entry.file_name().into_string() {\n Ok(admitted) => admitted,\n Err(native_name) => {\n listing_refusal = Some(format!(\"directory entry name {:?} is not valid UTF-8, and this listing encoding carries Unicode names only, so the entry cannot be represented faithfully. The rendering shown is a diagnostic and is not the operational pathname\", native_name));\n break;\n }\n };\n if entry_name.contains('\\n') {\n listing_refusal = Some(format!(\"directory entry name {:?} contains a line feed, which is the delimiter this listing encoding joins entries with, so one entry would be indistinguishable from two and a membership test could answer yes for an entry that is not there\", entry_name));\n break;\n }\n entry_names.push(entry_name);\n }\n match listing_refusal {\n Some(refused) => (false, String::new(), refused, 0i64),\n None => {\n entry_names.sort();\n let listing = entry_names.join(\"\\n\");\n let listing_bytes = listing.len() as i64;\n (true, listing, String::new(), listing_bytes)\n }\n }\n }\n Err(list_err) => (false, String::new(), format!(\"{}\", list_err), 0i64),\n};".to_string() + "match std::fs::read_dir(&file_path) {\n Ok(dir_entries) => {\n let mut entry_names: Vec = Vec::new();\n let mut listing_refusal: Option = None;\n for dir_entry in dir_entries {\n let dir_entry = match dir_entry {\n Ok(admitted) => admitted,\n Err(advance_err) => {\n listing_refusal = Some(format!(\"directory enumeration failed after {} entr(ies) were read, so this listing accounts for no known portion of the directory and establishes no absence: {}\", entry_names.len(), advance_err));\n break;\n }\n };\n let entry_name = match dir_entry.file_name().into_string() {\n Ok(admitted) => admitted,\n Err(native_name) => {\n listing_refusal = Some(format!(\"directory entry name {:?} is not valid UTF-8, and this listing encoding carries Unicode names only, so the entry cannot be represented faithfully. The rendering shown is a diagnostic and is not the operational pathname\", native_name));\n break;\n }\n };\n if entry_name.contains('\\n') {\n listing_refusal = Some(format!(\"directory entry name {:?} contains a line feed, which is the delimiter this listing encoding joins entries with, so one entry would be indistinguishable from two and a membership test could answer yes for an entry that is not there\", entry_name));\n break;\n }\n entry_names.push(entry_name);\n }\n match listing_refusal {\n Some(refused) => (false, String::new(), refused, 0i64, \"other\".to_string()),\n None => {\n entry_names.sort();\n let listing = entry_names.join(\"\\n\");\n let listing_bytes = listing.len() as i64;\n (true, listing, String::new(), listing_bytes, String::new())\n }\n }\n }\n Err(list_err) => (false, String::new(), format!(\"{}\", list_err), 0i64, file_io_error_kind(&list_err)),\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36358,7 +36369,7 @@ pub fn file_list_match_expr() -> String { pub fn file_write_expr() -> String { thread_local! { static CACHED: String = { - "{\n let payload_bytes = content.len() as i64;\n match std::fs::write(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(write_err) => (false, String::new(), format!(\"{}\", write_err), 0i64),\n }\n};".to_string() + "{\n let payload_bytes = content.len() as i64;\n match std::fs::write(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(write_err) => (false, String::new(), format!(\"{}\", write_err), 0i64, file_io_error_kind(&write_err)),\n }\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36367,7 +36378,7 @@ pub fn file_write_expr() -> String { pub fn file_write_create_new_expr() -> String { thread_local! { static CACHED: String = { - "{\n let payload_bytes = content.len() as i64;\n match v1_rt::gunbc_file_write_create_new(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(create_new_err) => (false, String::new(), format!(\"{}\", create_new_err), 0i64),\n }\n};".to_string() + "{\n let payload_bytes = content.len() as i64;\n match v1_rt::gunbc_file_write_create_new(&file_path, content.as_bytes()) {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(create_new_err) => (false, String::new(), format!(\"{}\", create_new_err), 0i64, file_io_error_kind(&create_new_err)),\n }\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36376,7 +36387,7 @@ pub fn file_write_create_new_expr() -> String { pub fn file_write_owner_only_expr() -> String { thread_local! { static CACHED: String = { - "{\n let payload_bytes = content.len() as i64;\n let owner_only_result = (|| -> std::io::Result<()> {\n #[cfg(unix)]\n {\n use std::io::Write;\n use std::os::unix::fs::OpenOptionsExt;\n let mut owner_only_file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&file_path)?;\n owner_only_file.write_all(content.as_bytes())?;\n return Ok(());\n }\n #[cfg(not(unix))]\n {\n return Err(std::io::Error::new(std::io::ErrorKind::Unsupported, \"write_owner_only refused: owner-only mode-at-creation is unavailable on this platform\"));\n }\n })();\n match owner_only_result {\n Ok(()) => (true, String::new(), String::new(), payload_bytes),\n Err(owner_only_err) => (false, String::new(), format!(\"{}\", owner_only_err), 0i64),\n }\n};".to_string() + "{\n let payload_bytes = content.len() as i64;\n let owner_only_result = (|| -> std::io::Result<()> {\n #[cfg(unix)]\n {\n use std::io::Write;\n use std::os::unix::fs::OpenOptionsExt;\n let mut owner_only_file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&file_path)?;\n owner_only_file.write_all(content.as_bytes())?;\n return Ok(());\n }\n #[cfg(not(unix))]\n {\n return Err(std::io::Error::new(std::io::ErrorKind::Unsupported, \"write_owner_only refused: owner-only mode-at-creation is unavailable on this platform\"));\n }\n })();\n match owner_only_result {\n Ok(()) => (true, String::new(), String::new(), payload_bytes, String::new()),\n Err(owner_only_err) => (false, String::new(), format!(\"{}\", owner_only_err), 0i64, file_io_error_kind(&owner_only_err)),\n }\n};".to_string() }; } CACHED.with(|c: &String| c.clone()) @@ -36389,6 +36400,7 @@ pub fn emit_file_channel_expr(channel: FileResultChannel, is_optional: bool) -> FileResultChannel::FileChanByteCount => "file_byte_count".to_string(), FileResultChannel::FileChanPath => "file_path.clone()".to_string(), FileResultChannel::FileChanError => "file_error.clone()".to_string(), + FileResultChannel::FileChanErrorKind => "file_error_kind.clone()".to_string(), FileResultChannel::FileChanContent => "file_content.clone()".to_string(), }; if is_optional.clone() { diff --git a/src/v1/stage0/src/v1_interpreter.rs b/src/v1/stage0/src/v1_interpreter.rs index e1d17559015..2b3e4918e41 100644 --- a/src/v1/stage0/src/v1_interpreter.rs +++ b/src/v1/stage0/src/v1_interpreter.rs @@ -14036,6 +14036,10 @@ mod write_file_create_new_tests { for r in results.iter().filter(|r| r.is_err()) { let e = r.as_ref().unwrap_err(); assert_eq!(e.kind(), std::io::ErrorKind::AlreadyExists); + // Every loser reaches the `error_kind` channel as the occupied arm, which is what lets + // a put-if-absent consumer (std.materialization_object store_commit_settle) read back + // and converge instead of refusing. + assert_eq!(super::io_error_kind_name(e), "already_exists"); } assert!(path.exists(), "the winner's target must be published"); assert_eq!(std::fs::read(&path).expect("target").len(), payload.len()); @@ -14159,9 +14163,26 @@ struct FileResult { byte_count: i64, path: String, error: String, + // The host error's kind, projected onto extdeps.filesystem.filesystem_io FilesystemFailureKind's + // closed names ("" on success). Carried so a consumer can separate NotFound from + // PermissionDenied / AlreadyExists without reading `error`'s text. + error_kind: String, content: String, } +/// The one projection from a host `std::io::Error` onto the `error_kind` channel's closed roster +/// (extdeps.filesystem.filesystem_io admit_filesystem_failure_kind). Every kind outside the three +/// named ones is "other" -- the host still distinguished it, but no consumer has asked for it yet. +fn io_error_kind_name(e: &std::io::Error) -> String { + match e.kind() { + std::io::ErrorKind::NotFound => "not_found", + std::io::ErrorKind::AlreadyExists => "already_exists", + std::io::ErrorKind::PermissionDenied => "permission_denied", + _ => "other", + } + .to_string() +} + fn dispatch_file( op_node: &Rc, transport: &Rc, @@ -14212,6 +14233,7 @@ fn dispatch_file( byte_count: 0, path, error: String::new(), + error_kind: String::new(), content: String::new(), }), Err(e) => Ok(FileResult { @@ -14219,6 +14241,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), }; @@ -14238,6 +14261,7 @@ fn dispatch_file( byte_count: content.len() as i64, path, error: String::new(), + error_kind: String::new(), content, }) } @@ -14246,6 +14270,7 @@ fn dispatch_file( byte_count: 0, path, error, + error_kind: "other".to_string(), content: String::new(), }), }, @@ -14254,6 +14279,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), }; @@ -14281,6 +14307,7 @@ fn dispatch_file( byte_count, path, error: String::new(), + error_kind: String::new(), content: String::new(), }), Err(e) => Ok(FileResult { @@ -14288,6 +14315,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), }; @@ -14315,6 +14343,7 @@ fn dispatch_file( byte_count, path, error: String::new(), + error_kind: String::new(), content: String::new(), }), // The refusal carries the host's message verbatim and does NOT classify itself. @@ -14326,6 +14355,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), }; @@ -14368,6 +14398,7 @@ fn dispatch_file( byte_count, path, error: String::new(), + error_kind: String::new(), content: String::new(), }), Err(e) => Ok(FileResult { @@ -14375,6 +14406,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), } @@ -14389,6 +14421,7 @@ fn dispatch_file( byte_count: s.len() as i64, path, error: String::new(), + error_kind: String::new(), content: s, }), Err(e) => Ok(FileResult { @@ -14396,6 +14429,7 @@ fn dispatch_file( byte_count: 0, path, error: format!("{}", e), + error_kind: io_error_kind_name(&e), content: String::new(), }), } @@ -14434,6 +14468,7 @@ fn map_file_outputs( "bytes_written" | "bytes" | "byte_count" => Value::Int(result.byte_count), "path" => str_value(result.path.clone()), "error" => str_value(result.error.clone()), + "error_kind" => str_value(result.error_kind.clone()), "content" | "entries" => str_value(result.content.clone()), _ => Value::Null, }; @@ -22828,7 +22863,9 @@ mod the_emitted_listing_producer_refuses_too { &src, format!( "fn main() {{\n let file_path = std::env::args().nth(1).unwrap();\n \ - let (file_success, file_content, file_error, file_byte_count): (bool, String, String, i64) = {}\n{}\n}}\n", + {}\n \ + let (file_success, file_content, file_error, file_byte_count, file_error_kind): (bool, String, String, i64, String) = {}\n{}\n}}\n", + crate::v1_compiler_emit_rust::file_io_error_kind_fn(), crate::v1_compiler_emit_rust::file_list_match_expr(), expectations ), diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 1ee1782dc10..7c8e0ae8526 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -909,8 +909,44 @@ fn floor_route_gap_expectation_chunk_07() -> List { } } +// The eight test.claim.materialization_store_local_wet_witness identities: the provider spine's +// executed controls against a real filesystem (commit and read-back, corruption, wrong kind, +// incomplete, a converging and a conflicting second writer, an unavailable store, a batch). Their +// first effect is shell.Mktemp.DirWithTemplate, which the hermetic envelope does not mock. Enrolled +// as the triple chunk_06's header requires: this expectation, the WetScheduledClaim rows in +// v2.workflow.local_repo_wet_terminal local_repo_wet_schedule, and the gunbc.ci_layer_roots +// witness_exclusion_frontier row classifying the file LocalRepoWetLane. Each was executed wet and +// held before these rows were written. +fn floor_route_gap_expectation_chunk_08() -> List { + Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.commit_then_read_back_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.corruption_is_an_integrity_refusal_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.wrong_kind_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.incomplete_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_second_writer_converges_on_the_committed_artifact_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_divergent_second_writer_conflicts_and_changes_nothing_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.an_unavailable_store_refuses_and_leaves_no_shadow_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_batch_answers_each_request_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Empty {} + } + } + } + } + } + } + } + } +} + fn floor_route_gap_expectation_chunks() -> List> { - Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Empty {} } } } } } } } } + Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Cons { head: floor_route_gap_expectation_chunk_08(), tail: Empty {} } } } } } } } } } } fn floor_route_gap_expectations() -> List { diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 4aa6ce5d903..e03669098f7 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -146,6 +146,16 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // and it holds. Had the substrate short-circuited, that fn would be ExpectedToRefuse and the // disagreement among the five would itself have been the finding. // +// EIGHT MATERIALIZATION-STORE MEMBERS (added 2026-09-11, the provider-spine lane). Their admitted +// effect is the evaluation-model probes' effect with one widening named here rather than folded +// in: shell.Mktemp.DirWithTemplate (a /tmp/gunbc_matstore.* throwaway directory, because the +// store grant admits only /tmp/gunbc_* scratch roots), Filesystem.Read, Filesystem.WriteCreateNew +// and Filesystem.Write INSIDE that directory, shell.Test.IsFile, and shell.Remove.RecursiveForce +// of the directory before returning. WriteCreateNew is the new admission: it stages, fsyncs and +// hard-links within the same directory. No network, no cargo, no remote host, no install media, +// nothing outside the directory the fn created. Each was executed wet through `gunbc run` over +// this tree and held before the rows were written. +// // EACH WAS MEASURED WET BEFORE BEING SCHEDULED, not assumed runnable from its exclusion note. A // member the executor cannot actually run would make this lane's route claim false, which is the // failure the ruling that ordered this lane forbids by name. The four contract-epoch members were @@ -435,6 +445,54 @@ fn local_repo_wet_schedule() -> List { entry: "dag/test/claim/eval_model_probe_test.dag", function: "and_with_true_left_removes_file", expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "commit_then_read_back_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "commit_then_read_back_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "corruption_is_an_integrity_refusal_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "corruption_is_an_integrity_refusal_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "wrong_kind_refuses_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "wrong_kind_refuses_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "incomplete_refuses_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "incomplete_refuses_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "a_second_writer_converges_on_the_committed_artifact_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "a_second_writer_converges_on_the_committed_artifact_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "a_divergent_second_writer_conflicts_and_changes_nothing_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "a_divergent_second_writer_conflicts_and_changes_nothing_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "an_unavailable_store_refuses_and_leaves_no_shadow_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "an_unavailable_store_refuses_and_leaves_no_shadow_by_real_execution", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "a_batch_answers_each_request_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "a_batch_answers_each_request_by_real_execution", + expectation: ExpectedToHold {} } ] } From 837cf7856a132b6863426da7df29a734b67f352b Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 14:36:35 +0000 Subject: [PATCH 02/10] Type the GCS interface frontier; fix one misindented line (review 63841) extdeps.cloud.gcp.storage gains gcs_interface_consumer_frontier: a std.roster_frontier FrontierRow naming the consumer (extdeps.realization.materialization_store_gcs gcs_store_read_observation) as a bound dissolution, per the operator's 2026-09-11 provider ruling (interface rows day one; handler at ruled dispatch step 5). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/extdeps/cloud/gcp/storage.dag | 27 +++++++++++++++++++++++---- src/v1/stage0/src/v1_interpreter.rs | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/dag/extdeps/cloud/gcp/storage.dag b/dag/extdeps/cloud/gcp/storage.dag index 1a56c76fa89..273621eb60c 100644 --- a/dag/extdeps/cloud/gcp/storage.dag +++ b/dag/extdeps/cloud/gcp/storage.dag @@ -3,6 +3,9 @@ module extdeps.cloud.gcp.storage import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import std.types { Int, List, String } +import std.decl_ref { decl_ref } +import std.dissolution { bound_dissolution } +import std.roster_frontier { FrontierRow, frontier_row_decl } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -15,10 +18,17 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { // WOULD BIND TO, AND NOTHING ELSE. This module owns the upstream's shape (DESIGN section 3: the // interface is extdeps', the transport handler is one realization bound to it, the policy is the // workflow's). It carries no handler and no dispatch: the handler lands when the operator says -// (brief of the provider-spine lane, 2026-09-11), and until then these rows are a DECLARED FRONTIER -// -- their first consumer is a GCS realization of std.materialization_object's observations, the -// same two observations extdeps.realization.materialization_store_local produces from a -// filesystem. TRIGGER: the operator names the GCS realization lane. +// (operator provider ruling, 2026-09-11). These rows are therefore a DECLARED FRONTIER in DESIGN +// section 3c's middle state -- consumed by a NAMED consumer in a NAMED later change -- and the row +// gcs_interface_consumer_frontier below carries that as data, not prose. The consumer is the GCS +// realization handler, extdeps.realization.materialization_store_gcs, the sibling of +// extdeps.realization.materialization_store_local: it binds these rows to +// std.materialization_object's observations exactly as the local handler binds the filesystem +// operations. The change is dispatch step 5 of the ruled order (1 recut #10940, 2 ancestry, 3 +// provider spine -- this change, 4 route integration, 5 NFS/GCS realization, 6 incubation), not an +// open-ended discretion. GCS-readiness TODAY is carried by std.materialization_object's path-free +// observation types, which are consumed now; these rows are the upstream facts (cited, upstream's +// own names, API version v1) that the handler binds, so that nothing re-coins them when it lands. // // What the rows establish, cited: // objects.get with alt=media returns the object's bytes; 404 is a typed absent object @@ -89,3 +99,12 @@ data gcs_object_status_facts: List = [ GcsObjectStatusFact { status: 500, meaning: GcsServerError }, GcsObjectStatusFact { status: 503, meaning: GcsServerError }, ] + +// The frontier, typed: the subject is this module's status-fact roster; it dissolves when the named +// consumer declaration appears (std.dissolution bound_dissolution -- DissolutionPending until then, +// mechanically, not by reading this annotation). +data gcs_interface_consumer_frontier: FrontierRow = frontier_row_decl( + ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_object_status_facts"), + reason: "GCS JSON API v1 interface rows land ahead of their handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration).", + dissolution: bound_dissolution(ref: decl_ref(module_path: "extdeps.realization.materialization_store_gcs", decl_name: "gcs_store_read_observation")) +) diff --git a/src/v1/stage0/src/v1_interpreter.rs b/src/v1/stage0/src/v1_interpreter.rs index 2b3e4918e41..b40d2bf37b5 100644 --- a/src/v1/stage0/src/v1_interpreter.rs +++ b/src/v1/stage0/src/v1_interpreter.rs @@ -14261,7 +14261,7 @@ fn dispatch_file( byte_count: content.len() as i64, path, error: String::new(), - error_kind: String::new(), + error_kind: String::new(), content, }) } From ba3a82aacc9d02076ae30ff6accbd7ba9ba09946 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 15:19:52 +0000 Subject: [PATCH 03/10] GCS frontier row: unbound dissolution until its consumer exists The required declarations phase refused CITED-MODULE-ABSENT on 837cf78: a bound dissolution cited extdeps.realization.materialization_store_gcs, which does not exist yet. The not-yet-landed consumer is carried in an unbound_dissolution description naming the declaration and ruled step 5; the handler's change rebinds it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/extdeps/cloud/gcp/storage.dag | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dag/extdeps/cloud/gcp/storage.dag b/dag/extdeps/cloud/gcp/storage.dag index 273621eb60c..7ce8091b0e1 100644 --- a/dag/extdeps/cloud/gcp/storage.dag +++ b/dag/extdeps/cloud/gcp/storage.dag @@ -4,7 +4,7 @@ import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import std.types { Int, List, String } import std.decl_ref { decl_ref } -import std.dissolution { bound_dissolution } +import std.dissolution { unbound_dissolution } import std.roster_frontier { FrontierRow, frontier_row_decl } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { @@ -100,11 +100,13 @@ data gcs_object_status_facts: List = [ GcsObjectStatusFact { status: 503, meaning: GcsServerError }, ] -// The frontier, typed: the subject is this module's status-fact roster; it dissolves when the named -// consumer declaration appears (std.dissolution bound_dissolution -- DissolutionPending until then, -// mechanically, not by reading this annotation). +// The frontier, typed: the subject is this module's status-fact roster, and the dissolution is +// UNBOUND because its consumer does not exist yet -- a bound dissolution may only cite a declaration +// the tree already carries (the required declarations phase refuses CITED-MODULE-ABSENT otherwise). +// The description names the consumer declaration and the ruled dispatch step, so the row becomes +// bindable to std.dissolution bound_dissolution in the change that lands the handler. data gcs_interface_consumer_frontier: FrontierRow = frontier_row_decl( ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_object_status_facts"), reason: "GCS JSON API v1 interface rows land ahead of their handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration).", - dissolution: bound_dissolution(ref: decl_ref(module_path: "extdeps.realization.materialization_store_gcs", decl_name: "gcs_store_read_observation")) + dissolution: unbound_dissolution(description: "dissolve-on: extdeps.realization.materialization_store_gcs gcs_store_read_observation appears (the GCS realization handler, sibling of extdeps.realization.materialization_store_local, landed at ruled dispatch step 5 NFS/GCS realization); that change rebinds this row to bound_dissolution on that declaration.") ) From 68e755b8f3f017403cc38b0ae94e294827b16d8a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 15:54:10 +0000 Subject: [PATCH 04/10] Enroll the GCS frontier group in the census roster (review 63874) gcs_storage_frontier_rows joins census_closure_frontier_row_groups beside the sibling extdeps frontier groups, so gunbc.dissolution_census folds it (well-formedness and expiry) instead of the row being untracked. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/extdeps/cloud/gcp/storage.dag | 18 +++++++++++------- dag/gunbc/census_closure_frontier.dag | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dag/extdeps/cloud/gcp/storage.dag b/dag/extdeps/cloud/gcp/storage.dag index 7ce8091b0e1..7ac7c888f36 100644 --- a/dag/extdeps/cloud/gcp/storage.dag +++ b/dag/extdeps/cloud/gcp/storage.dag @@ -20,7 +20,7 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { // workflow's). It carries no handler and no dispatch: the handler lands when the operator says // (operator provider ruling, 2026-09-11). These rows are therefore a DECLARED FRONTIER in DESIGN // section 3c's middle state -- consumed by a NAMED consumer in a NAMED later change -- and the row -// gcs_interface_consumer_frontier below carries that as data, not prose. The consumer is the GCS +// group gcs_storage_frontier_rows below carries that as data, not prose. The consumer is the GCS // realization handler, extdeps.realization.materialization_store_gcs, the sibling of // extdeps.realization.materialization_store_local: it binds these rows to // std.materialization_object's observations exactly as the local handler binds the filesystem @@ -100,13 +100,17 @@ data gcs_object_status_facts: List = [ GcsObjectStatusFact { status: 503, meaning: GcsServerError }, ] -// The frontier, typed: the subject is this module's status-fact roster, and the dissolution is +// The frontier, typed, and enrolled in gunbc.census_closure_frontier +// census_closure_frontier_row_groups so gunbc.dissolution_census folds it like every sibling +// extdeps frontier group: the subject is this module's status-fact roster, and the dissolution is // UNBOUND because its consumer does not exist yet -- a bound dissolution may only cite a declaration // the tree already carries (the required declarations phase refuses CITED-MODULE-ABSENT otherwise). // The description names the consumer declaration and the ruled dispatch step, so the row becomes // bindable to std.dissolution bound_dissolution in the change that lands the handler. -data gcs_interface_consumer_frontier: FrontierRow = frontier_row_decl( - ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_object_status_facts"), - reason: "GCS JSON API v1 interface rows land ahead of their handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration).", - dissolution: unbound_dissolution(description: "dissolve-on: extdeps.realization.materialization_store_gcs gcs_store_read_observation appears (the GCS realization handler, sibling of extdeps.realization.materialization_store_local, landed at ruled dispatch step 5 NFS/GCS realization); that change rebinds this row to bound_dissolution on that declaration.") -) +data gcs_storage_frontier_rows: List = [ + frontier_row_decl( + ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_object_status_facts"), + reason: "GCS JSON API v1 interface rows land ahead of their handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration).", + dissolution: unbound_dissolution(description: "dissolve-on: extdeps.realization.materialization_store_gcs gcs_store_read_observation appears (the GCS realization handler, sibling of extdeps.realization.materialization_store_local, landed at ruled dispatch step 5 NFS/GCS realization); that change rebinds this row to bound_dissolution on that declaration.") + ) +] diff --git a/dag/gunbc/census_closure_frontier.dag b/dag/gunbc/census_closure_frontier.dag index b00f1f07da7..f45c597ac50 100644 --- a/dag/gunbc/census_closure_frontier.dag +++ b/dag/gunbc/census_closure_frontier.dag @@ -34,6 +34,7 @@ import gunbc.cloudflare.r2_token_mint { cloudflare_r2_token_mint_frontier_rows } import extdeps.cloudflare.account_api_tokens { cloudflare_account_api_tokens_frontier_rows } import extdeps.cloudflare.r2 { cloudflare_r2_frontier_rows } import extdeps.backblaze.b2 { backblaze_b2_frontier_rows } +import extdeps.cloud.gcp.storage { gcs_storage_frontier_rows } import gunbc.cloudflare.r2_permission_group_observe { cloudflare_permission_group_observation_frontier_rows } fn census_closure_frontier_row_groups() -> List> { @@ -64,6 +65,7 @@ fn census_closure_frontier_row_groups() -> List> { cloudflare_account_api_tokens_frontier_rows, cloudflare_r2_frontier_rows, backblaze_b2_frontier_rows, + gcs_storage_frontier_rows, cloudflare_permission_group_observation_frontier_rows, colo_types_frontier_rows, ] From 27eed9173602f28563c8f9e285183fa18a3aae7d Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 16:27:53 +0000 Subject: [PATCH 05/10] Address review 63888: UTF-8 byte sizes, one StoreObjectLine fold, per-decl GCS frontier, no endpoint re-mint - payload_part sizes are UTF-8 byte counts (std.bytes), not code points; new control a_part_size_counts_utf8_bytes_not_code_points. - store_object_line_fold is the one match over StoreObjectLine; the predicates and projections derive from it. - gcs_storage_frontier_rows carries one row per unconsumed declaration. - gcs_objects_get names extdeps.cloud.gcp.gcp's GcpStorage endpoint instead of restating its base URL; the media-upload endpoint lives once, cited. - v1_interpreter write_create_new comment matches the error_kind channel. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/extdeps/cloud/gcp/storage.dag | 47 +++++++++++--- dag/std/materialization_object.dag | 63 +++++++++---------- .../materialization_store_witness_test.dag | 8 +++ src/v1/stage0/src/v1_interpreter.rs | 8 +-- 4 files changed, 82 insertions(+), 44 deletions(-) diff --git a/dag/extdeps/cloud/gcp/storage.dag b/dag/extdeps/cloud/gcp/storage.dag index 7ac7c888f36..6d41782390a 100644 --- a/dag/extdeps/cloud/gcp/storage.dag +++ b/dag/extdeps/cloud/gcp/storage.dag @@ -3,8 +3,9 @@ module extdeps.cloud.gcp.storage import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import std.types { Int, List, String } +import extdeps.cloud.gcp.gcp { GcpService, GcpStorage } import std.decl_ref { decl_ref } -import std.dissolution { unbound_dissolution } +import std.dissolution { DissolutionCondition, unbound_dissolution } import std.roster_frontier { FrontierRow, frontier_row_decl } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { @@ -49,10 +50,21 @@ type GcsObjectOperation = GcsObjectsGet | GcsObjectsInsert +// WHICH ENDPOINT, WITHOUT RESTATING ONE. objects.get is served by the JSON API, whose base URL is +// already owned by extdeps.cloud.gcp.gcp api_endpoints (the GcpStorage row) -- so the fact here +// names that authority and carries only the resource path relative to it (review 63888: the base +// literal is not re-minted). objects.insert with uploadType=media is served by the MEDIA UPLOAD +// endpoint, a distinct upstream URL that api_endpoints does not carry; it lives once, on this +// variant, cited to the upload documentation. +type GcsRequestEndpoint + = GcsJsonApiEndpoint { service: GcpService } + | GcsMediaUploadEndpoint { base_url: String, citation: String } + type GcsObjectOperationFacts { operation: GcsObjectOperation method: String - path_template: String + endpoint: GcsRequestEndpoint + resource_path: String required_query: List citation: String } @@ -60,7 +72,8 @@ type GcsObjectOperationFacts { data gcs_objects_get: GcsObjectOperationFacts = GcsObjectOperationFacts { operation: GcsObjectsGet, method: "GET", - path_template: "https://storage.googleapis.com/storage/v1/b/\{bucket\}/o/\{object\}", + endpoint: GcsJsonApiEndpoint { service: GcpStorage }, + resource_path: "/b/\{bucket\}/o/\{object\}", required_query: ["alt=media"], citation: "cloud.google.com/storage/docs/json_api/v1/objects/get" } @@ -68,7 +81,11 @@ data gcs_objects_get: GcsObjectOperationFacts = GcsObjectOperationFacts { data gcs_objects_insert_if_absent: GcsObjectOperationFacts = GcsObjectOperationFacts { operation: GcsObjectsInsert, method: "POST", - path_template: "https://storage.googleapis.com/upload/storage/v1/b/\{bucket\}/o", + endpoint: GcsMediaUploadEndpoint { + base_url: "https://storage.googleapis.com/upload/storage/v1", + citation: "cloud.google.com/storage/docs/uploading-objects" + }, + resource_path: "/b/\{bucket\}/o", required_query: ["uploadType=media", "name=\{object\}", "ifGenerationMatch=0"], citation: "cloud.google.com/storage/docs/request-preconditions" } @@ -102,15 +119,29 @@ data gcs_object_status_facts: List = [ // The frontier, typed, and enrolled in gunbc.census_closure_frontier // census_closure_frontier_row_groups so gunbc.dissolution_census folds it like every sibling -// extdeps frontier group: the subject is this module's status-fact roster, and the dissolution is +// extdeps frontier group, one row per declaration the handler will consume (review 63888), and the dissolution is // UNBOUND because its consumer does not exist yet -- a bound dissolution may only cite a declaration // the tree already carries (the required declarations phase refuses CITED-MODULE-ABSENT otherwise). // The description names the consumer declaration and the ruled dispatch step, so the row becomes // bindable to std.dissolution bound_dissolution in the change that lands the handler. +data gcs_frontier_reason: String = "GCS JSON API v1 interface row landing ahead of its handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration)." + +data gcs_frontier_dissolution: DissolutionCondition = unbound_dissolution(description: "dissolve-on: extdeps.realization.materialization_store_gcs gcs_store_read_observation appears (the GCS realization handler, sibling of extdeps.realization.materialization_store_local, landed at ruled dispatch step 5 NFS/GCS realization); that change rebinds this row to bound_dissolution on that declaration.") + data gcs_storage_frontier_rows: List = [ + frontier_row_decl( + ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_objects_get"), + reason: gcs_frontier_reason, + dissolution: gcs_frontier_dissolution + ), + frontier_row_decl( + ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_objects_insert_if_absent"), + reason: gcs_frontier_reason, + dissolution: gcs_frontier_dissolution + ), frontier_row_decl( ref: decl_ref(module_path: "extdeps.cloud.gcp.storage", decl_name: "gcs_object_status_facts"), - reason: "GCS JSON API v1 interface rows land ahead of their handler (operator provider ruling 2026-09-11: interface rows day one, handler later). Named consumer: extdeps.realization.materialization_store_gcs, which maps gcs_objects_get / gcs_objects_insert_if_absent and gcs_object_status_facts onto std.materialization_object StoreObjectRead / StoreObjectPublish. Named change: ruled dispatch step 5 (NFS/GCS realization), after 4 (route integration).", - dissolution: unbound_dissolution(description: "dissolve-on: extdeps.realization.materialization_store_gcs gcs_store_read_observation appears (the GCS realization handler, sibling of extdeps.realization.materialization_store_local, landed at ruled dispatch step 5 NFS/GCS realization); that change rebinds this row to bound_dissolution on that declaration.") - ) + reason: gcs_frontier_reason, + dissolution: gcs_frontier_dissolution + ), ] diff --git a/dag/std/materialization_object.dag b/dag/std/materialization_object.dag index ca72f9e14e9..3c4dea4f729 100644 --- a/dag/std/materialization_object.dag +++ b/dag/std/materialization_object.dag @@ -11,6 +11,7 @@ import std.content_hash { } import std.cache_identity { ArtifactKindId, CacheInterfaceId } import std.measure { ByteSize, byte_size, measure_count } +import std.bytes { bytes_octets, utf8_encode_bytes } import std.materialization_provider { ArtifactRequest, ArtifactOutputId, @@ -76,8 +77,10 @@ import std.materialization_provider { // minted from bytes, which is the grounding std.materialization_provider's derivation invariant // names for source (3): observed_digest is computed from the stored parts by this derivation, not // asserted by a caller. The digest is tagged so an empty payload (an empty diagnostics output is -// ordinary) still hashes a non-empty value. The size is the payload's string_length, the same -// measure the decode re-derives, so a size that disagrees with the bytes changes the content digest. +// ordinary) still hashes a non-empty value. The size is the payload's UTF-8 BYTE count +// (std.bytes utf8_encode_bytes, then bytes_octets) -- not string_length, which counts code points +// and would put a different quantity in a ByteSize (review 63888). The decode re-derives it the same +// way, so a size that disagrees with the bytes changes the content digest. type ArtifactPayload { id: ArtifactOutputId @@ -88,7 +91,7 @@ fn payload_part(payload: ArtifactPayload) -> CarriedOutput { CarriedOutput { id: payload.id, digest: content_hash_atom(value: concat("materialization_payload:", payload.content) as NonEmptyStr), - bytes: byte_size(count: string_length(payload.content)) + bytes: byte_size(count: bytes_octets(b: utf8_encode_bytes(s: payload.content)) |> count()) } } @@ -237,15 +240,31 @@ fn store_object_body_lines(lines: List) -> List { lines |> skip(n: 3) |> map(l => store_object_line(line: l)) } -fn store_object_line_malformed_text(l: StoreObjectLine) -> String? { +// ONE fold over StoreObjectLine; every projection below derives from it and carries no match of +// its own, so a fifth line kind is a type error at exactly one site. +fn store_object_line_fold( + l: StoreObjectLine, + part: fn(CarriedOutput) -> T, + payload: fn(ArtifactPayload) -> T, + end: T, + malformed: fn(String) -> T +) -> T { match l { - StoreMalformedLine { line: t } => Present { value: t } - StorePartLine { part: _ } => none - StorePayloadLine { payload: _ } => none - StoreEndLine => none + StorePartLine { part: p } => part(p) + StorePayloadLine { payload: p } => payload(p) + StoreEndLine => end + StoreMalformedLine { line: t } => malformed(t) } } +fn store_object_line_malformed_text(l: StoreObjectLine) -> String? { + store_object_line_fold(l: l, part: p => none, payload: p => none, end: none, malformed: t => Present { value: t }) +} + +fn store_object_line_is_end(l: StoreObjectLine) -> Bool { + store_object_line_fold(l: l, part: p => false, payload: p => false, end: true, malformed: t => false) +} + fn store_object_first_malformed(body: List) -> String? { match body |> filter(l => match store_object_line_malformed_text(l: l) { Present { value: _ } => true, Absent => false }) |> first() { Present { value: l } => store_object_line_malformed_text(l: l) @@ -254,45 +273,25 @@ fn store_object_first_malformed(body: List) -> String? { } fn store_object_end_count(body: List) -> Int { - body |> filter(l => match l { - StoreEndLine => true - StorePartLine { part: _ } => false - StorePayloadLine { payload: _ } => false - StoreMalformedLine { line: _ } => false - }) |> count() + body |> filter(l => store_object_line_is_end(l: l)) |> count() } fn store_object_last_is_end(body: List) -> Bool { let n = body |> count() if n == 0 { false } else { match get(xs: body, index: n - 1) { - Present { value: l } => match l { - StoreEndLine => true - StorePartLine { part: _ } => false - StorePayloadLine { payload: _ } => false - StoreMalformedLine { line: _ } => false - } + Present { value: l } => store_object_line_is_end(l: l) Absent => false } } } fn store_object_parts(body: List) -> List { - body |> flat_map(l => match l { - StorePartLine { part: p } => [p] - StorePayloadLine { payload: _ } => [] - StoreEndLine => [] - StoreMalformedLine { line: _ } => [] - }) + body |> flat_map(l => store_object_line_fold(l: l, part: p => [p], payload: p => [], end: [], malformed: t => [])) } fn store_object_payloads(body: List) -> List { - body |> flat_map(l => match l { - StorePayloadLine { payload: p } => [p] - StorePartLine { part: _ } => [] - StoreEndLine => [] - StoreMalformedLine { line: _ } => [] - }) + body |> flat_map(l => store_object_line_fold(l: l, part: p => [], payload: p => [p], end: [], malformed: t => [])) } // A TRUNCATED OBJECT IS MALFORMED, NOT INCOMPLETE. The trailing `end` line is required exactly once diff --git a/dag/test/claim/materialization_store_witness_test.dag b/dag/test/claim/materialization_store_witness_test.dag index 94041c2e029..eca4b9f1db4 100644 --- a/dag/test/claim/materialization_store_witness_test.dag +++ b/dag/test/claim/materialization_store_witness_test.dag @@ -3,6 +3,7 @@ module test.claim.materialization_store_witness import std.types { Bool, Int, List, String, NonEmptyStr } import std.content_hash { ContentHash, Fnv1a64Structural, content_hash_atom, content_hash_of_value } import std.cache_identity { CacheInterfaceId } +import std.measure { byte_size } import std.materialization_provider { DeclaredInput, ArtifactRequest, ArtifactManifest, NativeCompilerArtifactRequest, UniverseArtifactRequest, NativeModuleVerdictBundleRequest, @@ -344,3 +345,10 @@ test fn a_store_opens_only_on_its_own_marker() -> Bool { } opened && uninitialized && foreign } + +// A part's size is its UTF-8 BYTE count, not its code-point count: "é€" is 2 code points and 5 +// bytes, so a ByteSize filled from string_length would answer 2 here (review 63888). +test fn a_part_size_counts_utf8_bytes_not_code_points() -> Bool { + payload_part(payload: ArtifactPayload { id: output_module_test_verdicts, content: "é€" }).bytes == byte_size(count: 5) + && string_length("é€") == 2 +} diff --git a/src/v1/stage0/src/v1_interpreter.rs b/src/v1/stage0/src/v1_interpreter.rs index b40d2bf37b5..ea704691278 100644 --- a/src/v1/stage0/src/v1_interpreter.rs +++ b/src/v1/stage0/src/v1_interpreter.rs @@ -14346,10 +14346,10 @@ fn dispatch_file( error_kind: String::new(), content: String::new(), }), - // The refusal carries the host's message verbatim and does NOT classify itself. - // Deciding "already existed" from the error TEXT would be a heuristic standing in - // for an observation; the caller learns the create did not happen and why the - // host said so, which is what it needs to refuse. + // The refusal carries the host's message verbatim in `error` and the host + // error's KIND in `error_kind`, so "already existed" is classified by the host's + // own io::ErrorKind -- never by matching the error TEXT, which would be a + // heuristic standing in for an observation. Err(e) => Ok(FileResult { success: false, byte_count: 0, From bed50e8e9bd16276f6103d23e7bafa95395cca8c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 17:07:02 +0000 Subject: [PATCH 06/10] Derive the durable root position from its path (review 63898) materialization_store_durable_root is absolute_path_position of materialization_store_durable_root_path instead of a hand-written segment list, so the grant root and the admission target are one datum. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/std/materialization_store_grant.dag | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dag/std/materialization_store_grant.dag b/dag/std/materialization_store_grant.dag index 3f743c46662..f2da758013b 100644 --- a/dag/std/materialization_store_grant.dag +++ b/dag/std/materialization_store_grant.dag @@ -4,7 +4,6 @@ import std.types { String, Bool, List } import std.access { AccessDecision } import std.effect_grant { Verb, Read, Write, - filesystem_position, NamespacePosition, RealTransport, LifecycleByConvention, @@ -60,8 +59,10 @@ data materialization_store_frame: Frame = Frame { data materialization_store_durable_root_path: String = "/var/lib/gunbc/materialization-store" -data materialization_store_durable_root: NamespacePosition = filesystem_position( - path: ["var", "lib", "gunbc", "materialization-store"] +// DERIVED from the path, never restated as segments (review 63898): the grant root and the +// admission target are the same projection of one datum, so they cannot drift apart. +data materialization_store_durable_root: NamespacePosition = absolute_path_position( + path: materialization_store_durable_root_path ) fn materialization_store_root_path(root: MaterializationStoreRoot) -> String { From a2c0b087e27b10ede86b121b12570f3e2048f568 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 17:44:32 +0000 Subject: [PATCH 07/10] Refuse a repeated payload line in a stored object (review 63921) A second payload line for one output id now refuses as StoreLookupPayloadRepeated instead of serving the first copy, matching the repeated-part refusal on the manifest side. The three payload checks share store_payload_refusal; new control a_repeated_payload_line_is_refused_not_resolved. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/std/materialization_object.dag | 37 ++++++++++++++++--- .../materialization_store_witness_test.dag | 10 +++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/dag/std/materialization_object.dag b/dag/std/materialization_object.dag index 3c4dea4f729..5163af318cb 100644 --- a/dag/std/materialization_object.dag +++ b/dag/std/materialization_object.dag @@ -380,6 +380,7 @@ type StoreLookupRefusal | StoreLookupManifestUninterpretable { decode: ManifestDecode } | StoreLookupPayloadMissing { part: ArtifactOutputId } | StoreLookupPayloadUndeclared { part: ArtifactOutputId } + | StoreLookupPayloadRepeated { part: ArtifactOutputId } | StoreLookupContractRefused { lookup: ProviderLookup } type StoreLookup @@ -394,6 +395,16 @@ fn store_payload_undeclared(manifest: ArtifactManifest, payloads: List) -> ArtifactOutputId? { + match payloads |> filter(p => (payloads |> filter(q => q.id == p.id) |> count()) > 1) |> first() { + Present { value: p } => Present { value: p.id } + Absent => none + } +} + fn store_payload_missing(manifest: ArtifactManifest, payloads: List) -> ArtifactOutputId? { match manifest.parts |> filter(o => !(payloads |> any(p => p.id == o.id))) |> first() { Present { value: o } => Present { value: o.id } @@ -401,6 +412,23 @@ fn store_payload_missing(manifest: ArtifactManifest, payloads: List) -> StoreLookupRefusal? { + match store_payload_undeclared(manifest: manifest, payloads: payloads) { + Present { value: p } => Present { value: StoreLookupPayloadUndeclared { part: p } } + Absent => + match store_payload_repeated(payloads: payloads) { + Present { value: p } => Present { value: StoreLookupPayloadRepeated { part: p } } + Absent => + match store_payload_missing(manifest: manifest, payloads: payloads) { + Present { value: p } => Present { value: StoreLookupPayloadMissing { part: p } } + Absent => none + } + } + } +} + // THE OBSERVED DIGEST IS RE-DERIVED FROM THE PAYLOAD BYTES, in manifest order, and the artifact the // door judges is decoded from the manifest's CLAIMS. The contract compares the two: a flipped payload // byte, a payload truncated inside its line, or a part digest or size edited in the manifest all @@ -430,12 +458,9 @@ fn store_lookup_decide( StoreObjectMalformed { cause: cause } => StoreLookupRefused { key: key, refusal: StoreLookupObjectMalformed { cause: cause } } StoreObjectDecoded { manifest: m, payloads: ps } => - match store_payload_undeclared(manifest: m, payloads: ps) { - Present { value: p } => StoreLookupRefused { key: key, refusal: StoreLookupPayloadUndeclared { part: p } } + match store_payload_refusal(manifest: m, payloads: ps) { + Present { value: r } => StoreLookupRefused { key: key, refusal: r } Absent => - match store_payload_missing(manifest: m, payloads: ps) { - Present { value: p } => StoreLookupRefused { key: key, refusal: StoreLookupPayloadMissing { part: p } } - Absent => match artifact_from_manifest(m: m) { ManifestDecoded { artifact: a } => { let served = provider_serve( @@ -471,7 +496,6 @@ fn store_lookup_decide( ManifestPartRepeated { kind: _, part: _ } => StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } } - } } } } @@ -638,6 +662,7 @@ fn store_lookup_refusal_tag(l: StoreLookup) -> String { StoreLookupManifestUninterpretable { decode: _ } => "manifest_uninterpretable" StoreLookupPayloadMissing { part: _ } => "payload_missing" StoreLookupPayloadUndeclared { part: _ } => "payload_undeclared" + StoreLookupPayloadRepeated { part: _ } => "payload_repeated" StoreLookupContractRefused { lookup: pl } => concat("contract_", provider_lookup_outcome_tag(l: pl)) } ) diff --git a/dag/test/claim/materialization_store_witness_test.dag b/dag/test/claim/materialization_store_witness_test.dag index eca4b9f1db4..31ccdbd636d 100644 --- a/dag/test/claim/materialization_store_witness_test.dag +++ b/dag/test/claim/materialization_store_witness_test.dag @@ -352,3 +352,13 @@ test fn a_part_size_counts_utf8_bytes_not_code_points() -> Bool { payload_part(payload: ArtifactPayload { id: output_module_test_verdicts, content: "é€" }).bytes == byte_size(count: 5) && string_length("é€") == 2 } + +// A payload line repeated for one declared output is corruption: it refuses, like a repeated manifest +// part, instead of serving the first copy (review 63921). The same object without the duplicate hits. +test fn a_repeated_payload_line_is_refused_not_resolved() -> Bool { + let good = prepared_object(req: bundle_req(), payloads: bundle_payloads()) + let duplicated = replace(good, "\nend", join(["\npayload", output_module_test_verdicts as String, "forged\nend"], " ")) + (duplicated == good) == false + && store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: duplicated)) == "payload_repeated" + && store_lookup_is_hit(l: lookup_present(req: bundle_req(), object: good)) +} From 9d4c04c73546ebbc73b4984039cd5121f1a89e8d Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 18:23:48 +0000 Subject: [PATCH 08/10] Route the store fault detail through filesystem_failure_kind_name (review 63940) filesystem_fault now names the host's failure kind in StoreFault.detail, giving filesystem_failure_kind_name its consumer; new control an_unreadable_object_fault_names_the_host_kind. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- .../realization/materialization_store_local.dag | 12 ++++++++---- .../claim/materialization_store_witness_test.dag | 12 ++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 7c0164520ba..781bf31c514 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -122,12 +122,16 @@ fn store_marker_recognized(root: MaterializationStoreRoot, content: String) -> B // --------------------------------------------------------------------------------------------- // Transport observations -> the realization-agnostic observations. +// The detail names the host's KIND beside its message (filesystem_failure_kind_name), so a +// refusal read by a person or a later classifier carries the classification the channel made rather +// than only the host's free text. fn filesystem_fault(kind: FilesystemFailureKind, error: String) -> StoreFault { + let detail = join([filesystem_failure_kind_name(kind: kind), ": ", error], "") match kind { - FilesystemPermissionDenied => StoreFault { class: StoreFaultUnauthorized, detail: error } - FilesystemNotFound => StoreFault { class: StoreFaultUnreachable, detail: error } - FilesystemAlreadyExists => StoreFault { class: StoreFaultUnreachable, detail: error } - FilesystemOtherFailure => StoreFault { class: StoreFaultUnreachable, detail: error } + FilesystemPermissionDenied => StoreFault { class: StoreFaultUnauthorized, detail: detail } + FilesystemNotFound => StoreFault { class: StoreFaultUnreachable, detail: detail } + FilesystemAlreadyExists => StoreFault { class: StoreFaultUnreachable, detail: detail } + FilesystemOtherFailure => StoreFault { class: StoreFaultUnreachable, detail: detail } } } diff --git a/dag/test/claim/materialization_store_witness_test.dag b/dag/test/claim/materialization_store_witness_test.dag index 31ccdbd636d..4727214b884 100644 --- a/dag/test/claim/materialization_store_witness_test.dag +++ b/dag/test/claim/materialization_store_witness_test.dag @@ -30,6 +30,7 @@ import std.materialization_object { encode_store_object, percent_encode_payload, percent_decode_payload, } import extdeps.filesystem.filesystem_io { + FilesystemPermissionDenied, FilesystemExactPathAbsent, FilesystemExactPathUnreadable, FilesystemExactPathKindUnrecognized, FilesystemExactPathRead, FilesystemCreateTargetOccupied, FilesystemCreateRefused, filesystem_exact_read, filesystem_create_new, @@ -38,6 +39,7 @@ import extdeps.realization.materialization_store_local { LocalStoreOpened, LocalStoreUnavailable, LocalStoreNotInitialized, LocalStoreMarkerUnrecognized, LocalStoreMarkerUnavailable, local_store_open_decide, store_marker_content, materialization_store_local_id, + store_read_observation, } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } @@ -362,3 +364,13 @@ test fn a_repeated_payload_line_is_refused_not_resolved() -> Bool { && store_lookup_refusal_tag(l: lookup_present(req: bundle_req(), object: duplicated)) == "payload_repeated" && store_lookup_is_hit(l: lookup_present(req: bundle_req(), object: good)) } + +// An unreadable object reaches the store as unavailable, and its fault carries the host's KIND by +// name (filesystem_failure_kind_name), not only the host's free text (review 63940). +test fn an_unreadable_object_fault_names_the_host_kind() -> Bool { + match store_read_observation(read: FilesystemExactPathUnreadable { path: "/s/o", kind: FilesystemPermissionDenied, error: "denied" }) { + StoreObjectUnavailable { fault: f } => starts_with(s: f.detail, prefix: "permission_denied: ") + StoreObjectPresent { content: _ } => false + StoreObjectAbsent => false + } +} From 2c6f70249044a5dace1a566729232cc68bc436b1 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 11 Sep 2026 19:05:09 +0000 Subject: [PATCH 09/10] Share decoded / bound values across match arms (review 63959) store_lookup_decide binds the manifest decode once; store_commit_prepare binds the decode, the offered digest and the request binding once; store_commit_verify binds the committed digest once. No arm recomputes the value its match already holds. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/std/materialization_object.dag | 46 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/dag/std/materialization_object.dag b/dag/std/materialization_object.dag index 5163af318cb..f199e42d970 100644 --- a/dag/std/materialization_object.dag +++ b/dag/std/materialization_object.dag @@ -460,8 +460,9 @@ fn store_lookup_decide( StoreObjectDecoded { manifest: m, payloads: ps } => match store_payload_refusal(manifest: m, payloads: ps) { Present { value: r } => StoreLookupRefused { key: key, refusal: r } - Absent => - match artifact_from_manifest(m: m) { + Absent => { + let decode = artifact_from_manifest(m: m) + match decode { ManifestDecoded { artifact: a } => { let served = provider_serve( req: req, @@ -490,11 +491,12 @@ fn store_lookup_decide( ) } ManifestKindUndeclared { kind: _ } => - StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: decode } } ManifestPartUndeclared { kind: _, part: _ } => - StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: decode } } ManifestPartRepeated { kind: _, part: _ } => - StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: artifact_from_manifest(m: m) } } + StoreLookupRefused { key: key, refusal: StoreLookupManifestUninterpretable { decode: decode } } + } } } } @@ -531,30 +533,34 @@ fn store_commit_prepare(req: ArtifactRequest, payloads: List) - request_key: key, parts: payloads |> map(p => payload_part(payload: p)) } - match artifact_from_manifest(m: offered) { - ManifestDecoded { artifact: a } => - match request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) { + let decode = artifact_from_manifest(m: offered) + match decode { + ManifestDecoded { artifact: a } => { + let offered_digest = artifact_content_digest(artifact: a) + let binding = request_artifact_binding(req: req, artifact: a, observed_digest: offered_digest) + match binding { BindingOk => StoreCommitPrepared { key: key, object_name: store_object_name(key: key), object: encode_store_object(manifest: artifact_manifest(artifact: a), payloads: payloads), - offered_digest: artifact_content_digest(artifact: a) + offered_digest: offered_digest } BindingKindMismatch { requested: _, found: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: binding } } } BindingWrongArtifact { key: _, found_key: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: binding } } } BindingWrongContent { recorded: _, observed: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: binding } } } BindingIncomplete { missing: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: request_artifact_binding(req: req, artifact: a, observed_digest: artifact_content_digest(artifact: a)) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitContractRefused { binding: binding } } } } + } ManifestKindUndeclared { kind: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: decode } } } ManifestPartUndeclared { kind: _, part: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: decode } } } ManifestPartRepeated { kind: _, part: _ } => - StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: artifact_from_manifest(m: offered) } } } + StoreCommitNotPrepared { commit: StoreCommitRefused { key: key, refusal: StoreCommitManifestUninterpretable { decode: decode } } } } } @@ -593,8 +599,9 @@ fn store_commit_verify( disposition: RealizationDisposition ) -> StoreCommit { match read_back { - StoreLookupHit { artifact: a, payloads: _, receipt: r } => - if artifact_content_digest(artifact: a) == offered { + StoreLookupHit { artifact: a, payloads: _, receipt: r } => { + let committed = artifact_content_digest(artifact: a) + if committed == offered { StoreCommitSettled { receipt: RealizationReceipt { request_key: r.request_key, @@ -606,8 +613,9 @@ fn store_commit_verify( } } } else { - StoreCommitRefused { key: key, refusal: StoreCommitConflict { committed: artifact_content_digest(artifact: a), offered: offered } } + StoreCommitRefused { key: key, refusal: StoreCommitConflict { committed: committed, offered: offered } } } + } StoreLookupMiss { key: _ } => StoreCommitRefused { key: key, refusal: StoreCommitReadBackRefused { read_back: read_back } } StoreLookupRefused { key: _, refusal: _ } => From 128185c4afa29ffa5fa0e68e63727260237a76b0 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Sat, 12 Sep 2026 01:09:02 +0000 Subject: [PATCH 10/10] Key the compiler request from generation's five causal axes std.materialization_provider keyed NativeCompilerArtifactRequest on {emitted closure, toolchain, target platform}, a second spelling of an identity v2.compiler.self_host.generation already owns: a changed producer compiler or build configuration left the key fixed, so two generations read as one computation and a differing artifact read as a same-key content conflict rather than a changed input. generation now exposes PreMaterializationIdentity (the five CAUSAL axes) and pre_materialization_digest, derived on demand, no composite stored. The request carries that projection and derives its key from it. The sixth axis, materialized_artifact, stays out of a pre-build lookup: it is earned from verified bytes and joined back afterwards. Controls: every_generation_causal_axis_is_load_bearing_on_the_compiler_key perturbs each axis independently, and a_compiler_request_key_ignores_which_store_serves_it holds the provider out of the key. Reproducing the old three-field shape reds the first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016AYera4CKxLqZJn5YW5Mzw --- dag/std/materialization_provider.dag | 62 +++++++++++------ .../materialization_store_witness_test.dag | 68 +++++++++++++++++-- src/v2/compiler/self_host/generation.dag | 56 +++++++++++++++ 3 files changed, 158 insertions(+), 28 deletions(-) diff --git a/dag/std/materialization_provider.dag b/dag/std/materialization_provider.dag index 2ce30a9bcb0..9ae2d4516dc 100644 --- a/dag/std/materialization_provider.dag +++ b/dag/std/materialization_provider.dag @@ -33,6 +33,10 @@ import std.computation_identity { IdentityUnknownCause, MissingConcept, MissingEnforcement } import std.artifact_store { ArtifactStore, store_put, store_put_fold } +import v2.compiler.self_host.generation { + PreMaterializationIdentity, + pre_materialization_digest, +} import std.disposition { Disposition, Terminal } import std.measure { ByteSize, byte_size } @@ -155,11 +159,7 @@ type ArtifactRequest compiler: DeclaredInput } | ResolveClosureRequest { closure_digest: Fnv1a64Structural, compiler: DeclaredInput } - | NativeCompilerArtifactRequest { - emitted_closure_identity: Fnv1a64Structural, - build_toolchain: DeclaredInput, - target_platform: DeclaredInput - } + | NativeCompilerArtifactRequest { generation: PreMaterializationIdentity } | UniverseArtifactRequest { tested_tree_identity: Fnv1a64Structural, route_binary: DeclaredInput } | NativeModuleVerdictBundleRequest(NativeModuleVerdictBundleSubject) @@ -175,12 +175,28 @@ type ArtifactRequest // same artifact in every realization, which is what lets a later realization serve what an // earlier one committed without re-keying. // -// NativeCompilerArtifactRequest: the executable built from the seed-prepared emitted closure. -// Subject = the emitted closure's content identity (gunbc.witness.v2_native_route -// NativeRouteReceipt emitted_closure_identity); declared inputs = the build toolchain identity -// and the target platform identity, because the same closure built by a different rustc or for a -// different triple is a different executable. Requires the executable AND the build diagnostics: -// a build whose warnings were dropped is not the build that ran. +// NativeCompilerArtifactRequest: the executable built from the seed-prepared emitted closure. Its +// identity is NOT re-invented here. v2.compiler.self_host.generation is the sole authority for what +// identifies a compiler generation, and this request carries that module's PRE-MATERIALIZATION +// projection -- the five CAUSAL axes (producer compiler, source closure, target model, toolchain, +// build configuration) -- with the key derived by its own pre_materialization_digest. An earlier +// shape keyed on {emitted closure, toolchain, target platform}, which dropped the producer compiler +// and the build configuration: two generations differing only in those read as ONE computation, and +// the differing artifact then reads as a same-key content CONFLICT rather than as a changed input. +// That is the section 3 fork this lane was told not to create, and it is closed by consumption +// rather than by a check. The sixth axis, materialized_artifact, is deliberately absent: it is +// EARNED from verified stored bytes (a committed artifact read back and verified) and joined back to +// GenerationIdentity afterwards, so it can never be caller-authored into a pre-build lookup. +// Requires the executable AND the build diagnostics: a build whose warnings were dropped is not the +// build that ran. +// +// THE IMPORT DIRECTION IS A STATED DEPARTURE, not an oversight. This is the only std module that +// imports a v2 COMPILER module (other std modules import v2.std.*). The alternative was to restate +// the five axes here, which is precisely the fork this change exists to close, and DESIGN section 3 +// puts a fact's home in its LAYER rather than its folder, with acyclicity as the import graph's only +// structural law -- v2.compiler.self_host.generation imports nothing from this module, so the edge is +// acyclic. If the axes ever need a home below the compiler, the move is generation's to make and this +// consumer follows it; duplicating them here would not be that move. // // UniverseArtifactRequest: the derived test universe of one tested tree under one route binary -- // the ordered test-identity population and the per-module import closures the route ingests. @@ -215,7 +231,7 @@ fn request_fold( parse_module: fn(Fnv1a64Structural, DeclaredInput) -> T, typecheck_module: fn(Fnv1a64Structural, List, DeclaredInput) -> T, resolve_closure: fn(Fnv1a64Structural, DeclaredInput) -> T, - native_compiler: fn(Fnv1a64Structural, DeclaredInput, DeclaredInput) -> T, + native_compiler: fn(PreMaterializationIdentity) -> T, universe: fn(Fnv1a64Structural, DeclaredInput) -> T, module_verdict_bundle: fn(NativeModuleVerdictBundleSubject) -> T ) -> T { @@ -224,11 +240,7 @@ fn request_fold( TypecheckModuleRequest { source_digest: sd, direct_import_interfaces: ii, compiler: c } => typecheck_module(sd, ii, c) ResolveClosureRequest { closure_digest: cd, compiler: c } => resolve_closure(cd, c) - NativeCompilerArtifactRequest { - emitted_closure_identity: ec, - build_toolchain: bt, - target_platform: tp - } => native_compiler(ec, bt, tp) + NativeCompilerArtifactRequest { generation: g } => native_compiler(g) UniverseArtifactRequest { tested_tree_identity: tt, route_binary: rb } => universe(tt, rb) NativeModuleVerdictBundleRequest(subject) => module_verdict_bundle(subject) } @@ -262,7 +274,7 @@ fn request_kind(req: ArtifactRequest) -> ArtifactKindId { parse_module: (sd, c) => parse_source_artifact_kind, typecheck_module: (sd, ii, c) => typed_module_artifact_kind, resolve_closure: (cd, c) => resolved_graph_artifact_kind, - native_compiler: (ec, bt, tp) => native_compiler_artifact_kind, + native_compiler: g => native_compiler_artifact_kind, universe: (tt, rb) => native_test_universe_artifact_kind, module_verdict_bundle: subject => native_module_verdict_bundle_artifact_kind ) @@ -274,7 +286,7 @@ fn request_subject_digest(req: ArtifactRequest) -> Fnv1a64Structural { parse_module: (sd, c) => sd, typecheck_module: (sd, ii, c) => sd, resolve_closure: (cd, c) => cd, - native_compiler: (ec, bt, tp) => ec, + native_compiler: g => pre_materialization_digest(pre: g), universe: (tt, rb) => tt, module_verdict_bundle: subject => module_verdict_bundle_subject_digest(subject: subject) ) @@ -287,7 +299,13 @@ fn request_declared_inputs(req: ArtifactRequest) -> List { typecheck_module: (sd, ii, c) => concat(ii |> map(h => DeclaredInput { ref: "direct_import_interface", digest: h }), [c]), resolve_closure: (cd, c) => [c], - native_compiler: (ec, bt, tp) => [bt, tp], + native_compiler: g => [ + DeclaredInput { ref: "generation.producer_compiler", digest: g.producer_compiler }, + DeclaredInput { ref: "generation.source_closure", digest: g.source_closure }, + DeclaredInput { ref: "generation.target_model", digest: g.target_model }, + DeclaredInput { ref: "generation.toolchain", digest: g.toolchain }, + DeclaredInput { ref: "generation.build_configuration", digest: g.build_configuration } + ], universe: (tt, rb) => [rb], module_verdict_bundle: subject => [] ) @@ -300,7 +318,7 @@ fn request_required_outputs(req: ArtifactRequest) -> List { typecheck_module: (sd, ii, c) => [output_typed_module_interface, output_typecheck_diagnostics], resolve_closure: (cd, c) => [output_resolved_graph, output_source_indices, output_compile_clean_diagnostic_union], - native_compiler: (ec, bt, tp) => + native_compiler: g => [output_native_compiler_executable, output_native_compiler_build_diagnostics], universe: (tt, rb) => [output_universe_test_identity_population, output_universe_module_import_closures], @@ -372,7 +390,7 @@ fn request_key(req: ArtifactRequest) -> ContentHash { compiler_identity: c.digest ), resolve_closure: (cd, c) => local_request_key(req: req), - native_compiler: (ec, bt, tp) => local_request_key(req: req), + native_compiler: g => local_request_key(req: req), universe: (tt, rb) => local_request_key(req: req), module_verdict_bundle: subject => local_request_key(req: req) ) diff --git a/dag/test/claim/materialization_store_witness_test.dag b/dag/test/claim/materialization_store_witness_test.dag index 4727214b884..81c84720891 100644 --- a/dag/test/claim/materialization_store_witness_test.dag +++ b/dag/test/claim/materialization_store_witness_test.dag @@ -12,9 +12,10 @@ import std.materialization_provider { request_key, request_kind, request_required_outputs, output_module_test_verdicts, output_module_preparation_diagnostics, output_universe_test_identity_population, output_universe_module_import_closures, - output_native_compiler_executable, + output_native_compiler_executable, output_native_compiler_build_diagnostics, } import std.materialization_store_grant { WitnessScratchRoot, DurableHostVolumeRoot } +import v2.compiler.self_host.generation { PreMaterializationIdentity } import std.materialization_object { ArtifactPayload, payload_part, OpenedMaterializationStore, @@ -81,14 +82,20 @@ fn universe_req() -> ArtifactRequest { } } -fn compiler_req() -> ArtifactRequest { - NativeCompilerArtifactRequest { - emitted_closure_identity: h(v: "tree-1"), - build_toolchain: DeclaredInput { ref: "build_toolchain_identity", digest: h(v: "route-binary-1") }, - target_platform: DeclaredInput { ref: "target_platform_identity", digest: h(v: "x86_64-unknown-linux-gnu") } +fn generation_axes() -> PreMaterializationIdentity { + PreMaterializationIdentity { + producer_compiler: h(v: "tree-1"), + source_closure: h(v: "closure-1"), + target_model: h(v: "rust"), + toolchain: h(v: "rustc-1.93.0"), + build_configuration: h(v: "release") } } +fn compiler_req() -> ArtifactRequest { + NativeCompilerArtifactRequest { generation: generation_axes() } +} + fn bundle_payloads() -> List { [ ArtifactPayload { id: output_module_test_verdicts, content: "v2.test.alpha.t1 pass\nv2.test.alpha.t2 fail 100% sure" }, @@ -374,3 +381,52 @@ test fn an_unreadable_object_fault_names_the_host_kind() -> Bool { StoreObjectAbsent => false } } + +// EVERY CAUSAL AXIS OF THE GENERATION IS LOAD-BEARING ON THE COMPILER REQUEST KEY, and the axes are +// v2.compiler.self_host.generation's, not a second list: perturbing any one of the five moves the +// key, and the five perturbations are pairwise distinct, so no two axes are being folded into one +// position. The earlier shape keyed only the closure, the toolchain and the target, so a changed +// producer compiler or build configuration left the key fixed. +test fn every_generation_causal_axis_is_load_bearing_on_the_compiler_key() -> Bool { + let base = generation_axes() + let k = request_key(req: compiler_req()) + let variants = [ + PreMaterializationIdentity { producer_compiler: h(v: "tree-2"), source_closure: base.source_closure, target_model: base.target_model, toolchain: base.toolchain, build_configuration: base.build_configuration }, + PreMaterializationIdentity { producer_compiler: base.producer_compiler, source_closure: h(v: "closure-2"), target_model: base.target_model, toolchain: base.toolchain, build_configuration: base.build_configuration }, + PreMaterializationIdentity { producer_compiler: base.producer_compiler, source_closure: base.source_closure, target_model: h(v: "wasm"), toolchain: base.toolchain, build_configuration: base.build_configuration }, + PreMaterializationIdentity { producer_compiler: base.producer_compiler, source_closure: base.source_closure, target_model: base.target_model, toolchain: h(v: "rustc-1.94.0"), build_configuration: base.build_configuration }, + PreMaterializationIdentity { producer_compiler: base.producer_compiler, source_closure: base.source_closure, target_model: base.target_model, toolchain: base.toolchain, build_configuration: h(v: "debug") }, + ] + let keys = variants |> map(g => request_key(req: NativeCompilerArtifactRequest { generation: g })) + let same_again = request_key(req: NativeCompilerArtifactRequest { generation: generation_axes() }) == k + let all_apart = (keys |> any(x => x == k)) == false + let pairwise_apart = (keys |> any(x => (keys |> filter(y => y == x) |> count()) > 1)) == false + same_again && all_apart && pairwise_apart && (variants |> count()) == 5 +} + +// AND THE PROVIDER IS NOT IN THAT KEY. The same compiler request served from two different stores +// keys identically and yields the same content digest; only the receipt differs. A provider or store +// instance in the computation key would give one artifact as many identities as it has homes. +test fn a_compiler_request_key_ignores_which_store_serves_it() -> Bool { + let payloads = [ + ArtifactPayload { id: output_native_compiler_executable, content: "ELF-bytes-fixture" }, + ArtifactPayload { id: output_native_compiler_build_diagnostics, content: "" }, + ] + let object = prepared_object(req: compiler_req(), payloads: payloads) + let other_store = OpenedMaterializationStore { provider: "some_other_realization" as CacheInterfaceId, instance: content_hash_of_value(value: "other-instance") } + let a = store_lookup_decide(store: witness_store(), req: compiler_req(), read: StoreObjectPresent { content: object }) + let b = store_lookup_decide(store: other_store, req: compiler_req(), read: StoreObjectPresent { content: object }) + match a { + StoreLookupHit { artifact: _, payloads: _, receipt: ra } => + match b { + StoreLookupHit { artifact: _, payloads: _, receipt: rb } => + ra.request_key == rb.request_key && ra.content_digest == rb.content_digest + && ra.request_key == request_key(req: compiler_req()) + && (ra.provider == rb.provider) == false + StoreLookupMiss { key: _ } => false + StoreLookupRefused { key: _, refusal: _ } => false + } + StoreLookupMiss { key: _ } => false + StoreLookupRefused { key: _, refusal: _ } => false + } +} diff --git a/src/v2/compiler/self_host/generation.dag b/src/v2/compiler/self_host/generation.dag index cfd8dfb8ba5..490c1797938 100644 --- a/src/v2/compiler/self_host/generation.dag +++ b/src/v2/compiler/self_host/generation.dag @@ -95,6 +95,62 @@ type GenerationIdentity { materialized_artifact: GeneratedArtifactIdentity } +// THE PRE-MATERIALIZATION PROJECTION: the five CAUSAL axes, without the artifact. +// +// A consumer that must key a compiler build BEFORE the artifact exists -- a materialization request +// for the native compiler (std.materialization_provider NativeCompilerArtifactRequest) is the first +// one -- needs exactly the axes that CAUSE the build, and cannot have the sixth: materialized_artifact +// is EARNED from observed bytes and joined back afterwards, so admitting it into a pre-build lookup +// key would let a caller author the very fact the build is supposed to produce. +// +// It lives HERE because this module is the sole authority for what a generation's identity IS +// (DESIGN section 3). A consumer that re-listed the axes would fork that fact, and the fork is not +// hypothetical: a key over only {source closure, toolchain, target} leaves a changed PRODUCER +// COMPILER or BUILD CONFIGURATION invisible, so two different generations read as one computation +// and the differing artifact reads as a same-key content conflict rather than a changed input. +// Adding a seventh causal axis therefore changes this projection and every key derived from it, +// which is the property a second list cannot offer. +// +// NO COMPOSITE IS STORED, exactly as the head of this module requires: the digest is a projection +// derived on demand, folded in the declared axis order under a tag of its own so it cannot collide +// with any other fold over the same five digests. +type PreMaterializationIdentity { + producer_compiler: Hash + source_closure: Hash + target_model: Hash + toolchain: Hash + build_configuration: Hash +} + +fn pre_materialization_identity(identity: GenerationIdentity) -> PreMaterializationIdentity { + PreMaterializationIdentity { + producer_compiler: identity.producer_compiler, + source_closure: identity.source_closure, + target_model: identity.target_model, + toolchain: identity.toolchain, + build_configuration: identity.build_configuration + } +} + +fn pre_materialization_digest(pre: PreMaterializationIdentity) -> Hash { + combine_hash( + a: combine_hash( + a: combine_hash( + a: combine_hash( + a: combine_hash( + a: symbol_identity_digest(sym: ^generation_pre_materialization_identity), + b: pre.producer_compiler + ), + b: pre.source_closure + ), + b: pre.target_model + ), + b: pre.toolchain + ), + b: pre.build_configuration + ) +} + // Three generations, three distinct TYPES. A candidate cannot be constructed without // naming the seed that produced it, and a successor cannot be constructed without // naming the candidate that produced it, so "seed-generated candidate" and