From ebc99661af1119f987b3a37487d011ce48b15af6 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 01:35:26 -0400 Subject: [PATCH 01/11] WIP: Pipeline Authority --- src/v3/compiler/src/lib.rs | 2 +- src/v3/compiler/src/pipeline_authority.rs | 131 +++++----------------- 2 files changed, 26 insertions(+), 107 deletions(-) diff --git a/src/v3/compiler/src/lib.rs b/src/v3/compiler/src/lib.rs index c6662c3cded..88b1540976c 100644 --- a/src/v3/compiler/src/lib.rs +++ b/src/v3/compiler/src/lib.rs @@ -886,7 +886,7 @@ pub fn surface_top_level_let_names_for_test(module: &parse::SurfaceModule) -> Ve /// `pipeline.dag` — the same ordering as `materialize_pipeline_realizations`. #[doc(hidden)] pub fn pipeline_compile_order_stage_names() -> Result, String> { - pipeline_authority::pipeline_compile_order_names() + pipeline_authority::pipeline_compile_order_names(&dag::Dag::new()) } /// Top-level compile failure. Distinguishes three structural diff --git a/src/v3/compiler/src/pipeline_authority.rs b/src/v3/compiler/src/pipeline_authority.rs index d7a6ea6a742..c37a6ede543 100644 --- a/src/v3/compiler/src/pipeline_authority.rs +++ b/src/v3/compiler/src/pipeline_authority.rs @@ -1,13 +1,8 @@ -use std::collections::HashMap; - use crate::dag::{Dag, DeclarationId, FieldValue, TypeConnective, ValueBody}; -use crate::parse::{self, SurfaceItem}; -use crate::tokenize; pub(crate) const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; const PIPELINE_STAGE_BINDING_TYPE: &str = "PipelineStageBinding"; -const PIPELINE_COMPILE_FN: &str = "compile"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum PipelineSnapshotKind { @@ -25,42 +20,10 @@ pub(crate) struct PipelineStageAuthority { pub(crate) snapshot_kind: PipelineSnapshotKind, } +/// Pipeline stage order, read structurally from `PipelineStageBinding` +/// declarations in the Dag. The declaration order of the bindings in +/// `pipeline.dag` is the ordering authority. pub(crate) fn ordered_pipeline_stages(dag: &Dag) -> Result, String> { - let mut bindings = pipeline_stage_bindings(dag)?; - let stage_names = pipeline_compile_order_names()?; - let mut ordered = Vec::with_capacity(stage_names.len()); - - for stage_name in stage_names { - let stage_id = dag - .declaration_by_name(&stage_name) - .map(|decl| decl.id) - .ok_or_else(|| { - format!("pipeline compile body references missing stage declaration `{stage_name}`") - })?; - let authority = bindings.remove(&stage_id).ok_or_else(|| { - format!("pipeline stage `{stage_name}` is used by compile but has no stage binding") - })?; - ordered.push(authority); - } - - if !bindings.is_empty() { - let mut extras: Vec<_> = bindings - .into_values() - .map(|authority| authority.stage_name) - .collect(); - extras.sort(); - return Err(format!( - "pipeline stage bindings not used by compile: {}", - extras.join(", ") - )); - } - - Ok(ordered) -} - -fn pipeline_stage_bindings( - dag: &Dag, -) -> Result, String> { let binding_type_id = dag .declaration_by_name(PIPELINE_STAGE_BINDING_TYPE) .map(|decl| decl.id) @@ -68,7 +31,9 @@ fn pipeline_stage_bindings( format!("missing pipeline authority type `{PIPELINE_STAGE_BINDING_TYPE}`") })?; - let mut bindings = HashMap::new(); + let mut ordered: Vec = Vec::new(); + let mut seen_stages: Vec = Vec::new(); + for declaration in dag.declarations() { if declaration.meta_tag != Some(binding_type_id) { continue; @@ -108,29 +73,36 @@ fn pipeline_stage_bindings( ) })?; - if bindings.contains_key(&stage) { + if seen_stages.contains(&stage) { return Err(format!( "pipeline stage `{stage_name}` has multiple stage bindings" )); } + seen_stages.push(stage); - bindings.insert( + ordered.push(PipelineStageAuthority { stage, - PipelineStageAuthority { - stage, - stage_name, - realization, - realization_name, - snapshot_kind, - }, - ); + stage_name, + realization, + realization_name, + snapshot_kind, + }); } - if bindings.is_empty() { + if ordered.is_empty() { return Err("pipeline authority contains no stage bindings".to_string()); } - Ok(bindings) + Ok(ordered) +} + +/// Pipeline stage names in authority order. Thin wrapper over +/// `ordered_pipeline_stages` for callers that only need the names. +pub(crate) fn pipeline_compile_order_names(dag: &Dag) -> Result, String> { + Ok(ordered_pipeline_stages(dag)? + .into_iter() + .map(|stage| stage.stage_name) + .collect()) } fn require_decl_ref( @@ -221,56 +193,3 @@ fn require_snapshot_kind( )), } } - -pub(crate) fn pipeline_compile_order_names() -> Result, String> { - let source = include_str!("../pipeline.dag"); - let tokens = tokenize::tokenize(source, PIPELINE_AUTHORITY_FILE) - .map_err(|diag| format!("failed to tokenize pipeline authority: {diag:?}"))?; - let module = parse::parse(&tokens, PIPELINE_AUTHORITY_FILE) - .map_err(|diag| format!("failed to parse pipeline authority: {diag:?}"))?; - - let body_span = module - .items - .iter() - .find_map(|item| match item { - SurfaceItem::FnExternalBody { - name, body_span, .. - } if name == PIPELINE_COMPILE_FN => Some(body_span.clone()), - _ => None, - }) - .ok_or_else(|| { - format!("pipeline authority is missing external-body fn `{PIPELINE_COMPILE_FN}`") - })?; - - let body = source - .get(body_span.byte_start as usize..body_span.byte_end as usize) - .ok_or_else(|| "pipeline compile body span is out of bounds".to_string())?; - let body = body.trim(); - let body = body - .strip_prefix('{') - .and_then(|body| body.strip_suffix('}')) - .ok_or_else(|| "pipeline compile body must be a braced block".to_string())?; - - let mut stages = Vec::new(); - for line in body.lines() { - let candidate = line.trim(); - if candidate.is_empty() || candidate.starts_with("//") { - continue; - } - if !candidate - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') - { - return Err(format!( - "pipeline compile body contains unsupported stage expression `{candidate}`" - )); - } - stages.push(candidate.to_string()); - } - - if stages.is_empty() { - return Err("pipeline compile body does not list any stages".to_string()); - } - - Ok(stages) -} From 0e31b6a0e97fb19ec0ec696c636763d263c50628 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 01:46:23 -0400 Subject: [PATCH 02/11] WIP: Pipeline Authority --- src/v3/compiler/src/dag.rs | 20 ++++--- ...m1_fn_external_body_reconciliation_test.rs | 52 +++++++++++++++++++ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/v3/compiler/src/dag.rs b/src/v3/compiler/src/dag.rs index 82b740e1a7e..edecfc4a94d 100644 --- a/src/v3/compiler/src/dag.rs +++ b/src/v3/compiler/src/dag.rs @@ -576,15 +576,13 @@ include!("dag_scalar_generated.rs"); /// bootstrap rewrites those Arrow bodies to `ExternalRealization` before /// inference — so `Unparsed` does not persist for those stages in a /// bootstrapped DAG. **`fn compile` (case 2c)** has no `PipelineStageBinding`: -/// **`Unparsed` persists**; `pipeline_compile_order_stage_names` reads its -/// **body span** as pipeline ordering authority — **terminal for bootstrap -/// ordering**, not a host bridge (contrast 2a) and not parse-lag debt -/// (contrast case 1). **Dissolution for 2c:** a future substrate change that -/// records stage order structurally and supersedes span extraction (then this -/// path retires). The signature flows forward through the declaration table -/// so callers can type-check against it; the body source span is preserved so -/// M2+ parser extensions can reach in for case 1, or so pipeline authority can -/// parse ordering for `compile`. +/// **`Unparsed` persists** on its Arrow body. Pipeline ordering authority is +/// the declaration order of the `PipelineStageBinding` records in the Dag — +/// `ordered_pipeline_stages` reads that structural order directly; the +/// `compile` body span is no longer consulted. The signature still flows +/// forward through the declaration table so callers can type-check against +/// it, and the body source span is preserved so M2+ parser extensions can +/// reach in for case 1. /// **User-range boundary:** `reject_user_unparsed_scaffolds` in /// `src/v3/compiler/src/lower.rs` fails-closed any user-range /// declaration carrying this variant (R14 + M1(2.8) Scaffold @@ -599,8 +597,8 @@ include!("dag_scalar_generated.rs"); /// transition state: `Pending` and **case-1** `Unparsed` are scaffolds with /// named dissolution (M3 / M2 grammar). **`Unparsed` on `pipeline.dag`'s /// `compile` (DB-16 case 2c)** is different — **persistent bootstrap-range -/// ordering authority** until structural pipeline order supersedes span -/// extraction; it does **not** share case 1’s “wait for M2 parser” story. User-range +/// carrier** whose body span is no longer the ordering authority (structural +/// binding order is); it does **not** share case 1’s “wait for M2 parser” story. User-range /// `Unparsed` stays gated (R14). #[derive(Debug, Clone)] pub enum ArrowBody { diff --git a/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs b/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs index 4838e98543f..945069e5829 100644 --- a/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs +++ b/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs @@ -6,6 +6,58 @@ use v3_compiler::dag::{ArrowBody, Dag, TypeConnective}; +/// Pipeline stage order is read structurally from `PipelineStageBinding` +/// declarations in the Dag — the `fn compile { ... }` body span is no +/// longer the ordering authority. This test locks that structural path by +/// matching the public stage-name list against the in-Dag iteration order +/// of stage bindings. +#[test] +fn pipeline_stage_order_is_read_structurally_from_bindings() { + use v3_compiler::dag::{FieldValue, ValueBody}; + + let dag = Dag::new(); + let binding_type_id = dag + .declaration_by_name("PipelineStageBinding") + .expect("PipelineStageBinding type must be bootstrapped") + .id; + + let expected: Vec = dag + .declarations() + .iter() + .filter(|decl| decl.meta_tag == Some(binding_type_id)) + .map(|decl| { + let Some(ValueBody::Structural { fields }) = &decl.value_body else { + panic!("binding must have structural body"); + }; + let stage_id = fields + .iter() + .find(|(f, _)| f == "stage") + .and_then(|(_, v)| match v { + FieldValue::Reference(id) => Some(*id), + _ => None, + }) + .expect("stage field"); + dag.declaration(stage_id) + .name + .clone() + .expect("stage has name") + }) + .collect(); + + let actual = v3_compiler::pipeline_compile_order_stage_names() + .expect("pipeline stage order must resolve"); + + assert_eq!( + actual, expected, + "stage order must be structurally read from PipelineStageBinding \ + declaration order" + ); + assert!( + !actual.is_empty(), + "pipeline must have at least one bound stage" + ); +} + #[test] fn pipeline_stages_lower_to_external_realization_not_unparsed() { let dag = Dag::new(); From 953bb66dbbc24d165b68c8a6203eb61ea7194619 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 02:08:22 -0400 Subject: [PATCH 03/11] pipeline_authority: fail-closed reconciliation of compile body vs bindings Address REQUEST_CHANGES on #637. Bindings remain the runtime ordering authority, but `ordered_pipeline_stages` now fail-closes on any drift between the `fn compile { ... }` body and the `PipelineStageBinding` declaration order. Drift surfaces as a bootstrap diagnostic rather than silent divergence. Adds unit tests covering both the matching and reordered cases, and updates the DB-16 case-2c doc comment. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/v3/compiler/src/dag.rs | 13 ++- src/v3/compiler/src/pipeline_authority.rs | 116 ++++++++++++++++++++++ 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/src/v3/compiler/src/dag.rs b/src/v3/compiler/src/dag.rs index edecfc4a94d..deb502b75b1 100644 --- a/src/v3/compiler/src/dag.rs +++ b/src/v3/compiler/src/dag.rs @@ -578,11 +578,14 @@ include!("dag_scalar_generated.rs"); /// bootstrapped DAG. **`fn compile` (case 2c)** has no `PipelineStageBinding`: /// **`Unparsed` persists** on its Arrow body. Pipeline ordering authority is /// the declaration order of the `PipelineStageBinding` records in the Dag — -/// `ordered_pipeline_stages` reads that structural order directly; the -/// `compile` body span is no longer consulted. The signature still flows -/// forward through the declaration table so callers can type-check against -/// it, and the body source span is preserved so M2+ parser extensions can -/// reach in for case 1. +/// `ordered_pipeline_stages` reads that structural order directly at runtime. +/// The `compile` body is retained as a second surface-level expression of +/// the same ordering and `ordered_pipeline_stages` fail-closes on any drift +/// between the two (`reconcile_with_compile_body`) so the bindings remain +/// the single runtime authority without silently diverging from the +/// orchestrator surface. The signature still flows forward through the +/// declaration table so callers can type-check against it, and the body +/// source span is preserved so M2+ parser extensions can reach in for case 1. /// **User-range boundary:** `reject_user_unparsed_scaffolds` in /// `src/v3/compiler/src/lower.rs` fails-closed any user-range /// declaration carrying this variant (R14 + M1(2.8) Scaffold diff --git a/src/v3/compiler/src/pipeline_authority.rs b/src/v3/compiler/src/pipeline_authority.rs index c37a6ede543..e42e32c598a 100644 --- a/src/v3/compiler/src/pipeline_authority.rs +++ b/src/v3/compiler/src/pipeline_authority.rs @@ -1,8 +1,11 @@ use crate::dag::{Dag, DeclarationId, FieldValue, TypeConnective, ValueBody}; +use crate::parse::{self, SurfaceItem}; +use crate::tokenize; pub(crate) const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; const PIPELINE_STAGE_BINDING_TYPE: &str = "PipelineStageBinding"; +const PIPELINE_COMPILE_FN: &str = "compile"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum PipelineSnapshotKind { @@ -93,9 +96,90 @@ pub(crate) fn ordered_pipeline_stages(dag: &Dag) -> Result Result<(), String> { + let body_names = compile_body_stage_names_from_source()?; + let binding_names: Vec<&str> = ordered + .iter() + .map(|stage| stage.stage_name.as_str()) + .collect(); + let body_refs: Vec<&str> = body_names.iter().map(|name| name.as_str()).collect(); + if binding_names != body_refs { + return Err(format!( + "pipeline authority drift: `fn compile` body lists [{}] but \ + `PipelineStageBinding` declaration order is [{}]. The bindings \ + are the runtime authority — update `fn compile` to match, or \ + reorder the bindings to match `fn compile`.", + body_refs.join(", "), + binding_names.join(", ") + )); + } + Ok(()) +} + +fn compile_body_stage_names_from_source() -> Result, String> { + let source = include_str!("../pipeline.dag"); + let tokens = tokenize::tokenize(source, PIPELINE_AUTHORITY_FILE) + .map_err(|diag| format!("failed to tokenize pipeline authority: {diag:?}"))?; + let module = parse::parse(&tokens, PIPELINE_AUTHORITY_FILE) + .map_err(|diag| format!("failed to parse pipeline authority: {diag:?}"))?; + + let body_span = module + .items + .iter() + .find_map(|item| match item { + SurfaceItem::FnExternalBody { + name, body_span, .. + } if name == PIPELINE_COMPILE_FN => Some(body_span.clone()), + _ => None, + }) + .ok_or_else(|| { + format!("pipeline authority is missing external-body fn `{PIPELINE_COMPILE_FN}`") + })?; + + let body = source + .get(body_span.byte_start as usize..body_span.byte_end as usize) + .ok_or_else(|| "pipeline compile body span is out of bounds".to_string())?; + let body = body.trim(); + let body = body + .strip_prefix('{') + .and_then(|body| body.strip_suffix('}')) + .ok_or_else(|| "pipeline compile body must be a braced block".to_string())?; + + let mut stages = Vec::new(); + for line in body.lines() { + let candidate = line.trim(); + if candidate.is_empty() || candidate.starts_with("//") { + continue; + } + if !candidate + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + return Err(format!( + "pipeline compile body contains unsupported stage expression `{candidate}`" + )); + } + stages.push(candidate.to_string()); + } + + if stages.is_empty() { + return Err("pipeline compile body does not list any stages".to_string()); + } + + Ok(stages) +} + /// Pipeline stage names in authority order. Thin wrapper over /// `ordered_pipeline_stages` for callers that only need the names. pub(crate) fn pipeline_compile_order_names(dag: &Dag) -> Result, String> { @@ -193,3 +277,35 @@ fn require_snapshot_kind( )), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn bootstrapped_stages() -> Vec { + // Bootstrapping includes reconcile_with_compile_body, so any Dag + // we build by hand would already match. Start from a Dag that + // passes reconciliation and mutate the returned vector. + let dag = Dag::new(); + ordered_pipeline_stages(&dag).expect("bootstrap pipeline authority is clean") + } + + #[test] + fn reconcile_rejects_reordered_binding_list() { + let mut stages = bootstrapped_stages(); + assert!(stages.len() >= 2, "need >=2 stages to reorder"); + stages.swap(0, 1); + let err = reconcile_with_compile_body(&stages) + .expect_err("reordered bindings must be rejected"); + assert!( + err.contains("pipeline authority drift"), + "expected drift diagnostic, got: {err}" + ); + } + + #[test] + fn reconcile_accepts_matching_order() { + let stages = bootstrapped_stages(); + reconcile_with_compile_body(&stages).expect("matching order must reconcile"); + } +} From 1f2da955fc2b826d9552c4ddcaee65c3d757cb49 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 02:08:33 -0400 Subject: [PATCH 04/11] chore: apply cargo fmt --- src/v3/compiler/src/pipeline_authority.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v3/compiler/src/pipeline_authority.rs b/src/v3/compiler/src/pipeline_authority.rs index e42e32c598a..01526939c1f 100644 --- a/src/v3/compiler/src/pipeline_authority.rs +++ b/src/v3/compiler/src/pipeline_authority.rs @@ -295,8 +295,8 @@ mod tests { let mut stages = bootstrapped_stages(); assert!(stages.len() >= 2, "need >=2 stages to reorder"); stages.swap(0, 1); - let err = reconcile_with_compile_body(&stages) - .expect_err("reordered bindings must be rejected"); + let err = + reconcile_with_compile_body(&stages).expect_err("reordered bindings must be rejected"); assert!( err.contains("pipeline authority drift"), "expected drift diagnostic, got: {err}" From 0f3690c448615f12a7d4b00961ad493651d3225a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 02:09:33 -0400 Subject: [PATCH 05/11] test: pin canonical pipeline stage sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address APPROVE_WITH_COMMENTS on #637. The prior test rebuilt `expected` from the same binding iteration the reader uses, so a stray reorder in `pipeline.dag` would pass silently. Pin the literal stage sequence (parse → lower → infer → compute_ownership → lens_complexity → emit) so drift from the canonical L1.5 pipeline fails closed. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...m1_fn_external_body_reconciliation_test.rs | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs b/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs index 945069e5829..122db2afc19 100644 --- a/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs +++ b/src/v3/compiler/tests/integration/m1_fn_external_body_reconciliation_test.rs @@ -8,53 +8,27 @@ use v3_compiler::dag::{ArrowBody, Dag, TypeConnective}; /// Pipeline stage order is read structurally from `PipelineStageBinding` /// declarations in the Dag — the `fn compile { ... }` body span is no -/// longer the ordering authority. This test locks that structural path by -/// matching the public stage-name list against the in-Dag iteration order -/// of stage bindings. +/// longer the ordering authority. Pin the literal sequence so a stray +/// reorder of bindings in `pipeline.dag` fails closed (rather than +/// agreeing with itself through the same code path). #[test] fn pipeline_stage_order_is_read_structurally_from_bindings() { - use v3_compiler::dag::{FieldValue, ValueBody}; - - let dag = Dag::new(); - let binding_type_id = dag - .declaration_by_name("PipelineStageBinding") - .expect("PipelineStageBinding type must be bootstrapped") - .id; - - let expected: Vec = dag - .declarations() - .iter() - .filter(|decl| decl.meta_tag == Some(binding_type_id)) - .map(|decl| { - let Some(ValueBody::Structural { fields }) = &decl.value_body else { - panic!("binding must have structural body"); - }; - let stage_id = fields - .iter() - .find(|(f, _)| f == "stage") - .and_then(|(_, v)| match v { - FieldValue::Reference(id) => Some(*id), - _ => None, - }) - .expect("stage field"); - dag.declaration(stage_id) - .name - .clone() - .expect("stage has name") - }) - .collect(); - let actual = v3_compiler::pipeline_compile_order_stage_names() .expect("pipeline stage order must resolve"); + let expected = vec![ + "parse".to_string(), + "lower".to_string(), + "infer".to_string(), + "compute_ownership".to_string(), + "lens_complexity".to_string(), + "emit".to_string(), + ]; + assert_eq!( actual, expected, - "stage order must be structurally read from PipelineStageBinding \ - declaration order" - ); - assert!( - !actual.is_empty(), - "pipeline must have at least one bound stage" + "pipeline stage order must match the canonical L1.5 pipeline; \ + update `pipeline.dag` bindings and `fn compile` body together" ); } From 90d78a6a31c07e768633111825fa0cccc48ebb0c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 02:15:35 -0400 Subject: [PATCH 06/11] docs: explain why compile-body mirror is terminal, not a scaffold Address APPROVE_WITH_COMMENTS doc ask on #637. The prior DB-16 case-2c doc promised the body-span path would retire; this PR retains it as a drift detector. Name the new framing explicitly: the fn-body is the human-readable pipeline contract the bindings satisfy, bindings are the runtime authority, and the reconcile keeps the contract honest (P3) without letting the body become a second runtime source (P2). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/v3/compiler/src/dag.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/v3/compiler/src/dag.rs b/src/v3/compiler/src/dag.rs index deb502b75b1..67b5ebb4f1c 100644 --- a/src/v3/compiler/src/dag.rs +++ b/src/v3/compiler/src/dag.rs @@ -583,7 +583,14 @@ include!("dag_scalar_generated.rs"); /// the same ordering and `ordered_pipeline_stages` fail-closes on any drift /// between the two (`reconcile_with_compile_body`) so the bindings remain /// the single runtime authority without silently diverging from the -/// orchestrator surface. The signature still flows forward through the +/// orchestrator surface. The `compile` body is **terminal**, not a scaffold +/// awaiting dissolution: it is the human-readable pipeline contract that +/// the binding records satisfy — a reader sees the pipeline in one glance +/// as `{ parse; lower; infer; ... }` rather than reconstructing it from a +/// binding table. The bindings are the runtime authority; the body is the +/// surface the bindings commit to. The fail-closed reconcile is what keeps +/// that contract honest (P3) without letting the body become a second +/// runtime source (P2). The signature still flows forward through the /// declaration table so callers can type-check against it, and the body /// source span is preserved so M2+ parser extensions can reach in for case 1. /// **User-range boundary:** `reject_user_unparsed_scaffolds` in From 4304f0c6fbaa4484b03f39779c15d7c216fc58ab Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 02:20:00 -0400 Subject: [PATCH 07/11] docs: sync DB-16 case-2c narrative with structural-read cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address APPROVE doc-sync ask on #637. Update DB-16 receipts in docs/db-history/db-{14,16,17}.md, docs/history/roadmap-scheduled-deletions.md, and docs/design-fn-external-body-reconciliation.md to reflect that: - runtime pipeline ordering now reads PipelineStageBinding declaration order via ordered_pipeline_stages (no body-span parsing on the live path); - the fn compile body is the human-readable pipeline contract, fail- closed reconciled against the bindings (reconcile_with_compile_body); - case 2c is terminal (not a scheduled scaffold) — removed from the scheduled-deletions tables. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/db-history/db-14.md | 5 ++--- docs/db-history/db-16.md | 4 ++-- docs/db-history/db-17.md | 3 +-- docs/design-fn-external-body-reconciliation.md | 14 ++++++++------ docs/history/roadmap-scheduled-deletions.md | 3 +-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/db-history/db-14.md b/docs/db-history/db-14.md index 73a46a8f759..34c0b7c0277 100644 --- a/docs/db-history/db-14.md +++ b/docs/db-history/db-14.md @@ -2,7 +2,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROADMAP.md). -**DB-16 (`FnExternalBody` reconciliation, PR #524).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; `pipeline_compile_order_stage_names` reads `compile`'s body span for ordering authority)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1** only; **2c** and accessor interim have separate dissolution triggers. +**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — terminal surface, not a scaffold)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is split (case 1 vs DB-14 interim) — the M2 grammar milestone removes **case 1** only; accessor interim has a separate dissolution trigger; **2c is terminal** (no dissolution scheduled). **Deferral: E-9 substrate accessor bootstrap rewrite (substrate, DB-14 follow-on).** DB-14 substrate accessor callables currently keep `ArrowBody::Unparsed` through bootstrap; emitters pair accessor declarations with per-target realizations via `SubstrateAccessorBinding` (see `bootstrap.rs` DB-14 comment — target-specific realization choice cannot collapse to one id at `Dag::new()` without a redesign). **`INVARIANTS.md` §E-9** requires that external realization appear only as `ArrowBody::ExternalRealization(ref)` on the Arrow, to a target-neutral marker, with per-target specs resolving from that marker — no second “externality” channel. **Dissolution trigger:** a follow-up PR extends bootstrap (or one materialization pass) to rewrite accessor Arrow bodies from `Unparsed` to `ExternalRealization(accessor_marker_id)` for each declared substrate accessor, preserving multi-target resolution through the marker + spec tables (structurally parallel to `materialize_pipeline_realizations`). Until that lands, DB-16 and `parse.rs`/`dag.rs` DB-16-scoped comments avoid framing accessor `Unparsed` as a second legitimate steady-state meaning alongside parse lag. Design: [design-substrate-external-primitives.md](docs/design-substrate-external-primitives.md) (DB-14), E-9. @@ -12,7 +12,6 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | -| `ArrowBody::Unparsed` (**case 2c** — `pipeline.dag` `fn compile` ordering text) | Structural pipeline-order carrier | First-class ordered stage list (or successor substrate) supersedes `compile` body-span parsing | **Not** the M2 grammar milestone — dissolution per [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) case 2c; `pipeline_compile_order_stage_names` is the reader today | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -23,7 +22,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | `encoding_meet` / `encoding_join` (Rust fns) | Track 8 Phase 2 (user-defined generic emission) | User-defined generic emission for `Lattice` instance | **Compiler-source ratchet** (temporary; becomes lens-able when compiler.dag self-hosts and emission-generated code replaces these hand-written fns) | - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three dissolution stories, not one.** DB-16 / PR #524: **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — ordering text read by `pipeline_authority` until a structural pipeline-order fact supersedes span extraction — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete `compile`’s span authority or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/db-history/db-16.md b/docs/db-history/db-16.md index 92ba9841c0f..638c2677902 100644 --- a/docs/db-history/db-16.md +++ b/docs/db-history/db-16.md @@ -10,7 +10,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | 3a.4 surface generics (DB-12, S) | ✅ Shipped | PR #496 | Tests-only landing; infrastructure already wired. | | 3a.5 Disj dotted-path (DB-13, S) | ✅ Shipped | PR #496 | Tests-only landing; infrastructure already wired. | -**DB-16 (`FnExternalBody` reconciliation, PR #524).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; `pipeline_compile_order_stage_names` reads `compile`'s body span for ordering authority)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1** only; **2c** and accessor interim have separate dissolution triggers. +**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — terminal surface, not a scaffold)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1** only; **2c** is terminal (no dissolution scheduled — body is the pipeline contract, structural authority is the bindings); accessor interim has a separate dissolution trigger. **Deferral: E-9 substrate accessor bootstrap rewrite (substrate, DB-14 follow-on).** DB-14 substrate accessor callables currently keep `ArrowBody::Unparsed` through bootstrap; emitters pair accessor declarations with per-target realizations via `SubstrateAccessorBinding` (see `bootstrap.rs` DB-14 comment — target-specific realization choice cannot collapse to one id at `Dag::new()` without a redesign). **`INVARIANTS.md` §E-9** requires that external realization appear only as `ArrowBody::ExternalRealization(ref)` on the Arrow, to a target-neutral marker, with per-target specs resolving from that marker — no second “externality” channel. **Dissolution trigger:** a follow-up PR extends bootstrap (or one materialization pass) to rewrite accessor Arrow bodies from `Unparsed` to `ExternalRealization(accessor_marker_id)` for each declared substrate accessor, preserving multi-target resolution through the marker + spec tables (structurally parallel to `materialize_pipeline_realizations`). Until that lands, DB-16 and `parse.rs`/`dag.rs` DB-16-scoped comments avoid framing accessor `Unparsed` as a second legitimate steady-state meaning alongside parse lag. Design: [design-substrate-external-primitives.md](docs/design-substrate-external-primitives.md) (DB-14), E-9. @@ -25,7 +25,7 @@ Closed (DB-16, PR #522): Follow-up (not blocking): emission for narrowed ports currently errors if `emit_rust` is invoked on a DAG whose narrow ports lack a producer. Acceptable today because Lane 1e's single-emitter consolidation hasn't landed and the 3a.3 acceptance is compile-only; wire a Bind-alias or emission-local name shim alongside Lane 1e when it lands. DB-16's substituted-refined carriers inherit the same narrowed-port shim requirement. - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three dissolution stories, not one.** DB-16 / PR #524: **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — ordering text read by `pipeline_authority` until a structural pipeline-order fact supersedes span extraction — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete `compile`’s span authority or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/db-history/db-17.md b/docs/db-history/db-17.md index 6a9fc84feba..248e408330d 100644 --- a/docs/db-history/db-17.md +++ b/docs/db-history/db-17.md @@ -6,7 +6,6 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | -| `ArrowBody::Unparsed` (**case 2c** — `pipeline.dag` `fn compile` ordering text) | Structural pipeline-order carrier | First-class ordered stage list (or successor substrate) supersedes `compile` body-span parsing | **Not** the M2 grammar milestone — dissolution per [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) case 2c; `pipeline_compile_order_stage_names` is the reader today | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -17,7 +16,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | `encoding_meet` / `encoding_join` (Rust fns) | Track 8 Phase 2 (user-defined generic emission) | User-defined generic emission for `Lattice` instance | **Compiler-source ratchet** (temporary; becomes lens-able when compiler.dag self-hosts and emission-generated code replaces these hand-written fns) | - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three dissolution stories, not one.** DB-16 / PR #524: **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — ordering text read by `pipeline_authority` until a structural pipeline-order fact supersedes span extraction — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete `compile`’s span authority or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/design-fn-external-body-reconciliation.md b/docs/design-fn-external-body-reconciliation.md index 0c89d5e14f4..5a763e3eb25 100644 --- a/docs/design-fn-external-body-reconciliation.md +++ b/docs/design-fn-external-body-reconciliation.md @@ -48,11 +48,13 @@ Substrate accessor callables (DB-14) are **out of scope** for this design doc: d ### Case 2c: `compile` orchestrator (ordering authority) -`pipeline.dag` also declares `fn compile(...) -> String { parse \n lower \n infer \n ... }`. Like other block-bodied compiler fns, it parses as `FnExternalBody` → `ArrowBody::Unparsed(body_span)`. There is **no** `PipelineStageBinding` for `compile` itself — only per-stage fns get that rewrite — so **`Unparsed` persists for `compile` after bootstrap.** Downstream, `pipeline_compile_order_stage_names` reads **`compile`'s body span** as the authoritative ordering surface for which stages exist and in what sequence. This is not case 1 parse lag: the body is intentional structured text consumed by the compiler, not std/ grammar debt waiting for M2. +`pipeline.dag` also declares `fn compile(...) -> String { parse \n lower \n infer \n ... }`. Like other block-bodied compiler fns, it parses as `FnExternalBody` → `ArrowBody::Unparsed(body_span)`. There is **no** `PipelineStageBinding` for `compile` itself — only per-stage fns get that rewrite — so **`Unparsed` persists for `compile` after bootstrap.** -**Receipt — terminal (bootstrap-range):** 2c is **not** an interim bridge to `ExternalRealization` (nothing to “realize” for `compile` itself). For today’s substrate, the span is the **intended terminal carrier** of ordering facts at the `Arrow → body` edge — the meeting point is still structural (`Unparsed` holds the span token), but the *role* is compiler ordering authority, not user execution body. +**Runtime authority is structural (PR #637).** Pipeline stage order is read from the declaration order of `PipelineStageBinding` records in the Dag. `ordered_pipeline_stages` walks `dag.declarations()`, filters by `meta_tag == PipelineStageBinding`, and returns them in declaration order. No text slicing, no `body_span` on the runtime path. -**Dissolution trigger (future):** introduce a **first-class structural representation** of pipeline stage order (e.g. ordered data in `pipeline.dag` or a dedicated substrate carrier) that `pipeline_authority` can read without scraping `compile`'s text; migrate consumers, then retire span-based extraction. **Not** “wait for M2 to parse `compile` as `SurfaceExpr`” — that would confuse ordering metadata with user grammar growth. +**`compile` body role: surface contract, fail-closed cross-check.** The `fn compile { parse \n lower \n ... }` body is the **human-readable pipeline contract**: a reader sees the pipeline in one glance rather than reconstructing it from a binding table. `ordered_pipeline_stages` fail-closes (`reconcile_with_compile_body`) on any drift between the body and the bindings, so the two surfaces cannot silently diverge. Bindings are the single **runtime** authority (P2); the body is the **surface** the bindings commit to; the reconcile keeps that commitment honest (P3). + +**Receipt — terminal (bootstrap-range):** 2c is **not** an interim bridge to `ExternalRealization` (nothing to “realize” for `compile` itself). The body is **not** a scaffold awaiting dissolution — it is the intended terminal human-readable carrier of the pipeline contract. The prior doc framed span-based extraction as interim; PR #637 made the structural read the runtime authority and retained the body as the surface contract, not as ordering truth. ### Why the conflation is a real risk @@ -80,7 +82,7 @@ to something like: > > **Case 2a — target-native (per-stage).** Compiler-internal fns whose body is a host runtime (e.g., pipeline.dag's `fn parse(...) -> Dag { host parse }`). Dissolves via bootstrap: `PipelineStageBinding` rewrites the Arrow body from `Unparsed` to `ExternalRealization(realization_id)`. > -> **Case 2c — `compile` orchestrator.** `fn compile(...) { ... }` in `pipeline.dag`: **`Unparsed` persists**; `pipeline_compile_order_stage_names` reads the body span for ordering authority. Not dissolved by `PipelineStageBinding`. +> **Case 2c — `compile` orchestrator.** `fn compile(...) { ... }` in `pipeline.dag`: **`Unparsed` persists**. Runtime ordering authority is the declaration order of `PipelineStageBinding` records (`ordered_pipeline_stages`); the body is the human-readable pipeline contract, cross-checked fail-closed against the bindings (`reconcile_with_compile_body`). Not dissolved by `PipelineStageBinding` — the body is terminal surface, not a scaffold. > > The parser does not distinguish these cases — all are 'block body that isn't a SurfaceExpr.' The disambiguator is **downstream role** (binding rewrite vs ordering authority vs parse lag), not "no binding ⇒ case 1." @@ -100,7 +102,7 @@ fn pipeline_stages_lower_to_external_realization_not_unparsed() { let stages = v3_compiler::pipeline_compile_order_stage_names() .expect("pipeline.dag `compile` body must list stages"); for stage in stages { - // Names come from `compile`'s body span — excludes `compile` itself (case 2c: Unparsed persists). + // Names come from `PipelineStageBinding` declaration order — excludes `compile` itself (case 2c: Unparsed persists). let decl = dag.declaration_by_name(&stage).unwrap(); // ... assert ArrowBody::ExternalRealization for each listed stage } @@ -120,7 +122,7 @@ Considered: split `SurfaceItem::FnExternalBody` into two variants (`FnExternalBo - The divergence genuinely happens downstream, at bootstrap. Splitting at parse time forces the parser to know about compiler-internal concepts (pipeline stages) that are properly below it. - One parse-time variant + two bootstrap paths (parser-growth rewrite of `Unparsed` vs `PipelineStageBinding`-style rewrite to `ExternalRealization`) is the right layering. -The proper disambiguator is **downstream bootstrap / authority role**, not a single boolean: `PipelineStageBinding` for per-stage pipeline fns; `pipeline_compile_order_stage_names` + persisted `Unparsed` for `compile`; absence of those *and* no special pipeline role implies parse lag (case 1). +The proper disambiguator is **downstream bootstrap / authority role**, not a single boolean: `PipelineStageBinding` for per-stage pipeline fns; persisted `Unparsed` + fail-closed reconcile against binding order for `compile`; absence of those *and* no special pipeline role implies parse lag (case 1). --- diff --git a/docs/history/roadmap-scheduled-deletions.md b/docs/history/roadmap-scheduled-deletions.md index 07e92b85c32..db85cc2ad5e 100644 --- a/docs/history/roadmap-scheduled-deletions.md +++ b/docs/history/roadmap-scheduled-deletions.md @@ -20,7 +20,6 @@ Each scheduled deletion names one enforcement path. **Grep over source code is n |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | -| `ArrowBody::Unparsed` (**case 2c** — `pipeline.dag` `fn compile` ordering text) | Structural pipeline-order carrier | First-class ordered stage list (or successor substrate) supersedes `compile` body-span parsing | **Not** the M2 grammar milestone — dissolution per [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) case 2c; `pipeline_compile_order_stage_names` is the reader today | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -33,7 +32,7 @@ Each scheduled deletion names one enforcement path. **Grep over source code is n ### Notes on specific rows - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three dissolution stories, not one.** DB-16 / PR #524: **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — ordering text read by `pipeline_authority` until a structural pipeline-order fact supersedes span extraction — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete `compile`’s span authority or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`). **2c is terminal, not scheduled for deletion** — the body is the pipeline contract, the bindings are the structural authority — and is therefore no longer listed in the table above. Case 1 and **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands) remain scheduled. The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. From f5afd563a4f5be209ebf22686adf9744559748dd Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 03:02:49 -0400 Subject: [PATCH 08/11] WIP: Pipeline Authority --- docs/db-history/db-14.md | 3 ++- docs/db-history/db-16.md | 2 +- docs/history/roadmap-scheduled-deletions.md | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/db-history/db-14.md b/docs/db-history/db-14.md index 34c0b7c0277..0ecfc2183b1 100644 --- a/docs/db-history/db-14.md +++ b/docs/db-history/db-14.md @@ -12,6 +12,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | +| `ArrowBody::Unparsed` + `reconcile_with_compile_body` bridge (**case 2c** — `pipeline.dag` `fn compile` body vs `PipelineStageBinding` order) | Derivation: one carrier authored, the other emitted at regen/emit so there is only one authored surface | Requires a regen/emit path producing the `compile` body from binding order (or an inverse substrate making `compile` the sole authored carrier with bindings derived) | **Runtime reconcile today** — `reconcile_with_compile_body` in `pipeline_authority.rs` fails closed on drift; clears when the two carriers are unified to a single authored source | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -22,7 +23,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | `encoding_meet` / `encoding_join` (Rust fns) | Track 8 Phase 2 (user-defined generic emission) | User-defined generic emission for `Lattice` instance | **Compiler-source ratchet** (temporary; becomes lens-able when compiler.dag self-hosts and emission-generated code replaces these hand-written fns) | - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is still debt in bridge shape (two authored carriers, reconcile is the behavioral bridge) until derivation collapses it to a single authored source — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/db-history/db-16.md b/docs/db-history/db-16.md index 638c2677902..8722a5c7705 100644 --- a/docs/db-history/db-16.md +++ b/docs/db-history/db-16.md @@ -10,7 +10,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | 3a.4 surface generics (DB-12, S) | ✅ Shipped | PR #496 | Tests-only landing; infrastructure already wired. | | 3a.5 Disj dotted-path (DB-13, S) | ✅ Shipped | PR #496 | Tests-only landing; infrastructure already wired. | -**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — terminal surface, not a scaffold)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1** only; **2c** is terminal (no dissolution scheduled — body is the pipeline contract, structural authority is the bindings); accessor interim has a separate dissolution trigger. +**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — bridge shape: two authored carriers kept consistent by reconcile, scheduled until derivation collapses them to one authored source)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1**; **2c + `reconcile_with_compile_body` bridge** dissolves when derivation collapses the `compile` body and the bindings to a single authored source (PR #637 narrowed the prior body-span authority to runtime-structural + reconcile, but the two-carrier bridge itself stays scheduled); accessor interim has a separate dissolution trigger. **Deferral: E-9 substrate accessor bootstrap rewrite (substrate, DB-14 follow-on).** DB-14 substrate accessor callables currently keep `ArrowBody::Unparsed` through bootstrap; emitters pair accessor declarations with per-target realizations via `SubstrateAccessorBinding` (see `bootstrap.rs` DB-14 comment — target-specific realization choice cannot collapse to one id at `Dag::new()` without a redesign). **`INVARIANTS.md` §E-9** requires that external realization appear only as `ArrowBody::ExternalRealization(ref)` on the Arrow, to a target-neutral marker, with per-target specs resolving from that marker — no second “externality” channel. **Dissolution trigger:** a follow-up PR extends bootstrap (or one materialization pass) to rewrite accessor Arrow bodies from `Unparsed` to `ExternalRealization(accessor_marker_id)` for each declared substrate accessor, preserving multi-target resolution through the marker + spec tables (structurally parallel to `materialize_pipeline_realizations`). Until that lands, DB-16 and `parse.rs`/`dag.rs` DB-16-scoped comments avoid framing accessor `Unparsed` as a second legitimate steady-state meaning alongside parse lag. Design: [design-substrate-external-primitives.md](docs/design-substrate-external-primitives.md) (DB-14), E-9. diff --git a/docs/history/roadmap-scheduled-deletions.md b/docs/history/roadmap-scheduled-deletions.md index db85cc2ad5e..b5f5066fe42 100644 --- a/docs/history/roadmap-scheduled-deletions.md +++ b/docs/history/roadmap-scheduled-deletions.md @@ -20,6 +20,7 @@ Each scheduled deletion names one enforcement path. **Grep over source code is n |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | +| `ArrowBody::Unparsed` + `reconcile_with_compile_body` bridge (**case 2c** — `pipeline.dag` `fn compile` body vs `PipelineStageBinding` order) | Derivation: one carrier authored, the other emitted (e.g., `fn compile` body emitted from binding declaration order at regen/emit, or bindings emitted from a single ordered carrier) so there is only one authored surface | Requires a regen/emit path that produces the `compile` body from the structural order (or an inverse substrate that lets `compile` itself be the single authored carrier with bindings derived) | **Runtime reconcile today** — `reconcile_with_compile_body` in `pipeline_authority.rs` fails closed on drift; clears when the two carriers are unified to a single authored source | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -32,7 +33,7 @@ Each scheduled deletion names one enforcement path. **Grep over source code is n ### Notes on specific rows - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`). **2c is terminal, not scheduled for deletion** — the body is the pipeline contract, the bindings are the structural authority — and is therefore no longer listed in the table above. Case 1 and **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands) remain scheduled. The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`). **2c is still debt**, just in its post-dissolution-step shape: the reconcile is a behavioral bridge between two authored carriers of stage order, and the row stays scheduled until derivation collapses it to a single authored source. Case 1 and **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands) remain scheduled for their own triggers. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. From 68e9c65fafc63125d7724aad541d2839e6232417 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 03:04:37 -0400 Subject: [PATCH 09/11] docs: reclassify case 2c as bridge-shape debt, not terminal Address REQUEST_CHANGES on #637 from openai-pro review. Two authored carriers (fn compile body and PipelineStageBinding order) kept consistent by reconcile_with_compile_body is still bridge shape, not single-authority. Restore case 2c in the scheduled-deletion tables with its post-#637 dissolution trigger: derivation collapses the two carriers to a single authored source (e.g., regen emits the compile body from binding order). Updates dag.rs comment, design-fn-external-body-reconciliation.md, db-14.md, db-16.md, db-17.md, and roadmap-scheduled-deletions.md to describe 2c as bridge shape with a named derivation trigger, not as terminal surface. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/db-history/db-16.md | 2 +- docs/db-history/db-17.md | 3 +- .../design-fn-external-body-reconciliation.md | 2 +- src/v3/compiler/src/dag.rs | 30 ++++++++++++------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/db-history/db-16.md b/docs/db-history/db-16.md index 8722a5c7705..239d250685a 100644 --- a/docs/db-history/db-16.md +++ b/docs/db-history/db-16.md @@ -25,7 +25,7 @@ Closed (DB-16, PR #522): Follow-up (not blocking): emission for narrowed ports currently errors if `emit_rust` is invoked on a DAG whose narrow ports lack a producer. Acceptable today because Lane 1e's single-emitter consolidation hasn't landed and the 3a.3 acceptance is compile-only; wire a Bind-alias or emission-local name shim alongside Lane 1e when it lands. DB-16's substituted-refined carriers inherit the same narrowed-port shim requirement. - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete the `compile` body carrier or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c remains scheduled debt in bridge shape (two authored carriers, reconcile is the behavioral bridge) until derivation collapses them to a single authored source — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** by itself delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/db-history/db-17.md b/docs/db-history/db-17.md index 248e408330d..dedc04bc794 100644 --- a/docs/db-history/db-17.md +++ b/docs/db-history/db-17.md @@ -6,6 +6,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD |---|---|---|---| | `ArrowBody::Pending` | M3 ratchet | Every realization arrow bound to `ExternalRealization` | **Lens** — writable now; walk `d.nodes` for Arrow declarations with `body = Pending` reachable from user-range roots | | `ArrowBody::Unparsed` (**case 1** — `FnExternalBody` parse lag in std/) | M2+ parser surface | Match / pipe / lambda / block-body parsing so `FnExternalBody` lowers away | **Lens** — user-range + applicable std/ per R14; ratchet fires when block bodies become `SurfaceExpr` | +| `ArrowBody::Unparsed` + `reconcile_with_compile_body` bridge (**case 2c** — `pipeline.dag` `fn compile` body vs `PipelineStageBinding` order) | Derivation: one carrier authored, the other emitted at regen/emit so there is only one authored surface | Requires a regen/emit path producing the `compile` body from binding order (or an inverse substrate making `compile` the sole authored carrier with bindings derived) | **Runtime reconcile today** — `reconcile_with_compile_body` in `pipeline_authority.rs` fails closed on drift; clears when the two carriers are unified to a single authored source | | `ArrowBody::Unparsed` (**DB-14 accessor interim**, pre–E-9) | E-9 bootstrap materialization | **Deferral: E-9 substrate accessor bootstrap rewrite** below — `ExternalRealization(marker)` on `Arrow.body` | Clears with that deferral (not the case-1 lens) | | `ValueBody::Unparsed` | M2+ parser surface | Record / map / list literal parsing | **Lens** — writable now; walk `data` declarations | | `TransformTarget::Operator` | M2+ parser surface | Operator desugar into algebra-field calls | **Lens** — writable now; walk Transform targets | @@ -16,7 +17,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROAD | `encoding_meet` / `encoding_join` (Rust fns) | Track 8 Phase 2 (user-defined generic emission) | User-defined generic emission for `Lattice` instance | **Compiler-source ratchet** (temporary; becomes lens-able when compiler.dag self-hosts and emission-generated code replaces these hand-written fns) | - **`ArrowBody::Pending` lens-name-filter follow-up (#518) is cleared on PR #548.** The remaining anonymous operator-fallback site was reclassified to `ArrowBody::NoBody`, and `lens_structural_resolution` now keys on surviving `Arrow(Pending)` structurally rather than through a `Declaration.name` proxy. The scheduled-deletions row above stays live until the variant itself is removed; this follow-up only cleared the false-positive/false-negative gap around anonymous arrows. -- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is now structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c is terminal, not dissolving — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. +- **`ArrowBody::Unparsed` is three stories, not one.** DB-16 / PR #524 (and PR #637 structural-read cleanup): **case 1** (parse lag, M2 grammar, lens ratchet) is separate from **`pipeline.dag` `compile` (case 2c)** — runtime ordering is structural (`PipelineStageBinding` declaration order via `ordered_pipeline_stages`); the `compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings (`reconcile_with_compile_body`); 2c remains scheduled debt in bridge shape (two authored carriers, reconcile is the behavioral bridge) until derivation collapses them to a single authored source — and from **DB-14 accessors** (interim `Unparsed` until the **E-9** deferral lands). The M2 milestone deletes case-1 uses; it does **not** delete the `compile` body carrier or accessor interim encoding. - **`declaration_by_name` is a helper name, not a single debt class.** The function at `dag.rs:1459` has 83 call sites that split into distinct classes with separate dissolution paths. [DB-17 (reference-resolution provenance)](docs/design-reference-resolution-provenance.md) narrows its scope to **only the user-range AtomPayload fallback class** (lowering produces `ResolvedByName(id)` when a structural walk falls back to name lookup). DB-17's lens walks user-range AtomPayloads; compiler-internal call sites (bootstrap substrate_markers in `dag.rs`, pipeline authority wiring, emitter algebra lookups in `emit_go.rs`/`emit_python.rs`) are a separate compiler-source class that dissolves at self-hosting (or via per-site substrate amendments — e.g., substrate_markers becoming typed edges rather than name-keyed caches). Keying the scheduled deletion to the helper name conflates these. - **`Node.name` cluster**: closed in the current v3 compiler/substrate. The old generic `Node.name` carrier is gone; surviving `name` fields are declaration names and `BindNode.name`, which are distinct facts. Some older design docs still describe the deletion as in-flight; treat those as stale prose, not live debt. - **`keyword_to_name` (recon outcome 2026-04-17, no row added):** the bare `keyword_to_name` was renamed to `tok_keyword_to_name` during v2 Phase 0 parser restructure — see `src/v2/parser-design.md:403-408`. The new name still carries the scaffold (parser-side keyword-name logic that duplicates facts from the tokenizer's `SyntaxSpec`), but it lives in `src/v2/02_parse.dag:455` and `src/v2/stage0/src/v2_compiler_parse.rs:1321` — **v2 code**, not v3. Grep confirms zero equivalents in `src/v3/`. V2 is the reference-implementation / test oracle per `ROADMAP.md` §"Sketch vs Oracle framing"; v2 scaffolds dissolve when v3 supersedes v2 entirely, not individually. The v3 Scheduled Deletions list tracks v3-scope scaffolds only. diff --git a/docs/design-fn-external-body-reconciliation.md b/docs/design-fn-external-body-reconciliation.md index 5a763e3eb25..077b19904b7 100644 --- a/docs/design-fn-external-body-reconciliation.md +++ b/docs/design-fn-external-body-reconciliation.md @@ -54,7 +54,7 @@ Substrate accessor callables (DB-14) are **out of scope** for this design doc: d **`compile` body role: surface contract, fail-closed cross-check.** The `fn compile { parse \n lower \n ... }` body is the **human-readable pipeline contract**: a reader sees the pipeline in one glance rather than reconstructing it from a binding table. `ordered_pipeline_stages` fail-closes (`reconcile_with_compile_body`) on any drift between the body and the bindings, so the two surfaces cannot silently diverge. Bindings are the single **runtime** authority (P2); the body is the **surface** the bindings commit to; the reconcile keeps that commitment honest (P3). -**Receipt — terminal (bootstrap-range):** 2c is **not** an interim bridge to `ExternalRealization` (nothing to “realize” for `compile` itself). The body is **not** a scaffold awaiting dissolution — it is the intended terminal human-readable carrier of the pipeline contract. The prior doc framed span-based extraction as interim; PR #637 made the structural read the runtime authority and retained the body as the surface contract, not as ordering truth. +**Receipt — bridge shape (scheduled, not terminal):** 2c is **not** an interim bridge to `ExternalRealization` (nothing to "realize" for `compile` itself). PR #637 narrowed the prior "body-span is the ordering authority" framing to "bindings are the runtime authority; body is the human-readable contract; reconcile keeps them consistent." That is a real P2/P3 improvement over the prior shape, but the two authored carriers (`compile` body and `PipelineStageBinding` declaration order) still need a behavioral bridge (`reconcile_with_compile_body`) to stay consistent — so 2c remains scheduled debt. **Dissolution trigger:** derivation collapses the two carriers to a single authored source — e.g., a regen/emit step writes the `compile` body from binding declaration order at regen time (or an inverse substrate makes `compile` the sole authored carrier with bindings derived from it). When one surface is emitted from the other, the reconcile retires with it. ### Why the conflation is a real risk diff --git a/src/v3/compiler/src/dag.rs b/src/v3/compiler/src/dag.rs index 67b5ebb4f1c..9bab3d71dc6 100644 --- a/src/v3/compiler/src/dag.rs +++ b/src/v3/compiler/src/dag.rs @@ -583,14 +583,19 @@ include!("dag_scalar_generated.rs"); /// the same ordering and `ordered_pipeline_stages` fail-closes on any drift /// between the two (`reconcile_with_compile_body`) so the bindings remain /// the single runtime authority without silently diverging from the -/// orchestrator surface. The `compile` body is **terminal**, not a scaffold -/// awaiting dissolution: it is the human-readable pipeline contract that -/// the binding records satisfy — a reader sees the pipeline in one glance -/// as `{ parse; lower; infer; ... }` rather than reconstructing it from a -/// binding table. The bindings are the runtime authority; the body is the -/// surface the bindings commit to. The fail-closed reconcile is what keeps -/// that contract honest (P3) without letting the body become a second -/// runtime source (P2). The signature still flows forward through the +/// orchestrator surface. The `compile` body is the human-readable pipeline +/// contract that the binding records satisfy — a reader sees the pipeline +/// in one glance as `{ parse; lower; infer; ... }` rather than +/// reconstructing it from a binding table. The bindings are the runtime +/// authority; the body is the surface the bindings commit to; the +/// fail-closed reconcile keeps that contract honest (P3) without letting +/// the body become a second runtime source (P2). This is **bridge shape**, +/// not terminal: two authored carriers kept consistent by reconcile. PR #637 +/// narrowed the prior body-span-as-authority shape; the bridge itself +/// remains scheduled debt (see `docs/history/roadmap-scheduled-deletions.md` +/// case 2c) until derivation collapses the two carriers to a single +/// authored source — e.g., regen emits the `compile` body from binding +/// declaration order. The signature still flows forward through the /// declaration table so callers can type-check against it, and the body /// source span is preserved so M2+ parser extensions can reach in for case 1. /// **User-range boundary:** `reject_user_unparsed_scaffolds` in @@ -606,9 +611,12 @@ include!("dag_scalar_generated.rs"); /// `ExternalRealization`, `NoBody`). The 5-variant shape is a /// transition state: `Pending` and **case-1** `Unparsed` are scaffolds with /// named dissolution (M3 / M2 grammar). **`Unparsed` on `pipeline.dag`'s -/// `compile` (DB-16 case 2c)** is different — **persistent bootstrap-range -/// carrier** whose body span is no longer the ordering authority (structural -/// binding order is); it does **not** share case 1’s “wait for M2 parser” story. User-range +/// `compile` (DB-16 case 2c)** is different — a **bootstrap-range carrier +/// in bridge shape**: body span is no longer the ordering authority +/// (structural binding order is), but two authored carriers still require +/// `reconcile_with_compile_body` to stay consistent, so 2c remains scheduled +/// debt with its own dissolution trigger (derivation, not the M2 parser +/// milestone). User-range /// `Unparsed` stays gated (R14). #[derive(Debug, Clone)] pub enum ArrowBody { From 3585634dc0a965ee3e0b9c75ff3988c9e2130178 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 03:05:11 -0400 Subject: [PATCH 10/11] docs: drop stale "terminal" phrasing in db-14.md header Missed a copy of the old framing in the DB-14 narrative header. Align with the rest: 2c is bridge shape, scheduled until derivation. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/db-history/db-14.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/db-history/db-14.md b/docs/db-history/db-14.md index 0ecfc2183b1..46ebc2eb66f 100644 --- a/docs/db-history/db-14.md +++ b/docs/db-history/db-14.md @@ -2,7 +2,7 @@ Collected receipts and reconciliation notes moved out of [ROADMAP.md](../../ROADMAP.md). -**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — terminal surface, not a scaffold)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is split (case 1 vs DB-14 interim) — the M2 grammar milestone removes **case 1** only; accessor interim has a separate dissolution trigger; **2c is terminal** (no dissolution scheduled). +**DB-16 (`FnExternalBody` reconciliation, PR #524; structural-read cleanup, PR #637).** [design-fn-external-body-reconciliation.md](docs/design-fn-external-body-reconciliation.md) — documents the **`FnExternalBody` / `ArrowBody::Unparsed` split for pipeline work only:** parse lag (case 1), pipeline host stages (case 2a → bootstrap `ExternalRealization` via `PipelineStageBinding`), **`pipeline.dag` `compile` (case 2c → `Unparsed` persists; runtime ordering reads `PipelineStageBinding` declaration order via `ordered_pipeline_stages`; `fn compile` body is the human-readable pipeline contract, fail-closed reconciled against the bindings via `reconcile_with_compile_body` — bridge shape, scheduled until derivation collapses the two authored carriers to one)**. Invariant `pipeline_stages_lower_to_external_realization_not_unparsed` derives stage names from `pipeline_compile_order_stage_names()` (same authority as bootstrap; excludes `compile` itself). Intentionally **does not** canonically document substrate accessor `Unparsed` semantics (see deferral below). **Scheduled deletions:** `ArrowBody::Unparsed` is **split into three rows** (case 1 vs `compile` 2c + reconcile bridge vs DB-14 interim) in §Scheduled deletions — the M2 grammar milestone removes **case 1**; **2c** dissolves when derivation collapses the `compile` body and bindings to a single authored source; accessor interim has its own trigger. **Deferral: E-9 substrate accessor bootstrap rewrite (substrate, DB-14 follow-on).** DB-14 substrate accessor callables currently keep `ArrowBody::Unparsed` through bootstrap; emitters pair accessor declarations with per-target realizations via `SubstrateAccessorBinding` (see `bootstrap.rs` DB-14 comment — target-specific realization choice cannot collapse to one id at `Dag::new()` without a redesign). **`INVARIANTS.md` §E-9** requires that external realization appear only as `ArrowBody::ExternalRealization(ref)` on the Arrow, to a target-neutral marker, with per-target specs resolving from that marker — no second “externality” channel. **Dissolution trigger:** a follow-up PR extends bootstrap (or one materialization pass) to rewrite accessor Arrow bodies from `Unparsed` to `ExternalRealization(accessor_marker_id)` for each declared substrate accessor, preserving multi-target resolution through the marker + spec tables (structurally parallel to `materialize_pipeline_realizations`). Until that lands, DB-16 and `parse.rs`/`dag.rs` DB-16-scoped comments avoid framing accessor `Unparsed` as a second legitimate steady-state meaning alongside parse lag. Design: [design-substrate-external-primitives.md](docs/design-substrate-external-primitives.md) (DB-14), E-9. From cd6668e734f06d68f7ab00a5f2230ae7ca610dd4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Apr 2026 03:09:16 -0400 Subject: [PATCH 11/11] docs: drop stale "terminal surface" phrasing in design doc quoted block Last straggler of the terminal-vs-bridge reclassification. The quoted recipe-style block describing case 2c still said "terminal surface, not a scaffold." Align with the rest of the corpus: 2c is bridge shape, scheduled until derivation. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/design-fn-external-body-reconciliation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design-fn-external-body-reconciliation.md b/docs/design-fn-external-body-reconciliation.md index 077b19904b7..7e8e02cde49 100644 --- a/docs/design-fn-external-body-reconciliation.md +++ b/docs/design-fn-external-body-reconciliation.md @@ -82,7 +82,7 @@ to something like: > > **Case 2a — target-native (per-stage).** Compiler-internal fns whose body is a host runtime (e.g., pipeline.dag's `fn parse(...) -> Dag { host parse }`). Dissolves via bootstrap: `PipelineStageBinding` rewrites the Arrow body from `Unparsed` to `ExternalRealization(realization_id)`. > -> **Case 2c — `compile` orchestrator.** `fn compile(...) { ... }` in `pipeline.dag`: **`Unparsed` persists**. Runtime ordering authority is the declaration order of `PipelineStageBinding` records (`ordered_pipeline_stages`); the body is the human-readable pipeline contract, cross-checked fail-closed against the bindings (`reconcile_with_compile_body`). Not dissolved by `PipelineStageBinding` — the body is terminal surface, not a scaffold. +> **Case 2c — `compile` orchestrator.** `fn compile(...) { ... }` in `pipeline.dag`: **`Unparsed` persists**. Runtime ordering authority is the declaration order of `PipelineStageBinding` records (`ordered_pipeline_stages`); the body is the human-readable pipeline contract, cross-checked fail-closed against the bindings (`reconcile_with_compile_body`). Not dissolved by `PipelineStageBinding`: 2c is **bridge shape** (two authored carriers kept consistent by reconcile), scheduled until derivation collapses them to one authored source. > > The parser does not distinguish these cases — all are 'block body that isn't a SurfaceExpr.' The disambiguator is **downstream role** (binding rewrite vs ordering authority vs parse lag), not "no binding ⇒ case 1."