diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a3cecd3..ea6f62460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `persistence_postgres` typed `text_segment` SQL: insert/lookup of exact UTF-8 half-open byte spans on the existing `0006` table, cutoff-eligible document reads (`available_time <= knowledge_cutoff`), and live recovery of a known `hello` span. No new migration number (`#45` still owns `0007`). - Hourly contextual-orchestrator discovery records all provider models but routes OpenCode only through general-chat candidates, excluding embedding, image, reranker, transcription, moderation, safety, and other endpoint-only identifiers before price selection. - Live `docs/product-technical-gap-baseline.md` mapping operator-visible gaps to protected-main maturity, exact current PR/issue state, stacked delivery order, diff --git a/crates/persistence_postgres/src/error.rs b/crates/persistence_postgres/src/error.rs index bf5f7cd73..add355391 100644 --- a/crates/persistence_postgres/src/error.rs +++ b/crates/persistence_postgres/src/error.rs @@ -42,6 +42,8 @@ pub enum PersistenceError { ConcurrentWriteConflict, /// A restored snapshot failed integrity revalidation and is not usable. RestoreIntegrityFailed, + /// A text segment had a negative or inverted UTF-8 byte span. + InvalidTextSegment, /// A retention, hold, deletion, or tombstone record failed closed validation. InvalidRetentionLifecycle, /// An active legal hold blocked completed deletion. @@ -71,6 +73,7 @@ impl fmt::Display for PersistenceError { Self::InvalidAuditEvent => "invalid audit event", Self::ConcurrentWriteConflict => "concurrent write conflict", Self::RestoreIntegrityFailed => "restore integrity failed", + Self::InvalidTextSegment => "invalid text segment", Self::InvalidRetentionLifecycle => "invalid retention lifecycle", Self::LegalHoldBlocksDeletion => "legal hold blocks deletion", Self::UngovernedEvidenceRestore => "ungoverned evidence restore", @@ -209,6 +212,10 @@ mod tests { PersistenceError::RestoreIntegrityFailed.to_string(), "restore integrity failed" ); + assert_eq!( + PersistenceError::InvalidTextSegment.to_string(), + "invalid text segment" + ); assert_eq!( PersistenceError::InvalidRetentionLifecycle.to_string(), "invalid retention lifecycle" diff --git a/crates/persistence_postgres/src/lib.rs b/crates/persistence_postgres/src/lib.rs index 1b008eb62..02685e043 100644 --- a/crates/persistence_postgres/src/lib.rs +++ b/crates/persistence_postgres/src/lib.rs @@ -15,6 +15,8 @@ //! contracts chain immutable run identities to those manifests. Typed //! membership-assignment SQL (migration `0006`) replaces the polymorphic 0001 stub so documents //! can belong to multiple entities and projects without atomistic collapse. +//! Typed `text_segment` SQL persists exact UTF-8 byte spans and cutoff-eligible +//! document lookups so segment-level membership is not raw SQL. //! Concurrent document revises use one transactional `DO` block that requires //! exactly one open row to close, and live `SQLx` maps racing SQLSTATEs onto //! typed conflict errors. Restore integrity probes refuse to mark analytical @@ -42,6 +44,7 @@ mod naming; mod relation_sql; mod restore_integrity; mod retention_sql; +mod segment_sql; mod sql_session; mod sqlx_gate; #[cfg(feature = "live-sqlx")] @@ -194,6 +197,14 @@ pub use retention_sql::release_legal_hold_sql; pub use retention_sql::select_active_analysis_document_sql; /// Render supersede SQL for a successive retention policy. pub use retention_sql::supersede_retention_policy_sql; +/// Exact-span text segment row. +pub use segment_sql::TextSegmentRecord; +/// Render insert SQL for a validated text segment. +pub use segment_sql::insert_text_segment_sql; +/// Render selection SQL for a text segment by primary key. +pub use segment_sql::select_text_segment_by_id_sql; +/// Render cutoff-eligible text-segment selection for one document. +pub use segment_sql::select_text_segments_for_document_as_of_sql; /// Recording SQL transport for offline contract tests. pub use sql_session::RecordingSqlSession; /// Live SQL transport contract. diff --git a/crates/persistence_postgres/src/live_repository.rs b/crates/persistence_postgres/src/live_repository.rs index 82d7581a6..5c092b576 100644 --- a/crates/persistence_postgres/src/live_repository.rs +++ b/crates/persistence_postgres/src/live_repository.rs @@ -35,10 +35,14 @@ use crate::retention_sql::{ insert_evidence_tombstone_sql, insert_legal_hold_sql, insert_retention_policy_sql, select_active_analysis_document_sql, }; +use crate::segment_sql::{ + TextSegmentRecord, insert_text_segment_sql, select_text_segment_by_id_sql, + select_text_segments_for_document_as_of_sql, +}; use crate::sql_session::{SqlSession, apply_sql_batch}; use crate::tenant_session::set_session_tenant_sql; use crate::{MigrationContractError, PersistenceError}; -use temporal_core::{EventTime, SystemTime}; +use temporal_core::{EventTime, KnowledgeCutoff, SystemTime}; use uuid::Uuid; /// Fail-closed live document/audit repository backed by [`SqlSession`]. @@ -320,6 +324,47 @@ impl LiveDocumentRepository { self.session.execute(&sql) } + /// Insert an exact-span text segment under the active tenant. + /// + /// # Errors + /// + /// Returns inverted/negative span validation or transport failures. + pub fn insert_text_segment( + &mut self, + record: &TextSegmentRecord, + ) -> Result<(), PersistenceError> { + self.bind_session_tenant(record.tenant_record_id)?; + let sql = insert_text_segment_sql(record)?; + self.session.execute(&sql) + } + + /// Look up one text segment by primary key. + /// + /// # Errors + /// + /// Returns transport failures. + pub fn submit_text_segment_by_id( + &mut self, + text_segment_id: Uuid, + ) -> Result<(), PersistenceError> { + let sql = select_text_segment_by_id_sql(text_segment_id); + self.session.execute(&sql) + } + + /// Look up cutoff-eligible text segments for one document identity. + /// + /// # Errors + /// + /// Returns transport failures. + pub fn submit_text_segments_for_document_as_of( + &mut self, + document_record_id: Uuid, + knowledge_cutoff: &KnowledgeCutoff, + ) -> Result<(), PersistenceError> { + let sql = select_text_segments_for_document_as_of_sql(document_record_id, knowledge_cutoff); + self.session.execute(&sql) + } + /// Insert a bitemporal event-instance version. /// /// # Errors @@ -523,9 +568,10 @@ mod tests { use crate::migration::MigrationCatalog; use crate::model_run_sql::{CorpusSplitManifestRecord, ModelArtifactRecord, ModelRunRecord}; use crate::relation_sql::EventRelationRecord; + use crate::segment_sql::TextSegmentRecord; use crate::sql_session::RecordingSqlSession; use crate::{MigrationContractError, PersistenceError}; - use temporal_core::{AvailableTime, EventTime, SystemTime}; + use temporal_core::{AvailableTime, EventTime, KnowledgeCutoff, SystemTime}; fn sample_record() -> DocumentRecord { DocumentRecord { @@ -670,6 +716,42 @@ mod tests { ); } + fn exercise_text_segment(repo: &mut LiveDocumentRepository) { + let segment = TextSegmentRecord { + text_segment_id: uuid::Uuid::from_u128(7), + tenant_record_id: uuid::Uuid::nil(), + document_record_id: uuid::Uuid::from_u128(11), + start_byte: 0, + end_byte: 5, + system_time: SystemTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("s"), + available_time: AvailableTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("a"), + }; + repo.insert_text_segment(&segment).expect("segment insert"); + let executed = repo.session().executed(); + let segment_bind = executed + .iter() + .rposition(|sql| sql.contains("tepp.current_tenant_record_id")) + .expect("text segment insert must bind tenant session"); + assert!(executed[segment_bind + 1].contains("INSERT INTO text_segment")); + repo.submit_text_segment_by_id(segment.text_segment_id) + .expect("segment by id"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + repo.submit_text_segments_for_document_as_of(segment.document_record_id, &cutoff) + .expect("segments as of"); + let mut inverted = segment.clone(); + inverted.end_byte = 0; + assert_eq!( + repo.insert_text_segment(&inverted), + Err(PersistenceError::InvalidTextSegment) + ); + assert!( + repo.session() + .executed() + .iter() + .any(|sql| sql.contains("INSERT INTO text_segment")) + ); + } + fn exercise_event_mention(repo: &mut LiveDocumentRepository) { let mention = EventMentionRecord { event_mention_id: uuid::Uuid::from_u128(2), @@ -987,6 +1069,7 @@ mod tests { exercise_membership_assignment(&mut repo); exercise_event_relation(&mut repo); exercise_event_mention(&mut repo); + exercise_text_segment(&mut repo); exercise_event_instance(&mut repo); exercise_source_artifact(&mut repo); exercise_retention_legal_hold(&mut repo); diff --git a/crates/persistence_postgres/src/segment_sql.rs b/crates/persistence_postgres/src/segment_sql.rs new file mode 100644 index 000000000..30b5177fa --- /dev/null +++ b/crates/persistence_postgres/src/segment_sql.rs @@ -0,0 +1,145 @@ +//! SQL contracts for exact-span `text_segment` rows (ADR 0008 / ADR 0013). + +use crate::PersistenceError; +use temporal_core::{AvailableTime, KnowledgeCutoff, SystemTime}; +use uuid::Uuid; + +/// One append-only exact-span observation on a document. +/// +/// Maps to physical `text_segment` from migration `0006`. Byte offsets are +/// half-open `[start_byte, end_byte)` over the document UTF-8 bytes. +/// `document_record_id` is required; a foreign key remains a later migration +/// (`#45` owns `0007`). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TextSegmentRecord { + /// Segment identity used by membership and mention observed units. + pub text_segment_id: Uuid, + /// Owning tenant boundary. + pub tenant_record_id: Uuid, + /// Document whose UTF-8 bytes this span indexes. + pub document_record_id: Uuid, + /// Inclusive start offset in UTF-8 bytes; must be `>= 0`. + pub start_byte: i64, + /// Exclusive end offset in UTF-8 bytes; must be `> start_byte`. + pub end_byte: i64, + /// System/record time when the span was asserted. + pub system_time: SystemTime, + /// Availability time of the span evidence. + pub available_time: AvailableTime, +} + +impl TextSegmentRecord { + /// Fail-closed half-open byte-span validation. + /// + /// # Errors + /// + /// Returns [`PersistenceError::InvalidTextSegment`] when `start_byte` is + /// negative or `end_byte` is not strictly greater than `start_byte`. + pub fn validate(&self) -> Result<(), PersistenceError> { + if self.start_byte < 0 || self.end_byte <= self.start_byte { + return Err(PersistenceError::InvalidTextSegment); + } + Ok(()) + } +} + +/// Render insert SQL for a validated text segment. +/// +/// # Errors +/// +/// Returns [`PersistenceError::InvalidTextSegment`] before any SQL is produced. +pub fn insert_text_segment_sql(record: &TextSegmentRecord) -> Result { + record.validate()?; + Ok(format!( + "INSERT INTO text_segment (\ + text_segment_id, tenant_record_id, document_record_id, \ + start_byte, end_byte, system_time, available_time\ + ) VALUES (\ + '{segment}'::uuid, '{tenant}'::uuid, '{document}'::uuid, \ + {start_byte}, {end_byte}, '{system}'::timestamptz, '{available}'::timestamptz\ + )", + segment = record.text_segment_id, + tenant = record.tenant_record_id, + document = record.document_record_id, + start_byte = record.start_byte, + end_byte = record.end_byte, + system = record.system_time.to_rfc3339(), + available = record.available_time.to_rfc3339(), + )) +} + +/// Render selection of one text segment by primary key. +#[must_use] +pub fn select_text_segment_by_id_sql(text_segment_id: Uuid) -> String { + format!( + "SELECT text_segment_id, tenant_record_id, document_record_id, \ + start_byte, end_byte, system_time, available_time \ + FROM text_segment \ + WHERE text_segment_id = '{text_segment_id}'::uuid \ + LIMIT 1" + ) +} + +/// Render cutoff-eligible segments for one document identity. +/// +/// Enforces `available_time <= knowledge_cutoff` so a historical fit cannot +/// consume a span that was unavailable at the declared cutoff. +#[must_use] +pub fn select_text_segments_for_document_as_of_sql( + document_record_id: Uuid, + knowledge_cutoff: &KnowledgeCutoff, +) -> String { + format!( + "SELECT text_segment_id, tenant_record_id, document_record_id, \ + start_byte, end_byte, system_time, available_time \ + FROM text_segment \ + WHERE document_record_id = '{document_record_id}'::uuid \ + AND available_time <= '{cutoff}'::timestamptz \ + ORDER BY start_byte, text_segment_id", + cutoff = knowledge_cutoff.to_rfc3339(), + ) +} + +#[cfg(test)] +mod tests { + use super::{ + TextSegmentRecord, insert_text_segment_sql, select_text_segment_by_id_sql, + select_text_segments_for_document_as_of_sql, + }; + use crate::PersistenceError; + use temporal_core::{AvailableTime, KnowledgeCutoff, SystemTime}; + use uuid::Uuid; + + fn sample() -> TextSegmentRecord { + TextSegmentRecord { + text_segment_id: Uuid::from_u128(1), + tenant_record_id: Uuid::from_u128(2), + document_record_id: Uuid::from_u128(3), + start_byte: 0, + end_byte: 5, + system_time: SystemTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("s"), + available_time: AvailableTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("a"), + } + } + + #[test] + fn validate_and_select_helpers_cover_local_branches() { + let record = sample(); + record.validate().expect("valid"); + let insert = insert_text_segment_sql(&record).expect("insert"); + assert!(insert.contains("0, 5")); + assert_eq!( + insert_text_segment_sql(&TextSegmentRecord { + start_byte: 0, + end_byte: 0, + ..record.clone() + }), + Err(PersistenceError::InvalidTextSegment) + ); + let by_id = select_text_segment_by_id_sql(record.text_segment_id); + assert!(by_id.contains("LIMIT 1")); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-01-01T00:00:00Z").expect("c"); + let as_of = select_text_segments_for_document_as_of_sql(record.document_record_id, &cutoff); + assert!(as_of.contains("available_time <=")); + } +} diff --git a/crates/persistence_postgres/tests/live_postgres.rs b/crates/persistence_postgres/tests/live_postgres.rs index b1282bd3d..7b2479824 100644 --- a/crates/persistence_postgres/tests/live_postgres.rs +++ b/crates/persistence_postgres/tests/live_postgres.rs @@ -11,15 +11,15 @@ use persistence_postgres::{ EvidenceTombstoneRecord, LegalHoldRecord, LiveDocumentRepository, LiveSqlxPoolOptions, MembershipAssignmentRecord, MigrationCatalog, ModelArtifactRecord, ModelRunRecord, PersistenceError, ReproducibilityManifestRecord, RetentionPolicyRecord, SqlSession, - apply_sql_batch, assume_app_runtime_role_sql, clear_session_tenant_sql, open_live_sqlx_pool, - require_live_sqlx_config, reset_app_runtime_role_sql, select_active_analysis_document_sql, - set_session_tenant_sql, + TextSegmentRecord, apply_sql_batch, assume_app_runtime_role_sql, clear_session_tenant_sql, + open_live_sqlx_pool, require_live_sqlx_config, reset_app_runtime_role_sql, + select_active_analysis_document_sql, set_session_tenant_sql, }; use std::sync::mpsc; use std::sync::{Arc, Barrier}; use std::thread; use std::time::Duration; -use temporal_core::{AvailableTime, EventTime, SystemTime}; +use temporal_core::{AvailableTime, EventTime, KnowledgeCutoff, SystemTime}; use uuid::Uuid; const CONCURRENT_WRITERS: usize = 2; @@ -142,6 +142,7 @@ fn live_postgres_applies_migrations_and_document_sql() { exercise_model_run_artifact_chain(&mut repo, tenant_record_id, &manifest, available); exercise_typed_membership_assignments(&mut repo, tenant_record_id, available, system); + prove_text_segment_known_span(&mut repo, tenant_record_id, available, system); prove_retention_deletion_legal_hold( &mut repo, tenant_record_id, @@ -975,6 +976,79 @@ fn exercise_typed_membership_assignments( ); } +fn prove_text_segment_known_span( + repo: &mut LiveDocumentRepository, + tenant_record_id: Uuid, + available: AvailableTime, + system: SystemTime, +) { + const DOCUMENT_UTF8: &str = "hello world"; + assert_eq!(DOCUMENT_UTF8.len(), 11); + assert_eq!(&DOCUMENT_UTF8.as_bytes()[0..5], b"hello"); + let document_record_id = Uuid::now_v7(); + let hello = Uuid::now_v7(); + let world = Uuid::now_v7(); + let later = AvailableTime::parse_rfc3339("2026-06-01T00:00:00Z").expect("later"); + repo.insert_text_segment(&TextSegmentRecord { + text_segment_id: hello, + tenant_record_id, + document_record_id, + start_byte: 0, + end_byte: 5, + system_time: system, + available_time: available, + }) + .expect("insert known hello span"); + repo.insert_text_segment(&TextSegmentRecord { + text_segment_id: world, + tenant_record_id, + document_record_id, + start_byte: 6, + end_byte: 11, + system_time: system, + available_time: later, + }) + .expect("insert later world span"); + let inverted = TextSegmentRecord { + text_segment_id: Uuid::now_v7(), + tenant_record_id, + document_record_id, + start_byte: 5, + end_byte: 0, + system_time: system, + available_time: available, + }; + assert_eq!( + repo.insert_text_segment(&inverted), + Err(PersistenceError::InvalidTextSegment) + ); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + repo.submit_text_segment_by_id(hello) + .expect("select text_segment by id"); + repo.submit_text_segments_for_document_as_of(document_record_id, &cutoff) + .expect("select cutoff-eligible text_segment rows"); + let recovery = format!( + "DO $tepp_text_segment$ BEGIN \ + IF (SELECT start_byte FROM text_segment \ + WHERE text_segment_id = '{hello}'::uuid) <> 0 THEN \ + RAISE EXCEPTION 'hello start_byte did not recover'; \ + END IF; \ + IF (SELECT end_byte FROM text_segment \ + WHERE text_segment_id = '{hello}'::uuid) <> 5 THEN \ + RAISE EXCEPTION 'hello end_byte did not recover'; \ + END IF; \ + IF (SELECT COUNT(*) FROM text_segment \ + WHERE document_record_id = '{document_record_id}'::uuid \ + AND available_time <= '2026-02-01T00:00:00Z'::timestamptz) <> 1 THEN \ + RAISE EXCEPTION 'cutoff must keep only the available hello span'; \ + END IF; \ + END $tepp_text_segment$" + ); + repo.session_mut() + .execute(&recovery) + .expect("known hello span and cutoff eligibility must recover"); +} + fn prove_persisted_membership_rows( repo: &mut LiveDocumentRepository, document_record_id: Uuid, @@ -1041,20 +1115,16 @@ fn prove_membership_exactly_one_rejections( ); let text_segment_id = Uuid::now_v7(); - repo.session_mut() - .execute(&format!( - "INSERT INTO text_segment (\ - text_segment_id, tenant_record_id, document_record_id, \ - start_byte, end_byte, system_time, available_time\ - ) VALUES (\ - '{text_segment_id}'::uuid, '{tenant_record_id}'::uuid, \ - '{document_record_id}'::uuid, 0, 8, \ - '{system}'::timestamptz, '{available}'::timestamptz\ - )", - system = system.to_rfc3339(), - available = available.to_rfc3339(), - )) - .expect("insert text_segment"); + repo.insert_text_segment(&TextSegmentRecord { + text_segment_id, + tenant_record_id, + document_record_id, + start_byte: 0, + end_byte: 8, + system_time: system, + available_time: available, + }) + .expect("insert text_segment"); let dual_unit = format!( "INSERT INTO membership_assignment (\ membership_assignment_id, tenant_record_id, document_record_id, \ diff --git a/crates/persistence_postgres/tests/text_segment_sql_contract.rs b/crates/persistence_postgres/tests/text_segment_sql_contract.rs new file mode 100644 index 000000000..ee1b893e2 --- /dev/null +++ b/crates/persistence_postgres/tests/text_segment_sql_contract.rs @@ -0,0 +1,79 @@ +//! Typed `text_segment` SQL must recover known byte spans and refuse inverted ones. + +use persistence_postgres::{ + PersistenceError, TextSegmentRecord, insert_text_segment_sql, select_text_segment_by_id_sql, + select_text_segments_for_document_as_of_sql, +}; +use temporal_core::{AvailableTime, KnowledgeCutoff, SystemTime}; +use uuid::Uuid; + +/// UTF-8 `hello world` is 11 bytes; the true `hello` span is `[0, 5)`. +const TRUTH_START_BYTE: i64 = 0; +const TRUTH_END_BYTE: i64 = 5; +const DOCUMENT_UTF8: &str = "hello world"; + +fn clocks() -> (AvailableTime, SystemTime) { + ( + AvailableTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("available"), + SystemTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("system"), + ) +} + +fn segment(start_byte: i64, end_byte: i64) -> TextSegmentRecord { + let (available, system) = clocks(); + TextSegmentRecord { + text_segment_id: Uuid::from_u128(7), + tenant_record_id: Uuid::from_u128(3), + document_record_id: Uuid::from_u128(11), + start_byte, + end_byte, + system_time: system, + available_time: available, + } +} + +#[test] +fn insert_sql_recovers_the_known_hello_byte_span() { + assert_eq!(DOCUMENT_UTF8.len(), 11); + assert_eq!(&DOCUMENT_UTF8.as_bytes()[0..5], b"hello"); + let sql = insert_text_segment_sql(&segment(TRUTH_START_BYTE, TRUTH_END_BYTE)).expect("sql"); + assert!(sql.contains("INSERT INTO text_segment")); + assert!(sql.contains("start_byte")); + assert!(sql.contains("end_byte")); + assert!( + sql.contains(&format!("{TRUTH_START_BYTE}, {TRUTH_END_BYTE}")), + "rendered SQL must carry the known-truth span: {sql}" + ); + assert!( + !sql.contains("byte_start"), + "physical 0006 column is start_byte" + ); +} + +#[test] +fn inverted_empty_and_negative_spans_fail_closed_before_sql() { + assert_eq!( + insert_text_segment_sql(&segment(5, 5)), + Err(PersistenceError::InvalidTextSegment) + ); + assert_eq!( + insert_text_segment_sql(&segment(5, 4)), + Err(PersistenceError::InvalidTextSegment) + ); + assert_eq!( + insert_text_segment_sql(&segment(-1, 4)), + Err(PersistenceError::InvalidTextSegment) + ); +} + +#[test] +fn cutoff_select_binds_available_time_and_document_identity() { + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let document = Uuid::from_u128(11); + let sql = select_text_segments_for_document_as_of_sql(document, &cutoff); + assert!(sql.contains("FROM text_segment")); + assert!(sql.contains(&format!("document_record_id = '{document}'::uuid"))); + assert!(sql.contains("available_time <= '2026-02-01T00:00:00Z'::timestamptz")); + let by_id = select_text_segment_by_id_sql(Uuid::from_u128(7)); + assert!(by_id.contains("text_segment_id = '00000000-0000-0000-0000-000000000007'::uuid")); +} diff --git a/docs/ERD.md b/docs/ERD.md index f14d8eae3..94916b90f 100644 --- a/docs/ERD.md +++ b/docs/ERD.md @@ -392,6 +392,8 @@ Customer/partner/competitor/author/department/project/opportunity roles are cont All four identifiers are typed UUID foreign keys to their named entities; there is no untyped polymorphic `membership_target_id`. This permits document-level and exact-segment weighted membership while preserving relational integrity. If event-level membership is added later, it must be an explicit typed foreign key plus an updated exactly-one constraint and ADR/data-model change. +Physical `text_segment` from migration `0006` currently stores `start_byte` / `end_byte` (half-open UTF-8 offsets), tenant, document identity, and system/available clocks. Call `insert_text_segment` to write that row. The accepted ERD still lists scalar offsets, `segment_type_code`, and a `document_record` foreign key; those columns are later migrations (`#45` owns `0007`). + `valid_from_window` is a non-empty `tstzrange` containing the possible start instant; an exact start is encoded as the singleton closed range `[t,t]`. `valid_to_window` uses the same representation for an exact or uncertain end and is NULL only for an open-ended membership. `valid_time_precision_code` records the governed precision vocabulary used to construct both windows. Database/application validation must reject empty windows, a definitely-later start than end, and a precision code inconsistent with either bound; it must never coerce an uncertain or open bound to a false exact timestamp. ## Reproducibility and relation-aware split invariant diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 9d6f381ac..d1b4acc7c 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -7,8 +7,8 @@ The full APA 7th standards/literature register remains `docs/research/standards- | Requirement / decision | Canonical basis | Source/evidence boundary | Maturity | |---|---|---|---| -| immutable source evidence and exact spans | PRD; Architecture; ADR 0008 | `evidence_core`, Task 2 tests/doctoring; `persistence_postgres` source-artifact SQL insert/lookup plus idempotent retry (#40 implemented-main) | implemented-main | -| Rust numerical authority / CPU `f64` reference | ADR 0001 | current workspace foundation; `checkpoint_authority` checkpoint-versus-estimator gate on the active PR; future estimators | partial | +| immutable source evidence and exact spans | PRD; Architecture; ADR 0008 | `evidence_core`, Task 2 tests/doctoring; `persistence_postgres` source-artifact SQL insert/lookup plus idempotent retry (#40 implemented-main); typed `text_segment` byte-span SQL (active PR) | implemented-main | +| Rust numerical authority / CPU `f64` reference | ADR 0001 | current workspace foundation; future estimators | partial | | Rust workspace/quality foundation | ADR 0007 | workspace/CI/repository contract | implemented-main | | six distinct clocks and uncertain intervals | PRD; ADR 0002 | PR #8 `temporal_core` on protected main; PR #5 historical only | implemented-main | | Allen relation algebra/bounded closure | ADR 0002; temporal research | PR #9 `temporal_core` path-consistency on protected main | implemented-main | @@ -17,7 +17,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | time-varying cross-classified multiple membership | PRD; ADR 0003 | `membership_core` network on protected main; multilevel estimators remaining | partial | | leakage-safe availability/cutoff snapshots | PRD; ADR 0002/0013 | `corpus_split` on protected main | implemented-main | | recovery metrics (RMSE, bias, coverage, graph, temporal order, Monte Carlo SE gates) | PRD; Test Strategy; ADR 0007/0014 | `validation_core` on protected main (PR #19); SE-aware Monte Carlo gates included | implemented-main | -| PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts, in-memory adapters, live SQL session/document SQL port, tenant RLS (`0002` + session GUC/role helpers), `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` driver, exact-head live PostgreSQL CI with isolation proof, append-only immutability triggers (`0004`), temporal interval ordering CHECKs (`0005`), typed membership assignment (`0006` implemented-main), event-relation/mention/instance SQL (`#37–#39` implemented-main), source-artifact SQL (`#40` implemented-main), audit-event SQL (`#41` implemented-main), concurrent document-write stress (`#43` implemented-main), backup/restore integrity revalidation (`#44` implemented-main); remaining physical ERD constraints | partial | +| PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts, in-memory adapters, live SQL session/document SQL port, tenant RLS (`0002` + session GUC/role helpers), `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` driver, exact-head live PostgreSQL CI with isolation proof, append-only immutability triggers (`0004`), temporal interval ordering CHECKs (`0005`), typed membership assignment (`0006` implemented-main), event-relation/mention/instance SQL (#37–#39 implemented-main), source-artifact SQL (#40 implemented-main), audit-event SQL (#41 implemented-main), concurrent document-write stress (#43 implemented-main), backup/restore integrity revalidation (#44 implemented-main), typed `text_segment` SQL insert/cutoff lookup (active PR); remaining physical ERD constraints including `document_record` FK on `text_segment` | partial | | known-truth temporal/event simulation manifests | PRD; TRD; Test Strategy | `tepp_simulation` on protected main; recovery metrics in `validation_core` | implemented-main | | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index a593ddb16..f16e30045 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation on the active PR +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, concurrent document-write stress, and backup/restore integrity revalidation implemented-main; typed `text_segment` SQL on the active PR **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). diff --git a/docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md b/docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md index 89c3a045c..07cc8dd4e 100644 --- a/docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md +++ b/docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md @@ -21,17 +21,12 @@ fail-closed no-op. A dry run may print the task contract without credentials. When a PR or issue exists, normal review → repair → exact-head Checks → merge governance owns the hour. The scheduler does not create a competing branch. -Current executable queue while drafts remain open: - -1. Merge the predicted-versus-observed Allen coverage gate - (`prediction_contradiction` / the coverage-authority landing PR). Keep - superseded coverage drafts unmerged. -2. Next buyer-visible slice: naruon live HTTP loopback with stream deadline, - RFC 3339 cutoff, loopback `Host`, NIM/proxy header refusal, and export - over a real `TcpStream` (this PR). Keep PR #87 unmerged. -3. After that: `text_segment` SQL on migration `0006` (PR #99), then - production TLS bind (PR #100 / #90). Do not open a competing hourly - proposal until the open-PR inventory is empty. +Current preferred next gap after the open-PR queue drains: persist the +accepted ERD `document_record` foreign key on `text_segment` once `#45` +releases migration `0007`. Do not allocate that number from another lane. +Until then, land `#90` (production TLS bind policy), `#97` (prediction +coverage gate), and `#45` (retention/`0007`) rather than opening a fourth +writer for the same tables. ## Required repository configuration diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index 7b24026e7..ab68b4763 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -76,11 +76,15 @@ Davis, M., Iancu, L., & Whistler, K. (Eds.). (2024). *Unicode Standard Annex #15 Davis, M., Iancu, L., & Whistler, K. (Eds.). (2024). *Unicode Standard Annex #29: Unicode text segmentation*. Unicode Consortium. +Bird, S., & Liberman, M. (2001). A formal framework for linguistic annotation. *Speech Communication, 33*(1–2), 23–60. https://doi.org/10.1016/S0167-6393(00)00068-6 + +Wilde, E., & Duerst, M. (2008). *URI fragment identifiers for the text/plain media type* (RFC 5147). Internet Engineering Task Force. https://doi.org/10.17487/RFC5147 + Phillips, A., & Davis, M. (2009). *Tags for identifying languages* (RFC 5646). Internet Engineering Task Force. https://doi.org/10.17487/RFC5646 Nivre, J., de Marneffe, M.-C., Ginter, F., Hajič, J., Manning, C. D., Pyysalo, S., Schuster, S., Tyers, F., & Zeman, D. (2020). Universal Dependencies v2: An evergrowing multilingual treebank collection. In *Proceedings of the 12th Language Resources and Evaluation Conference* (pp. 4034–4043). European Language Resources Association. -The original source is preserved. NFC is used for canonical analysis views; compatibility normalization is limited to explicit auxiliary keys. Segmentation and morphology are language-tailored. Universal POS informs source priors rather than irreversible deletion. +The original source is preserved. NFC is used for canonical analysis views; compatibility normalization is limited to explicit auxiliary keys. Segmentation and morphology are language-tailored. Universal POS informs source priors rather than irreversible deletion. Persist an exact UTF-8 byte span through `text_segment` SQL when a membership or mention must point at a unit without copying source text (Bird & Liberman, 2001; Wilde & Duerst, 2008; Davis et al., 2024). ## Evidence identity, hashing, and interchange diff --git a/docs/research/text-segment-sql.md b/docs/research/text-segment-sql.md new file mode 100644 index 000000000..77c921b66 --- /dev/null +++ b/docs/research/text-segment-sql.md @@ -0,0 +1,52 @@ +# Typed text-segment SQL (doctoring) + +## Scope + +Call `insert_text_segment` when a document must expose an exact observed +unit for membership, mention, or later semantic work. The adapter writes +the existing `0006` `text_segment` row: half-open UTF-8 byte offsets +`[start_byte, end_byte)`, tenant, document identity, system time, and +availability time. Historical reads use +`select_text_segments_for_document_as_of_sql` so a span whose +`available_time` is after the declared cutoff cannot enter a fit +(Jensen & Snodgrass, 1999). + +This slice does **not** allocate `0007` or `0008`. It does not add the +accepted ERD foreign key from `text_segment.document_record_id` to +`document_record`, Unicode scalar columns, or `segment_type_code`. Those +remain later migrations. + +## Authority + +Exact character or byte positions are the durable way to point at a +substring without copying source text into every membership or mention +row (Bird & Liberman, 2001; Wilde & Duerst, 2008). Unicode text +segmentation defines language-appropriate units; TEPP stores the +resulting UTF-8 byte interval rather than a token string (Davis et al., +2024). Availability versus cutoff is the historical-inclusion rule +already used for documents and splits (Jensen & Snodgrass, 1999). + +Bird, S., & Liberman, M. (2001). A formal framework for linguistic +annotation. *Speech Communication, 33*(1–2), 23–60. +https://doi.org/10.1016/S0167-6393(00)00068-6 + +Wilde, E., & Duerst, M. (2008). *URI fragment identifiers for the +text/plain media type* (RFC 5147). Internet Engineering Task Force. +https://doi.org/10.17487/RFC5147 + +Davis, M., Iancu, L., & Whistler, K. (Eds.). (2024). *Unicode Standard +Annex #29: Unicode text segmentation*. Unicode Consortium. + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. +*IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. +https://doi.org/10.1109/69.755613 + +## Verification + +- contract tests recover the known `hello` span `[0, 5)` from + `hello world` and refuse inverted, empty, and negative spans before + SQL is rendered; +- cutoff selection SQL binds `available_time <= knowledge_cutoff`; +- live PostgreSQL CI inserts the known span, refuses an inverted + adapter write, and proves the later `world` span is excluded at a + February cutoff when `TEPP_LIVE_POSTGRES=1`. diff --git a/docs/research/typed-membership-assignment-persistence.md b/docs/research/typed-membership-assignment-persistence.md index ed48f5e4b..64af7d4ba 100644 --- a/docs/research/typed-membership-assignment-persistence.md +++ b/docs/research/typed-membership-assignment-persistence.md @@ -49,3 +49,6 @@ exactly-one constraints preserve those distinct contexts. Migration membership for the same document, asserts those three rows persist, and rejects both dual-target and dual observed-unit rows when `TEPP_LIVE_POSTGRES=1`. +- segment-level membership now inserts the observed `text_segment` + through the typed adapter rather than ad-hoc SQL. See + `docs/research/text-segment-sql.md`. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index f2a86391a..4a0a91093 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -18,6 +18,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Event mention/instance | `event_core` | partial | — | unit + fail-closed promotion | Task 5 / PR #13 | | Multiple membership | `membership_core` | partial | — | unit + ESS weights | Task 7 / PR #12 + #25 | | Forward transition DAG | `relation_graph` | implemented-main | — | unit + cycle rejection | Task 6 / PR #14 | +| Bitemporal persistence + live SQL port | `persistence_postgres` | partial | typed `text_segment` SQL | migration contracts + recording transport + optional PgPool + live CI + tenant RLS + `0005`/`0006` + event relation/mention/instance + source-artifact + audit-event + concurrent-write + restore integrity (#37–#44 implemented-main) + typed `text_segment` insert/cutoff lookup (active PR) | Task 8 / PR #16 + #23 + #26 + #27 + #29 + #30–#44 + text-segment SQL | | Bitemporal persistence + live SQL port | `persistence_postgres` | partial | — | migration contracts + recording transport + optional PgPool + live CI + tenant RLS + `0005`/`0006` interval and membership contracts + event relation/mention/instance + source-artifact + audit-event + concurrent-write + restore-integrity contracts implemented-main | Task 8 / PR #16 + #23 + #26 + #27 + #29 + #30–#44 | | Leakage-safe splits | `corpus_split` | implemented-main | — | cutoff + co-partition tests | Task 9 / PR #17 | | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 |