diff --git a/CHANGELOG.md b/CHANGELOG.md index b1217a3c3..033936c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `corpus_split` leakage-safe knowledge-cutoff snapshots, relation-connected co-partition groups, rolling-origin windows, and group-normalized ESS weight contracts. - `persistence_postgres` bitemporal foundation: multi-word migration contracts, knowledge-cutoff eligibility, and in-memory as-known-at / as-valid-at document replay (live SQLx/PostgreSQL execution remains accepted-target). - `event_core` mention/instance separation with explicit promotion, typed roles, event-time validity, and fail-closed mention-as-instance refusal. - `membership_core` time-varying weighted multiple-membership network with contextual roles, event-time validity, and atomistic-fallacy prevention contracts. diff --git a/Cargo.lock b/Cargo.lock index 3ced459a7..45ce80460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,10 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "corpus_split" version = "0.1.0" +dependencies = [ + "temporal_core", + "uuid", +] [[package]] name = "cpufeatures" diff --git a/crates/corpus_split/Cargo.toml b/crates/corpus_split/Cargo.toml index a1dc4dfac..0a61ed2df 100644 --- a/crates/corpus_split/Cargo.toml +++ b/crates/corpus_split/Cargo.toml @@ -13,5 +13,12 @@ keywords.workspace = true categories.workspace = true publish = false +[dependencies] +temporal_core = { path = "../temporal_core", version = "0.1.0" } +uuid.workspace = true + +[dev-dependencies] +uuid.workspace = true + [lints] workspace = true diff --git a/crates/corpus_split/src/connected_group.rs b/crates/corpus_split/src/connected_group.rs new file mode 100644 index 000000000..0a8c35df9 --- /dev/null +++ b/crates/corpus_split/src/connected_group.rs @@ -0,0 +1,214 @@ +//! Relation-connected leakage groups. + +use crate::CorpusSplitError; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use uuid::Uuid; + +/// Governed link kinds that force co-partitioning. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum LeakageLinkKind { + /// Document revision of another document. + Revision, + /// Translation of another document. + Translation, + /// Copied or template-derived variant. + CopiedVariant, + /// Shared event episode membership. + SameEpisode, +} + +/// Undirected leakage link between two document identities. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LeakageLink { + /// First endpoint. + pub left: Uuid, + /// Second endpoint. + pub right: Uuid, + /// Governed link kind. + pub kind: LeakageLinkKind, +} + +/// Union of documents that must remain in one split partition. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConnectedGroup { + members: BTreeSet, +} + +impl ConnectedGroup { + /// Borrow member identities. + #[must_use] + pub fn members(&self) -> &BTreeSet { + &self.members + } + + /// Return the number of members. + #[must_use] + pub fn len(&self) -> usize { + self.members.len() + } + + /// Return whether the group is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.members.is_empty() + } +} + +/// Build connected components over leakage links restricted to `universe`. +#[must_use] +pub fn build_connected_groups(universe: &[Uuid], links: &[LeakageLink]) -> Vec { + let allowed: BTreeSet = universe.iter().copied().collect(); + let mut adjacency: BTreeMap> = BTreeMap::new(); + for id in &allowed { + adjacency.entry(*id).or_default(); + } + for link in links { + if !allowed.contains(&link.left) || !allowed.contains(&link.right) { + continue; + } + if link.left == link.right { + continue; + } + adjacency.entry(link.left).or_default().insert(link.right); + adjacency.entry(link.right).or_default().insert(link.left); + } + + let mut seen = BTreeSet::new(); + let mut groups = Vec::new(); + for start in universe { + if !seen.insert(*start) { + continue; + } + let mut members = BTreeSet::new(); + let mut queue = VecDeque::from([*start]); + while let Some(node) = queue.pop_front() { + // Each node is enqueued at most once via `seen`, so membership insert always succeeds. + members.insert(node); + // Every universe identity is pre-inserted into `adjacency`. + for neighbor in &adjacency[&node] { + if seen.insert(*neighbor) { + queue.push_back(*neighbor); + } + } + } + groups.push(ConnectedGroup { members }); + } + groups +} + +/// Reject a proposed partition map that separates any connected group. +/// +/// # Errors +/// +/// Returns [`CorpusSplitError::RelationLeakage`] when linked members diverge. +pub fn assert_no_group_leakage( + groups: &[ConnectedGroup], + partition_by_document: &BTreeMap, +) -> Result<(), CorpusSplitError> { + for group in groups { + let mut partition = None; + for member in &group.members { + let Some(assigned) = partition_by_document.get(member) else { + continue; + }; + match partition { + None => partition = Some(*assigned), + Some(existing) if existing == *assigned => {} + Some(_) => return Err(CorpusSplitError::RelationLeakage), + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + ConnectedGroup, LeakageLink, LeakageLinkKind, assert_no_group_leakage, + build_connected_groups, + }; + use crate::CorpusSplitError; + use std::collections::{BTreeMap, BTreeSet}; + use uuid::Uuid; + + #[test] + fn revisions_and_translations_form_one_group() { + let a = Uuid::now_v7(); + let b = Uuid::now_v7(); + let c = Uuid::now_v7(); + let groups = build_connected_groups( + &[a, b, c], + &[ + LeakageLink { + left: a, + right: b, + kind: LeakageLinkKind::Revision, + }, + LeakageLink { + left: b, + right: c, + kind: LeakageLinkKind::Translation, + }, + ], + ); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 3); + assert!(!groups[0].is_empty()); + assert_eq!(groups[0].members(), &BTreeSet::from([a, b, c])); + } + + #[test] + fn cross_partition_assignment_is_rejected() { + let a = Uuid::now_v7(); + let b = Uuid::now_v7(); + let groups = build_connected_groups( + &[a, b], + &[LeakageLink { + left: a, + right: b, + kind: LeakageLinkKind::CopiedVariant, + }], + ); + let mut map = BTreeMap::new(); + map.insert(a, 0); + map.insert(b, 1); + assert_eq!( + assert_no_group_leakage(&groups, &map), + Err(CorpusSplitError::RelationLeakage) + ); + map.insert(b, 0); + assert_no_group_leakage(&groups, &map).expect("same partition"); + let empty = ConnectedGroup { + members: BTreeSet::new(), + }; + assert!(empty.is_empty()); + // Self-links and out-of-universe endpoints are ignored. + let lonely = Uuid::now_v7(); + let outside = Uuid::now_v7(); + let isolated = build_connected_groups( + &[lonely], + &[ + LeakageLink { + left: lonely, + right: lonely, + kind: LeakageLinkKind::Revision, + }, + LeakageLink { + left: lonely, + right: outside, + kind: LeakageLinkKind::Translation, + }, + LeakageLink { + left: outside, + right: lonely, + kind: LeakageLinkKind::CopiedVariant, + }, + ], + ); + assert_eq!(isolated.len(), 1); + assert_eq!(isolated[0].len(), 1); + // Unassigned members do not induce leakage. + let partial = BTreeMap::from([(a, 0_u8)]); + assert_no_group_leakage(&groups, &partial).expect("partial assignment"); + } +} diff --git a/crates/corpus_split/src/document.rs b/crates/corpus_split/src/document.rs new file mode 100644 index 000000000..c5562861e --- /dev/null +++ b/crates/corpus_split/src/document.rs @@ -0,0 +1,24 @@ +//! Document observations eligible for cutoff-aware snapshots. + +use temporal_core::AvailableTime; +use uuid::Uuid; + +/// One document observation with availability provenance. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CorpusDocument { + /// Stable analytical document identity. + pub document_id: Uuid, + /// When the document became available as evidence. + pub available_time: AvailableTime, +} + +impl CorpusDocument { + /// Construct a document observation. + #[must_use] + pub const fn new(document_id: Uuid, available_time: AvailableTime) -> Self { + Self { + document_id, + available_time, + } + } +} diff --git a/crates/corpus_split/src/error.rs b/crates/corpus_split/src/error.rs new file mode 100644 index 000000000..c9cc4cba5 --- /dev/null +++ b/crates/corpus_split/src/error.rs @@ -0,0 +1,56 @@ +//! Fail-closed corpus-split errors. + +use std::fmt; + +/// A fail-closed corpus-split domain error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum CorpusSplitError { + /// A partition assignment would separate linked records. + RelationLeakage, + /// A document was unavailable at the requested knowledge cutoff. + UnavailableAtCutoff, + /// A duplicate document identity was rejected. + DuplicateDocumentIdentity, + /// Split proportions or seeds were invalid. + InvalidSplitConfiguration, +} + +impl fmt::Display for CorpusSplitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::RelationLeakage => "relation-aware split leakage", + Self::UnavailableAtCutoff => "document unavailable at knowledge cutoff", + Self::DuplicateDocumentIdentity => "duplicate document identity", + Self::InvalidSplitConfiguration => "invalid split configuration", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for CorpusSplitError {} + +#[cfg(test)] +mod tests { + use super::CorpusSplitError; + + #[test] + fn messages_are_stable() { + assert_eq!( + CorpusSplitError::RelationLeakage.to_string(), + "relation-aware split leakage" + ); + assert_eq!( + CorpusSplitError::UnavailableAtCutoff.to_string(), + "document unavailable at knowledge cutoff" + ); + assert_eq!( + CorpusSplitError::DuplicateDocumentIdentity.to_string(), + "duplicate document identity" + ); + assert_eq!( + CorpusSplitError::InvalidSplitConfiguration.to_string(), + "invalid split configuration" + ); + } +} diff --git a/crates/corpus_split/src/lib.rs b/crates/corpus_split/src/lib.rs index a45cc7d50..313fae238 100644 --- a/crates/corpus_split/src/lib.rs +++ b/crates/corpus_split/src/lib.rs @@ -2,5 +2,66 @@ #![deny(missing_docs)] //! Leakage-safe corpus snapshots and relation-aware data splits. //! -//! This crate intentionally exposes no production behavior in the workspace-foundation -//! slice. Domain APIs are introduced test-first in the corresponding implementation task. +//! TEPP historical analyses may only consume documents whose availability time +//! does not exceed the declared knowledge cutoff. Train/validation/test +//! partitions keep revisions, translations, copied variants, and shared +//! episodes co-located so relation-aware leakage cannot invent independence. + +mod connected_group; +mod document; +mod error; +mod rolling_origin; +mod snapshot; +mod weights; + +use temporal_core::{AvailableTime, KnowledgeCutoff}; + +/// Knowledge-cutoff eligibility for snapshot membership. +#[must_use] +pub fn cutoff_eligible(available_time: &AvailableTime, knowledge_cutoff: &KnowledgeCutoff) -> bool { + available_time.instant() <= knowledge_cutoff.instant() +} + +/// Relation-connected leakage group. +pub use connected_group::ConnectedGroup; +/// Undirected leakage link. +pub use connected_group::LeakageLink; +/// Governed leakage link vocabulary. +pub use connected_group::LeakageLinkKind; +/// Reject partitions that separate linked groups. +pub use connected_group::assert_no_group_leakage; +/// Build connected components over leakage links. +pub use connected_group::build_connected_groups; +/// Document observation with availability provenance. +pub use document::CorpusDocument; +/// Fail-closed corpus-split errors. +pub use error::CorpusSplitError; +/// Rolling-origin train/test window. +pub use rolling_origin::RollingOriginWindow; +/// Build ordered rolling-origin windows. +pub use rolling_origin::rolling_origin_windows; +/// Cutoff-filtered corpus snapshot. +pub use snapshot::CorpusSnapshot; +/// Kish effective sample size. +pub use weights::effective_sample_size; +/// Group-normalized observation weights. +pub use weights::group_normalized_weights; + +#[cfg(test)] +mod tests { + use super::cutoff_eligible; + use temporal_core::{AvailableTime, KnowledgeCutoff}; + + #[test] + fn cutoff_boundary_is_inclusive() { + let stamp = "2026-05-01T00:00:00Z"; + assert!(cutoff_eligible( + &AvailableTime::parse_rfc3339(stamp).expect("a"), + &KnowledgeCutoff::parse_rfc3339(stamp).expect("c") + )); + assert!(!cutoff_eligible( + &AvailableTime::parse_rfc3339("2026-05-02T00:00:00Z").expect("a"), + &KnowledgeCutoff::parse_rfc3339(stamp).expect("c") + )); + } +} diff --git a/crates/corpus_split/src/rolling_origin.rs b/crates/corpus_split/src/rolling_origin.rs new file mode 100644 index 000000000..44d951278 --- /dev/null +++ b/crates/corpus_split/src/rolling_origin.rs @@ -0,0 +1,66 @@ +//! Rolling-origin evaluation windows over cutoff snapshots. + +use temporal_core::KnowledgeCutoff; + +/// One rolling-origin evaluation origin with train and test cutoffs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RollingOriginWindow { + /// Inclusive knowledge cutoff for training evidence. + pub train_cutoff: KnowledgeCutoff, + /// Inclusive knowledge cutoff for the test horizon. + pub test_cutoff: KnowledgeCutoff, +} + +/// Build contiguous rolling-origin windows from ordered cutoffs. +/// +/// Each window uses cutoff `i` for training and cutoff `i + 1` for testing. +/// +/// # Errors +/// +/// Returns [`crate::CorpusSplitError::InvalidSplitConfiguration`] when fewer +/// than two cutoffs are provided or order is not strictly increasing. +pub fn rolling_origin_windows( + ordered_cutoffs: &[KnowledgeCutoff], +) -> Result, crate::CorpusSplitError> { + if ordered_cutoffs.len() < 2 { + return Err(crate::CorpusSplitError::InvalidSplitConfiguration); + } + for window in ordered_cutoffs.windows(2) { + if window[0].instant() >= window[1].instant() { + return Err(crate::CorpusSplitError::InvalidSplitConfiguration); + } + } + Ok(ordered_cutoffs + .windows(2) + .map(|pair| RollingOriginWindow { + train_cutoff: pair[0], + test_cutoff: pair[1], + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::rolling_origin_windows; + use crate::CorpusSplitError; + use temporal_core::KnowledgeCutoff; + + #[test] + fn windows_require_strictly_increasing_cutoffs() { + let early = KnowledgeCutoff::parse_rfc3339("2026-01-01T00:00:00Z").expect("e"); + let mid = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("m"); + let late = KnowledgeCutoff::parse_rfc3339("2026-03-01T00:00:00Z").expect("l"); + let windows = rolling_origin_windows(&[early, mid, late]).expect("windows"); + assert_eq!(windows.len(), 2); + assert_eq!(windows[0].train_cutoff, early); + assert_eq!(windows[0].test_cutoff, mid); + assert_eq!( + rolling_origin_windows(&[early]), + Err(CorpusSplitError::InvalidSplitConfiguration) + ); + assert_eq!( + rolling_origin_windows(&[mid, early]), + Err(CorpusSplitError::InvalidSplitConfiguration) + ); + } +} diff --git a/crates/corpus_split/src/snapshot.rs b/crates/corpus_split/src/snapshot.rs new file mode 100644 index 000000000..8e74c72ba --- /dev/null +++ b/crates/corpus_split/src/snapshot.rs @@ -0,0 +1,102 @@ +//! Knowledge-cutoff corpus snapshots. + +use crate::CorpusDocument; +use crate::CorpusSplitError; +use crate::cutoff_eligible; +use std::collections::BTreeMap; +use temporal_core::KnowledgeCutoff; +use uuid::Uuid; + +/// Immutable snapshot of documents eligible under a knowledge cutoff. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CorpusSnapshot { + documents: BTreeMap, +} + +impl CorpusSnapshot { + /// Create an empty snapshot. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Insert a document if it is eligible under `knowledge_cutoff`. + /// + /// # Errors + /// + /// Returns unavailability or duplicate-identity errors. + pub fn insert_if_eligible( + &mut self, + document: CorpusDocument, + knowledge_cutoff: &KnowledgeCutoff, + ) -> Result<(), CorpusSplitError> { + if !cutoff_eligible(&document.available_time, knowledge_cutoff) { + return Err(CorpusSplitError::UnavailableAtCutoff); + } + if self.documents.contains_key(&document.document_id) { + return Err(CorpusSplitError::DuplicateDocumentIdentity); + } + self.documents.insert(document.document_id, document); + Ok(()) + } + + /// Return whether the snapshot contains a document identity. + #[must_use] + pub fn contains(&self, document_id: Uuid) -> bool { + self.documents.contains_key(&document_id) + } + + /// Return the number of eligible documents. + #[must_use] + pub fn len(&self) -> usize { + self.documents.len() + } + + /// Return whether the snapshot is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.documents.is_empty() + } + + /// Iterate document identities in sorted order. + pub fn document_ids(&self) -> impl Iterator + '_ { + self.documents.keys().copied() + } +} + +#[cfg(test)] +mod tests { + use super::CorpusSnapshot; + use crate::{CorpusDocument, CorpusSplitError}; + use temporal_core::{AvailableTime, KnowledgeCutoff}; + use uuid::Uuid; + + #[test] + fn late_available_documents_are_excluded() { + let mut snapshot = CorpusSnapshot::new(); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-03-01T00:00:00Z").expect("cutoff"); + let early = CorpusDocument::new( + Uuid::now_v7(), + AvailableTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("a"), + ); + let late = CorpusDocument::new( + Uuid::now_v7(), + AvailableTime::parse_rfc3339("2026-08-01T00:00:00Z").expect("a"), + ); + snapshot + .insert_if_eligible(early.clone(), &cutoff) + .expect("early"); + assert_eq!( + snapshot.insert_if_eligible(late, &cutoff), + Err(CorpusSplitError::UnavailableAtCutoff) + ); + assert_eq!( + snapshot.insert_if_eligible(early.clone(), &cutoff), + Err(CorpusSplitError::DuplicateDocumentIdentity) + ); + assert_eq!(snapshot.len(), 1); + assert!(!snapshot.is_empty()); + assert!(snapshot.contains(early.document_id)); + assert_eq!(snapshot.document_ids().count(), 1); + } +} diff --git a/crates/corpus_split/src/weights.rs b/crates/corpus_split/src/weights.rs new file mode 100644 index 000000000..eb42e8420 --- /dev/null +++ b/crates/corpus_split/src/weights.rs @@ -0,0 +1,86 @@ +//! Duplicate-aware effective sample size and group-normalized weights. + +use crate::ConnectedGroup; + +/// Compute Kish effective sample size for non-negative weights. +/// +/// Returns `0.0` when the weight vector is empty or all zeros. +#[must_use] +pub fn effective_sample_size(weights: &[f64]) -> f64 { + let sum: f64 = weights.iter().copied().sum(); + let sum_sq: f64 = weights.iter().map(|weight| weight * weight).sum(); + if sum_sq == 0.0 { + return 0.0; + } + (sum * sum) / sum_sq +} + +/// Normalize member weights within each connected group to sum to one. +/// +/// Groups with no positive mass are skipped. Members absent from `weights` +/// are treated as zero and omitted from the output map. +#[must_use] +pub fn group_normalized_weights( + groups: &[ConnectedGroup], + weights: &[(uuid::Uuid, f64)], +) -> Vec<(uuid::Uuid, f64)> { + let weight_map: std::collections::BTreeMap = weights + .iter() + .copied() + .filter(|(_, weight)| *weight > 0.0) + .collect(); + let mut normalized = Vec::new(); + for group in groups { + let total: f64 = group + .members() + .iter() + .filter_map(|member| weight_map.get(member).copied()) + .sum(); + if total <= 0.0 { + continue; + } + for member in group.members() { + if let Some(weight) = weight_map.get(member) { + normalized.push((*member, weight / total)); + } + } + } + normalized +} + +#[cfg(test)] +mod tests { + use super::{effective_sample_size, group_normalized_weights}; + use crate::connected_group::{LeakageLink, LeakageLinkKind, build_connected_groups}; + use uuid::Uuid; + + #[test] + fn ess_and_group_normalization_contracts() { + assert!((effective_sample_size(&[]) - 0.0).abs() < 1e-12); + assert!((effective_sample_size(&[0.0, 0.0]) - 0.0).abs() < 1e-12); + let ess = effective_sample_size(&[1.0, 1.0, 1.0, 1.0]); + assert!((ess - 4.0).abs() < 1e-12); + let unequal = effective_sample_size(&[1.0, 0.0, 0.0, 0.0]); + assert!((unequal - 1.0).abs() < 1e-12); + + let a = Uuid::now_v7(); + let b = Uuid::now_v7(); + let groups = build_connected_groups( + &[a, b], + &[LeakageLink { + left: a, + right: b, + kind: LeakageLinkKind::SameEpisode, + }], + ); + let normalized = group_normalized_weights(&groups, &[(a, 1.0), (b, 3.0)]); + let sum: f64 = normalized.iter().map(|(_, weight)| weight).sum(); + assert!((sum - 1.0).abs() < 1e-12); + let empty = group_normalized_weights(&groups, &[(a, 0.0), (b, 0.0)]); + assert!(empty.is_empty()); + // Partial weight maps omit members without positive mass. + let partial = group_normalized_weights(&groups, &[(a, 2.0)]); + assert_eq!(partial.len(), 1); + assert!((partial[0].1 - 1.0).abs() < 1e-12); + } +} diff --git a/crates/corpus_split/tests/leakage_contract.rs b/crates/corpus_split/tests/leakage_contract.rs new file mode 100644 index 000000000..caf5ba2b7 --- /dev/null +++ b/crates/corpus_split/tests/leakage_contract.rs @@ -0,0 +1,80 @@ +//! Integration contracts for cutoff snapshots and relation-aware splits. + +use corpus_split::{ + CorpusDocument, CorpusSnapshot, CorpusSplitError, LeakageLink, LeakageLinkKind, + assert_no_group_leakage, build_connected_groups, rolling_origin_windows, +}; +use std::collections::BTreeMap; +use temporal_core::{AvailableTime, KnowledgeCutoff}; +use uuid::Uuid; + +#[test] +fn retrospective_late_documents_cannot_enter_earlier_cutoffs() { + let mut snapshot = CorpusSnapshot::new(); + let cutoff = KnowledgeCutoff::parse_rfc3339("2025-12-31T00:00:00Z").expect("cutoff"); + let retrospective = CorpusDocument::new( + Uuid::now_v7(), + AvailableTime::parse_rfc3339("2026-06-01T00:00:00Z").expect("available"), + ); + assert_eq!( + snapshot.insert_if_eligible(retrospective, &cutoff), + Err(CorpusSplitError::UnavailableAtCutoff) + ); + assert!(snapshot.is_empty()); +} + +#[test] +fn linked_variants_never_cross_partitions() { + let original = Uuid::now_v7(); + let revision = Uuid::now_v7(); + let translation = Uuid::now_v7(); + let copy = Uuid::now_v7(); + let episode_peer = Uuid::now_v7(); + let groups = build_connected_groups( + &[original, revision, translation, copy, episode_peer], + &[ + LeakageLink { + left: original, + right: revision, + kind: LeakageLinkKind::Revision, + }, + LeakageLink { + left: original, + right: translation, + kind: LeakageLinkKind::Translation, + }, + LeakageLink { + left: original, + right: copy, + kind: LeakageLinkKind::CopiedVariant, + }, + LeakageLink { + left: original, + right: episode_peer, + kind: LeakageLinkKind::SameEpisode, + }, + ], + ); + assert_eq!(groups.len(), 1); + let mut map = BTreeMap::new(); + for member in groups[0].members() { + map.insert(*member, 0); + } + assert_no_group_leakage(&groups, &map).expect("co-located"); + map.insert(translation, 1); + assert_eq!( + assert_no_group_leakage(&groups, &map), + Err(CorpusSplitError::RelationLeakage) + ); +} + +#[test] +fn rolling_origin_uses_ordered_cutoffs() { + let cutoffs = [ + KnowledgeCutoff::parse_rfc3339("2026-01-01T00:00:00Z").expect("1"), + KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("2"), + KnowledgeCutoff::parse_rfc3339("2026-03-01T00:00:00Z").expect("3"), + ]; + let windows = rolling_origin_windows(&cutoffs).expect("windows"); + assert_eq!(windows.len(), 2); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index a6b556350..0903326e2 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -15,7 +15,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | forward-only transition subgraph | PRD; ADR 0002/0003 | future `relation_graph` validation | accepted-target | | event ontology/evidence mentions | PRD; ADR 0003 | future `event_core` | accepted-target | | time-varying cross-classified multiple membership | PRD; ADR 0003 | future `membership_core` | accepted-target | -| leakage-safe availability/cutoff snapshots | PRD; ADR 0002/0013 | future `corpus_split` | accepted-target | +| leakage-safe availability/cutoff snapshots | PRD; ADR 0002/0013 | `corpus_split` cutoff snapshots + relation co-partition contracts | partial | | PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts + in-memory bitemporal adapters; live SQLx remaining | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | future persistence/model-run artifact chain | accepted-target | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | diff --git a/docs/research/task-9-corpus-split-foundations.md b/docs/research/task-9-corpus-split-foundations.md new file mode 100644 index 000000000..889c6f524 --- /dev/null +++ b/docs/research/task-9-corpus-split-foundations.md @@ -0,0 +1,23 @@ +# Task 9 — Leakage-safe corpus split foundations + +## Scope + +Task 9 delivers storage-independent split contracts: + +1. knowledge-cutoff snapshots that reject `available_time > knowledge_cutoff`; +2. relation-connected groups over revision, translation, copied-variant, and same-episode links; +3. fail-closed leakage checks that reject partitions separating group members; +4. rolling-origin windows over strictly increasing cutoffs; +5. Kish effective sample size and group-normalized weights for duplicate-aware estimation. + +## Authoritative sources + +Kish, L. (1965). *Survey sampling*. John Wiley & Sons. + +Tashman, L. J. (2000). Out-of-sample tests of forecasting accuracy: An analysis and review. *International Journal of Forecasting, 16*(4), 437–450. https://doi.org/10.1016/S0169-2070(00)00065-0 + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. *IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. https://doi.org/10.1109/69.755613 + +## Verification + +Unit and integration contracts cover cutoff exclusion, co-partition of linked variants, rolling-origin order, ESS, and group-normalized weights. Workspace line and branch coverage gates must remain complete.