diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 179628119..91abdb5d8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `model_selection` | statistical/Pareto candidate-`K` gates; LLM votes are not numerical authority | | `checkpoint_authority` | a model checkpoint is not the CPU `f64` estimator | No crate exposes placeholder production behavior in Task 1. This prevents an diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b9e5d4a..4c9c3c437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `model_selection` candidate-`K` gates: statistical candidates require `K >= 2` and finite held-out log-likelihood/complexity, a Pareto front excludes dominated alternatives, LLM votes cannot define the numerical optimum, and selected `K` recovers known truth with computed RMSE. - `event_core` mention-confidence Brier score: known-truth binary outcomes recover a computed Brier of 0 for perfect forecasts and 0.25 for constant 0.5, with empty or mismatched streams failing closed. - `membership_core` nested ICC: CPU `f64` unbalanced ANOVA recovers a known cluster ICC and refuses to treat cross-classified or multiple-membership designs as a single hierarchy (ADR 0003). - `persistence_postgres` typed `text_segment` SQL: insert/lookup of exact UTF-8 half-open byte spans on the existing `0006` table, cutoff-eligible document reads (`available_time <= knowledge_cutoff`), and live recovery of a known `hello` span. No new migration number (`#45` still owns `0007`). diff --git a/Cargo.lock b/Cargo.lock index cfc9b1b29..80ee30816 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -737,6 +737,10 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "model_selection" +version = "0.1.0" + [[package]] name = "num-traits" version = "0.2.19" diff --git a/Cargo.toml b/Cargo.toml index 00a873052..35ec7f0f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/model_selection", "crates/checkpoint_authority", ] default-members = [ @@ -24,6 +25,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/model_selection", "crates/checkpoint_authority", ] diff --git a/README.md b/README.md index 79c0bc9e7..d402d22e7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ crates/corpus_split crates/tepp_simulation crates/validation_core crates/tepp_api +crates/model_selection crates/checkpoint_authority ``` diff --git a/crates/model_selection/Cargo.toml b/crates/model_selection/Cargo.toml new file mode 100644 index 000000000..ae6369529 --- /dev/null +++ b/crates/model_selection/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "model_selection" +description = "Statistical and Pareto candidate-K gates that refuse LLM numerical authority." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/model_selection/src/candidate.rs b/crates/model_selection/src/candidate.rs new file mode 100644 index 000000000..b692ef131 --- /dev/null +++ b/crates/model_selection/src/candidate.rs @@ -0,0 +1,156 @@ +//! Candidate topic counts with statistical diagnostics. + +use crate::ModelSelectionError; + +/// One candidate `K` together with the diagnostics that may admit it. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ModelCandidate { + candidate_k: u32, + held_out_log_likelihood: Option, + complexity: Option, + llm_vote_only: bool, +} + +impl ModelCandidate { + /// Construct a statistically supported candidate. + /// + /// # Errors + /// + /// Returns [`ModelSelectionError::NonPositiveCandidateK`] when `candidate_k` + /// is less than two, or [`ModelSelectionError::InvalidDiagnostic`] when a + /// diagnostic is non-finite. + pub fn statistical( + candidate_k: u32, + held_out_log_likelihood: f64, + complexity: f64, + ) -> Result { + if candidate_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + if !held_out_log_likelihood.is_finite() || !complexity.is_finite() || complexity < 0.0 { + return Err(ModelSelectionError::InvalidDiagnostic); + } + Ok(Self { + candidate_k, + held_out_log_likelihood: Some(held_out_log_likelihood), + complexity: Some(complexity), + llm_vote_only: false, + }) + } + + /// Construct a candidate whose only support is an LLM vote. + /// + /// The vote may later recommend among statistically admissible candidates. + /// It cannot itself define the numerical optimum. + /// + /// # Errors + /// + /// Returns [`ModelSelectionError::NonPositiveCandidateK`] when `candidate_k` + /// is less than two. + pub fn llm_vote_only(candidate_k: u32) -> Result { + if candidate_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + Ok(Self { + candidate_k, + held_out_log_likelihood: None, + complexity: None, + llm_vote_only: true, + }) + } + + /// Return the candidate topic count. + #[must_use] + pub const fn candidate_k(self) -> u32 { + self.candidate_k + } + + /// Return whether this candidate carries finite statistical diagnostics. + #[must_use] + pub const fn is_statistically_supported(self) -> bool { + match (self.held_out_log_likelihood, self.complexity) { + (Some(_), Some(_)) => !self.llm_vote_only, + _ => false, + } + } + + /// Held-out log-likelihood when the candidate is statistically supported. + #[must_use] + pub const fn held_out_log_likelihood(self) -> Option { + self.held_out_log_likelihood + } + + /// Complexity penalty (larger is worse) when statistically supported. + #[must_use] + pub const fn complexity(self) -> Option { + self.complexity + } + + /// Return whether the candidate is an LLM vote without statistical support. + #[must_use] + pub const fn is_llm_vote_only(self) -> bool { + self.llm_vote_only + } + + /// Return whether `self` Pareto-dominates `other` on likelihood and complexity. + #[must_use] + pub fn dominates(self, other: Self) -> bool { + let (Some(self_ll), Some(self_complexity), Some(other_ll), Some(other_complexity)) = ( + self.held_out_log_likelihood, + self.complexity, + other.held_out_log_likelihood, + other.complexity, + ) else { + return false; + }; + let no_worse = self_ll >= other_ll && self_complexity <= other_complexity; + let strictly_better = self_ll > other_ll || self_complexity < other_complexity; + no_worse && strictly_better + } +} + +#[cfg(test)] +mod tests { + use super::ModelCandidate; + use crate::ModelSelectionError; + + #[test] + fn statistical_candidate_accessors_and_dominance_cover_branches() { + let better = ModelCandidate::statistical(4, -10.0, 5.0).expect("better"); + let worse = ModelCandidate::statistical(8, -20.0, 9.0).expect("worse"); + assert_eq!(better.candidate_k(), 4); + assert_eq!(better.held_out_log_likelihood(), Some(-10.0)); + assert_eq!(better.complexity(), Some(5.0)); + assert!(better.is_statistically_supported()); + assert!(!better.is_llm_vote_only()); + assert!(better.dominates(worse)); + assert!(!worse.dominates(better)); + assert!(!better.dominates(better)); + + let llm = ModelCandidate::llm_vote_only(3).expect("valid llm candidate"); + assert!(llm.is_llm_vote_only()); + assert!(!llm.is_statistically_supported()); + assert!(!llm.dominates(better)); + assert!(!better.dominates(llm)); + assert_eq!( + ModelCandidate::statistical(0, -1.0, 1.0), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + ModelCandidate::statistical(2, -1.0, -0.1), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(2, f64::NAN, 1.0), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(2, -1.0, f64::INFINITY), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::llm_vote_only(1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + } +} diff --git a/crates/model_selection/src/error.rs b/crates/model_selection/src/error.rs new file mode 100644 index 000000000..9c34391e7 --- /dev/null +++ b/crates/model_selection/src/error.rs @@ -0,0 +1,60 @@ +//! Fail-closed model-selection errors. + +use std::fmt; + +/// A fail-closed model-selection error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ModelSelectionError { + /// Candidate `K` was less than two. + NonPositiveCandidateK, + /// A diagnostic was non-finite or otherwise unusable. + InvalidDiagnostic, + /// No candidates were supplied. + EmptyCandidateSet, + /// An LLM vote was asked to define the numerical optimum. + LlmVoteIsNotStatisticalAuthority, +} + +impl fmt::Display for ModelSelectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::NonPositiveCandidateK => "candidate k must be at least two", + Self::InvalidDiagnostic => "invalid model-selection diagnostic", + Self::EmptyCandidateSet => "empty model-selection candidate set", + Self::LlmVoteIsNotStatisticalAuthority => "llm vote is not statistical authority", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for ModelSelectionError {} + +#[cfg(test)] +mod tests { + use super::ModelSelectionError; + + #[test] + fn error_messages_are_stable() { + for (error, message) in [ + ( + ModelSelectionError::NonPositiveCandidateK, + "candidate k must be at least two", + ), + ( + ModelSelectionError::InvalidDiagnostic, + "invalid model-selection diagnostic", + ), + ( + ModelSelectionError::EmptyCandidateSet, + "empty model-selection candidate set", + ), + ( + ModelSelectionError::LlmVoteIsNotStatisticalAuthority, + "llm vote is not statistical authority", + ), + ] { + assert_eq!(error.to_string(), message); + } + } +} diff --git a/crates/model_selection/src/gate.rs b/crates/model_selection/src/gate.rs new file mode 100644 index 000000000..72fc9aac3 --- /dev/null +++ b/crates/model_selection/src/gate.rs @@ -0,0 +1,123 @@ +//! Pareto admission and selection among statistically supported candidates. + +use crate::{ModelCandidate, ModelSelectionError}; + +/// Select the unique admissible `K` from a Pareto-filtered statistical front. +/// +/// LLM-only candidates are ignored as recommenders and never become the +/// numerical optimum. Among non-dominated statistical candidates the gate +/// prefers higher held-out log-likelihood, then smaller `K`; complexity is +/// applied while constructing the Pareto front. +/// +/// # Errors +/// +/// Returns [`ModelSelectionError::EmptyCandidateSet`] when no candidates are +/// supplied or [`ModelSelectionError::LlmVoteIsNotStatisticalAuthority`] when +/// every candidate is an LLM vote. +pub fn select_candidate_k(candidates: &[ModelCandidate]) -> Result { + if candidates.is_empty() { + return Err(ModelSelectionError::EmptyCandidateSet); + } + if candidates + .iter() + .all(|candidate| candidate.is_llm_vote_only()) + { + return Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority); + } + + let statistical: Vec = candidates + .iter() + .copied() + .filter(|candidate| candidate.is_statistically_supported()) + .collect(); + let mut front: Vec = statistical + .iter() + .copied() + .filter(|candidate| !statistical.iter().any(|other| other.dominates(*candidate))) + .collect(); + front.sort_by(|left, right| { + let ll_ord = right + .held_out_log_likelihood() + .partial_cmp(&left.held_out_log_likelihood()) + .unwrap_or(std::cmp::Ordering::Equal); + if ll_ord != std::cmp::Ordering::Equal { + return ll_ord; + } + left.candidate_k().cmp(&right.candidate_k()) + }); + Ok(front[0].candidate_k()) +} + +/// RMSE of selected `K` replications against a known-truth topic count. +/// +/// # Errors +/// +/// Returns [`ModelSelectionError::EmptyCandidateSet`] when `selected` is +/// empty, [`ModelSelectionError::NonPositiveCandidateK`] when `truth_k` is +/// less than two, or [`ModelSelectionError::InvalidDiagnostic`] when a +/// selected replication is less than two. +pub fn selected_k_root_mean_square_error( + selected: &[u32], + truth_k: u32, +) -> Result { + if selected.is_empty() { + return Err(ModelSelectionError::EmptyCandidateSet); + } + if truth_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + let mut sum_squares = 0.0_f64; + for selected_k in selected { + if *selected_k < 2 { + return Err(ModelSelectionError::InvalidDiagnostic); + } + let residual = f64::from(*selected_k) - f64::from(truth_k); + sum_squares += residual * residual; + } + Ok((sum_squares / selected.len() as f64).sqrt()) +} + +#[cfg(test)] +mod tests { + use super::{select_candidate_k, selected_k_root_mean_square_error}; + use crate::{ModelCandidate, ModelSelectionError}; + + #[test] + fn gate_helpers_cover_local_branches() { + let a = ModelCandidate::statistical(2, -30.0, 8.0).expect("a"); + let b = ModelCandidate::statistical(4, -30.0, 8.0).expect("b"); + assert_eq!( + select_candidate_k(&[]), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!(select_candidate_k(&[a, b]).expect("tie"), 2); + let higher_likelihood = ModelCandidate::statistical(8, -20.0, 9.0).expect("likelihood"); + assert_eq!( + select_candidate_k(&[a, higher_likelihood]).expect("likelihood tie-break"), + 8 + ); + + assert_eq!( + selected_k_root_mean_square_error(&[], 4), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!( + selected_k_root_mean_square_error(&[4], 1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + selected_k_root_mean_square_error(&[1], 4), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert!( + selected_k_root_mean_square_error(&[4], 4) + .expect("valid rmse") + .abs() + < f64::EPSILON + ); + assert_eq!( + select_candidate_k(&[ModelCandidate::llm_vote_only(3).expect("valid llm candidate")]), + Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority) + ); + } +} diff --git a/crates/model_selection/src/lib.rs b/crates/model_selection/src/lib.rs new file mode 100644 index 000000000..599829af7 --- /dev/null +++ b/crates/model_selection/src/lib.rs @@ -0,0 +1,22 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +// Selected-K RMSE casts small finite topic counts to `f64`. +#![allow(clippy::cast_precision_loss)] +//! Statistical and Pareto candidate-`K` gates for TRSL-TM model selection. +//! +//! Model selection uses held-out log-likelihood and complexity before any +//! blinded LLM review. An LLM vote may recommend among statistically +//! admissible candidates but never defines the numerical optimum (ADR 0012). + +mod candidate; +mod error; +mod gate; + +/// One candidate `K` with statistical or LLM-only support. +pub use candidate::ModelCandidate; +/// Fail-closed model-selection errors. +pub use error::ModelSelectionError; +/// Select the admissible candidate `K` from a Pareto-filtered statistical front. +pub use gate::select_candidate_k; +/// RMSE of selected `K` replications against known truth. +pub use gate::selected_k_root_mean_square_error; diff --git a/crates/model_selection/tests/crate_contract.rs b/crates/model_selection/tests/crate_contract.rs new file mode 100644 index 000000000..cd4ec7bae --- /dev/null +++ b/crates/model_selection/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `model_selection` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "model_selection"); +} diff --git a/crates/model_selection/tests/pareto_k_gate_contract.rs b/crates/model_selection/tests/pareto_k_gate_contract.rs new file mode 100644 index 000000000..52e43c7c7 --- /dev/null +++ b/crates/model_selection/tests/pareto_k_gate_contract.rs @@ -0,0 +1,138 @@ +//! Statistical/Pareto K gates are deterministic and recover known K without LLM authority. + +use model_selection::{ + ModelCandidate, ModelSelectionError, select_candidate_k, selected_k_root_mean_square_error, +}; + +fn candidate(k: u32, log_likelihood: f64, complexity: f64) -> ModelCandidate { + ModelCandidate::statistical(k, log_likelihood, complexity).expect("statistical candidate") +} + +fn synthetic_candidates(truth_k: u32, noisy_replication: bool) -> [ModelCandidate; 3] { + let true_log_likelihood = if noisy_replication { -20.0 } else { -10.0 }; + [ + candidate(truth_k, true_log_likelihood, f64::from(truth_k)), + candidate(truth_k - 1, -30.0, f64::from(truth_k - 1)), + candidate( + truth_k + 1, + if noisy_replication { -19.0 } else { -25.0 }, + f64::from(truth_k + 1), + ), + ] +} + +#[test] +fn non_positive_k_and_non_finite_diagnostics_fail_closed() { + assert_eq!( + ModelCandidate::statistical(1, -10.0, 4.0), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + ModelCandidate::statistical(3, f64::NAN, 4.0), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(3, -10.0, f64::INFINITY), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(3, -10.0, -0.1), + Err(ModelSelectionError::InvalidDiagnostic) + ); +} + +#[test] +fn llm_vote_cannot_define_the_numerical_optimum() { + let only_llm = ModelCandidate::llm_vote_only(5).expect("valid llm candidate"); + assert_eq!( + select_candidate_k(&[only_llm]), + Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority) + ); +} + +#[test] +fn pareto_front_selects_known_truth_k_with_computed_rmse() { + let truth_k = 4_u32; + let candidates = [ + candidate(2, -100.0, 10.0), + candidate(truth_k, -40.0, 20.0), + candidate(8, -45.0, 40.0), + ModelCandidate::llm_vote_only(6).expect("valid llm candidate"), + ]; + + let selected = select_candidate_k(&candidates).expect("admissible statistical front"); + assert_eq!(selected, truth_k); + + let rmse = selected_k_root_mean_square_error(&[selected], truth_k).expect("rmse"); + let expected = { + let residual = f64::from(selected) - f64::from(truth_k); + (residual * residual).sqrt() + }; + assert!((rmse - expected).abs() < f64::EPSILON); + assert!(rmse < 0.5); + assert_eq!( + select_candidate_k(&[candidate(2, -30.0, 8.0), candidate(4, -30.0, 8.0)]), + Ok(2) + ); + assert_eq!( + selected_k_root_mean_square_error(&[], truth_k), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!( + selected_k_root_mean_square_error(&[selected], 1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); +} + +#[test] +fn repeated_synthetic_truth_recovers_k_with_bounded_error_and_bias() { + let truth = [3_u32, 4, 5, 6, 7, 8]; + let selected: Vec = truth + .iter() + .enumerate() + .map(|(replication, truth_k)| { + select_candidate_k(&synthetic_candidates(*truth_k, replication == 4)) + .expect("synthetic statistical front") + }) + .collect(); + + let matching = truth + .iter() + .zip(&selected) + .filter(|(truth_k, selected_k)| truth_k == selected_k) + .count(); + let sum_squared_error: f64 = truth + .iter() + .zip(&selected) + .map(|(truth_k, selected_k)| { + let residual = f64::from(*selected_k) - f64::from(*truth_k); + residual * residual + }) + .sum(); + let bias: f64 = truth + .iter() + .zip(&selected) + .map(|(truth_k, selected_k)| f64::from(*selected_k) - f64::from(*truth_k)) + .sum::() + / f64::from(u32::try_from(truth.len()).expect("small fixture")); + + assert_eq!(selected, vec![3, 4, 5, 6, 8, 8]); + assert_eq!(matching, 5); + let replication_count = f64::from(u32::try_from(truth.len()).expect("small fixture")); + assert!((sum_squared_error / replication_count).sqrt() < 0.5); + assert!((bias - (1.0 / 6.0)).abs() < f64::EPSILON); + for (truth_k, selected_k) in truth.iter().zip(&selected) { + assert!( + selected_k_root_mean_square_error(&[*selected_k], *truth_k).expect("replication RMSE") + <= 1.0 + ); + } +} + +#[test] +fn empty_candidate_sets_abstain() { + assert_eq!( + select_candidate_k(&[]), + Err(ModelSelectionError::EmptyCandidateSet) + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index c783444c7..1e3b41f6c 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -27,9 +27,9 @@ The full APA 7th standards/literature register remains `docs/research/standards- | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | -| candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | -| compositional topic correlation / stable clustering | ADR 0005/0012; Aitchison (1982) | future `network_analysis` | accepted-target | -| posterior ESEM / longitudinal invariance / DSEM | ADR 0005; Asparouhov & Muthén (2009); Asparouhov et al. (2018); Marsh et al. (2014); AERA/APA/NCME (2014) | future `psychometric_core` | accepted-target | +| candidate K statistical/Pareto gates | ADR 0012; research | `model_selection` statistical/Pareto `K` gate on the active PR; candidate blinding, blinded LLM review, and backend comparison remain accepted-target | active-PR | +| compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | +| posterior ESEM / longitudinal invariance / DSEM | ADR 0005 | future `psychometric_core` | accepted-target | | CPU bounded multithreading + GPU/VRAM streaming/parity | ADR 0001/0006 | future `compute_backend` | accepted-target | | TDT detection/tracking | ADR 0016; Allan (2002) | future `event_intelligence` | accepted-target | | neural event-schema induction and prediction | ADR 0016; Li et al. (2021) | future `event_intelligence` | accepted-target | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index cca839523..5e89b068d 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** active-PR — `model_selection` statistical/Pareto candidate-`K` gates and known-`K` RMSE live in the new crate; remaining TRSL-TM estimator, global topic identity, method effects, and backend interchange remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/README.md b/docs/adr/README.md index d48f45f7c..d77ad42e6 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -20,7 +20,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | active-PR | Statistical/Pareto candidate-`K` gates in `model_selection` on the active PR; remaining topic estimator/backend/global-K contract remains accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented; checkpoint-versus-estimator refusal is `checkpoint_authority` on the active PR; full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | diff --git a/docs/research/model-selection-pareto-gates.md b/docs/research/model-selection-pareto-gates.md new file mode 100644 index 000000000..f7da5c823 --- /dev/null +++ b/docs/research/model-selection-pareto-gates.md @@ -0,0 +1,43 @@ +# Candidate-K statistical and Pareto gates (doctoring) + +## Scope + +`model_selection` admits a topic count `K` only when it is statistically +supported (`K >= 2`, finite held-out log-likelihood, finite non-negative +complexity) and not Pareto-dominated on those two objectives. An LLM vote may +later recommend among admissible candidates. It cannot itself define the +numerical optimum or bypass diagnostics (ADR 0012). + +This slice does not fit a topic model, choose a neural architecture, or claim a +unique true `K` for every corpus. Known-truth recovery reports computed RMSE of +the selected `K` against the generating `K`. + +## Authority + +### Normative TEPP contract + +- `docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md` — + model selection uses statistical/recovery/stability/alignment/fairness gates + and a Pareto-style comparison before any future blinded LLM review; the LLM + never defines the numerical optimum. + +### Supporting model-selection literature + +Akaike (1974) and Burnham and Anderson (2002) provide background for +likelihood-and-complexity comparison of fitted candidates. Deb et al. (2002) +provides background for non-dominated (Pareto) filtering when two objectives +are compared simultaneously. Those sources do not by themselves validate the +exact TEPP thresholds, acceptance criteria, or orchestration boundary; ADR +0012 is normative for this repository. They do **not** authorize an LLM vote as +a statistical estimator. + +Akaike, H. (1974). A new look at the statistical model identification. *IEEE +Transactions on Automatic Control, 19*(6), 716–723. +https://doi.org/10.1109/TAC.1974.1100705 + +Burnham, K. P., & Anderson, D. R. (2002). *Model selection and multimodel +inference: A practical information-theoretic approach* (2nd ed.). Springer. + +Deb, K., Pratap, A., Agarwal, S., & Meyarivan, T. (2002). A fast and elitist +multiobjective genetic algorithm: NSGA-II. *IEEE Transactions on Evolutionary +Computation, 6*(2), 182–197. https://doi.org/10.1109/4235.996017 diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index a66aae0b9..e215ddc44 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -46,7 +46,18 @@ Stammbach, D., Zouhar, V., Hoyle, A., Sachan, M., & Ash, E. (2023). Revisiting a Yang, X., Zhao, H., Phung, D., Buntine, W., & Du, L. (2025). LLM reading tea leaves: Automatically evaluating topic models with large language models. *Transactions of the Association for Computational Linguistics, 13*. -LLM evaluation complements but never replaces predictive, posterior, stability, alignment, fairness, recovery, and human-validation evidence. Candidates are blinded and statistically gated before LLM review. +Akaike, H. (1974). A new look at the statistical model identification. *IEEE Transactions on Automatic Control, 19*(6), 716–723. https://doi.org/10.1109/TAC.1974.1100705 + +Burnham, K. P., & Anderson, D. R. (2002). *Model selection and multimodel inference: A practical information-theoretic approach* (2nd ed.). Springer. + +Deb, K., Pratap, A., Agarwal, S., & Meyarivan, T. (2002). A fast and elitist multiobjective genetic algorithm: NSGA-II. *IEEE Transactions on Evolutionary Computation, 6*(2), 182–197. https://doi.org/10.1109/4235.996017 + +LLM evaluation complements but never replaces predictive, posterior, stability, +alignment, fairness, recovery, and human-validation evidence. The current +`model_selection` crate performs statistical/Pareto gating; candidate blinding +and blinded LLM review remain accepted-target extensions and are not executed +by this crate. Pareto-filtered held-out log-likelihood and complexity admit a +candidate `K`; an LLM vote cannot define the numerical optimum. ## Compositional data, correlation, and clusters diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 963a75a4c..89f817b96 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -26,6 +26,9 @@ This report tracks exact-head scientific and engineering evidence required befor | Mention-confidence Brier score | `event_core` | active-PR | calibration vs binary truth | perfect 0 / half 0.25 RMSE | ADR 0003; `docs/research/mention-confidence-brier.md` | | Checkpoint is not the estimator | `checkpoint_authority` | accepted-target | active PR | refuse checkpoint-as-estimator + unvalidated artifact + recovery vs estimator collapse | ADR 0001/0014 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | +| Candidate-K statistical/Pareto gates | `model_selection` | active-PR | this PR | known-K RMSE + LLM-vote refusal | ADR 0012; estimator/backend remaining | +| Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | +| Adaptive orchestration router | `tepp_api` | accepted-target | active PR | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | | TDT/CHRONOS evidence-status gates | `event_core` | active-PR | PR #50 | admission + first-story rates | known-stream miss/FA; full tracking/calibration/schema extraction remains future; ADR 0016; `docs/research/event-intelligence-status-gates.md` | | Purpose-bound provider payloads | `tepp_api` | implemented-main | — | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | partial | — | mode selection, document-control denial, ablation, credential-free bind; live NIM execution remains future | ADR 0010; `docs/research/adaptive-orchestration-router.md` | diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index a4da9db39..678d7e9d4 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "model_selection", "checkpoint_authority", ) diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index da59f7997..ce20b427e 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -25,6 +25,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) + self.assertEqual(len(crate_roots), 11) self.assertEqual(len(crate_roots), len(workspace_contract.EXPECTED_CRATES)) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots))