diff --git a/cli/src/main.rs b/cli/src/main.rs index 2ed18240..93a0262f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -958,6 +958,9 @@ fn chunks_for_segments( u32::try_from(last).unwrap_or(0), )); let duplicate = duplicates.get(phrase).copied().flatten(); + // Persist the link onto the record too (#76, schema v8), so a + // downloaded chunk / manifest keeps it, not just the CLI print. + meta.duplicate = duplicate; PhraseChunk { meta, duplicate } }) .collect() @@ -1130,6 +1133,7 @@ fn build_chunk_meta( structure, gesture, complexity, + duplicate: None, style_cohort: Some(inputs.style_cohort), ensemble, rights: Some(inputs.rights.clone()), @@ -1956,6 +1960,13 @@ mod tests { .expect("phrase 2 near-duplicates phrase 0"); assert_eq!(dup.of, 0); assert!(dup.quote_share >= 0.8, "share {}", dup.quote_share); + // The link is also mirrored onto the persisted record (#76, schema v8), + // so a downloaded chunk / manifest keeps it — not just the CLI print. + assert_eq!( + chunks[2].meta.duplicate, chunks[2].duplicate, + "the duplicate link is persisted on the ChunkMeta" + ); + assert!(chunks[0].meta.duplicate.is_none()); } #[test] diff --git a/core/src/corpus.rs b/core/src/corpus.rs index ac0b142c..f7ef8fc5 100644 --- a/core/src/corpus.rs +++ b/core/src/corpus.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::complement::AxisScores; use crate::gesture::GestureStats; +use crate::novelty::PhraseDuplicate; use crate::structure::{ComplexityProfile, StructureMetrics}; /// Current corpus schema version. @@ -36,6 +37,11 @@ use crate::structure::{ComplexityProfile, StructureMetrics}; /// `rights` key) keep loading and re-serialize losslessly. Rights status is /// not derivable from content, so it is captured at curation time and cannot /// be backfilled. +/// - v8 — persisted near-duplicate link (#76): `ChunkMeta` gains optional +/// [`PhraseDuplicate`] under the same pattern; pre-v8 records (no `duplicate` +/// key) keep loading and re-serialize losslessly. It records which earlier +/// split phrase a later one repeats — a curation signal previously surfaced +/// only in the UI/CLI and dropped on download. /// /// Tag taxonomy is intentionally *not* versioned here: [`SwancoreTag`] grows /// additively (e.g. `let_ring`, #75) and `SCHEMA_VERSION` tracks structural @@ -43,7 +49,7 @@ use crate::structure::{ComplexityProfile, StructureMetrics}; /// not the tag set — a new tag only breaks readers that hard-reject unknown /// variants, a curation-tooling concern, not a corpus-structure one /// (decisions 2026-06-19). -pub const SCHEMA_VERSION: u32 = 7; +pub const SCHEMA_VERSION: u32 = 8; // ── identifiers ─────────────────────────────────────────────────────────────── @@ -354,6 +360,14 @@ pub struct ChunkMeta { /// as `null` — when unmeasured, so pre-v6 files round-trip byte-identically. #[serde(default, skip_serializing_if = "Option::is_none")] pub complexity: Option, + /// Near-duplicate link (schema v8, #76): the earlier split phrase this one + /// verbatim-quotes (transposition-aware) and by how much, or absent when the + /// phrase is distinct or was captured outside a split. `of` indexes the + /// phrase within the same split run — it pairs with the `_p` id suffix. + /// Skipped — not written as `null` — when absent, so pre-v8 files round-trip + /// byte-identically. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duplicate: Option, /// Style cohort (schema v4). Absent in pre-v4 records — unlabeled; the /// key is skipped when unset, so older files round-trip byte-identically. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/core/src/novelty.rs b/core/src/novelty.rs index 6907d05c..1f5f3e1b 100644 --- a/core/src/novelty.rs +++ b/core/src/novelty.rs @@ -30,6 +30,8 @@ use std::collections::BTreeSet; +use serde::{Deserialize, Serialize}; + use crate::score::{AtomEvent, LossReport, Score, Track}; use crate::scoring::{Axes, Axis, WeightPolicy}; @@ -193,7 +195,7 @@ pub fn novelty_weights_v1() -> WeightPolicy { pub const PHRASE_DUPLICATE_SHARE: f64 = 0.8; /// A phrase flagged as a near-duplicate of an earlier one (#76). -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct PhraseDuplicate { /// Index, within the phrase list, of the earlier phrase it most closely quotes. pub of: usize, diff --git a/core/tests/corpus_schema.rs b/core/tests/corpus_schema.rs index 5cf6def3..b53ca970 100644 --- a/core/tests/corpus_schema.rs +++ b/core/tests/corpus_schema.rs @@ -14,6 +14,7 @@ use griff_core::corpus::{ StyleCohort, SwancoreTag, SCHEMA_VERSION, }; use griff_core::gesture::GestureStats; +use griff_core::novelty::PhraseDuplicate; use griff_core::structure::{ComplexityProfile, StructureMetrics}; use proptest::{collection::vec as prop_vec, option::of as prop_opt, prelude::*}; @@ -85,6 +86,7 @@ fn minimal_chunk() -> ChunkMeta { structure: None, gesture: None, complexity: None, + duplicate: None, style_cohort: None, ensemble: None, rights: None, @@ -93,16 +95,44 @@ fn minimal_chunk() -> ChunkMeta { } } -// ── schema v7: rights + provenance (decisions 2026-06-12) ───────────────────── +// ── schema v8: near-duplicate link (#76) ────────────────────────────────────── + +#[test] +fn schema_version_is_8() { + assert_eq!( + SCHEMA_VERSION, 8, + "the persisted near-duplicate link bumps the corpus schema" + ); +} #[test] -fn schema_version_is_7() { +fn chunk_meta_persists_and_skips_the_near_duplicate_link() { + // Absent by default → the key is skipped (not written as null), so pre-v8 + // records round-trip byte-identically. + let plain_json = serde_json::to_string(&minimal_chunk()).expect("serialize"); + assert!( + !plain_json.contains("duplicate"), + "an unset duplicate link is skipped, not null: {plain_json}" + ); + + // Present → a nested object that round-trips losslessly. + let mut flagged = minimal_chunk(); + flagged.duplicate = Some(PhraseDuplicate { + of: 1, + quote_share: 0.9375, + }); + let json = serde_json::to_string(&flagged).expect("serialize"); + assert!(json.contains("\"duplicate\""), "{json}"); + assert!(json.contains("\"of\":1"), "{json}"); + let back: ChunkMeta = serde_json::from_str(&json).expect("deserialize"); assert_eq!( - SCHEMA_VERSION, 7, - "the rights record bumps the corpus schema" + back, flagged, + "the near-duplicate link round-trips losslessly" ); } +// ── schema v7: rights + provenance (decisions 2026-06-12) ───────────────────── + /// A representative rights record (the common scraped-community-tab case). fn sample_rights() -> RightsInfo { RightsInfo { @@ -732,6 +762,24 @@ fn arb_rights() -> impl Strategy> { prop_opt(info) } +/// Strategy: an optional near-duplicate link with a dyadic quote share, so the +/// JSON round-trip stays byte-identical (#76, schema v8). +fn arb_duplicate() -> impl Strategy> { + prop_opt( + ( + 0_usize..16, + prop_oneof![ + Just(0.0_f64), + Just(0.5), + Just(0.75), + Just(0.9375), + Just(1.0) + ], + ) + .prop_map(|(of, quote_share)| PhraseDuplicate { of, quote_share }), + ) +} + proptest! { #[test] fn prop_chunk_meta_json_roundtrip( @@ -753,6 +801,7 @@ proptest! { style_cohort in arb_cohort(), ensemble in arb_ensemble(), rights in arb_rights(), + duplicate in arb_duplicate(), ) { let meta = ChunkMeta { id: ChunkId(id), @@ -774,6 +823,7 @@ proptest! { structure, gesture, complexity, + duplicate, style_cohort, ensemble, rights, diff --git a/core/tests/similarity.rs b/core/tests/similarity.rs index 8c600d45..3c21dae2 100644 --- a/core/tests/similarity.rs +++ b/core/tests/similarity.rs @@ -144,6 +144,7 @@ fn chunk( structure, gesture, complexity, + duplicate: None, style_cohort: None, ensemble: None, rights: None, diff --git a/docs/decisions.log.md b/docs/decisions.log.md index 855b6295..e9c09850 100644 --- a/docs/decisions.log.md +++ b/docs/decisions.log.md @@ -1303,3 +1303,17 @@ Architectural decisions go to [`adr/`](adr/) instead. beat count), finer-than-eighth and triplet grids, and bars whose beat is not an even tick count go unmeasured in this first cut (under-tagging, never mis-tagging). + +- 2026-06-21 — In the context of persisting curation signals to the corpus, + facing that the split's near-duplicate flag (#76) lived only in the live + UI/CLI and the split envelope — so a downloaded `chunk.json` or a built + manifest lost which phrases are repeats — we decided for an optional + `ChunkMeta.duplicate` (`Option`) under the established + additive pattern (serde `default` + `skip_serializing_if`), bumping + `SCHEMA_VERSION` to 8, and against leaving it envelope-only or storing the + referenced chunk's full id, to achieve a corpus that keeps the repeat + relationship for dedup/curation, accepting that `duplicate.of` is an index + within the same split run (it pairs with the `_p` id suffix) and is + meaningful only alongside its sibling phrases. Unlike the #75 tag additions — + data within an existing field, deliberately *not* versioned — this is a + structural `ChunkMeta` field, so it bumps the schema like v2–v7 before it. diff --git a/preview/src/curation.rs b/preview/src/curation.rs index 45b79c3c..94622d0c 100644 --- a/preview/src/curation.rs +++ b/preview/src/curation.rs @@ -281,13 +281,16 @@ fn ticks_per_bar(meta: &ChunkMeta) -> u32 { .unwrap_or(0) } -/// Resets what a changed extent invalidates: the reviewer decision and the -/// whole-extent measurements (structure, gesture, complexity). +/// Resets what a changed extent invalidates: the reviewer decision, the +/// whole-extent measurements (structure, gesture, complexity), and the +/// near-duplicate link (its `of` indexes the original split's siblings, so it +/// is meaningless once the extent is re-cut). const fn fresh_extent(mut meta: ChunkMeta) -> ChunkMeta { meta.reviewer = None; meta.structure = None; meta.gesture = None; meta.complexity = None; + meta.duplicate = None; meta } diff --git a/preview/tests/curation.rs b/preview/tests/curation.rs index b00c7f5c..6a6b7836 100644 --- a/preview/tests/curation.rs +++ b/preview/tests/curation.rs @@ -40,6 +40,7 @@ fn record() -> ChunkMeta { structure: None, gesture: None, complexity: None, + duplicate: None, style_cohort: None, ensemble: None, rights: None, @@ -371,6 +372,33 @@ fn split_record_clears_the_ensemble_link() { } } +// TDD red phase (Codex P2, PR #96): a near-duplicate link's `of` indexes the +// original split's sibling phrases; once an extent is split (or merged) the +// halves take new ids and bar ranges, so that pointer goes stale. The split +// must clear it alongside the other extent-invalidated metadata. Fails until +// `fresh_extent` resets it. + +#[test] +fn split_record_clears_the_near_duplicate_link() { + use griff_core::novelty::PhraseDuplicate; + use griff_preview::curation::split_record; + + let mut meta = record_with_range(0, 4); + meta.duplicate = Some(PhraseDuplicate { + of: 1, + quote_share: 0.95, + }); + let json = serde_json::to_string(&meta).expect("serialize"); + let (a, b) = split_record(&json, 2, 2).expect("split ok"); + for half in [a, b] { + let half: ChunkMeta = serde_json::from_str(&half).expect("parses"); + assert_eq!( + half.duplicate, None, + "a stale sibling pointer cannot survive a split" + ); + } +} + #[test] fn split_record_rejects_an_out_of_range_point() { use griff_preview::curation::split_record; diff --git a/web/src/lib.rs b/web/src/lib.rs index 4a83e466..d6e09389 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -577,6 +577,7 @@ fn build_chunk_meta_record( structure: structure::measure_structure(score, track_index).ok(), gesture: gesture::measure_gesture(score, track_index).ok(), complexity: structure::measure_complexity(score, track_index).ok(), + duplicate: None, style_cohort: Some(cohort_from(cohort)), ensemble: None, rights: Some(RightsInfo { @@ -747,6 +748,9 @@ fn split_segments_to_json( let bar_lo = u32::try_from(*start).unwrap_or(0); let bar_hi = u32::try_from(end.saturating_sub(1)).unwrap_or(0); meta.source.bar_range = Some((bar_lo, bar_hi)); + // Persist the near-duplicate link onto the record too (#76, schema v8), + // so a downloaded chunk keeps it — not just the envelope below. + meta.duplicate = dups.get(phrase).copied().flatten(); let pretty = serde_json::to_string_pretty(&meta) .unwrap_or_else(|e| format!("{{\"error\":\"serialize: {e}\"}}"));