From 3a45cf1c50b20e80cc010794e431ca919bfe54af Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 21:54:23 +0500 Subject: [PATCH 1/2] =?UTF-8?q?test(cli):=20red=20=E2=80=94=20a=20shared?= =?UTF-8?q?=20source=20is=20parsed=20once=20per=20corpus=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full-corpus load benchmark over the 9,909-chunk ingest measured 111.7s and 9,909 source imports for only 400 unique sources — each tab parsed ~25×. A per-session sha256→Score cache cut it to 4.5s / 400 imports with an identical result (9,909 loaded both ways). `load_corpus_material_with` injects the importer so the count is observable. The test writes two chunks that share one source and asserts it is imported once; the loader currently imports per chunk, so it fails 2 != 1. The cache is the GREEN. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- cli/src/generation_input.rs | 177 ++++++++++++++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 7 deletions(-) diff --git a/cli/src/generation_input.rs b/cli/src/generation_input.rs index 74f2ec61..e912c935 100644 --- a/cli/src/generation_input.rs +++ b/cli/src/generation_input.rs @@ -10,12 +10,14 @@ //! **Experimental, `#[doc(hidden)]`**: a stability-exempt seam for tooling, not //! a public library surface. +use std::collections::HashMap; use std::fs; use std::path::Path; use griff_core::corpus::{source_sha256, ChunkMeta}; use griff_core::generation_input::{corpus_material, prepare_chunk, LoadedChunk}; use griff_core::import; +use griff_core::score::Score; pub use griff_core::generation_input::{ bar_rhythms, generation_request_from_score, gesture_control_from_chunks, CorpusMaterial, @@ -33,6 +35,17 @@ pub use griff_core::generation_input::{ /// # Errors /// [`GenerationInputError::Corpus`] when `dir` cannot be read. pub fn load_corpus_material(dir: &Path) -> Result { + load_corpus_material_with(dir, |bytes| import::import_score_auto(bytes).ok()) +} + +/// The load loop with the source importer injected, so a test can count how +/// often a source is parsed. The corpus is ~25x redundant (≈400 sources behind +/// ≈9,900 chunks), so parsing each tab once instead of once per chunk is the +/// difference between a ~112 s and a ~4.5 s full load — measured, not guessed. +fn load_corpus_material_with( + dir: &Path, + mut import: impl FnMut(&[u8]) -> Option, +) -> Result { let entries = fs::read_dir(dir).map_err(|e| { GenerationInputError::Corpus(format!("cannot read corpus dir {}: {e}", dir.display())) })?; @@ -43,10 +56,11 @@ pub fn load_corpus_material(dir: &Path) -> Result = HashMap::new(); let mut loaded = Vec::new(); let mut skipped = Vec::new(); for name in record_names { - match load_chunk(dir, &name) { + match load_chunk(dir, &name, &mut cache, &mut import) { Some(chunk) => loaded.push(chunk), None => skipped.push(name), } @@ -54,11 +68,17 @@ pub fn load_corpus_material(dir: &Path) -> Result Option { +/// Reads one chunk record and prepares it through core, importing its source +/// tab (or reusing an already-parsed one from `cache`). `None` when the record +/// does not parse, the source is missing/unimportable/hash-mismatched, or the +/// prepared slice carries no sounding track — the caller reports it as skipped. +fn load_chunk( + dir: &Path, + record_name: &str, + cache: &mut HashMap, + import: &mut impl FnMut(&[u8]) -> Option, +) -> Option { + let _ = cache; let meta: ChunkMeta = serde_json::from_str(&fs::read_to_string(dir.join(record_name)).ok()?).ok()?; let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; @@ -70,6 +90,149 @@ fn load_chunk(dir: &Path, record_name: &str) -> Option { return None; } } - let source = import::import_score_auto(&bytes).ok()?; + let source = import(&bytes)?; prepare_chunk(meta, &source) } + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::unwrap_used)] + + use super::{import, load_corpus_material_with, source_sha256, Score}; + use griff_core::corpus::{ChunkId, ChunkMeta, SourceFormat, SourceRef}; + use griff_core::event::{NoteMarks, Pitch, Tempo, Ticks, TimeSignature, Tuning, Velocity}; + use griff_core::midi; + use griff_core::score::{ + AtomEvent, AtomNote, EventGroup, EventGroupKind, LossReport, MasterBar, RepeatMarker, + Track, Voice, + }; + use griff_core::slice::TickRange; + use std::cell::Cell; + use std::{env, fs, process}; + + fn note(start: u32, pitch: u8) -> AtomEvent { + AtomEvent::Note(AtomNote { + absolute_start: Ticks(start), + duration: Ticks(480), + pitch: Pitch::new(pitch).unwrap(), + velocity: Velocity::new(90).unwrap(), + marks: NoteMarks::empty(), + position: None, + }) + } + + fn two_bar_source() -> Score { + let master_bars = (0..2usize) + .map(|i| { + let start = u32::try_from(i).unwrap().saturating_mul(1920); + MasterBar { + index: i, + tick_range: TickRange::new(Ticks(start), Ticks(start.saturating_add(1920))) + .unwrap(), + time_signature: TimeSignature { + numerator: 4, + denominator: 4, + }, + tempo: Tempo::new(120.0).unwrap(), + repeat: RepeatMarker::default(), + } + }) + .collect(); + let atoms = [ + note(0, 40), + note(480, 43), + note(960, 45), + note(1440, 47), + note(1920, 50), + note(2400, 47), + note(2880, 45), + note(3360, 43), + ]; + Score { + ticks_per_quarter: 480, + master_bars, + tracks: vec![Track { + name: None, + channel: 0, + voices: vec![Voice { + id: 0, + event_groups: atoms + .into_iter() + .map(|a| EventGroup { + kind: EventGroupKind::Single, + atoms: vec![a], + technique_spans: Vec::new(), + }) + .collect(), + }], + tuning: Tuning::standard_e(), + }], + source_meta: None, + loss: LossReport::new(), + } + } + + fn chunk_meta(id: &str, sha: &str, bars: (u32, u32)) -> ChunkMeta { + ChunkMeta { + id: ChunkId(id.to_owned()), + title: String::new(), + source: SourceRef { + filename: "s.mid".to_owned(), + format: SourceFormat::Midi, + bar_range: Some(bars), + track_index: Some(0), + sha256: Some(sha.to_owned()), + }, + tempo_bpm: 120.0, + ticks_per_quarter: 480, + time_signature: (4, 4), + tuning: "standard_e".to_owned(), + tags: Vec::new(), + boundaries: Vec::new(), + techniques: Vec::new(), + quality_flags: Vec::new(), + reviewer: None, + structure: None, + gesture: None, + complexity: None, + duplicate: None, + style_cohort: None, + ensemble: None, + rights: None, + created_at: String::new(), + updated_at: String::new(), + } + } + + #[test] + fn one_source_is_parsed_once_for_all_its_chunks() { + let dir = env::temp_dir().join(format!("griff_loadcache_{}", process::id())); + fs::create_dir_all(&dir).unwrap(); + let bytes = midi::export_score(&two_bar_source()).unwrap(); + let sha = source_sha256(&bytes); + fs::write(dir.join("s.mid"), &bytes).unwrap(); + for (id, bars) in [("a", (0_u32, 0_u32)), ("b", (1, 1))] { + fs::write( + dir.join(format!("{id}.chunk.json")), + serde_json::to_string(&chunk_meta(id, &sha, bars)).unwrap(), + ) + .unwrap(); + } + + let imports = Cell::new(0_usize); + let material = load_corpus_material_with(&dir, |b| { + imports.set(imports.get().saturating_add(1)); + import::import_score_auto(b).ok() + }) + .expect("corpus loads"); + + assert_eq!(material.references.len(), 2, "both chunks load"); + assert_eq!( + imports.get(), + 1, + "the shared source is parsed once, not once per chunk" + ); + + fs::remove_dir_all(&dir).ok(); + } +} From 4b5520bec15856c53ad34d10caf2bb5ad9796d89 Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 21:55:32 +0500 Subject: [PATCH 2/2] =?UTF-8?q?fix(cli):=20green=20=E2=80=94=20cache=20par?= =?UTF-8?q?sed=20sources=20by=20content=20hash=20within=20a=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `load_chunk` keys the parsed `Score` by the source's sha256 (filename for pre-v9 records) in a per-load `HashMap`, so a tab shared by many chunks is parsed once. The count test passes (1 import for 2 chunks), and the existing correctness test is unchanged — `prepare_chunk` slices an immutable `&Score`, so reuse is bit-identical to per-chunk parsing. The hash check lives on the cache-miss path, so a mismatched source is still a strict load failure, never bypassed by reuse. Full-corpus effect (measured): 111.7s → 4.5s, 9,909 source imports → 400, same 9,909 chunks loaded. No persisted-schema change; the cockpit's OPFS loader is a separate path and unaffected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- cli/src/generation_input.rs | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/cli/src/generation_input.rs b/cli/src/generation_input.rs index e912c935..4a18af64 100644 --- a/cli/src/generation_input.rs +++ b/cli/src/generation_input.rs @@ -78,20 +78,31 @@ fn load_chunk( cache: &mut HashMap, import: &mut impl FnMut(&[u8]) -> Option, ) -> Option { - let _ = cache; let meta: ChunkMeta = serde_json::from_str(&fs::read_to_string(dir.join(record_name)).ok()?).ok()?; - let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; - // A filename is not an identity: when the record pins the source's hash - // (schema v9), a same-named but different file must not silently supply the - // notes. A mismatch is a load failure, reported like a missing source. - if let Some(expected) = &meta.source.sha256 { - if &source_sha256(&bytes) != expected { - return None; + // Key the parsed source by its content hash (v9) — falling back to the + // filename for pre-v9 records — so every chunk of one tab reuses a single + // parse. Determinism is unchanged: `prepare_chunk` slices an immutable + // `&Score`, so a shared parse yields exactly the per-chunk-parse result. + let key = meta + .source + .sha256 + .clone() + .unwrap_or_else(|| meta.source.filename.clone()); + if !cache.contains_key(&key) { + let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; + // A filename is not an identity: when the record pins the source's hash + // (schema v9), a same-named but different file must not silently supply + // the notes. A mismatch is a load failure — and a cache miss, so the + // check runs for every distinct expected hash, never bypassed by reuse. + if let Some(expected) = &meta.source.sha256 { + if &source_sha256(&bytes) != expected { + return None; + } } + cache.insert(key.clone(), import(&bytes)?); } - let source = import(&bytes)?; - prepare_chunk(meta, &source) + prepare_chunk(meta, cache.get(&key)?) } #[cfg(test)]