diff --git a/middleware/package-lock.json b/middleware/package-lock.json index 77e63aa6c..1ba34646e 100644 --- a/middleware/package-lock.json +++ b/middleware/package-lock.json @@ -5215,9 +5215,9 @@ } }, "node_modules/hono": { - "version": "4.12.23", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.23.tgz", - "integrity": "sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==", + "version": "4.12.25", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz", + "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==", "license": "MIT", "engines": { "node": ">=16.9.0" diff --git a/middleware/packages/harness-orchestrator-extras/src/contextRetriever.ts b/middleware/packages/harness-orchestrator-extras/src/contextRetriever.ts index c164d0e1d..1276b1db4 100644 --- a/middleware/packages/harness-orchestrator-extras/src/contextRetriever.ts +++ b/middleware/packages/harness-orchestrator-extras/src/contextRetriever.ts @@ -710,7 +710,7 @@ export class ContextRetriever { const seenInsightIds = new Set(); const insights: RecalledInsight[] = []; for (const hit of durableHits) { - const insight = toRecalledInsight(hit); + const insight = toRecalledInsight(hit, true); if (seenInsightIds.has(insight.mkId)) continue; seenInsightIds.add(insight.mkId); insights.push(insight); @@ -1479,7 +1479,12 @@ function renderRecallBlocks( if (recalled.insights.length > 0 && budget > 200) { push('\n## Aus früheren Sessions — verwandte Erkenntnisse'); for (const ins of recalled.insights) { - const chunk = `- ${ins.kind}: ${truncate(ins.summary, 300)} (score ${ins.score.toFixed(2)})`; + // Durable (curated schema/reference) insights render in full so the + // agent can rely on them instead of re-running discovery tools; fuzzy + // insights stay capped. `ins.summary` is already bounded upstream by + // toRecalledInsight (durable → 2000, fuzzy → 300). + const rendered = ins.durable ? ins.summary : truncate(ins.summary, 300); + const chunk = `- ${ins.kind}: ${rendered} (score ${ins.score.toFixed(2)})`; if (!push(chunk)) break; } } @@ -1487,12 +1492,25 @@ function renderRecallBlocks( return parts.join('\n'); } -function toRecalledInsight(hit: MemoryRecallHit): RecalledInsight { +/** Full render length for DURABLE insights — curated schema/reference + * knowledge must reach the agent complete, or it re-discovers (e.g. re-runs + * `dynamics_describe`) what it already knows. Fuzzy insights stay capped. */ +const DURABLE_SUMMARY_MAX_CHARS = 2000; +const FUZZY_SUMMARY_MAX_CHARS = 300; + +function toRecalledInsight( + hit: MemoryRecallHit, + durable = false, +): RecalledInsight { return { mkId: hit.mk.id, kind: String(hit.mk.props['kind'] ?? 'memory'), - summary: truncate(String(hit.mk.props['summary'] ?? ''), 300), + summary: truncate( + String(hit.mk.props['summary'] ?? ''), + durable ? DURABLE_SUMMARY_MAX_CHARS : FUZZY_SUMMARY_MAX_CHARS, + ), score: hit.score, + ...(durable ? { durable: true } : {}), }; } diff --git a/middleware/packages/harness-orchestrator-extras/src/excerptExtractor.ts b/middleware/packages/harness-orchestrator-extras/src/excerptExtractor.ts index 176a7b6d6..f78af6546 100644 --- a/middleware/packages/harness-orchestrator-extras/src/excerptExtractor.ts +++ b/middleware/packages/harness-orchestrator-extras/src/excerptExtractor.ts @@ -73,7 +73,18 @@ Definitions: "decision" — A choice was made or recommended ("we will use X", "go with Y"). "insight" — A non-obvious finding or learning ("turns out the API caps at 100/min"). "preference" — A stated user/team preference ("always reply in German first"). - "reference" — Stable how-to / SOP / lookup material ("to deploy: run X then Y"). + "reference" — Stable, reusable lookup material that does NOT change per + request: how-to / SOP ("to deploy: run X then Y"), AND — + importantly — **data-model / schema / domain conventions**: + which table or entity holds which data, field names and + their meaning, entity-set names, how to filter/join, naming + rules. E.g. "Courses live in the Dynamics table ud_tutorial + (entitySet ud_tutorials); fields ud_name, ud_coursenumber, + ud_startdatetime; bookings in ud_booking". This kind of + learned structure is long-lived knowledge the agent must NOT + re-discover every session — classify it as "reference". + (A time-bound DATA snapshot — "29 courses next week" — is an + "insight", NOT a reference.) summary — Stand-alone sentence(s) the user could read in /memories years later. Do NOT start with "The user asked about…" — describe the answer's substance. rationale — Why it matters / preconditions / caveats. Use null when redundant with summary. diff --git a/middleware/packages/harness-orchestrator-extras/src/mergeCandidateDetector.ts b/middleware/packages/harness-orchestrator-extras/src/mergeCandidateDetector.ts index 3a246f61a..837441338 100644 --- a/middleware/packages/harness-orchestrator-extras/src/mergeCandidateDetector.ts +++ b/middleware/packages/harness-orchestrator-extras/src/mergeCandidateDetector.ts @@ -43,6 +43,17 @@ export interface MergeCandidateDetectorDeps { excerptMinSimilarity?: number; /** Max candidates checked per source. Default 5. */ topK?: number; + /** + * Auto-merge threshold. When set, a flagged MK pair whose cosine is at or + * above this value is RESOLVED automatically (the duplicate is retired via + * `resolveMergeCandidate`) instead of only being flagged for an operator — + * so re-learned knowledge stops accumulating on any deployment without a + * manual cleanup pass. SAFETY: a `manuallyAuthored` (durable) node is NEVER + * deleted; when exactly one side is durable it always wins; when BOTH are + * durable the pair is left for an operator; otherwise the OLDER node wins. + * Undefined → auto-merge disabled (flag-only, legacy behaviour). + */ + autoMergeThreshold?: number; log?: (msg: string) => void; } @@ -59,6 +70,24 @@ export function createMergeCandidateDetector( const excerptMinSim = deps.excerptMinSimilarity ?? DEFAULT_EXCERPT_MIN_SIMILARITY; const topK = deps.topK ?? DEFAULT_TOP_K; + const autoMergeThreshold = deps.autoMergeThreshold; + + /** + * Pick which of two near-duplicate MKs to KEEP, or null to leave for an + * operator. Durable (`manuallyAuthored`) is sacred: it is never the loser, + * and two durable nodes are never auto-merged. Otherwise the older node wins + * (it's the established one; the fresh re-statement is the duplicate). + */ + function decideKeeper(a: GraphNode, b: GraphNode): string | null { + const aDur = a.manuallyAuthored === true; + const bDur = b.manuallyAuthored === true; + if (aDur && bDur) return null; // both curated — hands off + if (aDur !== bDur) return aDur ? a.id : b.id; // the durable one wins + const aAt = String(a.props['created_at'] ?? ''); + const bAt = String(b.props['created_at'] ?? ''); + if (aAt && bAt && aAt !== bAt) return aAt < bAt ? a.id : b.id; // older wins + return b.id; // tie / unknown → keep the existing candidate, retire source + } async function detectFor( memorableKnowledgeNodeId: string, @@ -137,6 +166,38 @@ export function createMergeCandidateDetector( log( `[merge-detector] flagged ${source.id} vs ${candidate.mk.id} cosine=${candidate.cosineSim.toFixed(3)}`, ); + // Auto-merge: high-confidence + safe → retire the duplicate now so + // re-learned knowledge stops accumulating without a manual sweep. + if ( + autoMergeThreshold !== undefined && + candidate.cosineSim >= autoMergeThreshold + ) { + const keeper = decideKeeper(source, candidate.mk); + if (keeper === null) { + log( + `[merge-detector] auto-merge SKIP ${source.id} vs ${candidate.mk.id}: both durable`, + ); + } else { + // duplicateOf is sorted ascending; keep_a keeps [0] (deletes [1]), + // keep_b keeps [1] (deletes [0]). + const resolution = + persisted.duplicateOf[0] === keeper ? 'keep_a' : 'keep_b'; + try { + await deps.graph.resolveMergeCandidate( + persisted.id, + resolution, + { actorOmadiaUserId: viewer }, + ); + log( + `[merge-detector] auto-merged cosine=${candidate.cosineSim.toFixed(3)} kept=${keeper} (retired the duplicate)`, + ); + } catch (err) { + log( + `[merge-detector] auto-merge FAILED for ${persisted.id} (left flagged): ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + } } } catch (err) { log( diff --git a/middleware/packages/harness-orchestrator-extras/src/plugin.ts b/middleware/packages/harness-orchestrator-extras/src/plugin.ts index a3e6ab824..ed67eaa79 100644 --- a/middleware/packages/harness-orchestrator-extras/src/plugin.ts +++ b/middleware/packages/harness-orchestrator-extras/src/plugin.ts @@ -437,11 +437,36 @@ export async function activate( // order: capture-filter → inconsistency-trigger → merge-trigger // (outermost). Cosine-only, no Anthropic dependency. Always active // when embeddingClient is wired. + // Slice 13 — AUTOMATIC dedup. When enabled (default on), the merge detector + // RESOLVES high-confidence duplicate MK pairs itself (retires the duplicate; + // durable `manuallyAuthored` nodes are never deleted) instead of only + // flagging for an operator — so re-learned knowledge stops accumulating on + // ANY deployment without a manual cleanup pass. Aggressive default 0.90 so + // paraphrased re-statements also merge. The detector's flag floor is lowered + // to the same threshold so sub-0.95 near-dups actually surface to be merged. + // Disable with kg_auto_merge_enabled=false (reverts to flag-only at 0.95). + const autoMergeEnabledRaw = + ctx.config.get('kg_auto_merge_enabled') ?? + process.env['KG_AUTO_MERGE_ENABLED']; + const autoMergeDisabled = + typeof autoMergeEnabledRaw === 'string' + ? autoMergeEnabledRaw.toLowerCase() === 'false' + : autoMergeEnabledRaw === false; + const autoMergeThreshold = parseNumberOrDefault( + ctx.config.get('kg_auto_merge_threshold'), + 0.9, + ); const mergeCandidateDetector = createMergeCandidateDetector({ graph: inconsistencyWrappedKg, ...(embeddingClient ? { embeddingClient } : {}), + ...(autoMergeDisabled + ? {} + : { autoMergeThreshold, minSimilarity: autoMergeThreshold }), log: (msg) => { console.error(msg); }, }); + ctx.log( + `[harness-orchestrator-extras] auto-merge ${autoMergeDisabled ? 'off (flag-only @0.95)' : `on (resolve >= ${autoMergeThreshold.toFixed(2)}, durable-protected)`}`, + ); const disposeMergeCandidateDetector = ctx.services.provide( MERGE_CANDIDATE_DETECTOR_SERVICE_NAME, mergeCandidateDetector, diff --git a/middleware/packages/harness-orchestrator-extras/src/promotion.ts b/middleware/packages/harness-orchestrator-extras/src/promotion.ts index 93a51cb8a..7e60d37d3 100644 --- a/middleware/packages/harness-orchestrator-extras/src/promotion.ts +++ b/middleware/packages/harness-orchestrator-extras/src/promotion.ts @@ -86,6 +86,7 @@ export interface PromoteTurnResult { | 'promoted' | 'below-threshold' | 'no-significance' + | 'hygiene-skip' | 'already-promoted' | 'missing-user' | 'missing-turn' @@ -176,16 +177,28 @@ export async function promoteTurnIfSignificant( input.fallbackAssistantAnswer, ); + // Ingest hygiene — applied to ALL auto-harvest, not just the durable tier. + // First-person agent narration ("Ich schaue kurz in den Memory…") and + // trivially short fragments scored high enough to clear the significance + // threshold and were being stored as fuzzy MK every session, re-polluting + // recall. Drop them entirely so they never enter the KG. + if (!passesIngestHygiene(summary)) { + log( + `[promotion] skip turn=${input.turnId} reason=hygiene (agent narration) significance=${significance.toFixed(2)}`, + ); + return { promoted: false, reason: 'hygiene-skip', significance }; + } + // Trigger T3 — durable auto-promotion gate. Conservative by design: only - // high-significance reference knowledge that survives the hygiene check is - // marked durable, so conversational narration + time-bound snapshots never - // re-pollute the always-surface tier. + // high-significance, substantial reference knowledge is marked durable, so + // time-bound snapshots / short fragments never reach the always-surface + // tier (narration is already dropped above). const durableKinds = input.durableKinds ?? ['reference']; const durable = input.durableMinSignificance !== undefined && significance >= input.durableMinSignificance && durableKinds.includes(kind) && - passesDurableHygiene(summary, rationale); + isDurableContentLengthOk(summary, rationale); const result = await input.kg.createMemorableKnowledge({ kind, @@ -231,23 +244,27 @@ export async function promoteTurnIfSignificant( } /** - * Hygiene gate for durable auto-promotion (Trigger T3). Rejects the two - * pollution classes observed in the live KG — first-person agent narration - * ("Ich schaue zuerst in den Memory…") and trivially short fragments — so they - * stay in the fuzzy tier instead of re-polluting the always-surface durable - * tier. Conservative: returns false when unsure. + * Ingest hygiene gate for ALL auto-harvested MemorableKnowledge (not just the + * durable tier). Rejects first-person agent narration / meta-process preambles + * ("Ich schaue zuerst in den Memory…") — the dominant pollution class observed + * in the live KG — so they never enter the KG and re-pollute recall every + * session. Deliberately does NOT reject on length: short FACTS ("Preis 1200€", + * "Migration 0007 ist live.") are legitimate. The durable tier adds its own + * length floor. Conservative: returns false when unsure. */ -function passesDurableHygiene(summary: string, rationale?: string): boolean { +function passesIngestHygiene(summary: string): boolean { const head = summary.trim(); - const full = `${summary} ${rationale ?? ''}`.trim(); - if (full.length < 40) return false; - // First-person agent narration / meta-process preambles. const NARRATION = /^(ich\s+(schaue|schau|prüfe|pruefe|sehe|gucke|lese|checke|werde|muss)|lass\s+mich|du\s+hast\s+recht|moment\b|kurz\b|let me\b|i\s+will\b|i'?ll\b|looking\b|checking\b)/i; if (NARRATION.test(head)) return false; return true; } +/** Durable tier requires substantial content on top of ingest hygiene. */ +function isDurableContentLengthOk(summary: string, rationale?: string): boolean { + return `${summary} ${rationale ?? ''}`.trim().length >= 40; +} + function buildPayload( excerpt: PalaiaExcerpt | undefined, fallbackAnswer: string, diff --git a/middleware/packages/harness-orchestrator-extras/src/significanceScorer.ts b/middleware/packages/harness-orchestrator-extras/src/significanceScorer.ts index 284f9b51a..dc7948719 100644 --- a/middleware/packages/harness-orchestrator-extras/src/significanceScorer.ts +++ b/middleware/packages/harness-orchestrator-extras/src/significanceScorer.ts @@ -44,10 +44,18 @@ Definitions: 0.0 = trivial chit-chat, weather, "thanks", repeated greetings. 0.5 = useful answer, but no new fact about the user/world. 1.0 = high-signal: a decision, deadline, name, address, password - hint, customer-specific quirk, recurring pattern. + hint, customer-specific quirk, recurring pattern, OR + **learned data-model / schema / domain conventions** + (which table/entity holds which data, field names and + meaning, entity-set names, how to filter/join). This + reusable structure must survive across sessions so the + agent never re-discovers it — score it HIGH (>=0.85). + (A time-bound data snapshot like "29 courses next week" + is mid-signal ~0.5, NOT high.) entry_type "memory" — A general fact, preference, or note (default). - "process" — A repeatable how-to / SOP / workflow description. + "process" — A repeatable how-to / SOP / workflow description, OR a stable + data-model / schema / convention (entities, fields, lookups). "task" — Something the user explicitly asked to be done or tracked ("remind me", "add to my list", "follow up"). diff --git a/middleware/packages/harness-orchestrator/src/buildOrchestrator.ts b/middleware/packages/harness-orchestrator/src/buildOrchestrator.ts index 482be37d6..2be4df148 100644 --- a/middleware/packages/harness-orchestrator/src/buildOrchestrator.ts +++ b/middleware/packages/harness-orchestrator/src/buildOrchestrator.ts @@ -24,6 +24,7 @@ import type { import type { EntityRefBus, KnowledgeGraph, + MemorableKind, MemoryStore, NudgeRegistry, NudgeStateStore, @@ -118,6 +119,11 @@ export interface OrchestratorDeps { /** Merged from main 2026-05-26: KG-ACL auto-promotion env flags. */ readonly autoPromote?: boolean; readonly autoPromoteThreshold?: number; + /** Trigger T3 — durable auto-promotion. Threaded here so dynamic / registry + * agents self-curate the durable tier too (not just the static chatAgent@1). + * Undefined → durable auto-promotion off for this agent. */ + readonly autoPromoteDurableMinSignificance?: number; + readonly autoPromoteDurableKinds?: MemorableKind[]; /** Shared Postgres pool the Orchestrator may use for direct KG writes. */ readonly graphPool?: Pool; readonly graphTenantId?: string; @@ -265,6 +271,15 @@ export function buildOrchestratorForAgent( ...(deps.autoPromoteThreshold !== undefined ? { autoPromoteThreshold: deps.autoPromoteThreshold } : {}), + ...(deps.autoPromoteDurableMinSignificance !== undefined + ? { + autoPromoteDurableMinSignificance: + deps.autoPromoteDurableMinSignificance, + } + : {}), + ...(deps.autoPromoteDurableKinds !== undefined + ? { autoPromoteDurableKinds: deps.autoPromoteDurableKinds } + : {}), ...(deps.graphPool ? { graphPool: deps.graphPool } : {}), ...(deps.graphTenantId ? { graphTenantId: deps.graphTenantId } : {}), ...(deps.assistantIdentity diff --git a/middleware/packages/harness-orchestrator/src/orchestrator.ts b/middleware/packages/harness-orchestrator/src/orchestrator.ts index f44f09ee9..a15019254 100644 --- a/middleware/packages/harness-orchestrator/src/orchestrator.ts +++ b/middleware/packages/harness-orchestrator/src/orchestrator.ts @@ -668,6 +668,11 @@ Memory-Namensräume (Konvention): - Bei einem **Follow-up** im selben Chat (Variante, Bereinigung, Klarifikation, Nachfrage zum letzten Turn wie "und das Ganze nochmal ohne X", "und für Q4?", "zeig das als Line-Chart") **NICHT erneut** die Regeln lesen — der Verbatim-Tail im Gesprächskontext hat bereits den relevanten Stand. Direkt antworten (ggf. mit \`render_diagram\` für Chart-Varianten). Regel erneut lesen nur, wenn die Follow-up eine fachlich neue Dimension einführt (z. B. "jetzt das Gleiche auf HR-Ebene"). - Heuristik: enthält der Kontext-Block einen \`## Letzte Turns in diesem Chat\`-Abschnitt und bezieht sich die aktuelle Frage auf einen dieser Turns → Memory-Read überspringen. +**Dauerhaftes Schema-Wissen vertrauen (kein Re-Discovery):** +- Enthält der Kontext-Block den Abschnitt \`## Aus früheren Sessions — verwandte Erkenntnisse\` mit **kuratiertem Schema-/Referenzwissen** (z. B. Dynamics-Entitäten und ihre Felder: \`ud_tutorial\`/\`ud_tutorials\`, \`ud_name\`, \`ud_coursenumber\`, \`ud_startdatetime\` …), dann ist das **maßgeblich und sessionübergreifend stabil**. Nutze es direkt. +- **Rufe KEINE Discovery-Tools erneut** (z. B. \`dynamics_describe\`) für eine Entität, deren Struktur in diesem dauerhaften Wissen bereits beschrieben ist. Gehe direkt zur **Daten-Abfrage** (\`dynamics_query\` o. ä.) über. Discovery nur für Entitäten/Felder, die im dauerhaften Wissen NICHT vorkommen. +- Widerspricht eine Fach-Agent-Antwort dem dauerhaften Wissen, weise den Nutzer auf die Inkonsistenz hin — überschreibe das kuratierte Wissen nicht still. + **Antwort-Verzicht (NO_REPLY):** Wenn du nichts beizutragen hast, antworte mit dem **alleinigen, exakten** Token \`NO_REPLY\` (keine Erklärung, kein Präfix, kein Suffix). Das System fängt das Token ab und sendet **keine Nachricht** an den User. Anwendungsfälle: diff --git a/middleware/packages/plugin-api/src/knowledgeGraph.ts b/middleware/packages/plugin-api/src/knowledgeGraph.ts index 17e9cec65..39c3dd587 100644 --- a/middleware/packages/plugin-api/src/knowledgeGraph.ts +++ b/middleware/packages/plugin-api/src/knowledgeGraph.ts @@ -888,6 +888,11 @@ export interface RecalledInsight { kind: string; summary: string; score: number; + /** True when this insight comes from the always-surface DURABLE tier + * (curated `manuallyAuthored` reference/decision knowledge). Durable + * insights render at full length (not the fuzzy cap) so the agent can + * trust recalled schema instead of re-discovering it via tools. */ + durable?: boolean; } /** What the cross-session probe surfaced this turn. Empty arrays when a leg diff --git a/middleware/test/autoMerge.test.ts b/middleware/test/autoMerge.test.ts new file mode 100644 index 000000000..fa1d58bdb --- /dev/null +++ b/middleware/test/autoMerge.test.ts @@ -0,0 +1,118 @@ +import { strict as assert } from 'node:assert'; +import { describe, it } from 'node:test'; + +import type { EmbeddingClient } from '@omadia/embeddings'; +import { InMemoryKnowledgeGraph } from '@omadia/knowledge-graph-inmemory'; +import { createMergeCandidateDetector } from '@omadia/orchestrator-extras'; + +// Slice 13 · automatic dedup. The merge detector, when given an +// autoMergeThreshold, RESOLVES high-confidence duplicate MK pairs itself +// (retires the duplicate) instead of only flagging. Safety: a durable +// (manuallyAuthored) node is never deleted; both-durable is left alone; else +// the older node wins. + +const VEC: number[] = [1, 0, 0, 0]; // every seeded MK shares it ⇒ cosine ≈ 1 + +const embedder: EmbeddingClient = { + async embed(): Promise { + return [...VEC]; + }, +}; + +async function seedMk( + kg: InMemoryKnowledgeGraph, + summary: string, + durable: boolean, +): Promise { + const created = await kg.createMemorableKnowledge({ + kind: 'reference', + summary, + createdBy: 'web:bob', + involvedOmadiaUserIds: [], + aclOwners: ['bob'], + ...(durable ? { manuallyAuthored: true } : {}), + }); + kg.setEmbedding(created.memorableKnowledgeNodeId, VEC); + return created.memorableKnowledgeNodeId; +} + +function detector(kg: InMemoryKnowledgeGraph) { + return createMergeCandidateDetector({ + graph: kg, + embeddingClient: embedder, + autoMergeThreshold: 0.9, + minSimilarity: 0.9, + log: () => {}, + }); +} + +describe('Slice 13 · automatic high-confidence merge', () => { + it('retires a fuzzy duplicate, keeps the durable one', async () => { + const kg = new InMemoryKnowledgeGraph(); + const durableId = await seedMk(kg, '# Schema: courses in ud_tutorial', true); + const fuzzyId = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + // Source = the freshly-learned fuzzy MK; candidate = the durable one. + await detector(kg).detectFor(fuzzyId); + assert.equal( + await kg.getMemorableKnowledge(fuzzyId), + null, + 'fuzzy duplicate retired', + ); + assert.ok( + await kg.getMemorableKnowledge(durableId), + 'durable node survives (never deleted)', + ); + }); + + it('never deletes a durable node even when it is the source', async () => { + const kg = new InMemoryKnowledgeGraph(); + const fuzzyId = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + const durableId = await seedMk(kg, '# Schema: courses in ud_tutorial', true); + await detector(kg).detectFor(durableId); // source is durable + assert.ok( + await kg.getMemorableKnowledge(durableId), + 'durable source survives', + ); + assert.equal( + await kg.getMemorableKnowledge(fuzzyId), + null, + 'fuzzy duplicate retired', + ); + }); + + it('keeps both when BOTH are durable (left for an operator)', async () => { + const kg = new InMemoryKnowledgeGraph(); + const a = await seedMk(kg, '# Schema: courses in ud_tutorial', true); + const b = await seedMk(kg, '# Schema: courses in ud_tutorial', true); + await detector(kg).detectFor(b); + assert.ok(await kg.getMemorableKnowledge(a), 'durable A survives'); + assert.ok(await kg.getMemorableKnowledge(b), 'durable B survives'); + }); + + it('two fuzzy duplicates: the OLDER survives', async () => { + const kg = new InMemoryKnowledgeGraph(); + const older = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + const newer = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + await detector(kg).detectFor(newer); + assert.ok(await kg.getMemorableKnowledge(older), 'older survives'); + assert.equal( + await kg.getMemorableKnowledge(newer), + null, + 'newer duplicate retired', + ); + }); + + it('without autoMergeThreshold it only FLAGS (legacy, no deletion)', async () => { + const kg = new InMemoryKnowledgeGraph(); + const a = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + const b = await seedMk(kg, '# Schema: courses in ud_tutorial', false); + const flagOnly = createMergeCandidateDetector({ + graph: kg, + embeddingClient: embedder, + log: () => {}, + }); + await flagOnly.detectFor(b); + assert.ok(await kg.getMemorableKnowledge(a), 'a kept (flag-only)'); + assert.ok(await kg.getMemorableKnowledge(b), 'b kept (flag-only)'); + }); +}); diff --git a/middleware/test/durableRecall.test.ts b/middleware/test/durableRecall.test.ts index 1b11ca870..c14b58555 100644 --- a/middleware/test/durableRecall.test.ts +++ b/middleware/test/durableRecall.test.ts @@ -257,4 +257,39 @@ describe('B1 · durable-curation tier surfaces manual MK', () => { }); assert.equal(result.recalled.insights.length, 0, 'wrong kind not admitted'); }); + + it('(e) durable insight renders FULL-length (not truncated to the fuzzy 300 cap)', async () => { + const kg = new InMemoryKnowledgeGraph(); + // A long curated schema (>300 chars) — must reach the agent complete so it + // doesn't re-run discovery tools for fields it already has. + const longSchema = + '# Dynamics: Kurse / Seminare liegen in ud_tutorial (entitySet ud_tutorials). ' + + 'Felder: ud_name (Name), ud_coursenumber (Kursnummer), ud_startdatetime (Start), ' + + 'ud_enddatetime (Ende), ud_msa_city (Ort), ud_maxparticipants (Kapazität), ' + + 'ud_price (Preis). Buchungen: ud_booking. Teilnehmer: ud_participant. ' + + 'Registrierungen: ud_registration. Filter für Zeiträume über ud_startdatetime.'; + assert.ok(longSchema.length > 300, 'fixture must exceed fuzzy cap'); + const mkId = await seedDurableMk(kg, 'reference', longSchema); + const retriever = new ContextRetriever( + kg, + { teamVisibility: true }, + queryEmbedder, + ); + const result = await retriever.assembleForBudget({ + userMessage: 'Welche Kurse finden nächste Woche statt?', + agentId: 'test-agent', + sessionScope: 'sess-now', + userId: 'alice', + }); + assert.equal(result.recalled.insights.length, 1); + const ins = result.recalled.insights[0]!; + assert.equal(ins.mkId, mkId); + assert.equal(ins.durable, true, 'flagged durable'); + assert.ok( + ins.summary.length > 300, + `durable summary must NOT be cut to 300 (got ${String(ins.summary.length)})`, + ); + // The full schema text reaches the rendered recall block (agent prompt). + assert.match(result.text, /ud_participant/); + }); }); diff --git a/middleware/test/office.test.ts b/middleware/test/office.test.ts index a40308aaf..503089a87 100644 --- a/middleware/test/office.test.ts +++ b/middleware/test/office.test.ts @@ -59,10 +59,23 @@ describe('office xlsx renderer', () => { assert.match(ws.getCell('C2').numFmt ?? '', /€/); }); - it('is deterministic — same descriptor yields identical bytes', async () => { - const a = await renderXlsx(descriptor); - const b = await renderXlsx(descriptor); - assert.ok(a.buffer.equals(b.buffer), 'pinned metadata → byte-identical'); + it('is deterministic — same descriptor yields identical bytes', async (t) => { + // exceljs stamps the ZIP entry mtimes with the wall clock (DOS 2-second + // granularity) and exposes no API to pin them, so two renders that straddle + // a 2s boundary differ even though the logical workbook + pinned + // created/modified are identical — a timing flake in slow CI (passes + // locally where both renders land in the same window). Freeze the clock + // across both renders so the byte-equality verifies renderer determinism + // rather than wall-clock timing. (Freezing Date globally inside the + // renderer itself would be unsafe under concurrent async on the server.) + t.mock.timers.enable({ apis: ['Date'], now: 1_700_000_000_000 }); + try { + const a = await renderXlsx(descriptor); + const b = await renderXlsx(descriptor); + assert.ok(a.buffer.equals(b.buffer), 'pinned metadata → byte-identical'); + } finally { + t.mock.timers.reset(); + } }); it('counts rowsWritten across sheets', async () => { diff --git a/middleware/test/promoteTurnIfSignificant.test.ts b/middleware/test/promoteTurnIfSignificant.test.ts index 6f2e0f294..34a45f635 100644 --- a/middleware/test/promoteTurnIfSignificant.test.ts +++ b/middleware/test/promoteTurnIfSignificant.test.ts @@ -302,4 +302,47 @@ describe('Slice 4b · promoteTurnIfSignificant', () => { }); assert.ok(createCalls[0]!.summary.length <= 500); }); + + it('skips agent-narration turns (ingest hygiene) even above threshold', async () => { + const { pool } = makeFakePool({ + significanceRows: [{ significance: 0.85 }], + idempotencyRows: [], + }); + const { kg, createCalls } = makeFakeKg({}); + const out = await promoteTurnIfSignificant({ + pool: pool as never, + tenantId: 'byte5', + kg, + turnId: TURN_ID, + userId: USER_ID, + threshold: 0.7, + // First-person agent narration — high significance but pure meta-process. + fallbackAssistantAnswer: + 'Ich schaue kurz in den Memory für Konventionen und ob es schon Detail-Befunde gibt.', + log: () => {}, + }); + assert.equal(out.promoted, false); + assert.equal(out.reason, 'hygiene-skip'); + assert.equal(createCalls.length, 0, 'narration must NOT be stored as MK'); + }); + + it('still stores short factual turns (length is not a gate for fuzzy)', async () => { + const { pool } = makeFakePool({ + significanceRows: [{ significance: 0.85 }], + idempotencyRows: [], + }); + const { kg, createCalls } = makeFakeKg({}); + const out = await promoteTurnIfSignificant({ + pool: pool as never, + tenantId: 'byte5', + kg, + turnId: TURN_ID, + userId: USER_ID, + threshold: 0.7, + fallbackAssistantAnswer: 'Preis 1200 EUR.', + log: () => {}, + }); + assert.equal(out.promoted, true); + assert.equal(createCalls.length, 1); + }); });