feat(kg): automatic self-curation — durable coverage grows + duplicates auto-merge

middleware/packages/harness-orchestrator-extras/src/contextRetriever.ts

@@ -710,7 +710,7 @@ export class ContextRetriever {
     const seenInsightIds = new Set<string>();
     const insights: RecalledInsight[] = [];
     for (const hit of durableHits) {
-      const insight = toRecalledInsight(hit);
+      const insight = toRecalledInsight(hit, true);
       if (seenInsightIds.has(insight.mkId)) continue;
       seenInsightIds.add(insight.mkId);
       insights.push(insight);
@@ -1479,20 +1479,38 @@ function renderRecallBlocks(
   if (recalled.insights.length > 0 && budget > 200) {
     push('\n## Aus früheren Sessions — verwandte Erkenntnisse');
     for (const ins of recalled.insights) {
-      const chunk = `- ${ins.kind}: ${truncate(ins.summary, 300)} (score ${ins.score.toFixed(2)})`;
+      // Durable (curated schema/reference) insights render in full so the
+      // agent can rely on them instead of re-running discovery tools; fuzzy
+      // insights stay capped. `ins.summary` is already bounded upstream by
+      // toRecalledInsight (durable → 2000, fuzzy → 300).
+      const rendered = ins.durable ? ins.summary : truncate(ins.summary, 300);
+      const chunk = `- ${ins.kind}: ${rendered} (score ${ins.score.toFixed(2)})`;
       if (!push(chunk)) break;
     }
   }
 
   return parts.join('\n');
 }
 
-function toRecalledInsight(hit: MemoryRecallHit): RecalledInsight {
+/** Full render length for DURABLE insights — curated schema/reference
+ *  knowledge must reach the agent complete, or it re-discovers (e.g. re-runs
+ *  `dynamics_describe`) what it already knows. Fuzzy insights stay capped. */
+const DURABLE_SUMMARY_MAX_CHARS = 2000;
+const FUZZY_SUMMARY_MAX_CHARS = 300;
+
+function toRecalledInsight(
+  hit: MemoryRecallHit,
+  durable = false,
+): RecalledInsight {
   return {
     mkId: hit.mk.id,
     kind: String(hit.mk.props['kind'] ?? 'memory'),
-    summary: truncate(String(hit.mk.props['summary'] ?? ''), 300),
+    summary: truncate(
+      String(hit.mk.props['summary'] ?? ''),
+      durable ? DURABLE_SUMMARY_MAX_CHARS : FUZZY_SUMMARY_MAX_CHARS,
+    ),
     score: hit.score,
+    ...(durable ? { durable: true } : {}),
   };
 }
 

middleware/packages/harness-orchestrator-extras/src/excerptExtractor.ts

@@ -73,7 +73,18 @@ Definitions:
     "decision"   — A choice was made or recommended ("we will use X", "go with Y").
     "insight"    — A non-obvious finding or learning ("turns out the API caps at 100/min").
     "preference" — A stated user/team preference ("always reply in German first").
-    "reference"  — Stable how-to / SOP / lookup material ("to deploy: run X then Y").
+    "reference"  — Stable, reusable lookup material that does NOT change per
+                   request: how-to / SOP ("to deploy: run X then Y"), AND —
+                   importantly — **data-model / schema / domain conventions**:
+                   which table or entity holds which data, field names and
+                   their meaning, entity-set names, how to filter/join, naming
+                   rules. E.g. "Courses live in the Dynamics table ud_tutorial
+                   (entitySet ud_tutorials); fields ud_name, ud_coursenumber,
+                   ud_startdatetime; bookings in ud_booking". This kind of
+                   learned structure is long-lived knowledge the agent must NOT
+                   re-discover every session — classify it as "reference".
+                   (A time-bound DATA snapshot — "29 courses next week" — is an
+                   "insight", NOT a reference.)
   summary  — Stand-alone sentence(s) the user could read in /memories years later.
               Do NOT start with "The user asked about…" — describe the answer's substance.
   rationale — Why it matters / preconditions / caveats. Use null when redundant with summary.

middleware/packages/harness-orchestrator-extras/src/mergeCandidateDetector.ts

@@ -43,6 +43,17 @@ export interface MergeCandidateDetectorDeps {
   excerptMinSimilarity?: number;
   /** Max candidates checked per source. Default 5. */
   topK?: number;
+  /**
+   * Auto-merge threshold. When set, a flagged MK pair whose cosine is at or
+   * above this value is RESOLVED automatically (the duplicate is retired via
+   * `resolveMergeCandidate`) instead of only being flagged for an operator —
+   * so re-learned knowledge stops accumulating on any deployment without a
+   * manual cleanup pass. SAFETY: a `manuallyAuthored` (durable) node is NEVER
+   * deleted; when exactly one side is durable it always wins; when BOTH are
+   * durable the pair is left for an operator; otherwise the OLDER node wins.
+   * Undefined → auto-merge disabled (flag-only, legacy behaviour).
+   */
+  autoMergeThreshold?: number;
   log?: (msg: string) => void;
 }
 
@@ -59,6 +70,24 @@ export function createMergeCandidateDetector(
   const excerptMinSim =
     deps.excerptMinSimilarity ?? DEFAULT_EXCERPT_MIN_SIMILARITY;
   const topK = deps.topK ?? DEFAULT_TOP_K;
+  const autoMergeThreshold = deps.autoMergeThreshold;
+
+  /**
+   * Pick which of two near-duplicate MKs to KEEP, or null to leave for an
+   * operator. Durable (`manuallyAuthored`) is sacred: it is never the loser,
+   * and two durable nodes are never auto-merged. Otherwise the older node wins
+   * (it's the established one; the fresh re-statement is the duplicate).
+   */
+  function decideKeeper(a: GraphNode, b: GraphNode): string | null {
+    const aDur = a.manuallyAuthored === true;
+    const bDur = b.manuallyAuthored === true;
+    if (aDur && bDur) return null; // both curated — hands off
+    if (aDur !== bDur) return aDur ? a.id : b.id; // the durable one wins
+    const aAt = String(a.props['created_at'] ?? '');
+    const bAt = String(b.props['created_at'] ?? '');
+    if (aAt && bAt && aAt !== bAt) return aAt < bAt ? a.id : b.id; // older wins
+    return b.id; // tie / unknown → keep the existing candidate, retire source
+  }
 
   async function detectFor(
     memorableKnowledgeNodeId: string,
@@ -137,6 +166,38 @@ export function createMergeCandidateDetector(
           log(
             `[merge-detector] flagged ${source.id} vs ${candidate.mk.id} cosine=${candidate.cosineSim.toFixed(3)}`,
           );
+          // Auto-merge: high-confidence + safe → retire the duplicate now so
+          // re-learned knowledge stops accumulating without a manual sweep.
+          if (
+            autoMergeThreshold !== undefined &&
+            candidate.cosineSim >= autoMergeThreshold
+          ) {
+            const keeper = decideKeeper(source, candidate.mk);
+            if (keeper === null) {
+              log(
+                `[merge-detector] auto-merge SKIP ${source.id} vs ${candidate.mk.id}: both durable`,
+              );
+            } else {
+              // duplicateOf is sorted ascending; keep_a keeps [0] (deletes [1]),
+              // keep_b keeps [1] (deletes [0]).
+              const resolution =
+                persisted.duplicateOf[0] === keeper ? 'keep_a' : 'keep_b';
+              try {
+                await deps.graph.resolveMergeCandidate(
+                  persisted.id,
+                  resolution,
+                  { actorOmadiaUserId: viewer },
+                );
+                log(
+                  `[merge-detector] auto-merged cosine=${candidate.cosineSim.toFixed(3)} kept=${keeper} (retired the duplicate)`,
+                );
+              } catch (err) {
+                log(
+                  `[merge-detector] auto-merge FAILED for ${persisted.id} (left flagged): ${err instanceof Error ? err.message : String(err)}`,
+                );
+              }
+            }
+          }
         }
       } catch (err) {
         log(

middleware/packages/harness-orchestrator-extras/src/plugin.ts

@@ -437,11 +437,36 @@ export async function activate(
   // order: capture-filter → inconsistency-trigger → merge-trigger
   // (outermost). Cosine-only, no Anthropic dependency. Always active
   // when embeddingClient is wired.
+  // Slice 13 — AUTOMATIC dedup. When enabled (default on), the merge detector
+  // RESOLVES high-confidence duplicate MK pairs itself (retires the duplicate;
+  // durable `manuallyAuthored` nodes are never deleted) instead of only
+  // flagging for an operator — so re-learned knowledge stops accumulating on
+  // ANY deployment without a manual cleanup pass. Aggressive default 0.90 so
+  // paraphrased re-statements also merge. The detector's flag floor is lowered
+  // to the same threshold so sub-0.95 near-dups actually surface to be merged.
+  // Disable with kg_auto_merge_enabled=false (reverts to flag-only at 0.95).
+  const autoMergeEnabledRaw =
+    ctx.config.get<unknown>('kg_auto_merge_enabled') ??
+    process.env['KG_AUTO_MERGE_ENABLED'];
+  const autoMergeDisabled =
+    typeof autoMergeEnabledRaw === 'string'
+      ? autoMergeEnabledRaw.toLowerCase() === 'false'
+      : autoMergeEnabledRaw === false;
+  const autoMergeThreshold = parseNumberOrDefault(
+    ctx.config.get<unknown>('kg_auto_merge_threshold'),
+    0.9,
+  );
   const mergeCandidateDetector = createMergeCandidateDetector({
     graph: inconsistencyWrappedKg,
     ...(embeddingClient ? { embeddingClient } : {}),
+    ...(autoMergeDisabled
+      ? {}
+      : { autoMergeThreshold, minSimilarity: autoMergeThreshold }),
     log: (msg) => { console.error(msg); },
   });
+  ctx.log(
+    `[harness-orchestrator-extras] auto-merge ${autoMergeDisabled ? 'off (flag-only @0.95)' : `on (resolve >= ${autoMergeThreshold.toFixed(2)}, durable-protected)`}`,
+  );
   const disposeMergeCandidateDetector = ctx.services.provide(
     MERGE_CANDIDATE_DETECTOR_SERVICE_NAME,
     mergeCandidateDetector,

middleware/packages/harness-orchestrator-extras/src/promotion.ts

@@ -86,6 +86,7 @@ export interface PromoteTurnResult {
     | 'promoted'
     | 'below-threshold'
     | 'no-significance'
+    | 'hygiene-skip'
     | 'already-promoted'
     | 'missing-user'
     | 'missing-turn'
@@ -176,16 +177,28 @@ export async function promoteTurnIfSignificant(
       input.fallbackAssistantAnswer,
     );
 
+    // Ingest hygiene — applied to ALL auto-harvest, not just the durable tier.
+    // First-person agent narration ("Ich schaue kurz in den Memory…") and
+    // trivially short fragments scored high enough to clear the significance
+    // threshold and were being stored as fuzzy MK every session, re-polluting
+    // recall. Drop them entirely so they never enter the KG.
+    if (!passesIngestHygiene(summary)) {
+      log(
+        `[promotion] skip turn=${input.turnId} reason=hygiene (agent narration) significance=${significance.toFixed(2)}`,
+      );
+      return { promoted: false, reason: 'hygiene-skip', significance };
+    }
+
     // Trigger T3 — durable auto-promotion gate. Conservative by design: only
-    // high-significance reference knowledge that survives the hygiene check is
-    // marked durable, so conversational narration + time-bound snapshots never
-    // re-pollute the always-surface tier.
+    // high-significance, substantial reference knowledge is marked durable, so
+    // time-bound snapshots / short fragments never reach the always-surface
+    // tier (narration is already dropped above).
     const durableKinds = input.durableKinds ?? ['reference'];
     const durable =
       input.durableMinSignificance !== undefined &&
       significance >= input.durableMinSignificance &&
       durableKinds.includes(kind) &&
-      passesDurableHygiene(summary, rationale);
+      isDurableContentLengthOk(summary, rationale);
 
     const result = await input.kg.createMemorableKnowledge({
       kind,
@@ -231,23 +244,27 @@ export async function promoteTurnIfSignificant(
 }
 
 /**
- * Hygiene gate for durable auto-promotion (Trigger T3). Rejects the two
- * pollution classes observed in the live KG — first-person agent narration
- * ("Ich schaue zuerst in den Memory…") and trivially short fragments — so they
- * stay in the fuzzy tier instead of re-polluting the always-surface durable
- * tier. Conservative: returns false when unsure.
+ * Ingest hygiene gate for ALL auto-harvested MemorableKnowledge (not just the
+ * durable tier). Rejects first-person agent narration / meta-process preambles
+ * ("Ich schaue zuerst in den Memory…") — the dominant pollution class observed
+ * in the live KG — so they never enter the KG and re-pollute recall every
+ * session. Deliberately does NOT reject on length: short FACTS ("Preis 1200€",
+ * "Migration 0007 ist live.") are legitimate. The durable tier adds its own
+ * length floor. Conservative: returns false when unsure.
  */
-function passesDurableHygiene(summary: string, rationale?: string): boolean {
+function passesIngestHygiene(summary: string): boolean {
   const head = summary.trim();
-  const full = `${summary} ${rationale ?? ''}`.trim();
-  if (full.length < 40) return false;
-  // First-person agent narration / meta-process preambles.
   const NARRATION =
     /^(ich\s+(schaue|schau|prüfe|pruefe|sehe|gucke|lese|checke|werde|muss)|lass\s+mich|du\s+hast\s+recht|moment\b|kurz\b|let me\b|i\s+will\b|i'?ll\b|looking\b|checking\b)/i;
   if (NARRATION.test(head)) return false;
   return true;
 }
 
+/** Durable tier requires substantial content on top of ingest hygiene. */
+function isDurableContentLengthOk(summary: string, rationale?: string): boolean {
+  return `${summary} ${rationale ?? ''}`.trim().length >= 40;
+}
+
 function buildPayload(
   excerpt: PalaiaExcerpt | undefined,
   fallbackAnswer: string,

middleware/packages/harness-orchestrator-extras/src/significanceScorer.ts

@@ -44,10 +44,18 @@ Definitions:
                 0.0 = trivial chit-chat, weather, "thanks", repeated greetings.
                 0.5 = useful answer, but no new fact about the user/world.
                 1.0 = high-signal: a decision, deadline, name, address, password
-                      hint, customer-specific quirk, recurring pattern.
+                      hint, customer-specific quirk, recurring pattern, OR
+                      **learned data-model / schema / domain conventions**
+                      (which table/entity holds which data, field names and
+                      meaning, entity-set names, how to filter/join). This
+                      reusable structure must survive across sessions so the
+                      agent never re-discovers it — score it HIGH (>=0.85).
+                      (A time-bound data snapshot like "29 courses next week"
+                      is mid-signal ~0.5, NOT high.)
   entry_type
     "memory"  — A general fact, preference, or note (default).
-    "process" — A repeatable how-to / SOP / workflow description.
+    "process" — A repeatable how-to / SOP / workflow description, OR a stable
+                 data-model / schema / convention (entities, fields, lookups).
     "task"    — Something the user explicitly asked to be done or
                  tracked ("remind me", "add to my list", "follow up").
 

middleware/packages/harness-orchestrator/src/buildOrchestrator.ts

@@ -24,6 +24,7 @@ import type {
 import type {
   EntityRefBus,
   KnowledgeGraph,
+  MemorableKind,
   MemoryStore,
   NudgeRegistry,
   NudgeStateStore,
@@ -118,6 +119,11 @@ export interface OrchestratorDeps {
   /** Merged from main 2026-05-26: KG-ACL auto-promotion env flags. */
   readonly autoPromote?: boolean;
   readonly autoPromoteThreshold?: number;
+  /** Trigger T3 — durable auto-promotion. Threaded here so dynamic / registry
+   *  agents self-curate the durable tier too (not just the static chatAgent@1).
+   *  Undefined → durable auto-promotion off for this agent. */
+  readonly autoPromoteDurableMinSignificance?: number;
+  readonly autoPromoteDurableKinds?: MemorableKind[];
   /** Shared Postgres pool the Orchestrator may use for direct KG writes. */
   readonly graphPool?: Pool;
   readonly graphTenantId?: string;
@@ -265,6 +271,15 @@ export function buildOrchestratorForAgent(
     ...(deps.autoPromoteThreshold !== undefined
       ? { autoPromoteThreshold: deps.autoPromoteThreshold }
       : {}),
+    ...(deps.autoPromoteDurableMinSignificance !== undefined
+      ? {
+          autoPromoteDurableMinSignificance:
+            deps.autoPromoteDurableMinSignificance,
+        }
+      : {}),
+    ...(deps.autoPromoteDurableKinds !== undefined
+      ? { autoPromoteDurableKinds: deps.autoPromoteDurableKinds }
+      : {}),
     ...(deps.graphPool ? { graphPool: deps.graphPool } : {}),
     ...(deps.graphTenantId ? { graphTenantId: deps.graphTenantId } : {}),
     ...(deps.assistantIdentity

middleware/packages/harness-orchestrator/src/orchestrator.ts

@@ -668,6 +668,11 @@ Memory-Namensräume (Konvention):
 - Bei einem **Follow-up** im selben Chat (Variante, Bereinigung, Klarifikation, Nachfrage zum letzten Turn wie "und das Ganze nochmal ohne X", "und für Q4?", "zeig das als Line-Chart") **NICHT erneut** die Regeln lesen — der Verbatim-Tail im Gesprächskontext hat bereits den relevanten Stand. Direkt antworten (ggf. mit \`render_diagram\` für Chart-Varianten). Regel erneut lesen nur, wenn die Follow-up eine fachlich neue Dimension einführt (z. B. "jetzt das Gleiche auf HR-Ebene").
 - Heuristik: enthält der Kontext-Block einen \`## Letzte Turns in diesem Chat\`-Abschnitt und bezieht sich die aktuelle Frage auf einen dieser Turns → Memory-Read überspringen.
 
+**Dauerhaftes Schema-Wissen vertrauen (kein Re-Discovery):**
+- Enthält der Kontext-Block den Abschnitt \`## Aus früheren Sessions — verwandte Erkenntnisse\` mit **kuratiertem Schema-/Referenzwissen** (z. B. Dynamics-Entitäten und ihre Felder: \`ud_tutorial\`/\`ud_tutorials\`, \`ud_name\`, \`ud_coursenumber\`, \`ud_startdatetime\` …), dann ist das **maßgeblich und sessionübergreifend stabil**. Nutze es direkt.
+- **Rufe KEINE Discovery-Tools erneut** (z. B. \`dynamics_describe\`) für eine Entität, deren Struktur in diesem dauerhaften Wissen bereits beschrieben ist. Gehe direkt zur **Daten-Abfrage** (\`dynamics_query\` o. ä.) über. Discovery nur für Entitäten/Felder, die im dauerhaften Wissen NICHT vorkommen.
+- Widerspricht eine Fach-Agent-Antwort dem dauerhaften Wissen, weise den Nutzer auf die Inkonsistenz hin — überschreibe das kuratierte Wissen nicht still.
+
 **Antwort-Verzicht (NO_REPLY):**
 
 Wenn du nichts beizutragen hast, antworte mit dem **alleinigen, exakten** Token \`NO_REPLY\` (keine Erklärung, kein Präfix, kein Suffix). Das System fängt das Token ab und sendet **keine Nachricht** an den User. Anwendungsfälle:

middleware/packages/plugin-api/src/knowledgeGraph.ts

@@ -888,6 +888,11 @@ export interface RecalledInsight {
   kind: string;
   summary: string;
   score: number;
+  /** True when this insight comes from the always-surface DURABLE tier
+   *  (curated `manuallyAuthored` reference/decision knowledge). Durable
+   *  insights render at full length (not the fuzzy cap) so the agent can
+   *  trust recalled schema instead of re-discovering it via tools. */
+  durable?: boolean;
 }
 
 /** What the cross-session probe surfaced this turn. Empty arrays when a leg