diff --git a/app/admin/memory/compaction/page.tsx b/app/admin/memory/compaction/page.tsx new file mode 100644 index 0000000..065572c --- /dev/null +++ b/app/admin/memory/compaction/page.tsx @@ -0,0 +1,9 @@ +import { resolvePandoraRuntimeSafetyConfig } from "@/lib/config/pandora-runtime-safety-config"; +export default function MemoryCompactionAdminPage(){const r=resolvePandoraRuntimeSafetyConfig(); const safe=[ + ["Distillation gate",r.config.memoryDistillationEnabled], + ["Model calls",r.config.modelCallsEnabled], + ["Embeddings",r.config.embeddingsEnabled], + ["Semantic retrieval",r.config.semanticRetrievalEnabled], + ["Public memory read",r.config.publicMemoryReadEnabled], + ["Public persistence",r.config.publicMemoryPersistenceEnabled], +]; return

Phase 5C Memory Compaction

Daily compaction deterministically turns reviewed/captured memory into versioned profiles, open loops, and daily context packs. It does not call models, create embeddings, enable retrieval, or expose public reads/writes.

Safety status

Internal job

POST /api/memory/jobs/daily-digest with Authorization: Bearer $PANDORA_INTERNAL_JOB_TOKEN.

{JSON.stringify({namespace:"real_life",dry_run:true,since:new Date(Date.now()-86400000).toISOString()},null,2)}

Operator notes

} diff --git a/app/api/memory/jobs/daily-digest/route.ts b/app/api/memory/jobs/daily-digest/route.ts new file mode 100644 index 0000000..c82333d --- /dev/null +++ b/app/api/memory/jobs/daily-digest/route.ts @@ -0,0 +1,8 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { resolvePandoraRuntimeSafetyConfig } from "@/lib/config/pandora-runtime-safety-config"; +import { createSupabaseBridgeAdminClient } from "@/lib/supabase/bridge-admin"; +import { runDailyMemoryCompaction } from "@/lib/services/memory-compaction-service"; +const schema=z.object({namespace:z.enum(["real_life","au"]),since:z.string().datetime().optional(),dry_run:z.boolean().optional(),user_id:z.string().min(1).optional()}); +function bearer(request:NextRequest){const [scheme,token]=(request.headers.get("authorization")??"").split(" "); return scheme?.toLowerCase()==="bearer"?token:undefined;} +export async function POST(request:NextRequest){const token=process.env.PANDORA_INTERNAL_JOB_TOKEN; if(!token||bearer(request)!==token)return NextResponse.json({ok:false,blockers:["internal_job_token_required"]},{status:401}); const runtime=resolvePandoraRuntimeSafetyConfig(); if(!runtime.config.memoryDistillationEnabled)return NextResponse.json({ok:false,blockers:["memoryDistillationEnabled_disabled"],warnings:[],next_step:"Set PANDORA_ENABLE_MEMORY_DISTILLATION=true in a reviewed environment."},{status:403}); const parsed=schema.safeParse(await request.json().catch(()=>({}))); if(!parsed.success)return NextResponse.json({ok:false,blockers:["invalid_request"],issues:parsed.error.flatten()},{status:400}); const user_id=parsed.data.user_id??process.env.PANDORA_MEMORY_BRIDGE_USER_ID; if(!user_id)return NextResponse.json({ok:false,blockers:["job_user_id_required"],next_step:"Send user_id with the internal token or configure PANDORA_MEMORY_BRIDGE_USER_ID."},{status:400}); const result=await runDailyMemoryCompaction(createSupabaseBridgeAdminClient() as never,{user_id,namespace:parsed.data.namespace,since:parsed.data.since,dry_run:parsed.data.dry_run}); return NextResponse.json({ok:result.ok,namespace:result.namespace,dry_run:result.dry_run,daily_pack:result.daily_pack,updated_profiles:result.updated_profiles,opened_or_updated_loops:result.opened_or_updated_loops,warnings:result.warnings,blockers:result.blockers,next_step:result.next_step});} diff --git a/docs/pandora-memory-autopilot.md b/docs/pandora-memory-autopilot.md index 7878315..fb83e5e 100644 --- a/docs/pandora-memory-autopilot.md +++ b/docs/pandora-memory-autopilot.md @@ -55,3 +55,22 @@ Phase 5A is deterministic and keyword-based. It does not implement the future re ## Phase 5B Candidate Review Phase 5A queues durable memory candidates automatically, but it does not make them permanent memory by default. Phase 5B adds the operator review console at `/admin/memory/candidates`, where Joven can approve, reject, edit, mark duplicate, or capture candidates. This keeps Pandora teachable: autopilot surfaces what might matter, while human review prevents noisy, sensitive, wrong, duplicate, or wrong-namespace items from polluting durable memory. + +## Phase 5C Daily Compaction + +Phase 5A queues candidates. Phase 5B reviews and captures candidates. Phase 5C compacts reviewed/captured memory into versioned profiles, open loops, and daily context packs. This is the deterministic compounding layer that lets Pandora improve over time without enabling model calls, embeddings, semantic retrieval, public reads, public writes, or permanent auto-capture. + +Recommended environment posture: + +```bash +PANDORA_ENABLE_MEMORY_DISTILLATION=true +PANDORA_INTERNAL_JOB_TOKEN= +PANDORA_ENABLE_MODEL_CALLS=false +PANDORA_ENABLE_EMBEDDINGS=false +PANDORA_ENABLE_SEMANTIC_RETRIEVAL=false +PANDORA_ENABLE_PUBLIC_MEMORY_READ=false +PANDORA_ENABLE_PUBLIC_MEMORY_PERSISTENCE=false +PANDORA_ENABLE_AUTO_CAPTURE=false +``` + +`PANDORA_INTERNAL_JOB_TOKEN` must remain server-only and must never be exposed as a `NEXT_PUBLIC_*` variable. Do not enable model calls or embeddings just for compaction, and do not enable permanent auto-capture yet. diff --git a/docs/pandora-memory-compaction.md b/docs/pandora-memory-compaction.md new file mode 100644 index 0000000..2d7ab0f --- /dev/null +++ b/docs/pandora-memory-compaction.md @@ -0,0 +1,94 @@ +# Pandora Memory Phase 5C: Daily Compaction + +Phase 5C adds deterministic daily compaction for Pandora Memory. It turns reviewed and captured source-backed memory into clean operating context: versioned profiles, open loops, and daily context packs. + +## Why compaction matters + +Phase 5A queues candidates. Phase 5B lets an operator approve, reject, edit, duplicate, or capture them. Phase 5C makes the reviewed/captured layer compound over time instead of becoming an infinite event log. + +## Data layers + +- **Candidates** are proposed memory items awaiting review or capture. +- **Memory events** are captured, source-backed memory rows. +- **Session digests** summarize sessions and can feed compaction wrappers. +- **Profiles** store durable operating, project, risk, relationship, people, deployment, and AU canon state. +- **Open loops** track unresolved blockers, risks, review backlog, and follow-up actions. +- **Context packs** are generated daily or master summaries for downstream private context use. + +## Deterministic profile synthesis + +`runDailyMemoryCompaction` scans recent rows by `user_id`, `namespace`, and ISO `since`. It uses deterministic keyword signals only. It does not call models, create embeddings, or use semantic retrieval. + +Profile updates are versioned. Existing active profiles are marked `superseded`, and a new active row is inserted with merged evidence references. + +## Feedback-aware compaction + +Approved and captured candidates are eligible profile/open-loop inputs. Rejected and duplicate candidates remain visible in the compaction result but are not promoted as source material. Edited candidate fields are preferred because compaction reads the reviewed candidate title/summary/type/sensitivity fields. + +Blocked secrets are excluded and redacted before pack creation. + +## Open-loop updates + +The open-loop engine creates or updates loops by `user_id`, `namespace`, `loop_type`, and `subject_key`. Existing open/acknowledged loops are updated, evidence references are merged, and severity can increase but not downgrade. + +## Daily context packs + +Daily compaction creates a `memory_context_packs` row with `pack_type=daily`, key points, projects, people, decisions, risks, open loops, and generated event ids. Older active daily packs for the same user/namespace are superseded when possible. + +## Internal job endpoint + +`POST /api/memory/jobs/daily-digest` + +Headers: + +```http +Authorization: Bearer +``` + +Body: + +```json +{ "namespace": "real_life", "since": "2026-06-27T00:00:00.000Z", "dry_run": true } +``` + +The endpoint requires `PANDORA_ENABLE_MEMORY_DISTILLATION=true` and is internal-token protected. If needed for the first single-user rollout, send `user_id` only through this internal-token route or configure `PANDORA_MEMORY_BRIDGE_USER_ID` server-side. + +## Env posture + +```bash +PANDORA_ENABLE_MEMORY_DISTILLATION=true +PANDORA_INTERNAL_JOB_TOKEN= +PANDORA_ENABLE_MODEL_CALLS=false +PANDORA_ENABLE_EMBEDDINGS=false +PANDORA_ENABLE_SEMANTIC_RETRIEVAL=false +PANDORA_ENABLE_PUBLIC_MEMORY_READ=false +PANDORA_ENABLE_PUBLIC_MEMORY_PERSISTENCE=false +PANDORA_ENABLE_AUTO_CAPTURE=false +``` + +Never expose `PANDORA_INTERNAL_JOB_TOKEN` as `NEXT_PUBLIC_*`. + +## Safety boundaries + +- deterministic only +- no model calls required +- no embeddings required +- no semantic retrieval required +- no public memory reads/writes +- blocked secrets excluded/redacted +- `real_life` and `au` remain separated +- permanent auto-capture remains disabled by default + +## Rollout + +1. Deploy with `PANDORA_INTERNAL_JOB_TOKEN` configured. +2. Keep model calls, embeddings, semantic retrieval, public read/write, and auto-capture off. +3. Enable `PANDORA_ENABLE_MEMORY_DISTILLATION=true`. +4. Run the daily digest endpoint with `dry_run=true`. +5. Inspect profile, loop, and pack output. +6. Run for `real_life`. +7. Run for `au`. +8. Verify daily context packs appear. +9. Verify active profiles version correctly. +10. Verify open loops update without duplicate spam. +11. Only then schedule the job externally. diff --git a/lib/services/memory-compaction-service.ts b/lib/services/memory-compaction-service.ts new file mode 100644 index 0000000..ee60bd2 --- /dev/null +++ b/lib/services/memory-compaction-service.ts @@ -0,0 +1,35 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { MemoryBridgeDbClient, MemoryBridgeNamespace, MemoryEvent } from "@/lib/services/memory-bridge-service"; +import { buildDailyContextPack } from "@/lib/services/memory-distillation-service"; +import { upsertVersionedMemoryProfile } from "@/lib/services/memory-profile-service"; +import { upsertOpenLoop } from "@/lib/services/memory-open-loop-service"; +import { redactSecrets } from "@/lib/services/memory-redaction-service"; +export type MemoryCompactionScope={user_id:string;namespace:MemoryBridgeNamespace;since?:string;dry_run?:boolean}; +export type MemoryCompactionResult={ok:boolean;namespace:MemoryBridgeNamespace;dry_run:boolean;daily_pack?:unknown;updated_profiles:unknown[];opened_or_updated_loops:unknown[];reviewed_candidates:unknown[];captured_events:unknown[];feedback_events:unknown[];blockers:string[];warnings:string[];next_step?:string}; +const projects=["Pandora Memory","PLP","Hatid","Speedcash","SpeedyPay","RetargetOS","AI GrowthOS","Red-Apple"]; +const personNames=["Patty","Melodee","Mang Rudy","Mang Bert","Ate Dhes"]; +const technicalWords=["GitHub","PR","merge","Vercel","Supabase","migration","deployment","build","CI","repo"]; +const relationshipWords=["relationship loop","emotional loop","relapse","unresolved emotional pattern"]; +const auOnlyNames=["Melodee","Mang Rudy","Mang Bert","Ate Dhes"]; +const blocked="[REDACTED_SECRET]"; +const clean=(s:string)=>redactSecrets(s).replace(/\s+/g," ").trim().slice(0,500); +const textOf=(x:any)=>clean([x.extracted_summary,x.summary,x.title,x.redacted_excerpt,x.raw_text,x.raw_excerpt].filter(Boolean).join(" — ")); +const has=(t:string,words:string[])=>words.some(w=>t.toLowerCase().includes(w.toLowerCase())); +const ref=(kind:string,x:any)=>({kind,id:x.id,source:x.source,source_ref:x.source_ref,created_at:x.created_at??x.reviewed_at}); +const subject=(s:string)=>s.toLowerCase().replace(/[^a-z0-9]+/g,"_").replace(/^_|_$/g,"")||"global"; +async function list(client:MemoryBridgeDbClient,table:string,input:MemoryCompactionScope,statuses?:string[]){const q:any=client.from(table).select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).order("created_at",{ascending:false}).limit(100);const r=await (q as Promise<{data:any[]|null;error:{message:string}|null}>); if(r.error)throw new Error(`${table}: ${r.error.message}`); return (r.data??[]).filter(x=>(!input.since||String(x.created_at??x.reviewed_at??"")>=input.since!)&&(!statuses||statuses.includes(x.status))).filter(x=>x.status!=="blocked_secret"&&textOf(x)!==blocked);} +function profileInputs(user_id:string,namespace:MemoryBridgeNamespace,sources:any[]){const out:any[]=[];const add=(profile_type:string,subject_key:string,title:string,items:any[],bucket:string)=>{if(!items.length)return; out.push({user_id,namespace,profile_type,subject_key,title,summary:items.map(textOf).filter(Boolean).slice(0,8).join("\n"),[bucket]:items.map((x:any)=>({text:textOf(x),ref:ref(x.raw_text?"event":"candidate",x)})),evidence_refs:items.map((x:any)=>ref(x.raw_text?"event":"candidate",x)),confidence:Math.min(0.95,0.55+items.length*0.05)});}; + add("operating_profile","global","Global operating profile",sources.filter(x=>has(textOf(x),["I prefer","from now on","always","don't","dont","be blunt","be concise","don't overpraise","check repo first","analyze automatically","maximize Codex task"])),"preferences"); + add("style_profile","global","Global style profile",sources.filter(x=>has(textOf(x),["tone","style","format","writing","prompt style","direct","Taglish","cinematic"])),"preferences"); + add("risk_profile","global","Global risk profile",sources.filter(x=>has(textOf(x),["gambling","money risk","reputation risk","avoidance","fantasy-vs-execution","legal risk","deployment claim not verified","fake progress","risk"])),"risks"); + for(const p of projects){const projectHits=sources.filter(x=>has(textOf(x),[p])); add("project_profile",subject(p),`${p} project profile`,projectHits,"facts"); add("technical_deployment_profile",subject(p),`${p} deployment profile`,projectHits.filter(x=>has(textOf(x),technicalWords)),"facts");} + add("technical_activity_profile","global","Global technical activity profile",sources.filter(x=>has(textOf(x),technicalWords)),"facts"); + for(const n of personNames){if(namespace==="real_life"&&auOnlyNames.includes(n))continue; const personHits=sources.filter(x=>has(textOf(x),[n])); add(namespace==="au"?"people_profile":"relationship_loop_profile",subject(n),`${n} profile`,personHits,"patterns");} + add("relationship_loop_profile","global","Global relationship/emotional loop profile",sources.filter(x=>has(textOf(x),relationshipWords)),"patterns"); + if(namespace==="au")add("writing_canon_profile","global","AU writing canon",sources.filter(x=>has(textOf(x),["AU","canon","chapter","story","fictional continuity","Melodee","Mang Rudy","Mang Bert","Ate Dhes"])),"facts"); + return out.filter(p=>p.summary&&!p.summary.includes(blocked));} +function loopInputs(user_id:string,namespace:MemoryBridgeNamespace,sources:any[]){const specs=[ ["deployment_blocker","deployment","Deployment blocker",["deployment blocked","Vercel failed","CI failed","failed migration","blocked"],8], ["review_backlog","candidate_review","Candidate review backlog",["candidate review backlog","pending review"],5], ["risk","global","Risk follow-up",["gambling","money risk","reputation risk","legal risk"],9], ["relationship_loop","relationship","Relationship/emotional loop",["relationship loop","emotional loop","relapse","unresolved"],7], ["project_followup","project","Project follow-up",["what next","next action","todo","follow up","pending"],5] ] as const; return specs.flatMap(([loop_type,sk,title,words,severity])=>{const items=sources.filter(x=>has(textOf(x),words as unknown as string[]));return items.length?[{user_id,namespace,loop_type,subject_key:sk,title,description:items.map(textOf).slice(0,5).join("\n"),severity,evidence_refs:items.map((x:any)=>ref(x.raw_text?"event":"candidate",x)),next_action:"Review and close when source evidence is resolved."}]:[];});} +export async function runDailyMemoryCompaction(client:MemoryBridgeDbClient,input:MemoryCompactionScope):Promise{const since=input.since??new Date(Date.now()-24*60*60*1000).toISOString(); const base={ok:false,namespace:input.namespace,dry_run:!!input.dry_run,updated_profiles:[],opened_or_updated_loops:[],reviewed_candidates:[],captured_events:[],feedback_events:[],blockers:[],warnings:[]} as MemoryCompactionResult; try{const events=await list(client,"memory_events",{...input,since},["captured","reviewed","promoted"]);const candidates=await list(client,"memory_capture_candidates",{...input,since},["approved","captured","rejected","duplicate"]);const feedback=await list(client,"memory_feedback_events",{...input,since});base.reviewed_candidates=candidates;base.captured_events=events;base.feedback_events=feedback;const usable=[...events,...candidates.filter(c=>["approved","captured"].includes(c.status))]; if(!usable.length){return {...base,ok:true,warnings:["no_source_data_available"],next_step:"Capture or approve memory before compaction."};} + for(const p of profileInputs(input.user_id,input.namespace,usable)){const r=await upsertVersionedMemoryProfile(client,{...p,dry_run:input.dry_run}); if(r.ok)base.updated_profiles.push(r.profile); else base.warnings.push(...r.warnings,...r.blockers);} + for(const l of loopInputs(input.user_id,input.namespace,usable)){const r=await upsertOpenLoop(client,{...l,dry_run:input.dry_run}); if(r.ok)base.opened_or_updated_loops.push(r.open_loop); else base.warnings.push(...r.warnings,...r.blockers);} + const safeEvents=events.map((e:MemoryEvent)=>({...e,raw_text:clean(e.raw_text),extracted_summary:e.extracted_summary?clean(e.extracted_summary):e.extracted_summary})); const pack={...buildDailyContextPack(input.namespace,input.user_id,safeEvents),open_loops:base.opened_or_updated_loops,risks:(base.updated_profiles as any[]).filter(p=>p.profile_type==="risk_profile")}; if(input.dry_run)base.daily_pack=pack; else {await client.from("memory_context_packs").update({status:"superseded",updated_at:new Date().toISOString()}).eq("user_id",input.user_id).eq("namespace",input.namespace).eq("pack_type","daily").eq("status","active").select("*").single(); const ins=await client.from("memory_context_packs").insert({...pack,status:"active"}).select("*").single(); if(ins.error||!ins.data)base.warnings.push(ins.error?.message??"context_pack_write_failed"); else base.daily_pack=ins.data;} return {...base,ok:true};}catch(e){return {...base,blockers:["compaction_failed"],warnings:[e instanceof Error?e.message:"unknown failure"],next_step:"Check compaction source table schemas and RLS."};}} \ No newline at end of file diff --git a/lib/services/memory-open-loop-service.ts b/lib/services/memory-open-loop-service.ts index bc33fb4..7067760 100644 --- a/lib/services/memory-open-loop-service.ts +++ b/lib/services/memory-open-loop-service.ts @@ -3,3 +3,15 @@ import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services export async function getOpenLoops(client:MemoryBridgeDbClient,userId:string,namespace:MemoryBridgeNamespace){ return client.from("memory_open_loops").select("*").eq("user_id",userId).eq("namespace",namespace).eq("status","open").order("severity",{ascending:false}) as unknown as Promise; } export async function createOpenLoop(client:MemoryBridgeDbClient,input:any){ return client.from("memory_open_loops").insert(input).select("*").single(); } export async function resolveOpenLoop(client:MemoryBridgeDbClient,id:string,userId:string){ return client.from("memory_open_loops").update({status:"resolved",resolved_at:new Date().toISOString()}).eq("id",id).eq("user_id",userId).select("*").single(); } +const arr=(v:unknown)=>Array.isArray(v)?v:[]; +function mergeEvidence(a:unknown,b:unknown){const seen=new Set();return [...arr(a),...arr(b)].filter(x=>{const k=JSON.stringify(x);if(seen.has(k))return false;seen.add(k);return true;});} +export async function upsertOpenLoop(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;loop_type:string;subject_key:string;title:string;description:string;severity:number;evidence_refs:unknown[];next_action?:string;dry_run?:boolean}){ + const existing=await (client.from("memory_open_loops").select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).eq("loop_type",input.loop_type).eq("subject_key",input.subject_key).order("updated_at",{ascending:false}).limit(5) as any as Promise<{data:any[]|null;error:{message:string}|null}>); + if(existing.error)return {ok:false,dry_run:!!input.dry_run,blockers:["open_loop_read_failed"],warnings:[existing.error.message],next_step:"Check memory_open_loops schema and RLS."}; + const active=(existing.data??[]).find(l=>["open","acknowledged"].includes(l.status)); + const row={user_id:input.user_id,namespace:input.namespace,loop_type:input.loop_type,subject_key:input.subject_key,title:input.title,description:input.description,severity:Math.max(1,Math.min(10,Number(input.severity)||3)),evidence_refs:input.evidence_refs,next_action:input.next_action,status:"open",updated_at:new Date().toISOString()}; + if(input.dry_run)return {ok:true,dry_run:true,open_loop:active?{...active,...row,severity:Math.max(Number(active.severity??0),row.severity),evidence_refs:mergeEvidence(active.evidence_refs,input.evidence_refs)}:row,blockers:[],warnings:[]}; + const result=active?await client.from("memory_open_loops").update({...row,severity:Math.max(Number(active.severity??0),row.severity),evidence_refs:mergeEvidence(active.evidence_refs,input.evidence_refs)}).eq("id",active.id).eq("user_id",input.user_id).eq("namespace",input.namespace).select("*").single():await client.from("memory_open_loops").insert(row).select("*").single(); + if(result.error||!result.data)return {ok:false,dry_run:false,blockers:["open_loop_write_failed"],warnings:[result.error?.message??"unknown open loop write failure"],next_step:"Check memory_open_loops schema and RLS."}; + return {ok:true,dry_run:false,open_loop:result.data,blockers:[],warnings:[]}; +} diff --git a/lib/services/memory-profile-service.ts b/lib/services/memory-profile-service.ts index 421da94..b637cf6 100644 --- a/lib/services/memory-profile-service.ts +++ b/lib/services/memory-profile-service.ts @@ -1,7 +1,68 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services/memory-bridge-service"; -export async function getActiveProfiles(client:MemoryBridgeDbClient,userId:string,namespace:MemoryBridgeNamespace,_filters?:{profile_types?:string[];subject_keys?:string[]}){ const q=client.from("memory_profiles").select("*").eq("user_id",userId).eq("namespace",namespace).eq("status","active"); return q as unknown as Promise; } -export async function upsertProfileFromMemoryEvents(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;profile_type:string;subject_key:string;summary:string;evidence_refs:unknown[];dry_run?:boolean}){ if(input.dry_run)return {dry_run:true,profile:input}; return client.from("memory_profiles").insert({user_id:input.user_id,namespace:input.namespace,profile_type:input.profile_type,subject_key:input.subject_key,title:input.subject_key,summary:input.summary,evidence_refs:input.evidence_refs,status:"active",version:1}).select("*").single(); } -export const updateOperatingProfile=upsertProfileFromMemoryEvents; export const updateStyleProfile=upsertProfileFromMemoryEvents; export const updateRiskProfile=upsertProfileFromMemoryEvents; export const updateProjectProfile=upsertProfileFromMemoryEvents; -export async function versionProfile(client:MemoryBridgeDbClient,id:string,userId:string){ return client.from("memory_profiles").update({status:"superseded",updated_at:new Date().toISOString()}).eq("id",id).eq("user_id",userId).select("*").single(); } -export async function retireProfile(client:MemoryBridgeDbClient,id:string,userId:string){ return client.from("memory_profiles").update({status:"retired",updated_at:new Date().toISOString()}).eq("id",id).eq("user_id",userId).select("*").single(); } + +export type MemoryProfileInput = { + user_id: string; + namespace: MemoryBridgeNamespace; + profile_type: string; + subject_key: string; + title: string; + summary: string; + facts?: unknown[]; + preferences?: unknown[]; + patterns?: unknown[]; + risks?: unknown[]; + open_loops?: unknown[]; + decisions?: unknown[]; + evidence_refs?: unknown[]; + confidence?: number; + dry_run?: boolean; +}; + +export type MemoryProfileUpsertResult = { ok: true; dry_run: boolean; profile: any; previous_profile?: any; blockers: string[]; warnings: string[] } | { ok: false; dry_run: boolean; blockers: string[]; warnings: string[]; next_step: string }; + +function arr(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } +function mergeEvidence(a: unknown, b: unknown) { + const seen = new Set(); + return [...arr(a), ...arr(b)].filter((item) => { const key = JSON.stringify(item); if (seen.has(key)) return false; seen.add(key); return true; }); +} + +export async function upsertVersionedMemoryProfile(client: MemoryBridgeDbClient, input: MemoryProfileInput): Promise { + const existing = await (client.from("memory_profiles").select("*").eq("user_id", input.user_id).eq("namespace", input.namespace).eq("profile_type", input.profile_type).eq("subject_key", input.subject_key).eq("status", "active").order("version", { ascending: false }).limit(1) as any as Promise<{ data: any[] | null; error: { message: string } | null }>); + if (existing.error) return { ok: false, dry_run: !!input.dry_run, blockers: ["profile_read_failed"], warnings: [existing.error.message], next_step: "Check memory_profiles RLS and schema." }; + const previous = existing.data?.[0]; + const version = Number(previous?.version ?? 0) + 1; + const row = { + user_id: input.user_id, + namespace: input.namespace, + profile_type: input.profile_type, + subject_key: input.subject_key, + title: input.title, + summary: input.summary, + facts: input.facts ?? [], + preferences: input.preferences ?? [], + patterns: input.patterns ?? [], + risks: input.risks ?? [], + open_loops: input.open_loops ?? [], + decisions: input.decisions ?? [], + evidence_refs: mergeEvidence(previous?.evidence_refs, input.evidence_refs), + confidence: input.confidence ?? previous?.confidence ?? 0.6, + status: "active", + version, + supersedes_profile_id: previous?.id, + updated_at: new Date().toISOString(), + }; + if (input.dry_run) return { ok: true, dry_run: true, profile: row, previous_profile: previous, blockers: [], warnings: [] }; + if (previous?.id) await client.from("memory_profiles").update({ status: "superseded", superseded_at: new Date().toISOString(), updated_at: new Date().toISOString() }).eq("id", previous.id).eq("user_id", input.user_id).eq("namespace", input.namespace).select("*").single(); + const inserted = await client.from("memory_profiles").insert(row).select("*").single(); + if (inserted.error || !inserted.data) return { ok: false, dry_run: false, blockers: ["profile_write_failed"], warnings: [inserted.error?.message ?? "unknown profile write failure"], next_step: "Check memory_profiles schema and RLS." }; + return { ok: true, dry_run: false, profile: inserted.data, previous_profile: previous, blockers: [], warnings: [] }; +} + +export async function upsertProfileFromMemoryEvents(client: MemoryBridgeDbClient, input: { user_id: string; namespace: MemoryBridgeNamespace; profile_type: string; subject_key: string; summary: string; evidence_refs: unknown[]; dry_run?: boolean }) { + return upsertVersionedMemoryProfile(client, { ...input, title: input.subject_key, confidence: input.evidence_refs.length ? 0.65 : 0.4 }); +} +export const updateOperatingProfile = upsertProfileFromMemoryEvents; +export const updateStyleProfile = upsertProfileFromMemoryEvents; +export const updateRiskProfile = upsertProfileFromMemoryEvents; +export const updateProjectProfile = upsertProfileFromMemoryEvents; diff --git a/lib/services/memory-session-digest-service.ts b/lib/services/memory-session-digest-service.ts index 73f6e5c..5b0ac65 100644 --- a/lib/services/memory-session-digest-service.ts +++ b/lib/services/memory-session-digest-service.ts @@ -2,5 +2,11 @@ import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services/memory-bridge-service"; import { summarizeSession } from "@/lib/services/memory-model-provider"; import { createCandidatesFromSession } from "@/lib/services/memory-candidate-service"; +import { runDailyMemoryCompaction } from "@/lib/services/memory-compaction-service"; +import { buildMasterContextPack } from "@/lib/services/memory-distillation-service"; export async function createSessionDigest(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;source:string;source_ref?:string;transcript_or_summary:string;auto_capture?:boolean;update_profiles?:boolean;distill?:boolean}){ const s=await summarizeSession({text:input.transcript_or_summary}); const c=await createCandidatesFromSession(client,{user_id:input.user_id,namespace:input.namespace,source:input.source,source_ref:input.source_ref,text:input.transcript_or_summary}); const row={user_id:input.user_id,namespace:input.namespace,source:input.source,source_ref:input.source_ref,title:s.title,summary:s.summary,durable_updates:c.candidates,decisions:[],open_loops:[],risks:[],people:[],projects:[],style_updates:[],candidate_ids:(c.candidates as any[]).map(x=>x.id).filter(Boolean),captured_event_ids:[],profile_ids:[]}; const r=await client.from("memory_session_digests").insert(row).select("*").single(); return {digest:r.data??row,warnings:[...(c.warnings??[]),...(r.error?[r.error.message]:[])]}; } -export const digestSessionTranscript=createSessionDigest; export const digestSessionSummary=createSessionDigest; export async function compoundSessionIntoProfiles(){return {ok:true,warnings:["profile_compounding_minimal"]};} export async function createDailyPackFromSessionDigests(){return {ok:true};} export async function createMasterPackFromProfiles(){return {ok:true};} +export const digestSessionTranscript=createSessionDigest; +export const digestSessionSummary=createSessionDigest; +export async function compoundSessionIntoProfiles(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;since?:string;dry_run?:boolean}){ const r=await runDailyMemoryCompaction(client,input); return {ok:r.ok,data:{updated_profiles:r.updated_profiles,opened_or_updated_loops:r.opened_or_updated_loops},blockers:r.blockers,warnings:r.warnings}; } +export async function createDailyPackFromSessionDigests(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;since?:string;dry_run?:boolean}){ const r=await runDailyMemoryCompaction(client,input); return {ok:r.ok,data:r.daily_pack,blockers:r.blockers,warnings:r.warnings}; } +export async function createMasterPackFromProfiles(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;dry_run?:boolean}){ const events=await (client.from("memory_events").select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).order("created_at",{ascending:false}).limit(50) as any as Promise<{data:any[]|null;error:{message:string}|null}>); if(events.error)return {ok:false,data:null,blockers:["event_read_failed"],warnings:[events.error.message]}; const pack=buildMasterContextPack(input.namespace,input.user_id,events.data??[]); if(input.dry_run)return {ok:true,data:pack,blockers:[],warnings:[]}; const ins=await client.from("memory_context_packs").insert({...pack,status:"active"}).select("*").single(); return {ok:!ins.error,data:ins.data??pack,blockers:ins.error?["master_pack_write_failed"]:[],warnings:ins.error?[ins.error.message]:[]}; } diff --git a/tests/phase5c-memory-compaction.test.ts b/tests/phase5c-memory-compaction.test.ts new file mode 100644 index 0000000..961d0e3 --- /dev/null +++ b/tests/phase5c-memory-compaction.test.ts @@ -0,0 +1,18 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { describe, expect, it, vi } from "vitest"; +import { runDailyMemoryCompaction } from "../lib/services/memory-compaction-service"; +import { upsertVersionedMemoryProfile } from "../lib/services/memory-profile-service"; +import { upsertOpenLoop } from "../lib/services/memory-open-loop-service"; +function client(seed:Partial>={}){const rows:Record={memory_events:[],memory_capture_candidates:[],memory_feedback_events:[],memory_profiles:[],memory_open_loops:[],memory_context_packs:[],...seed}; return {rows,from(table:string){let selected=[...(rows[table]??[])];let patch:any;let insertValue:any;const api:any={select:()=>api,eq:(k:string,v:any)=>{selected=selected.filter(r=>r[k]===v);return api;},order:(k:string,opt:any)=>{selected.sort((a,b)=>opt?.ascending?String(a[k]??"").localeCompare(String(b[k]??"")):String(b[k]??"").localeCompare(String(a[k]??"")));return api;},limit:(n:number)=>{selected=selected.slice(0,n);return api;},update:(v:any)=>{patch=v;return api;},insert:(v:any)=>{insertValue=v;return api;},single:async()=>{if(insertValue){const row={id:`${table}-${rows[table].length+1}`,...insertValue};rows[table].push(row);return {data:row,error:null};} if(patch){for(const r of selected)Object.assign(r,patch);return {data:selected[0]??null,error:selected[0]?null:{message:"not found"}};} return {data:selected[0]??null,error:selected[0]?null:{message:"not found"}};},then:(resolve:any)=>resolve({data:selected,error:null})};return api;}} as any;} +const ev=(id:string,raw_text:string,namespace="real_life",over:any={})=>({id,user_id:"u1",namespace,source:"project_update",raw_text,extracted_summary:raw_text,importance:7,sensitivity:"medium",status:"captured",created_at:"2026-06-28T00:00:00.000Z",...over}); + +describe("Phase 5C deterministic compaction",()=>{ + it("creates project, operating, style, risk, relationship, and daily pack outputs",async()=>{const c=client({memory_events:[ev("e1","Pandora Memory PR pending review in GitHub; Vercel deployment blocked"),ev("e2","I prefer be concise and don't overpraise; use direct tone"),ev("e3","money risk and reputation risk detected"),ev("e4","Patty relationship loop unresolved")],memory_capture_candidates:[{id:"c1",user_id:"u1",namespace:"real_life",status:"approved",summary:"Supabase migration pending",created_at:"2026-06-28T00:00:00.000Z"}],memory_feedback_events:[{id:"f1",user_id:"u1",namespace:"real_life",action:"approve",created_at:"2026-06-28T00:00:00.000Z"}]}); const r=await runDailyMemoryCompaction(c,{user_id:"u1",namespace:"real_life",since:"2026-06-27T00:00:00.000Z"}); expect(r.ok).toBe(true); expect(c.rows.memory_profiles.map((p:any)=>p.profile_type)).toEqual(expect.arrayContaining(["project_profile","operating_profile","style_profile","risk_profile","relationship_loop_profile"])); expect(c.rows.memory_open_loops.length).toBeGreaterThan(0); expect(c.rows.memory_context_packs[0].pack_type).toBe("daily"); expect(JSON.stringify(r.daily_pack)).not.toContain("password"); }); + it("keeps AU canon out of real_life but creates writing canon in au",async()=>{const au=client({memory_events:[ev("e1","AU canon chapter with Melodee fictional continuity","au")]}); await runDailyMemoryCompaction(au,{user_id:"u1",namespace:"au",since:"2026-06-27T00:00:00.000Z"}); expect(au.rows.memory_profiles.some((p:any)=>p.profile_type==="writing_canon_profile")).toBe(true); const real=client({memory_events:[ev("e1","AU canon chapter with Melodee fictional continuity","real_life")]}); await runDailyMemoryCompaction(real,{user_id:"u1",namespace:"real_life",since:"2026-06-27T00:00:00.000Z"}); expect(real.rows.memory_profiles.some((p:any)=>p.profile_type==="writing_canon_profile")).toBe(false); }); + it("versions active profiles and dry_run does not write",async()=>{const c=client({memory_profiles:[{id:"p1",user_id:"u1",namespace:"real_life",profile_type:"operating_profile",subject_key:"global",status:"active",version:1,evidence_refs:[{id:"old"}]}]}); const r=await upsertVersionedMemoryProfile(c,{user_id:"u1",namespace:"real_life",profile_type:"operating_profile",subject_key:"global",title:"t",summary:"s",evidence_refs:[{id:"new"}]}); expect(r.ok).toBe(true); expect(c.rows.memory_profiles[0].status).toBe("superseded"); expect(c.rows.memory_profiles[1].version).toBe(2); const d=client(); await runDailyMemoryCompaction(d,{user_id:"u1",namespace:"real_life",dry_run:true}); expect(d.rows.memory_profiles).toHaveLength(0); }); + it("updates open loops, merges evidence, and does not downgrade severity",async()=>{const c=client({memory_open_loops:[{id:"l1",user_id:"u1",namespace:"real_life",loop_type:"deployment",subject_key:"pandora",status:"open",severity:8,evidence_refs:[{id:"a"}]}]}); await upsertOpenLoop(c,{user_id:"u1",namespace:"real_life",loop_type:"deployment",subject_key:"pandora",title:"Deployment",description:"CI pending",severity:5,evidence_refs:[{id:"b"}]}); expect(c.rows.memory_open_loops).toHaveLength(1); expect(c.rows.memory_open_loops[0].severity).toBe(8); expect(c.rows.memory_open_loops[0].evidence_refs).toHaveLength(2); }); +}); + +describe("Phase 5C job endpoint",()=>{ + it("requires token and distillation gate",async()=>{vi.resetModules(); process.env.PANDORA_INTERNAL_JOB_TOKEN="secret"; process.env.PANDORA_ENABLE_MEMORY_DISTILLATION="false"; const route=await import("../app/api/memory/jobs/daily-digest/route"); let res=await route.POST(new Request("http://x",{method:"POST",body:"{}"}) as any); expect(res.status).toBe(401); res=await route.POST(new Request("http://x",{method:"POST",headers:{authorization:"Bearer wrong"},body:"{}"}) as any); expect(res.status).toBe(401); res=await route.POST(new Request("http://x",{method:"POST",headers:{authorization:"Bearer secret"},body:JSON.stringify({namespace:"real_life",dry_run:true,user_id:"u1"})}) as any); expect(res.status).toBe(403); }); +});