storage: migrate core execution state to SQLite

README.md

@@ -152,7 +152,7 @@ Start with [ARCHITECTURE.md](./ARCHITECTURE.md). It provides the system map, cod
 apps/desktop/       Electron main / preload / React renderer
 
 packages/core/      Pure contracts for Sessions, Events, Permissions, and Connections
-packages/storage/   File-backed stores and run ledgers
+packages/storage/   SQLite operational state, legacy importers, and payload stores
 packages/runtime/   AgentRun, model adapters, tools, context, and recovery
 packages/headless/  TaskRun, Autonomous Loop, Self-check, eval, and AHE
 packages/cli/       TUI and non-interactive CLI
@@ -184,7 +184,7 @@ Current boundaries that matter:
 
 Read [SECURITY.md](./SECURITY.md) for security reporting and policy, and [docs/README.md](./docs/README.md) for current privacy and sandbox contracts.
 
-## Experimental runtime recovery flag
+## Runtime storage and recovery
 
 RuntimeEvent persistence is always canonical in `runtime.sqlite`. On the first
 write, Maka batch-idempotently imports legacy RuntimeEvent JSONL without
@@ -200,6 +200,16 @@ rows; a legacy database changed after cutover fails closed. The old database is
 retained as migration evidence but is no longer a production writer. Session
 transcript bodies remain append-only JSONL.
 
+Core execution state now shares that authority too: AgentRun headers and event
+ledgers, event projections, root-turn admissions and source proofs,
+Interactions, Host Epoch message receipts, and ShellRun records are canonical
+in `runtime.sqlite` across CLI, Desktop, Runtime Host, and Headless. Each legacy
+file store is fingerprinted and imported through its own durable
+`cutover_journal` entry before the corresponding repository opens. Copy and
+validation are one SQLite transaction, retries are idempotent, and a changed
+legacy source after cutover fails closed. The legacy files are retained only as
+migration evidence; new execution writes do not modify them.
+
 Runtime continuation remains opt-in:
 
 - `MAKA_RUNTIME_SAFE_BOUNDARY_RESUME=1` enables the Desktop interrupted-turn

packages/cli/src/__tests__/runtime-bootstrap.test.ts

@@ -8,7 +8,7 @@ import {
   createConnectionStore,
   createFileCredentialStore,
   createSessionStore,
-  createShellRunStore,
+  createSqliteShellRunStore,
 } from '@maka/storage';
 import {
   BackendRegistry,
@@ -592,10 +592,10 @@ describe('Maka CLI runtime bootstrap', () => {
         assert.equal(detail.output?.stdout, 'start');
 
         await context.close();
-        const record = await createShellRunStore(workspaceRoot).readShellRun(
-          'session-1',
-          backgroundTaskId(result.ref),
-        );
+        const shellRuns = createSqliteShellRunStore(workspaceRoot);
+        await shellRuns.ready();
+        const record = await shellRuns.readShellRun('session-1', backgroundTaskId(result.ref));
+        shellRuns.close();
         assert.equal(record.status, 'cancelled');
         assert.equal(record.exitCode, 130);
       } finally {
@@ -744,10 +744,10 @@ describe('Maka CLI runtime bootstrap', () => {
           return snapshot?.result.status === 'completed' ? snapshot : undefined;
         });
         assert.equal(hydrated.result.status, 'completed');
-        const stored = await createShellRunStore(workspaceRoot).readShellRun(
-          'session-1',
-          backgroundTaskId(started.ref),
-        );
+        const shellRuns = createSqliteShellRunStore(workspaceRoot);
+        await shellRuns.ready();
+        const stored = await shellRuns.readShellRun('session-1', backgroundTaskId(started.ref));
+        shellRuns.close();
         assert.equal(stored.observedAt, undefined);
       } finally {
         await context.close();

packages/cli/src/runtime-bootstrap.ts

@@ -58,7 +58,7 @@ import {
   type ModelMessage,
 } from '@maka/runtime';
 import {
-  createAgentRunStore,
+  createSqliteAgentRunStore,
   createAttachmentByteReader,
   createArtifactStore,
   createAutomationStore,
@@ -70,7 +70,7 @@ import {
   createReadImageSnapshotter,
   createSessionStore,
   createSettingsStore,
-  createShellRunStore,
+  createSqliteShellRunStore,
   assertSessionBundleRootLayout,
   type ForeignSessionStore,
   persistProviderRequestCaptureArtifact,
@@ -222,12 +222,19 @@ export async function createMakaCliRuntimeContext(
   }
   await resolveStorageRoot({ path: stateRoot, kind: 'interactive' });
   const store = createSessionStore(stateRoot);
-  const runStore = createAgentRunStore(stateRoot);
+  const runStore = createSqliteAgentRunStore(stateRoot);
   const runtimePersistence = await openRuntimeEventPersistence({
     workspaceRoot: stateRoot,
   });
   const runtimeEventStore = runtimePersistence.runtimeEventStore;
-  const shellRunStore = createShellRunStore(stateRoot);
+  const shellRunStore = createSqliteShellRunStore(stateRoot);
+  await Promise.all([runStore.ready?.(), shellRunStore.ready()]).catch(async (error) => {
+    await store.close?.().catch(() => {});
+    runtimePersistence.close();
+    runStore.close?.();
+    shellRunStore.close();
+    throw error;
+  });
   const artifactStore = createArtifactStore(stateRoot);
   const agentGraphControlStore = agentGraphEnabled
     ? createAgentGraphControlStore(stateRoot)
@@ -1093,6 +1100,8 @@ export async function createMakaCliRuntimeContext(
       shellRunListeners.clear();
       await store.close?.();
       runtimePersistence.close();
+      runStore.close?.();
+      shellRunStore.close();
     },
   };
 }

packages/headless/src/__tests__/provider-request-trace.test.ts

@@ -5,6 +5,7 @@ import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import type { AgentRunEvent, AgentRunHeader } from '@maka/core';
 import type { InvocationResult } from '@maka/runtime';
+import { acquireOperationalStateDatabase } from '@maka/storage';
 
 import { writeHarborTaskRunTrace } from '../harbor-cell.js';
 import { openHeadlessStorageForWrite } from '../headless-storage.js';
@@ -263,11 +264,7 @@ test('exports a torn AgentRun tail as incomplete provider-request evidence', asy
     await runStore.createRun(header);
     await runStore.appendEvent(identity.sessionId, identity.runId, capture as AgentRunEvent);
     await runStore.appendEvent(identity.sessionId, identity.runId, attempt as AgentRunEvent);
-    await writeFile(
-      join(storageRoot, 'sessions', identity.sessionId, 'runs', identity.runId, 'events.jsonl'),
-      '{"type":"provider_request_attempt_recorded"',
-      { flag: 'a' },
-    );
+    corruptAgentRunEvent(storageRoot, identity.sessionId, identity.runId, 1);
 
     const traceEventsPath = await writeHarborTaskRunTrace({
       outputDir,
@@ -322,10 +319,14 @@ test('also diagnoses missing provider evidence when the only run event is corrup
       updatedAt: 4,
       completedAt: 4,
     });
-    await writeFile(
-      join(storageRoot, 'sessions', identity.sessionId, 'runs', identity.runId, 'events.jsonl'),
-      '{"type":"tool_failed"',
-    );
+    await storage.executionStores.agentRunStore.appendEvent(identity.sessionId, identity.runId, {
+      type: 'tool_failed',
+      id: 'corrupt-event',
+      ...identity,
+      ts: 2,
+      message: 'will be corrupted',
+    });
+    corruptAgentRunEvent(storageRoot, identity.sessionId, identity.runId, 0);
 
     const traceEventsPath = await writeHarborTaskRunTrace({
       outputDir,
@@ -347,6 +348,27 @@ test('also diagnoses missing provider evidence when the only run event is corrup
   }
 });
 
+function corruptAgentRunEvent(
+  storageRoot: string,
+  sessionId: string,
+  runId: string,
+  sequence: number,
+): void {
+  const lease = acquireOperationalStateDatabase(storageRoot);
+  try {
+    const result = lease.database
+      .prepare(`
+        UPDATE core_agent_run_events
+        SET record_json = ?
+        WHERE session_id = ? AND run_id = ? AND sequence = ?
+      `)
+      .run('{"type":"corrupt"', sessionId, runId, sequence);
+    assert.equal(result.changes, 1);
+  } finally {
+    lease.close();
+  }
+}
+
 test('exports missing provider-request evidence for every continuation invocation', async () => {
   const root = await mkdtemp(join(tmpdir(), 'maka-provider-trace-export-'));
   const storageRoot = join(root, 'storage');

packages/headless/src/__tests__/task-agent-controller.test.ts

@@ -21,7 +21,11 @@ import {
 } from '@maka/core';
 import type { BackendSendInput } from '@maka/core/backend-types';
 import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary';
-import { createSessionStore, openRuntimeEventReadPersistence } from '@maka/storage';
+import {
+  createSessionStore,
+  createSqliteAgentRunStore,
+  openRuntimeEventReadPersistence,
+} from '@maka/storage';
 import { StorageRootAuthorityError } from '@maka/storage/root-authority';
 import type { Config, Task } from '../contracts.js';
 import { openHeadlessStorageForWrite } from '../headless-storage.js';
@@ -1160,8 +1164,13 @@ async function readAgentRunHeader(
   sessionId: string,
   runId: string,
 ): Promise<AgentRunHeader> {
-  const runPath = join(storageRoot, 'sessions', sessionId, 'runs', runId, 'run.json');
-  return JSON.parse(await readFile(runPath, 'utf8')) as AgentRunHeader;
+  const store = createSqliteAgentRunStore(storageRoot);
+  try {
+    await store.ready?.();
+    return await store.readRun(sessionId, runId);
+  } finally {
+    store.close?.();
+  }
 }
 
 describe('runTaskOnce', () => {

packages/runtime-host/src/__tests__/canonical-session-projection.test.ts

@@ -1,5 +1,5 @@
 import assert from 'node:assert/strict';
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { mkdtemp, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { test } from 'node:test';
@@ -9,6 +9,7 @@ import {
   type ExecutionStoresWriter,
 } from '@maka/storage/execution-stores';
 import type { StoredInteractionRequest } from '@maka/storage/interaction-store';
+import { acquireOperationalStateDatabase } from '@maka/storage';
 import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '@maka/storage/root-authority';
 import { type SessionMessageQueueProjection } from '../protocol/index.js';
 import {
@@ -311,19 +312,47 @@ test('fails closed when the owned tip durable identity changes', async () => {
       sourceMessages: [],
       admittedAt: 10,
     });
-    const admissionPath = join(root, 'sessions', session.id, 'turn-admissions', 'turn-1.json');
-    const original = await readFile(admissionPath, 'utf8');
-    const durable = JSON.parse(original) as Record<string, unknown>;
+    const database = acquireOperationalStateDatabase(root);
+    const row = database.database
+      .prepare(`
+        SELECT admitted_at, record_json
+        FROM core_root_turn_admissions
+        WHERE session_id = ? AND turn_id = 'turn-1'
+      `)
+      .get(session.id) as { admitted_at?: unknown; record_json?: unknown } | undefined;
+    assert.equal(typeof row?.admitted_at, 'number');
+    assert.equal(typeof row?.record_json, 'string');
+    const durable = JSON.parse(row!.record_json as string) as Record<string, unknown>;
 
-    await rm(admissionPath);
+    database.transaction('write', () => {
+      database.database
+        .prepare(`
+          DELETE FROM core_root_turn_admissions
+          WHERE session_id = ? AND turn_id = 'turn-1'
+        `)
+        .run(session.id);
+    });
     const missingReader = new CanonicalSessionProjectionReader({
       stores,
       rootAdmissions,
       messages: createMessages(session.id, stores),
     });
     await assert.rejects(() => missingReader.read(session.id), /missing from durable storage/);
 
-    await writeFile(admissionPath, `${JSON.stringify({ ...durable, runId: 'run-drifted' })}\n`);
+    database.transaction('write', () => {
+      database.database
+        .prepare(`
+          INSERT INTO core_root_turn_admissions(
+            session_id, turn_id, admitted_at, record_json
+          ) VALUES (?, 'turn-1', ?, ?)
+        `)
+        .run(
+          session.id,
+          row!.admitted_at as number,
+          JSON.stringify({ ...durable, runId: 'run-drifted' }),
+        );
+    });
+    database.close();
 
     const reader = new CanonicalSessionProjectionReader({
       stores,

packages/runtime-host/src/__tests__/execution-host.test.ts

@@ -1,11 +1,20 @@
 import assert from 'node:assert/strict';
 import { fork, type ChildProcess } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { appendFile, chmod, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import {
+  appendFile,
+  chmod,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readdir,
+  rm,
+  writeFile,
+} from 'node:fs/promises';
 import { createServer, type Server } from 'node:http';
 import { connect, type Socket } from 'node:net';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
 import { test } from 'node:test';
 import type { AgentRunHeader } from '@maka/core/agent-run';
 import type { MessageContent } from '@maka/core/events';
@@ -1071,6 +1080,7 @@ test('startup recovery imports past a truncated legacy RuntimeEvent tail without
       'recover after a partial RuntimeEvent write',
     );
     const runtimeEventsPath = fixture.runtimeEventsPath(runId);
+    await mkdir(dirname(runtimeEventsPath), { recursive: true });
     await writeFile(runtimeEventsPath, '{"id":"truncated"', 'utf8');
 
     const host = await fixture.startHost();
@@ -1101,6 +1111,7 @@ test('startup recovery fails closed on a complete malformed RuntimeEvent record'
     );
     const runtimeEventsPath = fixture.runtimeEventsPath(runId);
     const malformed = '{"id":"malformed"\n';
+    await mkdir(dirname(runtimeEventsPath), { recursive: true });
     await writeFile(runtimeEventsPath, malformed, 'utf8');
 
     await fixture.expectHostStartupFailure();
@@ -1134,6 +1145,7 @@ test('startup recovery fails closed on a complete malformed AgentRun record', as
     );
     const eventsPath = fixture.eventsPath(runId);
     const malformed = '{"type":"run_started"\n';
+    await mkdir(dirname(eventsPath), { recursive: true });
     await writeFile(eventsPath, malformed, 'utf8');
 
     await fixture.expectHostStartupFailure();