fix(usage): report incomplete transcript scans

apps/mobile/src/features/usage/UsageRouteScreen.tsx

@@ -1,7 +1,7 @@
 import { EnvironmentId, USAGE_CONTRACT_VERSION } from "@t3tools/contracts";
 import { type RouteProp, useNavigation, useRoute } from "@react-navigation/native";
 import {
-  isCompatibleUsageContractVersion,
+  isMergeableUsageSummary,
   isModelCostUnknown,
   type DailyTotals,
   type MergedUsage,
@@ -647,7 +647,7 @@ function isUsageLoading(environment: EnvironmentUsageStatus) {
 function usageEnvironmentStatus(environment: EnvironmentUsageStatus): string {
   if (
     environment.summary &&
-    !isCompatibleUsageContractVersion(environment.summary.contractVersion, USAGE_CONTRACT_VERSION)
+    !isMergeableUsageSummary(environment.summary, USAGE_CONTRACT_VERSION)
   ) {
     return "Older server · excluded from usage totals";
   }

apps/server/src/usage/UsageService.test.ts

@@ -4,6 +4,10 @@ import * as NodeFSP from "node:fs/promises";
 import * as NodeOS from "node:os";
 import * as NodePath from "node:path";
 
+import { vi } from "vite-plus/test";
+
+vi.mock("node:fs/promises", { spy: true });
+
 import { assert, describe, it } from "@effect/vitest";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";
@@ -199,6 +203,19 @@ describe("UsageService", () => {
       );
       const summary = yield* service.readSummary(WINDOW);
       assert.strictEqual(totalOutputTokens(summary), 36);
+      assert.deepStrictEqual(
+        summary.buckets
+          .filter((bucket) => bucket.provider === "claude")
+          .map((bucket) => ({
+            path: summary.sources[bucket.sourceIndex!]?.fingerprint.resolvedHomePath,
+            output: bucket.totals.outputTokens,
+          }))
+          .sort((a, b) => a.output - b.output),
+        [
+          { path: NodePath.join(home, "claude", "projects"), output: 5 },
+          { path: NodePath.join(claudeHome, "projects"), output: 7 },
+        ],
+      );
       const sources = summary.sources.filter((source) => source.status === "ok");
       assert.strictEqual(sources.length, 4);
       assert.strictEqual(
@@ -408,8 +425,8 @@ describe("UsageService", () => {
         const service = yield* UsageService.make.pipe(
           Effect.provideService(FileSystem.FileSystem, {
             ...fileSystem,
-            exists: (path) =>
-              fileSystem.exists(path).pipe(
+            realPath: (path) =>
+              fileSystem.realPath(path).pipe(
                 Effect.tap(() => {
                   if (path !== NodePath.join(home, "claude", "projects")) return Effect.void;
                   homeProbes += 1;
@@ -607,3 +624,113 @@ describe("UsageService", () => {
     }).pipe(Effect.scoped),
   );
 });
+
+describe("UsageService scan coverage", () => {
+  it.live("reports read failures and retries unchanged bytes without caching an empty result", () =>
+    Effect.gen(function* () {
+      const { transcript, settings, home } = yield* setup;
+      yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5)));
+      const service = yield* UsageService.make.pipe(
+        Effect.provide(
+          serviceLayers({ prefix: "usage-service-read-failure-test", home, settings }),
+        ),
+      );
+      const first = yield* service.readSummary(WINDOW);
+      assert.strictEqual(totalOutputTokens(first), 5);
+      yield* Effect.promise(() => NodeFSP.appendFile(transcript, claudeLine(2, 7)));
+      const open = vi
+        .spyOn(NodeFSP, "open")
+        .mockRejectedValueOnce(
+          Object.assign(new Error("private transcript contents"), { code: "EACCES" }),
+        );
+      const failed = yield* service
+        .readSummary(WINDOW)
+        .pipe(Effect.ensuring(Effect.sync(() => open.mockRestore())));
+      const source = failed.sources.find((source) => source.fingerprint.provider === "claude");
+      assert.strictEqual(source?.status, "partial");
+      assert.strictEqual(source?.skippedFiles, 1);
+      assert.notInclude(source?.message ?? "", "private transcript contents");
+      const recovered = yield* service.readSummary(WINDOW);
+      assert.strictEqual(
+        recovered.sources.find((source) => source.fingerprint.provider === "claude")?.status,
+        "ok",
+      );
+      assert.strictEqual(totalOutputTokens(recovered), 12);
+    }).pipe(Effect.scoped),
+  );
+
+  it.live.each(["EACCES", "ENOENT"])(
+    "keeps cached transcripts hidden by a %s directory walk",
+    (code) =>
+      Effect.gen(function* () {
+        const { transcript, settings, home } = yield* setup;
+        yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5)));
+        yield* Effect.promise(() =>
+          NodeFSP.writeFile(
+            NodePath.join(home, "claude", "projects", "readable.jsonl"),
+            claudeLine(2, 7),
+          ),
+        );
+        const service = yield* UsageService.make.pipe(
+          Effect.provide(
+            serviceLayers({ prefix: "usage-service-partial-cache-test", home, settings }),
+          ),
+        );
+        assert.strictEqual(totalOutputTokens(yield* service.readSummary(WINDOW)), 12);
+        const actual = yield* Effect.promise(() =>
+          vi.importActual<typeof NodeFSP>("node:fs/promises"),
+        );
+        const readdir = vi
+          .mocked(NodeFSP.readdir)
+          .mockImplementationOnce((...args) => actual.readdir(...args))
+          .mockRejectedValueOnce(
+            Object.assign(new Error("cannot list nested directory"), { code }),
+          );
+        const partial = yield* service
+          .readSummary(WINDOW)
+          .pipe(Effect.ensuring(Effect.sync(() => readdir.mockRestore())));
+        assert.strictEqual(
+          partial.sources.find((source) => source.fingerprint.provider === "claude")?.status,
+          "partial",
+        );
+        assert.strictEqual(totalOutputTokens(partial), 7);
+        // The unchanged file should remain cached. A reread would fail here.
+        const open = vi.mocked(NodeFSP.open).mockRejectedValueOnce(new Error("unexpected reread"));
+        const recovered = yield* service
+          .readSummary(WINDOW)
+          .pipe(Effect.ensuring(Effect.sync(() => open.mockRestore())));
+        assert.strictEqual(totalOutputTokens(recovered), 12);
+        assert.strictEqual(
+          recovered.sources.find((source) => source.fingerprint.provider === "claude")?.status,
+          "ok",
+        );
+      }).pipe(Effect.scoped),
+  );
+
+  it.live("distinguishes an unreadable root from absent provider directories", () =>
+    Effect.gen(function* () {
+      const { settings, home } = yield* setup;
+      const service = yield* UsageService.make.pipe(
+        Effect.provide(
+          serviceLayers({ prefix: "usage-service-root-failure-test", home, settings }),
+        ),
+      );
+      const readdir = vi
+        .spyOn(NodeFSP, "readdir")
+        .mockRejectedValueOnce(
+          Object.assign(new Error("private directory detail"), { code: "EACCES" }),
+        );
+      const summary = yield* service
+        .readSummary(WINDOW)
+        .pipe(Effect.ensuring(Effect.sync(() => readdir.mockRestore())));
+      assert.strictEqual(
+        summary.sources.find((source) => source.fingerprint.provider === "claude")?.status,
+        "failed",
+      );
+      assert.strictEqual(
+        summary.sources.find((source) => source.fingerprint.provider === "codex")?.status,
+        "missing",
+      );
+    }).pipe(Effect.scoped),
+  );
+});

apps/server/src/usage/UsageService.ts

@@ -338,7 +338,7 @@ export const make = Effect.gen(function* () {
     size: number,
     mtimeMs: number,
     provider: UsageProviderKind,
-  ): Effect.Effect<readonly UsageRecord[]> =>
+  ): Effect.Effect<readonly UsageRecord[] | null> =>
     Effect.gen(function* () {
       const cached = fileCache.get(filePath);
       // Provider is part of the identity: if both providers were ever pointed
@@ -366,7 +366,7 @@ export const make = Effect.gen(function* () {
       );
       // A read failure is not an empty transcript: caching it under this
       // (size, mtime) would silently drop the file's usage until it changes.
-      if (parsed === null) return [];
+      if (parsed === null) return null;
 
       // Stored already de-duplicated within the file, which is 99% of all
       // duplicates. The aggregator still runs the cross-file dedupe pass. One
@@ -394,40 +394,42 @@ export const make = Effect.gen(function* () {
     readonly provider: UsageProviderKind;
     readonly dir: string;
     readonly volumeId: string;
-    /** Parsed records per file, or `null` when the directory does not exist. */
-    readonly files:
-      | readonly { readonly path: string; readonly records: readonly UsageRecord[] }[]
-      | null;
+    readonly status: "ok" | "partial" | "missing" | "failed";
+    readonly failedEntries: number;
+    readonly files: readonly {
+      readonly path: string;
+      readonly records: readonly UsageRecord[] | null;
+    }[];
   }
 
   const collectDirs = Effect.fn("UsageService.collectDirs")(function* (
     windowStartMs: number,
     settings: ServerSettingsValue,
   ) {
-    // The home resolvers ask for `Path` themselves; satisfy them from the
-    // instance we already hold so the scan stays context-free.
     const dirs = yield* resolveTranscriptDirs(settings).pipe(
       Effect.provideService(Path.Path, path),
     );
     const scanned: ScannedDir[] = [];
     for (const { provider, dir, fileName } of dirs) {
       const volumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir));
-      const exists = yield* fileSystem
-        .exists(dir)
-        .pipe(Effect.catchCause(() => Effect.succeed(false)));
-      if (!exists) {
-        scanned.push({ provider, dir, volumeId, files: null });
-        continue;
-      }
-      const files = yield* Effect.promise(() =>
+      const listing = yield* Effect.promise(() =>
         listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }),
       );
-      const parsedFiles: { path: string; records: readonly UsageRecord[] }[] = [];
-      for (const file of files) {
+      const parsedFiles: { path: string; records: readonly UsageRecord[] | null }[] = [];
+      let failedEntries = listing.failedEntries;
+      for (const file of listing.files) {
         const records = yield* readFileRecords(file.path, file.size, file.mtimeMs, provider);
+        if (records === null) failedEntries += 1;
         parsedFiles.push({ path: file.path, records });
       }
-      scanned.push({ provider, dir, volumeId, files: parsedFiles });
+      scanned.push({
+        provider,
+        dir,
+        volumeId,
+        files: parsedFiles,
+        failedEntries,
+        status: listing.status === "ok" && failedEntries > 0 ? "partial" : listing.status,
+      });
     }
     return scanned;
   });
@@ -503,21 +505,25 @@ export const make = Effect.gen(function* () {
     const livePaths = new Set<string>();
     const walkedRoots: string[] = [];
 
-    for (const { provider, dir, volumeId, files } of scannedDirs) {
-      if (files === null) {
+    for (const { provider, dir, volumeId, files, status, failedEntries } of scannedDirs) {
+      if (status === "missing" || status === "failed") {
         sources.push({
           fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },
-          status: "missing",
+          status,
           scannedFiles: 0,
           skippedFiles: 0,
           malformedRecords: 0,
           distinctSessions: 0,
-          message: "No transcript directory on this environment.",
+          message:
+            status === "missing"
+              ? "No transcript directory on this environment."
+              : "Could not read the transcript directory.",
         });
         continue;
       }
 
-      walkedRoots.push(dir);
+      // An incomplete walk cannot establish which cached files disappeared.
+      if (status === "ok") walkedRoots.push(dir);
       let scannedFiles = 0;
       let skippedFiles = 0;
       // Distinct per directory. Buckets carry per-cell session counts, but a
@@ -526,28 +532,31 @@ export const make = Effect.gen(function* () {
 
       for (const file of files) {
         livePaths.add(file.path);
-        if (file.records.length === 0) {
+        if (file.records === null || file.records.length === 0) {
           skippedFiles += 1;
           continue;
         }
         scannedFiles += 1;
         for (const record of file.records) {
           // Only sessions that contributed in-window count: the mtime slack
           // admits boundary files whose records fall outside the range.
-          if (aggregator.add(record) && record.sessionId.length > 0) {
+          if (aggregator.add(record, sources.length) && record.sessionId.length > 0) {
             sessionIds.add(record.sessionId);
           }
         }
       }
 
       sources.push({
         fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },
-        status: "ok",
+        status,
         scannedFiles,
         skippedFiles,
         malformedRecords: 0,
         distinctSessions: sessionIds.size,
-        message: null,
+        message:
+          status === "partial"
+            ? `Usage is incomplete: ${failedEntries} transcript files or directory entries could not be read.`
+            : null,
       });
     }
 

apps/server/src/usage/usageAggregation.test.ts

@@ -87,6 +87,32 @@ describe("UsageAggregator", () => {
     expect(result.buckets[0]?.totals.outputTokens).toBe(50);
   });
 
+  it("attributes buckets and deduplicates within each physical source", () => {
+    const aggregator = new UsageAggregator({
+      timeZone: "UTC",
+      sinceDay: "2026-08-01",
+      untilDay: "2026-08-31",
+      rates,
+    });
+    const item = record({ dedupeKey: "msg_1:" });
+    expect(aggregator.add(item, 0)).toBe(true);
+    expect(aggregator.add(item, 0)).toBe(false);
+    expect(aggregator.add(item, 1)).toBe(true);
+    expect(aggregator.add(item, 1)).toBe(false);
+    const result = aggregator.finish();
+    expect(result.duplicatesDropped).toBe(2);
+    expect(
+      result.buckets.map((bucket) => [
+        bucket.sourceIndex,
+        bucket.records,
+        bucket.totals.outputTokens,
+      ]),
+    ).toEqual([
+      [0, 1, 50],
+      [1, 1, 50],
+    ]);
+  });
+
   it("still sums records that carry no dedupe key", () => {
     const result = aggregate([record(), record()]);