diff --git a/packages/core/src/services/memoryDiagnosticsDumper.test.ts b/packages/core/src/services/memoryDiagnosticsDumper.test.ts new file mode 100644 index 00000000000..a5755ba227e --- /dev/null +++ b/packages/core/src/services/memoryDiagnosticsDumper.test.ts @@ -0,0 +1,250 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js'; +import type { Config } from '../config/config.js'; + +vi.mock('node:fs', () => ({ + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +vi.mock('node:v8', () => ({ + getHeapStatistics: vi.fn().mockReturnValue({ + heap_size_limit: 4_096_000_000, + total_heap_size: 2_048_000_000, + used_heap_size: 1_800_000_000, + total_available_size: 2_000_000_000, + }), +})); + +vi.mock('../utils/memoryDiagnostics.js', () => ({ + collectMemoryDiagnostics: vi.fn().mockResolvedValue({ + timestamp: '2026-05-31T00:00:00.000Z', + memoryUsage: { + rss: 2_000_000_000, + heapUsed: 1_800_000_000, + heapTotal: 2_048_000_000, + external: 50_000_000, + arrayBuffers: 10_000_000, + }, + v8HeapStats: { + heapSizeLimit: 4_096_000_000, + totalHeapSize: 2_048_000_000, + usedHeapSize: 1_800_000_000, + }, + }), +})); + +function createMockConfig(overrides: Partial> = {}) { + return { + getSessionId: vi.fn().mockReturnValue('test-session-id-12345678'), + getCliVersion: vi.fn().mockReturnValue('0.17.0'), + getGeminiClient: vi.fn().mockReturnValue({ + getChat: () => ({ + getHistoryLength: () => 500, + }), + }), + storage: { + getProjectDir: vi.fn().mockReturnValue('/tmp/test-project'), + }, + ...overrides, + } as unknown as Config; +} + +describe('MemoryDiagnosticsDumper', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('writes diagnostics JSON on first dump', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + const result = await dumper.dump('hard'); + + expect(result).toBeDefined(); + expect(result!.trigger).toBe('hard'); + expect(result!.filePath).toContain( + path.join('/tmp/test-project', 'diagnostics') + path.sep, + ); + expect(result!.filePath).toContain('memory-test-ses'); + expect(fs.mkdirSync).toHaveBeenCalledWith( + expect.stringContaining('diagnostics'), + { recursive: true }, + ); + // Two-phase write: Phase 1 (minimal) + Phase 2 (full) + expect(fs.writeFileSync).toHaveBeenCalledTimes(2); + + const phase1Content = JSON.parse( + vi.mocked(fs.writeFileSync).mock.calls[0][1] as string, + ); + expect(phase1Content.trigger).toBe('hard'); + expect(phase1Content.dumpNumber).toBe(1); + expect(phase1Content.collectionComplete).toBe(false); + expect(phase1Content.memoryUsage).toBeDefined(); + expect(phase1Content.v8HeapStats).toBeDefined(); + + const phase2Content = JSON.parse( + vi.mocked(fs.writeFileSync).mock.calls[1][1] as string, + ); + expect(phase2Content.trigger).toBe('hard'); + expect(phase2Content.dumpNumber).toBe(1); + expect(phase2Content.collectionComplete).toBe(true); + expect(phase2Content.memoryUsage.rss).toBe(2_000_000_000); + expect(phase2Content.session.historyEntries).toBe(500); + expect(phase2Content.suggestion).toContain('/compress'); + }); + + it('respects per-session cap of 3 dumps', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + // Bypass cooldown by mocking Date.now + let mockNow = 1000000; + vi.spyOn(Date, 'now').mockImplementation(() => { + mockNow += 60_000; + return mockNow; + }); + + const r1 = await dumper.dump('hard'); + const r2 = await dumper.dump('critical'); + const r3 = await dumper.dump('hard'); + const r4 = await dumper.dump('critical'); + + expect(r1).toBeDefined(); + expect(r2).toBeDefined(); + expect(r3).toBeDefined(); + expect(r4).toBeUndefined(); + // 3 successful dumps × 2 writes each (Phase 1 + Phase 2) + expect(fs.writeFileSync).toHaveBeenCalledTimes(6); + }); + + it('respects cooldown between dumps', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + const mockNow = 1000000; + vi.spyOn(Date, 'now').mockReturnValue(mockNow); + + const r1 = await dumper.dump('hard'); + const r2 = await dumper.dump('hard'); + + expect(r1).toBeDefined(); + expect(r2).toBeUndefined(); + // 1 successful dump × 2 writes (Phase 1 + Phase 2) + expect(fs.writeFileSync).toHaveBeenCalledTimes(2); + }); + + it('resets state on new session', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + let mockNow = 1000000; + vi.spyOn(Date, 'now').mockImplementation(() => { + mockNow += 60_000; + return mockNow; + }); + + await dumper.dump('hard'); + await dumper.dump('hard'); + await dumper.dump('hard'); + + // Cap reached + const r4 = await dumper.dump('hard'); + expect(r4).toBeUndefined(); + + // Reset + dumper.resetForNewSession(); + + const r5 = await dumper.dump('critical'); + expect(r5).toBeDefined(); + expect(r5!.trigger).toBe('critical'); + }); + + it('includes critical suggestion for critical pressure', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + await dumper.dump('critical'); + + // Phase 2 (full payload) is the second write + const writtenContent = JSON.parse( + vi.mocked(fs.writeFileSync).mock.calls[1][1] as string, + ); + expect(writtenContent.suggestion).toContain('critically high'); + expect(writtenContent.collectionComplete).toBe(true); + }); + + it('writes Phase 1 synchronously before any await (survives crash during Phase 2)', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + // Fire dump() but do not await — Phase 1 must have already written to disk + // because async functions execute synchronously up to the first await. + const promise = dumper.dump('hard'); + + // At this point Phase 2 has not run yet (its await is pending), but Phase 1 + // must have completed its writeFileSync call. + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + const phase1Content = JSON.parse( + vi.mocked(fs.writeFileSync).mock.calls[0][1] as string, + ); + expect(phase1Content.collectionComplete).toBe(false); + + await promise; + // Phase 2 has now overwritten the file + expect(fs.writeFileSync).toHaveBeenCalledTimes(2); + }); + + it('reserves slot synchronously to prevent concurrent dumps from bypassing cap', async () => { + const config = createMockConfig(); + const dumper = new MemoryDiagnosticsDumper(config); + + let mockNow = 1000000; + vi.spyOn(Date, 'now').mockImplementation(() => { + mockNow += 60_000; + return mockNow; + }); + + // Fire 4 concurrent dumps without awaiting between them. The synchronous + // slot reservation must enforce the cap of 3 even though all 4 calls happen + // before any of them complete their async Phase 2. + const results = await Promise.all([ + dumper.dump('hard'), + dumper.dump('hard'), + dumper.dump('hard'), + dumper.dump('hard'), + ]); + + const successful = results.filter((r) => r !== undefined); + expect(successful).toHaveLength(3); + expect(results[3]).toBeUndefined(); + }); + + it('handles missing geminiClient gracefully', async () => { + const config = createMockConfig({ + getGeminiClient: vi.fn().mockReturnValue(null), + }); + const dumper = new MemoryDiagnosticsDumper(config); + + const result = await dumper.dump('hard'); + + expect(result).toBeDefined(); + // Phase 2 (full payload) is the second write + const writtenContent = JSON.parse( + vi.mocked(fs.writeFileSync).mock.calls[1][1] as string, + ); + expect(writtenContent.session.available).toBe(false); + }); +}); diff --git a/packages/core/src/services/memoryDiagnosticsDumper.ts b/packages/core/src/services/memoryDiagnosticsDumper.ts new file mode 100644 index 00000000000..81fb51f54f3 --- /dev/null +++ b/packages/core/src/services/memoryDiagnosticsDumper.ts @@ -0,0 +1,175 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Writes a lightweight memory diagnostics JSON to disk when the memory + * pressure monitor detects hard or critical pressure. The file survives + * a subsequent OOM crash, giving maintainers actionable data from bug + * reports without requiring the user to manually run `/doctor memory`. + * + * Design: diagnostics JSON is written BEFORE any expensive operation + * (like heap snapshots) so it lands on disk even if the process crashes + * during the heavier step. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as v8 from 'node:v8'; +import { collectMemoryDiagnostics } from '../utils/memoryDiagnostics.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; +import { getErrorMessage } from '../utils/errors.js'; +import type { Config } from '../config/config.js'; + +const debugLogger = createDebugLogger('MEMORY_DUMP'); + +/** Maximum dumps per session to avoid disk flooding. */ +const MAX_DUMPS_PER_SESSION = 3; + +/** Minimum interval between dumps (ms). */ +const MIN_DUMP_INTERVAL_MS = 30_000; + +export interface MemoryDumpResult { + filePath: string; + trigger: string; +} + +export class MemoryDiagnosticsDumper { + private dumpCount = 0; + private lastDumpTime = 0; + + constructor(private readonly config: Config) {} + + /** + * Resets session-scoped state. Called when a new session starts. + */ + resetForNewSession(): void { + this.dumpCount = 0; + this.lastDumpTime = 0; + } + + /** + * Writes a diagnostics snapshot to disk if within per-session limits. + * + * Uses a two-phase write strategy: + * - Phase 1 (synchronous): writes a minimal JSON with process.memoryUsage() + * and v8.getHeapStatistics() — no fork/exec, so it lands on disk even + * under extreme memory pressure. + * - Phase 2 (async): collects full diagnostics (may spawn subprocesses) + * and overwrites the file with the complete payload. If Phase 2 crashes, + * Phase 1's file still survives for debugging. + * + * Slot is reserved synchronously before any await to prevent concurrent + * invocations from bypassing the cap/cooldown guards. + */ + async dump( + trigger: 'hard' | 'critical', + ): Promise { + if (this.dumpCount >= MAX_DUMPS_PER_SESSION) { + debugLogger.debug( + `Skipping dump: session cap reached (${MAX_DUMPS_PER_SESSION})`, + ); + return undefined; + } + + const now = Date.now(); + if (now - this.lastDumpTime < MIN_DUMP_INTERVAL_MS) { + debugLogger.debug('Skipping dump: cooldown not elapsed'); + return undefined; + } + + // Reserve slot synchronously to prevent race between concurrent dumps + const dumpNumber = ++this.dumpCount; + this.lastDumpTime = now; + + try { + const diagnosticsDir = this.ensureDiagnosticsDir(); + const sessionId = this.config.getSessionId(); + const timestamp = new Date() + .toISOString() + .replace(/:/g, '-') + .replace(/\./g, '_'); + const fileName = `memory-${sessionId.slice(0, 8)}-${timestamp}.json`; + const filePath = path.join(diagnosticsDir, fileName); + + // Phase 1: synchronous minimal write — survives crash during Phase 2 + const minimalPayload = { + trigger, + dumpNumber, + timestamp: new Date().toISOString(), + memoryUsage: process.memoryUsage(), + v8HeapStats: v8.getHeapStatistics(), + session: this.collectSessionStats(), + suggestion: this.getSuggestion(trigger), + collectionComplete: false, + }; + fs.writeFileSync( + filePath, + JSON.stringify(minimalPayload, null, 2), + 'utf8', + ); + + debugLogger.info( + `Phase 1 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`, + ); + + // Phase 2: full collection (may fork subprocesses — risky under pressure) + const diagnostics = await collectMemoryDiagnostics({ + sessionId, + qwenVersion: this.config.getCliVersion(), + }); + + const fullPayload = { + trigger, + dumpNumber, + ...diagnostics, + session: this.collectSessionStats(), + suggestion: this.getSuggestion(trigger), + collectionComplete: true, + }; + fs.writeFileSync(filePath, JSON.stringify(fullPayload, null, 2), 'utf8'); + + debugLogger.info( + `Phase 2 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`, + ); + + return { filePath, trigger }; + } catch (err) { + // Slot stays consumed — a failed write should not open the door to more + // attempts that would likely also fail under the same pressure conditions. + debugLogger.error( + `Failed to write memory diagnostics: ${getErrorMessage(err)}`, + ); + return undefined; + } + } + + private ensureDiagnosticsDir(): string { + const projectDir = this.config.storage.getProjectDir(); + const diagnosticsDir = path.join(projectDir, 'diagnostics'); + fs.mkdirSync(diagnosticsDir, { recursive: true }); + return diagnosticsDir; + } + + private collectSessionStats(): Record { + try { + const geminiClient = this.config.getGeminiClient?.(); + if (!geminiClient) return { available: false }; + const historyLength = geminiClient.getChat?.()?.getHistoryLength?.() ?? 0; + return { + historyEntries: historyLength, + }; + } catch { + return { available: false }; + } + } + + private getSuggestion(trigger: 'hard' | 'critical'): string { + if (trigger === 'critical') { + return 'Memory is critically high. Consider running /compress or starting a fresh session to avoid OOM.'; + } + return 'Memory pressure detected. Running /compress may help reduce memory usage.'; + } +} diff --git a/packages/core/src/services/memoryPressureMonitor.ts b/packages/core/src/services/memoryPressureMonitor.ts index 956755a0308..babecda4d4f 100644 --- a/packages/core/src/services/memoryPressureMonitor.ts +++ b/packages/core/src/services/memoryPressureMonitor.ts @@ -11,6 +11,7 @@ import { getHeapStatistics } from 'node:v8'; import { createDebugLogger } from '../utils/debugLogger.js'; import { getErrorMessage } from '../utils/errors.js'; import type { Config } from '../config/config.js'; +import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js'; // Types @@ -104,6 +105,7 @@ export class MemoryPressureMonitor extends EventEmitter { private consecutiveIneffectiveAggressiveCleanups = 0; private cleanupGeneration = 0; private readonly effectiveMemoryLimit: number; + private readonly diagnosticsDumper: MemoryDiagnosticsDumper; constructor(coreConfig: Config, pressureConfig?: MemoryPressureConfig) { super(); @@ -111,6 +113,7 @@ export class MemoryPressureMonitor extends EventEmitter { this.config = { ...(pressureConfig ?? DEFAULT_PRESSURE_CONFIG) }; validateMemoryPressureConfig(this.config); this.effectiveMemoryLimit = this.computeEffectiveMemoryLimit(); + this.diagnosticsDumper = new MemoryDiagnosticsDumper(coreConfig); const heapSizeLimit = getHeapStatistics().heap_size_limit; debugLogger.info( `Effective memory limit: ${formatMiB(this.effectiveMemoryLimit)} MiB; ` + @@ -142,6 +145,7 @@ export class MemoryPressureMonitor extends EventEmitter { resetForNewSession(): void { this.cleanupGeneration++; this.resetConsecutiveFailures(); + this.diagnosticsDumper.resetForNewSession(); this.cleanupInProgress = false; this.activeCleanupAction = 'none'; this.queuedCleanupRecommendation = undefined; @@ -196,6 +200,16 @@ export class MemoryPressureMonitor extends EventEmitter { return; } + // Write diagnostics to disk before cleanup. dump() uses a two-phase strategy: + // Phase 1 (synchronous, before the first await) writes a minimal JSON via + // writeFileSync — guaranteed to complete before executeCleanup starts because + // async functions run synchronously up to the first await. Phase 2 enriches + // the file asynchronously in the background; if it fails the minimal file + // still survives for debugging. + if (pressure === 'hard' || pressure === 'critical') { + void this.diagnosticsDumper.dump(pressure); + } + this.executeCleanup(recommendation); }