fix(cli): replace hardcoded state lists with dynamic scan

.changeset/dynamic-state-discovery.md

@@ -0,0 +1,5 @@
+---
+'@bradygaster/squad-cli': patch
+---
+
+Replace hardcoded state artifact lists with dynamic directory scanning in externalize/internalize, preventing silent orphaning when new `.squad/` artifacts are added.

packages/squad-cli/src/cli/commands/externalize.ts

@@ -20,17 +20,24 @@ import { fatal } from '../core/errors.js';
 
 const storage = new FSStorageProvider();
 
-/** Directories under .squad/ that hold the actual state */
-const STATE_DIRS = [
-  'agents', 'casting', 'decisions', 'identity', 'orchestration-log',
-  'log', 'plugins', 'templates', 'skills', 'sessions', '.scratch',
-];
-
-/** Files under .squad/ that hold state (not config.json — that stays) */
-const STATE_FILES = [
-  'team.md', 'routing.md', 'ceremonies.md', 'decisions.md',
-  'decisions-archive.md', '.first-run',
-];
+/** Entries under .squad/ that must NOT be externalized (they stay in the repo).
+ *  All of these are read from the working tree by runtime code that does not
+ *  go through external-state resolution (detectSquadDir → local .squad/).
+ *  - config.json: thin marker read by the walk-up resolver
+ *  - manifest.json: public contract read by cross-squad discovery
+ *  - workstreams.json: workstream config read by streams resolver
+ *  - upstream.json: upstream registry read by cross-squad discover/delegate
+ *  - squad-registry.json: squad registry read by cross-squad discovery
+ *  - _upstream_repos: git clone cache read locally by upstream/cross-squad resolvers
+ */
+const KEEP_LOCAL = new Set([
+  'config.json',
+  'manifest.json',
+  'workstreams.json',
+  'upstream.json',
+  'squad-registry.json',
+  '_upstream_repos',
+]);
 
 /**
  * Move .squad/ state to the external directory.
@@ -52,29 +59,24 @@ export function runExternalize(projectDir: string, projectKey?: string): void {
 
   let movedCount = 0;
 
-  // Move directories
-  for (const dir of STATE_DIRS) {
-    const src = path.join(squadDir, dir);
-    if (!storage.existsSync(src)) continue;
-    const dest = path.join(externalDir, dir);
-    // Copy contents into the external state directory. If destination files
-    // already exist, they may be overwritten by copyDirRecursive().
-    copyDirRecursive(src, dest);
-    storage.deleteDirSync?.(src);
-    movedCount++;
-  }
-
-  // Move files
-  for (const file of STATE_FILES) {
-    const src = path.join(squadDir, file);
-    if (!storage.existsSync(src)) continue;
-    const dest = path.join(externalDir, file);
-    const content = storage.readSync(src);
-    if (content != null) {
-      storage.mkdirSync(path.dirname(dest), { recursive: true });
-      storage.writeSync(dest, content);
+  // Dynamically discover everything in .squad/ — move it all except KEEP_LOCAL.
+  // This eliminates silent orphaning when new state artifacts are added.
+  const entries = storage.listSync?.(squadDir) ?? [];
+  for (const entry of entries) {
+    if (KEEP_LOCAL.has(entry)) continue;
+    const src = path.join(squadDir, entry);
+    const dest = path.join(externalDir, entry);
+    if (storage.isDirectorySync(src)) {
+      copyDirRecursive(src, dest);
+      storage.deleteDirSync?.(src);
+    } else {
+      const content = storage.readSync(src);
+      if (content != null) {
+        storage.mkdirSync(path.dirname(dest), { recursive: true });
+        storage.writeSync(dest, content);
+      }
+      storage.deleteSync?.(src);
     }
-    storage.deleteSync?.(src);
     movedCount++;
   }
 
@@ -147,22 +149,19 @@ export function runInternalize(projectDir: string): void {
 
   let movedCount = 0;
 
-  // Copy directories back
-  for (const dir of STATE_DIRS) {
-    const src = path.join(externalDir, dir);
-    if (!storage.existsSync(src)) continue;
-    const dest = path.join(squadDir, dir);
-    copyDirRecursive(src, dest);
-    movedCount++;
-  }
-
-  // Copy files back
-  for (const file of STATE_FILES) {
-    const src = path.join(externalDir, file);
-    if (!storage.existsSync(src)) continue;
-    const content = storage.readSync(src);
-    if (content != null) {
-      storage.writeSync(path.join(squadDir, file), content);
+  // Dynamically discover everything in the external dir and copy it back.
+  const entries = storage.listSync?.(externalDir) ?? [];
+  for (const entry of entries) {
+    if (KEEP_LOCAL.has(entry)) continue;
+    const src = path.join(externalDir, entry);
+    const dest = path.join(squadDir, entry);
+    if (storage.isDirectorySync(src)) {
+      copyDirRecursive(src, dest);
+    } else {
+      const content = storage.readSync(src);
+      if (content != null) {
+        storage.writeSync(dest, content);
+      }
     }
     movedCount++;
   }

test/external-state.test.ts

@@ -1,8 +1,9 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import { resolveExternalStateDir, deriveProjectKey } from '@bradygaster/squad-sdk';
 import { mkdirSync, rmSync, existsSync, writeFileSync, readFileSync, readdirSync } from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
+import { runExternalize, runInternalize } from '../packages/squad-cli/src/cli/commands/externalize.js';
 
 const TEST_ROOT = path.join(os.tmpdir(), `squad-external-test-${Date.now()}`);
 
@@ -121,3 +122,171 @@ describe('resolveExternalStateDir security', () => {
     expect(dir).not.toContain('/project\\');
   });
 });
+
+// ============================================================================
+// runExternalize / runInternalize — CLI function tests
+// ============================================================================
+
+describe('runExternalize / runInternalize', () => {
+  let projectDir: string;
+  let squadDir: string;
+  let consoleSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    projectDir = path.join(TEST_ROOT, 'fake-project');
+    squadDir = path.join(projectDir, '.squad');
+    mkdirSync(squadDir, { recursive: true });
+    consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  /** Helper: list entries in a directory (returns [] if missing). */
+  function lsDir(dir: string): string[] {
+    if (!existsSync(dir)) return [];
+    return readdirSync(dir);
+  }
+
+  // T1: runExternalize basic — files move, KEEP_LOCAL entries stay
+  it('moves state to external dir, leaving only KEEP_LOCAL entries', () => {
+    // Create typical .squad/ state
+    writeFileSync(path.join(squadDir, 'team.md'), '# Team\n');
+    writeFileSync(path.join(squadDir, 'manifest.json'), '{"name":"test"}');
+    writeFileSync(path.join(squadDir, 'workstreams.json'), '{"workstreams":[]}');
+    writeFileSync(path.join(squadDir, 'upstream.json'), '{"upstreams":[]}');
+    writeFileSync(path.join(squadDir, 'squad-registry.json'), '[]');
+    const agentsDir = path.join(squadDir, 'agents');
+    mkdirSync(agentsDir, { recursive: true });
+    writeFileSync(path.join(agentsDir, 'bot.md'), '# Bot\n');
+    // _upstream_repos is a git clone cache that must stay local
+    const upstreamRepos = path.join(squadDir, '_upstream_repos', 'org');
+    mkdirSync(upstreamRepos, { recursive: true });
+    writeFileSync(path.join(upstreamRepos, 'README.md'), '# Upstream\n');
+
+    runExternalize(projectDir);
+
+    // .squad/ should only retain KEEP_LOCAL entries + config.json (written by externalize)
+    const remaining = lsDir(squadDir).sort();
+    expect(remaining).toContain('config.json');
+    expect(remaining).toContain('manifest.json');
+    expect(remaining).toContain('workstreams.json');
+    expect(remaining).toContain('upstream.json');
+    expect(remaining).toContain('squad-registry.json');
+    expect(remaining).toContain('_upstream_repos');
+    expect(remaining).not.toContain('team.md');
+    expect(remaining).not.toContain('agents');
+
+    // External dir should have the moved files
+    const key = deriveProjectKey(projectDir);
+    const externalDir = resolveExternalStateDir(key, false);
+    expect(existsSync(path.join(externalDir, 'team.md'))).toBe(true);
+    expect(existsSync(path.join(externalDir, 'agents', 'bot.md'))).toBe(true);
+    // KEEP_LOCAL entries must NOT be externalized
+    expect(existsSync(path.join(externalDir, 'manifest.json'))).toBe(false);
+    expect(existsSync(path.join(externalDir, 'workstreams.json'))).toBe(false);
+    expect(existsSync(path.join(externalDir, 'upstream.json'))).toBe(false);
+    expect(existsSync(path.join(externalDir, 'squad-registry.json'))).toBe(false);
+    expect(existsSync(path.join(externalDir, '_upstream_repos'))).toBe(false);
+  });
+
+  // T2: runExternalize with unknown entries — proves dynamic scan
+  it('moves unknown entries not in old hardcoded lists', () => {
+    writeFileSync(path.join(squadDir, 'custom-state.json'), '{"x":1}');
+    const pluginsDir = path.join(squadDir, 'plugins');
+    mkdirSync(pluginsDir, { recursive: true });
+    writeFileSync(path.join(pluginsDir, 'my-plugin.txt'), 'plugin data');
+
+    runExternalize(projectDir);
+
+    const remaining = lsDir(squadDir);
+    expect(remaining).not.toContain('custom-state.json');
+    expect(remaining).not.toContain('plugins');
+
+    const key = deriveProjectKey(projectDir);
+    const externalDir = resolveExternalStateDir(key, false);
+    expect(readFileSync(path.join(externalDir, 'custom-state.json'), 'utf-8')).toBe('{"x":1}');
+    expect(readFileSync(path.join(externalDir, 'plugins', 'my-plugin.txt'), 'utf-8')).toBe('plugin data');
+  });
+
+  // T3: runExternalize preserves existing config fields
+  it('preserves extra config.json fields through externalization', () => {
+    writeFileSync(path.join(squadDir, 'team.md'), '# Team\n');
+    writeFileSync(
+      path.join(squadDir, 'config.json'),
+      JSON.stringify({ version: 1, consult: true, stateBackend: 'fs' }),
+    );
+
+    runExternalize(projectDir);
+
+    const config = JSON.parse(readFileSync(path.join(squadDir, 'config.json'), 'utf-8'));
+    expect(config.consult).toBe(true);
+    expect(config.stateBackend).toBe('fs');
+    expect(config.stateLocation).toBe('external');
+    expect(config.projectKey).toBe(deriveProjectKey(projectDir));
+  });
+
+  // T4: runInternalize round-trip — externalize then internalize restores files
+  it('round-trips files through externalize → internalize', () => {
+    writeFileSync(path.join(squadDir, 'team.md'), '# My Team\n');
+    writeFileSync(path.join(squadDir, 'custom-data.json'), '{"round":"trip"}');
+    const logDir = path.join(squadDir, 'log');
+    mkdirSync(logDir, { recursive: true });
+    writeFileSync(path.join(logDir, 'session.md'), '## Session 1\n');
+
+    runExternalize(projectDir);
+
+    // State is gone from .squad/
+    expect(existsSync(path.join(squadDir, 'team.md'))).toBe(false);
+
+    runInternalize(projectDir);
+
+    // State is back
+    expect(readFileSync(path.join(squadDir, 'team.md'), 'utf-8')).toBe('# My Team\n');
+    expect(readFileSync(path.join(squadDir, 'custom-data.json'), 'utf-8')).toBe('{"round":"trip"}');
+    expect(readFileSync(path.join(squadDir, 'log', 'session.md'), 'utf-8')).toBe('## Session 1\n');
+  });
+
+  // T5: runInternalize cleans config — external-state fields removed
+  it('removes stateLocation/projectKey/teamRoot from config after internalize', () => {
+    // Seed config with an extra field so config.json survives (not deleted)
+    writeFileSync(
+      path.join(squadDir, 'config.json'),
+      JSON.stringify({ version: 1, consult: true }),
+    );
+    writeFileSync(path.join(squadDir, 'team.md'), '# Team\n');
+
+    runExternalize(projectDir);
+
+    // Verify external fields are present
+    const extConfig = JSON.parse(readFileSync(path.join(squadDir, 'config.json'), 'utf-8'));
+    expect(extConfig.stateLocation).toBe('external');
+    expect(extConfig.projectKey).toBeDefined();
+    expect(extConfig.teamRoot).toBe('.');
+
+    runInternalize(projectDir);
+
+    // External fields must be gone; user fields preserved
+    const intConfig = JSON.parse(readFileSync(path.join(squadDir, 'config.json'), 'utf-8'));
+    expect(intConfig.stateLocation).toBeUndefined();
+    expect(intConfig.projectKey).toBeUndefined();
+    expect(intConfig.teamRoot).toBeUndefined();
+    expect(intConfig.consult).toBe(true);
+  });
+
+  // T6: runExternalize on empty .squad/ — succeeds with nothing to move
+  it('succeeds when .squad/ contains only config.json', () => {
+    writeFileSync(
+      path.join(squadDir, 'config.json'),
+      JSON.stringify({ version: 1 }),
+    );
+
+    // Should not throw
+    expect(() => runExternalize(projectDir)).not.toThrow();
+
+    // config.json is still there with external marker
+    const config = JSON.parse(readFileSync(path.join(squadDir, 'config.json'), 'utf-8'));
+    expect(config.stateLocation).toBe('external');
+  });
+});