perf: parallelize local environment sync

DEVELOPMENT.md

@@ -244,6 +244,7 @@ should pass against the local PostgreSQL database.
 | `KILO_PORT_OFFSET=auto pnpm dev:start` | Start all local services in a tmux dashboard with worktree-safe ports |
 | `pnpm dev:stop` | Stop the tmux session and all services |
 | `pnpm dev:env` | Sync `.dev.vars` files from `.env.local` (see [Worker `.dev.vars` setup](#worker-dev-vars-setup)) |
+| `pnpm dev:env --missing-secrets-only` | Sync env files and create missing local Secrets Store entries without refreshing existing secrets |
 | `pnpm test` | Run web, web environment, and local development-tool tests |
 | `pnpm typecheck` | Run the TypeScript type checker |
 | `pnpm lint` | Lint all source files |
@@ -405,7 +406,9 @@ Most workers require a `.dev.vars` file with secrets like `NEXTAUTH_SECRET` and
 pnpm dev:env
 ```
 
-The script (`dev/local/env-sync/`) scans every `.dev.vars.example` in the repo and `apps/web/.env.development.local.example`, resolves each variable's value, and writes (or patches) the corresponding generated local env file. Before applying, it shows a diff of what will change and asks for confirmation.
+The script (`dev/local/env-sync/`) scans every `.dev.vars.example` in the repo and `apps/web/.env.development.local.example`, resolves each variable's value, and writes (or patches) the corresponding generated local env file. Before applying, it shows a diff of what will change and asks for confirmation. Local Secrets Store checks and writes run across four independent Worker persistence directories at a time; set `KILO_ENV_SYNC_CONCURRENCY` to an integer from 1 through 32 to override that limit.
+
+For fast worktree setup, pass `--missing-secrets-only`. It creates bindings that are absent from the local Secrets Store while preserving existing source-backed values; a normal `pnpm dev:env` continues to refresh those values from their configured sources.
 
 Values are resolved using annotations in example env file comment lines:
 

dev/local/cli.ts

@@ -1116,6 +1116,7 @@ async function cmdStop(repoRoot: string, force: boolean): Promise<void> {
 
 async function cmdEnv(args: string[], repoRoot: string): Promise<void> {
   const check = args.includes('--check') || args.includes('check');
+  const missingSecretsOnly = args.includes('--missing-secrets-only');
 
   // Runs before the sync: worker and Next.js env values are derived from this
   // worktree's ports, and a fresh worktree has published none yet.
@@ -1129,6 +1130,7 @@ async function cmdEnv(args: string[], repoRoot: string): Promise<void> {
   const result = await syncEnvVars({
     repoRoot,
     check,
+    missingSecretsOnly,
     yes,
     targets: targets.length > 0 ? targets : undefined,
   });
@@ -1160,6 +1162,8 @@ Usage:
   dev:env [targets...]    Sync env vars (.dev.vars + .env.development.local)
   dev:env --check         Validate env vars (CI mode)
   dev:env -y              Sync without confirmation
+  dev:env --missing-secrets-only
+                          Create missing Secrets Store entries without refreshing existing ones
 
 Targets: app, app-builder, agents, code-review, security-agent, mobile, all, or any service/group name
 Multiple targets can be specified: dev:start kiloclaw security-agent`);

dev/local/env-sync/concurrency.ts

@@ -0,0 +1,39 @@
+const DEFAULT_ENV_SYNC_CONCURRENCY = 4;
+const MAX_ENV_SYNC_CONCURRENCY = 32;
+
+function resolveEnvSyncConcurrency(value = process.env.KILO_ENV_SYNC_CONCURRENCY): number {
+  if (value === undefined || value === '') return DEFAULT_ENV_SYNC_CONCURRENCY;
+
+  const concurrency = Number(value);
+  if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_ENV_SYNC_CONCURRENCY) {
+    throw new Error(
+      `KILO_ENV_SYNC_CONCURRENCY must be an integer between 1 and ${MAX_ENV_SYNC_CONCURRENCY}`
+    );
+  }
+  return concurrency;
+}
+
+async function mapWithConcurrency<T, R>(
+  values: readonly T[],
+  concurrency: number,
+  map: (value: T, index: number) => Promise<R>
+): Promise<R[]> {
+  if (values.length === 0) return [];
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error('Concurrency must be a positive integer');
+  }
+
+  const results = new Array<R>(values.length);
+  let nextIndex = 0;
+  const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => {
+    while (true) {
+      const index = nextIndex++;
+      if (index >= values.length) return;
+      results[index] = await map(values[index] as T, index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
+export { mapWithConcurrency, resolveEnvSyncConcurrency };

dev/local/env-sync/index.ts

@@ -2,7 +2,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import * as readline from 'node:readline';
 import { resolveTargets } from '../services';
-import { computePlan, findDevVarsExamples } from './plan';
+import { computePlanAsync, findDevVarsExamples } from './plan';
 import { planHasChanges, displayPlan, applyEnvLocalAutoCreates, applyPlan } from './output';
 import type { SyncResult, CheckResult } from './types';
 
@@ -39,12 +39,14 @@ function resolveServiceFilter(targets?: string[]): Set<string> | undefined {
 async function syncEnvVars(options: {
   repoRoot: string;
   check?: boolean;
+  missingSecretsOnly?: boolean;
   yes?: boolean;
   targets?: string[];
 }): Promise<SyncResult> {
-  const { repoRoot, check = false, yes = false, targets } = options;
+  const { repoRoot, check = false, missingSecretsOnly = false, yes = false, targets } = options;
   const serviceFilter = resolveServiceFilter(targets);
-  const plan = computePlan(repoRoot, serviceFilter, !check);
+  const refreshSourceBackedSecrets = !check && !missingSecretsOnly;
+  const plan = await computePlanAsync(repoRoot, serviceFilter, refreshSourceBackedSecrets);
 
   if (plan.missingEnvLocal) {
     displayPlan(plan);
@@ -74,8 +76,10 @@ async function syncEnvVars(options: {
     if (shouldApply) {
       applyEnvLocalAutoCreates(plan.envLocalAutoCreates, repoRoot);
       const applyReadyPlan =
-        plan.envLocalAutoCreates.length > 0 ? computePlan(repoRoot, serviceFilter, true) : plan;
-      applyPlan(applyReadyPlan, repoRoot);
+        plan.envLocalAutoCreates.length > 0
+          ? await computePlanAsync(repoRoot, serviceFilter, refreshSourceBackedSecrets)
+          : plan;
+      await applyPlan(applyReadyPlan, repoRoot, { missingSecretsOnly });
       console.log(`\n${GREEN}✓ Applied${RESET}`);
     } else {
       console.log('Skipped.');
@@ -99,7 +103,7 @@ async function checkEnvVars(repoRoot: string, targets?: string[]): Promise<Check
   }
 
   const serviceFilter = resolveServiceFilter(targets);
-  const plan = computePlan(repoRoot, serviceFilter, false);
+  const plan = await computePlanAsync(repoRoot, serviceFilter, false);
   const totalMissing =
     plan.devVarsChanges.reduce((sum, c) => sum + c.missingValues.length, 0) +
     plan.secretStoreWarnings.reduce((sum, c) => sum + c.bindings.length, 0);
@@ -144,6 +148,7 @@ function findRepoRoot(): string {
 async function main() {
   const args = process.argv.slice(2);
   const checkMode = args.includes('--check');
+  const missingSecretsOnly = args.includes('--missing-secrets-only');
   const yesMode = args.includes('--yes') || args.includes('-y');
   const targets = args.filter(a => !a.startsWith('-'));
 
@@ -165,6 +170,7 @@ async function main() {
 
   const result = await syncEnvVars({
     repoRoot,
+    missingSecretsOnly,
     yes: yesMode,
     targets: targets.length > 0 ? targets : undefined,
   });

dev/local/env-sync/output.test.ts

@@ -0,0 +1,73 @@
+import assert from 'node:assert/strict';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import test from 'node:test';
+import { applyPlan } from './output';
+import type { EnvSyncPlan, SecretStoreAutoCreate } from './types';
+
+function secretCreate(workerDir: string, secretName: string): SecretStoreAutoCreate {
+  return {
+    workerDir,
+    binding: {
+      binding: secretName,
+      store_id: 'store-id',
+      secret_name: secretName,
+    },
+    sourceKey: secretName,
+    value: `value-for-${secretName}`,
+  };
+}
+
+test('creates missing secrets concurrently across stores and rechecks existing secrets', async () => {
+  const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'env-sync-output-'));
+  const existing = secretCreate('services/event-service', 'EXISTING_SECRET');
+  const missing = secretCreate('services/model-eval-ingest', 'MISSING_SECRET');
+  for (const create of [existing, missing]) {
+    fs.mkdirSync(path.join(repoRoot, create.workerDir, '.wrangler'), { recursive: true });
+  }
+
+  const plan: EnvSyncPlan = {
+    lanIp: undefined,
+    devVarsChanges: [],
+    envDevLocalChanges: [],
+    envLocalAutoCreates: [],
+    secretStoreWarnings: [],
+    secretStoreAutoCreates: [existing, missing],
+    consistencyWarnings: [],
+    execWarnings: [],
+    missingEnvLocal: false,
+  };
+  let active = 0;
+  let maxActive = 0;
+  const created: string[] = [];
+  try {
+    await applyPlan(plan, repoRoot, {
+      concurrency: 2,
+      missingSecretsOnly: true,
+      runWrangler: async (_repoRoot, workerDir, args, input) => {
+        active++;
+        maxActive = Math.max(maxActive, active);
+        await new Promise(resolve => setTimeout(resolve, 20));
+        active--;
+
+        if (args.includes('list')) {
+          return {
+            status: 0,
+            stderr: '',
+            stdout: workerDir === existing.workerDir ? 'EXISTING_SECRET\n' : '',
+          };
+        }
+        assert.ok(input?.startsWith('value-for-'));
+        assert.ok(!args.some(arg => arg.startsWith('value-for-')));
+        created.push(args[args.indexOf('--name') + 1] ?? '');
+        return { status: 0, stderr: '', stdout: '' };
+      },
+    });
+
+    assert.equal(maxActive, 2);
+    assert.deepEqual(created, ['MISSING_SECRET']);
+  } finally {
+    fs.rmSync(repoRoot, { recursive: true, force: true });
+  }
+});