fix(indexing): replace chokidar file watcher with @parcel/watcher

.changeset/kilo-indexing-file-watcher.md

@@ -0,0 +1,5 @@
+---
+"@kilocode/kilo-indexing": patch
+---
+
+Fix code index hanging at "Initializing file watcher…": replace the chokidar watcher with @parcel/watcher so subscribing no longer blocks the bundled runtime, prune ignored and per-repo gitignored directories from the native watch so large repositories don't exhaust OS file-watch limits, and fall back to a full scan (incremental updates disabled) when the native watcher can't start instead of failing indexing.

packages/kilo-indexing/package.json

@@ -39,9 +39,9 @@
     "@kilocode/kilo-gateway": "workspace:*",
     "@kilocode/plugin": "workspace:*",
     "@lancedb/lancedb": "0.26.2",
+    "@parcel/watcher": "2.5.1",
     "@qdrant/js-client-rest": "1.17.0",
     "async-mutex": "0.5.0",
-    "chokidar": "4.0.3",
     "effect": "catalog:",
     "glob": "13.0.6",
     "hono": "catalog:",

packages/kilo-indexing/src/indexing/constants/index.ts

@@ -35,6 +35,8 @@ export const DEFAULT_MAX_SEARCH_RESULTS = CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH
 /**File Watcher */
 export const QDRANT_CODE_BLOCK_NAMESPACE = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
 export const MAX_FILE_SIZE_BYTES = 1 * 1024 * 1024 // 1MB
+// Defensive bound on establishing the native file-watcher subscription.
+export const PARCEL_SUBSCRIBE_TIMEOUT_MS = 10_000
 
 /**Directory Scanner */
 export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000

packages/kilo-indexing/src/indexing/processors/file-watcher.ts

@@ -1,15 +1,16 @@
-import { watch as chokidarWatch, type FSWatcher as ChokidarFSWatcher } from "chokidar"
+import type ParcelWatcher from "@parcel/watcher"
 import { stat, readFile } from "fs/promises"
 import { createHash } from "crypto"
 import path from "path"
 import { v5 as uuidv5 } from "uuid"
-import { Emitter, type Disposable } from "../runtime"
+import { Emitter } from "../runtime"
 import {
   QDRANT_CODE_BLOCK_NAMESPACE,
   MAX_FILE_SIZE_BYTES,
   BATCH_SEGMENT_THRESHOLD,
   MAX_BATCH_RETRIES,
   INITIAL_RETRY_DELAY_MS,
+  PARCEL_SUBSCRIBE_TIMEOUT_MS,
 } from "../constants"
 import { scannerExtensions } from "../shared/supported-extensions"
 import {
@@ -38,15 +39,119 @@ import { isBinary } from "../shared/is-binary"
 
 const log = Log.create({ service: "file-watcher" })
 
-/**
- * Implementation of the file watcher interface.
- *
- * RATIONALE: Uses chokidar instead of vscode.workspace.createFileSystemWatcher
- * so the watcher works outside VS Code (CLI, tests, headless).
- */
+declare const KILO_LIBC: string | undefined
+
+export interface FileWatchEvent {
+  path: string
+  type: "create" | "update" | "delete"
+}
+
+export interface FileWatchSubscription {
+  unsubscribe(): Promise<void>
+}
+
+// Injectable so tests can supply a fake watcher instead of a real subscription.
+export type FileWatchSubscribe = (
+  directory: string,
+  onEvents: (events: readonly FileWatchEvent[]) => void,
+  ignore: readonly string[],
+) => Promise<FileWatchSubscription>
+
+function watcherBackend(): ParcelWatcher.BackendType | undefined {
+  if (process.platform === "win32") return "windows"
+  if (process.platform === "darwin") return "fs-events"
+  if (process.platform === "linux") return "inotify"
+  return undefined
+}
+
+let parcelModule: typeof import("@parcel/watcher") | null | undefined
+function loadParcelWatcher(): typeof import("@parcel/watcher") | undefined {
+  if (parcelModule !== undefined) return parcelModule ?? undefined
+  try {
+    // Load the platform binding directly; this is what resolves inside the
+    // bundled bun-compiled runtime (mirrors @kilocode/core's workspace watcher).
+    const libc = typeof KILO_LIBC === "undefined" ? undefined : KILO_LIBC
+    const suffix = process.platform === "linux" ? `-${libc || "glibc"}` : ""
+    // @ts-ignore - the wrapper subpath ships without type declarations
+    const { createWrapper } = require("@parcel/watcher/wrapper")
+    const binding = require(`@parcel/watcher-${process.platform}-${process.arch}${suffix}`)
+    parcelModule = createWrapper(binding) as typeof import("@parcel/watcher")
+  } catch {
+    try {
+      parcelModule = require("@parcel/watcher") as typeof import("@parcel/watcher") // dev / CLI / test fallback
+    } catch {
+      parcelModule = null
+    }
+  }
+  return parcelModule ?? undefined
+}
+
+// Kilo's always-ignored infrastructure dirs as globs the native watcher prunes.
+// Per-repo .gitignore/.kilocodeignore dirs are unioned in initialize() from the
+// ignore matcher; correctness for indexed files stays in shouldIndex().
+function watcherIgnoreGlobs(): string[] {
+  return FileIgnore.PATTERNS.map((pattern) => (pattern.includes("/") ? pattern : `**/${pattern}`))
+}
+
+function withSubscribeTimeout(
+  pending: Promise<FileWatchSubscription>,
+  directory: string,
+): Promise<FileWatchSubscription> {
+  let timedOut = false
+  let timer: NodeJS.Timeout | undefined
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      timedOut = true
+      reject(new Error(`Timed out establishing the file watcher subscription for "${directory}".`))
+    }, PARCEL_SUBSCRIBE_TIMEOUT_MS)
+    timer.unref()
+  })
+  // Reconcile a subscription that only settles after we've already given up:
+  // tear a live one down, or note that the subscribe itself failed — without
+  // mislabeling one as the other.
+  void pending.then(
+    (subscription) => {
+      if (!timedOut) return
+      void subscription
+        .unsubscribe()
+        .catch((err) =>
+          log.warn("failed to tear down late file watcher subscription", { err, workspacePath: directory }),
+        )
+    },
+    (err) => {
+      if (timedOut) log.warn("file watcher subscribe failed after the timeout", { err, workspacePath: directory })
+    },
+  )
+  return Promise.race([pending, timeout]).finally(() => {
+    if (timer) clearTimeout(timer)
+  })
+}
+
+// @parcel/watcher subscribes without a blocking initial walk (reporting only
+// future changes), so it does not stall startup the way chokidar did under bun.
+const parcelSubscribe: FileWatchSubscribe = (directory, onEvents, ignore) => {
+  const parcel = loadParcelWatcher()
+  if (!parcel) {
+    return Promise.reject(new Error("native file watcher backend (@parcel/watcher) is unavailable"))
+  }
+  const pending = parcel.subscribe(
+    directory,
+    (error, events) => {
+      if (error) {
+        log.error("file watcher reported an error", { workspacePath: directory, err: error })
+        return
+      }
+      onEvents(events)
+    },
+    { ignore: [...ignore], backend: watcherBackend() },
+  )
+  return withSubscribeTimeout(pending, directory)
+}
+
+/** Watches the workspace via @parcel/watcher and feeds changes into the incremental indexer. */
 export class FileWatcher implements IFileWatcher {
   private ignoreInstance?: IgnoreMatcher
-  private watcher?: ChokidarFSWatcher
+  private subscription?: FileWatchSubscription
   private accumulatedEvents: Map<string, { path: string; type: "create" | "change" | "delete" }> = new Map()
   private batchProcessDebounceTimer?: NodeJS.Timeout
   private readonly BATCH_DEBOUNCE_DELAY_MS = 500
@@ -68,6 +173,15 @@ export class FileWatcher implements IFileWatcher {
   }>()
   public readonly onDidFinishBatchProcessing = new Emitter<BatchProcessingSummary>()
 
+  // @parcel/watcher reports "update" where the pipeline expects "change".
+  private readonly onWatchEvents = (events: readonly FileWatchEvent[]): void => {
+    for (const event of events) {
+      const type: "create" | "change" | "delete" =
+        event.type === "create" ? "create" : event.type === "delete" ? "delete" : "change"
+      this.handleFileEvent(event.path, type)
+    }
+  }
+
   constructor(
     private workspacePath: string,
     private readonly cacheManager: CacheManager,
@@ -80,6 +194,7 @@ export class FileWatcher implements IFileWatcher {
     private readonly telemetryMeta?: IndexingTelemetryMeta,
     extensions: readonly string[] = scannerExtensions,
     private readonly parser: ICodeParser = codeParser,
+    private readonly subscribeFn: FileWatchSubscribe = parcelSubscribe,
   ) {
     if (ignoreInstance) {
       this.ignoreInstance = ignoreInstance
@@ -124,10 +239,10 @@ export class FileWatcher implements IFileWatcher {
   }
 
   /**
-   * Initializes the file watcher using chokidar.
-   *
-   * RATIONALE: chokidar watches the filesystem directly using native OS events,
-   * removing the dependency on VS Code's file system watcher API.
+   * Subscribes to the workspace for incremental updates. The watcher is only
+   * needed for incremental updates, so a subscription failure (missing native
+   * backend, inotify limits, ...) degrades to a warning and lets the full scan
+   * proceed rather than failing indexing.
    */
   async initialize(): Promise<void> {
     if (this.ready) {
@@ -137,26 +252,40 @@ export class FileWatcher implements IFileWatcher {
 
     log.info("initializing file watcher", { workspacePath: this.workspacePath })
 
-    this.watcher = chokidarWatch(this.workspacePath, {
-      ignored: (filePath: string) => {
-        const relativeFilePath = generateRelativeIgnorePath(filePath, this.workspacePath)
-        if (!relativeFilePath) return false
-        if (FileIgnore.match(relativeFilePath)) return true
-        return this.ignoreInstance?.ignores(relativeFilePath) ?? false
+    // Prune Kilo's infra dirs plus (best-effort) the per-repo gitignored dirs
+    // from the native watch, so a large repo doesn't exhaust inotify descriptors
+    // (ENOSPC) watching trees we never index.
+    const ignore = [...new Set([...watcherIgnoreGlobs(), ...(this.ignoreInstance?.watchIgnoreGlobs?.() ?? [])])]
+
+    const ready = this.subscribeFn(this.workspacePath, this.onWatchEvents, ignore).then(
+      (subscription) => {
+        // If shutdown()/dispose() cleared this.ready, or a newer initialize()
+        // superseded us while subscribing, don't store this subscription — it
+        // would leak and keep feeding accumulatedEvents. Tear it down instead.
+        if (this.ready !== ready) {
+          void subscription
+            .unsubscribe()
+            .catch((err) =>
+              log.error("failed to unsubscribe superseded file watcher", { err, workspacePath: this.workspacePath }),
+            )
+          return
+        }
+        this.subscription = subscription
+        log.info("file watcher subscribed", { workspacePath: this.workspacePath })
       },
-      persistent: true,
-      ignoreInitial: true,
-    })
-
-    this.watcher.on("add", (filePath) => this.handleFileEvent(filePath, "create"))
-    this.watcher.on("change", (filePath) => this.handleFileEvent(filePath, "change"))
-    this.watcher.on("unlink", (filePath) => this.handleFileEvent(filePath, "delete"))
-    this.ready = new Promise((resolve, reject) => {
-      this.watcher?.once("ready", resolve)
-      this.watcher?.once("error", reject)
-    })
+      (error) => {
+        // The watcher only powers incremental updates, so degrade to a warning
+        // and let the full scan proceed. Clear this.ready (when still current)
+        // so a later run retries a transient failure such as inotify ENOSPC.
+        if (this.ready === ready) this.ready = undefined
+        log.warn("file watcher unavailable; continuing without incremental updates", {
+          workspacePath: this.workspacePath,
+          err: error,
+        })
+      },
+    )
+    this.ready = ready
     await this.ready
-    log.info("file watcher ready", { workspacePath: this.workspacePath })
   }
 
   setOverlay(overlay?: WorktreeOverlay): void {
@@ -191,14 +320,24 @@ export class FileWatcher implements IFileWatcher {
     this.collecting = false
     if (this.batchProcessDebounceTimer) clearTimeout(this.batchProcessDebounceTimer)
     this.batchProcessDebounceTimer = undefined
-    await this.watcher?.close()
+    const subscription = this.subscription
+    this.subscription = undefined
+    await subscription
+      ?.unsubscribe()
+      .catch((err) => log.error("failed to unsubscribe file watcher", { err, workspacePath: this.workspacePath }))
     await this.drainTask
     this.dispose()
   }
 
   dispose(): void {
     this.collecting = false
-    void this.watcher?.close()
+    const subscription = this.subscription
+    this.subscription = undefined
+    if (subscription) {
+      void subscription
+        .unsubscribe()
+        .catch((err) => log.error("failed to unsubscribe file watcher", { err, workspacePath: this.workspacePath }))
+    }
     if (this.batchProcessDebounceTimer) clearTimeout(this.batchProcessDebounceTimer)
     this.batchProcessDebounceTimer = undefined
     this.onDidStartBatchProcessing.dispose()
@@ -209,7 +348,7 @@ export class FileWatcher implements IFileWatcher {
   }
 
   /**
-   * Handles a file event from chokidar by accumulating it and scheduling batch processing.
+   * Handles a file event reported by the watcher by accumulating it and scheduling batch processing.
    */
   private handleFileEvent(filePath: string, type: "create" | "change" | "delete"): void {
     if (!this.shouldIndex(filePath)) return