From e88dd504ccc4fadd09c02a33e2769fa3fb63e7fc Mon Sep 17 00:00:00 2001 From: Josh Lambert Date: Tue, 4 Aug 2026 17:51:59 -0400 Subject: [PATCH] fix(cli): repair corrupted LanceDB indexes --- .changeset/repair-missing-lance-files.md | 5 ++++ .../vector-store/lancedb-vector-store.ts | 25 ++++++++++++++----- .../vector-store/lancedb-vector-store.test.ts | 19 ++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 .changeset/repair-missing-lance-files.md diff --git a/.changeset/repair-missing-lance-files.md b/.changeset/repair-missing-lance-files.md new file mode 100644 index 00000000000..04c3f0c15ac --- /dev/null +++ b/.changeset/repair-missing-lance-files.md @@ -0,0 +1,5 @@ +--- +"@kilocode/cli": patch +--- + +Repair code indexes with missing LanceDB data files and prevent startup cleanup from invalidating active index readers diff --git a/packages/kilo-indexing/src/indexing/vector-store/lancedb-vector-store.ts b/packages/kilo-indexing/src/indexing/vector-store/lancedb-vector-store.ts index f97363af68c..44e12b35978 100644 --- a/packages/kilo-indexing/src/indexing/vector-store/lancedb-vector-store.ts +++ b/packages/kilo-indexing/src/indexing/vector-store/lancedb-vector-store.ts @@ -277,6 +277,15 @@ export class LanceDBVectorStore implements IVectorStore { } async initialize(): Promise { + return this.init(true) + } + + private missing(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + return message.includes("Object at location ") && message.includes(".lance not found:") + } + + private async init(repair: boolean): Promise { try { await this.closeConnect() const db = await this.getDb() @@ -321,7 +330,6 @@ export class LanceDBVectorStore implements IVectorStore { await this._dropTableIfExists(db, this.metadataTableName) await this._createVectorTable(db) await this._createMetadataTable(db) - this.optimizeTable() log.info("LanceDB store reinitialized for embedding profile change", { workspacePath: this.workspacePath, @@ -332,7 +340,6 @@ export class LanceDBVectorStore implements IVectorStore { return true } - this.optimizeTable() log.info("LanceDB store initialized", { workspacePath: this.workspacePath, dbPath: this.dbPath, @@ -341,6 +348,15 @@ export class LanceDBVectorStore implements IVectorStore { }) return false } catch (error) { + if (repair && this.missing(error)) { + log.warn("Rebuilding LanceDB store with missing data files", { + workspacePath: this.workspacePath, + dbPath: this.dbPath, + error, + }) + await this.deleteCollection() + return this.init(false) + } log.error("Failed to initialize LanceDB store", { error }) throw new Error(`Failed to initialize LanceDB store: ${(error as Error).message}`, { cause: error }) } @@ -577,10 +593,7 @@ export class LanceDBVectorStore implements IVectorStore { try { const table = await this.getTable() - await table.optimize({ - cleanupOlderThan: new Date(), - deleteUnverified: false, - }) + await table.optimize() } catch (error) { log.error("Failed to optimize table", { error }) } diff --git a/packages/kilo-indexing/test/kilocode/indexing/vector-store/lancedb-vector-store.test.ts b/packages/kilo-indexing/test/kilocode/indexing/vector-store/lancedb-vector-store.test.ts index 0044da62667..aa96cc4f162 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/vector-store/lancedb-vector-store.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/vector-store/lancedb-vector-store.test.ts @@ -36,6 +36,7 @@ const mockTable = { remove: mock(), createIndex: mock(), dropIndex: mock(), + optimize: mock().mockResolvedValue(undefined), indexes: [], columns: [], primaryKey: "id", @@ -100,6 +101,7 @@ const allMocks = [ mockTable.remove, mockTable.createIndex, mockTable.dropIndex, + mockTable.optimize, mockTable.batch, mockTable.distanceRange, mockDb.openTable, @@ -132,6 +134,7 @@ function resetAllMocks() { mockTable.openTable.mockResolvedValue(undefined) mockTable.search.mockReturnThis() mockTable.distanceRange.mockReturnThis() + mockTable.optimize.mockResolvedValue(undefined) mockDb.openTable.mockResolvedValue(mockTable) mockDb.createTable.mockResolvedValue(mockTable) mockDb.dropTable.mockResolvedValue(undefined) @@ -243,6 +246,7 @@ describe("LocalVectorStore", () => { store["_getMetadataValue"] = mock().mockResolvedValue("2") const result = await store.initialize() expect(result).toBe(false) + expect(mockTable.optimize).not.toHaveBeenCalled() }) test("recreates an index using the legacy payload schema", async () => { @@ -268,6 +272,20 @@ describe("LocalVectorStore", () => { expect(mockDb.createTable).not.toHaveBeenCalled() }) + test("rebuilds an index whose manifest references a missing data file", async () => { + const error = new Error( + "Failed to get next batch from stream: LanceError(IO): Object at location metadata.lance/data/missing.lance not found: os error 2", + ) + spyOn(fs, "existsSync").mockReturnValue(true) + const remove = spyOn(fs, "rmSync").mockImplementation(() => {}) + mockDb.tableNames.mockResolvedValueOnce(["vector", "metadata"]).mockResolvedValueOnce([]) + store["_getStoredVectorSize"] = mock().mockRejectedValue(error) + + expect(await store.initialize()).toBe(true) + expect(remove).toHaveBeenCalledWith(store["dbPath"], { recursive: true, force: true }) + expect(mockDb.createTable).toHaveBeenCalledTimes(2) + }) + test("does not recreate when profile metadata cannot be read", async () => { mockTable.countRows.mockResolvedValue(1) store["_getStoredVectorSize"] = mock().mockResolvedValue(vectorSize) @@ -515,6 +533,7 @@ describe("LocalVectorStore", () => { mockTable.delete.mockResolvedValue(undefined) await expect(store.clearCollection()).resolves.toBeUndefined() expect(mockTable.delete).toHaveBeenCalledWith("true") + expect(mockTable.optimize).toHaveBeenCalledWith() }) test("should warn if metadata table clear fails", async () => {