From 703503687a0acbbb856c5bb8b9fbd725ceb7cc94 Mon Sep 17 00:00:00 2001
From: Jason <jason@outland.art>
Date: Sun, 8 Mar 2026 15:49:33 +0800
Subject: [PATCH] fix(ingestion): increase bufferSize from 256KB to 2MB for
 large files

Fixes #198

Problem: Files between 256KB-512KB crash tree-sitter with
'Invalid argument' because bufferSize (256KB) is too small.

Fix: Increased bufferSize from 1024*256 (256KB) to 1024*1024*2 (2MB)

This allows parsing of files up to MAX_FILE_SIZE (512KB) without errors.
---
 gitnexus/src/core/ingestion/call-processor.ts       | 2 +-
 gitnexus/src/core/ingestion/heritage-processor.ts   | 2 +-
 gitnexus/src/core/ingestion/import-processor.ts     | 2 +-
 gitnexus/src/core/ingestion/parsing-processor.ts    | 2 +-
 gitnexus/src/core/ingestion/workers/parse-worker.ts | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts
index 9b6dfc3f43..fddf4c62eb 100644
--- a/gitnexus/src/core/ingestion/call-processor.ts
+++ b/gitnexus/src/core/ingestion/call-processor.ts
@@ -174,7 +174,7 @@ export const processCalls = async (
       // Cache Miss: Re-parse
       // Use larger bufferSize for files > 32KB
       try {
-        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 1024 * 2 });
       } catch (parseError) {
         // Skip files that can't be parsed
         continue;
diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts
index dbb7bac8ca..611d1137c6 100644
--- a/gitnexus/src/core/ingestion/heritage-processor.ts
+++ b/gitnexus/src/core/ingestion/heritage-processor.ts
@@ -47,7 +47,7 @@ export const processHeritage = async (
     if (!tree) {
       // Use larger bufferSize for files > 32KB
       try {
-        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 1024 * 2 });
       } catch (parseError) {
         // Skip files that can't be parsed
         continue;
diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts
index 990f968afd..e6247a55ab 100644
--- a/gitnexus/src/core/ingestion/import-processor.ts
+++ b/gitnexus/src/core/ingestion/import-processor.ts
@@ -794,7 +794,7 @@ export const processImports = async (
 
     if (!tree) {
       try {
-        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+        tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 1024 * 2 });
       } catch (parseError) {
         continue;
       }
diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts
index ae0e7026f2..3495c0c6f8 100644
--- a/gitnexus/src/core/ingestion/parsing-processor.ts
+++ b/gitnexus/src/core/ingestion/parsing-processor.ts
@@ -297,7 +297,7 @@ const processParsingSequential = async (
 
     let tree;
     try {
-      tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+      tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 1024 * 2 });
     } catch (parseError) {
       console.warn(`Skipping unparseable file: ${file.path}`);
       continue;
diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts
index b9d0b2bdb7..38f4f345ed 100644
--- a/gitnexus/src/core/ingestion/workers/parse-worker.ts
+++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts
@@ -1109,7 +1109,7 @@ const processFileGroup = (
 
     let tree;
     try {
-      tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+      tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 1024 * 2 });
     } catch {
       continue;
     }