Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gitnexus-web/src/core/ingestion/call-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ export const processCalls = async (
if (!queryStr) continue;

// 2. ALWAYS load the language before querying (parser is stateful)
await loadLanguage(language, file.path);
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST (Try Cache First)
let tree = astCache.get(file.path);
Expand Down
4 changes: 2 additions & 2 deletions gitnexus-web/src/core/ingestion/heritage-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ export const processHeritage = async (
const queryStr = LANGUAGE_QUERIES[language];
if (!queryStr) continue;

// 2. Load the language
await loadLanguage(language, file.path);
// 2. Load the language (skip if unavailable)
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST
let tree = astCache.get(file.path);
Expand Down
2 changes: 1 addition & 1 deletion gitnexus-web/src/core/ingestion/import-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ export const processImports = async (
if (!queryStr) continue;

// 2. ALWAYS load the language before querying (parser is stateful)
await loadLanguage(language, file.path);
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST (Try Cache First)
let tree = astCache.get(file.path);
Expand Down
2 changes: 1 addition & 1 deletion gitnexus-web/src/core/ingestion/parsing-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ export const processParsing = async (

if (!language) continue;

await loadLanguage(language, file.path);
if (!await loadLanguage(language, file.path)) continue;

// 3. Parse the text content into an AST
const tree = parser.parse(file.content);
Expand Down
14 changes: 6 additions & 8 deletions gitnexus-web/src/core/tree-sitter/parser-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,29 +46,27 @@ const getWasmPath = (language: SupportedLanguages, filePath?: string): string =>
return languageFileMap[language];
};

export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<void> => {
export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<boolean> => {
if (!parser) await loadParser();
const wasmPath = getWasmPath(language, filePath);

if (languageCache.has(wasmPath)) {
parser!.setLanguage(languageCache.get(wasmPath)!);
return;
return true;
}

if (!wasmPath) {
console.error(`❌ [Parser] No WASM path configured for language: ${language}`);
throw new Error(`Unsupported language: ${language}`);
return false;
}

try {
const loadedLanguage = await Parser.Language.load(wasmPath);
languageCache.set(wasmPath, loadedLanguage);
parser!.setLanguage(loadedLanguage);
return true;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(`❌ [Parser] Failed to load WASM grammar for ${language}`);
console.error(` WASM Path: ${wasmPath}`);
console.error(` Error: ${errorMessage}`);
throw new Error(`Failed to load grammar for ${language}: ${errorMessage}`);
console.warn(`[Parser] Failed to load WASM grammar for ${language}: ${errorMessage}`);
return false;
}
}
2 changes: 1 addition & 1 deletion gitnexus/src/core/ingestion/call-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ export const processCalls = async (
if (!queryStr) continue;

// 2. ALWAYS load the language before querying (parser is stateful)
await loadLanguage(language, file.path);
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST (Try Cache First)
let tree = astCache.get(file.path);
Expand Down
4 changes: 2 additions & 2 deletions gitnexus/src/core/ingestion/heritage-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ export const processHeritage = async (
const queryStr = LANGUAGE_QUERIES[language];
if (!queryStr) continue;

// 2. Load the language
await loadLanguage(language, file.path);
// 2. Load the language (skip if unavailable, e.g. optional tree-sitter grammar)
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST
let tree = astCache.get(file.path);
Expand Down
2 changes: 1 addition & 1 deletion gitnexus/src/core/ingestion/import-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -786,7 +786,7 @@ export const processImports = async (
if (!queryStr) continue;

// 2. ALWAYS load the language before querying (parser is stateful)
await loadLanguage(language, file.path);
if (!await loadLanguage(language, file.path)) continue;

// 3. Get AST (Try Cache First)
let tree = astCache.get(file.path);
Expand Down
6 changes: 1 addition & 5 deletions gitnexus/src/core/ingestion/parsing-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,11 +289,7 @@ const processParsingSequential = async (
// Skip very large files — they can crash tree-sitter or cause OOM
if (file.content.length > 512 * 1024) continue;

try {
await loadLanguage(language, file.path);
} catch {
continue; // parser unavailable — already warned in pipeline
}
if (!await loadLanguage(language, file.path)) continue;

let tree;
try {
Expand Down
15 changes: 5 additions & 10 deletions gitnexus/src/core/ingestion/workers/parse-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,14 @@ const languageMap: Record<string, any> = {
...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
};

const setLanguage = (language: SupportedLanguages, filePath: string): void => {
const setLanguage = (language: SupportedLanguages, filePath: string): boolean => {
const key = language === SupportedLanguages.TypeScript && filePath.endsWith('.tsx')
? `${language}:tsx`
: language;
const lang = languageMap[key];
if (!lang) throw new Error(`Unsupported language: ${language}`);
if (!lang) return false;
parser.setLanguage(lang);
return true;
};

// ============================================================================
Expand Down Expand Up @@ -589,21 +590,15 @@ const processBatch = (files: ParseWorkerInput[], onProgress?: (filesProcessed: n

// Process regular files for this language
if (regularFiles.length > 0) {
try {
setLanguage(language, regularFiles[0].path);
if (setLanguage(language, regularFiles[0].path)) {
processFileGroup(regularFiles, language, queryString, result, onFileProcessed);
} catch {
// parser unavailable — skip this language group
}
}

// Process tsx files separately (different grammar)
if (tsxFiles.length > 0) {
try {
setLanguage(language, tsxFiles[0].path);
if (setLanguage(language, tsxFiles[0].path)) {
processFileGroup(tsxFiles, language, queryString, result, onFileProcessed);
} catch {
// parser unavailable — skip this language group
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions gitnexus/src/core/tree-sitter/parser-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,16 @@ export const loadParser = async (): Promise<Parser> => {
return parser;
};

export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<void> => {
export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<boolean> => {
if (!parser) await loadParser();
const key = language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
? `${language}:tsx`
: language;

const lang = languageMap[key];
if (!lang) {
throw new Error(`Unsupported language: ${language}`);
return false;
}
parser!.setLanguage(lang);
return true;
};
41 changes: 18 additions & 23 deletions gitnexus/test/unit/parser-loader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,69 +19,64 @@ describe('parser-loader', () => {

describe('loadLanguage', () => {
it('loads TypeScript language', async () => {
await expect(loadLanguage(SupportedLanguages.TypeScript)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.TypeScript)).toBe(true);
});

it('loads JavaScript language', async () => {
await expect(loadLanguage(SupportedLanguages.JavaScript)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.JavaScript)).toBe(true);
});

it('loads Python language', async () => {
await expect(loadLanguage(SupportedLanguages.Python)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.Python)).toBe(true);
});

it('loads Java language', async () => {
await expect(loadLanguage(SupportedLanguages.Java)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.Java)).toBe(true);
});

it('loads C language', async () => {
await expect(loadLanguage(SupportedLanguages.C)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.C)).toBe(true);
});

it('loads C++ language', async () => {
await expect(loadLanguage(SupportedLanguages.CPlusPlus)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.CPlusPlus)).toBe(true);
});

it('loads C# language', async () => {
await expect(loadLanguage(SupportedLanguages.CSharp)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.CSharp)).toBe(true);
});

it('loads Go language', async () => {
await expect(loadLanguage(SupportedLanguages.Go)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.Go)).toBe(true);
});

it('loads Rust language', async () => {
await expect(loadLanguage(SupportedLanguages.Rust)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.Rust)).toBe(true);
});

it('loads PHP language', async () => {
await expect(loadLanguage(SupportedLanguages.PHP)).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.PHP)).toBe(true);
});

it('loads TSX grammar for .tsx files', async () => {
// TSX uses a different grammar (TypeScript.tsx vs TypeScript.typescript)
await expect(loadLanguage(SupportedLanguages.TypeScript, 'Component.tsx')).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.TypeScript, 'Component.tsx')).toBe(true);
});

it('loads TS grammar for .ts files', async () => {
await expect(loadLanguage(SupportedLanguages.TypeScript, 'utils.ts')).resolves.not.toThrow();
expect(await loadLanguage(SupportedLanguages.TypeScript, 'utils.ts')).toBe(true);
});

it('throws for unsupported language', async () => {
await expect(loadLanguage('ruby' as SupportedLanguages)).rejects.toThrow('Unsupported language');
it('returns false for unsupported language', async () => {
const result = await loadLanguage('ruby' as SupportedLanguages);
expect(result).toBe(false);
});
});

describe('Swift optional dependency', () => {
it('handles Swift loading gracefully', async () => {
// Swift is optional — it either loads successfully or throws an error about unsupported language
try {
await loadLanguage(SupportedLanguages.Swift);
// If it succeeds, tree-sitter-swift is installed
} catch (e: any) {
// If it fails, it should be because tree-sitter-swift is not installed
expect(e.message).toContain('Unsupported language');
}
const result = await loadLanguage(SupportedLanguages.Swift);
// Returns true if tree-sitter-swift is installed, false otherwise
expect(typeof result).toBe('boolean');
});
});
});