Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions gitnexus/src/core/ingestion/call-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1701,7 +1701,7 @@ export const processAssignmentsFromExtracted = (
};

/**
* Resolve pre-extracted Laravel routes to CALLS edges from route files to controller methods.
* Resolve pre-extracted framework routes to CALLS edges from handler files to controller methods.
*/
export const processRoutesFromExtracted = async (
graph: KnowledgeGraph,
Expand Down Expand Up @@ -1739,7 +1739,7 @@ export const processRoutesFromExtracted = async (
targetId: guessedId,
type: 'CALLS',
confidence: confidence * 0.8,
reason: 'laravel-route',
reason: 'framework-route',
});
continue;
}
Expand All @@ -1751,7 +1751,7 @@ export const processRoutesFromExtracted = async (
targetId: methodId,
type: 'CALLS',
confidence,
reason: 'laravel-route',
reason: 'framework-route',
});
}

Expand Down
12 changes: 12 additions & 0 deletions gitnexus/src/core/ingestion/field-extractors/configs/jvm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { SupportedLanguages } from 'gitnexus-shared';
import type { FieldExtractionConfig } from '../generic.js';
import { findVisibility, hasKeyword, hasModifier, typeFromField } from './helpers.js';
import { extractSimpleTypeName } from '../../type-extractors/shared.js';
import { extractJavaStringLiteral } from '../../utils/java-strings.js';
import type { FieldVisibility } from '../../field-types.js';

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -65,6 +66,17 @@ export const javaConfig: FieldExtractionConfig = {
isReadonly(node) {
return hasKeyword(node, 'final') || hasModifier(node, 'modifiers', 'final');
},

extractConstantValue(node) {
const isStatic = hasKeyword(node, 'static') || hasModifier(node, 'modifiers', 'static');
const isReadonly = hasKeyword(node, 'final') || hasModifier(node, 'modifiers', 'final');
if (!isStatic || !isReadonly) return undefined;
const declarator =
node.childForFieldName('declarator') ??
node.namedChildren.find((child) => child.type === 'variable_declarator');
const valueNode = declarator?.childForFieldName('value');
return extractJavaStringLiteral(valueNode);
},
};

// ---------------------------------------------------------------------------
Expand Down
5 changes: 5 additions & 0 deletions gitnexus/src/core/ingestion/field-extractors/generic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ export interface FieldExtractionConfig {
isStatic: (node: SyntaxNode) => boolean;
/** Check if a field is readonly/final/const */
isReadonly: (node: SyntaxNode) => boolean;
/** Extract compile-time constant string value when available */
extractConstantValue?: (node: SyntaxNode) => string | undefined;
/** Extract fields from primary constructor parameters on the owner node itself
* (e.g. C# record positional parameters, C# 12 class primary constructors). */
extractPrimaryFields?: (ownerNode: SyntaxNode, context: FieldExtractorContext) => FieldInfo[];
Expand Down Expand Up @@ -176,12 +178,15 @@ export function createFieldExtractor(config: FieldExtractionConfig): FieldExtrac
if (resolved) type = resolved;
}

const constantValue = config.extractConstantValue?.(node);

return {
name,
type,
visibility: config.extractVisibility(node),
isStatic: config.isStatic(node),
isReadonly: config.isReadonly(node),
...(constantValue !== undefined ? { constantValue } : {}),
sourceFile: context.filePath,
line: node.startPosition.row + 1,
};
Expand Down
2 changes: 2 additions & 0 deletions gitnexus/src/core/ingestion/field-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ export interface FieldInfo {
isStatic: boolean;
/** Is this readonly/const? */
isReadonly: boolean;
/** Compile-time constant string value when available */
constantValue?: string;
/** Source file path */
sourceFile: string;
/** Line number */
Expand Down
16 changes: 16 additions & 0 deletions gitnexus/src/core/ingestion/language-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* so adding a language to the enum without creating a provider is a compiler error.
*/

import type Parser from 'tree-sitter';
import type { SupportedLanguages } from 'gitnexus-shared';
import type { LanguageTypeConfig } from './type-extractors/types.js';
import type { CallRouter } from './call-routing.js';
Expand All @@ -19,6 +20,9 @@ import type { ImportResolverFn } from './import-resolvers/types.js';
import type { NamedBindingExtractorFn } from './named-bindings/types.js';
import type { SyntaxNode } from './utils/ast-helpers.js';
import type { NodeLabel } from 'gitnexus-shared';
import type { DeferredRouteCandidate } from './route-extractors/deferred-route-types.js';
import type { ResolutionContext } from './resolution-context.js';
import type { ExtractedRoute } from './workers/parse-worker.js';

// ── Shared type aliases ────────────────────────────────────────────────────
/** Tree-sitter query captures: capture name → AST node (or undefined if not captured). */
Expand Down Expand Up @@ -143,6 +147,18 @@ interface LanguageProviderConfig {
* When true, the worker extracts routes via the language's route extraction logic.
* Default: undefined (no route files). */
readonly isRouteFile?: (filePath: string) => boolean;
/** Extract deferred framework route candidates that need finalize after imports/symbols.
* Default: undefined (no deferred route extraction). */
readonly deferredRouteExtractor?: (
tree: Parser.Tree,
filePath: string,
) => DeferredRouteCandidate[];
/** Finalize deferred route candidates after imports/symbols are available.
* Default: undefined (no deferred route finalization). */
readonly deferredRouteFinalizer?: (
candidates: DeferredRouteCandidate[],
ctx: ResolutionContext,
) => ExtractedRoute[];

// ── Noise filtering ────────────────────────────────────────────────
/** Built-in/stdlib names that should be filtered from the call graph for this language.
Expand Down
24 changes: 24 additions & 0 deletions gitnexus/src/core/ingestion/languages/java.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,29 @@ import { JAVA_QUERIES } from '../tree-sitter-queries.js';
import { createFieldExtractor } from '../field-extractors/generic.js';
import { javaConfig } from '../field-extractors/configs/jvm.js';
import { createMethodExtractor } from '../method-extractors/generic.js';
import {
extractSpringJavaRouteCandidates,
finalizeSpringJavaRoutes,
} from '../route-extractors/spring-java.js';
import { javaMethodConfig } from '../method-extractors/configs/jvm.js';

const SPRING_ROUTE_FILE_SUFFIXES = [
'Controller.java',
'Resource.java',
'Endpoint.java',
'Api.java',
'Handler.java',
];
const SPRING_ROUTE_DIR_HINTS = ['/controller/', '/controllers/', '/rest/', '/api/', '/web/'];

function isSpringRouteFile(filePath: string): boolean {
const normalized = filePath.replace(/\\/g, '/');
return (
SPRING_ROUTE_FILE_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) ||
SPRING_ROUTE_DIR_HINTS.some((segment) => normalized.includes(segment))
);
}

export const javaProvider = defineLanguage({
id: SupportedLanguages.Java,
extensions: ['.java'],
Expand All @@ -31,4 +52,7 @@ export const javaProvider = defineLanguage({
mroStrategy: 'implements-split',
fieldExtractor: createFieldExtractor(javaConfig),
methodExtractor: createMethodExtractor(javaMethodConfig),
isRouteFile: isSpringRouteFile,
deferredRouteExtractor: extractSpringJavaRouteCandidates,
deferredRouteFinalizer: finalizeSpringJavaRoutes,
});
76 changes: 55 additions & 21 deletions gitnexus/src/core/ingestion/parsing-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ import {
} from './utils/ast-helpers.js';
import { detectFrameworkFromAST } from './framework-detection.js';
import { buildTypeEnv } from './type-env.js';
import type { DeferredRouteCandidate } from './route-extractors/deferred-route-types.js';
import type { FieldInfo, FieldExtractorContext } from './field-types.js';
import type { LanguageProvider } from './language-provider.js';
import { WorkerPool } from './workers/worker-pool.js';
import { getTreeSitterBufferSize, TREE_SITTER_MAX_BUFFER } from './constants.js';
import type {
ParseWorkerResult,
ParseWorkerInput,
Expand All @@ -36,7 +38,6 @@ import type {
FileTypeEnvBindings,
ExtractedORMQuery,
} from './workers/parse-worker.js';
import { getTreeSitterBufferSize, TREE_SITTER_MAX_BUFFER } from './constants.js';

export type FileProgressCallback = (current: number, total: number, filePath: string) => void;

Expand All @@ -48,12 +49,30 @@ export interface WorkerExtractedData {
routes: ExtractedRoute[];
fetchCalls: ExtractedFetchCall[];
decoratorRoutes: ExtractedDecoratorRoute[];
deferredRouteCandidates: DeferredRouteCandidate[];
toolDefs: ExtractedToolDef[];
ormQueries: ExtractedORMQuery[];
constructorBindings: FileConstructorBindings[];
typeEnvBindings: FileTypeEnvBindings[];
}

function createEmptyExtractedData(): WorkerExtractedData {
return {
imports: [],
calls: [],
assignments: [],
heritage: [],
routes: [],
fetchCalls: [],
decoratorRoutes: [],
deferredRouteCandidates: [],
toolDefs: [],
ormQueries: [],
constructorBindings: [],
typeEnvBindings: [],
};
}

// ============================================================================
// Worker-based parallel parsing
// ============================================================================
Expand All @@ -73,20 +92,7 @@ const processParsingWithWorkers = async (
if (lang) parseableFiles.push({ path: file.path, content: file.content });
}

if (parseableFiles.length === 0)
return {
imports: [],
calls: [],
assignments: [],
heritage: [],
routes: [],
fetchCalls: [],
decoratorRoutes: [],
toolDefs: [],
ormQueries: [],
constructorBindings: [],
typeEnvBindings: [],
};
if (parseableFiles.length === 0) return createEmptyExtractedData();

const total = files.length;

Expand All @@ -106,6 +112,7 @@ const processParsingWithWorkers = async (
const allRoutes: ExtractedRoute[] = [];
const allFetchCalls: ExtractedFetchCall[] = [];
const allDecoratorRoutes: ExtractedDecoratorRoute[] = [];
const allDeferredRouteCandidates: DeferredRouteCandidate[] = [];
const allToolDefs: ExtractedToolDef[] = [];
const allORMQueries: ExtractedORMQuery[] = [];
const allConstructorBindings: FileConstructorBindings[] = [];
Expand All @@ -130,6 +137,7 @@ const processParsingWithWorkers = async (
parameterTypes: sym.parameterTypes,
returnType: sym.returnType,
declaredType: sym.declaredType,
constantValue: sym.constantValue,
ownerId: sym.ownerId,
});
}
Expand All @@ -141,6 +149,7 @@ const processParsingWithWorkers = async (
allRoutes.push(...result.routes);
allFetchCalls.push(...result.fetchCalls);
allDecoratorRoutes.push(...result.decoratorRoutes);
allDeferredRouteCandidates.push(...result.deferredRouteCandidates);
allToolDefs.push(...result.toolDefs);
if (result.ormQueries) allORMQueries.push(...result.ormQueries);
allConstructorBindings.push(...result.constructorBindings);
Expand Down Expand Up @@ -171,6 +180,7 @@ const processParsingWithWorkers = async (
routes: allRoutes,
fetchCalls: allFetchCalls,
decoratorRoutes: allDecoratorRoutes,
deferredRouteCandidates: allDeferredRouteCandidates,
toolDefs: allToolDefs,
ormQueries: allORMQueries,
constructorBindings: allConstructorBindings,
Expand Down Expand Up @@ -250,8 +260,9 @@ const processParsingSequential = async (
symbolTable: SymbolTable,
astCache: ASTCache,
onFileProgress?: FileProgressCallback,
) => {
): Promise<WorkerExtractedData> => {
const parser = await loadParser();
const extractedData = createEmptyExtractedData();
const total = files.length;
const skippedLanguages = new Map<string, number>();

Expand Down Expand Up @@ -421,6 +432,7 @@ const processParsingSequential = async (
let seqVisibility: string | undefined;
let seqIsStatic: boolean | undefined;
let seqIsReadonly: boolean | undefined;
let seqConstantValue: string | undefined;
if (nodeLabel === 'Property' && definitionNode) {
// FieldExtractor is the single source of truth when available
if (provider.fieldExtractor && typeEnv) {
Expand All @@ -438,6 +450,7 @@ const processParsingSequential = async (
seqVisibility = info.visibility;
seqIsStatic = info.isStatic;
seqIsReadonly = info.isReadonly;
seqConstantValue = info.constantValue;
}
}
}
Expand All @@ -448,6 +461,7 @@ const processParsingSequential = async (
if (seqVisibility !== undefined) node.properties.visibility = seqVisibility;
if (seqIsStatic !== undefined) node.properties.isStatic = seqIsStatic;
if (seqIsReadonly !== undefined) node.properties.isReadonly = seqIsReadonly;
if (seqConstantValue !== undefined) node.properties.constantValue = seqConstantValue;
if (declaredType !== undefined) node.properties.declaredType = declaredType;

symbolTable.add(file.path, nodeName, nodeId, nodeLabel, {
Expand All @@ -456,6 +470,7 @@ const processParsingSequential = async (
parameterTypes: methodSig?.parameterTypes,
returnType: methodSig?.returnType,
declaredType,
constantValue: seqConstantValue,
ownerId: enclosingClassId ?? undefined,
});

Expand Down Expand Up @@ -487,6 +502,12 @@ const processParsingSequential = async (
});
}
});

if (provider.isRouteFile?.(file.path) && provider.deferredRouteExtractor) {
extractedData.deferredRouteCandidates.push(
...provider.deferredRouteExtractor(tree, file.path),
);
}
}

if (skippedLanguages.size > 0) {
Expand All @@ -495,30 +516,41 @@ const processParsingSequential = async (
.join(', ');
console.warn(` Skipped unsupported languages: ${summary}`);
}

return extractedData;
};

// ============================================================================
// Public API
// ============================================================================

export interface ParsingResult {
data: WorkerExtractedData;
usedWorkers: boolean;
}

export const processParsing = async (
graph: KnowledgeGraph,
files: { path: string; content: string }[],
symbolTable: SymbolTable,
astCache: ASTCache,
onFileProgress?: FileProgressCallback,
workerPool?: WorkerPool,
): Promise<WorkerExtractedData | null> => {
): Promise<ParsingResult> => {
if (workerPool) {
try {
return await processParsingWithWorkers(
const data = await processParsingWithWorkers(
graph,
files,
symbolTable,
astCache,
workerPool,
onFileProgress,
);
return {
data,
usedWorkers: true,
};
} catch (err) {
console.warn(
'Worker pool parsing failed, falling back to sequential:',
Expand All @@ -527,7 +559,9 @@ export const processParsing = async (
}
}

// Fallback: sequential parsing (no pre-extracted data)
await processParsingSequential(graph, files, symbolTable, astCache, onFileProgress);
return null;
const data = await processParsingSequential(graph, files, symbolTable, astCache, onFileProgress);
return {
data,
usedWorkers: false,
};
};
Loading