diff --git a/src/lib/utils/csvProcessor.ts b/src/lib/utils/csvProcessor.ts index 704dcc9012..fdbbd1928e 100644 --- a/src/lib/utils/csvProcessor.ts +++ b/src/lib/utils/csvProcessor.ts @@ -106,12 +106,7 @@ function detectValueType(value: string): CSVColumnDataType { return "empty"; } - // Check boolean first (before numbers since "1" and "0" could be both) - if (BOOLEAN_VALUES.has(trimmed.toLowerCase())) { - return "boolean"; - } - - // Check integer + // Check integer (before boolean since "1"/"0" are in BOOLEAN_VALUES) if (INTEGER_REGEX.test(trimmed)) { return "integer"; } @@ -121,6 +116,11 @@ function detectValueType(value: string): CSVColumnDataType { return "float"; } + // Check boolean (after numeric checks so "1"/"0" are classified as integers) + if (BOOLEAN_VALUES.has(trimmed.toLowerCase())) { + return "boolean"; + } + // Check email if (EMAIL_REGEX.test(trimmed)) { return "email"; @@ -515,11 +515,13 @@ function detectHasHeaders( const textRatio = textLikeCount / nonEmptyHeaders; - // Check 2: Headers should be unique - const uniqueHeaders = new Set( - headerValues.map((v) => v?.trim().toLowerCase()), - ); - const hasUniqueHeaders = uniqueHeaders.size === headerValues.length; + // Check 2: Headers should be unique (empty/blank names excluded from uniqueness check) + const normalizedNonEmptyHeaders = headerValues + .map((v) => v?.trim().toLowerCase()) + .filter((v) => v !== ""); + const uniqueHeaders = new Set(normalizedNonEmptyHeaders); + const hasUniqueHeaders = + uniqueHeaders.size === normalizedNonEmptyHeaders.length; // Check 3: Compare with data rows if available if (dataRows && dataRows.length > 0) { @@ -679,7 +681,6 @@ export class CSVProcessor { logger.info("[CSVProcessor] ✅ Processed CSV file", { formatStyle: "raw", rowCount, - columnCount: (limitedLines[0] || "").split(",").length, truncated: wasTruncated, }); @@ -691,6 +692,20 @@ export class CSVProcessor { const { columnMetadata, dataQualityWarnings, dataQualityScore } = analyzeColumns(sampleForAnalysis); + // Derive headers from the properly parsed sample (handles quoted/escaped commas correctly) + const parsedHeaders = + sampleForAnalysis.length > 0 && + sampleForAnalysis[0] && + typeof sampleForAnalysis[0] === "object" + ? Object.keys(sampleForAnalysis[0] as Record) + : (limitedLines[0] || "").split(","); + const parsedColumnCount = parsedHeaders.length; + + // Detect delimiter from sep= metadata line if present, otherwise assume comma + const firstLineTrimmed = (limitedLines[0] || "").trim(); + const sepMatch = firstLineTrimmed.match(/^sep=(.)/i); + const detectedDelimiter = sepMatch ? sepMatch[1] : ","; + // Log data quality summary if (dataQualityWarnings.length > 0) { logger.debug("[CSVProcessor] Data quality warnings detected", { @@ -708,16 +723,18 @@ export class CSVProcessor { size: content.length, rowCount, totalLines: limitedLines.length, - columnCount: (limitedLines[0] || "").split(",").length, + columnCount: parsedColumnCount, extension, columnMetadata, dataQualityWarnings, dataQualityScore, hasHeaders: detectHasHeaders( - (limitedLines[0] || "").split(","), - undefined, + parsedHeaders.map(String), + sampleForAnalysis.length > 0 + ? (sampleForAnalysis as Record[]) + : undefined, ), - detectedDelimiter: ",", + detectedDelimiter, }, }; } diff --git a/test/integration/openrouter.test.ts b/test/integration/openrouter.test.ts index 03ccec157e..7908ec0c05 100644 --- a/test/integration/openrouter.test.ts +++ b/test/integration/openrouter.test.ts @@ -192,10 +192,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("Invalid API key"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("Invalid OpenRouter API key"); - expect(handledError.message).toContain("https://openrouter.ai/keys"); + expect(() => provider.handleProviderError(error)).toThrow( + /Invalid OpenRouter API key/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /https:\/\/openrouter\.ai\/keys/, + ); }); it("should handle API_KEY_INVALID error specifically", () => { @@ -204,10 +206,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("API_KEY_INVALID"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("Invalid OpenRouter API key"); - expect(handledError.message).toContain("https://openrouter.ai/keys"); + expect(() => provider.handleProviderError(error)).toThrow( + /Invalid OpenRouter API key/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /https:\/\/openrouter\.ai\/keys/, + ); }); it("should handle Unauthorized error", () => { @@ -216,9 +220,9 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("Unauthorized"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("Invalid OpenRouter API key"); + expect(() => provider.handleProviderError(error)).toThrow( + /Invalid OpenRouter API key/, + ); }); it("should handle rate limit error", () => { @@ -227,10 +231,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("rate limit exceeded"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("rate limit exceeded"); - expect(handledError.message).toContain("https://openrouter.ai/credits"); + expect(() => provider.handleProviderError(error)).toThrow( + /rate limit exceeded/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /https:\/\/openrouter\.ai\/credits/, + ); }); it("should handle model not found error", () => { @@ -239,10 +245,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider("invalid/model"); const error = new Error("model not found"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("not available on OpenRouter"); - expect(handledError.message).toContain("https://openrouter.ai/models"); + expect(() => provider.handleProviderError(error)).toThrow( + /not available on OpenRouter/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /https:\/\/openrouter\.ai\/models/, + ); }); it("should handle insufficient credits error", () => { @@ -251,10 +259,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("insufficient_credits"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("Insufficient OpenRouter credits"); - expect(handledError.message).toContain("https://openrouter.ai/credits"); + expect(() => provider.handleProviderError(error)).toThrow( + /Insufficient OpenRouter credits/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /https:\/\/openrouter\.ai\/credits/, + ); }); it("should handle network connection error", () => { @@ -263,10 +273,12 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = new Error("ECONNREFUSED"); - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("not available"); - expect(handledError.message).toContain("network connection"); + expect(() => provider.handleProviderError(error)).toThrow( + /not available/, + ); + expect(() => provider.handleProviderError(error)).toThrow( + /network connection/, + ); }); it("should handle timeout error", () => { @@ -275,9 +287,7 @@ describe("OpenRouter Provider Integration Tests", () => { const provider = new OpenRouterProvider(); const error = { name: "TimeoutError", message: "Request timed out" }; - const handledError = provider.handleProviderError(error); - - expect(handledError.message).toContain("timed out"); + expect(() => provider.handleProviderError(error)).toThrow(/timed out/); }); }); diff --git a/test/unit/utils/csvProcessor.test.ts b/test/unit/utils/csvProcessor.test.ts index 72f2cbe3a1..0edca3ded0 100644 --- a/test/unit/utils/csvProcessor.test.ts +++ b/test/unit/utils/csvProcessor.test.ts @@ -583,8 +583,8 @@ Charlie,35,Chicago`; it("should consolidate integers and floats as number type", async () => { // Mix of integers and floats should be classified as "number", not "mixed" - // Using values > 1 to avoid boolean detection (0 and 1 are treated as boolean) - const csvData = Buffer.from("value\n10\n2.5\n30\n4.5\n50\n6.5\n70\n8.5"); + // 0 and 1 are now correctly detected as integers (numeric check runs before boolean) + const csvData = Buffer.from("value\n0\n2.5\n1\n4.5\n50\n6.5\n70\n8.5"); const result = await CSVProcessor.process(csvData, { formatStyle: "raw", });