Skip to content

Fix/standardize provider errors #811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
y-naaz wants to merge 1 commit into from
Closed

Fix/standardize provider errors #811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 33 additions & 16 deletions src/lib/utils/csvProcessor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,7 @@ function detectValueType(value: string): CSVColumnDataType {
return "empty";
}

// Check boolean first (before numbers since "1" and "0" could be both)
if (BOOLEAN_VALUES.has(trimmed.toLowerCase())) {
return "boolean";
}

// Check integer
// Check integer (before boolean since "1"/"0" are in BOOLEAN_VALUES)
if (INTEGER_REGEX.test(trimmed)) {
return "integer";
}
Expand All @@ -121,6 +116,11 @@ function detectValueType(value: string): CSVColumnDataType {
return "float";
}

// Check boolean (after numeric checks so "1"/"0" are classified as integers)
if (BOOLEAN_VALUES.has(trimmed.toLowerCase())) {
return "boolean";
}

// Check email
if (EMAIL_REGEX.test(trimmed)) {
return "email";
Expand Down Expand Up @@ -515,11 +515,13 @@ function detectHasHeaders(

const textRatio = textLikeCount / nonEmptyHeaders;

// Check 2: Headers should be unique
const uniqueHeaders = new Set(
headerValues.map((v) => v?.trim().toLowerCase()),
);
const hasUniqueHeaders = uniqueHeaders.size === headerValues.length;
// Check 2: Headers should be unique (empty/blank names excluded from uniqueness check)
const normalizedNonEmptyHeaders = headerValues
.map((v) => v?.trim().toLowerCase())
.filter((v) => v !== "");
const uniqueHeaders = new Set(normalizedNonEmptyHeaders);
const hasUniqueHeaders =
uniqueHeaders.size === normalizedNonEmptyHeaders.length;

// Check 3: Compare with data rows if available
if (dataRows && dataRows.length > 0) {
Expand Down Expand Up @@ -679,7 +681,6 @@ export class CSVProcessor {
logger.info("[CSVProcessor] ✅ Processed CSV file", {
formatStyle: "raw",
rowCount,
columnCount: (limitedLines[0] || "").split(",").length,
truncated: wasTruncated,
});

Expand All @@ -691,6 +692,20 @@ export class CSVProcessor {
const { columnMetadata, dataQualityWarnings, dataQualityScore } =
analyzeColumns(sampleForAnalysis);

// Derive headers from the properly parsed sample (handles quoted/escaped commas correctly)
const parsedHeaders =
sampleForAnalysis.length > 0 &&
sampleForAnalysis[0] &&
typeof sampleForAnalysis[0] === "object"
? Object.keys(sampleForAnalysis[0] as Record<string, unknown>)
: (limitedLines[0] || "").split(",");
const parsedColumnCount = parsedHeaders.length;

// Detect delimiter from sep= metadata line if present, otherwise assume comma
const firstLineTrimmed = (limitedLines[0] || "").trim();
const sepMatch = firstLineTrimmed.match(/^sep=(.)/i);
const detectedDelimiter = sepMatch ? sepMatch[1] : ",";

// Log data quality summary
if (dataQualityWarnings.length > 0) {
logger.debug("[CSVProcessor] Data quality warnings detected", {
Expand All @@ -708,16 +723,18 @@ export class CSVProcessor {
size: content.length,
rowCount,
totalLines: limitedLines.length,
columnCount: (limitedLines[0] || "").split(",").length,
columnCount: parsedColumnCount,
extension,
columnMetadata,
dataQualityWarnings,
dataQualityScore,
hasHeaders: detectHasHeaders(
(limitedLines[0] || "").split(","),
undefined,
parsedHeaders.map(String),
sampleForAnalysis.length > 0
? (sampleForAnalysis as Record<string, unknown>[])
: undefined,
),
detectedDelimiter: ",",
detectedDelimiter,
},
};
}
Expand Down
70 changes: 40 additions & 30 deletions test/integration/openrouter.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("Invalid API key");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("Invalid OpenRouter API key");
expect(handledError.message).toContain("https://openrouter.ai/keys");
expect(() => provider.handleProviderError(error)).toThrow(
/Invalid OpenRouter API key/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/https:\/\/openrouter\.ai\/keys/,
);
});

it("should handle API_KEY_INVALID error specifically", () => {
Expand All @@ -204,10 +206,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("API_KEY_INVALID");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("Invalid OpenRouter API key");
expect(handledError.message).toContain("https://openrouter.ai/keys");
expect(() => provider.handleProviderError(error)).toThrow(
/Invalid OpenRouter API key/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/https:\/\/openrouter\.ai\/keys/,
);
});

it("should handle Unauthorized error", () => {
Expand All @@ -216,9 +220,9 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("Unauthorized");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("Invalid OpenRouter API key");
expect(() => provider.handleProviderError(error)).toThrow(
/Invalid OpenRouter API key/,
);
});

it("should handle rate limit error", () => {
Expand All @@ -227,10 +231,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("rate limit exceeded");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("rate limit exceeded");
expect(handledError.message).toContain("https://openrouter.ai/credits");
expect(() => provider.handleProviderError(error)).toThrow(
/rate limit exceeded/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/https:\/\/openrouter\.ai\/credits/,
);
});

it("should handle model not found error", () => {
Expand All @@ -239,10 +245,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider("invalid/model");

const error = new Error("model not found");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("not available on OpenRouter");
expect(handledError.message).toContain("https://openrouter.ai/models");
expect(() => provider.handleProviderError(error)).toThrow(
/not available on OpenRouter/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/https:\/\/openrouter\.ai\/models/,
);
});

it("should handle insufficient credits error", () => {
Expand All @@ -251,10 +259,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("insufficient_credits");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("Insufficient OpenRouter credits");
expect(handledError.message).toContain("https://openrouter.ai/credits");
expect(() => provider.handleProviderError(error)).toThrow(
/Insufficient OpenRouter credits/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/https:\/\/openrouter\.ai\/credits/,
);
});

it("should handle network connection error", () => {
Expand All @@ -263,10 +273,12 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = new Error("ECONNREFUSED");
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("not available");
expect(handledError.message).toContain("network connection");
expect(() => provider.handleProviderError(error)).toThrow(
/not available/,
);
expect(() => provider.handleProviderError(error)).toThrow(
/network connection/,
);
});

it("should handle timeout error", () => {
Expand All @@ -275,9 +287,7 @@ describe("OpenRouter Provider Integration Tests", () => {
const provider = new OpenRouterProvider();

const error = { name: "TimeoutError", message: "Request timed out" };
const handledError = provider.handleProviderError(error);

expect(handledError.message).toContain("timed out");
expect(() => provider.handleProviderError(error)).toThrow(/timed out/);
});
});

Expand Down
4 changes: 2 additions & 2 deletions test/unit/utils/csvProcessor.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -583,8 +583,8 @@ Charlie,35,Chicago`;

it("should consolidate integers and floats as number type", async () => {
// Mix of integers and floats should be classified as "number", not "mixed"
// Using values > 1 to avoid boolean detection (0 and 1 are treated as boolean)
const csvData = Buffer.from("value\n10\n2.5\n30\n4.5\n50\n6.5\n70\n8.5");
// 0 and 1 are now correctly detected as integers (numeric check runs before boolean)
const csvData = Buffer.from("value\n0\n2.5\n1\n4.5\n50\n6.5\n70\n8.5");
const result = await CSVProcessor.process(csvData, {
formatStyle: "raw",
});
Expand Down
Loading