diff --git a/apps/expo/app/(app)/current-pack/[id].tsx b/apps/expo/app/(app)/current-pack/[id].tsx
index db114c3fe7..2a37709625 100644
--- a/apps/expo/app/(app)/current-pack/[id].tsx
+++ b/apps/expo/app/(app)/current-pack/[id].tsx
@@ -155,7 +155,7 @@ export default function CurrentPackScreen() {
{t('packs.lastUpdated', {
- time: getRelativeTime(pack.localUpdatedAt ?? pack.updatedAt, t),
+ time: getRelativeTime(pack.localUpdatedAt ?? pack.updatedAt, t as any),
})}
diff --git a/apps/expo/app/(app)/recent-packs.tsx b/apps/expo/app/(app)/recent-packs.tsx
index 20fa2db13a..6ab73c53fb 100644
--- a/apps/expo/app/(app)/recent-packs.tsx
+++ b/apps/expo/app/(app)/recent-packs.tsx
@@ -34,7 +34,7 @@ function RecentPackCard({ pack }: { pack: Pack }) {
{pack.totalWeight ?? 0} g
- {getRelativeTime(pack.localCreatedAt ?? pack.createdAt, t)}
+ {getRelativeTime(pack.localCreatedAt ?? pack.createdAt, t as any)}
@@ -45,7 +45,7 @@ function RecentPackCard({ pack }: { pack: Pack }) {
{t('packs.lastUpdated', {
- time: getRelativeTime(pack.localUpdatedAt ?? pack.updatedAt, t),
+ time: getRelativeTime(pack.localUpdatedAt ?? pack.updatedAt, t as any),
})}
diff --git a/packages/api/scripts/reset-stuck-etl-jobs.sql b/packages/api/scripts/reset-stuck-etl-jobs.sql
new file mode 100644
index 0000000000..f5595b867e
--- /dev/null
+++ b/packages/api/scripts/reset-stuck-etl-jobs.sql
@@ -0,0 +1,7 @@
+-- Reset ETL jobs stuck in 'running' state for more than 3 hours.
+-- 3h accounts for large first-time imports (~500K rows + embedding generation).
+-- Run manually when zombie jobs are detected.
+UPDATE etl_jobs
+SET status = 'failed', completed_at = NOW()
+WHERE status = 'running'
+ AND started_at < NOW() - INTERVAL '3 hours';
diff --git a/packages/api/src/services/catalogService.ts b/packages/api/src/services/catalogService.ts
index a416d67a85..5fb5d15b13 100644
--- a/packages/api/src/services/catalogService.ts
+++ b/packages/api/src/services/catalogService.ts
@@ -343,7 +343,20 @@ export class CatalogService {
.onConflictDoUpdate({
target: catalogItems.sku,
set: Object.values(columns).reduce>((acc, col) => {
- acc[col.name] = sql.raw(`COALESCE(catalog_items.${col.name}, excluded."${col.name}")`);
+ if (col.name === 'id' || col.name === 'created_at') {
+ // Never overwrite PK or original creation timestamp
+ acc[col.name] = sql`COALESCE(${col}, excluded.${sql.identifier(col.name)})`;
+ } else if (col.name === 'weight') {
+ // Keep old weight if new weight is missing or invalid (0 / negative)
+ acc[col.name] =
+ sql`CASE WHEN excluded.${sql.identifier('weight')} IS NOT NULL AND excluded.${sql.identifier('weight')} > 0 THEN excluded.${sql.identifier('weight')} ELSE COALESCE(${catalogItems.weight}, excluded.${sql.identifier('weight')}) END`;
+ } else if (col.name === 'weight_unit') {
+ // weight_unit stays in sync with weight validity
+ acc[col.name] =
+ sql`CASE WHEN excluded.${sql.identifier('weight')} IS NOT NULL AND excluded.${sql.identifier('weight')} > 0 THEN excluded.${sql.identifier('weight_unit')} ELSE COALESCE(${catalogItems.weightUnit}, excluded.${sql.identifier('weight_unit')}) END`;
+ } else {
+ acc[col.name] = sql`excluded.${sql.identifier(col.name)}`;
+ }
return acc;
}, {}),
})
diff --git a/packages/api/src/services/etl/CatalogItemValidator.ts b/packages/api/src/services/etl/CatalogItemValidator.ts
index 478b2c54a4..6788ba475d 100644
--- a/packages/api/src/services/etl/CatalogItemValidator.ts
+++ b/packages/api/src/services/etl/CatalogItemValidator.ts
@@ -31,23 +31,9 @@ export class CatalogItemValidator {
});
}
- if (!item.weight || !isNumber(item.weight) || item.weight <= 0) {
- errors.push({
- field: 'weight',
- reason: 'Weight is required and must be a positive number',
- value: item.weight,
- });
- }
-
- if (!item.weightUnit || !isString(item.weightUnit) || item.weightUnit.trim().length === 0) {
- errors.push({
- field: 'weightUnit',
- reason: 'Weight unit is required and must be a non-empty string',
- value: item.weightUnit,
- });
- }
-
// Additional validations
+ // Note: weight and weightUnit are intentionally not required — clothing/footwear brands often
+ // omit weight data. Items without weight are ingested but won't appear in weight comparisons.
if (item.productUrl && !this.isValidUrl(item.productUrl)) {
errors.push({
field: 'productUrl',
diff --git a/packages/api/src/services/etl/processCatalogEtl.ts b/packages/api/src/services/etl/processCatalogEtl.ts
index 7bda4487e3..dfe396f2e5 100644
--- a/packages/api/src/services/etl/processCatalogEtl.ts
+++ b/packages/api/src/services/etl/processCatalogEtl.ts
@@ -107,16 +107,38 @@ export async function processCatalogETL({
}
rowIndex++;
+
+ // Flush valid batch to DB every BATCH_SIZE rows to avoid Worker OOM on large files
+ if (validItemsBatch.length >= BATCH_SIZE) {
+ await processValidItemsBatch({ jobId, items: [...validItemsBatch], env });
+ await db
+ .update(etlJobs)
+ .set({ totalProcessed: sql`COALESCE(${etlJobs.totalProcessed}, 0) + ${BATCH_SIZE}` })
+ .where(eq(etlJobs.id, jobId));
+ validItemsBatch.length = 0;
+ }
+ // Flush invalid batch to DB every BATCH_SIZE rows
+ if (invalidItemsBatch.length >= BATCH_SIZE) {
+ await processLogsBatch({ jobId, logs: [...invalidItemsBatch], env });
+ await db
+ .update(etlJobs)
+ .set({ totalProcessed: sql`COALESCE(${etlJobs.totalProcessed}, 0) + ${BATCH_SIZE}` })
+ .where(eq(etlJobs.id, jobId));
+ invalidItemsBatch.length = 0;
+ }
}
- console.log(`🔍 [TRACE] Streaming complete - processing batches`);
+ console.log(`🔍 [TRACE] Streaming complete - processing remaining batches`);
- const itemsProcessed = validItemsBatch.length + invalidItemsBatch.length;
+ // Flush remaining items after the stream ends
+ const remainingItems = validItemsBatch.length + invalidItemsBatch.length;
- await db
- .update(etlJobs)
- .set({ totalProcessed: sql`COALESCE(${etlJobs.totalProcessed}, 0) + ${itemsProcessed}` })
- .where(eq(etlJobs.id, jobId));
+ if (remainingItems > 0) {
+ await db
+ .update(etlJobs)
+ .set({ totalProcessed: sql`COALESCE(${etlJobs.totalProcessed}, 0) + ${remainingItems}` })
+ .where(eq(etlJobs.id, jobId));
+ }
if (validItemsBatch.length > 0) {
console.log(`🔍 [TRACE] Processing valid items batch - size: ${validItemsBatch.length}`);
diff --git a/tsconfig.json b/tsconfig.json
index bd2a8c48e0..8d3088ab3a 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,9 +1,8 @@
{
"compilerOptions": {
"allowJs": true,
- "baseUrl": ".",
"esModuleInterop": true,
- "ignoreDeprecations": "6.0",
+ "ignoreDeprecations": "5.0",
"jsx": "react-native",
"lib": ["DOM", "ESNext"],
"module": "preserve",