[OPDATA-3889] Ftse EA Parsing Utils

package.json

@@ -60,7 +60,7 @@
     "@types/babel__preset-env": "7.9.7",
     "@types/eslint": "8.56.12",
     "@types/jest": "^29.5.14",
-    "@types/node": "22.14.1",
+    "@types/node": "^24.3.1",
     "husky": "7.0.4",
     "jest": "^29.7.0",
     "lint-staged": "11.2.6",

packages/sources/ftse-sftp/src/parsing/base-parser.ts

@@ -1,15 +1,15 @@
-import { CSVParser, ParsedData, CSVParserConfig, defaultCSVConfig } from './interfaces'
-import * as csvParse from 'csv-parse/sync'
+import { CSVParser, ParsedData } from './interfaces'
+import { parse, Options } from 'csv-parse/sync'
 
 /**
  * Abstract base class for CSV parsers
  * Uses the csv-parse library for robust CSV parsing
  */
 export abstract class BaseCSVParser implements CSVParser {
-  protected config: CSVParserConfig
+  protected config: Record<string, any>
 
-  constructor(config: Partial<CSVParserConfig> = {}) {
-    this.config = { ...defaultCSVConfig, ...config }
+  constructor(config: Record<string, any> = {}) {
+    this.config = { ...config }
   }
 
   /**
@@ -26,11 +26,11 @@
   /**
    * Helper method to parse CSV content using csv-parse library
    */
-  protected parseCSV(csvContent: string, options?: Partial<CSVParserConfig>): any[] {
-    const finalConfig = { ...this.config, ...options }
+  protected parseCSV(csvContent: string, options?: Options): any[] {
+    const finalConfig: Options = { ...this.config, ...options }
 
     try {
-      return csvParse.parse(csvContent, finalConfig)
+      return parse(csvContent, finalConfig)
     } catch (error) {
       throw new Error(`Error parsing CSV: ${error}`)
     }
@@ -47,22 +47,20 @@
       return null
     }
 
-    const trimmedValue = value.trim()
-
     switch (expectedType) {
       case 'number': {
         const numValue = parseFloat(trimmedValue.replace(/,/g, ''))
         return isNaN(numValue) ? null : numValue
       }
 
       case 'date': {
         const dateValue = new Date(trimmedValue)
         return isNaN(dateValue.getTime()) ? null : dateValue
       }
 
       case 'string':
       default:
         return trimmedValue
     }
   }
 }

packages/sources/ftse-sftp/src/parsing/factory.ts

@@ -6,7 +6,6 @@ import { RussellDailyValuesParser } from './russell'
  * Supported CSV parser types
  */
 export const instrumentToElementMap = {
-  FTSE100INDEX: 'UKX',
   Russell1000INDEX: 'Russell 1000® Index',
   Russell2000INDEX: 'Russell 2000® Index',
   Russell3000INDEX: 'Russell 3000® Index',

packages/sources/ftse-sftp/src/parsing/ftse100.ts

@@ -1,6 +1,16 @@
 import { BaseCSVParser } from './base-parser'
 import { ParsedData } from './interfaces'
 
+/**
+ * Spreadsheet consts
+ */
+const FTSE_100_INDEX_CODE = 'UKX'
+const FTSE_INDEX_CODE_COLUMN = 'Index Code'
+const FTSE_INDEX_SECTOR_NAME_COLUMN = 'Index/Sector Name'
+const FTSE_NUMBER_OF_CONSTITUENTS_COLUMN = 'Number of Constituents'
+const FTSE_INDEX_BASE_CURRENCY_COLUMN = 'Index Base Currency'
+const FTSE_GBP_INDEX_COLUMN = 'GBP Index'
+
 /**
  * Specific data structure for FTSE data
  * Based on the actual FTSE CSV format with Index Code, Index/Sector Name, Number of Constituents, Index Base Currency, and GBP Index
@@ -26,6 +36,7 @@ export class FTSE100Parser extends BaseCSVParser {
       trim: true,
       quote: '"',
       escape: '"',
+      relax_column_count: true, // Allow rows with different number of columns
     })
   }
 
@@ -42,35 +53,58 @@ export class FTSE100Parser extends BaseCSVParser {
     for (const row of parsed) {
       try {
         // Only include records where indexCode is "UKX" (FTSE 100 Index)
-        if (row['Index Code'] === 'UKX') {
-          const data: FTSE100Data = {
-            indexCode: this.convertValue(row['Index Code'], 'string') as string,
-            indexSectorName: this.convertValue(row['Index/Sector Name'], 'string') as string,
-            numberOfConstituents: this.convertValue(row['Number of Constituents'], 'number') as
-              | number
-              | null,
-            indexBaseCurrency: this.convertValue(row['Index Base Currency'], 'string') as string,
-            gbpIndex: this.convertValue(row['GBP Index'], 'number') as number | null,
-          }
-
-          // Additional validation for required fields
-          if (!data.indexCode || data.indexCode === '') {
-            console.warn(`Missing required Index Code field`)
-            continue
-          }
-
-          results.push(data)
+        if (row[FTSE_INDEX_CODE_COLUMN] != FTSE_100_INDEX_CODE) {
+          continue
         }
+
+        const data = this.createFTSE100Data(row)
+        results.push(data)
       } catch (error) {
         console.error(`Error parsing row:`, error)
       }
     }
 
+    if (results.length > 1) {
+      throw new Error('Multiple FTSE 100 index records found, expected only one')
+    }
+
     return results
   }
 
   /**
-   * Enhanced validation specific to FTSE format
+   * Creates FTSE100Data object from a CSV row
+   */
+  private createFTSE100Data(row: any): FTSE100Data {
+    // Validate that all required columns are present in the row
+    const requiredColumns = [
+      FTSE_INDEX_CODE_COLUMN,
+      FTSE_INDEX_SECTOR_NAME_COLUMN,
+      FTSE_NUMBER_OF_CONSTITUENTS_COLUMN,
+      FTSE_INDEX_BASE_CURRENCY_COLUMN,
+      FTSE_GBP_INDEX_COLUMN,
+    ]
+
+    const missingColumns = requiredColumns.filter((column) => row[column] === undefined)
+    if (missingColumns.length > 0) {
+      throw new Error(`Missing required columns in row: ${missingColumns.join(', ')}`)
+    }
+
+    return {
+      indexCode: this.convertValue(row[FTSE_INDEX_CODE_COLUMN], 'string') as string,
+      indexSectorName: this.convertValue(row[FTSE_INDEX_SECTOR_NAME_COLUMN], 'string') as string,
+      numberOfConstituents: this.convertValue(row[FTSE_NUMBER_OF_CONSTITUENTS_COLUMN], 'number') as
+        | number
+        | null,
+      indexBaseCurrency: this.convertValue(
+        row[FTSE_INDEX_BASE_CURRENCY_COLUMN],
+        'string',
+      ) as string,
+      gbpIndex: this.convertValue(row[FTSE_GBP_INDEX_COLUMN], 'number') as number | null,
+    }
+  }
+
+  /**
+   * Validate that the first 2 rows actually match the expected FTSE format
    */
   validateFormat(csvContent: string): boolean {
     if (!csvContent || csvContent.trim().length === 0) {
@@ -81,7 +115,7 @@ export class FTSE100Parser extends BaseCSVParser {
       // Parse from line 4 (header) to line 6 to validate the format
       const parsed = this.parseCSV(csvContent, {
         from_line: 4,
-        to_line: 6, // Parse header and a couple data rows for validation
+        to_line: 6,
         relax_column_count: true,
       })
 
@@ -114,7 +148,7 @@ export class FTSE100Parser extends BaseCSVParser {
 
       return true
     } catch (error) {
-      return false
+      throw new Error(`Error validating CSV format: ${error}`)
     }
   }
 }

packages/sources/ftse-sftp/src/parsing/index.ts

@@ -1,4 +1,4 @@
-export { CSVParser, ParsedData, CSVParserConfig, defaultCSVConfig } from './interfaces'
+export { CSVParser, ParsedData } from './interfaces'
 export { BaseCSVParser } from './base-parser'
 export { FTSE100Parser, FTSE100Data } from './ftse100'
 export { RussellDailyValuesParser, RussellDailyValuesData } from './russell'

packages/sources/ftse-sftp/src/parsing/interfaces.ts

@@ -23,32 +23,3 @@ export interface CSVParser {
 export interface ParsedData {
   [key: string]: string | number | Date | null
 }
-
-/**
- * Configuration options for CSV parsing using csv-parse library
- * Contains only the options we actually use in this project
- */
-export interface CSVParserConfig {
-  delimiter?: string
-  columns?: boolean | string[]
-  skip_empty_lines?: boolean
-  trim?: boolean
-  encoding?: BufferEncoding
-  from_line?: number
-  to_line?: number
-  relax_column_count?: boolean
-  quote?: string
-  escape?: string
-}
-
-/**
- * Base configuration with default values
- */
-export const defaultCSVConfig: CSVParserConfig = {
-  delimiter: ',',
-  columns: true,
-  skip_empty_lines: true,
-  trim: true,
-  encoding: 'utf8',
-  relax_column_count: true, // Allow rows with different column counts
-}