Revision #3

Author: chray-zhang

package.json

@@ -60,7 +60,7 @@
     "@types/babel__preset-env": "7.9.7",
     "@types/eslint": "8.56.12",
     "@types/jest": "^29.5.14",
-    "@types/node": "^24.3.1",
+    "@types/node": "22.14.1",
     "husky": "7.0.4",
     "jest": "^29.7.0",
     "lint-staged": "11.2.6",

packages/sources/ftse-sftp/src/parsing/base-parser.ts

@@ -6,9 +6,9 @@ import { parse, Options } from 'csv-parse/sync'
  * Uses the csv-parse library for robust CSV parsing
  */
 export abstract class BaseCSVParser implements CSVParser {
-  protected config: Record<string, any>
+  protected config: Options
 
-  constructor(config: Record<string, any> = {}) {
+  constructor(config: Options = {}) {
     this.config = { ...config }
   }
 
@@ -17,12 +17,6 @@ export abstract class BaseCSVParser implements CSVParser {
    */
   abstract parse(csvContent: string): Promise<ParsedData[]>
 
-  /**
-   * Abstract method that must be implemented by concrete classes
-   * Should validate the CSV content format and structure
-   */
-  abstract validateFormat(csvContent: string): boolean
-
   /**
    * Helper method to parse CSV content using csv-parse library
    */
@@ -49,18 +43,18 @@ export abstract class BaseCSVParser implements CSVParser {
 
     switch (expectedType) {
       case 'number': {
-        const numValue = parseFloat(trimmedValue.replace(/,/g, ''))
+        const numValue = parseFloat(value.replace(/,/g, ''))
         return isNaN(numValue) ? null : numValue
       }
 
       case 'date': {
-        const dateValue = new Date(trimmedValue)
+        const dateValue = new Date(value)
         return isNaN(dateValue.getTime()) ? null : dateValue
       }
 
       case 'string':
       default:
-        return trimmedValue
+        return value
     }
   }
 }

packages/sources/ftse-sftp/src/parsing/factory.ts

@@ -23,9 +23,7 @@ export class CSVParserFactory {
       case 'FTSE100INDEX':
         return new FTSE100Parser()
       case 'Russell1000INDEX':
-        return new RussellDailyValuesParser(instrumentToElementMap[instrument])
       case 'Russell2000INDEX':
-        return new RussellDailyValuesParser(instrumentToElementMap[instrument])
       case 'Russell3000INDEX':
         return new RussellDailyValuesParser(instrumentToElementMap[instrument])
       default:

packages/sources/ftse-sftp/src/parsing/ftse100.ts

@@ -10,6 +10,7 @@ const FTSE_INDEX_SECTOR_NAME_COLUMN = 'Index/Sector Name'
 const FTSE_NUMBER_OF_CONSTITUENTS_COLUMN = 'Number of Constituents'
 const FTSE_INDEX_BASE_CURRENCY_COLUMN = 'Index Base Currency'
 const FTSE_GBP_INDEX_COLUMN = 'GBP Index'
+const FIRST_DATA_ROW = 4 // Start from header line (line 4)
 
 /**
  * Specific data structure for FTSE data
@@ -41,31 +42,20 @@ export class FTSE100Parser extends BaseCSVParser {
   }
 
   async parse(csvContent: string): Promise<FTSE100Data[]> {
-    if (!this.validateFormat(csvContent)) {
-      throw new Error('Invalid CSV format for FTSE data')
-    }
-
     const parsed = this.parseCSV(csvContent, {
-      from_line: 4, // Start parsing from line 4 (includes header)
+      from_line: FIRST_DATA_ROW, // Start parsing from line 5 (includes header)
     })
-    const results: FTSE100Data[] = []
-
-    for (const row of parsed) {
-      try {
-        // Only include records where indexCode is "UKX" (FTSE 100 Index)
-        if (row[FTSE_INDEX_CODE_COLUMN] != FTSE_100_INDEX_CODE) {
-          continue
-        }
 
-        const data = this.createFTSE100Data(row)
-        results.push(data)
-      } catch (error) {
-        console.error(`Error parsing row:`, error)
-      }
-    }
+    const results: FTSE100Data[] = parsed
+      .filter((row) => {
+        return row[FTSE_INDEX_CODE_COLUMN] === FTSE_100_INDEX_CODE
+      })
+      .map((row) => this.createFTSE100Data(row))
 
     if (results.length > 1) {
       throw new Error('Multiple FTSE 100 index records found, expected only one')
+    } else if (results.length === 0) {
+      throw new Error('No FTSE 100 index record found')
     }
 
     return results
@@ -102,53 +92,4 @@ export class FTSE100Parser extends BaseCSVParser {
       gbpIndex: this.convertValue(row[FTSE_GBP_INDEX_COLUMN], 'number') as number | null,
     }
   }
-
-  /**
-   * Validate that the first 2 rows actually match the expected FTSE format
-   */
-  validateFormat(csvContent: string): boolean {
-    if (!csvContent || csvContent.trim().length === 0) {
-      return false
-    }
-
-    try {
-      // Parse from line 4 (header) to line 6 to validate the format
-      const parsed = this.parseCSV(csvContent, {
-        from_line: 4,
-        to_line: 6,
-        relax_column_count: true,
-      })
-
-      if (!parsed || parsed.length === 0) {
-        return false
-      }
-
-      // Check if we can access the expected columns from the first data row
-      const firstDataRow = parsed[0]
-      if (!firstDataRow) {
-        console.error('No data rows found in CSV for validation')
-        return false
-      }
-
-      const requiredColumns = [
-        'Index Code',
-        'Index/Sector Name',
-        'Number of Constituents',
-        'Index Base Currency',
-        'GBP Index',
-      ]
-
-      const missingColumns = requiredColumns.filter((column) => firstDataRow[column] === undefined)
-
-      if (missingColumns.length > 0) {
-        console.error(`Missing required columns in FTSE CSV: ${missingColumns.join(', ')}`)
-        console.error(`Available columns: ${Object.keys(firstDataRow).join(', ')}`)
-        return false
-      }
-
-      return true
-    } catch (error) {
-      throw new Error(`Error validating CSV format: ${error}`)
-    }
-  }
 }

packages/sources/ftse-sftp/src/parsing/interfaces.ts

@@ -8,13 +8,6 @@ export interface CSVParser {
    * @returns Promise of parsed data
    */
   parse(csvContent: string): Promise<ParsedData[]>
-
-  /**
-   * Validate the CSV format
-   * @param csvContent - Raw CSV content as string
-   * @returns boolean indicating if the format is valid
-   */
-  validateFormat(csvContent: string): boolean
 }
 
 /**

packages/sources/ftse-sftp/src/parsing/russell.ts

@@ -3,7 +3,8 @@ import { ParsedData } from './interfaces'
 
 // Column indices for Russell CSV format
 const INDEX_NAME_COLUMN = 0
-const VALUE_COLUMN = 4
+const CLOSE_VALUE_COLUMN = 4
+const FIRST_DATA_ROW = 7
 
 /**
  * Specific data structure for Russell Daily Values data
@@ -32,104 +33,33 @@ export class RussellDailyValuesParser extends BaseCSVParser {
   }
 
   async parse(csvContent: string): Promise<RussellDailyValuesData[]> {
-    const results: RussellDailyValuesData[] = []
-
-    if (!this.validateFormat(csvContent)) {
-      throw new Error('Invalid CSV format for Russell data')
-    }
-
     // Russell data starts after the header rows, which vary in position
     // Parse the entire CSV and find Russell rows dynamically
     const parsed = this.parseCSV(csvContent, {
+      from_line: FIRST_DATA_ROW, // Start parsing from line 7 (includes header)
       relax_column_count: true, // Allow rows with different number of columns
     })
 
-    for (const row of parsed) {
-      if (!row || row.length < 5) {
-        // Need at least 5 columns for index name and close value
-        continue // Skip rows with insufficient fields
-      }
-
-      // Skip empty rows (CSV contains separator rows with multiple empty fields)
-      const hasContent = row.some((field: any) => field && String(field).trim() !== '')
-      if (!hasContent) {
-        continue // Skip empty rows
-      }
-
-      const indexName = this.convertValue(row[INDEX_NAME_COLUMN], 'string') as string
-
-      // Only process rows that start with "Russell" and contain the ® symbol
-      if (
-        !indexName ||
-        !indexName.includes('Russell') ||
-        (!indexName.includes('®') && !indexName.includes('�'))
-      ) {
-        continue
-      }
-
-      const data: RussellDailyValuesData = {
-        indexName,
-        close: this.convertValue(row[VALUE_COLUMN], 'number') as number | null,
-      }
-
-      // Filter by instrument if specified
-      if (this.instrument) {
-        // Normalize both strings for comparison (remove special characters and extra spaces)
-        const normalizeString = (str: string) =>
-          str.replace(/[®�™]/g, '').replace(/\s+/g, ' ').trim()
-        const normalizedIndexName = normalizeString(indexName)
-        const normalizedInstrument = normalizeString(this.instrument)
+    const results: RussellDailyValuesData[] = parsed
+      .filter((row) => row.length > CLOSE_VALUE_COLUMN) // Keep rows with enough columns
+      .map((row) => ({
+        row,
+        indexName: this.convertValue(row[INDEX_NAME_COLUMN], 'string') as string,
+      }))
+      .filter(({ indexName }) => indexName && indexName === this.instrument) // Only process rows that match the instrument
+      .map(
+        ({ row, indexName }): RussellDailyValuesData => ({
+          indexName,
+          close: this.convertValue(row[CLOSE_VALUE_COLUMN], 'number') as number | null,
+        }),
+      )
 
-        if (normalizedIndexName === normalizedInstrument) {
-          results.push(data)
-        }
-      } else {
-        results.push(data)
-      }
+    if (results.length === 0) {
+      throw new Error('No matching Russell index records found')
+    } else if (results.length > 1) {
+      throw new Error('Multiple matching Russell index records found, expected only one')
     }
 
     return results
   }
-
-  /**
-   * Validate that the CSV contains Russell index data
-   */
-  validateFormat(csvContent: string): boolean {
-    if (!csvContent || csvContent.trim().length === 0) {
-      return false
-    }
-
-    try {
-      // Parse the entire CSV with relaxed column count to find Russell data
-      const parsed = this.parseCSV(csvContent, {
-        relax_column_count: true,
-      })
-
-      if (!parsed || parsed.length === 0) {
-        console.error('No data rows found in CSV for Russell validation')
-        return false
-      }
-
-      // Check if any row contains valid Russell index data
-      const hasValidRussellData = parsed.some(
-        (row) =>
-          row &&
-          row.length >= 5 && // Must have at least 5 columns for index name and close value
-          row[INDEX_NAME_COLUMN] &&
-          String(row[INDEX_NAME_COLUMN]).includes('Russell') &&
-          (String(row[INDEX_NAME_COLUMN]).includes('®') ||
-            String(row[INDEX_NAME_COLUMN]).includes('�')),
-      )
-
-      if (!hasValidRussellData) {
-        console.error('No valid Russell index data found in CSV validation')
-        return false
-      }
-
-      return true
-    } catch (error) {
-      console.error('Error during Russell CSV validation:', error)
-      return false
-    }
-  }
 }