[ML] Switch file data visualizer to use Papa Parse for CSV parsing (#24329)

Author: peteharverson

x-pack/package.json

@@ -154,7 +154,6 @@
     "concat-stream": "1.5.1",
     "cookie": "^0.3.1",
     "copy-to-clipboard": "^3.0.8",
-    "csv-parse": "^3.1.2",
     "d3": "3.5.6",
     "d3-scale": "1.0.6",
     "dataloader": "^1.4.0",

x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_importer.js

@@ -6,8 +6,7 @@
 
 
 import { Importer } from './importer';
-import { parse } from './csv_parse';
-// import csvParse from 'csv-parse';
+import Papa from 'papaparse';
 
 export class CsvImporter extends Importer {
   constructor(results) {
@@ -21,57 +20,33 @@ export class CsvImporter extends Importer {
   }
 
   async read(csv) {
-    console.log('read delimited file');
-    console.time('read delimited file');
-
     try {
-      const settings = {
-        skip_empty_lines: true,
+      const config = {
+        header: false,
+        skipEmptyLines: 'greedy',
         delimiter: this.delimiter,
-        quote: this.quote,
-        columns: this.columnNames,
-        relax_column_count: true,
+        quoteChar: this.quote,
       };
 
-      // commented out code left in for future work to optimise
-      // csv parsing
-
-      // const parser = csvParse(settings);
-      // parser.write(csv);
-      // parser.end();
-
-      this.data = await parse(csv, settings);
-
-      // const output = [];
-      // parser.on('readable', () => {
-      //   let record;
-      //   while (record = parser.read()) {
-      //     output.push(record);
-      //   }
-      // });
+      const parserOutput = Papa.parse(csv, config);
 
-      // parser.on('error', (err) => {
-      //   console.error(err.message);
-      // });
+      if (parserOutput.errors.length) {
+        // throw an error with the message of the first error encountered
+        throw parserOutput.errors[0].message;
+      }
 
-      // parser.on('end', () => {
-      //   console.log('output', output);
-      //   console.timeEnd('read file');
-      // });
+      this.data = parserOutput.data;
 
       if (this.hasHeaderRow) {
         this.data.shift();
       }
 
-      this.docArray = this.data;
-      // this.docArray = formatToJson(this.data, this.columnNames);
-      console.timeEnd('read delimited file');
+      this.docArray = formatToJson(this.data, this.columnNames);
 
       return {
         success: true,
       };
     } catch (error) {
-      console.timeEnd('read delimited file');
       return {
         success: false,
         error,
@@ -80,15 +55,15 @@ export class CsvImporter extends Importer {
   }
 }
 
-// function formatToJson(data, columnNames) {
-//   const docArray = [];
-//   for (let i = 0; i < data.length; i++) {
-//     const line = {};
-//     for (let c = 0; c < columnNames.length; c++) {
-//       const col = columnNames[c];
-//       line[col] = data[i][c];
-//     }
-//     docArray.push(line);
-//   }
-//   return docArray;
-// }
+function formatToJson(data, columnNames) {
+  const docArray = [];
+  for (let i = 0; i < data.length; i++) {
+    const line = {};
+    for (let c = 0; c < columnNames.length; c++) {
+      const col = columnNames[c];
+      line[col] = data[i][c];
+    }
+    docArray.push(line);
+  }
+  return docArray;
+}

x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_parse.js

@@ -2803,11 +2803,6 @@ csstype@^2.2.0:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.5.5.tgz#4125484a3d42189a863943f23b9e4b80fedfa106"
   integrity sha512-EGMjeoiN3aqEX5u/cyH5mSdGBDGdLcCQvcEcBWNGFSPXKd9uOTIeVG91YQ22OxI44DKpvI+4C7VUSmEpsHWJaA==
 
-csv-parse@^3.1.2:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-3.1.2.tgz#51bfd2d39b13e66f6da8656360b8ec82ca159494"
-  integrity sha512-NtcS4h3rLcwawpJJDRolYeuQW/2RvutpHpzbokbk5xMCNEDSyvAK3SmuCtd6tKOftKiMa2wn4viOU32A/5VDTQ==
-
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"

x-pack/yarn.lock

@@ -4331,11 +4331,6 @@ csstype@^2.2.0:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.5.2.tgz#4534308476ceede8fbe148b9b99f9baf1c80fa06"
   integrity sha512-epoHrKPywwUhFTXadQNA5PPx4GChtkM03SkBeFZTaFtJcn6QfXpBkX2IAc4J9Oe18nha5NrTOo2OMOge+qH1mw==
 
-csv-parse@^3.1.2:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-3.1.3.tgz#b96227e85e308f685133cdb2b79ec9a03fde3c1c"
-  integrity sha512-fs+nzn0lU+e7uZHkY/SvEYiRT3PCvb746csfl238iUXh1D+gzzN4crzgZ7gkVDutoPWYwfrFSQaSAulSeynTlg==
-
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"