diff --git a/x-pack/package.json b/x-pack/package.json index 9c20ebc3efe37..6186ad5b4959c 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -154,7 +154,6 @@ "concat-stream": "1.5.1", "cookie": "^0.3.1", "copy-to-clipboard": "^3.0.8", - "csv-parse": "^3.1.2", "d3": "3.5.6", "d3-scale": "1.0.6", "dataloader": "^1.4.0", diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_importer.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_importer.js index af389e217862a..a9aa52f104c9a 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_importer.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_importer.js @@ -6,8 +6,7 @@ import { Importer } from './importer'; -import { parse } from './csv_parse'; -// import csvParse from 'csv-parse'; +import Papa from 'papaparse'; export class CsvImporter extends Importer { constructor(results) { @@ -21,57 +20,33 @@ export class CsvImporter extends Importer { } async read(csv) { - console.log('read delimited file'); - console.time('read delimited file'); - try { - const settings = { - skip_empty_lines: true, + const config = { + header: false, + skipEmptyLines: 'greedy', delimiter: this.delimiter, - quote: this.quote, - columns: this.columnNames, - relax_column_count: true, + quoteChar: this.quote, }; - // commented out code left in for future work to optimise - // csv parsing - - // const parser = csvParse(settings); - // parser.write(csv); - // parser.end(); - - this.data = await parse(csv, settings); - - // const output = []; - // parser.on('readable', () => { - // let record; - // while (record = parser.read()) { - // output.push(record); - // } - // }); + const parserOutput = Papa.parse(csv, config); - // parser.on('error', (err) => { - // console.error(err.message); - // }); + if (parserOutput.errors.length) { + // throw an error with the message of the first error encountered + throw parserOutput.errors[0].message; + } - // parser.on('end', () => { - // console.log('output', output); - // console.timeEnd('read file'); - // }); + this.data = parserOutput.data; if (this.hasHeaderRow) { this.data.shift(); } - this.docArray = this.data; - // this.docArray = formatToJson(this.data, this.columnNames); - console.timeEnd('read delimited file'); + this.docArray = formatToJson(this.data, this.columnNames); return { success: true, }; } catch (error) { - console.timeEnd('read delimited file'); return { success: false, error, @@ -80,15 +55,15 @@ export class CsvImporter extends Importer { } } -// function formatToJson(data, columnNames) { -// const docArray = []; -// for (let i = 0; i < data.length; i++) { -// const line = {}; -// for (let c = 0; c < columnNames.length; c++) { -// const col = columnNames[c]; -// line[col] = data[i][c]; -// } -// docArray.push(line); -// } -// return docArray; -// } +function formatToJson(data, columnNames) { + const docArray = []; + for (let i = 0; i < data.length; i++) { + const line = {}; + for (let c = 0; c < columnNames.length; c++) { + const col = columnNames[c]; + line[col] = data[i][c]; + } + docArray.push(line); + } + return docArray; +} diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_parse.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_parse.js deleted file mode 100644 index 57d6f8e8290cb..0000000000000 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/csv_parse.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - -// promisifying the csv-parse parse function - -import csvParse from 'csv-parse'; - -export function parse(data, settings) { - return new Promise((resolve, reject) => { - csvParse(data, settings, (err, out) => { - if (err !== null) { - reject(err); - } else { - resolve(out); - } - }); - }); -} diff --git a/x-pack/yarn.lock b/x-pack/yarn.lock index d607b53f592a7..dba91eff930c2 100644 --- a/x-pack/yarn.lock +++ b/x-pack/yarn.lock @@ -2803,11 +2803,6 @@ csstype@^2.2.0: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.5.5.tgz#4125484a3d42189a863943f23b9e4b80fedfa106" integrity sha512-EGMjeoiN3aqEX5u/cyH5mSdGBDGdLcCQvcEcBWNGFSPXKd9uOTIeVG91YQ22OxI44DKpvI+4C7VUSmEpsHWJaA== -csv-parse@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-3.1.2.tgz#51bfd2d39b13e66f6da8656360b8ec82ca159494" - integrity sha512-NtcS4h3rLcwawpJJDRolYeuQW/2RvutpHpzbokbk5xMCNEDSyvAK3SmuCtd6tKOftKiMa2wn4viOU32A/5VDTQ== - currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" diff --git a/yarn.lock b/yarn.lock index 6f9428338d08a..2446eb2e3642c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4331,11 +4331,6 @@ csstype@^2.2.0: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.5.2.tgz#4534308476ceede8fbe148b9b99f9baf1c80fa06" integrity sha512-epoHrKPywwUhFTXadQNA5PPx4GChtkM03SkBeFZTaFtJcn6QfXpBkX2IAc4J9Oe18nha5NrTOo2OMOge+qH1mw== -csv-parse@^3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-3.1.3.tgz#b96227e85e308f685133cdb2b79ec9a03fde3c1c" - integrity sha512-fs+nzn0lU+e7uZHkY/SvEYiRT3PCvb746csfl238iUXh1D+gzzN4crzgZ7gkVDutoPWYwfrFSQaSAulSeynTlg== - currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"