From 288800bd4f3d5c5b9ba76129725b435eec3c1869 Mon Sep 17 00:00:00 2001 From: Juanjo Diaz Date: Wed, 29 Jun 2022 18:12:39 +0200 Subject: [PATCH] perf: bring back v5 perf improvements and add some new ones --- packages/formatters/src/number.js | 16 ++++----------- packages/formatters/src/string.js | 2 +- packages/parsers/src/BaseParser.js | 22 +++++++++++--------- packages/parsers/src/Parser.js | 29 +++++++++++++++------------ packages/parsers/src/StreamParser.js | 4 ++-- packages/parsers/src/utils.js | 30 +++++++++++++++++++++++++++- packages/transforms/src/unwind.js | 5 +++-- packages/transforms/src/utils.js | 11 ++++++++++ 8 files changed, 79 insertions(+), 40 deletions(-) diff --git a/packages/formatters/src/number.js b/packages/formatters/src/number.js index 8ae9b8e..019dc11 100644 --- a/packages/formatters/src/number.js +++ b/packages/formatters/src/number.js @@ -1,24 +1,16 @@ -function toFixedDecimals(value, decimals) { - return value.toFixed(decimals); -} - -function replaceSeparator(value, separator) { - return value.replace('.', separator); -} - export default function numberFormatter(opts = {}) { if (opts.separator) { if (opts.decimals) { return (value) => - replaceSeparator(toFixedDecimals(value, opts.decimals), opts.separator); + value.toFixed(opts.decimals).replace('.', opts.separator); } - return (value) => replaceSeparator(value.toString(), opts.separator); + return (value) => `${value}`.replace('.', opts.separator); } if (opts.decimals) { - return (value) => toFixedDecimals(value, opts.decimals); + return (value) => value.toFixed(opts.decimals); } - return (value) => value.toString(); + return (value) => `${value}`; } diff --git a/packages/formatters/src/string.js b/packages/formatters/src/string.js index 35b0b0d..48e4ea0 100644 --- a/packages/formatters/src/string.js +++ b/packages/formatters/src/string.js @@ -5,7 +5,7 @@ export default function stringFormatter(opts = {}) { ? opts.escapedQuote : `${quote}${quote}`; - if (!quote) { + if (!quote || quote === escapedQuote) { return (value) => value; } diff --git a/packages/parsers/src/BaseParser.js b/packages/parsers/src/BaseParser.js index fdf1d46..081a6ac 100644 --- a/packages/parsers/src/BaseParser.js +++ b/packages/parsers/src/BaseParser.js @@ -1,10 +1,10 @@ import lodashGet from 'lodash.get'; -import { getProp } from './utils.js'; import defaultFormatter from '@json2csv/formatters/default'; import numberFormatterCtor from '@json2csv/formatters/number'; import stringFormatterCtor from '@json2csv/formatters/string'; import symbolFormatterCtor from '@json2csv/formatters/symbol'; import objectFormatterCtor from '@json2csv/formatters/object'; +import { getProp, flattenReducer, fastJoin } from './utils.js'; export default class JSON2CSVBase { constructor(opts) { @@ -118,10 +118,13 @@ export default class JSON2CSVBase { * * @returns {String} titles as a string */ - getHeader(fields) { - return fields - .map((fieldInfo) => this.opts.formatters.header(fieldInfo.label)) - .join(this.opts.delimiter); + getHeader() { + return fastJoin( + this.opts.fields.map((fieldInfo) => + this.opts.formatters.header(fieldInfo.label) + ), + this.opts.delimiter + ); } /** @@ -130,7 +133,8 @@ export default class JSON2CSVBase { */ preprocessRow(row) { return this.opts.transforms.reduce( - (rows, transform) => rows.flatMap((row) => transform(row)), + (rows, transform) => + rows.map((row) => transform(row)).reduce(flattenReducer, []), [row] ); } @@ -141,12 +145,12 @@ export default class JSON2CSVBase { * @param {Object} row JSON object to be converted in a CSV row * @returns {String} CSV string (row) */ - processRow(row, fields) { + processRow(row) { if (!row) { return undefined; } - const processedRow = fields.map((fieldInfo) => + const processedRow = this.opts.fields.map((fieldInfo) => this.processCell(row, fieldInfo) ); @@ -157,7 +161,7 @@ export default class JSON2CSVBase { return undefined; } - return processedRow.join(this.opts.delimiter); + return fastJoin(processedRow, this.opts.delimiter); } /** diff --git a/packages/parsers/src/Parser.js b/packages/parsers/src/Parser.js index 6d1e48d..38151ae 100644 --- a/packages/parsers/src/Parser.js +++ b/packages/parsers/src/Parser.js @@ -1,4 +1,5 @@ import JSON2CSVBase from './BaseParser.js'; +import { flattenReducer, fastJoin } from './utils.js'; export default class JSON2CSVParser extends JSON2CSVBase { constructor(opts) { @@ -11,12 +12,12 @@ export default class JSON2CSVParser extends JSON2CSVBase { * @returns {String} The CSV formated data as a string */ parse(data) { - const processedData = this.preprocessData(data, this.opts.fields); + const preprocessedData = this.preprocessData(data); - const fields = + this.opts.fields = this.opts.fields || this.preprocessFieldsInfo( - processedData.reduce((fields, item) => { + preprocessedData.reduce((fields, item) => { Object.keys(item).forEach((field) => { if (!fields.includes(field)) { fields.push(field); @@ -27,8 +28,8 @@ export default class JSON2CSVParser extends JSON2CSVBase { }, []) ); - const header = this.opts.header ? this.getHeader(fields) : ''; - const rows = this.processData(processedData, fields); + const header = this.opts.header ? this.getHeader() : ''; + const rows = this.processData(preprocessedData); const csv = (this.opts.withBOM ? '\ufeff' : '') + header + @@ -44,11 +45,11 @@ export default class JSON2CSVParser extends JSON2CSVBase { * * @param {Array|Object} data Array or object to be converted to CSV */ - preprocessData(data, fields) { + preprocessData(data) { const processedData = Array.isArray(data) ? data : [data]; if ( - !fields && + !this.opts.fields && (processedData.length === 0 || typeof processedData[0] !== 'object') ) { throw new Error( @@ -58,7 +59,9 @@ export default class JSON2CSVParser extends JSON2CSVBase { if (this.opts.transforms.length === 0) return processedData; - return processedData.flatMap((row) => this.preprocessRow(row)); + return processedData + .map((row) => this.preprocessRow(row)) + .reduce(flattenReducer, []); } /** @@ -67,10 +70,10 @@ export default class JSON2CSVParser extends JSON2CSVBase { * @param {Array} data Array of JSON objects to be converted to CSV * @returns {String} CSV string (body) */ - processData(data, fields) { - return data - .map((row) => this.processRow(row, fields)) - .filter((row) => row) // Filter empty rows - .join(this.opts.eol); + processData(data) { + return fastJoin( + data.map((row) => this.processRow(row)).filter((row) => row), // Filter empty rows + this.opts.eol + ); } } diff --git a/packages/parsers/src/StreamParser.js b/packages/parsers/src/StreamParser.js index d6db293..108a02b 100644 --- a/packages/parsers/src/StreamParser.js +++ b/packages/parsers/src/StreamParser.js @@ -124,7 +124,7 @@ export default class JSON2CSVStreamParser extends JSON2CSVBase { } if (this.opts.header) { - const header = this.getHeader(this.opts.fields); + const header = this.getHeader(); this.onHeader(header); this.onData(header); this._hasWritten = true; @@ -147,7 +147,7 @@ export default class JSON2CSVStreamParser extends JSON2CSVBase { } processedData.forEach((row) => { - const line = this.processRow(row, this.opts.fields); + const line = this.processRow(row); if (line === undefined) return; this.onLine(line); this.onData(this._hasWritten ? this.opts.eol + line : line); diff --git a/packages/parsers/src/utils.js b/packages/parsers/src/utils.js index 68e2a9d..0592034 100644 --- a/packages/parsers/src/utils.js +++ b/packages/parsers/src/utils.js @@ -1,3 +1,31 @@ export function getProp(obj, path, defaultValue) { - return obj[path] === undefined ? defaultValue : obj[path]; + const value = obj[path]; + return value === undefined ? defaultValue : value; +} + +export function flattenReducer(acc, arr) { + try { + // This is faster but susceptible to `RangeError: Maximum call stack size exceeded` + acc.push(...arr); + return acc; + } catch (err) { + // Fallback to a slower but safer option + return acc.concat(arr); + } +} + +export function fastJoin(arr, separator) { + let isFirst = true; + return arr.reduce((acc, elem) => { + if (elem === null || elem === undefined) { + elem = ''; + } + + if (isFirst) { + isFirst = false; + return `${elem}`; + } + + return `${acc}${separator}${elem}`; + }, ''); } diff --git a/packages/transforms/src/unwind.js b/packages/transforms/src/unwind.js index 1acc68f..2e937e1 100644 --- a/packages/transforms/src/unwind.js +++ b/packages/transforms/src/unwind.js @@ -1,5 +1,5 @@ import lodashGet from 'lodash.get'; -import { setProp, unsetProp } from './utils.js'; +import { setProp, unsetProp, flattenReducer } from './utils.js'; function getUnwindablePaths(obj, currentPath) { return Object.keys(obj).reduce((unwindablePaths, key) => { @@ -20,7 +20,8 @@ function getUnwindablePaths(obj, currentPath) { unwindablePaths.push(newPath); unwindablePaths = unwindablePaths.concat( value - .flatMap((arrObj) => getUnwindablePaths(arrObj, newPath)) + .map((arrObj) => getUnwindablePaths(arrObj, newPath)) + .reduce(flattenReducer, []) .filter((item, index, arr) => arr.indexOf(item) !== index) ); } diff --git a/packages/transforms/src/utils.js b/packages/transforms/src/utils.js index 792404a..b060115 100644 --- a/packages/transforms/src/utils.js +++ b/packages/transforms/src/utils.js @@ -32,3 +32,14 @@ export function unsetProp(obj, path) { {} ); } + +export function flattenReducer(acc, arr) { + try { + // This is faster but susceptible to `RangeError: Maximum call stack size exceeded` + acc.push(...arr); + return acc; + } catch (err) { + // Fallback to a slower but safer option + return acc.concat(arr); + } +}