From 50062c3e155ff2c12b1bb417085188a2156885a8 Mon Sep 17 00:00:00 2001 From: Juanjo Diaz Date: Thu, 28 Jan 2021 04:32:29 +0200 Subject: [PATCH] fix: simplify stringExcel formatter and support proper escaping (#513) --- README.md | 12 +++------ bin/json2csv.js | 13 +++++----- lib/formatters/stringExcel.js | 7 ++--- test/CLI.js | 11 ++++++++ test/JSON2CSVAsyncParser.js | 21 ++++++++++++++- test/JSON2CSVParser.js | 16 +++++++++++- test/JSON2CSVTransform.js | 26 ++++++++++++++++++- .../csv/excelStringsWithEscapedQuoted.csv | 3 +++ 8 files changed, 88 insertions(+), 21 deletions(-) create mode 100644 test/fixtures/csv/excelStringsWithEscapedQuoted.csv diff --git a/README.md b/README.md index c076859f..8ada41d3 100644 --- a/README.md +++ b/README.md @@ -363,17 +363,11 @@ The formatter needs to be instantiated and takes an options object as arguments Converts string data into normalized Excel style data after formatting it using the given string formatter. -The formatter needs to be instantiated and takes an options object as arguments containing: - -- `stringFormatter` - Boolean, whether to flatten JSON objects or not. Defaults to our built-in `stringFormatter`. +The formatter needs to be instantiated and takes no arguments. ```js { - // Uses the default string formatter - string: stringExcelFormatter(), - - // Uses custom string formatter - string: stringExcelFormatter(myStringFormatter()), + string: stringExcelFormatter, } ``` @@ -614,7 +608,7 @@ should be replaced by const { Parser, formatter: { stringExcel: stringExcelFormatter } } = require('json2csv'); const json2csvParser = new Parser({ formatters: { - string: stringExcelFormatter(stringFormatter({ quote: '\'', escapedQuote: '\\\'' }))), + string: stringExcelFormatter, } }); const csv = json2csvParser.parse(myData); diff --git a/bin/json2csv.js b/bin/json2csv.js index da472f66..539e4e46 100755 --- a/bin/json2csv.js +++ b/bin/json2csv.js @@ -16,7 +16,7 @@ const readFile = promisify(readFileOrig); const writeFile = promisify(writeFileOrig); const { unwind, flatten } = json2csv.transforms; -const { string: stringFormatterCtr, stringExcel: stringExcelFormatter } = json2csv.formatters; +const { string: stringFormatter, stringExcel: stringExcelFormatter } = json2csv.formatters; const JSON2CSVParser = json2csv.Parser; const Json2csvTransform = json2csv.Transform; @@ -151,12 +151,13 @@ async function processStream(config, opts) { })); } - const stringFormatter = stringFormatterCtr({ - quote: config.quote, - escapedQuote: config.escapedQuote, - }); const formatters = { - string: config.excelStrings ? stringExcelFormatter({ stringFormatter }) : stringFormatter + string: config.excelStrings + ? stringExcelFormatter + : stringFormatter({ + quote: config.quote, + escapedQuote: config.escapedQuote, + }) }; const opts = { diff --git a/lib/formatters/stringExcel.js b/lib/formatters/stringExcel.js index 3b59bb20..1d5784c6 100644 --- a/lib/formatters/stringExcel.js +++ b/lib/formatters/stringExcel.js @@ -1,7 +1,8 @@ -const defaulStringFormatter = require('./string'); +const quote = '"'; +const escapedQuote = '""""'; -function stringExcel(opts = { stringFormatter: defaulStringFormatter() }) { - return (value) => `"="${opts.stringFormatter(value)}""`; +function stringExcel(value) { + return `"=""${value.replace(new RegExp(quote, 'g'), escapedQuote)}"""`; } module.exports = stringExcel; diff --git a/test/CLI.js b/test/CLI.js index 2f0be8c3..969daec4 100644 --- a/test/CLI.js +++ b/test/CLI.js @@ -801,6 +801,17 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); }); + testRunner.add('should format strings to force excel to view the values as strings with escaped quotes', (t) => { + const opts = '--excel-strings'; + + exec(`${cli} -i "${getFixturePath('/json/quotes.json')}" ${opts}`, (err, stdout, stderr) => { + t.notOk(stderr); + const csv = stdout; + t.equal(csv, csvFixtures.excelStringsWithEscapedQuoted); + t.end(); + }); + }); + // String Escaping and preserving values testRunner.add('should parse JSON values with trailing backslashes', (t) => { diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js index ef9d9d1c..fb2787e6 100644 --- a/test/JSON2CSVAsyncParser.js +++ b/test/JSON2CSVAsyncParser.js @@ -1202,7 +1202,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = const opts = { fields: ['carModel', 'price', 'color'], formatters: { - string: stringExcelFormatter() + string: stringExcelFormatter } }; const parser = new AsyncParser(opts); @@ -1217,6 +1217,25 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = t.end(); }); + testRunner.add('should format strings to force excel to view the values as strings with escaped quotes', async (t) => { + const opts = { + formatters: { + string: stringExcelFormatter + } + }; + const parser = new AsyncParser(opts); + + try { + const csv = await parser.parse(jsonFixtures.quotes()).promise(); + t.equal(csv, csvFixtures.excelStringsWithEscapedQuoted); + } catch(err) { + t.fail(err.message); + } + + t.end(); + }); + + // String Escaping and preserving values testRunner.add('should parse JSON values with trailing backslashes', async (t) => { diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js index 37b3eb41..0aa98e24 100644 --- a/test/JSON2CSVParser.js +++ b/test/JSON2CSVParser.js @@ -875,7 +875,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { const opts = { fields: ['carModel', 'price', 'color'], formatters: { - string: stringExcelFormatter() + string: stringExcelFormatter } }; @@ -886,6 +886,20 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.end(); }); + testRunner.add('should format strings to force excel to view the values as strings with escaped quotes', (t) => { + const opts = { + formatters: { + string: stringExcelFormatter + } + }; + + const parser = new Json2csvParser(opts); + const csv = parser.parse(jsonFixtures.quotes); + + t.equal(csv, csvFixtures.excelStringsWithEscapedQuoted); + t.end(); + }); + // String Escaping and preserving values testRunner.add('should parse JSON values with trailing backslashes', (t) => { diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js index 4c9195dd..61da8f40 100644 --- a/test/JSON2CSVTransform.js +++ b/test/JSON2CSVTransform.js @@ -1381,7 +1381,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = const opts = { fields: ['carModel', 'price', 'color'], formatters: { - string: stringExcelFormatter() + string: stringExcelFormatter } }; @@ -1401,6 +1401,30 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = }); }); + testRunner.add('should format strings to force excel to view the values as strings with escaped quotes', (t) => { + const opts = { + formatters: { + string: stringExcelFormatter + } + }; + + const transform = new Json2csvTransform(opts); + const processor = jsonFixtures.quotes().pipe(transform); + + let csv = ''; + processor + .on('data', chunk => (csv += chunk.toString())) + .on('end', () => { + t.equal(csv, csvFixtures.excelStringsWithEscapedQuoted); + t.end(); + }) + .on('error', err => { + t.fail(err.message); + t.end(); + }); + }); + + // String Escaping and preserving values testRunner.add('should parse JSON values with trailing backslashes', (t) => { diff --git a/test/fixtures/csv/excelStringsWithEscapedQuoted.csv b/test/fixtures/csv/excelStringsWithEscapedQuoted.csv new file mode 100644 index 00000000..e8045c12 --- /dev/null +++ b/test/fixtures/csv/excelStringsWithEscapedQuoted.csv @@ -0,0 +1,3 @@ +"=""a string""" +"=""with a description""" +"=""with a description and """"quotes""""""" \ No newline at end of file