From 5c5de9fb71fa657d257b56b54673056081e3281f Mon Sep 17 00:00:00 2001 From: Ivan Kuznetsov <ivankuznetsov316@gmail.com> Date: Wed, 1 Aug 2018 02:22:57 +0700 Subject: [PATCH] feat: Added flattenSeparator option (#314) * Added flattenSeparator option * Updated docstring --- README.md | 44 +++++++-------- bin/json2csv.js | 10 ++-- json2csv | 1 + lib/JSON2CSVBase.js | 54 ++++++++++--------- test/JSON2CSVTransform.js | 19 +++++++ .../csv/flattenedCustomSeparatorDeepJSON.csv | 2 + 6 files changed, 80 insertions(+), 50 deletions(-) create mode 160000 json2csv create mode 100644 test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv diff --git a/README.md b/README.md index 49fea070..dc079580 100644 --- a/README.md +++ b/README.md @@ -45,27 +45,28 @@ $ npm install json2csv --save Options: - -V, --version output the version number - -i, --input <input> Path and name of the incoming json file. If not provided, will read from stdin. - -o, --output [output] Path and name of the resulting csv file. Defaults to stdout. - -n, --ndjson Treat the input as NewLine-Delimited JSON. - -s, --no-streaming Process the whole JSON array in memory instead of doing it line by line. - -f, --fields <fields> Specify the fields to convert. - -c, --fields-config <path> Specify a file with a fields configuration as a JSON array. - -u, --unwind <paths> Creates multiple rows from a single JSON document similar to MongoDB unwind. - -B, --unwind-blank When unwinding, blank out instead of repeating data. - -F, --flatten Flatten nested objects - -v, --default-value [defaultValue] Specify a default value other than empty string. - -q, --quote [value] Specify an alternate quote value. - -Q, --double-quote [value] Specify a value to replace double quote in strings - -d, --delimiter [delimiter] Specify a delimiter other than the default comma to use. - -e, --eol [value] Specify an End-of-Line value for separating rows. - -E, --excel-strings Converts string data into normalized Excel style data - -H, --no-header Disable the column name header - -a, --include-empty-rows Includes empty rows in the resulting CSV output. - -b, --with-bom Includes BOM character at the beginning of the csv. - -p, --pretty Use only when printing to console. Logs output in pretty tables. - -h, --help output usage information + -V, --version output the version number + -i, --input <input> Path and name of the incoming json file. If not provided, will read from stdin. + -o, --output [output] Path and name of the resulting csv file. Defaults to stdout. + -n, --ndjson Treat the input as NewLine-Delimited JSON. + -s, --no-streaming Process the whole JSON array in memory instead of doing it line by line. + -f, --fields <fields> Specify the fields to convert. + -c, --fields-config <path> Specify a file with a fields configuration as a JSON array. + -u, --unwind <paths> Creates multiple rows from a single JSON document similar to MongoDB unwind. + -B, --unwind-blank When unwinding, blank out instead of repeating data. + -F, --flatten Flatten nested objects. + -S, --flatten-separator <separator> Flattened keys separator. + -v, --default-value [defaultValue] Specify a default value other than empty string. + -q, --quote [value] Specify an alternate quote value. + -Q, --double-quote [value] Specify a value to replace double quote in strings. + -d, --delimiter [delimiter] Specify a delimiter other than the default comma to use. + -e, --eol [value] Specify an End-of-Line value for separating rows. + -E, --excel-strings Converts string data into normalized Excel style data. + -H, --no-header Disable the column name header. + -a, --include-empty-rows Includes empty rows in the resulting CSV output. + -b, --with-bom Includes BOM character at the beginning of the csv. + -p, --pretty Use only when printing to console. Logs output in pretty tables. + -h, --help output usage information ``` An input file `-i` and fields `-f` are required. If no output `-o` is specified the result is logged to the console. @@ -158,6 +159,7 @@ The programatic APIs take a configuration object very equivalent to the CLI opti - `unwind` - Array of Strings, creates multiple rows from a single JSON document similar to MongoDB's $unwind - `unwindBlank` - Boolean, unwind using blank values instead of repeating data. - `flatten` - Boolean, flattens nested JSON using [flat]. Defaults to `false`. +- `flattenSeparator` - String, separator to use between nested JSON keys when `flatten` option enabled. Defaults to `.` if not specified. - `defaultValue` - String, default value to use when missing data. Defaults to `<empty>` if not specified. (Overridden by `fields[].default`) - `quote` - String, quote around cell values and column names. Defaults to `"` if not specified. - `doubleQuote` - String, the value to replace double quote in strings. Defaults to 2x`quotes` (for example `""`) if not specified. diff --git a/bin/json2csv.js b/bin/json2csv.js index 4de2c863..b7a4bab4 100755 --- a/bin/json2csv.js +++ b/bin/json2csv.js @@ -24,14 +24,15 @@ program .option('-c, --fields-config <path>', 'Specify a file with a fields configuration as a JSON array.') .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.') - .option('-F, --flatten', 'Flatten nested objects') + .option('-F, --flatten', 'Flatten nested objects.') + .option('-S, --flatten-separator <separator>', 'Flattened keys separator.') .option('-v, --default-value [defaultValue]', 'Specify a default value other than empty string.') .option('-q, --quote [value]', 'Specify an alternate quote value.') - .option('-Q, --double-quote [value]', 'Specify a value to replace double quote in strings') + .option('-Q, --double-quote [value]', 'Specify a value to replace double quote in strings.') .option('-d, --delimiter [delimiter]', 'Specify a delimiter other than the default comma to use.') .option('-e, --eol [value]', 'Specify an End-of-Line value for separating rows.') - .option('-E, --excel-strings','Converts string data into normalized Excel style data') - .option('-H, --no-header', 'Disable the column name header') + .option('-E, --excel-strings','Converts string data into normalized Excel style data.') + .option('-H, --no-header', 'Disable the column name header.') .option('-a, --include-empty-rows', 'Includes empty rows in the resulting CSV output.') .option('-b, --with-bom', 'Includes BOM character at the beginning of the csv.') .option('-p, --pretty', 'Use only when printing to console. Logs output in pretty tables.') @@ -156,6 +157,7 @@ Promise.resolve() unwind: program.unwind ? program.unwind.split(',') : [], unwindBlank: program.unwindBlank, flatten: program.flatten, + flattenSeparator: program.flattenSeparator, defaultValue: program.defaultValue, quote: program.quote, doubleQuote: program.doubleQuote, diff --git a/json2csv b/json2csv new file mode 160000 index 00000000..b2690fee --- /dev/null +++ b/json2csv @@ -0,0 +1 @@ +Subproject commit b2690fee0d0a2ec1a2a59516704a225461406a2d diff --git a/lib/JSON2CSVBase.js b/lib/JSON2CSVBase.js index 680c904b..8b8dfa62 100644 --- a/lib/JSON2CSVBase.js +++ b/lib/JSON2CSVBase.js @@ -22,6 +22,7 @@ class JSON2CSVBase { ? (processedOpts.unwind ? [processedOpts.unwind] : []) : processedOpts.unwind processedOpts.delimiter = processedOpts.delimiter || ','; + processedOpts.flattenSeparator = processedOpts.flattenSeparator || '.'; processedOpts.eol = processedOpts.eol || os.EOL; processedOpts.quote = typeof processedOpts.quote === 'string' ? opts.quote @@ -63,7 +64,7 @@ class JSON2CSVBase { : [row]; if (this.opts.flatten) { - return processedRow.map(this.flatten); + return processedRow.map(this.flatten()); } return processedRow; @@ -208,34 +209,37 @@ class JSON2CSVBase { /** * Performs the flattening of a data row recursively * - * @param {Object} dataRow Original JSON object - * @returns {Object} Flattened object + * @returns {Function} Function that receives dataRow as input and outputs flattened object */ - flatten(dataRow) { - function step (obj, flatDataRow, currentPath) { - Object.keys(obj).forEach((key) => { - const value = obj[key]; - - const newPath = currentPath - ? `${currentPath}.${key}` - : key; - - if (typeof value !== 'object' - || value === null - || Array.isArray(value) - || Object.prototype.toString.call(value.toJSON) === '[object Function]' - || !Object.keys(value).length) { - flatDataRow[newPath] = value; - return; - } + flatten() { + return (dataRow) => { + const separator = this.opts.flattenSeparator; + + function step (obj, flatDataRow, currentPath) { + Object.keys(obj).forEach((key) => { + const value = obj[key]; + + const newPath = currentPath + ? `${currentPath}${separator}${key}` + : key; + + if (typeof value !== 'object' + || value === null + || Array.isArray(value) + || Object.prototype.toString.call(value.toJSON) === '[object Function]' + || !Object.keys(value).length) { + flatDataRow[newPath] = value; + return; + } - step(value, flatDataRow, newPath); - }); + step(value, flatDataRow, newPath); + }); - return flatDataRow; - } + return flatDataRow; + } - return step(dataRow, {}); + return step(dataRow, {}); + } } /** diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js index 1b90296f..20db1937 100644 --- a/test/JSON2CSVTransform.js +++ b/test/JSON2CSVTransform.js @@ -453,6 +453,25 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { .on('error', err => t.notOk(true, err.message)); }); + testRunner.add('should support custom flatten separator', (t) => { + const opts = { + flatten: true, + flattenSeparator: '__', + }; + + const transform = new Json2csvTransform(opts); + const processor = jsonFixtures.deepJSON().pipe(transform); + + let csv = ''; + processor + .on('data', chunk => (csv += chunk.toString())) + .on('end', () => { + t.equal(csv, csvFixtures.flattenedCustomSeparatorDeepJSON); + t.end(); + }) + .on('error', err => t.notOk(true, err.message)); + }); + testRunner.add('should unwind and flatten an object in the right order', (t) => { const opts = { unwind: ['items'], diff --git a/test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv b/test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv new file mode 100644 index 00000000..6901f534 --- /dev/null +++ b/test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv @@ -0,0 +1,2 @@ +"field1__embeddedField1","field1__embeddedField2" +"embeddedValue1","embeddedValue2" \ No newline at end of file