Comment on stages of the pipeline.

bitdivine · Dec 5, 2014 · be76a68 · be76a68
1 parent d77a7f5
commit be76a68
Show file tree

Hide file tree

Showing 22 changed files with 52 additions and 29 deletions.
diff --git a/bin/antijoin.js b/bin/antijoin.js
@@ -1,10 +1,9 @@
 #!/usr/bin/env node
 
-var args = require('../lib/opt').parse({filename:__filename, usage:
+var args = require('../lib/opt').fancy({filename:__filename, usage:
 [ 'Usage: jline-antijoin (--help|--version)'
 , '       jline-antijoin <join_key> <file1> <file2>'
 ].join("\n")});
-process.stdout.on('error',process.exit);
 
 var by    = args['<join_key>']
   , files = [args['<file1>'], args['<file1>']]

diff --git a/bin/by.js b/bin/by.js
@@ -7,6 +7,7 @@ if ((process.argv[2] === undefined) || (process.argv[2] === '--help') || (divide
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var parseStream   = require('./clean')
   , parsePath     = require('./parsePath')

diff --git a/bin/cast.js b/bin/cast.js
@@ -6,6 +6,7 @@ if ((process.argv[2] === undefined) || (process.argv[2] === '--help')) {
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var parseStream = require('./clean');
 var parsePath   = require('./parsePath');

diff --git a/bin/chart.js b/bin/chart.js
@@ -6,6 +6,7 @@ if ((process.argv[2] === undefined) || (process.argv[2] === '--help')) {
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var fs = require('fs');
 var html = fs.readFileSync(__dirname+'/chart.html', {encoding:'utf8'});

diff --git a/bin/clean.js b/bin/clean.js
@@ -2,8 +2,8 @@
 
 function parseJlineStream(stream, options) {
   options = options || {};
-  var logcode = {none:0, error:1, warn:2}[options.loglevel];
-  var diecode = {none:0, error:1, warn:2}[options.dielevel];
+  var logcode = {none:0, error:1, warn:2, undefined:0}[options.loglevel];
+  var diecode = {none:0, error:1, warn:2, undefined:0}[options.dielevel];
   var log     = function(msg,code){console.error(msg);};
   log = options.logger || log;
 
@@ -18,28 +18,30 @@ function parseJlineStream(stream, options) {
         record = JSON.parse(line);
         recordNumber++;
         this.emit('jline', record, lineNumber, line, recordNumber);
-        this.emit('line',  record, lineNumber, line, recordNumber);
+        this.emit('line',  line,   lineNumber, line, recordNumber);
     } catch(e){
-      // Does it look like an error?
-      var serious = !/^\w*(#|$)/.test(line);
-      var code = serious?1:2;
-      var name = serious?'parseError':'parseWarn';
-      this.emit(name, e, lineNumber, line);
-      this.emit('line', e, lineNumber, line);
+      // Does it look like an error? a comment?  an empty line?
+      var match   = line.match(/^[ \t]*(#.*|$)/);
+      var code=(match === null )?1 // malformed line
+              :(match[1].length)?2 // comment
+              :                  3;// blank line
       if (code <= logcode) log("Malformed JSON on line "+String(lineNumber)+" "+String(e));
       if (code <= diecode) throw(e);
+      if (code === 1) this.emit('parseError'  , e       , lineNumber, line);
+      if (code === 2) this.emit('comment'     , match[1], lineNumber, line);
+                      this.emit('line'        , line    , lineNumber, line);
     }
   });
 }
 
 module.exports = parseJlineStream;
 
 if(require.main === module) {
-  require('../lib/opt').parse({filename:__filename});
-  process.stdout.on('error',process.exit);
+  require('../lib/opt').fancy({filename:__filename});
   var exit = 0;
   parseJlineStream(process.stdin)
-  .on('jline',      function(record, lineNumber, line, recordNumber){console.log(line);})
+  .on('jline',      function(record,  lineNumber, line, recordNumber){console.log(line);})
+  .on('comment',    function(comment, lineNumber, line, recordNumber){console.log(line);})
   .on('parseError', function(e,n,l){console.error("Malformed JSON on line", n, e); exit = 1;})
   .on('end',        function(){process.exit(exit);});
 }
diff --git a/bin/compare.js b/bin/compare.js
@@ -8,6 +8,7 @@ var by    = process.argv[2]
   , split = require('split');
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var lefts = {};
 var ans = {};

diff --git a/bin/csv.js b/bin/csv.js
@@ -32,6 +32,7 @@ if(require.main === module) {
     process.exit(2);
   }
   process.stdout.on('error',process.exit);
+  console.log.apply(null,['#'].concat(process.argv));
   streamToCsv(process.stdin)
   .on('csvHeader', console.log)
   .on('csv'      , console.log);

diff --git a/bin/csv2jl.js b/bin/csv2jl.js
@@ -8,6 +8,7 @@ if(require.main === module) {
     }
 
     process.stdout.on('error',process.exit);
+    console.log.apply(null,['#'].concat(process.argv));
 
     var csv = require('ya-csv');
     var reader = csv.createCsvStreamReader(process.openStdin(), {

diff --git a/bin/csv2jla.js b/bin/csv2jla.js
@@ -8,6 +8,7 @@ if(require.main === module) {
     }
 
     process.stdout.on('error',process.exit);
+    console.log.apply(null,['#'].concat(process.argv));
 
     var csv = require('ya-csv');
     var reader = csv.createCsvStreamReader(process.openStdin(), {

diff --git a/bin/cumulative.js b/bin/cumulative.js
@@ -11,6 +11,7 @@ var code  = function(record){
 };
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 process.stdin
 .pipe(require('split')())

diff --git a/bin/filter.js b/bin/filter.js
@@ -19,6 +19,7 @@ if(require.main === module) {
     process.exit(2);
   }
   process.stdout.on('error',process.exit);
+  console.log.apply(null,['#'].concat(process.argv));
 
   var code  = (process.argv[2] === '-f')
             ? process.argv[3]

diff --git a/bin/foreach.js b/bin/foreach.js
@@ -2,11 +2,10 @@
 
 // Execute some code for each record.
 
-require('../lib/opt').parse({filename:__filename, usage:
+require('../lib/opt').fancy({filename:__filename, usage:
 [ 'Usage: jline-foreach [--help|--version]'
 , '       jline-foreach <cmd>...'
 ].join("\n")});
-process.stdout.on('error',process.exit);
 
 var eventHandlers = [];
 process.argv.slice(2).forEach(function(arg){
@@ -32,6 +31,7 @@ function keyvals(dict, callback){
 }
 
 var source = require('./clean')(process.stdin);
+source.on('comment', function(c){console.log(c);});
 eventHandlers.forEach(function(tuple){
     var event = tuple[0].toLowerCase();
     var code  = Function('require','emit','keyvals', 'record', 'lineNumber', 'line', 'recordNumber', tuple[1])

diff --git a/bin/intersect.js b/bin/intersect.js
@@ -8,6 +8,7 @@ var by    = process.argv[2]
   , split = require('split');
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var ans = {};
 

diff --git a/bin/join_left.js b/bin/join_left.js
@@ -8,6 +8,7 @@ var by    = process.argv[2]
   , split = require('split');
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var ans = {};
 

diff --git a/bin/map.js b/bin/map.js
@@ -16,6 +16,7 @@ function map(record, lineNumber, line, recordNumber){
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 require('./clean')(process.stdin)
 .on('jline', map);
 
diff --git a/bin/pretty.js b/bin/pretty.js
@@ -8,6 +8,7 @@ if (process.argv[2] === '--help') {
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 require('./clean')(process.stdin)
 .on('jline', function(record){

diff --git a/bin/select.js b/bin/select.js
@@ -6,6 +6,7 @@ if ((process.argv[2] === undefined) || (process.argv[2] === '--help')) {
 }
 
 process.stdout.on('error',process.exit);
+console.log.apply(null,['#'].concat(process.argv));
 
 var parseStream = require('./clean');
 var parsePath   = require('./parsePath');
@@ -33,8 +34,10 @@ function setPath(dict, path, val){
     dict[last] = val;
 }
 
+console.log.apply(null,['#'].concat(process.argv));
 var headers;
 parseStream(process.stdin)
+.on('comment', function(c){console.log(c);})
 .on('jline', function(record){
         var filtered = {};
         fields.forEach(function(field){

diff --git a/bin/sort.js b/bin/sort.js
@@ -1,22 +1,21 @@
 #!/usr/bin/env node
 
-if ((process.argv[2] === undefined) || (process.argv[2] === '--help')) {
-    console.error(require('fs').readFileSync(__filename.replace(/.js$/,'.md'),{encoding:'utf8'}));
-    process.exit(1);
-}
-
-process.stdout.on('error',process.exit);
+var args = require('../lib/opt').fancy({filename:__filename, usage:
+[ 'Usage: jline-sort [--help|--version]'
+, '       jline-sort <key>'
+].join("\n")});
 
 var split     = require('split')
   , parsePath = require('./parsePath')
   , parseData = require('./clean')
   , getPath   = require('tree-math').getPath;
 
-var by = parsePath(process.argv[2]) // sort by this
+var by = parsePath(args['<key>']) // sort by this
   , lines = [];
 
 
 parseData(process.stdin)
+.on('comment', function(c){console.log(c);})
 .on('jline', function(line){
     lines.push(line);
 })

diff --git a/lib/opt.js b/lib/opt.js
@@ -25,11 +25,18 @@ function handle(opts, vars, handlers){
     if(opts['--'+handler.name]) handler(opts, vars);
   });
 }
+function fancy(vars){
+    var ans = parse(vars);
+    process.stdout.on('error',process.exit);
+    console.log.apply(null,['#'].concat(process.argv.map(JSON.stringify)));
+    return ans;
+}
 
 module.exports = 
 { parse:  parse
 , handle: handle
 , help:   help
+, fancy:  fancy
 , version:version
 };
 

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "jline",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "description": "Sort, grep and join files where every line is JSON.",
   "main": "index.js",
   "bin": {

diff --git a/test/clean.js b/test/clean.js
@@ -8,10 +8,10 @@
 
     parseJlineStream(stream)
     .on('record',     function(parsed, lineNumber, line){console.log(parsed);})
-    .on('parseWarn',  function(error,  lineNumber, line){console.error("Comment or blank line on line", lineNumber);})
+    .on('comment',    function(comment,lineNumber, line){console.error("Comment or blank line on line", lineNumber);})
     .on('parseError', function(error,  lineNumber, line){console.error("Malformed JSON on line", lineNumber, error);})
     .on('end',        function(){console.log("Finito!");});
 
-    stream = require('fs').createReadStream(__dirname+"/data.clean.jsonl");
-    parseJlineStream(stream, {loglevel:'warn', dielevel:'error'})
-    .on('jline',      function(parsed, lineNumber, line){console.log(parsed);});
+//    stream = require('fs').createReadStream(__dirname+"/data.clean.jsonl");
+//    parseJlineStream(stream, {loglevel:'warn', dielevel:'error'})
+//    .on('jline',      function(parsed, lineNumber, line){console.log(parsed);});
diff --git a/test/clean.sh b/test/clean.sh
@@ -5,6 +5,6 @@ set -eux
 
     ./bin/clean.js --help                || (($? == 2))
     ./bin/clean.js <test/data.jsonl      || (($? == 1))
-    ./bin/clean.js <test/data.clean.jsonl
+    ./bin/clean.js <test/data.clean.jsonl|| (($? == 0))
 
 echo OK