Zip archives if zippedArchive is true

pixtron · pixtron · commit 44862ff1956a · 2019-12-02T22:54:41.000-05:00
* Zip archives during rotation if zippedArchive is true * Remove gzip stream which was never written to (see winstonjs#1128 (comment)) * ads tests checking the contents of zipped archives * ads tests for rolling files with tailable: false * Fixes previously failing test tests/tail.file.test.js
diff --git a/lib/winston/transports/file.js b/lib/winston/transports/file.js
@@ -566,11 +566,6 @@ module.exports = class File extends TransportStream {
       });
 
     debug('create stream ok', fullpath);
-    if (this.zippedArchive) {
-      const gzip = zlib.createGzip();
-      gzip.pipe(dest);
-      return gzip;
-    }
 
     return dest;
   }
@@ -585,12 +580,30 @@ module.exports = class File extends TransportStream {
     const ext = path.extname(this._basename);
     const basename = path.basename(this._basename, ext);
 
-    if (!this.tailable) {
-      this._created += 1;
-      this._checkMaxFilesIncrementing(ext, basename, callback);
-    } else {
-      this._checkMaxFilesTailable(ext, basename, callback);
+    const tasks = [];
+
+    if (this.zippedArchive) {
+      const num = this._created > 0 && !this.tailable ? this._created : '';
+
+      tasks.push(function (cb) {
+        this._compressFile(
+          path.join(this.dirname, `${basename}${num}${ext}`),
+          path.join(this.dirname, `${basename}${num}${ext}.gz`),
+          cb
+        );
+      }.bind(this));
     }
+
+    tasks.push(function (cb) {
+      if (!this.tailable) {
+        this._created += 1;
+        this._checkMaxFilesIncrementing(ext, basename, cb);
+      } else {
+        this._checkMaxFilesTailable(ext, basename, cb);
+      }
+    }.bind(this));
+
+    asyncSeries(tasks, callback);
   }
 
   /**
@@ -609,13 +622,9 @@ module.exports = class File extends TransportStream {
     // Caveat emptor (indexzero): rotationFormat() was broken by design When
     // combined with max files because the set of files to unlink is never
     // stored.
-    const target = !this.tailable && this._created
+    return !this.tailable && this._created
       ? `${basename}${isRotation}${ext}`
       : `${basename}${ext}`;
-
-    return this.zippedArchive && !this.tailable
-      ? `${target}.gz`
-      : target;
   }
 
   /**
@@ -658,7 +667,6 @@ module.exports = class File extends TransportStream {
       return;
     }
 
-    // const isZipped = this.zippedArchive ? '.gz' : '';
     const isZipped = this.zippedArchive ? '.gz' : '';
     for (let x = this.maxFiles - 1; x > 1; x--) {
       tasks.push(function (i, cb) {
@@ -678,13 +686,38 @@ module.exports = class File extends TransportStream {
 
     asyncSeries(tasks, () => {
       fs.rename(
-        path.join(this.dirname, `${basename}${ext}`),
+        path.join(this.dirname, `${basename}${ext}${isZipped}`),
         path.join(this.dirname, `${basename}1${ext}${isZipped}`),
         callback
       );
     });
   }
 
+  /**
+   * Compresses src to dest with gzip and unlinks src
+   * @param {string} src - path to source file.
+   * @param {string} dest - path to zipped destination file.
+   * @param {Function} callback - callback called after file has been compressed.
+   * @returns {undefined}
+   * @private
+   */
+  _compressFile(src, dest, callback) {
+    fs.exists(src, exists => {
+      if (!exists) return callback();
+
+      var gzip = zlib.createGzip();
+
+      var inp = fs.createReadStream(src);
+      var out = fs.createWriteStream(dest);
+
+      out.on('finish', () => {
+        fs.unlink(src, callback);
+      });
+
+      inp.pipe(gzip).pipe(out);
+    });
+  }
+
   _createLogDirIfNotExist(dirPath) {
     /* eslint-disable no-sync */
     if (!fs.existsSync(dirPath)) {
diff --git a/test/transports/file-archive.test.js b/test/transports/file-archive.test.js
@@ -7,6 +7,7 @@
  */
 
 /* eslint-disable no-sync */
+const zlib = require('zlib');
 const assert = require('assert');
 const rimraf = require('rimraf');
 const fs = require('fs');
@@ -25,6 +26,16 @@ function removeFixtures(done) {
 
 let archiveTransport = null;
 
+function data(ch, kb) {
+  return String.fromCharCode(65 + ch).repeat(kb * 1024 - 1);
+}
+
+function logKbytes(kbytes, txt) {
+  const toLog = {};
+  toLog[MESSAGE] = data(txt, kbytes);
+  archiveTransport.log(toLog);
+}
+
 describe('winston/transports/file/zippedArchive', function () {
   describe('An instance of the File Transport with tailable true', function () {
     before(removeFixtures);
@@ -47,16 +58,6 @@ describe('winston/transports/file/zippedArchive', function () {
       let created = 0;
       let loggedTotal = 0;
 
-      function data(ch, kb) {
-        return String.fromCharCode(65 + ch).repeat(kb * 1024 - 1);
-      }
-
-      function logKbytes(kbytes, txt) {
-        const toLog = {};
-        toLog[MESSAGE] = data(txt, kbytes);
-        archiveTransport.log(toLog);
-      }
-
       archiveTransport.on('logged', function (info) {
         loggedTotal += info[MESSAGE].length + 1;
         if (loggedTotal >= 14 * 1024) { // just over 3 x 4kb files
@@ -89,5 +90,107 @@ describe('winston/transports/file/zippedArchive', function () {
         }, Error);
       }
     });
+
+    it('testarchive.log should be ascii, testarchive1.log.gz and testarchive2.log.gz should be zipped', function () {
+      const letters = ['D', 'C', 'B'];
+
+      for (var num = 0; num < 3; num++) {
+        let content;
+        const letter = letters[num];
+        const file = !num ? 'testarchive.log' : 'testarchive' + num + '.log.gz';
+        const fileContent = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'logs', file));
+
+        if (num > 0) {
+          // archives should be zipped
+          assert.doesNotThrow(function () {
+            content = zlib.gunzipSync(fileContent).toString('ascii');
+          });
+        } else {
+          // current log file should be plain text
+          content = fileContent.toString('ascii');
+        }
+
+        assert(content.match(new RegExp(letter, 'g'))[0].length, content.length);
+      }
+    });
+  });
+
+  describe('An instance of the File Transport with tailable false', function () {
+    before(removeFixtures);
+    after(removeFixtures);
+
+    it('init logger AFTER cleaning up old files', function () {
+      archiveTransport = new winston.transports.File({
+        timestamp: true,
+        json: false,
+        zippedArchive: true,
+        tailable: false,
+        filename: 'testarchive.log',
+        dirname: path.join(__dirname, '..', 'fixtures', 'logs'),
+        maxsize: 4096,
+        maxFiles: 3
+      });
+    });
+
+    it('when created archived files are rolled', function (done) {
+
+      let created = 0;
+      let loggedTotal = 0;
+
+      archiveTransport.on('logged', function (info) {
+        loggedTotal += info[MESSAGE].length + 1;
+        if (loggedTotal >= 14 * 1024) { // just over 3 x 4kb files
+          return done();
+        }
+
+        if (loggedTotal % 4096 === 0) {
+          created++;
+        }
+        // eslint-disable-next-line max-nested-callbacks
+        setTimeout(() => logKbytes(1, created), 1);
+      });
+
+      logKbytes(1, created);
+    });
+
+    it('should be only 3 files called testarchive3.log, testarchive2.log.gz and testarchive1.log.gz', function () {
+      for (var num = 0; num < 4; num++) {
+        const file = num === 3 ? 'testarchive3.log' : 'testarchive' + (num > 0 ? num : '') + '.log.gz';
+        const fullpath = path.join(__dirname, '..', 'fixtures', 'logs', file);
+
+        const statFile = function () {
+          fs.statSync(fullpath);
+        };
+
+        if (num === 0) {
+          assert.throws(statFile, Error);
+        } else {
+          assert.doesNotThrow(statFile, Error);
+        }
+      }
+    });
+
+    it('testarchive3.log should be ascii, testarchive2.log.gz and testarchive1.log.gz should be zipped', function () {
+      const letters = ['B', 'C', 'D'];
+
+      for (var num = 1; num < 4; num++) {
+        let content;
+        const letter = letters[num - 1];
+        const file = num === 3 ? 'testarchive3.log' : 'testarchive' + num + '.log.gz';
+        const fileContent = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'logs', file));
+
+        if (num !== 3) {
+          // archives should be zipped
+          assert.doesNotThrow(function () {
+            content = zlib.gunzipSync(fileContent).toString('ascii');
+          });
+        } else {
+          // current log file should be plain text
+          content = fileContent.toString('ascii');
+        }
+
+        assert(content.match(new RegExp(letter, 'g'))[0].length, content.length);
+      }
+    });
   });
 });
diff --git a/test/transports/file-rolling.test.js b/test/transports/file-rolling.test.js
@@ -0,0 +1,92 @@
+/* eslint-disable no-sync */
+const assert = require('assert');
+const rimraf = require('rimraf');
+const fs = require('fs');
+const path = require('path');
+const winston = require('../../lib/winston');
+
+const { MESSAGE } = require('triple-beam');
+
+//
+// Remove all log fixtures
+//
+function removeFixtures(done) {
+  rimraf(path.join(__dirname, '..', 'fixtures', 'logs', 'testrollingfiles*'), done);
+}
+
+
+
+let rollTransport = null;
+
+describe('winston/transports/file/rolling', function () {
+  describe('An instance of the File Transport', function () {
+    before(removeFixtures);
+    after(removeFixtures);
+
+    it('init logger AFTER cleaning up old files', function () {
+      rollTransport = new winston.transports.File({
+        timestamp: false,
+        json: false,
+        filename: path.join(__dirname, '..', 'fixtures', 'logs', 'testrollingfiles.log'),
+        maxsize: 4096,
+        maxFiles: 3,
+        tailable: false
+      })
+    });
+
+    it('and when passed more files than the maxFiles', function (done) {
+      let created = 0;
+      let loggedTotal = 0;
+
+      function data(ch, kb) {
+        return String.fromCharCode(65 + ch).repeat(kb * 1024 - 1);
+      }
+
+      function logKbytes(kbytes, txt) {
+        const toLog = {};
+	      toLog[MESSAGE] = data(txt, kbytes);
+        rollTransport.log(toLog);
+      }
+
+      rollTransport.on('logged', function (info) {
+        loggedTotal += info[MESSAGE].length + 1;
+        if (loggedTotal >= 14 * 1024) { // just over 3 x 4kb files
+          return done();
+        }
+
+        if (loggedTotal % 4096 === 0) {
+          created++;
+        }
+        // eslint-disable-next-line max-nested-callbacks
+        setTimeout(() => logKbytes(1, created), 1);
+      });
+
+      logKbytes(1, created);
+    });
+
+    it('should be 3 log files, base to maxFiles - 1', function () {
+      for (var num = 0; num < 4; num++) {
+        const file = !num ? 'testrollingfiles.log' : 'testrollingfiles' + num + '.log';
+        const fullpath = path.join(__dirname, '..', 'fixtures', 'logs', file);
+
+        if (num === 0) {
+          return assert.ok(!fs.existsSync(fullpath));
+        }
+
+        assert.ok(fs.existsSync(fullpath));
+      }
+
+      return false;
+    });
+
+    it('should have files in correct order', function () {
+      ['B', 'C', 'D'].forEach(function (letter, i) {
+        const file = 'testrollingfiles' + (i + 1) + '.log';
+        let content = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'logs', file), 'ascii');
+        content = content.replace(/\s+/g, '');
+
+        assert(content.match(new RegExp(letter, 'g'))[0].length, content.length);
+      });
+    });
+  });
+});
