From 02d426748d859c00caa71bd457e914cda6f3160b Mon Sep 17 00:00:00 2001 From: Pulkit Aggarwal <82268257+Pulkit0729@users.noreply.github.com> Date: Wed, 3 Jan 2024 09:40:38 +0530 Subject: [PATCH] Added functionality to long broken zippedArchive option (#2337) * Test for file Transport option rotationFormat * Test for file Transport option rotationFormat * Added Lazy option in file transport * Lint and test * removed only statement * Update test/unit/winston/transports/01-file-maxsize.test.js Co-authored-by: David Hyde * Update test/unit/winston/transports/01-file-maxsize.test.js Co-authored-by: David Hyde * Added lazy in FileTransportOptions types * Added lazy type in FileTransportInstance * Add functionality to zippedArchive option --------- Co-authored-by: myAlapi Co-authored-by: David Hyde --- lib/winston/transports/file.js | 69 +++- .../transports/02-file-archive.test.js | 314 ++++++++++++++++++ 2 files changed, 365 insertions(+), 18 deletions(-) create mode 100644 test/unit/winston/transports/02-file-archive.test.js diff --git a/lib/winston/transports/file.js b/lib/winston/transports/file.js index b5801038b..99682453b 100644 --- a/lib/winston/transports/file.js +++ b/lib/winston/transports/file.js @@ -198,7 +198,7 @@ module.exports = class File extends TransportStream { return; } if (this.lazy) { - this._endStream(() => {this.emit('fileclosed')}); + this._endStream(() => {this.emit('fileclosed');}); return; } @@ -603,12 +603,6 @@ module.exports = class File extends TransportStream { }); debug('create stream ok', fullpath); - if (this.zippedArchive) { - const gzip = zlib.createGzip(); - gzip.pipe(dest); - return gzip; - } - return dest; } @@ -621,13 +615,33 @@ module.exports = class File extends TransportStream { debug('_incFile', this.filename); const ext = path.extname(this._basename); const basename = path.basename(this._basename, ext); + const tasks = []; - if (!this.tailable) { - this._created += 1; - this._checkMaxFilesIncrementing(ext, basename, callback); - } else { - this._checkMaxFilesTailable(ext, basename, callback); + if (this.zippedArchive) { + tasks.push( + function (cb) { + const num = this._created > 0 && !this.tailable ? this._created : ''; + this._compressFile( + path.join(this.dirname, `${basename}${num}${ext}`), + path.join(this.dirname, `${basename}${num}${ext}.gz`), + cb + ); + }.bind(this) + ); } + + tasks.push( + function (cb) { + if (!this.tailable) { + this._created += 1; + this._checkMaxFilesIncrementing(ext, basename, cb); + } else { + this._checkMaxFilesTailable(ext, basename, cb); + } + }.bind(this) + ); + + asyncSeries(tasks, callback); } /** @@ -646,13 +660,9 @@ module.exports = class File extends TransportStream { // Caveat emptor (indexzero): rotationFormat() was broken by design When // combined with max files because the set of files to unlink is never // stored. - const target = !this.tailable && this._created + return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`; - - return this.zippedArchive && !this.tailable - ? `${target}.gz` - : target; } /** @@ -715,13 +725,36 @@ module.exports = class File extends TransportStream { asyncSeries(tasks, () => { fs.rename( - path.join(this.dirname, `${basename}${ext}`), + path.join(this.dirname, `${basename}${ext}${isZipped}`), path.join(this.dirname, `${basename}1${ext}${isZipped}`), callback ); }); } + /** + * Compresses src to dest with gzip and unlinks src + * @param {string} src - path to source file. + * @param {string} dest - path to zipped destination file. + * @param {Function} callback - callback called after file has been compressed. + * @returns {undefined} + * @private + */ + _compressFile(src, dest, callback) { + fs.access(src, fs.F_OK, (err) => { + if (err) { + return callback(); + } + var gzip = zlib.createGzip(); + var inp = fs.createReadStream(src); + var out = fs.createWriteStream(dest); + out.on('finish', () => { + fs.unlink(src, callback); + }); + inp.pipe(gzip).pipe(out); + }); + } + _createLogDirIfNotExist(dirPath) { /* eslint-disable no-sync */ if (!fs.existsSync(dirPath)) { diff --git a/test/unit/winston/transports/02-file-archive.test.js b/test/unit/winston/transports/02-file-archive.test.js new file mode 100644 index 000000000..dd230fa0a --- /dev/null +++ b/test/unit/winston/transports/02-file-archive.test.js @@ -0,0 +1,314 @@ +/* + * file-archive-test.js: Tests for instances of the File transport setting the archive option, + * + */ + +/* eslint-disable no-sync */ +const rimraf = require('rimraf'); +const fs = require('fs'); +const path = require('path'); +const { MESSAGE } = require('triple-beam'); +const assume = require('assume'); +const zlib = require('zlib'); +const winston = require('../../../../lib/winston'); +const testLogFixturesPath = path.join( + __dirname, + '..', + '..', + '..', + 'fixtures', + 'logs' +); + + +function removeFixtures(done) { + rimraf(path.join(testLogFixturesPath, 'testarchive*'), done); +} + +describe('winston/transports/file/zippedArchive', function () { + this.beforeEach(removeFixtures); + this.afterEach(removeFixtures); + + it('should not create zip when file is being used', function (done) { + let archiveTransport = new winston.transports.File({ + zippedArchive: true, + level: 'info', + format: winston.format.printf(info => info.message), + filename: path.join(testLogFixturesPath, 'testarchive.log'), + maxsize: 4096 + }); + var info = { [MESSAGE]: 'this is my log message' }; + setTimeout(() => { + archiveTransport.log(info); + }, 100); + + archiveTransport.once('logged', function () { + assume(fs.existsSync(path.join(testLogFixturesPath, 'testarchive.log'))); + assume( + fs.existsSync(path.join(testLogFixturesPath, 'testarchive.log.gz')) + ).false(); + }); + done(); + }); + + it('should create multiple zip files', function (done) { + const fillWith = ['a', 'b', 'c', 'd', 'e']; + + let archiveTransport = new winston.transports.File({ + zippedArchive: true, + level: 'info', + format: winston.format.printf(info => info.message), + filename: path.join(testLogFixturesPath, 'testarchive.log'), + maxsize: 4096 + }); + const logger = winston.createLogger({ + transports: [archiveTransport] + }); + const files = []; + + function logKbytes(kbytes) { + // + // Shift the next fill char off the array then push it back + // to rotate the chars. + // + const filler = fillWith.shift(); + fillWith.push(filler); + + // + // + // To not make each file not fail the assertion of the filesize we can + // make the array 1023 characters long. + // + const kbStr = Array(1023).fill(filler).join(''); + for (var i = 0; i < kbytes; i++) { + logger.log({ level: 'info', message: kbStr }); + } + } + + function assumeFilesCreated() { + files.map(function (file, i) { + assume(fs.existsSync(file)); + const text = fs.readFileSync(file); + + content = zlib.gunzipSync(text).toString('utf8'); + assume(content.length).equal(4096); + assume(content[0]).equal(fillWith[i]); + }); + + done(); + } + + archiveTransport.on('open', function (file) { + if (files.length === 5) { + assumeFilesCreated(); + return; + } + files.push(file + '.gz'); + setImmediate(() => logKbytes(4)); + }); + }); + + it('should have correct no zip files with maxfiles', function (done) { + const fillWith = ['a', 'b', 'c', 'd', 'e']; + + let archiveTransport = new winston.transports.File({ + zippedArchive: true, + level: 'info', + format: winston.format.printf(info => info.message), + filename: path.join(testLogFixturesPath, 'testarchive.log'), + maxsize: 4096, + maxFiles: 3 + }); + const logger = winston.createLogger({ + transports: [archiveTransport] + }); + const files = []; + + function logKbytes(kbytes) { + // + // Shift the next fill char off the array then push it back + // to rotate the chars. + // + const filler = fillWith.shift(); + fillWith.push(filler); + + // + // + // To not make each file not fail the assertion of the filesize we can + // make the array 1023 characters long. + // + const kbStr = Array(1023).fill(filler).join(''); + for (var i = 0; i < kbytes; i++) { + logger.log({ level: 'info', message: kbStr }); + } + } + + function assumeFilesCreated() { + files.map(function (file, i) { + if (i <= 2) { + assume(fs.existsSync(file)).false(); + return; + } + const text = fs.readFileSync(file); + content = zlib.gunzipSync(text).toString('utf8'); + assume(content.length).equal(4096); + assume(content[0]).equal(fillWith[i]); + }); + + done(); + } + + archiveTransport.on('open', function (file) { + const match = file.match(/(\d+)\.log$/); + const count = match ? match[1] : 0; + + if (files.length === 5) { + assumeFilesCreated(); + return; + } + files.push(file + '.gz'); + setImmediate(() => logKbytes(4)); + }); + }); + it('should have zip files with tailable', function (done) { + const fillWith = ['a', 'b', 'c', 'd', 'e']; + + let archiveTransport = new winston.transports.File({ + zippedArchive: true, + level: 'info', + format: winston.format.printf(info => info.message), + filename: path.join(testLogFixturesPath, 'testarchive.log'), + maxsize: 4096, + tailable: true, + maxFiles: 3 + }); + const logger = winston.createLogger({ + transports: [archiveTransport] + }); + let count = 0; + function logKbytes(kbytes) { + // + // Shift the next fill char off the array then push it back + // to rotate the chars. + // + const filler = fillWith.shift(); + fillWith.push(filler); + + // + // + // To not make each file not fail the assertion of the filesize we can + // make the array 1023 characters long. + // + const kbStr = Array(1023).fill(filler).join(''); + for (var i = 0; i < kbytes; i++) { + logger.log({ level: 'info', message: kbStr }); + } + } + + function assumeFilesCreated() { + for (let i = 0; i < 4; i++) { + const file = !i ? 'testarchive.log' : 'testarchive' + i + '.log'; + if (i == 0) { + const fullpath = path.join(testLogFixturesPath, file); + assume(fs.existsSync(file)); + continue; + } + const fullpath = path.join(testLogFixturesPath, file + '.gz'); + if (i == 3) { + assume(fs.existsSync(fullpath)).false(); + continue; + } + assume(fs.existsSync(fullpath)); + } + done(); + } + + archiveTransport.on('open', function (file) { + if (count === 5) { + assumeFilesCreated(); + return; + } + count++; + setImmediate(() => logKbytes(4)); + }); + }); + it('should not create extra file', function (done) { + const fillWith = ['a', 'b', 'c', 'd', 'e']; + let archiveTransport = new winston.transports.File({ + zippedArchive: true, + level: 'info', + format: winston.format.printf(info => info.message), + filename: path.join(testLogFixturesPath, 'testarchive.log'), + maxsize: 4096, + lazy: true + }); + const logger = winston.createLogger({ + transports: [archiveTransport] + }); + // + // Setup a list of files which we will later stat. + // + const files = []; + + // + // Assets the no of files and all the files have been created with the + // correct filesize + // + function assumeFilesCreated() { + files.map(function (file, i) { + if (i == fillWith.length - 1) { + assume(fs.existsSync(file)).false(); + return; + } + const text = fs.readFileSync(file); + content = zlib.gunzipSync(text).toString('utf8'); + assume(content.length).equal(4096); + assume(content[0]).equal(fillWith[i]); + }); + done(); + } + + // + // Log the specified kbytes to the transport + // + function logKbytes(kbytes) { + // + // Shift the next fill char off the array then push it back + // to rotate the chars. + // + const filler = fillWith.shift(); + fillWith.push(filler); + + // + // + // To not make each file not fail the assertion of the filesize we can + // make the array 1023 characters long. + // + const kbStr = Array(1023).fill(filler).join(''); + for (var i = 0; i < kbytes; i++) { + logger.log({ level: 'info', message: kbStr }); + } + } + + // Initial Log + let count = 0; + logKbytes(4); + + // Listen to file close event called when the file is closed + archiveTransport.on('fileclosed', () => { + count += 1; + if (count === fillWith.length) { + assumeFilesCreated(); + return; + } + setImmediate(() => { + logKbytes(4); + }); + }); + + //Listent to file open event called when the file is opened + archiveTransport.on('open', file => { + files.push(file + '.gz'); + }); + }); +});