From bceb568f54bad0fcf9759f54c97fd6e23dcd386d Mon Sep 17 00:00:00 2001 From: Smit Shah Date: Mon, 20 Jul 2020 16:03:28 -0500 Subject: [PATCH 1/5] changed file from .txt to .log --- packages/apex-node/src/logs/logService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apex-node/src/logs/logService.ts b/packages/apex-node/src/logs/logService.ts index ca77889a..d3ff2107 100644 --- a/packages/apex-node/src/logs/logService.ts +++ b/packages/apex-node/src/logs/logService.ts @@ -49,7 +49,7 @@ export class LogService { if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }); } - const filePath = path.join(`${outputDir}`, `${id}.txt`); + const filePath = path.join(`${outputDir}`, `${id}.log`); const stream = fs.createWriteStream(filePath); stream.write(logRecord); stream.end(); From 63a7224c0560a4361737edbfed37af7a4b225542 Mon Sep 17 00:00:00 2001 From: Smit Shah Date: Tue, 21 Jul 2020 16:03:37 -0500 Subject: [PATCH 2/5] added .log file test --- .../apex-node/test/logs/logService.test.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/apex-node/test/logs/logService.test.ts b/packages/apex-node/test/logs/logService.test.ts index 2bd54caf..c74a8012 100644 --- a/packages/apex-node/test/logs/logService.test.ts +++ b/packages/apex-node/test/logs/logService.test.ts @@ -165,4 +165,21 @@ describe('Apex Log Service Tests', () => { expect(existsStub.callCount).to.eql(2); expect(mkdirStub.callCount).to.eql(1); }); -}); + + it('should successfully create a .log file', async () => { + const apexLogGet = new LogService(mockConnection); + const filePath = path.join('Users', 'smit.shah', 'Desktop', 'mod'); + const logIds = ['07WgsWfad', '9SiomgS']; + const logs = ['48jnskd', '57fskjf']; + const logsPath = path.join(filePath, `${logIds[0]}.log`) + const createStreamStub = sandboxStub.stub(fs, 'createWriteStream').returns({ + //@ts-ignore + write: () => {}, + end: () => {} + }); + toolingRequestStub.onFirstCall().resolves(logs[0]); + toolingRequestStub.onSecondCall().resolves(logs[1]); + await apexLogGet.writeLog(filePath,logs[0],logIds[0]); + expect(createStreamStub.calledWith(logsPath)).to.be.ok; + }); + }); From c1fa737f23aa20365caf31c173c84062a410b372 Mon Sep 17 00:00:00 2001 From: Smit Shah Date: Tue, 21 Jul 2020 16:36:53 -0500 Subject: [PATCH 3/5] updated .ok to be .true --- packages/apex-node/test/logs/logService.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apex-node/test/logs/logService.test.ts b/packages/apex-node/test/logs/logService.test.ts index c74a8012..8b5c2f18 100644 --- a/packages/apex-node/test/logs/logService.test.ts +++ b/packages/apex-node/test/logs/logService.test.ts @@ -180,6 +180,6 @@ describe('Apex Log Service Tests', () => { toolingRequestStub.onFirstCall().resolves(logs[0]); toolingRequestStub.onSecondCall().resolves(logs[1]); await apexLogGet.writeLog(filePath,logs[0],logIds[0]); - expect(createStreamStub.calledWith(logsPath)).to.be.ok; + expect(createStreamStub.calledWith(logsPath)).to.be.true; }); }); From 039f1f41cf11495aded94b9427eda24f07c2333c Mon Sep 17 00:00:00 2001 From: Luis Campos Date: Tue, 21 Jul 2020 19:43:27 -0700 Subject: [PATCH 4/5] Fix ERR_STREAM_WRITE_AFTER_END when processing more than one log file --- packages/apex-node/src/logs/logService.ts | 23 ++---- .../apex-node/src/utils/fileSystemHandler.ts | 37 ++++++++++ packages/apex-node/src/utils/index.ts | 8 +++ .../apex-node/test/logs/logService.test.ts | 57 +++++++-------- .../test/utils/fileSystemHandler.test.ts | 70 +++++++++++++++++++ 5 files changed, 146 insertions(+), 49 deletions(-) create mode 100644 packages/apex-node/src/utils/fileSystemHandler.ts create mode 100644 packages/apex-node/src/utils/index.ts create mode 100644 packages/apex-node/test/utils/fileSystemHandler.test.ts diff --git a/packages/apex-node/src/logs/logService.ts b/packages/apex-node/src/logs/logService.ts index d3ff2107..c1058805 100644 --- a/packages/apex-node/src/logs/logService.ts +++ b/packages/apex-node/src/logs/logService.ts @@ -8,8 +8,8 @@ import { Connection } from '@salesforce/core'; import { ApexLogGetOptions } from '../types'; import { QueryResult } from '../types/common'; import { nls } from '../i18n'; -import * as fs from 'fs'; import * as path from 'path'; +import { createFiles } from '../utils'; const MAX_NUM_LOGS = 25; @@ -29,30 +29,21 @@ export class LogService { logIdList.push(options.logId); } + const saveLogsMap = new Map(); + const connectionRequests = logIdList.map(async id => { const url = `${this.connection.tooling._baseUrl()}/sobjects/ApexLog/${id}/Body`; const logRecord = await this.toolingRequest(url); if (options.outputDir) { - this.writeLog(options.outputDir, logRecord, id); + saveLogsMap.set(path.join(options.outputDir, `${id}.log`), logRecord); } return logRecord; }); + const result = await Promise.all(connectionRequests); - return result; - } + createFiles(saveLogsMap); - public async writeLog( - outputDir: string, - logRecord: string, - id: string - ): Promise { - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - const filePath = path.join(`${outputDir}`, `${id}.log`); - const stream = fs.createWriteStream(filePath); - stream.write(logRecord); - stream.end(); + return result; } public async getLogIds(numberOfLogs: number): Promise { diff --git a/packages/apex-node/src/utils/fileSystemHandler.ts b/packages/apex-node/src/utils/fileSystemHandler.ts new file mode 100644 index 00000000..b327249e --- /dev/null +++ b/packages/apex-node/src/utils/fileSystemHandler.ts @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +export function ensureDirectoryExists(filePath: string): void { + if (fs.existsSync(filePath)) { + return; + } + ensureDirectoryExists(path.dirname(filePath)); + fs.mkdirSync(filePath); +} + +export function ensureFileExists(filePath: string): void { + ensureDirectoryExists(path.dirname(filePath)); + fs.closeSync(fs.openSync(filePath, 'w')); +} + +/** + * Method to save multiple files on disk. + * + * @param fileMap key = filePath, value = file contents + */ +export function createFiles(fileMap: Map): void { + for (const filePath of fileMap.keys()) { + ensureFileExists(filePath); + + const writeStream = fs.createWriteStream(filePath); + writeStream.write(fileMap.get(filePath)); + writeStream.end(); + } +} diff --git a/packages/apex-node/src/utils/index.ts b/packages/apex-node/src/utils/index.ts new file mode 100644 index 00000000..daa1fb31 --- /dev/null +++ b/packages/apex-node/src/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +export { createFiles } from './fileSystemHandler'; diff --git a/packages/apex-node/test/logs/logService.test.ts b/packages/apex-node/test/logs/logService.test.ts index 8b5c2f18..ce3de61e 100644 --- a/packages/apex-node/test/logs/logService.test.ts +++ b/packages/apex-node/test/logs/logService.test.ts @@ -12,6 +12,7 @@ import * as fs from 'fs'; import { createSandbox, SinonSandbox, SinonStub } from 'sinon'; import { LogService } from '../../src/logs/logService'; import * as path from 'path'; +import * as stream from 'stream'; const $$ = testSetup(); @@ -19,7 +20,6 @@ describe('Apex Log Service Tests', () => { const testData = new MockTestOrgData(); let mockConnection: Connection; let sandboxStub: SinonSandbox; - let mkdirStub: SinonStub; let toolingRequestStub: SinonStub; beforeEach(async () => { @@ -32,7 +32,6 @@ describe('Apex Log Service Tests', () => { username: testData.username }) }); - mkdirStub = sandboxStub.stub(fs, 'mkdirSync'); toolingRequestStub = sandboxStub.stub( LogService.prototype, 'toolingRequest' @@ -132,7 +131,15 @@ describe('Apex Log Service Tests', () => { const filePath = path.join('file', 'path', 'logs'); const logIds = ['07WgsWfad', '9SiomgS']; sandboxStub.stub(LogService.prototype, 'getLogIds').resolves(logIds); + const createStreamStub = sandboxStub.stub(fs, 'createWriteStream'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createStreamStub.onCall(0).returns(new stream.PassThrough() as any); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createStreamStub.onCall(1).returns(new stream.PassThrough() as any); + sandboxStub.stub(fs, 'closeSync'); + sandboxStub.stub(fs, 'openSync'); + const logs = ['48jnskd', '57fskjf']; toolingRequestStub.onFirstCall().resolves(logs[0]); toolingRequestStub.onSecondCall().resolves(logs[1]); @@ -144,42 +151,26 @@ describe('Apex Log Service Tests', () => { expect(createStreamStub.callCount).to.eql(2); }); - it('should create directory if it does not exist', async () => { + it('should successfully create a .log file', async () => { const apexLogGet = new LogService(mockConnection); - const filePath = path.join('Users', 'smit.shah', 'Desktop', 'mod'); - const logIds = ['07WgsWfad', '9SiomgS']; - const logs = ['48jnskd', '57fskjf']; - sandboxStub.stub(LogService.prototype, 'getLogIds').resolves(logIds); - const existsStub = sandboxStub.stub(fs, 'existsSync'); - existsStub.onFirstCall().returns(false); - existsStub.onSecondCall().returns(true); + const filePath = path.join('path', 'to', 'logs'); + const logIds = ['07WgsWfad']; + const logs = ['log content']; + const logsPath = path.join(filePath, `${logIds[0]}.log`); + sandboxStub.stub(fs, 'existsSync').returns(true); const createStreamStub = sandboxStub.stub(fs, 'createWriteStream'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createStreamStub.onCall(0).returns(new stream.PassThrough() as any); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createStreamStub.onCall(1).returns(new stream.PassThrough() as any); + sandboxStub.stub(fs, 'closeSync'); + sandboxStub.stub(fs, 'openSync'); toolingRequestStub.onFirstCall().resolves(logs[0]); toolingRequestStub.onSecondCall().resolves(logs[1]); - const response = await apexLogGet.getLogs({ - numberOfLogs: 2, + await apexLogGet.getLogs({ + logId: '07WgsWfad', outputDir: filePath }); - expect(response.length).to.eql(2); - expect(createStreamStub.callCount).to.eql(2); - expect(existsStub.callCount).to.eql(2); - expect(mkdirStub.callCount).to.eql(1); - }); - - it('should successfully create a .log file', async () => { - const apexLogGet = new LogService(mockConnection); - const filePath = path.join('Users', 'smit.shah', 'Desktop', 'mod'); - const logIds = ['07WgsWfad', '9SiomgS']; - const logs = ['48jnskd', '57fskjf']; - const logsPath = path.join(filePath, `${logIds[0]}.log`) - const createStreamStub = sandboxStub.stub(fs, 'createWriteStream').returns({ - //@ts-ignore - write: () => {}, - end: () => {} - }); - toolingRequestStub.onFirstCall().resolves(logs[0]); - toolingRequestStub.onSecondCall().resolves(logs[1]); - await apexLogGet.writeLog(filePath,logs[0],logIds[0]); expect(createStreamStub.calledWith(logsPath)).to.be.true; }); - }); +}); diff --git a/packages/apex-node/test/utils/fileSystemHandler.test.ts b/packages/apex-node/test/utils/fileSystemHandler.test.ts new file mode 100644 index 00000000..8d36e357 --- /dev/null +++ b/packages/apex-node/test/utils/fileSystemHandler.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import * as fsUtil from '../../src/utils/fileSystemHandler'; +import { createSandbox, SinonStub } from 'sinon'; +import { expect } from 'chai'; +import { join } from 'path'; +import * as fs from 'fs'; + +const sb = createSandbox(); + +describe('File System Utils', () => { + describe('ensureDirectoryExists', () => { + let mkdirStub: SinonStub; + let existsStub: SinonStub; + + beforeEach(() => { + mkdirStub = sb.stub(fs, 'mkdirSync'); + existsStub = sb.stub(fs, 'existsSync'); + }); + + afterEach(() => { + sb.restore(); + }); + + it('should return immediately if file or directory already exists', () => { + const path = join('path', 'to', 'dir'); + existsStub.withArgs(path).returns(true); + + fsUtil.ensureDirectoryExists(path); + + expect(mkdirStub.notCalled).to.be.true; + }); + + it('should create nested directories as needed', () => { + const path = join('path', 'to'); + const path2 = join(path, 'dir'); + const path3 = join(path2, 'dir2'); + existsStub.returns(false); + existsStub.withArgs(path).returns(true); + + fsUtil.ensureDirectoryExists(path3); + + expect(mkdirStub.firstCall.args[0]).to.equal(path2); + expect(mkdirStub.secondCall.args[0]).to.equal(path3); + }); + }); + + describe('ensureFileExists', () => { + afterEach(() => { + sb.restore(); + }); + + it('should ensure file exists', () => { + const path = join('path', 'to', 'a', 'file.x'); + const closeStub = sb.stub(fs, 'closeSync'); + const openStub = sb.stub(fs, 'openSync'); + openStub.returns(123); + const existsSyncStub = sb.stub(fs, 'existsSync').returns(true); + + fsUtil.ensureFileExists(path); + + expect(existsSyncStub.calledBefore(openStub)).to.be.true; + expect(closeStub.firstCall.args[0]).to.equal(123); + }); + }); +}); From aefc0acd9f394c09b4d2da24476a3db9b04f1408 Mon Sep 17 00:00:00 2001 From: Luis Campos Date: Wed, 22 Jul 2020 19:20:38 -0700 Subject: [PATCH 5/5] Save log file immediately after getting the api response --- packages/apex-node/src/logs/logService.ts | 7 ++----- packages/apex-node/src/utils/fileSystemHandler.ts | 12 +++++------- packages/apex-node/src/utils/index.ts | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/apex-node/src/logs/logService.ts b/packages/apex-node/src/logs/logService.ts index c1058805..2d5f1879 100644 --- a/packages/apex-node/src/logs/logService.ts +++ b/packages/apex-node/src/logs/logService.ts @@ -9,7 +9,7 @@ import { ApexLogGetOptions } from '../types'; import { QueryResult } from '../types/common'; import { nls } from '../i18n'; import * as path from 'path'; -import { createFiles } from '../utils'; +import { createFile } from '../utils'; const MAX_NUM_LOGS = 25; @@ -29,19 +29,16 @@ export class LogService { logIdList.push(options.logId); } - const saveLogsMap = new Map(); - const connectionRequests = logIdList.map(async id => { const url = `${this.connection.tooling._baseUrl()}/sobjects/ApexLog/${id}/Body`; const logRecord = await this.toolingRequest(url); if (options.outputDir) { - saveLogsMap.set(path.join(options.outputDir, `${id}.log`), logRecord); + createFile(path.join(options.outputDir, `${id}.log`), logRecord); } return logRecord; }); const result = await Promise.all(connectionRequests); - createFiles(saveLogsMap); return result; } diff --git a/packages/apex-node/src/utils/fileSystemHandler.ts b/packages/apex-node/src/utils/fileSystemHandler.ts index b327249e..7eebc3a6 100644 --- a/packages/apex-node/src/utils/fileSystemHandler.ts +++ b/packages/apex-node/src/utils/fileSystemHandler.ts @@ -22,16 +22,14 @@ export function ensureFileExists(filePath: string): void { } /** - * Method to save multiple files on disk. + * Method to save a file on disk. * - * @param fileMap key = filePath, value = file contents + * @param filePath path where to + * @param fileContent file contents */ -export function createFiles(fileMap: Map): void { - for (const filePath of fileMap.keys()) { +export function createFile(filePath: string, fileContent: string): void { ensureFileExists(filePath); const writeStream = fs.createWriteStream(filePath); - writeStream.write(fileMap.get(filePath)); - writeStream.end(); - } + writeStream.write(fileContent); } diff --git a/packages/apex-node/src/utils/index.ts b/packages/apex-node/src/utils/index.ts index daa1fb31..f0594503 100644 --- a/packages/apex-node/src/utils/index.ts +++ b/packages/apex-node/src/utils/index.ts @@ -5,4 +5,4 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -export { createFiles } from './fileSystemHandler'; +export { createFile } from './fileSystemHandler';