From 757bb07da83532639c272f7c90d410e16864d480 Mon Sep 17 00:00:00 2001 From: Andrew Boni Signori Date: Tue, 17 Dec 2024 17:12:01 -0800 Subject: [PATCH] Fed restrictions and receipt refactors --- ...receipts-integration.scheduler.e2e-spec.ts | 191 +++++++++--------- ...ursement-receipts-integration.scheduler.ts | 64 +++--- ...eral-restrictions-integration.scheduler.ts | 4 +- .../test/helpers/mock-utils/job-mock-utils.ts | 10 +- ...disbursement-receipt.processing.service.ts | 80 ++++---- 5 files changed, 178 insertions(+), 171 deletions(-) diff --git a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/_tests_/disbursement-receipts-integration.scheduler.e2e-spec.ts b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/_tests_/disbursement-receipts-integration.scheduler.e2e-spec.ts index 4af87ca869..9c7032c1b0 100644 --- a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/_tests_/disbursement-receipts-integration.scheduler.e2e-spec.ts +++ b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/_tests_/disbursement-receipts-integration.scheduler.e2e-spec.ts @@ -136,20 +136,23 @@ describe( return createFileFromStructuredRecords(file); }, ); - // Queued job. - const { job } = mockBullJob(); - - // Act - const [result] = await processor.processDisbursementReceipts(job); - - // Assert - const downloadedFile = path.join( + const expectedFileName = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_ONLY_FULL_TIME_FILE, ); - expect(result.errorsSummary).toContain( - `Error downloading file ${downloadedFile}. Error: Error: Invalid file header.`, - ); + // Queued job. + const mockedJob = mockBullJob(); + + // Act/Assert + await expect(processor.processQueue(mockedJob.job)).rejects.toThrowError( + "One or more errors were reported during the process, please see logs for details.", + ); + expect( + mockedJob.containLogMessages([ + `Error downloading file ${expectedFileName}.`, + "Invalid file header.", + ]), + ).toBe(true); }); it("Should log 'SIN Hash validation failed' error when the footer has an invalid SIN total hash.", async () => { @@ -167,20 +170,24 @@ describe( return createFileFromStructuredRecords(file); }, ); - // Queued job. - const { job } = mockBullJob(); - - // Act - const [result] = await processor.processDisbursementReceipts(job); - - // Assert - const downloadedFile = path.join( + // Expected file name. + const expectedFileName = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_ONLY_FULL_TIME_FILE, ); - expect(result.errorsSummary).toContain( - `Error downloading file ${downloadedFile}. Error: Error: SIN Hash validation failed.`, - ); + // Queued job. + const mockedJob = mockBullJob(); + + // Act/Assert + await expect(processor.processQueue(mockedJob.job)).rejects.toThrowError( + "One or more errors were reported during the process, please see logs for details.", + ); + expect( + mockedJob.containLogMessages([ + `Error downloading file ${expectedFileName}.`, + "SIN Hash validation failed.", + ]), + ).toBe(true); }); it("Should log 'Invalid file footer' error when the footer has a record code different than 'T'.", async () => { @@ -195,20 +202,23 @@ describe( return createFileFromStructuredRecords(file); }, ); - // Queued job. - const { job } = mockBullJob(); - - // Act - const [result] = await processor.processDisbursementReceipts(job); - - // Assert - const downloadedFile = path.join( + const expectedFileName = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_ONLY_FULL_TIME_FILE, ); - expect(result.errorsSummary).toContain( - `Error downloading file ${downloadedFile}. Error: Error: Invalid file footer.`, - ); + // Queued job. + const mockedJob = mockBullJob(); + + // Act/Assert + await expect(processor.processQueue(mockedJob.job)).rejects.toThrow( + "One or more errors were reported during the process, please see logs for details.", + ); + expect( + mockedJob.containLogMessages([ + `Error downloading file ${expectedFileName}.`, + "Invalid file footer.", + ]), + ).toBe(true); }); it("Should import disbursement receipt file and create federal and provincial awards receipts with proper awards code mappings for a full-time application when the file contains federal and provincial receipts.", async () => { @@ -222,29 +232,29 @@ describe( }); mockDownloadFiles(sftpClientMock, [FEDERAL_PROVINCIAL_FULL_TIME_FILE]); // Queued job. - const { job } = mockBullJob(); + const mockedJob = mockBullJob(); // Act - const result = await processor.processDisbursementReceipts(job); + const result = await processor.processQueue(mockedJob.job); // Assert + expect(result).toStrictEqual([ + "Completed disbursement receipts integration.", + ]); const downloadedFile = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_PROVINCIAL_FULL_TIME_FILE, ); - expect(result).toStrictEqual([ - { - processSummary: [ - `Processing file ${downloadedFile}.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, - `Processing file ${downloadedFile} completed.`, - `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, - "Provincial daily disbursement CSV report generated.", - ], - errorsSummary: [], - }, - ]); + expect( + mockedJob.containLogMessages([ + `Processing file ${downloadedFile}.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, + `Processing file ${downloadedFile} completed.`, + `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, + "Provincial daily disbursement CSV report generated.", + ]), + ).toBe(true); // Assert imported receipts. const { bcReceipt, feReceipt } = await getReceiptsForAssert( FILE_DATE, @@ -353,27 +363,27 @@ describe( }); mockDownloadFiles(sftpClientMock, [FEDERAL_ONLY_FULL_TIME_FILE]); // Queued job. - const { job } = mockBullJob(); + const mockedJob = mockBullJob(); // Act - const result = await processor.processDisbursementReceipts(job); + const result = await processor.processQueue(mockedJob.job); // Assert + expect(result).toStrictEqual([ + "Completed disbursement receipts integration.", + ]); const downloadedFile = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_ONLY_FULL_TIME_FILE, ); - expect(result).toStrictEqual([ - { - processSummary: [ - `Processing file ${downloadedFile}.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, - `Processing file ${downloadedFile} completed.`, - `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, - ], - errorsSummary: [], - }, - ]); + expect( + mockedJob.containLogMessages([ + `Processing file ${downloadedFile}.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, + `Processing file ${downloadedFile} completed.`, + `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, + ]), + ).toBe(true); // Assert imported receipts. const { feReceipt, bcReceipt } = await getReceiptsForAssert( FILE_DATE, @@ -430,10 +440,10 @@ describe( }); mockDownloadFiles(sftpClientMock, [FEDERAL_PROVINCIAL_PART_TIME_FILE]); // Queued job. - const { job } = mockBullJob(); + const mockedJob = mockBullJob(); // Act - const result = await processor.processDisbursementReceipts(job); + const result = await processor.processQueue(mockedJob.job); // Assert const downloadedFile = path.join( @@ -441,18 +451,18 @@ describe( FEDERAL_PROVINCIAL_PART_TIME_FILE, ); expect(result).toStrictEqual([ - { - processSummary: [ - `Processing file ${downloadedFile}.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, - `Processing file ${downloadedFile} completed.`, - `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, - "Provincial daily disbursement CSV report generated.", - ], - errorsSummary: [], - }, + "Completed disbursement receipts integration.", ]); + expect( + mockedJob.containLogMessages([ + `Processing file ${downloadedFile}.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, + `Processing file ${downloadedFile} completed.`, + `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, + "Provincial daily disbursement CSV report generated.", + ]), + ).toBe(true); // Assert imported receipts. const { bpReceipt, feReceipt } = await getReceiptsForAssert( FILE_DATE, @@ -563,30 +573,31 @@ describe( FEDERAL_PROVINCIAL_FULL_TIME_PART_TIME_FILE, ]); // Queued job. - const { job } = mockBullJob(); + const mockedJob = mockBullJob(); // Act - const result = await processor.processDisbursementReceipts(job); + const result = await processor.processQueue(mockedJob.job); // Assert + expect(result).toStrictEqual([ + "Completed disbursement receipts integration.", + ]); const downloadedFile = path.join( process.env.ESDC_RESPONSE_FOLDER, FEDERAL_PROVINCIAL_FULL_TIME_PART_TIME_FILE, ); - expect(result).toEqual([ - { - processSummary: [ - `Processing file ${downloadedFile}.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, - `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 4 inserted successfully.`, - `Processing file ${downloadedFile} completed.`, - `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, - "Provincial daily disbursement CSV report generated.", - ], - errorsSummary: [], - }, - ]); + expect( + mockedJob.containLogMessages([ + `Processing file ${downloadedFile}.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 2 inserted successfully.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 3 inserted successfully.`, + `Record with document number ${SHARED_DOCUMENT_NUMBER} at line 4 inserted successfully.`, + `Processing file ${downloadedFile} completed.`, + `Processing provincial daily disbursement CSV file on ${FILE_DATE}.`, + "Provincial daily disbursement CSV report generated.", + ]), + ).toBe(true); + // Assert imported receipts. const { feReceipt, bcReceipt, bpReceipt } = await getReceiptsForAssert( FILE_DATE, diff --git a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/disbursement-receipts-integration.scheduler.ts b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/disbursement-receipts-integration.scheduler.ts index 6d6111158e..26e3f81ad2 100644 --- a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/disbursement-receipts-integration.scheduler.ts +++ b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/ecert-integration/disbursement-receipts-integration.scheduler.ts @@ -1,12 +1,15 @@ -import { InjectQueue, Process, Processor } from "@nestjs/bull"; +import { InjectQueue, Processor } from "@nestjs/bull"; import { DisbursementReceiptProcessingService } from "@sims/integrations/esdc-integration"; import { QueueService } from "@sims/services/queue"; import { SystemUsersService } from "@sims/services/system-users"; import { QueueNames } from "@sims/utilities"; import { Job, Queue } from "bull"; -import { ProcessSummary } from "@sims/utilities/logger"; import { BaseScheduler } from "../../base-scheduler"; -import { ESDCFileResponse } from "../models/esdc.models"; +import { + InjectLogger, + LoggerService, + ProcessSummary, +} from "@sims/utilities/logger"; @Processor(QueueNames.DisbursementReceiptsFileIntegration) export class DisbursementReceiptsFileIntegrationScheduler extends BaseScheduler { @@ -21,43 +24,30 @@ export class DisbursementReceiptsFileIntegrationScheduler extends BaseScheduler< } /** - * To be removed once the method {@link process} is implemented. - * This method "hides" the {@link Process} decorator from the base class. + * Process all the disbursement receipt files from remote SFTP location. + * @param _job process job. + * @param processSummary process summary for logging. + * @returns processing result. */ - async processQueue(): Promise { - throw new Error("Method not implemented."); - } - - /** - * When implemented in a derived class, process the queue job. - * To be implemented. - */ - protected async process(): Promise { - throw new Error("Method not implemented."); + protected async process( + _job: Job, + processSummary: ProcessSummary, + ): Promise { + const auditUser = this.systemUsersService.systemUser; + await this.disbursementReceiptProcessingService.process( + auditUser.id, + processSummary, + ); + return "Completed disbursement receipts integration."; } /** - * Process all the disbursement receipt files from remote sftp location. - * @params job job details. - * @returns Summary details of processing. + * Logger for SFAS integration scheduler. + * Setting the logger here allows the correct context to be set + * during the property injection. + * Even if the logger is not used, it is required to be set, to + * allow the base classes to write logs using the correct context. */ - @Process() - async processDisbursementReceipts( - job: Job, - ): Promise { - const processSummary = new ProcessSummary(); - processSummary.info( - `Processing full time disbursement receipts integration job ${job.id} of type ${job.name}.`, - ); - const auditUser = this.systemUsersService.systemUser; - const processResponse = - await this.disbursementReceiptProcessingService.process(auditUser.id); - processSummary.info( - `Completed full time disbursement receipts integration job ${job.id} of type ${job.name}.`, - ); - return processResponse.map((response) => ({ - processSummary: response.processSummary, - errorsSummary: response.errorsSummary, - })); - } + @InjectLogger() + logger: LoggerService; } diff --git a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/federal-restrictions-integration/federal-restrictions-integration.scheduler.ts b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/federal-restrictions-integration/federal-restrictions-integration.scheduler.ts index 9ee1582626..396fa2130e 100644 --- a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/federal-restrictions-integration/federal-restrictions-integration.scheduler.ts +++ b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/federal-restrictions-integration/federal-restrictions-integration.scheduler.ts @@ -23,12 +23,12 @@ export class FederalRestrictionsIntegrationScheduler extends BaseScheduler /** * Federal restriction import. - * @param job process job. + * @param _job process job. * @param processSummary process summary for logging. * @returns processing result. */ protected async process( - job: Job, + _job: Job, processSummary: ProcessSummary, ): Promise { processSummary.info("Starting federal restrictions import."); diff --git a/sources/packages/backend/apps/queue-consumers/test/helpers/mock-utils/job-mock-utils.ts b/sources/packages/backend/apps/queue-consumers/test/helpers/mock-utils/job-mock-utils.ts index 75db17a417..f66f190f80 100644 --- a/sources/packages/backend/apps/queue-consumers/test/helpers/mock-utils/job-mock-utils.ts +++ b/sources/packages/backend/apps/queue-consumers/test/helpers/mock-utils/job-mock-utils.ts @@ -11,17 +11,17 @@ export class MockBullJobResult { ) {} /** - * Checks if the logs contains a entry that ends with the provided parameter. - * @param logMessage log message to be found using string endsWith method. + * Checks if the logs contains a entry that includes the provided parameter. + * @param logMessage log message to be found using string includes method. * @returns true if the log message was found. otherwise false. */ containLogMessage(logMessage: string): boolean { - return this.logMessages.some((message) => message.endsWith(logMessage)); + return this.logMessages.some((message) => message.includes(logMessage)); } /** - * Checks if the logs contains a entry that ends with the provided parameter. - * @param logMessages log messages to be found using string endsWith method. + * Checks if the logs contains a entry that includes the provided parameter. + * @param logMessages log messages to be found using string includes method. * @returns true if the log message was found. otherwise false. */ containLogMessages(logMessages: string[]): boolean { diff --git a/sources/packages/backend/libs/integrations/src/esdc-integration/disbursement-receipt-integration/disbursement-receipt.processing.service.ts b/sources/packages/backend/libs/integrations/src/esdc-integration/disbursement-receipt-integration/disbursement-receipt.processing.service.ts index de49783ffc..c2fb200843 100644 --- a/sources/packages/backend/libs/integrations/src/esdc-integration/disbursement-receipt-integration/disbursement-receipt.processing.service.ts +++ b/sources/packages/backend/libs/integrations/src/esdc-integration/disbursement-receipt-integration/disbursement-receipt.processing.service.ts @@ -1,7 +1,10 @@ import { Injectable } from "@nestjs/common"; -import { LoggerService, InjectLogger } from "@sims/utilities/logger"; +import { + LoggerService, + InjectLogger, + ProcessSummary, +} from "@sims/utilities/logger"; import { DisbursementReceiptIntegrationService } from "./disbursement-receipt.integration.service"; -import { ProcessSFTPResponseResult } from "../models/esdc-integration.model"; import { DisbursementReceiptDownloadResponse } from "./models/disbursement-receipt-integration.model"; import { ConfigService, ESDCIntegrationConfig } from "@sims/utilities/config"; import { @@ -14,7 +17,7 @@ import { ReportsFilterModel, } from "@sims/services"; import { DAILY_DISBURSEMENT_REPORT_NAME } from "@sims/services/constants"; -import { getISODateOnlyString, parseJSONError } from "@sims/utilities"; +import { getISODateOnlyString } from "@sims/utilities"; /** * Disbursement schedule map which consists of disbursement schedule id for a document number. @@ -45,9 +48,13 @@ export class DisbursementReceiptProcessingService { * Process all the available disbursement receipt files in SFTP location. * Once the file is processed, it gets archived. * @param auditUserId user that should be considered the one that is causing the changes. + * @param processSummary process summary for logging. * @returns Summary details of the processing. */ - async process(auditUserId: number): Promise { + async process( + auditUserId: number, + processSummary: ProcessSummary, + ): Promise { // Get the list of all files from SFTP ordered by file name. const fileSearch = new RegExp( `^${this.esdcConfig.environmentCode}EDU\\.PBC\\.DIS.[0-9]{8}\\.[0-9]{3}$`, @@ -57,11 +64,15 @@ export class DisbursementReceiptProcessingService { this.esdcConfig.ftpResponseFolder, fileSearch, ); - const result: ProcessSFTPResponseResult[] = []; for (const filePath of filePaths) { - result.push(await this.processAllReceiptsInFile(filePath, auditUserId)); + const fileProcessSummary = new ProcessSummary(); + processSummary.children(fileProcessSummary); + await this.processAllReceiptsInFile( + filePath, + auditUserId, + fileProcessSummary, + ); } - return result; } /** @@ -69,14 +80,15 @@ export class DisbursementReceiptProcessingService { * insert the records to database. * @param remoteFilePath file which is to be processed. * @param auditUserId user that should be considered the one that is causing the changes. + * @param processSummary process summary for logging. * @returns result processing summary. */ private async processAllReceiptsInFile( remoteFilePath: string, auditUserId: number, - ): Promise { - const result = new ProcessSFTPResponseResult(); - result.processSummary.push(`Processing file ${remoteFilePath}.`); + processSummary: ProcessSummary, + ): Promise { + processSummary.info(`Processing file ${remoteFilePath}.`); this.logger.log(`Starting download of file ${remoteFilePath}.`); let responseData: DisbursementReceiptDownloadResponse; try { @@ -85,12 +97,10 @@ export class DisbursementReceiptProcessingService { ); } catch (error: unknown) { this.logger.error(error); - result.errorsSummary.push( - `Error downloading file ${remoteFilePath}. Error: ${error}`, - ); - return result; + processSummary.error(`Error downloading file ${remoteFilePath}.`, error); + return; } - + // File download is successful, import the receipts. const documentNumbers = responseData.records.map( (record) => record.documentNumber, ); @@ -128,33 +138,32 @@ export class DisbursementReceiptProcessingService { createdAt, ); if (generatedIdentifier) { - result.processSummary.push( + processSummary.info( `Record with document number ${response.documentNumber} at line ${response.lineNumber} inserted successfully.`, ); } else { - result.processSummary.push( + processSummary.info( `Record with document number ${response.documentNumber} at line ${response.lineNumber} has been ignored as the receipt already exist.`, ); } } else { - result.processSummary.push( + processSummary.info( `Document number ${response.documentNumber} at line ${response.lineNumber} not found in SIMS.`, ); } } catch (error: unknown) { - this.logger.error(error); const logMessage = `Unexpected error while processing disbursement receipt record at line ${response.lineNumber}`; - result.errorsSummary.push(logMessage); - this.logger.error(logMessage); - this.logger.error(error); + this.logger.error(logMessage, error); + processSummary.error(logMessage, error); + return; } } - result.processSummary.push(`Processing file ${remoteFilePath} completed.`); + processSummary.info(`Processing file ${remoteFilePath} completed.`); await this.processSendingFile( - result, responseData.header.fileDate, responseData.header.sequenceNumber, + processSummary, ); try { @@ -162,26 +171,24 @@ export class DisbursementReceiptProcessingService { await this.integrationService.archiveFile(remoteFilePath); } catch (error) { const logMessage = `Unexpected error while archiving disbursement receipt file: ${remoteFilePath}.`; - result.errorsSummary.push(logMessage); - result.errorsSummary.push(parseJSONError(error)); this.logger.error(logMessage, error); + processSummary.error(logMessage, error); } - return result; } /** * Create a provincial daily disbursement CSV file and * email the content to the ministry users. - * @param result output of the processing steps. + * @param processSummary process summary for logging. * @param fileDate File date to be a part of filename. * @param sequenceNumber Sequence number to be a part of filename. */ private async processSendingFile( - result: ProcessSFTPResponseResult, fileDate: Date, sequenceNumber: number, + processSummary: ProcessSummary, ): Promise { - result.processSummary.push( + processSummary.info( `Processing provincial daily disbursement CSV file on ${getISODateOnlyString( fileDate, )}.`, @@ -220,18 +227,17 @@ export class DisbursementReceiptProcessingService { fileName: disbursementFileName, }, ); - result.processSummary.push( + processSummary.info( "Provincial daily disbursement CSV report generated.", ); this.logger.log( "Completed provincial daily disbursement report generation.", ); - } catch (error) { - this.logger.error(error); - result.errorsSummary.push( - "Error while generating provincial daily disbursement CSV report file.", - ); - result.errorsSummary.push(error); + } catch (error: unknown) { + const errorMessage = + "Error while generating provincial daily disbursement CSV report file."; + this.logger.error(errorMessage, error); + processSummary.error(errorMessage, error); } }