#4079 - Queue Monitoring - Schedulers Refactor - (MSFAA) (#4141)

- Refactored MSFAA-related schedulers and adjusted E2E tests. - full-time-msfaa-integration - part-time-msfaa-process-integration - part-time-msfaa-process-response-integration - full-time-msfaa-process-response-integration - Additional Refactor - As agreed in the previous PR, added extra logs to the `BaseQueue` start and end.
bcgov · Dec 20, 2024 · 31d6777 · 31d6777
1 parent 60166a6
commit 31d6777
Show file tree

Hide file tree

Showing 10 changed files with 301 additions and 357 deletions.
diff --git a/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/base-queue.ts b/sources/packages/backend/apps/queue-consumers/src/processors/schedulers/base-queue.ts
@@ -26,11 +26,16 @@ export abstract class BaseQueue<T> {
   async processQueue(job: Job<T>): Promise<string | string[]> {
     const processSummary = new ProcessSummary();
     try {
-      processSummary.info(
-        `Processing queue ${job.queue.name}, job ID ${job.id}.`,
-      );
+      // Log the start of the process for the summary and logger.
+      const startLogMessage = `Processing queue ${job.queue.name}, job ID ${job.id}.`;
+      processSummary.info(startLogMessage);
+      this.logger.log(startLogMessage);
+      // Execute the process.
       const result = await this.process(job, processSummary);
-      processSummary.info(`${job.queue.name}, job ID ${job.id}, executed.`);
+      // Log the end of the process for the summary and logger.
+      const endLogMessage = `${job.queue.name}, job ID ${job.id}, finished.`;
+      processSummary.info(endLogMessage);
+      this.logger.log(endLogMessage);
       const logsSum = processSummary.getLogLevelSum();
       if (logsSum.error) {
         throw new CustomNamedError(

diff --git a/...aa-integration/_tests_/msfaa-full-time-process-response-integration.scheduler.e2e-spec.ts b/...aa-integration/_tests_/msfaa-full-time-process-response-integration.scheduler.e2e-spec.ts
@@ -1,9 +1,10 @@
-import { DeepMocked, createMock } from "@golevelup/ts-jest";
+import { DeepMocked } from "@golevelup/ts-jest";
 import { INestApplication } from "@nestjs/common";
 import { QueueNames, getISODateOnlyString } from "@sims/utilities";
 import {
   createTestingAppModule,
   describeProcessorRootTest,
+  mockBullJob,
 } from "../../../../../../test/helpers";
 import { FullTimeMSFAAProcessResponseIntegrationScheduler } from "../msfaa-full-time-process-response-integration.scheduler";
 import {
@@ -27,7 +28,6 @@ import {
   NotificationMessageType,
 } from "@sims/sims-db";
 import * as Client from "ssh2-sftp-client";
-import { Job } from "bull";
 import * as path from "path";
 import { FULL_TIME_SAMPLE_MSFAA_NUMBER } from "./msfaa-process-integration.scheduler.models";
 import { IsNull } from "typeorm";
@@ -140,26 +140,29 @@ describe(
         },
       );
       // Queued job.
-      const job = createMock<Job<void>>();
+      const mockedJob = mockBullJob<void>();
       mockDownloadFiles(sftpClientMock, [
         MSFAA_FULL_TIME_RECEIVE_FILE_WITH_REACTIVATION_RECORD,
       ]);
+
       // Act
       // Now reactivate the cancelled MSFAA.
-      const processResult = await processor.processMSFAA(job);
+      const result = await processor.processQueue(mockedJob.job);
+
       // Assert
-      expect(processResult).toStrictEqual([
-        {
-          processSummary: [
-            `Processing file ${MSFAA_FULL_TIME_RECEIVE_FILE_WITH_REACTIVATION_RECORD}.`,
-            "File contains:",
-            "Confirmed MSFAA records (type R): 1.",
-            "Cancelled MSFAA records (type C): 0.",
-            "Record from line 2, updated as confirmed.",
-          ],
-          errorsSummary: [],
-        },
+      expect(result).toStrictEqual([
+        "MSFAA full-time response files processed.",
       ]);
+      expect(
+        mockedJob.containLogMessages([
+          `Processing file ${MSFAA_FULL_TIME_RECEIVE_FILE_WITH_REACTIVATION_RECORD}.`,
+          "File contains:",
+          "Confirmed MSFAA records (type R): 1.",
+          "Cancelled MSFAA records (type C): 0.",
+          "Record from line 2, updated as confirmed.",
+        ]),
+      ).toBe(true);
+
       // Assert that the file was archived from SFTP.
       expect(sftpClientMock.rename).toHaveBeenCalled();
       // Find the updated MSFAA records previously created.
@@ -259,12 +262,27 @@ describe(
         ),
       );
       // Queued job.
-      const job = createMock<Job<void>>();
+      const mockedJob = mockBullJob<void>();
       mockDownloadFiles(sftpClientMock, [
         MSFAA_FULL_TIME_RECEIVE_FILE_WITH_SINGLE_CANCELLATION_RECORD,
       ]);
       // Act
-      const processResult = await processor.processMSFAA(job);
+      const result = await processor.processQueue(mockedJob.job);
+
+      // Assert
+      expect(result).toStrictEqual([
+        "MSFAA full-time response files processed.",
+      ]);
+      expect(
+        mockedJob.containLogMessages([
+          `Processing file ${MSFAA_FULL_TIME_RECEIVE_FILE_WITH_SINGLE_CANCELLATION_RECORD}.`,
+          "File contains:",
+          "Confirmed MSFAA records (type R): 0.",
+          "Cancelled MSFAA records (type C): 1.",
+          "Record from line 2, updated as cancelled.",
+        ]),
+      ).toBe(true);
+      // Assert database changes.
       const cancelledMSFAARecord = await db.msfaaNumber.findOne({
         select: {
           cancelledDate: true,
@@ -292,19 +310,6 @@ describe(
           },
         },
       });
-      // Assert
-      expect(processResult).toStrictEqual([
-        {
-          processSummary: [
-            `Processing file ${MSFAA_FULL_TIME_RECEIVE_FILE_WITH_SINGLE_CANCELLATION_RECORD}.`,
-            "File contains:",
-            "Confirmed MSFAA records (type R): 0.",
-            "Cancelled MSFAA records (type C): 1.",
-            "Record from line 2, updated as cancelled.",
-          ],
-          errorsSummary: [],
-        },
-      ]);
       expect(cancelledMSFAARecord.cancelledDate).toBe("2021-11-24");
       expect(cancelledMSFAARecord.newIssuingProvince).toBe("ON");
       expect(notification.dateSent).toBe(null);

diff --git a/...ation/msfaa-integration/_tests_/msfaa-part-time-process-integration.scheduler.e2e-spec.ts b/...ation/msfaa-integration/_tests_/msfaa-part-time-process-integration.scheduler.e2e-spec.ts
@@ -1,10 +1,10 @@
-import { Job } from "bull";
-import { DeepMocked, createMock } from "@golevelup/ts-jest";
+import { DeepMocked } from "@golevelup/ts-jest";
 import { INestApplication } from "@nestjs/common";
 import { QueueNames, getISODateOnlyString } from "@sims/utilities";
 import {
   createTestingAppModule,
   describeProcessorRootTest,
+  mockBullJob,
 } from "../../../../../../test/helpers";
 import { PartTimeMSFAAProcessIntegrationScheduler } from "../msfaa-part-time-process-integration.scheduler";
 import { E2EDataSources, createE2EDataSources } from "@sims/test-utils";
@@ -70,13 +70,13 @@ describe(
         msfaaInputData,
       );
       // Queued job.
-      const job = createMock<Job<void>>();
+      const mockedJob = mockBullJob<void>();
       // Spy on the MSFAA content creation to intercept the process date and time used.
       const createMSFAARequestContentMock =
         createMSFAARequestContentSpyOnMock(app);
 
       // Act
-      const msfaaRequestResults = await processor.processMSFAA(job);
+      const result = await processor.processQueue(mockedJob.job);
 
       // Assert
       expect(createMSFAARequestContentMock).toHaveBeenCalledTimes(1);
@@ -87,14 +87,10 @@ describe(
         `MSFT-Request\\DPBC.EDU.MSFA.SENT.PT.${processDateFormatted}.010`,
       );
       // Assert process result.
-      expect(msfaaRequestResults).toStrictEqual([
-        {
-          generatedFile: uploadedFile.remoteFilePath,
-          uploadedRecords: msfaaInputData.length,
-          offeringIntensity: OfferingIntensity.partTime,
-        },
+      expect(result).toStrictEqual([
+        `Generated file: ${uploadedFile.remoteFilePath}`,
+        `Uploaded records: ${msfaaInputData.length}`,
       ]);
-
       // Assert file output.
       expect(uploadedFile.fileLines?.length).toBe(msfaaInputData.length + 2);
       const [