#2207 - Investigate Root Cause of Camunda Workers Getting Stuck

sources/packages/backend/apps/workers/src/controllers/application/application.controller.ts

@@ -1,4 +1,4 @@
-import { Controller } from "@nestjs/common";
+import { Controller, Logger } from "@nestjs/common";
 import { ZeebeWorker } from "../../zeebe";
 import {
   ZeebeJob,
@@ -24,6 +24,7 @@ import {
 import { APPLICATION_ID } from "@sims/services/workflow/variables/assessment-gateway";
 import { MaxJobsToActivate } from "../../types";
 import { Workers } from "@sims/services/constants";
+import { createUnexpectedJobFail } from "../../utilities";
 
 @Controller()
 export class ApplicationController {
@@ -52,18 +53,26 @@ export class ApplicationController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    const updateResult = await this.applicationService.updateStatus(
-      job.variables.applicationId,
-      job.customHeaders.fromStatus,
-      job.customHeaders.toStatus,
-    );
-    if (!updateResult.affected) {
-      return job.error(
-        APPLICATION_STATUS_NOT_UPDATED,
-        "The application status was not updated either because the application id was not found or the application is not in the expected status.",
+    const jobLogger = new Logger(job.type);
+    try {
+      const updateResult = await this.applicationService.updateStatus(
+        job.variables.applicationId,
+        job.customHeaders.fromStatus,
+        job.customHeaders.toStatus,
       );
+      if (!updateResult.affected) {
+        const message =
+          "The application status was not updated either because the application id was not found or the application is not in the expected status.";
+        jobLogger.error(message);
+        return job.error(APPLICATION_STATUS_NOT_UPDATED, message);
+      }
+      jobLogger.log("Updated the application status.");
+      return job.complete();
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
     }
-    return job.complete();
   }
 
   /**
@@ -85,36 +94,48 @@ export class ApplicationController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    const application = await this.applicationService.getApplicationById(
-      job.variables.applicationId,
-      { loadDynamicData: true },
-    );
-    if (!application) {
-      return job.error(APPLICATION_NOT_FOUND, "Application id not found.");
-    }
-    if (application.applicationException) {
-      // The exceptions were already processed for this application.
+    const jobLogger = new Logger(job.type);
+    try {
+      const application = await this.applicationService.getApplicationById(
+        job.variables.applicationId,
+        { loadDynamicData: true },
+      );
+      if (!application) {
+        const message = "Application id not found.";
+        jobLogger.error(message);
+        return job.error(APPLICATION_NOT_FOUND, message);
+      }
+      if (application.applicationException) {
+        // The exceptions were already processed for this application.
+        jobLogger.log("Exceptions were already processed for the application.");
+        return job.complete({
+          applicationExceptionStatus:
+            application.applicationException.exceptionStatus,
+        });
+      }
+      // Check for application exceptions present in the application dynamic data.
+      const exceptions = this.applicationExceptionService.searchExceptions(
+        application.data,
+      );
+      if (exceptions.length) {
+        const createdException =
+          await this.applicationExceptionService.createException(
+            job.variables.applicationId,
+            exceptions,
+          );
+        jobLogger.log("Exception created.");
+        return job.complete({
+          applicationExceptionStatus: createdException.exceptionStatus,
+        });
+      }
+      jobLogger.log("Verified application exception.");
       return job.complete({
-        applicationExceptionStatus:
-          application.applicationException.exceptionStatus,
+        applicationExceptionStatus: ApplicationExceptionStatus.Approved,
       });
-    }
-    // Check for application exceptions present in the application dynamic data.
-    const exceptions = this.applicationExceptionService.searchExceptions(
-      application.data,
-    );
-    if (exceptions.length) {
-      const createdException =
-        await this.applicationExceptionService.createException(
-          job.variables.applicationId,
-          exceptions,
-        );
-      return job.complete({
-        applicationExceptionStatus: createdException.exceptionStatus,
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
       });
     }
-    return job.complete({
-      applicationExceptionStatus: ApplicationExceptionStatus.Approved,
-    });
   }
 }

sources/packages/backend/apps/workers/src/controllers/assessment/assessment.controller.ts

@@ -25,7 +25,10 @@ import {
   SupportingUser,
   SupportingUserType,
 } from "@sims/sims-db";
-import { filterObjectProperties } from "../../utilities";
+import {
+  createUnexpectedJobFail,
+  filterObjectProperties,
+} from "../../utilities";
 import {
   ASSESSMENT_ALREADY_ASSOCIATED_TO_WORKFLOW,
   ASSESSMENT_INVALID_OPERATION_IN_THE_CURRENT_STATE,
@@ -62,25 +65,29 @@ export class AssessmentController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
+    const jobLogger = new Logger(job.type);
     try {
       await this.studentAssessmentService.associateWorkflowId(
         job.variables.assessmentId,
         job.processInstanceKey,
       );
+      jobLogger.log("Associated the assessment id.");
       return job.complete();
     } catch (error: unknown) {
       if (error instanceof CustomNamedError) {
         switch (error.name) {
           case ASSESSMENT_ALREADY_ASSOCIATED_TO_WORKFLOW:
+            jobLogger.log(`${error.name} ${error.message}`);
             return job.complete();
           case ASSESSMENT_NOT_FOUND:
           case ASSESSMENT_INVALID_OPERATION_IN_THE_CURRENT_STATE:
+            jobLogger.error(`${error.name} ${error.message}`);
             return job.error(error.name, error.message);
         }
       }
-      return job.fail(
-        `Not able to associate the assessment id ${job.variables.assessmentId} with the workflow instance id ${job.processInstanceKey}. ${error}`,
-      );
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
     }
   }
 
@@ -103,18 +110,28 @@ export class AssessmentController {
       ZeebeJob<AssessmentDataJobInDTO, ICustomHeaders, IOutputVariables>
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    const assessment = await this.studentAssessmentService.getById(
-      job.variables.assessmentId,
-    );
-    if (!assessment) {
-      return job.error(ASSESSMENT_NOT_FOUND, "Assessment not found.");
+    const jobLogger = new Logger(job.type);
+    try {
+      const assessment = await this.studentAssessmentService.getById(
+        job.variables.assessmentId,
+      );
+      if (!assessment) {
+        const message = "Assessment not found.";
+        jobLogger.error(message);
+        return job.error(ASSESSMENT_NOT_FOUND, message);
+      }
+      const assessmentDTO = this.transformToAssessmentDTO(assessment);
+      const outputVariables = filterObjectProperties(
+        assessmentDTO,
+        job.customHeaders,
+      );
+      jobLogger.log("Assessment consolidated data loaded.");
+      return job.complete(outputVariables);
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
     }
-    const assessmentDTO = this.transformToAssessmentDTO(assessment);
-    const outputVariables = filterObjectProperties(
-      assessmentDTO,
-      job.customHeaders,
-    );
-    return job.complete(outputVariables);
   }
 
   /**
@@ -129,11 +146,19 @@ export class AssessmentController {
       ZeebeJob<SaveAssessmentDataJobInDTO, ICustomHeaders, IOutputVariables>
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    await this.studentAssessmentService.updateAssessmentData(
-      job.variables.assessmentId,
-      job.variables.assessmentData,
-    );
-    return job.complete();
+    const jobLogger = new Logger(job.type);
+    try {
+      await this.studentAssessmentService.updateAssessmentData(
+        job.variables.assessmentId,
+        job.variables.assessmentData,
+      );
+      jobLogger.log("Assessment data saved.");
+      return job.complete();
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
+    }
   }
 
   /**
@@ -152,11 +177,19 @@ export class AssessmentController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    await this.studentAssessmentService.updateNOAApprovalStatus(
-      job.variables.assessmentId,
-      job.customHeaders.status,
-    );
-    return job.complete();
+    const jobLogger = new Logger(job.type);
+    try {
+      await this.studentAssessmentService.updateNOAApprovalStatus(
+        job.variables.assessmentId,
+        job.customHeaders.status,
+      );
+      jobLogger.log("NOA status updated.");
+      return job.complete();
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
+    }
   }
 
   /**
@@ -171,17 +204,18 @@ export class AssessmentController {
       ZeebeJob<WorkflowWrapUpJobInDTO, ICustomHeaders, IOutputVariables>
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
+    const jobLogger = new Logger(job.type);
     try {
       await this.studentAssessmentService.updateAssessmentStatusAndSaveWorkflowData(
         job.variables.assessmentId,
         job.variables.workflowData,
       );
+      jobLogger.log("Updated assessment status and saved the workflow data.");
       return job.complete();
     } catch (error: unknown) {
-      const jobLogger = new Logger(job.type);
-      const errorMessage = `Failed while updating assessment status and saving workflow data. ${error}`;
-      jobLogger.error(errorMessage);
-      return job.fail(errorMessage);
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
     }
   }
 

sources/packages/backend/apps/workers/src/controllers/cra-integration/cra-integration.controller.ts

@@ -1,4 +1,4 @@
-import { Controller } from "@nestjs/common";
+import { Controller, Logger } from "@nestjs/common";
 import { ZeebeWorker } from "../../zeebe";
 import {
   ZeebeJob,
@@ -22,6 +22,7 @@ import {
 } from "@sims/services/workflow/variables/cra-integration-income-verification";
 import { MaxJobsToActivate } from "../../types";
 import { Workers } from "@sims/services/constants";
+import { createUnexpectedJobFail } from "../../utilities";
 
 @Controller()
 export class CRAIntegrationController {
@@ -51,20 +52,27 @@ export class CRAIntegrationController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    const incomeRequest =
-      await this.incomeVerificationService.createIncomeVerification(
-        job.variables.applicationId,
-        job.variables.taxYear,
-        job.variables.reportedIncome,
-        job.variables.supportingUserId,
-      );
-    const [identifier] = incomeRequest.identifiers;
-
-    await this.incomeVerificationService.checkForCRAIncomeVerificationBypass(
-      identifier.id,
-    );
+    const jobLogger = new Logger(job.type);
+    try {
+      const incomeRequest =
+        await this.incomeVerificationService.createIncomeVerification(
+          job.variables.applicationId,
+          job.variables.taxYear,
+          job.variables.reportedIncome,
+          job.variables.supportingUserId,
+        );
+      const [identifier] = incomeRequest.identifiers;
 
-    return job.complete({ incomeVerificationId: identifier.id });
+      await this.incomeVerificationService.checkForCRAIncomeVerificationBypass(
+        identifier.id,
+      );
+      jobLogger.log("CRA income verification created.");
+      return job.complete({ incomeVerificationId: identifier.id });
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
+    }
   }
 
   /**
@@ -85,10 +93,18 @@ export class CRAIntegrationController {
       >
     >,
   ): Promise<MustReturnJobActionAcknowledgement> {
-    const incomeVerificationCompleted =
-      await this.incomeVerificationService.isIncomeVerificationCompleted(
-        job.variables.incomeVerificationId,
-      );
-    return job.complete({ incomeVerificationCompleted });
+    const jobLogger = new Logger(job.type);
+    try {
+      const incomeVerificationCompleted =
+        await this.incomeVerificationService.isIncomeVerificationCompleted(
+          job.variables.incomeVerificationId,
+        );
+      jobLogger.log("CRA income verification completed.");
+      return job.complete({ incomeVerificationCompleted });
+    } catch (error: unknown) {
+      return createUnexpectedJobFail(error, job, {
+        logger: jobLogger,
+      });
+    }
   }
 }