Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#2207 - Investigate Root Cause of Camunda Workers Getting Stuck #2219

Merged
merged 10 commits into from
Aug 31, 2023
Merged

#2207 - Investigate Root Cause of Camunda Workers Getting Stuck #2219

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7a7b499
added try catch
ann-aot Aug 25, 2023
75d83fb
global handler fix and log
ann-aot Aug 30, 2023
279eb78
comments
ann-aot Aug 30, 2023
df70c1f
refactot
ann-aot Aug 30, 2023
d9a9c7f
comment
ann-aot Aug 30, 2023
485fb5a
created a util function for unexcepted error
ann-aot Aug 30, 2023
46f3060
refactor
ann-aot Aug 30, 2023
d952c64
refactor
ann-aot Aug 30, 2023
c0feb43
upstream
ann-aot Aug 30, 2023
99da3f7
feedback
ann-aot Aug 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 59 additions & 39 deletions sources/packages/backend/apps/workers/src/controllers/application/application.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Controller } from "@nestjs/common";
import { Controller, Logger } from "@nestjs/common";
import { ZeebeWorker } from "../../zeebe";
import {
ZeebeJob,
Expand Down Expand Up @@ -52,18 +52,26 @@ export class ApplicationController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const updateResult = await this.applicationService.updateStatus(
job.variables.applicationId,
job.customHeaders.fromStatus,
job.customHeaders.toStatus,
);
if (!updateResult.affected) {
return job.error(
APPLICATION_STATUS_NOT_UPDATED,
"The application status was not updated either because the application id was not found or the application is not in the expected status.",
const jobLogger = new Logger(job.type);
try {
const updateResult = await this.applicationService.updateStatus(
job.variables.applicationId,
job.customHeaders.fromStatus,
job.customHeaders.toStatus,
);
if (!updateResult.affected) {
const message =
"The application status was not updated either because the application id was not found or the application is not in the expected status.";
jobLogger.error(message);
return job.error(APPLICATION_STATUS_NOT_UPDATED, message);
}
jobLogger.log("Updated the application status.");
return job.complete();
} catch (error: unknown) {
const errorMessage = `Unexpected error while updating the application status. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
return job.complete();
}

/**
Expand All @@ -85,36 +93,48 @@ export class ApplicationController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const application = await this.applicationService.getApplicationById(
job.variables.applicationId,
{ loadDynamicData: true },
);
if (!application) {
return job.error(APPLICATION_NOT_FOUND, "Application id not found.");
}
if (application.applicationException) {
// The exceptions were already processed for this application.
return job.complete({
applicationExceptionStatus:
application.applicationException.exceptionStatus,
});
}
// Check for application exceptions present in the application dynamic data.
const exceptions = this.applicationExceptionService.searchExceptions(
application.data,
);
if (exceptions.length) {
const createdException =
await this.applicationExceptionService.createException(
job.variables.applicationId,
exceptions,
);
const jobLogger = new Logger(job.type);
try {
const application = await this.applicationService.getApplicationById(
job.variables.applicationId,
{ loadDynamicData: true },
);
if (!application) {
const message = "Application id not found.";
jobLogger.error(message);
return job.error(APPLICATION_NOT_FOUND, message);
}
if (application.applicationException) {
// The exceptions were already processed for this application.
jobLogger.log("Exceptions were already processed for the application.");
return job.complete({
applicationExceptionStatus:
application.applicationException.exceptionStatus,
});
}
// Check for application exceptions present in the application dynamic data.
const exceptions = this.applicationExceptionService.searchExceptions(
application.data,
);
if (exceptions.length) {
const createdException =
await this.applicationExceptionService.createException(
job.variables.applicationId,
exceptions,
);
jobLogger.log("Exception created.");
return job.complete({
applicationExceptionStatus: createdException.exceptionStatus,
});
}
jobLogger.log("Verified application exception.");
return job.complete({
applicationExceptionStatus: createdException.exceptionStatus,
applicationExceptionStatus: ApplicationExceptionStatus.Approved,
});
} catch (error: unknown) {
const errorMessage = `Unexpected error while verifying the application exceptions. ${error}`;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because we would be repeating this along all controller and jobs, I was expecting that a basic helper would be created, as shown below.

} catch (error: unknown) {
  return createUnexpectedJobFail(error, job);
}

The helper method can be something like below.

createUnexpectedJobFail(
    error: unknown,
    job: ZeebeJob,
    logger?: Logger,
  ): MustReturnJobActionAcknowledgement {
    const jobLogger = logger ?? new Logger(job.type);
    const errorMessage = `Unexpected error while processing job. ${parseException(error)}`;
    jobLogger.error(errorMessage);
    return job.fail(errorMessage);
  }

As an out-of-PR-scope suggestion, I believe that we can create the below error formatter to avoid logging as jobLogger.log(${error.name} ${error.message});.

parseException(error: unknown): string {
  return JSON.stringify(error, null, 2);
}

jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
return job.complete({
applicationExceptionStatus: ApplicationExceptionStatus.Approved,
});
}
}
80 changes: 55 additions & 25 deletions sources/packages/backend/apps/workers/src/controllers/assessment/assessment.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Controller } from "@nestjs/common";
import { Controller, Logger } from "@nestjs/common";
import { ZeebeWorker } from "../../zeebe";
import {
ZeebeJob,
Expand Down Expand Up @@ -60,25 +60,29 @@ export class AssessmentController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const jobLogger = new Logger(job.type);
try {
await this.studentAssessmentService.associateWorkflowId(
job.variables.assessmentId,
job.processInstanceKey,
);
jobLogger.log("Associated the assessment id.");
return job.complete();
} catch (error: unknown) {
if (error instanceof CustomNamedError) {
switch (error.name) {
case ASSESSMENT_ALREADY_ASSOCIATED_TO_WORKFLOW:
jobLogger.log(`${error.name} ${error.message}`);
return job.complete();
case ASSESSMENT_NOT_FOUND:
case ASSESSMENT_INVALID_OPERATION_IN_THE_CURRENT_STATE:
jobLogger.error(`${error.name} ${error.message}`);
return job.error(error.name, error.message);
}
}
return job.fail(
`Not able to associate the assessment id ${job.variables.assessmentId} with the workflow instance id ${job.processInstanceKey}. ${error}`,
);
const errorMessage = `Not able to associate the assessment id ${job.variables.assessmentId} with the workflow instance id ${job.processInstanceKey}. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}

Expand All @@ -101,18 +105,28 @@ export class AssessmentController {
ZeebeJob<AssessmentDataJobInDTO, ICustomHeaders, IOutputVariables>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const assessment = await this.studentAssessmentService.getById(
job.variables.assessmentId,
);
if (!assessment) {
return job.error(ASSESSMENT_NOT_FOUND, "Assessment not found.");
const jobLogger = new Logger(job.type);
try {
const assessment = await this.studentAssessmentService.getById(
job.variables.assessmentId,
);
if (!assessment) {
const message = "Assessment not found.";
jobLogger.error(message);
return job.error(ASSESSMENT_NOT_FOUND, message);
}
const assessmentDTO = this.transformToAssessmentDTO(assessment);
const outputVariables = filterObjectProperties(
assessmentDTO,
job.customHeaders,
);
jobLogger.log("Assessment consolidated data loaded.");
return job.complete(outputVariables);
} catch (error: unknown) {
const errorMessage = `Unexpected error while loading assessment consolidated data. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
const assessmentDTO = this.transformToAssessmentDTO(assessment);
const outputVariables = filterObjectProperties(
assessmentDTO,
job.customHeaders,
);
return job.complete(outputVariables);
}

/**
Expand All @@ -127,11 +141,19 @@ export class AssessmentController {
ZeebeJob<SaveAssessmentDataJobInDTO, ICustomHeaders, IOutputVariables>
>,
): Promise<MustReturnJobActionAcknowledgement> {
await this.studentAssessmentService.updateAssessmentData(
job.variables.assessmentId,
job.variables.assessmentData,
);
return job.complete();
const jobLogger = new Logger(job.type);
try {
await this.studentAssessmentService.updateAssessmentData(
job.variables.assessmentId,
job.variables.assessmentData,
);
jobLogger.log("Assessment data saved.");
return job.complete();
} catch (error: unknown) {
const errorMessage = `Unexpected error saving the assessment data. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}

/**
Expand All @@ -150,11 +172,19 @@ export class AssessmentController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
await this.studentAssessmentService.updateNOAApprovalStatus(
job.variables.assessmentId,
job.customHeaders.status,
);
return job.complete();
const jobLogger = new Logger(job.type);
try {
await this.studentAssessmentService.updateNOAApprovalStatus(
job.variables.assessmentId,
job.customHeaders.status,
);
jobLogger.log("NOA status updated.");
return job.complete();
} catch (error: unknown) {
const errorMessage = `Unexpected error while updating the NOA status. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}

/**
Expand Down
53 changes: 34 additions & 19 deletions ...ckages/backend/apps/workers/src/controllers/cra-integration/cra-integration.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Controller } from "@nestjs/common";
import { Controller, Logger } from "@nestjs/common";
import { ZeebeWorker } from "../../zeebe";
import {
ZeebeJob,
Expand Down Expand Up @@ -51,20 +51,27 @@ export class CRAIntegrationController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const incomeRequest =
await this.incomeVerificationService.createIncomeVerification(
job.variables.applicationId,
job.variables.taxYear,
job.variables.reportedIncome,
job.variables.supportingUserId,
);
const [identifier] = incomeRequest.identifiers;

await this.incomeVerificationService.checkForCRAIncomeVerificationBypass(
identifier.id,
);
const jobLogger = new Logger(job.type);
try {
const incomeRequest =
await this.incomeVerificationService.createIncomeVerification(
job.variables.applicationId,
job.variables.taxYear,
job.variables.reportedIncome,
job.variables.supportingUserId,
);
const [identifier] = incomeRequest.identifiers;

return job.complete({ incomeVerificationId: identifier.id });
await this.incomeVerificationService.checkForCRAIncomeVerificationBypass(
identifier.id,
);
jobLogger.log("CRA income verification created.");
return job.complete({ incomeVerificationId: identifier.id });
} catch (error: unknown) {
const errorMessage = `Unexpected error while creating the CRA income verification. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}

/**
Expand All @@ -85,10 +92,18 @@ export class CRAIntegrationController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const incomeVerificationCompleted =
await this.incomeVerificationService.isIncomeVerificationCompleted(
job.variables.incomeVerificationId,
);
return job.complete({ incomeVerificationCompleted });
const jobLogger = new Logger(job.type);
try {
const incomeVerificationCompleted =
await this.incomeVerificationService.isIncomeVerificationCompleted(
job.variables.incomeVerificationId,
);
jobLogger.log("CRA income verification completed.");
return job.complete({ incomeVerificationCompleted });
} catch (error: unknown) {
const errorMessage = `Unexpected error while checking the CRA income verification. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}
}
22 changes: 15 additions & 7 deletions ...ces/packages/backend/apps/workers/src/controllers/disbursement/disbursement.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Controller } from "@nestjs/common";
import { Controller, Logger } from "@nestjs/common";
import { ZeebeWorker } from "../../zeebe";
import {
ZeebeJob,
Expand Down Expand Up @@ -56,25 +56,29 @@ export class DisbursementController {
>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const jobLogger = new Logger(job.type);
try {
await this.disbursementScheduleSharedService.createDisbursementSchedules(
job.variables.assessmentId,
job.variables.disbursementSchedules,
);
jobLogger.log("Created disbursement schedule");
return job.complete();
} catch (error: unknown) {
if (error instanceof CustomNamedError) {
switch (error.name) {
case DISBURSEMENT_SCHEDULES_ALREADY_CREATED:
jobLogger.log(`${error.name} ${error.message}`);
return job.complete();
case ASSESSMENT_NOT_FOUND:
case ASSESSMENT_INVALID_OPERATION_IN_THE_CURRENT_STATE:
jobLogger.error(`${error.name} ${error.message}`);
return job.error(error.name, error.message);
}
}
return job.fail(
`Unexpected error while creating disbursement schedules. ${error}`,
);
const errorMessage = `Unexpected error while creating disbursement schedules. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}

Expand All @@ -92,25 +96,29 @@ export class DisbursementController {
ZeebeJob<AssignMSFAAJobInDTO, ICustomHeaders, IOutputVariables>
>,
): Promise<MustReturnJobActionAcknowledgement> {
const jobLogger = new Logger(job.type);
try {
await this.disbursementScheduleService.associateMSFAANumber(
job.variables.assessmentId,
);
jobLogger.error("Associated the MSFAA number to the disbursements.");
return job.complete();
} catch (error: unknown) {
if (error instanceof CustomNamedError) {
switch (error.name) {
case DISBURSEMENT_NOT_FOUND:
case INVALID_OPERATION_IN_THE_CURRENT_STATUS:
case APPLICATION_INVALID_DATA_TO_CREATE_MSFAA_ERROR:
jobLogger.error(`${error.name} ${error.message}`);
return job.error(error.name, error.message);
case DISBURSEMENT_MSFAA_ALREADY_ASSOCIATED:
jobLogger.log(`${error.name} ${error.message}`);
return job.complete();
}
}
return job.fail(
`Unexpected error while associating the MSFAA number to the disbursements. ${error}`,
);
const errorMessage = `Unexpected error while associating the MSFAA number to the disbursements. ${error}`;
jobLogger.error(errorMessage);
return job.fail(errorMessage);
}
}
}
Loading