Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#3745 - Modify process that reads SIN & CRA verification response files #3835

Merged
merged 25 commits into from
Oct 30, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
7200 20240101 BCSAP00000 00000001 0
720110000000120240001NAME JOHN 1999010110000000000 0
7201100000001202400025077 111 AA DUMMY OO V0Y 0V0 0
7201100000001202400220001010101010101010010000VERIFICATION_ID:CRA_INCOME_VERIFICATIONBCSA 0
720110000000120240150000050000 15000 0
7201100000001202400220001010101010101010010000010101010100100010101010100100010101010BCSA 0
720110000000120241010000050000 10100 0
72011000000012024A010BC 0
72011000000012024A015BC 0
72011000000012024A030N00000000 0
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { createMock, DeepMocked } from "@golevelup/ts-jest";
import { INestApplication } from "@nestjs/common";
import { QueueNames } from "@sims/utilities";
import {
createTestingAppModule,
describeProcessorRootTest,
} from "../../../../../test/helpers";
import {
E2EDataSources,
createE2EDataSources,
createFakeCRAIncomeVerification,
saveFakeApplication,
saveFakeStudent,
} from "@sims/test-utils";
import * as Client from "ssh2-sftp-client";
import * as path from "path";
import { CRAResponseIntegrationScheduler } from "../cra-response-integration.scheduler";
import {
createFileFromStructuredRecords,
getStructuredRecords,
mockDownloadFiles,
} from "@sims/test-utils/mocks";
import { Job } from "bull";
import { ApplicationStatus } from "@sims/sims-db";

const CRA_FILENAME = "CRA_200_PBCSA00000.TXT";

describe(describeProcessorRootTest(QueueNames.CRAResponseIntegration), () => {
let app: INestApplication;
let processor: CRAResponseIntegrationScheduler;
let db: E2EDataSources;
let sftpClientMock: DeepMocked<Client>;
let craResponseFolder: string;

beforeAll(async () => {
craResponseFolder = path.join(__dirname, "cra-receive-files");
process.env.CRA_RESPONSE_FOLDER = craResponseFolder;
const { nestApplication, dataSource, sshClientMock } =
await createTestingAppModule();
app = nestApplication;
db = createE2EDataSources(dataSource);
sftpClientMock = sshClientMock;
processor = app.get(CRAResponseIntegrationScheduler);
});

beforeEach(async () => {
jest.clearAllMocks();
});

it("Should process SIN response file ignoring non-SIMS records when the file contains responses from requests that were not created by SIMS.", async () => {
// Arrange.
const student = await saveFakeStudent(db.dataSource);

const application = await saveFakeApplication(
db.dataSource,
{ student },
{ applicationStatus: ApplicationStatus.InProgress },
);

// Create CRA income verifications for student.
const studentCRAIncomeVerification = createFakeCRAIncomeVerification({
application,
});
await db.craIncomeVerification.save([studentCRAIncomeVerification]);
// Queued job.
const job = createMock<Job<void>>();
mockDownloadFiles(sftpClientMock, [CRA_FILENAME]);

mockDownloadFiles(sftpClientMock, [CRA_FILENAME], (fileContent: string) => {
const file = getStructuredRecords(fileContent);
file.records[2] = file.records[2].replace(
"CRA_INCOME_VERIFICATION",
studentCRAIncomeVerification.id.toString().padStart(9, "0"),
);
return createFileFromStructuredRecords(file);
});

// Act
const processResult = await processor.processResponses(job);
// Assert
const downloadedFile = path.join(
process.env.CRA_RESPONSE_FOLDER,
CRA_FILENAME,
);

// Assert
expect(processResult).toStrictEqual([
{
processSummary: [
`Processing file ${downloadedFile}.`,
"File contains 2 verifications.",
"Processed income verification. Total income record line 5. Status record from line 4.",
],
errorsSummary: [],
},
]);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
00100500220220921BC1
0026000000014Y100000001YYNY
0026000000025Y100000002YYNY
999000014000000200000003
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
import { createMock, DeepMocked } from "@golevelup/ts-jest";
import { INestApplication } from "@nestjs/common";
import { QueueNames } from "@sims/utilities";
import {
createTestingAppModule,
describeProcessorRootTest,
} from "../../../../../../test/helpers";
import {
E2EDataSources,
createE2EDataSources,
saveFakeStudent,
} from "@sims/test-utils";
import * as Client from "ssh2-sftp-client";
import * as path from "path";
import { SINValidationResponseIntegrationScheduler } from "../sin-validation-process-response-integration.scheduler";
import {
createFileFromStructuredRecords,
getStructuredRecords,
mockDownloadFiles,
} from "@sims/test-utils/mocks";
import { Job } from "bull";

const SIN_VALIDATION_FILENAME = "PCSLP.PBC.BC0000.ISR";

describe(
describeProcessorRootTest(QueueNames.SINValidationResponseIntegration),
() => {
let app: INestApplication;
let processor: SINValidationResponseIntegrationScheduler;
let db: E2EDataSources;
let sftpClientMock: DeepMocked<Client>;
let sinValidationResponseFolder: string;

beforeAll(async () => {
sinValidationResponseFolder = path.join(__dirname, "sin-receive-files");
process.env.ESDC_RESPONSE_FOLDER = sinValidationResponseFolder;
const { nestApplication, dataSource, sshClientMock } =
await createTestingAppModule();
app = nestApplication;
db = createE2EDataSources(dataSource);
sftpClientMock = sshClientMock;
processor = app.get(SINValidationResponseIntegrationScheduler);
});

beforeEach(async () => {
jest.clearAllMocks();
});

it("Should skip process SIN response file when no SIN validation was updated because the record id is already present.", async () => {
// Arrange
// Create a SIN record with REFERENCE_IDX = 600000001
const validSinStudent = await saveFakeStudent(db.dataSource, undefined, {
sinValidationInitialValue: {
sin: "100000001",
isValidSIN: true,
},
});

// Queued job.
const job = createMock<Job<void>>();
mockDownloadFiles(sftpClientMock, [SIN_VALIDATION_FILENAME]);

mockDownloadFiles(
sftpClientMock,
[SIN_VALIDATION_FILENAME],
(fileContent: string) => {
const file = getStructuredRecords(fileContent);
file.records[0] = file.records[0].replace(
// Assuming the first 3 characters are some identifier, replace everything between that and position 12
file.records[0].substring(3, 12),
validSinStudent.sinValidation.id.toString().padStart(9, "0"),
);
return createFileFromStructuredRecords(file);
},
);

// Act
const processResult = await processor.processSINValidationResponse(job);
// Assert
const downloadedFile = path.join(
process.env.ESDC_RESPONSE_FOLDER,
SIN_VALIDATION_FILENAME,
);

// Assert
expect(processResult).toStrictEqual([
{
processSummary: [
`Processing file ${downloadedFile}.`,
"File contains 2 SIN validations.",
"Processed SIN validation record from line 2: No SIN validation was updated because the record id is already present and this is not the most updated.",
"Processed SIN validation record from line 3: Not able to find the SIN validation on line number 3 to be updated with the ESDC response.",
],
errorsSummary: [],
},
]);
});

it("Should update one SIN validation record and skip one when one SIN response is from SIMS and the other is from SFAS.", async () => {
// Arrange
// Create a SIN record with REFERENCE_IDX = 600000002 and dateReceived = null to process the SIN validation record updated.
const validSinStudent = await saveFakeStudent(db.dataSource, undefined, {
sinValidationInitialValue: {
sin: "100000002",
isValidSIN: false,
dateReceived: null,
},
});

// Queued job.
const job = createMock<Job<void>>();
mockDownloadFiles(sftpClientMock, [SIN_VALIDATION_FILENAME]);

mockDownloadFiles(
sftpClientMock,
[SIN_VALIDATION_FILENAME],
(fileContent: string) => {
const file = getStructuredRecords(fileContent);
file.records[0] = file.records[0].replace(
// Assuming the first 3 characters are some identifier, replace everything between that and position 12
file.records[0].substring(3, 12),
validSinStudent.sinValidation.id.toString().padStart(9, "0"),
);
return createFileFromStructuredRecords(file);
},
);

// Act
const processResult = await processor.processSINValidationResponse(job);
// Assert
const downloadedFile = path.join(
process.env.ESDC_RESPONSE_FOLDER,
SIN_VALIDATION_FILENAME,
);

// Assert
expect(processResult).toStrictEqual([
{
processSummary: [
`Processing file ${downloadedFile}.`,
"File contains 2 SIN validations.",
"Processed SIN validation record from line 2: SIN validation record updated.",
"Processed SIN validation record from line 3: Not able to find the SIN validation on line number 3 to be updated with the ESDC response.",
],
errorsSummary: [],
},
]);
});
},
);
Original file line number Diff line number Diff line change
@@ -57,6 +57,6 @@ export interface SINValidationRecord {
* description of the operation executed.
*/
export interface SINValidationUpdateResult {
record: SINValidation;
record?: SINValidation;
operationDescription: string;
}
Original file line number Diff line number Diff line change
@@ -121,9 +121,9 @@ export class SINValidationService extends RecordDataModelService<SINValidation>
.getOne();

if (!existingValidation) {
throw new Error(
`Not able to find the SIN validation id ${validationResponse.referenceIndex} to be updated with the ESDC response.`,
);
return {
operationDescription: `Not able to find the SIN validation on line number ${validationResponse.lineNumber} to be updated with the ESDC response.`,
};
}

const sinValidationNeverUpdated = !existingValidation.dateReceived;
Original file line number Diff line number Diff line change
@@ -23,7 +23,10 @@ export function createFakeSINValidation(
options?.initialValue?.sin ??
faker.datatype.number({ min: 100000000, max: 899999999 }).toString();
sinValidation.dateSent = now;
sinValidation.dateReceived = now;
sinValidation.dateReceived =
options?.initialValue?.dateReceived !== undefined
? options.initialValue.dateReceived
: now;
sinValidation.fileSent = null;
sinValidation.fileReceived = null;
sinValidation.givenNameSent = null;