#3738 - SIMS to SFAS Part 2

.github/workflows/env-setup-deploy-secrets.yml

@@ -36,6 +36,7 @@ jobs:
       ESDC_RESPONSE_FOLDER: ${{ secrets.ESDC_RESPONSE_FOLDER }}
       ESDC_ENVIRONMENT_CODE: ${{ secrets.ESDC_ENVIRONMENT_CODE }}
       SFAS_RECEIVE_FOLDER: ${{ secrets.SFAS_RECEIVE_FOLDER }}
+      SFAS_SEND_FOLDER: ${{ vars.SFAS_SEND_FOLDER }}
       SIMS_DB_NAME: ${{ secrets.SIMS_DB_NAME }}
       INSTITUTION_REQUEST_FOLDER: ${{ secrets.INSTITUTION_REQUEST_FOLDER }}
       INSTITUTION_RESPONSE_FOLDER: ${{ vars.INSTITUTION_RESPONSE_FOLDER }}

configs/env-example

@@ -102,6 +102,7 @@ FILE_UPLOAD_ALLOWED_EXTENSIONS=.pdf,.doc,.docx,.jpg,.png
 
 #SFAS Integration
 SFAS_RECEIVE_FOLDER=
+SFAS_SEND_FOLDER=
 
 #Institution Integration
 INSTITUTION_REQUEST_FOLDER=

devops/Makefile

@@ -197,6 +197,7 @@ init-secrets:
 		-p ESDC_RESPONSE_FOLDER=$(ESDC_RESPONSE_FOLDER) \
 		-p ESDC_ENVIRONMENT_CODE=$(ESDC_ENVIRONMENT_CODE) \
 		-p SFAS_RECEIVE_FOLDER=$(SFAS_RECEIVE_FOLDER) \
+		-p SFAS_SEND_FOLDER=$(SFAS_SEND_FOLDER) \
 		-p SIMS_DB_NAME=$(SIMS_DB_NAME) \
 		-p INSTITUTION_REQUEST_FOLDER=$(INSTITUTION_REQUEST_FOLDER) \
 		-p INSTITUTION_RESPONSE_FOLDER=$(INSTITUTION_RESPONSE_FOLDER) \

devops/openshift/init-secrets.yml

@@ -23,6 +23,7 @@ objects:
       esdc-response-folder: ${ESDC_RESPONSE_FOLDER}
       esdc-environment-code: ${ESDC_ENVIRONMENT_CODE}
       sfas-receive-folder: ${SFAS_RECEIVE_FOLDER}
+      sfas-send-folder: ${SFAS_SEND_FOLDER}
       sims-db-name: ${SIMS_DB_NAME}
       institution-request-folder: ${INSTITUTION_REQUEST_FOLDER}
       institution-response-folder: ${INSTITUTION_RESPONSE_FOLDER}
@@ -142,6 +143,11 @@ parameters:
     required: true
     description: |
       Folder on the SFTP where SFAS integration files will be placed.
+  - name: SFAS_SEND_FOLDER
+    displayName: SFAS send folder
+    required: true
+    description: |
+      Folder on the SFTP where files sent to SFAS will be placed.
   - name: SIMS_DB_NAME
     displayName: SIMS database name
     required: true

devops/openshift/queue-consumers-deploy.yml

@@ -147,6 +147,11 @@ objects:
                     secretKeyRef:
                       key: ${SFAS_RECEIVE_FOLDER_NAME_KEY}
                       name: ${QUEUE_CONSUMERS_SECRET_NAME}
+                - name: SFAS_SEND_FOLDER
+                  valueFrom:
+                    secretKeyRef:
+                      key: ${SFAS_SEND_FOLDER_NAME_KEY}
+                      name: ${QUEUE_CONSUMERS_SECRET_NAME}
                 - name: CRA_REQUEST_FOLDER
                   valueFrom:
                     secretKeyRef:
@@ -448,6 +453,8 @@ parameters:
     value: gc-notify-api-key
   - name: SFAS_RECEIVE_FOLDER_NAME_KEY
     value: sfas-receive-folder
+  - name: SFAS_SEND_FOLDER_NAME_KEY
+    value: sfas-send-folder
   - name: ATBC_LOGIN_ENDPOINT
     required: true
   - name: ATBC_ENDPOINT

sources/Makefile

@@ -40,6 +40,7 @@ export FILE_UPLOAD_MAX_FILE_SIZE := $(or $(FILE_UPLOAD_MAX_FILE_SIZE), 15728640)
 export FILE_UPLOAD_ALLOWED_EXTENSIONS := $(or $(FILE_UPLOAD_ALLOWED_EXTENSIONS), .pdf,.doc,.docx,.jpg,.png)
 #SFAS Integration
 export SFAS_RECEIVE_FOLDER := $(or $(SFAS_RECEIVE_FOLDER), SFAS-Receive)
+export SFAS_SEND_FOLDER := $(or $(SFAS_SEND_FOLDER), OUT)
 # Fulltime Allowed
 export IS_FULLTIME_ALLOWED := $(or $(IS_FULLTIME_ALLOWED), true)
 #Institution Integration

sources/docker-compose.yml

@@ -159,6 +159,7 @@ services:
       - ZONE_B_SFTP_PRIVATE_KEY_PASSPHRASE=${ZONE_B_SFTP_PRIVATE_KEY_PASSPHRASE}
       - ZONE_B_SFTP_PRIVATE_KEY=${ZONE_B_SFTP_PRIVATE_KEY}
       - SFAS_RECEIVE_FOLDER=${SFAS_RECEIVE_FOLDER}
+      - SFAS_SEND_FOLDER=${SFAS_SEND_FOLDER}
       - ATBC_LOGIN_ENDPOINT=${ATBC_LOGIN_ENDPOINT}
       - ATBC_USERNAME=${ATBC_USERNAME}
       - ATBC_PASSWORD=${ATBC_PASSWORD}

sources/packages/backend/apps/queue-consumers/src/processors/schedulers/sfas-integration/sims-to-sfas-integration.scheduler.ts

@@ -12,20 +12,25 @@ import {
   logProcessSummaryToJobLogger,
 } from "../../../utilities";
 import { QueueNames } from "@sims/utilities";
+import { SIMSToSFASProcessingService } from "@sims/integrations/sfas-integration";
+import { SIMSToSFASService } from "@sims/integrations/services/sfas";
+import { SIMS_TO_SFAS_BRIDGE_FILE_INITIAL_DATE } from "@sims/integrations/constants";
 
 @Processor(QueueNames.SIMSToSFASIntegration)
 export class SIMSToSFASIntegrationScheduler extends BaseScheduler<void> {
   constructor(
     @InjectQueue(QueueNames.SIMSToSFASIntegration)
     schedulerQueue: Queue<void>,
     queueService: QueueService,
+    private readonly simsToSFASService: SIMSToSFASService,
+    private readonly simsToSFASIntegrationProcessingService: SIMSToSFASProcessingService,
   ) {
     super(schedulerQueue, queueService);
   }
 
   /**
-   * Generate data file consisting of all student and application updates in SIMS since the previous file generation
-   * and send the data file to SFAS.
+   * Generate data file consisting of all student, application and restriction updates in SIMS since the previous file generation
+   * until the start of the current job and send the data file to SFAS.
    * @param job job.
    * @returns process summary.
    */
@@ -37,13 +42,38 @@ export class SIMSToSFASIntegrationScheduler extends BaseScheduler<void> {
       processSummary.info(
         `Processing SIMS to SFAS integration job. Job id: ${job.id} and Job name: ${job.name}.`,
       );
-      // TODO: Processing implementation of SIMS to SFAS integration.
+      // Set the bridge data extracted date as current date-time
+      // before staring to process the bridge data.
+      const bridgeDataExtractedDate = new Date();
+      const latestBridgeFileReferenceDate =
+        await this.simsToSFASService.getLatestBridgeFileLogDate();
+      // If the bridge is being executed for the first time, set the modified since date to
+      // a safe initial date.
+      const modifiedSince =
+        latestBridgeFileReferenceDate ?? SIMS_TO_SFAS_BRIDGE_FILE_INITIAL_DATE;
+      processSummary.info(
+        `Processing data since ${modifiedSince} until ${bridgeDataExtractedDate}.`,
+      );
+      const integrationProcessSummary = new ProcessSummary();
+      processSummary.children(integrationProcessSummary);
+      const { studentRecordsSent, uploadedFileName } =
+        await this.simsToSFASIntegrationProcessingService.processSIMSUpdates(
+          integrationProcessSummary,
+          modifiedSince,
+          bridgeDataExtractedDate,
+        );
+      processSummary.info("Processing SIMS to SFAS integration job completed.");
       return getSuccessMessageWithAttentionCheck(
-        ["Process finalized with success."],
+        [
+          "Process finalized with success.",
+          `Student records sent: ${studentRecordsSent}`,
+          `Uploaded file name: ${uploadedFileName}`,
+        ],
         processSummary,
       );
     } catch (error: unknown) {
-      const errorMessage = "Unexpected error while executing the job.";
+      const errorMessage =
+        "Unexpected error while executing the SIMS to SFAS integration job.";
       processSummary.error(errorMessage, error);
       throw new Error(errorMessage, { cause: error });
     } finally {

sources/packages/backend/libs/integrations/src/constants/system-configurations-constants.ts

@@ -35,3 +35,8 @@ export const APPLICATION_CHANGES_REPORT_PREFIX = "PBC.EDU.APPCHANGES";
  * SFTP directory name used to archive files.
  */
 export const SFTP_ARCHIVE_DIRECTORY = "Archive";
+
+/**
+ * Initial date for the SIMS to SFAS bridge first ever execution.
+ */
+export const SIMS_TO_SFAS_BRIDGE_FILE_INITIAL_DATE = new Date("2024-01-01");

sources/packages/backend/libs/integrations/src/institution-integration/ier12-integration/ier12-file-detail.ts

@@ -13,9 +13,8 @@ import {
   NUMBER_FILLER,
   SPACE_FILLER,
   ScholasticStandingCode,
-  YNFlag,
 } from "./models/ier12-integration.model";
-import { FullTimeAwardTypes } from "@sims/integrations/models";
+import { FullTimeAwardTypes, YNFlag } from "@sims/integrations/models";
 
 /**
  * Record of a IER12 file.

sources/packages/backend/libs/integrations/src/institution-integration/ier12-integration/ier12.integration.service.ts

@@ -9,7 +9,6 @@ import {
   IER12Record,
   IERAward,
   ScholasticStandingCode,
-  YNFlag,
 } from "./models/ier12-integration.model";
 import {
   ApplicationStatus,
@@ -22,6 +21,7 @@ import {
   getTotalYearsOfStudy,
   replaceLineBreaks,
 } from "@sims/utilities";
+import { YNFlag } from "@sims/integrations/models";
 
 /**
  * Manages the creation of the content files that needs to be sent

sources/packages/backend/libs/integrations/src/institution-integration/ier12-integration/models/ier12-integration.model.ts

@@ -131,11 +131,6 @@ export enum ScholasticStandingCode {
   ST = "ST",
 }
 
-export enum YNFlag {
-  Y = "Y",
-  N = "N",
-}
-
 export type IERAddressInfo = Omit<AddressInfo, "country" | "selectedCountry">;
 
 export type IERAward = Pick<

sources/packages/backend/libs/integrations/src/models/common.model.ts

@@ -34,3 +34,8 @@ export enum FullTimeAwardTypes {
   BGPD = "BGPD",
   SBSD = "SBSD",
 }
+
+export enum YNFlag {
+  Y = "Y",
+  N = "N",
+}

sources/packages/backend/libs/integrations/src/services/sfas/index.ts

@@ -3,3 +3,5 @@ export * from "./sfas-data-importer";
 export * from "./sfas-individual.service";
 export * from "./sfas-part-time-application.service";
 export * from "./sfas-restriction.service";
+export * from "./sims-to-sfas.model";
+export * from "./sims-to-sfas.service";

sources/packages/backend/libs/integrations/src/services/sfas/sims-to-sfas.model.ts

@@ -0,0 +1,6 @@
+import { Student } from "@sims/sims-db";
+
+export type StudentDetail = Student & {
+  cslfOverawardTotal?: string;
+  bcslOverawardTotal?: string;
+};

sources/packages/backend/libs/integrations/src/services/sfas/sims-to-sfas.service.ts

@@ -0,0 +1,175 @@
+import { Injectable } from "@nestjs/common";
+import { InjectRepository } from "@nestjs/typeorm";
+import { StudentDetail } from "./sims-to-sfas.model";
+import {
+  BC_STUDENT_LOAN_AWARD_CODE,
+  CANADA_STUDENT_LOAN_FULL_TIME_AWARD_CODE,
+} from "@sims/services/constants";
+import {
+  Application,
+  ApplicationStatus,
+  mapFromRawAndEntities,
+  SFASBridgeLog,
+  Student,
+} from "@sims/sims-db";
+import { Brackets, Repository } from "typeorm";
+
+/**
+ * SIMS to SFAS services.
+ */
+@Injectable()
+export class SIMSToSFASService {
+  constructor(
+    @InjectRepository(Student)
+    private readonly studentRepo: Repository<Student>,
+    @InjectRepository(Application)
+    private readonly applicationRepo: Repository<Application>,
+    @InjectRepository(SFASBridgeLog)
+    private readonly sfasBridgeLogRepo: Repository<SFASBridgeLog>,
+  ) {}
+
+  /**
+   * Get the latest bridge file log date.
+   * When there is no log, null will be returned.
+   * @returns latest bridge file log date.
+   */
+  async getLatestBridgeFileLogDate(): Promise<Date | null> {
+    const [latestBridgeFileLog] = await this.sfasBridgeLogRepo.find({
+      select: { id: true, referenceDate: true },
+      order: { referenceDate: "DESC" },
+      take: 1,
+    });
+    return latestBridgeFileLog ? latestBridgeFileLog.referenceDate : null;
+  }
+
+  /**
+   * Log the details of bridge file that was sent to SFAS.
+   * @param referenceDate date when the bridge file data was extracted.
+   * @param fileName bridge file name.
+   */
+  async logBridgeFileDetails(
+    referenceDate: Date,
+    fileName: string,
+  ): Promise<void> {
+    await this.sfasBridgeLogRepo.insert({
+      referenceDate,
+      generatedFileName: fileName,
+    });
+  }
+
+  /**
+   * Get all student ids of students who have one or more updates
+   * between the given period.
+   * The updates can be one or more of the following:
+   * - Student or User data
+   * - SIN validation data
+   * - CAS supplier data
+   * - Overawards data
+   * @param modifiedSince the date after which the student data was updated.
+   * @param modifiedUntil the date until which the student data was updated.
+   */
+  async getAllStudentsWithUpdates(
+    modifiedSince: Date,
+    modifiedUntil: Date,
+  ): Promise<number[]> {
+    const applicationsWithStudentUpdates = await this.applicationRepo
+      .createQueryBuilder("application")
+      .select(["application.id", "student.id"])
+      .distinctOn(["student.id"])
+      .innerJoin("application.student", "student")
+      .innerJoin("student.user", "user")
+      .innerJoin("student.sinValidation", "sinValidation")
+      .innerJoin("student.casSupplier", "casSupplier")
+      .leftJoin("student.overawards", "overaward")
+      .where("application.applicationStatus != :overwritten")
+      .andWhere("application.currentAssessment is not null")
+      // Check if the student data was updated in the given period.
+      .andWhere(
+        new Brackets((qb) => {
+          qb.where(
+            "student.updatedAt > :modifiedSince AND student.updatedAt <= :modifiedUntil",
+          )
+            .orWhere(
+              "user.updatedAt > :modifiedSince AND user.updatedAt <= :modifiedUntil",
+            )
+            .orWhere(
+              "sinValidation.updatedAt > :modifiedSince AND sinValidation.updatedAt <= :modifiedUntil",
+            )
+            .orWhere(
+              "casSupplier.updatedAt > :modifiedSince AND casSupplier.updatedAt <= :modifiedUntil",
+            )
+            .orWhere(
+              "overaward.updatedAt > :modifiedSince AND overaward.updatedAt <= :modifiedUntil",
+            );
+        }),
+      )
+      .setParameters({
+        overwritten: ApplicationStatus.Overwritten,
+        modifiedSince,
+        modifiedUntil,
+      })
+      .getMany();
+    // Extract the student ids from the applications.
+    const modifiedStudentIds = applicationsWithStudentUpdates.map(
+      (application) => application.student.id,
+    );
+
+    return modifiedStudentIds;
+  }
+
+  /**
+   * Get student details of students who have one or more updates.
+   * @param studentIds student ids.
+   * @returns student details.
+   */
+  async getStudentRecordsByStudentIds(
+    studentIds: number[],
+  ): Promise<StudentDetail[]> {
+    const queryResult = await this.studentRepo
+      .createQueryBuilder("student")
+      .select([
+        "student.id",
+        "student.birthDate",
+        "student.disabilityStatus",
+        "student.disabilityStatusEffectiveDate",
+        "user.firstName",
+        "user.lastName",
+        "sinValidation.sin",
+        "casSupplier.supplierNumber",
+        "casSupplier.supplierAddress",
+      ])
+      .addSelect("SUM(cslfOveraward.overawardValue)", "cslfOverawardTotal")
+      .addSelect("SUM(bcslOveraward.overawardValue)", "bcslOverawardTotal")
+      .innerJoin("student.user", "user")
+      .innerJoin("student.sinValidation", "sinValidation")
+      .innerJoin("student.casSupplier", "casSupplier")
+      .leftJoin(
+        "student.overawards",
+        "cslfOveraward",
+        "cslfOveraward.disbursementValueCode = :cslfAwardCode",
+      )
+      .leftJoin(
+        "student.overawards",
+        "bcslOveraward",
+        "bcslOveraward.disbursementValueCode = :bcslAwardCode",
+      )
+      .groupBy("student.id")
+      .addGroupBy("user.id")
+      .addGroupBy("sinValidation.id")
+      .addGroupBy("casSupplier.id")
+      .where("student.id IN (:...studentIds)")
+      .setParameters({
+        cslfAwardCode: CANADA_STUDENT_LOAN_FULL_TIME_AWARD_CODE,
+        bcslAwardCode: BC_STUDENT_LOAN_AWARD_CODE,
+        studentIds,
+      })
+      .getRawAndEntities();
+
+    return mapFromRawAndEntities<StudentDetail>(
+      queryResult,
+      "cslfOverawardTotal",
+      "bcslOverawardTotal",
+    );
+  }
+  // TODO: SIMS to SFAS - Add methods to extract application and restriction data.
+}