#3889 - Increase Logging for SFTP and file handling

sources/packages/backend/apps/api/src/app.exception.filter.ts

@@ -10,9 +10,10 @@ export class AppAllExceptionsFilter extends BaseExceptionFilter {
     const request: Request = ctx.getRequest();
 
     // Logging Additional info
-    this.logger.error("Unhandled exception");
-    this.logger.error(`Request path [${request.path}]`);
-    this.logger.error(`${JSON.stringify(exception)}`);
+    this.logger.error(
+      `Unhandled exception, request path: ${request.path}`,
+      exception,
+    );
 
     // Calling super
     super.catch(exception, host);

sources/packages/backend/apps/api/src/main.ts

@@ -93,17 +93,13 @@ async function bootstrap() {
   await app.listen(port);
   // Logging node http server error
   app.getHttpServer().on("error", (error: unknown) => {
-    logger.error(`Application server receive ${error}`, undefined, "Bootstrap");
+    logger.error("Application server receive.", error, "Bootstrap");
     exit(1);
   });
   logger.log(`Application is listing on port ${port}`, "Bootstrap");
 }
 bootstrap().catch((error: unknown) => {
-  const logger = new LoggerService();
-  logger.error(
-    `Application bootstrap exception: ${error}`,
-    undefined,
-    "Bootstrap-Main",
-  );
+  const logger = new LoggerService("Bootstrap-Main");
+  logger.error("Application bootstrap exception.", error);
   exit(1);
 });

sources/packages/backend/libs/integrations/src/cra-integration/cra-income-verification.processing.service.ts

@@ -13,12 +13,11 @@ import {
   CRASFTPResponseFile,
   ProcessSftpResponseResult,
 } from "./cra-integration.models";
-import { getUTCNow } from "@sims/utilities";
+import { getUTCNow, parseJSONError } from "@sims/utilities";
 import * as path from "path";
 import { ConfigService } from "@sims/utilities/config";
 import { CRAIntegrationService } from "./cra.integration.service";
 import { CRAIncomeVerificationsService } from "../services";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
 
 const INCOME_VERIFICATION_TAG = "VERIFICATION_ID";
 
@@ -275,14 +274,15 @@ export class CRAIncomeVerificationProcessingService {
 
     try {
       // Archive file.
-      await this.craService.archiveFile(remoteFilePath, SFTP_ARCHIVE_DIRECTORY);
+      await this.craService.archiveFile(remoteFilePath);
     } catch (error) {
       // Log the error but allow the process to continue.
       // If there was an issue only during the file archiving, it will be
       // processed again and could be archived in the second attempt.
       const logMessage = `Error while archiving CRA response file: ${remoteFilePath}`;
-      this.logger.error(logMessage);
       result.errorsSummary.push(logMessage);
+      result.errorsSummary.push(parseJSONError(error));
+      this.logger.error(logMessage, error);
     }
 
     return result;

sources/packages/backend/libs/integrations/src/esdc-integration/disbursement-receipt-integration/disbursement-receipt.processing.service.ts

@@ -14,8 +14,7 @@ import {
   ReportsFilterModel,
 } from "@sims/services";
 import { DAILY_DISBURSEMENT_REPORT_NAME } from "@sims/services/constants";
-import { getISODateOnlyString } from "@sims/utilities";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
+import { getISODateOnlyString, parseJSONError } from "@sims/utilities";
 
 /**
  * Disbursement schedule map which consists of disbursement schedule id for a document number.
@@ -160,15 +159,12 @@ export class DisbursementReceiptProcessingService {
 
     try {
       // Archiving the file once it has been processed.
-      await this.integrationService.archiveFile(
-        remoteFilePath,
-        SFTP_ARCHIVE_DIRECTORY,
-      );
+      await this.integrationService.archiveFile(remoteFilePath);
     } catch (error) {
-      result.errorsSummary.push(
-        `Error while archiving disbursement receipt file: ${remoteFilePath}`,
-      );
-      result.errorsSummary.push(error);
+      const logMessage = `Unexpected error while archiving disbursement receipt file: ${remoteFilePath}.`;
+      result.errorsSummary.push(logMessage);
+      result.errorsSummary.push(parseJSONError(error));
+      this.logger.error(logMessage, error);
     }
     return result;
   }

sources/packages/backend/libs/integrations/src/esdc-integration/e-cert-integration/e-cert-file-handler.ts

@@ -27,7 +27,6 @@ import { ConfigService, ESDCIntegrationConfig } from "@sims/utilities/config";
 import { ECertGenerationService } from "@sims/integrations/services";
 import { ECertResponseRecord } from "./e-cert-files/e-cert-response-record";
 import * as path from "path";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
 
 /**
  * Used to abort the e-Cert generation process, cancel the current transaction,
@@ -521,10 +520,7 @@ export abstract class ECertFileHandler extends ESDCFileHandler {
     processSummary: ProcessSummary,
   ) {
     try {
-      await eCertIntegrationService.archiveFile(
-        filePath,
-        SFTP_ARCHIVE_DIRECTORY,
-      );
+      await eCertIntegrationService.archiveFile(filePath);
     } catch (error) {
       // Log the error but allow the process to continue.
       // If there was an issue only during the file archiving, it will be

sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts

@@ -3,7 +3,7 @@ import { Injectable } from "@nestjs/common";
 import { FedRestrictionIntegrationService } from "./fed-restriction.integration.service";
 import { DataSource, InsertResult } from "typeorm";
 import { FederalRestriction, Restriction } from "@sims/sims-db";
-import { getISODateOnlyString } from "@sims/utilities";
+import { getISODateOnlyString, parseJSONError } from "@sims/utilities";
 import { FedRestrictionFileRecord } from "./fed-restriction-files/fed-restriction-file-record";
 import { ProcessSFTPResponseResult } from "../models/esdc-integration.model";
 import { ConfigService, ESDCIntegrationConfig } from "@sims/utilities/config";
@@ -14,7 +14,6 @@ import {
 } from "@sims/integrations/services";
 import { SystemUsersService } from "@sims/services/system-users";
 import { StudentRestrictionSharedService } from "@sims/services";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
 
 /**
  * Used to limit the number of asynchronous operations that will
@@ -77,15 +76,12 @@ export class FedRestrictionProcessingService {
       // Only the most updated file matters because it represents the entire data snapshot.
       for (const remoteFilePath of filePaths) {
         try {
-          await this.integrationService.archiveFile(
-            remoteFilePath,
-            SFTP_ARCHIVE_DIRECTORY,
-          );
+          await this.integrationService.archiveFile(remoteFilePath);
         } catch (error) {
-          result.errorsSummary.push(
-            `Error while archiving federal restrictions file: ${remoteFilePath}`,
-          );
-          result.errorsSummary.push(error);
+          const logMessage = `Error while archiving federal restrictions file: ${remoteFilePath}`;
+          result.errorsSummary.push(logMessage);
+          result.errorsSummary.push(parseJSONError(error));
+          this.logger.error(logMessage, error);
         }
       }
     } else {

sources/packages/backend/libs/integrations/src/esdc-integration/msfaa-integration/msfaa-response.processing.service.ts

@@ -10,7 +10,7 @@ import {
 import { MSFAAResponseCancelledRecord } from "./msfaa-files/msfaa-response-cancelled-record";
 import { MSFAAResponseReceivedRecord } from "./msfaa-files/msfaa-response-received-record";
 import { MSFAAIntegrationService } from "./msfaa.integration.service";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
+import { parseJSONError } from "@sims/utilities";
 
 @Injectable()
 export class MSFAAResponseProcessingService {
@@ -95,17 +95,15 @@ export class MSFAAResponseProcessingService {
     }
     try {
       // Archive file.
-      await this.msfaaService.archiveFile(
-        responseFile.filePath,
-        SFTP_ARCHIVE_DIRECTORY,
-      );
+      await this.msfaaService.archiveFile(responseFile.filePath);
     } catch (error) {
       // Log the error but allow the process to continue.
       // If there was an issue only during the file archiving, it will be
       // processed again and could be archived in the second attempt.
       const logMessage = `Error while archiving MSFAA response file: ${responseFile.filePath}`;
-      this.logger.error(logMessage);
       result.errorsSummary.push(logMessage);
+      result.errorsSummary.push(parseJSONError(error));
+      this.logger.error(logMessage, error);
     }
 
     return result;

sources/packages/backend/libs/integrations/src/esdc-integration/sin-validation/sin-validation.processing.service.ts

@@ -17,7 +17,7 @@ import {
   SINValidationService,
   StudentService,
 } from "@sims/integrations/services";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
+import { parseJSONError } from "@sims/utilities";
 
 /**
  * Manages the process to generate SIN validations requests to ESDC and allow
@@ -212,17 +212,15 @@ export class SINValidationProcessingService {
 
     try {
       // Archive file.
-      await this.sinValidationIntegrationService.archiveFile(
-        remoteFilePath,
-        SFTP_ARCHIVE_DIRECTORY,
-      );
+      await this.sinValidationIntegrationService.archiveFile(remoteFilePath);
     } catch (error) {
       // Log the error but allow the process to continue.
       // If there was an issue only during the file archiving, it will be
       // processed again and could be archived in the second attempt.
-      const logMessage = `Error while archiving ESDC SIN validation response file: ${remoteFilePath}`;
-      this.logger.error(logMessage);
+      const logMessage = `Error while archiving ESDC SIN validation response file: ${remoteFilePath}.`;
       result.errorsSummary.push(logMessage);
+      result.errorsSummary.push(parseJSONError(logMessage));
+      this.logger.error(logMessage, error);
     }
 
     return result;

sources/packages/backend/libs/integrations/src/esdc-integration/student-loan-balances/student-loan-balances.processing.service.ts

@@ -16,7 +16,6 @@ import {
   User,
   isDatabaseConstraintError,
 } from "@sims/sims-db";
-import { SFTP_ARCHIVE_DIRECTORY } from "@sims/integrations/constants";
 
 /**
  * Manages to process the Student Loan Balances files
@@ -155,14 +154,13 @@ export class StudentLoanBalancesProcessingService {
         // Archive file.
         await this.studentLoanBalancesIntegrationService.archiveFile(
           remoteFilePath,
-          SFTP_ARCHIVE_DIRECTORY,
         );
       } catch (error: unknown) {
         // Log the error but allow the process to continue.
         // If there was an issue only during the file archiving, it will be
         // processed again and could be archived in the second attempt.
         const logMessage = `Error while archiving Student Loan Balances response file: ${remoteFilePath}`;
-        childrenProcessSummary.error(logMessage);
+        childrenProcessSummary.error(logMessage, error);
       }
     }
   }

sources/packages/backend/libs/integrations/src/institution-integration/ece-integration/ece-response.processing.service.ts

@@ -10,14 +10,14 @@ import { ProcessSummaryResult } from "@sims/integrations/models";
 import {
   ECE_RESPONSE_COE_DECLINED_REASON,
   ECE_RESPONSE_FILE_NAME,
-  SFTP_ARCHIVE_DIRECTORY,
 } from "@sims/integrations/constants";
 import { InstitutionLocationService } from "@sims/integrations/services";
 import {
   COE_DENIED_REASON_OTHER_ID,
   CustomNamedError,
   END_OF_LINE,
   StringBuilder,
+  parseJSONError,
   processInParallel,
 } from "@sims/utilities";
 import { ECEResponseFileDetail } from "./ece-files/ece-response-file-detail";
@@ -509,17 +509,15 @@ export class ECEResponseProcessingService {
   ): Promise<void> {
     try {
       // Archiving the file once it has been processed.
-      await this.integrationService.archiveFile(
-        remoteFilePath,
-        SFTP_ARCHIVE_DIRECTORY,
-      );
+      await this.integrationService.archiveFile(remoteFilePath);
       processSummary.summary.push(
         `The file ${remoteFilePath} has been archived after processing.`,
       );
     } catch (error: unknown) {
-      processSummary.errors.push(
-        `Error while archiving the file: ${remoteFilePath}. ${error}`,
-      );
+      const logMessage = `Error while archiving the file: ${remoteFilePath}.`;
+      processSummary.errors.push(logMessage);
+      processSummary.errors.push(parseJSONError(error));
+      this.logger.error(logMessage, error);
     }
   }