Fix logs and memory tracking (#82)

## Description

<!--
    A brief description of what the PR does/changes.
    Use active voice and present tense, e.g., This PR fixes ...
-->
This PR fixes how log returned from connector is shown in SDK when
skipped and improve memory tracking logging.

## Connected Issues

<!--
DevRev issue(s) full link(s) (e.g.
https://app.devrev.ai/devrev/works/ISS-123).
-->
https://app.devrev.ai/devrev/works/ISS-215007

## Checklist

- [x] Tests added/updated and ran with `npm run test` OR no tests
needed.
- [x] Ran backwards compatibility tests with `npm run
test:backwards-compatibility`.
- [x] Tested airdrop-template linked to this PR.
- [x] Documentation updated and provided a link to PR / new docs OR
`no-docs` written:
<!-- Add this once we have eslint prepared:
- [ ] Code formatted and checked with `npm run lint`. -->

Author: radovanjorgic

src/attachments-streaming/attachments-streaming-pool.ts

@@ -123,7 +123,8 @@ export class AttachmentsStreamingPool<ConnectorState> {
 
         if (response?.error) {
           console.warn(
-            `Skipping attachment with ID ${attachment.id} due to error: ${response.error}`
+            `Skipping attachment with ID ${attachment.id} due to error returned by the stream function`,
+            response.error
           );
           await this.updateProgress();
           continue;
@@ -142,7 +143,8 @@ export class AttachmentsStreamingPool<ConnectorState> {
         await this.updateProgress();
       } catch (error) {
         console.warn(
-          `Skipping attachment with ID ${attachment.id} due to error: ${error}`
+          `Skipping attachment with ID ${attachment.id} due to error in processAttachment function`,
+          error
         );
 
         await this.updateProgress();

src/common/constants.ts

@@ -69,6 +69,6 @@ export const LIBRARY_VERSION = getLibraryVersion();
 
 export const DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
 export const HARD_TIMEOUT_MULTIPLIER = 1.3;
-export const MEMORY_LOG_INTERVAL = 10 * 1000; // 10 seconds
+export const MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
 
 export const DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes

src/common/helpers.ts

@@ -236,7 +236,7 @@ export interface MemoryInfo {
   formattedMessage: string;
 }
 
-export function getMemoryUsage(): MemoryInfo | null {
+export function getMemoryUsage(): MemoryInfo {
   try {
     const memUsage = process.memoryUsage();
     const heapStats = v8.getHeapStatistics();
@@ -274,7 +274,7 @@ export function getMemoryUsage(): MemoryInfo | null {
       formattedMessage,
     };
   } catch (err) {
-    console.error('Error retrieving memory usage:', (err as Error).message);
-    return null;
+    console.warn('Error retrieving memory usage', err);
+    throw err;
   }
 }

src/workers/spawn.ts

@@ -231,11 +231,23 @@ export class Spawn {
       }
     });
 
-    // Log memory usage every 10 seconds
+    // Log memory usage every 30 seconds
     this.memoryMonitoringInterval = setInterval(() => {
-      const memoryInfo = getMemoryUsage();
-      if (memoryInfo) {
-        this.logger.info(memoryInfo.formattedMessage);
+      try {
+        const memoryInfo = getMemoryUsage();
+        if (memoryInfo) {
+          this.logger.info(memoryInfo.formattedMessage);
+        }
+      } catch (error) {
+        // If memory monitoring fails, log the warning and clear the interval to prevent further issues
+        this.logger.warn(
+          'Memory monitoring failed, stopping logging of memory usage interval',
+          error
+        );
+        if (this.memoryMonitoringInterval) {
+          clearInterval(this.memoryMonitoringInterval);
+          this.memoryMonitoringInterval = undefined;
+        }
       }
     }, MEMORY_LOG_INTERVAL);
   }
@@ -253,6 +265,8 @@ export class Spawn {
   }
 
   private async exitFromMainThread(): Promise<void> {
+    this.clearTimeouts();
+
     if (this.alreadyEmitted) {
       this.resolve();
       return;