feat: add configurable task intervals

Chia-Network · Apr 14, 2023 · 287ed22 · 287ed22
1 parent b02c88f
commit 287ed22
Show file tree

Hide file tree

Showing 8 changed files with 87 additions and 38 deletions.
diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js
@@ -4,7 +4,6 @@ import fs from 'fs';
 import path from 'path';
 import request from 'request-promise';
 import { getConfig } from '../utils/config-loader';
-import { decodeHex } from '../utils/datalayer-utils';
 import fullNode from './fullNode';
 import { publicIpv4 } from '../utils/ip-tools';
 import wallet from './wallet';
@@ -183,23 +182,15 @@ const getStoreData = async (storeId, rootHash) => {
 
     if (data.success) {
       if (!_.isEmpty(data.keys_values)) {
-        logger.info(
-          `Downloaded Data: ${JSON.stringify(
-            data.keys_values.map((record) => {
-              return {
-                key: decodeHex(record.key),
-              };
-            }),
-            null,
-            2,
-          )}`,
-        );
+        logger.info(`Downloaded Data, root hash: ${rootHash || 'latest'}`);
       }
       return data;
     }
   }
 
-  logger.info(`Unable to find store data for ${storeId}}`);
+  logger.info(
+    `Unable to find store data for ${storeId} at root ${rootHash || 'latest'}`,
+  );
   return false;
 };
 

diff --git a/src/tasks/sync-audit-table.js b/src/tasks/sync-audit-table.js
@@ -17,15 +17,15 @@ logger.info('climate-warehouse:task:audit');
 dotenv.config();
 import { getConfig } from '../utils/config-loader';
 
-const { USE_SIMULATOR } = getConfig().APP;
+const CONFIG = getConfig().APP;
 
 const task = new Task('sync-audit', async () => {
   try {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
 
     logger.info('Syncing Audit Information');
-    if (!USE_SIMULATOR) {
+    if (!CONFIG.USE_SIMULATOR) {
       const organizations = await Organization.findAll({
         where: { subscribed: true },
         raw: true,
@@ -37,12 +37,18 @@ const task = new Task('sync-audit', async () => {
       );
     }
   } catch (error) {
-    logger.error(`Retrying in 30 seconds`, error);
+    logger.error(
+      `Retrying in ${CONFIG?.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30} seconds`,
+      error,
+    );
   }
 });
 
 const job = new SimpleIntervalJob(
-  { seconds: 30, runImmediately: true },
+  {
+    seconds: CONFIG?.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30,
+    runImmediately: true,
+  },
   task,
   'sync-audit',
 );

diff --git a/src/tasks/sync-datalayer.js b/src/tasks/sync-datalayer.js
@@ -10,6 +10,8 @@ import { logger } from '../config/logger.cjs';
 
 const Spinner = cliSpinner.Spinner;
 dotenv.config();
+import { getConfig } from '../utils/config-loader';
+const CONFIG = getConfig().APP;
 
 logger.info('climate-warehouse:task:sync-datalayer');
 
@@ -26,13 +28,18 @@ const task = new Task('sync-datalayer', async () => {
     spinner.start();
     datalayer.startDataLayerUpdatePolling();
   } catch (error) {
-    logger.error('Retrying in 60 seconds', error);
+    logger.error(
+      `Retrying in ${
+        CONFIG?.TASKS?.DATAMODEL_SYNC_TASK_INTERVAL || 60
+      } seconds`,
+      error,
+    );
   }
 });
 
 let seconds = 5;
 if (process.env.NODE_ENV !== 'test') {
-  seconds = 60;
+  seconds = CONFIG?.TASKS?.DATAMODEL_SYNC_TASK_INTERVAL || 60;
 }
 
 const job = new SimpleIntervalJob(

diff --git a/src/tasks/sync-default-organizations.js b/src/tasks/sync-default-organizations.js
@@ -6,7 +6,7 @@ import {
 } from '../utils/data-assertions';
 import { logger } from '../config/logger.cjs';
 import { getConfig } from '../utils/config-loader';
-const { USE_SIMULATOR } = getConfig().APP;
+const CONFIG = getConfig().APP;
 
 import dotenv from 'dotenv';
 dotenv.config();
@@ -17,16 +17,24 @@ const task = new Task('sync-default-organizations', async () => {
   try {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
-    if (!USE_SIMULATOR) {
+    if (!CONFIG.USE_SIMULATOR) {
       Organization.subscribeToDefaultOrganizations();
     }
   } catch (error) {
-    logger.error('Retrying in 30 seconds', error);
+    logger.error(
+      `Retrying in ${
+        CONFIG?.TASK?.GOVERNANCE_SYNC_TASK_INTERVAL || 30
+      } seconds`,
+      error,
+    );
   }
 });
 
 const job = new SimpleIntervalJob(
-  { seconds: 30, runImmediately: true },
+  {
+    seconds: CONFIG?.TASK?.GOVERNANCE_SYNC_TASK_INTERVAL || 30,
+    runImmediately: true,
+  },
   task,
   'sync-default-organizations',
 );

diff --git a/src/tasks/sync-governance-body.js b/src/tasks/sync-governance-body.js
@@ -9,7 +9,7 @@ import { getConfig } from '../utils/config-loader';
 import { logger } from '../config/logger.cjs';
 import { Organization } from '../models';
 
-const { GOVERNANCE_BODY_ID } = getConfig().GOVERNANCE;
+const CONFIG = getConfig();
 
 import dotenv from 'dotenv';
 dotenv.config();
@@ -22,22 +22,36 @@ const task = new Task('sync-governance-meta', async () => {
     await assertWalletIsSynced();
 
     logger.info('Syncing governance data');
-    if (GOVERNANCE_BODY_ID) {
-      logger.info(`Governance Config Found ${GOVERNANCE_BODY_ID}`);
+    if (CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID) {
+      logger.info(
+        `Governance Config Found ${CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID}`,
+      );
 
       const myOrganization = await Organization.getHomeOrg();
 
-      if (_.get(myOrganization, 'orgUid', '') !== GOVERNANCE_BODY_ID) {
+      if (
+        _.get(myOrganization, 'orgUid', '') !==
+        CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID
+      ) {
         Governance.sync();
       }
     }
   } catch (error) {
-    logger.error('Cant download Goverance data, Retrying in 24 hours', error);
+    logger.error(
+      `Cant download Goverance data, Retrying in ${
+        CONFIG?.APP?.TASKS?.GOVERNANCE_SYNC_TASK_INTERVAL || 86400
+      } seconds`,
+      error,
+    );
   }
 });
 
 const job = new SimpleIntervalJob(
-  { days: 1, runImmediately: true },
+  {
+    // DEFAULT 1 day
+    seconds: CONFIG?.APP?.TASKS?.GOVERNANCE_SYNC_TASK_INTERVAL || 86400,
+    runImmediately: true,
+  },
   task,
   'sync-governance-meta',
 );

diff --git a/src/tasks/sync-organization-meta.js b/src/tasks/sync-organization-meta.js
@@ -7,7 +7,7 @@ import {
 } from '../utils/data-assertions';
 import { logger } from '../config/logger.cjs';
 
-const { USE_SIMULATOR } = getConfig().APP;
+const CONFIG = getConfig().APP;
 
 import dotenv from 'dotenv';
 dotenv.config();
@@ -19,16 +19,25 @@ const task = new Task('sync-organization-meta', async () => {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
     logger.info('Syncing subscribed organizations');
-    if (!USE_SIMULATOR) {
+    if (!CONFIG.USE_SIMULATOR) {
       Organization.syncOrganizationMeta();
     }
   } catch (error) {
-    logger.error('Retrying in 24 hours', error);
+    logger.error(
+      `Retrying in ${
+        CONFIG?.APP?.TASKS?.ORGANIZATION_META_SYNC_TASK_INTERVAL || 86400
+      } seconds`,
+      error,
+    );
   }
 });
 
 const job = new SimpleIntervalJob(
-  { days: 1, runImmediately: true },
+  {
+    // DEFAULT 1
+    seconds: CONFIG?.APP?.TASKS?.ORGANIZATION_META_SYNC_TASK_INTERVAL || 86400,
+    runImmediately: true,
+  },
   task,
   'sync-organization-meta',
 );

diff --git a/src/tasks/sync-picklists.js b/src/tasks/sync-picklists.js
@@ -5,23 +5,30 @@ import {
   assertDataLayerAvailable,
   assertWalletIsSynced,
 } from '../utils/data-assertions';
+import { getConfig } from '../utils/config-loader';
 
-logger.info('climate-warehouse:task:sync-picklists');
+const CONFIG = getConfig().APP;
 
-const retryInSeconds = 30;
+logger.info('climate-warehouse:task:sync-picklists');
 
 const task = new Task('sync-picklist', async () => {
   try {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
     pullPickListValues();
   } catch (error) {
-    logger.error(`Retrying in ${retryInSeconds} seconds`, error);
+    logger.error(
+      `Retrying in ${CONFIG?.TASKS?.PICKLIST_SYNC_TASK_INTERVAL || 30} seconds`,
+      error,
+    );
   }
 });
 
 const job = new SimpleIntervalJob(
-  { seconds: retryInSeconds, runImmediately: true },
+  {
+    seconds: CONFIG?.TASKS?.PICKLIST_SYNC_TASK_INTERVAL || 30,
+    runImmediately: true,
+  },
   task,
   'sync-picklist',
 );

diff --git a/src/utils/defaultConfig.json b/src/utils/defaultConfig.json
@@ -17,7 +17,14 @@
     "IS_GOVERNANCE_BODY": false,
     "DEFAULT_FEE": 300000000,
     "DEFAULT_COIN_AMOUNT": 300000000,
-    "DATALAYER_FILE_SERVER_URL": null
+    "DATALAYER_FILE_SERVER_URL": null,
+    "TASKS": {
+      "AUDIT_SYNC_TASK_INTERVAL": 30,
+      "DATAMODEL_SYNC_TASK_INTERVAL": 60,
+      "GOVERNANCE_SYNC_TASK_INTERVAL": 86400,
+      "ORGANIZATION_META_SYNC_TASK_INTERVAL": 86400,
+      "PICKLIST_SYNC_TASK_INTERVAL": 30
+    }
   },
   "GOVERNANCE": {
     "GOVERNANCE_BODY_ID": "23f6498e015ebcd7190c97df30c032de8deb5c8934fc1caa928bc310e2b8a57e"