From 287ed22a27a7a6ed2863357d5daa44cf4bf15360 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 14 Apr 2023 15:00:35 -0400 Subject: [PATCH] feat: add configurable task intervals --- src/datalayer/persistance.js | 17 ++++------------ src/tasks/sync-audit-table.js | 14 +++++++++---- src/tasks/sync-datalayer.js | 11 +++++++++-- src/tasks/sync-default-organizations.js | 16 +++++++++++---- src/tasks/sync-governance-body.js | 26 +++++++++++++++++++------ src/tasks/sync-organization-meta.js | 17 ++++++++++++---- src/tasks/sync-picklists.js | 15 ++++++++++---- src/utils/defaultConfig.json | 9 ++++++++- 8 files changed, 87 insertions(+), 38 deletions(-) diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js index 2c0bcff3..09a242b5 100644 --- a/src/datalayer/persistance.js +++ b/src/datalayer/persistance.js @@ -4,7 +4,6 @@ import fs from 'fs'; import path from 'path'; import request from 'request-promise'; import { getConfig } from '../utils/config-loader'; -import { decodeHex } from '../utils/datalayer-utils'; import fullNode from './fullNode'; import { publicIpv4 } from '../utils/ip-tools'; import wallet from './wallet'; @@ -183,23 +182,15 @@ const getStoreData = async (storeId, rootHash) => { if (data.success) { if (!_.isEmpty(data.keys_values)) { - logger.info( - `Downloaded Data: ${JSON.stringify( - data.keys_values.map((record) => { - return { - key: decodeHex(record.key), - }; - }), - null, - 2, - )}`, - ); + logger.info(`Downloaded Data, root hash: ${rootHash || 'latest'}`); } return data; } } - logger.info(`Unable to find store data for ${storeId}}`); + logger.info( + `Unable to find store data for ${storeId} at root ${rootHash || 'latest'}`, + ); return false; }; diff --git a/src/tasks/sync-audit-table.js b/src/tasks/sync-audit-table.js index ccd2a418..d1c68177 100644 --- a/src/tasks/sync-audit-table.js +++ b/src/tasks/sync-audit-table.js @@ -17,7 +17,7 @@ logger.info('climate-warehouse:task:audit'); dotenv.config(); import { getConfig } from '../utils/config-loader'; -const { USE_SIMULATOR } = getConfig().APP; +const CONFIG = getConfig().APP; const task = new Task('sync-audit', async () => { try { @@ -25,7 +25,7 @@ const task = new Task('sync-audit', async () => { await assertWalletIsSynced(); logger.info('Syncing Audit Information'); - if (!USE_SIMULATOR) { + if (!CONFIG.USE_SIMULATOR) { const organizations = await Organization.findAll({ where: { subscribed: true }, raw: true, @@ -37,12 +37,18 @@ const task = new Task('sync-audit', async () => { ); } } catch (error) { - logger.error(`Retrying in 30 seconds`, error); + logger.error( + `Retrying in ${CONFIG?.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30} seconds`, + error, + ); } }); const job = new SimpleIntervalJob( - { seconds: 30, runImmediately: true }, + { + seconds: CONFIG?.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30, + runImmediately: true, + }, task, 'sync-audit', ); diff --git a/src/tasks/sync-datalayer.js b/src/tasks/sync-datalayer.js index 09ad2a42..0e8d471a 100644 --- a/src/tasks/sync-datalayer.js +++ b/src/tasks/sync-datalayer.js @@ -10,6 +10,8 @@ import { logger } from '../config/logger.cjs'; const Spinner = cliSpinner.Spinner; dotenv.config(); +import { getConfig } from '../utils/config-loader'; +const CONFIG = getConfig().APP; logger.info('climate-warehouse:task:sync-datalayer'); @@ -26,13 +28,18 @@ const task = new Task('sync-datalayer', async () => { spinner.start(); datalayer.startDataLayerUpdatePolling(); } catch (error) { - logger.error('Retrying in 60 seconds', error); + logger.error( + `Retrying in ${ + CONFIG?.TASKS?.DATAMODEL_SYNC_TASK_INTERVAL || 60 + } seconds`, + error, + ); } }); let seconds = 5; if (process.env.NODE_ENV !== 'test') { - seconds = 60; + seconds = CONFIG?.TASKS?.DATAMODEL_SYNC_TASK_INTERVAL || 60; } const job = new SimpleIntervalJob( diff --git a/src/tasks/sync-default-organizations.js b/src/tasks/sync-default-organizations.js index ad0daf01..fee6c22a 100644 --- a/src/tasks/sync-default-organizations.js +++ b/src/tasks/sync-default-organizations.js @@ -6,7 +6,7 @@ import { } from '../utils/data-assertions'; import { logger } from '../config/logger.cjs'; import { getConfig } from '../utils/config-loader'; -const { USE_SIMULATOR } = getConfig().APP; +const CONFIG = getConfig().APP; import dotenv from 'dotenv'; dotenv.config(); @@ -17,16 +17,24 @@ const task = new Task('sync-default-organizations', async () => { try { await assertDataLayerAvailable(); await assertWalletIsSynced(); - if (!USE_SIMULATOR) { + if (!CONFIG.USE_SIMULATOR) { Organization.subscribeToDefaultOrganizations(); } } catch (error) { - logger.error('Retrying in 30 seconds', error); + logger.error( + `Retrying in ${ + CONFIG?.TASK?.GOVERNANCE_SYNC_TASK_INTERVAL || 30 + } seconds`, + error, + ); } }); const job = new SimpleIntervalJob( - { seconds: 30, runImmediately: true }, + { + seconds: CONFIG?.TASK?.GOVERNANCE_SYNC_TASK_INTERVAL || 30, + runImmediately: true, + }, task, 'sync-default-organizations', ); diff --git a/src/tasks/sync-governance-body.js b/src/tasks/sync-governance-body.js index fc494286..1160dec7 100644 --- a/src/tasks/sync-governance-body.js +++ b/src/tasks/sync-governance-body.js @@ -9,7 +9,7 @@ import { getConfig } from '../utils/config-loader'; import { logger } from '../config/logger.cjs'; import { Organization } from '../models'; -const { GOVERNANCE_BODY_ID } = getConfig().GOVERNANCE; +const CONFIG = getConfig(); import dotenv from 'dotenv'; dotenv.config(); @@ -22,22 +22,36 @@ const task = new Task('sync-governance-meta', async () => { await assertWalletIsSynced(); logger.info('Syncing governance data'); - if (GOVERNANCE_BODY_ID) { - logger.info(`Governance Config Found ${GOVERNANCE_BODY_ID}`); + if (CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID) { + logger.info( + `Governance Config Found ${CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID}`, + ); const myOrganization = await Organization.getHomeOrg(); - if (_.get(myOrganization, 'orgUid', '') !== GOVERNANCE_BODY_ID) { + if ( + _.get(myOrganization, 'orgUid', '') !== + CONFIG.GOVERNANCE.GOVERNANCE_BODY_ID + ) { Governance.sync(); } } } catch (error) { - logger.error('Cant download Goverance data, Retrying in 24 hours', error); + logger.error( + `Cant download Goverance data, Retrying in ${ + CONFIG?.APP?.TASKS?.GOVERNANCE_SYNC_TASK_INTERVAL || 86400 + } seconds`, + error, + ); } }); const job = new SimpleIntervalJob( - { days: 1, runImmediately: true }, + { + // DEFAULT 1 day + seconds: CONFIG?.APP?.TASKS?.GOVERNANCE_SYNC_TASK_INTERVAL || 86400, + runImmediately: true, + }, task, 'sync-governance-meta', ); diff --git a/src/tasks/sync-organization-meta.js b/src/tasks/sync-organization-meta.js index 75d44526..2864859a 100644 --- a/src/tasks/sync-organization-meta.js +++ b/src/tasks/sync-organization-meta.js @@ -7,7 +7,7 @@ import { } from '../utils/data-assertions'; import { logger } from '../config/logger.cjs'; -const { USE_SIMULATOR } = getConfig().APP; +const CONFIG = getConfig().APP; import dotenv from 'dotenv'; dotenv.config(); @@ -19,16 +19,25 @@ const task = new Task('sync-organization-meta', async () => { await assertDataLayerAvailable(); await assertWalletIsSynced(); logger.info('Syncing subscribed organizations'); - if (!USE_SIMULATOR) { + if (!CONFIG.USE_SIMULATOR) { Organization.syncOrganizationMeta(); } } catch (error) { - logger.error('Retrying in 24 hours', error); + logger.error( + `Retrying in ${ + CONFIG?.APP?.TASKS?.ORGANIZATION_META_SYNC_TASK_INTERVAL || 86400 + } seconds`, + error, + ); } }); const job = new SimpleIntervalJob( - { days: 1, runImmediately: true }, + { + // DEFAULT 1 + seconds: CONFIG?.APP?.TASKS?.ORGANIZATION_META_SYNC_TASK_INTERVAL || 86400, + runImmediately: true, + }, task, 'sync-organization-meta', ); diff --git a/src/tasks/sync-picklists.js b/src/tasks/sync-picklists.js index 49d98309..8f137f05 100644 --- a/src/tasks/sync-picklists.js +++ b/src/tasks/sync-picklists.js @@ -5,10 +5,11 @@ import { assertDataLayerAvailable, assertWalletIsSynced, } from '../utils/data-assertions'; +import { getConfig } from '../utils/config-loader'; -logger.info('climate-warehouse:task:sync-picklists'); +const CONFIG = getConfig().APP; -const retryInSeconds = 30; +logger.info('climate-warehouse:task:sync-picklists'); const task = new Task('sync-picklist', async () => { try { @@ -16,12 +17,18 @@ const task = new Task('sync-picklist', async () => { await assertWalletIsSynced(); pullPickListValues(); } catch (error) { - logger.error(`Retrying in ${retryInSeconds} seconds`, error); + logger.error( + `Retrying in ${CONFIG?.TASKS?.PICKLIST_SYNC_TASK_INTERVAL || 30} seconds`, + error, + ); } }); const job = new SimpleIntervalJob( - { seconds: retryInSeconds, runImmediately: true }, + { + seconds: CONFIG?.TASKS?.PICKLIST_SYNC_TASK_INTERVAL || 30, + runImmediately: true, + }, task, 'sync-picklist', ); diff --git a/src/utils/defaultConfig.json b/src/utils/defaultConfig.json index e59efef8..7f0617c8 100644 --- a/src/utils/defaultConfig.json +++ b/src/utils/defaultConfig.json @@ -17,7 +17,14 @@ "IS_GOVERNANCE_BODY": false, "DEFAULT_FEE": 300000000, "DEFAULT_COIN_AMOUNT": 300000000, - "DATALAYER_FILE_SERVER_URL": null + "DATALAYER_FILE_SERVER_URL": null, + "TASKS": { + "AUDIT_SYNC_TASK_INTERVAL": 30, + "DATAMODEL_SYNC_TASK_INTERVAL": 60, + "GOVERNANCE_SYNC_TASK_INTERVAL": 86400, + "ORGANIZATION_META_SYNC_TASK_INTERVAL": 86400, + "PICKLIST_SYNC_TASK_INTERVAL": 30 + } }, "GOVERNANCE": { "GOVERNANCE_BODY_ID": "23f6498e015ebcd7190c97df30c032de8deb5c8934fc1caa928bc310e2b8a57e"