From 4a9cd0b4eb6e55df8ae5e0f8762e3678536bb451 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Tue, 25 Jan 2022 14:22:32 -0500 Subject: [PATCH] feat: sync the orgUid back to cw --- src/controllers/project.controller.js | 2 +- src/fullnode/syncService.js | 79 +++++++++++-------- .../organizations/organizations.model.js | 7 +- .../qualificationUnits.modeltypes.cjs | 4 + .../qualificationUnits.stub.json | 6 +- 5 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js index 4d4de276..d3a16486 100644 --- a/src/controllers/project.controller.js +++ b/src/controllers/project.controller.js @@ -41,7 +41,7 @@ export const create = async (req, res) => { newRecord.warehouseProjectId = uuid; // All new projects are assigned to the home orgUid - const orgUid = _.head(Object.keys(await Organization.getHomeOrg())); + const { orgUid } = await Organization.getHomeOrg(); newRecord.orgUid = orgUid; // The new project is getting created in this registry diff --git a/src/fullnode/syncService.js b/src/fullnode/syncService.js index 42cdf2dc..ca96d8b4 100644 --- a/src/fullnode/syncService.js +++ b/src/fullnode/syncService.js @@ -22,9 +22,9 @@ export const startDataLayerUpdatePolling = () => { updateInterval = setInterval(async () => { const tablesToUpdate = await dataLayerWasUpdated(); _.keys(tablesToUpdate).forEach((storeId) => { - if (tablesToUpdate[storeId]) { - syncDataLayerStore(storeId); - } + // if (tablesToUpdate[storeId]) { + syncDataLayerStore(storeId); + // } }); }, 10000); }; @@ -42,45 +42,60 @@ export const syncDataLayerStore = async (storeId) => { storeData = await dataLayer.getStoreData(storeId); } - await Promise.all([ - Unit.destroy({ where: {}, truncate: true }), - Project.destroy({ where: {}, truncate: true }), - RelatedProject.destroy({ where: {}, truncate: true }), - QualificationUnit.destroy({ where: {}, truncate: true }), - CoBenefit.destroy({ where: {}, truncate: true }), - Vintage.destroy({ where: {}, truncate: true }), - ProjectLocation.destroy({ where: {}, truncate: true }), - Qualification.destroy({ where: {}, truncate: true }), - ]); + // Extract out the orgUid thats being updated so we can + // clean that orgs tables and repopulate without truncating + // the entire db + const { decodedStoreData, orgUid } = storeData.keys_values.reduce( + (accum, kv) => { + const decodedRecord = { + key: new Buffer(kv.key.replace(`${storeId}_`, ''), 'hex').toString(), + value: JSON.parse(new Buffer(kv.value, 'hex').toString()), + }; - await Promise.all( - storeData.keys_values.map(async (kv) => { - const key = new Buffer( - kv.key.replace(`${storeId}_`, ''), - 'hex', - ).toString(); - const value = new Buffer(kv.value, 'hex').toString(); + accum.decodedStoreData.push(decodedRecord); + if (_.get(decodedRecord, 'value.orgUid')) { + accum.orgUid = _.get(decodedRecord, 'value.orgUid'); + } + + return accum; + }, + { + decodedStoreData: [], + orgUid: null, + }, + ); + + if (orgUid) { + await Promise.all([ + // the child table records should cascade delete so we only need to + // truncate the primary tables + Unit.destroy({ where: { orgUid } }), + Project.destroy({ where: { orgUid } }), + QualificationUnit.destroy({ where: { orgUid } }), + ]); + } + await Promise.all( + decodedStoreData.map(async (kv) => { + const { key, value } = kv; if (key.includes('unit')) { - const data = JSON.parse(value); - await Unit.upsert(data); - await Staging.destroy({ where: { uuid: data.warehouseUnitId } }); + await Unit.upsert(value); + await Staging.destroy({ where: { uuid: value.warehouseUnitId } }); } else if (key.includes('project')) { - const data = JSON.parse(value); - await Project.upsert(data); - await Staging.destroy({ where: { uuid: data.warehouseProjectId } }); + await Project.upsert(value); + await Staging.destroy({ where: { uuid: value.warehouseProjectId } }); } else if (key.includes('relatedProjects')) { - await RelatedProject.upsert(JSON.parse(value)); + await RelatedProject.upsert(value); } else if (key.includes('qualification_units')) { - await QualificationUnit.upsert(JSON.parse(value)); + await QualificationUnit.upsert(value); } else if (key.includes('coBenefits')) { - await CoBenefit.upsert(JSON.stringify(value)); + await CoBenefit.upsert(value); } else if (key.includes('vintages')) { - await Vintage.upsert(JSON.parse(value)); + await Vintage.upsert(value); } else if (key.includes('projectLocations')) { - await ProjectLocation.upsert(JSON.parse(value)); + await ProjectLocation.upsert(value); } else if (key.includes('qualifications')) { - await Qualification.upsert(JSON.parse(value)); + await Qualification.upsert(value); } }), ); diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index cd6690d8..9f69a766 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -29,12 +29,9 @@ class Organization extends Model { }); return organizations.reduce((map, current) => { - if (map) { - map = {}; - } - map[current.orgUid] = current; - }); + return map; + }, {}); } static async createHomeOrganization(name, icon, dataVersion = 'v1') { diff --git a/src/models/qualificationUnits/qualificationUnits.modeltypes.cjs b/src/models/qualificationUnits/qualificationUnits.modeltypes.cjs index 95ddabae..4014d257 100644 --- a/src/models/qualificationUnits/qualificationUnits.modeltypes.cjs +++ b/src/models/qualificationUnits/qualificationUnits.modeltypes.cjs @@ -1,6 +1,10 @@ const Sequelize = require('sequelize'); module.exports = { + orgUid: { + type: Sequelize.STRING, + required: true, + }, warehouseUnitId: Sequelize.STRING, qualificationId: Sequelize.STRING, createdAt: { diff --git a/src/models/qualificationUnits/qualificationUnits.stub.json b/src/models/qualificationUnits/qualificationUnits.stub.json index 46acd246..7f47d636 100644 --- a/src/models/qualificationUnits/qualificationUnits.stub.json +++ b/src/models/qualificationUnits/qualificationUnits.stub.json @@ -1,10 +1,12 @@ [ { "warehouseUnitId": "5c960ac1-a180-45a4-9850-be177e26d2fb", - "qualificationId": "702cafbb-c624-4273-9cdc-c617ad5675df" + "qualificationId": "702cafbb-c624-4273-9cdc-c617ad5675df", + "orgUid": "f1c54511-865e-4611-976c-7c3c1f704662" }, { "warehouseUnitId": "5c960ac1-a180-45a4-9850-be177e26d2fb", - "qualificationId": "76903895-840e-406c-b2a0-f90244acf02d" + "qualificationId": "76903895-840e-406c-b2a0-f90244acf02d", + "orgUid": "f1c54511-865e-4611-976c-7c3c1f704662" } ]