diff --git a/migrations/20220222204323-create-audit-table.cjs b/migrations/20220222204323-create-audit-table.cjs new file mode 100644 index 00000000..0873af2f --- /dev/null +++ b/migrations/20220222204323-create-audit-table.cjs @@ -0,0 +1,13 @@ +'use strict'; + +const modelTypes = require('../src/models/audit/audit.modeltypes.cjs'); + +module.exports = { + async up(queryInterface) { + await queryInterface.createTable('audit', modelTypes); + }, + + async down(queryInterface) { + await queryInterface.dropTable('audit'); + }, +}; diff --git a/src/controllers/audit.controller.js b/src/controllers/audit.controller.js new file mode 100644 index 00000000..a69316a2 --- /dev/null +++ b/src/controllers/audit.controller.js @@ -0,0 +1,14 @@ +import { Audit } from '../models'; + +export const findAll = async (req, res) => { + try { + const { orgUid } = req.query; + const auditResults = await Audit.findAll({ where: { orgUid } }); + return res.json(auditResults); + } catch (error) { + res.status(400).json({ + message: 'Can not retreive issuances', + error: error.message, + }); + } +}; diff --git a/src/controllers/index.js b/src/controllers/index.js index 19a495e5..30328c18 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -4,3 +4,4 @@ export * as StagingController from './staging.controller'; export * as OrganizationController from './organization.controller'; export * as IssuanceController from './issuance.controller'; export * as LabelController from './label.controller'; +export * as AuditController from './audit.controller'; diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js index 00c880b0..9e19366a 100644 --- a/src/controllers/units.controller.js +++ b/src/controllers/units.controller.js @@ -196,6 +196,7 @@ export const findAll = async (req, res) => { ); } } catch (error) { + console.trace(error); res.status(400).json({ message: 'Error retrieving units', error: error.message, diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js index 28f34232..67342b01 100644 --- a/src/datalayer/persistance.js +++ b/src/datalayer/persistance.js @@ -1,3 +1,5 @@ +import _ from 'lodash'; + import fs from 'fs'; import path from 'path'; import request from 'request-promise'; @@ -213,3 +215,55 @@ export const subscribeToStoreOnDataLayer = async (storeId, ip, port) => { return false; } }; + +export const getRootHistory = async (storeId) => { + const options = { + url: `${rpcUrl}/get_root_history`, + body: JSON.stringify({ + id: storeId, + }), + }; + + try { + const response = await request( + Object.assign({}, getBaseOptions(), options), + ); + + const data = JSON.parse(response); + + if (data.success) { + return _.get(data, 'root_history', []); + } + + return []; + } catch (error) { + return []; + } +}; + +export const getRootDiff = async (storeId, root1, root2) => { + const options = { + url: `${rpcUrl}/get_kv_diff`, + body: JSON.stringify({ + id: storeId, + hash_1: root1, + hash_2: root2, + }), + }; + + try { + const response = await request( + Object.assign({}, getBaseOptions(), options), + ); + + const data = JSON.parse(response); + + if (data.success) { + return _.get(data, 'diff', []); + } + + return []; + } catch (error) { + return []; + } +}; diff --git a/src/datalayer/syncService.js b/src/datalayer/syncService.js index 932fa0a3..5f317883 100644 --- a/src/datalayer/syncService.js +++ b/src/datalayer/syncService.js @@ -216,11 +216,25 @@ const getSubscribedStoreData = async ( }, {}); }; +const getRootHistory = (storeId) => { + if (process.env.USE_SIMULATOR !== 'true') { + return dataLayer.getRootHistory(storeId); + } +}; + +const getRootDiff = (storeId, root1, root2) => { + if (process.env.USE_SIMULATOR !== 'true') { + return dataLayer.getRootDiff(storeId, root1, root2); + } +}; + export default { startDataLayerUpdatePolling, syncDataLayerStoreToClimateWarehouse, dataLayerWasUpdated, subscribeToStoreOnDataLayer, getSubscribedStoreData, + getRootHistory, + getRootDiff, POLLING_INTERVAL, }; diff --git a/src/models/audit/audit.mock.js b/src/models/audit/audit.mock.js new file mode 100644 index 00000000..58d12483 --- /dev/null +++ b/src/models/audit/audit.mock.js @@ -0,0 +1,8 @@ +import stub from './audit.stub.json'; + +export const SimulatorMock = { + findAll: () => stub, + findOne: (id) => { + return stub.find((record) => record.id == id); + }, +}; diff --git a/src/models/audit/audit.model.js b/src/models/audit/audit.model.js new file mode 100644 index 00000000..f0937d8b --- /dev/null +++ b/src/models/audit/audit.model.js @@ -0,0 +1,35 @@ +'use strict'; + +import Sequelize from 'sequelize'; +const { Model } = Sequelize; +import { sequelize, safeMirrorDbHandler } from '../database'; +import { AuditMirror } from './audit.model.mirror'; +import ModelTypes from './audit.modeltypes.cjs'; + +class Audit extends Model { + static async create(values, options) { + safeMirrorDbHandler(() => AuditMirror.create(values, options)); + return super.create(values, options); + } + + static async destroy(values, options) { + safeMirrorDbHandler(() => AuditMirror.destroy(values, options)); + return super.destroy(values, options); + } + + static async upsert(values, options) { + safeMirrorDbHandler(() => AuditMirror.upsert(values, options)); + return super.upsert(values, options); + } +} + +Audit.init(ModelTypes, { + sequelize, + modelName: 'audit', + freezeTableName: true, + timestamps: true, + createdAt: true, + updatedAt: true, +}); + +export { Audit }; diff --git a/src/models/audit/audit.model.mirror.js b/src/models/audit/audit.model.mirror.js new file mode 100644 index 00000000..3c65b068 --- /dev/null +++ b/src/models/audit/audit.model.mirror.js @@ -0,0 +1,22 @@ +'use strict'; + +import Sequelize from 'sequelize'; +const { Model } = Sequelize; + +import { sequelizeMirror, safeMirrorDbHandler } from '../database'; +import ModelTypes from './audit.modeltypes.cjs'; + +class AuditMirror extends Model {} + +safeMirrorDbHandler(() => { + AuditMirror.init(ModelTypes, { + sequelize: sequelizeMirror, + modelName: 'audit', + freezeTableName: true, + timestamps: true, + createdAt: true, + updatedAt: true, + }); +}); + +export { AuditMirror }; diff --git a/src/models/audit/audit.modeltypes.cjs b/src/models/audit/audit.modeltypes.cjs new file mode 100644 index 00000000..b2e5439d --- /dev/null +++ b/src/models/audit/audit.modeltypes.cjs @@ -0,0 +1,52 @@ +const Sequelize = require('sequelize'); + +module.exports = { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + orgUid: { + type: Sequelize.STRING, + required: true, + allowNull: false, + }, + registryId: { + type: Sequelize.STRING, + required: true, + allowNull: false, + }, + rootHash: { + type: Sequelize.STRING, + required: true, + allowNull: false, + }, + type: { + type: Sequelize.STRING, + required: true, + allowNull: false, + }, + change: { + type: Sequelize.STRING, + required: true, + allowNull: true, + }, + table: { + type: Sequelize.STRING, + required: true, + allowNull: true, + }, + onchainConfirmationTimeStamp: { + type: 'TIMESTAMP', + required: true, + allowNull: false, + }, + createdAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.NOW, + }, + updatedAt: { + type: Sequelize.DATE, + defaultValue: Sequelize.NOW, + }, +}; diff --git a/src/models/audit/audit.stub.json b/src/models/audit/audit.stub.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/src/models/audit/audit.stub.json @@ -0,0 +1 @@ +[] diff --git a/src/models/audit/index.js b/src/models/audit/index.js new file mode 100644 index 00000000..2268346e --- /dev/null +++ b/src/models/audit/index.js @@ -0,0 +1,2 @@ +export * from './audit.model.js'; +export * from './audit.mock.js'; diff --git a/src/models/index.js b/src/models/index.js index 96cf7ffe..5d374370 100644 --- a/src/models/index.js +++ b/src/models/index.js @@ -33,6 +33,7 @@ export * from './meta'; export * from './simulator'; export * from './labelUnits'; export * from './estimations'; +export * from './audit'; export const ModelKeys = { unit: Unit, diff --git a/src/routes/v1/index.js b/src/routes/v1/index.js index 24ad2025..288613c7 100644 --- a/src/routes/v1/index.js +++ b/src/routes/v1/index.js @@ -10,6 +10,7 @@ import { OrganizationRouter, IssuanceRouter, LabelRouter, + AuditRouter, } from './resources'; V1Router.use('/projects', ProjectRouter); @@ -18,5 +19,6 @@ V1Router.use('/staging', StagingRouter); V1Router.use('/organizations', OrganizationRouter); V1Router.use('/issuances', IssuanceRouter); V1Router.use('/labels', LabelRouter); +V1Router.use('/audit', AuditRouter); export { V1Router }; diff --git a/src/routes/v1/resources/audit.js b/src/routes/v1/resources/audit.js new file mode 100644 index 00000000..fc9449d9 --- /dev/null +++ b/src/routes/v1/resources/audit.js @@ -0,0 +1,16 @@ +'use strict'; + +import express from 'express'; +import joiExpress from 'express-joi-validation'; + +import { AuditController } from '../../../controllers'; +import { auditGetSchema } from '../../../validations'; + +const validator = joiExpress.createValidator({ passError: true }); +const AuditRouter = express.Router(); + +AuditRouter.get('/', validator.query(auditGetSchema), (req, res) => { + return AuditController.findAll(req, res); +}); + +export { AuditRouter }; diff --git a/src/routes/v1/resources/index.js b/src/routes/v1/resources/index.js index 32b2caf1..ed480d88 100644 --- a/src/routes/v1/resources/index.js +++ b/src/routes/v1/resources/index.js @@ -4,3 +4,4 @@ export * from './staging'; export * from './organization'; export * from './issuances'; export * from './labels'; +export * from './audit'; diff --git a/src/tasks/index.js b/src/tasks/index.js index 0a7d07e6..a78d23e5 100644 --- a/src/tasks/index.js +++ b/src/tasks/index.js @@ -3,6 +3,7 @@ import { ToadScheduler } from 'toad-scheduler'; import syncDataLayer from './sync-datalayer'; import syncOrganizations from './sync-organizations'; import syncPickLists from './sync-picklists'; +import syncAudit from './sync-audit-table'; const scheduler = new ToadScheduler(); @@ -15,7 +16,12 @@ const addJobToScheduler = (job) => { const start = () => { // add default jobs - const defaultJobs = [syncDataLayer, syncOrganizations, syncPickLists]; + const defaultJobs = [ + syncDataLayer, + syncOrganizations, + syncPickLists, + syncAudit, + ]; defaultJobs.forEach((defaultJob) => { jobRegistry[defaultJob.id] = defaultJob; scheduler.addSimpleIntervalJob(defaultJob); diff --git a/src/tasks/sync-audit-table.js b/src/tasks/sync-audit-table.js new file mode 100644 index 00000000..09f1db14 --- /dev/null +++ b/src/tasks/sync-audit-table.js @@ -0,0 +1,106 @@ +import _ from 'lodash'; + +import { SimpleIntervalJob, Task } from 'toad-scheduler'; +import { Organization, Audit } from '../models'; +import datalayer from '../datalayer'; +import { decodeHex } from '../utils/datalayer-utils'; +import dotenv from 'dotenv'; +dotenv.config(); + +const task = new Task('sync-audit', async () => { + console.log('Syncing Audit Information'); + if (process.env.USE_SIMULATOR === 'false') { + const organizations = await Organization.findAll({ raw: true }); + await Promise.all( + organizations.map((organization) => syncOrganizationAudit(organization)), + ); + } +}); + +const job = new SimpleIntervalJob( + { minutes: 5, runImmediately: true }, + task, + 'sync-audit', +); + +const syncOrganizationAudit = async (organization) => { + try { + console.log('Syncing Audit:', organization.name); + const rootHistory = await datalayer.getRootHistory(organization.registryId); + + const lastRootSaved = await Audit.findOne({ + where: { registryId: organization.registryId }, + order: [['createdAt', 'DESC']], + raw: true, + }); + + if (!rootHistory.length) { + return; + } + + let rootHash = _.get(rootHistory, '[0].root_hash'); + + if (lastRootSaved) { + rootHash = lastRootSaved.rootHash; + } + + const historyIndex = rootHistory.findIndex( + (root) => root.root_hash === rootHash, + ); + + if (!lastRootSaved) { + Audit.create({ + orgUid: organization.orgUid, + registryId: organization.registryId, + rootHash: _.get(rootHistory, '[0].root_hash'), + type: 'CREATE REGISTRY', + change: null, + table: null, + onchainConfirmationTimeStamp: _.get(rootHistory, '[0].timestamp'), + }); + + return; + } + + if (historyIndex === rootHistory.length) { + return; + } + + const root1 = _.get(rootHistory, `[${historyIndex}]`); + const root2 = _.get(rootHistory, `[${historyIndex + 1}]`); + + if (!_.get(root2, 'confirmed')) { + return; + } + + const kvDiff = await datalayer.getRootDiff( + organization.registryId, + root1.root_hash, + root2.root_hash, + ); + + if (_.isEmpty(kvDiff)) { + return; + } + + await Promise.all( + kvDiff.map(async (diff) => { + const key = decodeHex(diff.key); + const modelKey = key.split('|')[0]; + Audit.create({ + orgUid: organization.orgUid, + registryId: organization.registryId, + rootHash: root2.root_hash, + type: diff.type, + table: modelKey, + change: decodeHex(diff.value), + onchainConfirmationTimeStamp: root2.timestamp, + }); + }), + ); + } catch (error) { + console.log(error); + } +}; + +export default job; diff --git a/src/utils/xls.js b/src/utils/xls.js index 9ce45b3a..2b39e2a6 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -36,7 +36,7 @@ export const sendXls = (name, bytes, response) => { export const encodeValue = (value, hex = false) => { // Todo: address this elsewhere (hide these columns). This is a quick fix for complex relationships in xlsx - if (typeof value === 'object') { + if (value && typeof value === 'object') { value = value.id; } diff --git a/src/validations/audit.validations.js b/src/validations/audit.validations.js new file mode 100644 index 00000000..7170293e --- /dev/null +++ b/src/validations/audit.validations.js @@ -0,0 +1,5 @@ +import Joi from 'joi'; + +export const auditGetSchema = Joi.object().keys({ + orgUid: Joi.string().required(), +}); diff --git a/src/validations/index.js b/src/validations/index.js index 278be55c..93898982 100644 --- a/src/validations/index.js +++ b/src/validations/index.js @@ -9,3 +9,4 @@ export * from './relatedProjects.validations'; export * from './estimations.validations'; export * from './projects.validations'; export * from './units.validations'; +export * from './audit.validations'; diff --git a/src/validations/relatedProjects.validations.js b/src/validations/relatedProjects.validations.js index 324407a3..8a536fb6 100644 --- a/src/validations/relatedProjects.validations.js +++ b/src/validations/relatedProjects.validations.js @@ -4,7 +4,7 @@ export const relatedProjectSchema = Joi.object({ // orgUid - derived upon creation // warehouseProjectId - derived upon creation id: Joi.string().optional(), - relatedProjectId: Joi.string().optional(), + relatedProjectId: Joi.string().required(), relationshipType: Joi.string().optional(), registry: Joi.string().optional(), updatedAt: Joi.date().optional(), diff --git a/tests/test-data/new-project.json b/tests/test-data/new-project.json index 3bf5c914..c7121ddf 100644 --- a/tests/test-data/new-project.json +++ b/tests/test-data/new-project.json @@ -52,6 +52,7 @@ ], "relatedProjects": [ { + "relatedProjectId": "A34398", "relationshipType": "Pasir Ris Park Conservation", "registry": "Singapore National Registry" } diff --git a/tests/test-data/update-project.json b/tests/test-data/update-project.json index ca9f4550..e6b6d97e 100644 --- a/tests/test-data/update-project.json +++ b/tests/test-data/update-project.json @@ -52,6 +52,7 @@ ], "relatedProjects": [ { + "relatedProjectId": "UPDATED", "relationshipType": "UPDATED", "registry": "UPDATED" }