feat: attach comments to commits

src/controllers/staging.controller.js

@@ -83,7 +83,10 @@ export const commit = async (req, res) => {
     await assertWalletIsSynced();
     await assertNoPendingCommits();
 
-    await Staging.pushToDataLayer(_.get(req, 'query.table', null));
+    await Staging.pushToDataLayer(
+      _.get(req, 'query.table', null),
+      _.get(req, 'body.comment', ''),
+    );
     res.json({ message: 'Staging Table committed to full node' });
   } catch (error) {
     res.status(400).json({

src/database/migrations/20220428144558-add-comment-column-to-all-datamodels.js

@@ -0,0 +1,22 @@
+'use strict';
+
+export default {
+  async up(queryInterface, Sequelize) {
+    return Promise.all(
+      ['audit'].map((table) => {
+        queryInterface.addColumn(table, 'comment', {
+          type: Sequelize.STRING,
+          allowNull: true,
+        });
+      }),
+    );
+  },
+
+  async down(queryInterface) {
+    return Promise.all(
+      ['audit'].map((table) => {
+        queryInterface.removeColumn(table, 'comment');
+      }),
+    );
+  },
+};

src/database/migrations/index.js

@@ -18,6 +18,7 @@ import CreateEstimationTable from './20220127190529-create-estimation-table';
 import CreateAuditTable from './20220222204323-create-audit-table';
 import CreateMetaTable from './20220119211024-create-meta-table';
 import CreateGoveranceTable from './20220315134151-create-governance-table';
+import AddCommentColumnToDataModelTables from './20220428144558-add-comment-column-to-all-datamodels';
 
 export const migrations = [
   {
@@ -104,4 +105,8 @@ export const migrations = [
     migration: CreateGoveranceTable,
     name: '20220315134151-create-governance-table',
   },
+  {
+    migration: AddCommentColumnToDataModelTables,
+    name: '20220428144558-add-comment-column-to-all-datamodels',
+  },
 ];

src/datalayer/persistance.js

@@ -164,7 +164,7 @@ export const getStoreData = async (storeId, rootHash) => {
 
     if (data.success) {
       if (!_.isEmpty(data.keys_values)) {
-        logger.info(`Downloaded Data: ${data}`);
+        logger.info(`Downloaded Data: ${JSON.stringify(data)}`);
       }
       return data;
     }

src/datalayer/syncService.js

@@ -71,21 +71,29 @@ const syncDataLayerStoreToClimateWarehouse = async (storeId, rootHash) => {
         storeData.keys_values.map(async (kv) => {
           const key = decodeHex(kv.key.replace(`${storeId}_`, ''));
           const modelKey = key.split('|')[0];
-          const value = JSON.parse(decodeHex(kv.value));
+          let value;
 
-          await ModelKeys[modelKey].create(value);
-
-          const stagingUuid =
-            modelKey === 'unit'
-              ? value.warehouseUnitId
-              : modelKey === 'project'
-              ? value.warehouseProjectId
-              : undefined;
+          try {
+            value = JSON.parse(decodeHex(kv.value));
+          } catch (err) {
+            logger.error(`Cant parse json value: ${decodeHex(kv.value)}`);
+          }
 
-          if (stagingUuid) {
-            await Staging.destroy({
-              where: { uuid: stagingUuid },
-            });
+          if (ModelKeys[modelKey]) {
+            await ModelKeys[modelKey].upsert(value);
+
+            const stagingUuid =
+              modelKey === 'unit'
+                ? value.warehouseUnitId
+                : modelKey === 'project'
+                ? value.warehouseProjectId
+                : undefined;
+
+            if (stagingUuid) {
+              await Staging.destroy({
+                where: { uuid: stagingUuid },
+              });
+            }
           }
         }),
       );
@@ -256,6 +264,11 @@ const getStoreData = async (storeId, callback, onFail, retry = 0) => {
   }
 };
 
+const getCurrentStoreData = async (storeId) => {
+  const encodedData = await dataLayer.getStoreData(storeId);
+  return decodeDataLayerResponse(encodedData);
+};
+
 const getStoreIfUpdated = async (
   storeId,
   lastRootHash,
@@ -282,4 +295,5 @@ export default {
   getStoreData,
   getStoreIfUpdated,
   POLLING_INTERVAL,
+  getCurrentStoreData,
 };

src/datalayer/writeService.js

@@ -23,7 +23,7 @@ const createDataLayerStore = async () => {
 };
 
 const syncDataLayer = async (storeId, data, failedCallback) => {
-  logger.info(`Syncing ${storeId}: ${JSON.stringify(data)}`);
+  logger.info(`Syncing ${storeId}`);
   const changeList = Object.keys(data).map((key) => {
     return {
       action: 'insert',
@@ -71,7 +71,9 @@ const pushChangesWhenStoreIsAvailable = async (
     const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId);
 
     if (!hasUnconfirmedTransactions && storeExistAndIsConfirmed) {
-      logger.info('pushing to datalayer', { storeId, changeList });
+      logger.info(
+        `pushing to datalayer ${storeId} ${JSON.stringify(changeList)}`,
+      );
       const success = await dataLayer.pushChangeListToDataLayer(
         storeId,
         changeList,

src/models/audit/audit.modeltypes.cjs

@@ -41,6 +41,9 @@ module.exports = {
     required: true,
     allowNull: false,
   },
+  comment: {
+    type: Sequelize.STRING,
+  },
   createdAt: {
     type: Sequelize.DATE,
     defaultValue: Sequelize.NOW,

src/models/organizations/organizations.model.js

@@ -21,14 +21,14 @@ logger.info('climate-warehouse:organizations');
 import ModelTypes from './organizations.modeltypes.cjs';
 
 class Organization extends Model {
-  static async getHomeOrg() {
+  static async getHomeOrg(includeAddress = true) {
     const myOrganization = await Organization.findOne({
-      attributes: ['orgUid', 'name', 'icon', 'subscribed'],
+      attributes: ['orgUid', 'name', 'icon', 'subscribed', 'registryId'],
       where: { isHome: true },
       raw: true,
     });
 
-    if (myOrganization) {
+    if (myOrganization && includeAddress) {
       myOrganization.xchAddress = await datalayer.getPublicAddress();
       return myOrganization;
     }

src/models/projects/projects.model.js

@@ -20,6 +20,7 @@ import {
   Staging,
   Estimation,
   Rating,
+  Organization,
 } from '../';
 
 import {
@@ -34,6 +35,8 @@ import {
   formatModelAssociationName,
   getDeletedItems,
 } from '../../utils/model-utils.js';
+import { keyValueToChangeList } from '../../utils/datalayer-utils';
+import dataLayer from '../../datalayer';
 
 class Project extends Model {
   static stagingTableName = 'Projects';
@@ -291,7 +294,7 @@ class Project extends Model {
     };
   }
 
-  static async generateChangeListFromStagedData(stagedData) {
+  static async generateChangeListFromStagedData(stagedData, comment) {
     const [insertRecords, updateRecords, deleteChangeList] =
       Staging.seperateStagingDataIntoActionGroups(stagedData, 'Projects');
 
@@ -349,6 +352,20 @@ class Project extends Model {
       primaryKeyMap,
     );
 
+    const { registryId } = await Organization.getHomeOrg();
+    const currentDataLayer = await dataLayer.getCurrentStoreData(registryId);
+    const currentComment = currentDataLayer.filter(
+      (kv) => kv.key === 'comment',
+    );
+    const isUpdateComment = currentComment.length > 0;
+    const commentChangeList = keyValueToChangeList(
+      'comment',
+      `{"comment": "${comment}"}`,
+      isUpdateComment,
+    );
+
+    console.log('#############', commentChangeList);
+
     return {
       projects: [
         ..._.get(insertChangeList, 'project', []),
@@ -390,6 +407,7 @@ class Project extends Model {
         ..._.get(updateChangeList, 'projectRatings', []),
         ..._.get(deletedAssociationsChangeList, 'projectRatings', []),
       ],
+      comment: commentChangeList,
     };
   }
 }

src/models/staging/staging.model.js

@@ -205,7 +205,7 @@ class Staging extends Model {
     return [insertRecords, updateRecords, deleteChangeList];
   };
 
-  static async pushToDataLayer(tableToPush) {
+  static async pushToDataLayer(tableToPush, comment) {
     let stagedRecords;
     if (tableToPush) {
       stagedRecords = await Staging.findAll({
@@ -218,10 +218,12 @@ class Staging extends Model {
 
     const unitsChangeList = await Unit.generateChangeListFromStagedData(
       stagedRecords,
+      comment,
     );
 
     const projectsChangeList = await Project.generateChangeListFromStagedData(
       stagedRecords,
+      comment,
     );
 
     const unifiedChangeList = {

src/models/units/units.model.js

@@ -8,16 +8,18 @@ import {
   safeMirrorDbHandler,
   sanitizeSqliteFtsQuery,
 } from '../../database';
-import { Label, Issuance, Staging } from '../../models';
+import { Label, Issuance, Staging, Organization } from '../../models';
 import { UnitMirror } from './units.model.mirror';
 import ModelTypes from './units.modeltypes.cjs';
 import { transformSerialNumberBlock } from '../../utils/helpers';
 import {
   createXlsFromSequelizeResults,
   transformFullXslsToChangeList,
 } from '../../utils/xls';
+import { keyValueToChangeList } from '../../utils/datalayer-utils';
 import { unitsUpdateSchema } from '../../validations/index.js';
 import { getDeletedItems } from '../../utils/model-utils.js';
+import dataLayer from '../../datalayer';
 
 const { Model } = Sequelize;
 
@@ -308,7 +310,7 @@ class Unit extends Model {
     };
   }
 
-  static async generateChangeListFromStagedData(stagedData) {
+  static async generateChangeListFromStagedData(stagedData, comment) {
     const [insertRecords, updateRecords, deleteChangeList] =
       Staging.seperateStagingDataIntoActionGroups(stagedData, 'Units');
 
@@ -384,6 +386,18 @@ class Unit extends Model {
       primaryKeyMap,
     );
 
+    const { registryId } = await Organization.getHomeOrg();
+    const currentDataLayer = await dataLayer.getCurrentStoreData(registryId);
+    const currentComment = currentDataLayer.filter(
+      (kv) => kv.key === 'comment',
+    );
+    const isUpdateComment = currentComment.length > 0;
+    const commentChangeList = keyValueToChangeList(
+      'comment',
+      `{"comment": "${comment}"}`,
+      isUpdateComment,
+    );
+
     return {
       units: [
         ..._.get(insertChangeList, 'unit', []),
@@ -405,6 +419,7 @@ class Unit extends Model {
         ..._.get(updateChangeList, 'label_units', []),
         ..._.get(deletedAssociationsChangeList, 'label_units', []),
       ],
+      comment: commentChangeList,
     };
   }
 }

src/routes/v1/resources/staging.js

@@ -8,6 +8,7 @@ import {
   stagingDeleteSchema,
   stagingGetQuerySchema,
   stagingRetrySchema,
+  commitStagingSchema,
 } from '../../../validations';
 
 const validator = joiExpress.createValidator({ passError: true });
@@ -27,7 +28,11 @@ StagingRouter.post('/retry', validator.body(stagingRetrySchema), (req, res) => {
   return StagingController.retryRecrod(req, res);
 });
 
-StagingRouter.post('/commit', StagingController.commit);
+StagingRouter.post(
+  '/commit',
+  validator.body(commitStagingSchema),
+  StagingController.commit,
+);
 
 // Empty entire stagin table
 StagingRouter.delete('/clean', StagingController.clean);

src/tasks/sync-audit-table.js

@@ -49,7 +49,7 @@ const job = new SimpleIntervalJob(
 
 const syncOrganizationAudit = async (organization) => {
   try {
-    logger.info('Syncing Audit:', organization);
+    logger.info(`Syncing Audit: ${_.get(organization, 'name')}`);
     const rootHistory = await datalayer.getRootHistory(organization.registryId);
 
     const lastRootSaved = await Audit.findOne({
@@ -107,19 +107,32 @@ const syncOrganizationAudit = async (organization) => {
       return;
     }
 
+    // 0x636f6d6d656e74 is hex for 'comment'
+    const comment = kvDiff.filter(
+      (diff) =>
+        diff.key === '636f6d6d656e74' || diff.key === '0x636f6d6d656e74',
+    );
+
     await Promise.all(
       kvDiff.map(async (diff) => {
         const key = decodeHex(diff.key);
         const modelKey = key.split('|')[0];
-        Audit.create({
-          orgUid: organization.orgUid,
-          registryId: organization.registryId,
-          rootHash: root2.root_hash,
-          type: diff.type,
-          table: modelKey,
-          change: decodeHex(diff.value),
-          onchainConfirmationTimeStamp: root2.timestamp,
-        });
+        if (key !== 'comment') {
+          Audit.create({
+            orgUid: organization.orgUid,
+            registryId: organization.registryId,
+            rootHash: root2.root_hash,
+            type: diff.type,
+            table: modelKey,
+            change: decodeHex(diff.value),
+            onchainConfirmationTimeStamp: root2.timestamp,
+            comment: _.get(
+              JSON.parse(decodeHex(_.get(comment, '[0].value', '7b7d'))),
+              'comment',
+              '',
+            ),
+          });
+        }
       }),
     );
   } catch (error) {

src/validations/staging.validations.js

@@ -8,6 +8,10 @@ export const stagingRetrySchema = Joi.object({
   uuid: Joi.string().required(),
 });
 
+export const commitStagingSchema = Joi.object({
+  comment: Joi.string().optional(),
+});
+
 export const stagingGetQuerySchema = Joi.object()
   .keys({
     page: Joi.number(),