feat: finalize import/export

Chia-Network · Jan 30, 2022 · 52965eb · 52965eb
1 parent 5417a89
commit 52965eb
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 13 deletions.
diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js
@@ -33,7 +33,7 @@ import {
   tableDataFromXlsx,
   createXlsFromSequelizeResults,
   sendXls,
-  updateTablesWithData,
+  updateTableWithData, collapseTablesData,
 } from '../utils/xls';
 
 export const create = async (req, res) => {
@@ -190,7 +190,7 @@ export const updateFromXLS = async (req, res) => {
 
     const xlsxParsed = xlsx.parse(files.xlsx.data);
     const stagedDataItems = tableDataFromXlsx(xlsxParsed, Project);
-    await updateTablesWithData(stagedDataItems);
+    await updateTableWithData(collapseTablesData(stagedDataItems, Project), Project);
 
     res.json({
       message: 'Updates from xlsx added to staging',

diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js
@@ -22,10 +22,11 @@ import {
 
 import { createUnitRecordsFromCsv } from '../utils/csv-utils';
 import {
+  collapseTablesData,
   createXlsFromSequelizeResults,
   sendXls,
   tableDataFromXlsx,
-  updateTablesWithData,
+  updateTableWithData,
 } from '../utils/xls';
 import xlsx from 'node-xlsx';
 
@@ -196,7 +197,7 @@ export const updateFromXLS = async (req, res) => {
 
     const xlsxParsed = xlsx.parse(files.xlsx.data);
     const stagedDataItems = tableDataFromXlsx(xlsxParsed, Unit);
-    await updateTablesWithData(stagedDataItems);
+    await updateTableWithData(collapseTablesData(stagedDataItems, Unit), Unit);
 
     res.json({
       message: 'Updates from xlsx added to staging',

diff --git a/src/models/projects/projects.model.js b/src/models/projects/projects.model.js
@@ -36,6 +36,8 @@ class Project extends Model {
   static changes = new rxjs.Subject();
   static defaultColumns = Object.keys(ModelTypes);
   static validateImport = projectsUpdateSchema
+  static virtualFieldList = {};
+
   static getAssociatedModels = () => [
     ProjectLocation,
     Label,

diff --git a/src/models/units/units.model.js b/src/models/units/units.model.js
@@ -60,6 +60,7 @@ class Unit extends Model {
   static stagingTableName = 'Units';
   static changes = new rxjs.Subject();
   static validateImport = unitsUpdateSchema;
+  static virtualFieldList = virtualFields;
 
   static defaultColumns = Object.keys(
     Object.assign({}, ModelTypes, virtualFields),

diff --git a/src/models/units/units.stub.json b/src/models/units/units.stub.json
@@ -1,6 +1,8 @@
 [
   {
     "warehouseUnitId": "5c960ac1-a180-45a4-9850-be177e26d2fb",
+    "projectLocationId": "89adcb9a-dc10-41aa-8606-c1ce1545dcbd",
+    "vintageYear": 2020,
     "orgUid": "f1c54511-865e-4611-976c-7c3c1f704662",
     "unitOwner": "f1c54511-865e-4611-976c-7c3c1f704662",
     "countryJurisdictionOfOwner": "USA",

diff --git a/src/utils/xls.js b/src/utils/xls.js
@@ -77,7 +77,9 @@ export const createXlsFromSequelizeResults = (
   const initialReduceValue = {};
   initialReduceValue[model.name] = {
     name: model.name + 's',
-    data: [columnsInMainSheet],
+    data: [columnsInMainSheet.filter(colName => {
+      return !(excludeOrgUid && colName === 'orgUid');
+    })],
   };
 
   const xlsData = rows.reduce((sheets, row) => {
@@ -97,13 +99,17 @@ export const createXlsFromSequelizeResults = (
           sheets[mainColName + 's'] = {
             name: mainColName + 's',
             data: [
-              Object.keys(row[mainColName].filter(colName => {
-                return !(excludeOrgUid && colName === 'orgUid');
-              })).concat([
+              Object.keys(row[mainColName]).concat([
                 model.name.split('_').join('') + 'Id',
               ]),
             ],
-          };
+          }
+
+          console.log(Object.keys(row[mainColName]).filter(colName => {
+            return !(excludeOrgUid && colName === 'orgUid');
+          }).concat([
+            model.name.split('_').join('') + 'Id',
+          ]))
         }
         sheets[mainColName + 's'].data.push(
           Object.values(row[mainColName])
@@ -116,7 +122,9 @@ export const createXlsFromSequelizeResults = (
         if (row[mainColName] === null) {
           row[mainColName] = 'null';
         }
+        //console.log(mainColName, "//")
         if (!(excludeOrgUid && mainColName === 'orgUid')) {
+          //console.log(mainColName, "!!!");
           mainXlsRow.push(encodeValue(row[mainColName], hex));
         }
       }
@@ -210,7 +218,10 @@ export const tableDataFromXlsx = (xlsx, model) => {
           if (columnData === 'null') {
             columnData = null;
           }
-          row[columnNames[columnIndex]] = columnData;
+          // Ignore virtual fields
+          if (!Object.keys(model.virtualFieldList).includes(columnNames[columnIndex])) {
+            row[columnNames[columnIndex]] = columnData;
+          }
         }
         delete row.orgUid;
         stagingData[dataModel.name].data.push(row);
@@ -220,11 +231,42 @@ export const tableDataFromXlsx = (xlsx, model) => {
   }, {});
 };
 
-export const updateTablesWithData = async (tableData) => {
+export const collapseTablesData = (tableData, model) => {
+  const collapsed = {[model.name]: tableData[model.name]};
+
+  let associations = model.getAssociatedModels().map((model) => {
+    if (typeof model === 'object') {
+      return model.model;
+    } else {
+      return model;
+    }
+  });
+
+  for (const [i, data] of collapsed[model.name].data.entries()) {
+    for (const {name: association} of associations) {
+      if (['issuance'].includes(association)) {
+        collapsed[model.name].data[i][association] = tableData[association].data.find((row) => {
+          return row[model.name + 'Id'] === collapsed[model.name].data[i][association + 'Id'];
+        })
+      } else {
+        collapsed[model.name].data[i][association + 's'] = tableData[association].data.filter((row) => {
+          return row[model.name + 'Id'] === collapsed[model.name].data[i][tableData[model.name].model.primaryKeyAttributes[0]];
+        });
+      }
+    }
+  }
+
+  return collapsed;
+}
+
+export const updateTableWithData = async (tableData, model) => {
+  if (!['project', 'unit'].includes(model.name)) {
+    throw 'Bulk import is only supported for projects and units';
+  }
   // using a transaction ensures either everything is uploaded or everything fails
   await sequelize.transaction(async () => {
     const { orgUid } = await Organization.getHomeOrg();
-
+    
     for (let [, { model, data }] of Object.values(tableData).entries()) {
       for (let row of data) {
         const existingRecord = await model.findByPk(
@@ -241,7 +283,7 @@ export const updateTablesWithData = async (tableData) => {
           row.orgUid = orgUid;
         }
 
-        const validation = model.validateImport.validate(row);
+        const validation = model.validateImport.validate(row, { stripUnknown: true });
 
         if (!validation.error) {
           await Staging.upsert({