Import resources from csv/xlsx

packages/server/package.json

@@ -25,6 +25,7 @@
     "@types/i18n": "^0.8.7",
     "@types/knex": "^0.16.1",
     "@types/mathjs": "^6.0.12",
+    "@types/yup": "^0.29.13",
     "accepts": "^1.3.7",
     "accounting": "^0.4.1",
     "agenda": "^4.2.1",
@@ -53,7 +54,6 @@
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-boom": "^3.0.0",
-    "express-fileupload": "^1.1.7-alpha.3",
     "express-oauth-server": "^2.0.0",
     "express-validator": "^6.12.2",
     "form-data": "^4.0.0",
@@ -77,6 +77,7 @@
     "moment-timezone": "^0.5.43",
     "mongodb": "^6.1.0",
     "mongoose": "^5.10.0",
+    "multer": "1.4.5-lts.1",
     "mustache": "^3.0.3",
     "mysql": "^2.17.1",
     "mysql2": "^1.6.5",
@@ -105,7 +106,8 @@
     "typedi": "^0.8.0",
     "uniqid": "^5.2.0",
     "winston": "^3.2.1",
-    "xlsx": "^0.18.5"
+    "xlsx": "^0.18.5",
+    "yup": "^0.28.1"
   },
   "devDependencies": {
     "@types/lodash": "^4.14.158",

packages/server/src/api/controllers/Import/ImportController.ts

@@ -0,0 +1,180 @@
+import { Inject, Service } from 'typedi';
+import { Router, Request, Response, NextFunction } from 'express';
+import { body, param } from 'express-validator';
+import BaseController from '@/api/controllers/BaseController';
+import { ServiceError } from '@/exceptions';
+import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication';
+import { uploadImportFile } from './_utils';
+
+@Service()
+export class ImportController extends BaseController {
+  @Inject()
+  private importResourceApp: ImportResourceApplication;
+
+  /**
+   * Router constructor method.
+   */
+  router() {
+    const router = Router();
+
+    router.post(
+      '/file',
+      uploadImportFile.single('file'),
+      this.importValidationSchema,
+      this.validationResult,
+      this.asyncMiddleware(this.fileUpload.bind(this)),
+      this.catchServiceErrors
+    );
+    router.post(
+      '/:import_id/import',
+      this.asyncMiddleware(this.import.bind(this)),
+      this.catchServiceErrors
+    );
+    router.post(
+      '/:import_id/mapping',
+      [
+        param('import_id').exists().isString(),
+        body('mapping').exists().isArray({ min: 1 }),
+        body('mapping.*.from').exists(),
+        body('mapping.*.to').exists(),
+      ],
+      this.validationResult,
+      this.asyncMiddleware(this.mapping.bind(this)),
+      this.catchServiceErrors
+    );
+    router.post(
+      '/:import_id/preview',
+      this.asyncMiddleware(this.preview.bind(this)),
+      this.catchServiceErrors
+    );
+    return router;
+  }
+
+  /**
+   * Import validation schema.
+   * @returns {ValidationSchema[]}
+   */
+  private get importValidationSchema() {
+    return [body('resource').exists()];
+  }
+
+  /**
+   * Imports xlsx/csv to the given resource type.
+   * @param {Request} req -
+   * @param {Response} res -
+   * @param {NextFunction} next -
+   */
+  private async fileUpload(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+
+    try {
+      const data = await this.importResourceApp.import(
+        tenantId,
+        req.body.resource,
+        req.file.filename
+      );
+      return res.status(200).send(data);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Maps the columns of the imported file.
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
+   */
+  private async mapping(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+    const { import_id: importId } = req.params;
+    const body = this.matchedBodyData(req);
+
+    try {
+      const mapping = await this.importResourceApp.mapping(
+        tenantId,
+        importId,
+        body?.mapping
+      );
+      return res.status(200).send(mapping);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Preview the imported file before actual importing.
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
+   */
+  private async preview(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+    const { import_id: importId } = req.params;
+
+    try {
+      const preview = await this.importResourceApp.preview(tenantId, importId);
+
+      return res.status(200).send(preview);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Importing the imported file to the application storage.
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
+   */
+  private async import(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+    const { import_id: importId } = req.params;
+
+    try {
+      const result = await this.importResourceApp.process(tenantId, importId);
+
+      return res.status(200).send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Transforms service errors to response.
+   * @param {Error}
+   * @param {Request} req
+   * @param {Response} res
+   * @param {ServiceError} error
+   */
+  private catchServiceErrors(
+    error,
+    req: Request,
+    res: Response,
+    next: NextFunction
+  ) {
+    if (error instanceof ServiceError) {
+      if (error.errorType === 'INVALID_MAP_ATTRS') {
+        return res.status(400).send({
+          errors: [{ type: 'INVALID_MAP_ATTRS' }],
+        });
+      }
+      if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') {
+        return res.status(400).send({
+          errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }],
+        });
+      }
+      if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') {
+        return res.status(400).send({
+          errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }],
+        });
+      }
+      if (error.errorType === 'IMPORTED_FILE_EXTENSION_INVALID') {
+        return res.status(400).send({
+          errors: [{ type: 'IMPORTED_FILE_EXTENSION_INVALID' }],
+        });
+      }
+    }
+    next(error);
+  }
+}

packages/server/src/api/controllers/Import/_utils.ts

@@ -0,0 +1,20 @@
+import Multer from 'multer';
+import { ServiceError } from '@/exceptions';
+
+export function allowSheetExtensions(req, file, cb) {
+  if (
+    file.mimetype !== 'text/csv' &&
+    file.mimetype !== 'application/vnd.ms-excel'
+  ) {
+    cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID'));
+
+    return;
+  }
+  cb(null, true);
+}
+
+export const uploadImportFile = Multer({
+  dest: './public/imports',
+  limits: { fileSize: 5 * 1024 * 1024 },
+  fileFilter: allowSheetExtensions,
+});

packages/server/src/api/index.ts

@@ -56,6 +56,7 @@ import { ProjectsController } from './controllers/Projects/Projects';
 import { ProjectTasksController } from './controllers/Projects/Tasks';
 import { ProjectTimesController } from './controllers/Projects/Times';
 import { TaxRatesController } from './controllers/TaxRates/TaxRates';
+import { ImportController } from './controllers/Import/ImportController';
 import { BankingController } from './controllers/Banking/BankingController';
 import { Webhooks } from './controllers/Webhooks/Webhooks';
 
@@ -135,6 +136,9 @@ export default () => {
   dashboard.use('/warehouses', Container.get(WarehousesController).router());
   dashboard.use('/projects', Container.get(ProjectsController).router());
   dashboard.use('/tax-rates', Container.get(TaxRatesController).router());
+
+  dashboard.use('/import', Container.get(ImportController).router());
+
   dashboard.use('/', Container.get(ProjectTasksController).router());
   dashboard.use('/', Container.get(ProjectTimesController).router());
 

packages/server/src/database/migrations/20231209230719_create_imports_table.js

@@ -0,0 +1,15 @@
+exports.up = function (knex) {
+  return knex.schema.createTable('imports', (table) => {
+    table.increments();
+    table.string('filename');
+    table.string('import_id');
+    table.string('resource');
+    table.json('columns');
+    table.json('mapping');
+    table.timestamps();
+  });
+};
+
+exports.down = function (knex) {
+  return knex.schema.dropTableIfExists('imports');
+};

packages/server/src/interfaces/Model.ts

@@ -34,6 +34,7 @@ export interface IModelMetaFieldCommon {
   columnable?: boolean;
   fieldType: IModelColumnType;
   customQuery?: Function;
+  required?: boolean;
 }
 
 export interface IModelMetaFieldNumber {
@@ -77,5 +78,6 @@ export type IModelMetaRelationField = IModelMetaRelationFieldCommon & (
 export interface IModelMeta {
   defaultFilterField: string;
   defaultSort: IModelMetaDefaultSort;
+  importable?: boolean;
   fields: { [key: string]: IModelMetaField };
 }

packages/server/src/loaders/express.ts

@@ -4,7 +4,6 @@ import helmet from 'helmet';
 import boom from 'express-boom';
 import errorHandler from 'errorhandler';
 import bodyParser from 'body-parser';
-import fileUpload from 'express-fileupload';
 import { Server } from 'socket.io';
 import Container from 'typedi';
 import routes from 'api';
@@ -47,13 +46,6 @@ export default ({ app }) => {
 
   app.use('/public', express.static(path.join(global.__storage_dir)));
 
-  // Handle multi-media requests.
-  app.use(
-    fileUpload({
-      createParentPath: true,
-    })
-  );
-
   // Logger middleware.
   app.use(LoggerMiddleware);
 

packages/server/src/loaders/tenantModels.ts

@@ -61,6 +61,7 @@ import Task from 'models/Task';
 import TaxRate from 'models/TaxRate';
 import TaxRateTransaction from 'models/TaxRateTransaction';
 import Attachment from 'models/Attachment';
+import Import from 'models/Import';
 import PlaidItem from 'models/PlaidItem';
 import UncategorizedCashflowTransaction from 'models/UncategorizedCashflowTransaction';
 
@@ -127,6 +128,7 @@ export default (knex) => {
     TaxRate,
     TaxRateTransaction,
     Attachment,
+    Import,
     PlaidItem,
     UncategorizedCashflowTransaction
   };