Merge pull request #757 from openkfw/650-dms-no-auth

650 DMS with no authorization

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 <!-- ### Added -->
 
 <!-- ### Changed -->
+- Min.io storage for files.
 
 <!-- ### Deprecated -->
 

api/package.json

@@ -77,6 +77,7 @@
     "joi": "^14.3.1",
     "jsonwebtoken": "^8.5.0",
     "lodash.isequal": "^4.5.0",
+    "minio": "^7.0.17",
     "pino": "^5.8.0",
     "pino-pretty": "^2.2.3",
     "raw-body": "^2.3.3",
@@ -90,6 +91,7 @@
     "@types/joi": "^14.3.2",
     "@types/jsonwebtoken": "^8.0.0",
     "@types/lodash.isequal": "^4.5.5",
+    "@types/minio": "^7.0.6",
     "@types/mocha": "^5.2.6",
     "@types/node": "^14.6.4",
     "@types/pino": "^6.3.0",

api/src/config/index.ts

@@ -0,0 +1,10 @@
+export const host = process.env.API_HOST || "master-api";
+export const port = process.env.PORT || 8080;
+export const isSsl = process.env.USE_SSL === "ssl" ? true : false;
+export const hostPort = `${isSsl ? "https" : "http"}://${host}:${port}`;
+
+export const minioEndPoint = process.env.MINIO_ENDPOINT; // nginx in development
+export const minioPort = process.env.MINIO_PORT && parseInt(process.env.MINIO_PORT as string, 10) || 9000;
+export const minioUseSSL = process.env.MINIO_USE_SSL === "true" ? true : false;
+export const minioAccessKey =  process.env.MINIO_ACCESS_KEY || "minio";
+export const minioSecretKey = process.env.MINIO_SECRET_KEY || "minio123";

api/src/http_errors/not_found.ts

@@ -0,0 +1,17 @@
+export const schema = {
+  description: "Not found",
+  type: "object",
+  properties: {
+    apiVersion: { type: "string", example: "1.0" },
+    error: {
+      type: "object",
+      properties: {
+        code: { type: "string", example: "404" },
+        message: {
+          type: "string",
+          example: "The route you are looking for was not found.",
+        },
+      },
+    },
+  },
+};

api/src/httpd/server.ts

@@ -126,7 +126,7 @@ export const createBasicApp = (
 
   server.addContentTypeParser("application/gzip", async function (request, payload) {
     request.headers["content-length"] = "1024mb";
-    return payload; 
+    return payload;
   });
 
   // app.use(logging);

api/src/index.ts

@@ -84,6 +84,7 @@ import * as WorkflowitemAssignService from "./service/workflowitem_assign";
 import * as WorkflowitemCloseService from "./service/workflowitem_close";
 import * as WorkflowitemCreateService from "./service/workflowitem_create";
 import * as WorkflowitemDocumentDownloadService from "./service/workflowitem_document_download";
+import * as WorkflowitemDocumentDownloadMinioService from "./service/workflowitem_document_minio_download";
 import * as WorkflowitemGetService from "./service/workflowitem_get";
 import * as WorkflowitemViewHistoryService from "./service/workflowitem_history_get";
 import * as WorkflowitemListService from "./service/workflowitem_list";
@@ -119,6 +120,7 @@ import * as WorkflowitemAssignAPI from "./workflowitem_assign";
 import * as WorkflowitemCloseAPI from "./workflowitem_close";
 import * as WorkflowitemCreateAPI from "./workflowitem_create";
 import * as WorkflowitemsDocumentDownloadAPI from "./workflowitem_download_document";
+import * as WorkflowitemsDocumentDownloadMinioAPI from "./workflowitem_download_document_minio";
 import * as WorkflowitemListAPI from "./workflowitem_list";
 import * as WorkflowitemPermissionGrantAPI from "./workflowitem_permission_grant";
 import * as WorkflowitemPermissionRevokeAPI from "./workflowitem_permission_revoke";
@@ -159,6 +161,7 @@ if (!organizationVaultSecret) {
 
 const SWAGGER_BASEPATH = process.env.SWAGGER_BASEPATH || "/";
 
+
 /*
  * Initialize the components:
  */
@@ -774,6 +777,18 @@ WorkflowitemsDocumentDownloadAPI.addHttpHandler(server, URL_PREFIX, {
     ),
 });
 
+WorkflowitemsDocumentDownloadMinioAPI.addHttpHandler(server, URL_PREFIX, {
+  getDocumentMinio: (ctx, projectId, subprojectId, workflowitemId, documentId) =>
+    WorkflowitemDocumentDownloadMinioService.getDocumentMinio(
+      db,
+      ctx,
+      projectId,
+      subprojectId,
+      workflowitemId,
+      documentId,
+    ),
+});
+
 /*
  * Run the server.
  */

api/src/lib/minio.ts

@@ -0,0 +1,164 @@
+import * as Minio from "minio";
+import { minioEndPoint, minioPort, minioUseSSL, minioAccessKey, minioSecretKey } from "../config";
+
+const Readable = require("stream").Readable;
+
+interface Metadata {
+  "Content-Type"?: string,
+  fileName: string,
+}
+
+interface MetadataWithName extends Metadata {
+  name: string
+}
+
+const minioClient: any = new Minio.Client({
+    endPoint: minioEndPoint || "nginx",
+    port: minioPort,
+    useSSL: minioUseSSL,
+    accessKey: minioAccessKey,
+    secretKey: minioSecretKey,
+});
+
+const bucketName: string = "trubudget";
+
+
+const makeBucket = (bucket: string, cb: Function) => {
+  minioClient.bucketExists(bucket, (err, exists) => {
+    if (err) {
+      console.error("Error during searching for bucket", err);
+      return cb(err);
+    }
+
+    if (!exists) {
+      minioClient.makeBucket(bucket, "us-east-1", (err) => {
+        if (err) {
+          console.error("Error creating bucket.", err);
+          return cb(err);
+        }
+        console.log(`Minio: Bucket ${bucket} created.`);
+        return cb(null, true);
+      });
+    }
+  });
+};
+
+export const makeBucketAsPromised = (bucket: string) => {
+  return new Promise((resolve, reject) => {
+    makeBucket(bucket, (err) => {
+      if (err) return reject(err);
+
+      resolve(true);
+    });
+  });
+};
+
+
+const upload = (file: string, content: string, metaData: Metadata, cb: Function) => {
+  const s = new Readable();
+  s._read = () => {};
+  s.push(content);
+  s.push(null);
+
+  const metaDataWithName: MetadataWithName = { ...metaData, name: file };
+  // Using putObject API upload your file to the bucket .
+  minioClient.putObject(bucketName, file, s, metaDataWithName, (err, etag) => {
+    if (err) {
+      console.error("minioClient.putObject", err);
+      return cb(err);
+    }
+
+    return cb(null, etag);
+  });
+};
+
+export const uploadAsPromised = (file: string, content: string, metaData: Metadata = {fileName: "default"}) => {
+  return new Promise((resolve, reject) => {
+    upload(file, content, metaData, (err, etag) => {
+      if (err) return reject(err);
+
+      resolve(etag);
+    });
+  });
+};
+
+const download = (file: string, cb: Function) => {
+  let fileContent: string = "";
+  minioClient.getObject(bucketName, file, (err, dataStream) => {
+    if (err) {
+      console.error("Error during getting file object", err);
+      cb(err);
+    }
+    dataStream.on("data", (chunk: string) => {
+      fileContent += chunk;
+    });
+    dataStream.on("end", () => {
+      cb(null, fileContent);
+    });
+    dataStream.on("error", function (err) {
+      console.error("Error during getting file object datastream", err);
+    });
+  });
+};
+
+export const downloadAsPromised = (file: string) => {
+  return new Promise((resolve, reject) => {
+    download(file, (err, fileContent: string) => {
+      if (err) return reject(err);
+
+      resolve(fileContent);
+    });
+  });
+};
+
+const getMetadata = (fileHash: string, cb: Function) => {
+  minioClient.statObject(bucketName, fileHash, (err, stat: MetadataWithName) => {
+    if (err) {
+      console.error(err);
+      return cb(err);
+    }
+    cb(null, stat);
+  });
+};
+
+export const getMetadataAsPromised = (fileHash: string) => {
+  return new Promise((resolve, reject) => {
+    getMetadata(fileHash, (err, metaData: MetadataWithName) => {
+      if (err) return reject(err);
+
+      resolve(metaData);
+    });
+  });
+};
+
+const sleep = (ms) => {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+};
+
+const establishConnection = async () => {
+  const retries = 20;
+  for (let i = 0; i <= retries; i++) {
+    try {
+      await sleep(20000);
+
+      await makeBucketAsPromised(bucketName);
+
+      console.log("Connection with min.io established.");
+      break;
+    } catch (e) {
+      console.error("Problem with establishing connection to min.io and creating bucket.");
+
+      if (i === retries) {
+        console.error("Unable to connect with min.io. EXITING!");
+        process.exit(1);
+      }
+    }
+
+  }
+};
+
+establishConnection();
+
+export default minioClient;

api/src/service/domain/workflow/document.ts

@@ -22,6 +22,7 @@ export interface UploadedDocument {
   id: string;
   base64: string;
   fileName: string;
+  url?: string;
 }
 
 export const uploadedDocumentSchema = Joi.object({
@@ -30,6 +31,7 @@ export const uploadedDocumentSchema = Joi.object({
     .required()
     .error(() => new Error("Document can't be an empty file")),
   fileName: Joi.string(),
+  orgAccess: Joi.array().items(Joi.string()).optional(),
 });
 
 export async function hashDocument(

api/src/service/domain/workflow/workflowitem_create.spec.ts

@@ -42,6 +42,7 @@ describe("Create workflowitem", () => {
       workflowitemExists: async (_projectId, _subprojectId, _workflowitemId) => false,
       getSubproject: async () => baseSubproject,
       applyWorkflowitemType: () => [],
+      uploadDocument: () => new Promise(() => undefined),
     });
 
     assert.isTrue(Result.isErr(result));
@@ -61,6 +62,7 @@ describe("Create workflowitem", () => {
       workflowitemExists: async (_projectId, _subprojectId, _workflowitemId) => false,
       getSubproject: async () => baseSubproject,
       applyWorkflowitemType: () => [],
+      uploadDocument: () => new Promise(() => undefined),
     });
 
     assert.isTrue(Result.isErr(result));

api/src/service/domain/workflow/workflowitem_create.ts

@@ -1,5 +1,6 @@
 import Joi = require("joi");
 import { VError } from "verror";
+import { minioEndPoint, hostPort } from "../../../config";
 import Intent, { workflowitemIntents } from "../../../authz/intents";
 import { Ctx } from "../../../lib/ctx";
 import * as Result from "../../../result";
@@ -77,6 +78,9 @@ interface Repository {
     event: BusinessEvent,
     workflowitem: Workflowitem.Workflowitem,
   ): Result.Type<BusinessEvent[]>;
+  uploadDocument(
+    document: UploadedDocument
+  ): Promise<void>;
 }
 
 export async function createWorkflowitem(
@@ -199,7 +203,16 @@ export async function createWorkflowitem(
     if (Result.isErr(result)) {
       return result;
     }
-    documentUploadedEvents.push(result);
+    const { document } = result as WorkflowitemDocumentUploaded.Event;
+    // document should be private
+    if (minioEndPoint) {
+      await repository.uploadDocument(document);
+      const eventData = {...result, document: {...document, base64: "", url: hostPort}};
+      documentUploadedEvents.push(eventData);
+    } else {
+      documentUploadedEvents.push(result);
+    }
+
   }
 
   // Check the workflowitem type

api/src/service/domain/workflow/workflowitem_document_download.ts

@@ -6,6 +6,7 @@ import { ServiceUser } from "../organization/service_user";
 import * as WorkflowitemDocument from "./document";
 import * as Workflowitem from "./workflowitem";
 import * as WorkflowitemDocumentUploaded from "./workflowitem_document_uploaded";
+import { getDocument as getDocumentService } from "../../workflowitem_document_download";
 import VError = require("verror");
 
 interface Repository {
@@ -31,7 +32,10 @@ export async function getDocument(
     return new NotAuthorized({ ctx, userId: user.id, intent, target: workflowitem });
   }
 
-  // Get all events from one document
+  /**
+   * Get all events from one document
+   * @see getDocumentService
+   */
   const documentEvents = await repository.getDocumentEvents(documentId);
   if (Result.isErr(documentEvents)) {
     return new VError(