Merge pull request #6 from jembi/CU-86c12bkmm_Climate-mediator---Dupl…

…icate-CSV-Validation 86c12bkmm - Climate mediator - Duplicate CSV Validation
jembi · Nov 19, 2024 · 3c87811 · 3c87811
2 parents 8e66062 + 8dda3c3
commit 3c87811
Show file tree

Hide file tree

Showing 2 changed files with 114 additions and 21 deletions.
diff --git a/src/routes/index.ts b/src/routes/index.ts
@@ -6,6 +6,7 @@ import logger from '../logger';
 import fs from 'fs';
 import path from 'path';
 import { uploadToMinio } from '../utils/minioClient';
+import e from 'express';
 const routes = express.Router();
 
 const bodySizeLimit = getConfig().bodySizeLimit;
@@ -31,6 +32,20 @@ const saveCsvToTmp = (fileBuffer: Buffer, fileName: string): string => {
   return fileUrl;
 };
 
+const isValidFileType = (file: Express.Multer.File): boolean => {
+  const validMimeTypes = ['text/csv', 'application/json'];
+  return validMimeTypes.includes(file.mimetype);
+};
+
+function validateJsonFile(buffer: Buffer): boolean {
+  try {
+    JSON.parse(buffer.toString());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 routes.post('/upload', upload.single('file'), async (req, res) => {
   const file = req.file;
   const bucket = req.query.bucket;
@@ -45,20 +60,44 @@ routes.post('/upload', upload.single('file'), async (req, res) => {
     return res.status(400).send('No bucket provided');
   }
 
-  const headers = getCsvHeaders(file.buffer);
-
-  if (!headers) {
-    return res.status(400).send('Invalid file type, please upload a valid CSV file');
+  if (!isValidFileType(file)) {
+    logger.error(`Invalid file type: ${file.mimetype}`);
+    return res.status(400).send('Invalid file type. Please upload either a CSV or JSON file');
   }
-  const fileUrl = saveCsvToTmp(file.buffer, file.originalname);
 
-  const uploadResult = await uploadToMinio(fileUrl,file.originalname, bucket as string);
-  // const tableCreated = await createTable(headers, bucket as string);
-  logger.info(`file created: ${file.originalname}`);
+  // For CSV files, validate headers
+  if (file.mimetype === 'text/csv') {
+    const headers = getCsvHeaders(file.buffer);
+    if (!headers) {
+      return res.status(400).send('Invalid CSV file format');
+    }
+    try {
+      const fileUrl = saveCsvToTmp(file.buffer, file.originalname);
+      const uploadResult = await uploadToMinio(fileUrl, file.originalname, bucket as string, file.mimetype);
+      // Clean up the temporary file
+      fs.unlinkSync(fileUrl);
 
-  fs.unlinkSync(fileUrl);
+      if (uploadResult) {
+        return res.status(201).send(`File ${file.originalname} uploaded in bucket ${bucket}`);
+      } else {
+        return res.status(400).send(`Object ${file.originalname} already exists in bucket ${bucket}`);
+      }
+    } catch (error) {
+      // Clean up the temporary file in case of error
+      fs.unlinkSync(fileUrl);
+      logger.error('Error uploading file to Minio:', error);
+      return res.status(500).send('Error uploading file');
+    }
+  } else if (file.mimetype === 'application/json') {
+    if (!validateJsonFile(file.buffer)) {
+      return res.status(400).send('Invalid JSON file format');
+    }
 
-  return res.status(201).send('File uploaded successfully');
+    return res.status(200).send('JSON file is valid - Future implementation');
+  } else {
+    return res.status(400).send('Invalid file type. Please upload either a CSV or JSON file');
+  }
+
 });
 
 export default routes;
diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts
@@ -12,29 +12,83 @@ const {endPoint, port, useSSL, bucketRegion, accessKey, secretKey, prefix, suffi
  * @param {Object} [customMetadata={}] - Optional custom metadata
  * @returns {Promise<void>}
  */
-export async function uploadToMinio(sourceFile: string, destinationObject: string, bucket: string, customMetadata = {}) {
+export async function uploadToMinio(sourceFile: string, destinationObject: string, bucket: string, fileType: string, customMetadata = {}) {
     const minioClient = new Minio.Client({
         endPoint,
         port,
         useSSL,
         accessKey,
         secretKey
     });
-
     // Check if bucket exists, create if it doesn't
     const exists = await minioClient.bucketExists(bucket);
     if (!exists) {
         await minioClient.makeBucket(bucket, bucketRegion);
-        logger.debug(`Bucket ${bucket} created in "${bucketRegion}".`);
+        logger.info(`Bucket ${bucket} created in "${bucketRegion}".`);
     }
 
-    // Set the object metadata
-    const metaData = {
-        'Content-Type': 'text/plain',
-        ...customMetadata
-    };
 
-    // Upload the file
-    await minioClient.fPutObject(bucket, destinationObject, sourceFile, metaData);
-    logger.debug(`File ${sourceFile} uploaded as object ${destinationObject} in bucket ${bucket}`);
+    try {
+        const fileExists = await checkFileExists(destinationObject, bucket, fileType);
+        if (fileExists) {
+           return false;
+        } else {
+            const metaData = {
+                'Content-Type': fileType,
+                'X-Upload-Id': crypto.randomUUID(), 
+                ...customMetadata
+            };
+
+            // Upload the file
+            await minioClient.fPutObject(bucket, destinationObject, sourceFile, metaData);
+            logger.info(`File ${sourceFile} uploaded as object ${destinationObject} in bucket ${bucket}`);
+            return true;
+        }
+    } catch (error) {
+        console.error('Error checking file:', error);
+    }   
+}
+
+/**
+ * Checks if a CSV file exists in the specified Minio bucket
+ * @param {string} fileName - Name of the CSV file to check
+ * @param {string} bucket - Bucket name
+ * @returns {Promise<boolean>} - Returns true if file exists, false otherwise
+ */
+export async function checkFileExists(fileName: string, bucket: string, fileType: string): Promise<boolean> {
+    const minioClient = new Minio.Client({
+        endPoint,
+        port,
+        useSSL,
+        accessKey,
+        secretKey
+    });
+
+    try {
+        // Check if bucket exists first
+        const bucketExists = await minioClient.bucketExists(bucket);
+        if (!bucketExists) {
+            logger.info(`Bucket ${bucket} does not exist`);
+            return false;
+        }
+
+        // Get object stats to check if file exists
+        const stats = await minioClient.statObject(bucket, fileName);     // Optionally verify it's a CSV file by checking Content-Type
+        if (stats.metaData && stats.metaData['content-type'] === fileType) {
+            logger.info(`File ${fileName} exists in bucket ${bucket}`);
+            return true;
+        } else {
+            logger.info(`File ${fileName} does not exist in bucket ${bucket}`);
+            return false;
+        }
+    } catch (err: any) {
+        if (err.code === 'NotFound') {
+            logger.debug(`File ${fileName} not found in bucket ${bucket}`);
+            return false;
+        }
+        // For any other error, log it and rethrow
+        logger.error(`Error checking file existence: ${err.message}`);
+        throw err;
+    }
+
 }