Restructure logging architecture with file-based storage and automated rotation

.gitignore

@@ -9,6 +9,9 @@ flask_logging_server/logs/exceptions/*
 # DICOM server logs 
 dicom_server/exceptions/*.log
 
+# API logs
+API/logs/*.log
+
 # Python cache files
 dicom_server/**/__pycache__/
 *.pyc
@@ -24,4 +27,9 @@ __pycache__/
 *.so 
 
 # Node.js
-node_modules/ 
+node_modules/ 
+
+data/
+
+
+

API/app.js

@@ -3,12 +3,11 @@ const express = require('express');
 const path = require('path');
 const favicon = require('serve-favicon');
 const multer = require('multer');
-logger = require("./logger.js")
+const logger = require('./logger.js');
 const app = express();
 app.set('trust proxy', true);
 app.use(favicon(path.join(__dirname, 'static', 'favicon.ico')));
 
-
 //middleware
 app.use(express.json());
 app.use(express.urlencoded({ extended: true }));
@@ -28,18 +27,13 @@ const csrfMiddleware = require("./middleware/csrfMiddleware.js");
 const sessionMiddleware = require('./middleware/sessionMiddleware.js');
 app.use(sessionMiddleware);
 
-
-
-
-
 const USERS = {
   username: 'test', password: 'test',
-}
+};
 
 // const cors = require('cors');
 // const { verifyCSRFToken } = require('./middleware/csrfMiddleware.js');
 
-
 //Routes
 const mainRoutes = require('./routes/main.routes.js');
 app.use('/main', mainRoutes);
@@ -50,23 +44,18 @@ app.use('/', adminRoutes);
 //const authRoutes = require('./routes/auth.routes.js');
 //app.use('/', authRoutes);
 
-
-
 const storage = multer.diskStorage({
   destination: (req, file, cb) => {
     cb(null, 'uploads/');
   },
   filename: (req, file, cb) => {
     cb(null, `${Date.now()}-${file.originalname}`);
-  }
+  },
 });
 
 const upload = multer({ storage });
 
 app.post('/upload', (req, res) => {
-
-
-
   upload.array('dicomFiles[]', 1)(req, res, function (err) {
     if (err) {
       if (err.code === 'LIMIT_UNEXPECTED_FILE') {
@@ -83,38 +72,27 @@ app.post('/upload', (req, res) => {
       }))
     });
   });
-
 });
 
-
-
 app.post('/login', async (req, res) => {
-
-
   try {
-
     if (!req.body.username || !req.body.password) {
-      return res.status(401).json({ message: 'Empty username or password!' })
-    }
-
-
+      return res.status(401).json({ message: 'Empty username or password!' });
+    } 
     else if (req.body.username === "oolnie" && req.body.password === "iamtcte4*") {
       req.session.user = { username: req.body.username };
 
       const accessToken = jwtUtils.generateAccessToken({ username: req.body.username });
       const refreshToken = jwtUtils.generateRefreshToken({ username: req.body.username });
       res.cookie("accessToken", accessToken, { httpOnly: true, secure: false, sameSite: "Strict" });
       res.cookie("refreshToken", refreshToken, { httpOnly: true, secure: false, sameSite: "Strict" });
-      logger.logEvent("Hidden credintials accessed", req)
+      logger.logEvent("Hidden credentials accessed", req);
       res.status(200).json({ message: 'Login dev' });
       // Convert JSON object to a string
-
-    }
-
+    } 
     else if (req.body.username === USERS.username && req.body.password === USERS.password) {
-      logger.logEvent(`Successful login User ${req.body.username} and password ${req.body.password} `, req)
+      logger.logEvent(`Successful login User ${req.body.username} and password ${req.body.password} `, req);
       req.session.user = { username: req.body.username };
-      // console.log('User ','"',req.body.username,'"',' logged in. Session ID:', req.sessionID); // Log the session ID
 
       const accessToken = jwtUtils.generateAccessToken({ username: req.body.username });
       const refreshToken = jwtUtils.generateRefreshToken({ username: req.body.username });
@@ -125,29 +103,25 @@ app.post('/login', async (req, res) => {
       res.cookie("refreshToken", refreshToken, { httpOnly: true, secure: false, sameSite: "Strict" });
       // res.cookie("csrfToken", csrfToken, {httpOnly: false, secure: true, sameSite:"Strict"});
       // console.log("cookie", res.cookie);
-      //onsole.log("CSRF token:::", csrfToken);
-
+      // console.log("CSRF token:::", csrfToken);
 
       return res.status(200).json({ message: 'Login successful!' });
 
     } else {
       //it has been "invalid credentials before"
-      logger.logEvent(`Fail login User ${req.body.username} and password ${req.body.password}`, req)
+      logger.logEvent(`Fail login User ${req.body.username} and password ${req.body.password}`, req);
 
       return res.status(401).json({ message: 'Invalid username or password!' });
     }
   } catch (error) {
     res.status(500).json({ message: error.message });
   }
-
 });
 
-
-
 app.get("/get-ip", (req, res) => {
   const ip = req.headers["x-forwarded-for"] || req.socket.remoteAddress;
-  const port = req.connection.remotePort
-  res.json({ ip,port });
+  const port = req.connection.remotePort;
+  res.json({ ip, port });
 });
 app.post('/logout', (req, res) => {
   req.session.destroy(err => {
@@ -158,7 +132,7 @@ app.post('/logout', (req, res) => {
     if (err) {
       return res.status(500).send('Error logging out');
     }
-  })
+  });
   res.clearCookie('accessToken');
   //console.log("aaaaaaaa",res.clearCookie('accessToken'))
   res.clearCookie('refreshToken');
@@ -186,23 +160,17 @@ app.get('/refresh', (req, res) => {
     res.cookie("refreshToken", newRefreshToken, { httpOnly: true, secure: false, sameSite: "Strict" });
 
     res.redirect(req.query.redirectFrom);
-
   } catch (error) {
     res.status(403).json({ message: "Invalid or expired refresh token" });
   }
-
 });
 
 app.get('/', async (req, res) => {
   if (req.session.user) {
-    res.sendFile((path.join(__dirname, 'static', 'main.html')));
+    res.sendFile(path.join(__dirname, 'static', 'main.html'));
   } else {
-    res.sendFile((path.join(__dirname, 'static', 'login.html')));
+    res.sendFile(path.join(__dirname, 'static', 'login.html'));
   }
-
-
-
-
 });
 
 //check-admin-token is called loginAdmin to avoid fingerprinting
@@ -215,7 +183,6 @@ app.get('/loginAdmin', (req, res) => {
   }
 });
 
-
 // See all instances table has the option to download a single instance
 app.get('/downloadInstance', (req, res) => {
   const file = path.join(__dirname, './static/hf/image-000008sssssss.dcm.url');
@@ -228,7 +195,7 @@ app.get('/downloadInstance', (req, res) => {
 
 // See all studies table has the option to download a zip folder
 app.get('/downloadStudyFiles', (req, res) => {
-  logger.logEvent("StudyFilesDownloaded",req)
+  logger.logEvent("StudyFilesDownloaded", req);
   const file = path.join(__dirname, './static/hf/StudyFiles.zip');
   res.download(file, 'StudyFiles.zip', (err) => {
     if (err) {
@@ -237,7 +204,6 @@ app.get('/downloadStudyFiles', (req, res) => {
   });
 });
 
-
 //run the app
 try {
   app.listen(port, '0.0.0.0', () => {
@@ -246,4 +212,3 @@ try {
 } catch (error) {
   console.error("Error. Server did not start", error);
 }
-

API/dbConnectors/sqliteConnector.js

@@ -2,7 +2,7 @@
 const sqlite3 = require('sqlite3').verbose();
 var databasePath = "./../dicom_server/storage/db.db"
 if (process.env.Docker_ENV=="True") {
-    databasePath = '/app/db.db'
+    databasePath = '/opt/dicomhawk/storage/db.db'
 }
 
 // Connect to the SQLite database

API/logger.js

@@ -1,65 +1,111 @@
-const redisClient= require("./dbConnectors/redisConnector");
-const fs = require('fs');
-const threat_intelligence= require('./theatIntelligence')
-function getClientIp(req) {
-    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || req.socket.remoteAddress || '';
-    return ip.includes('::ffff:') ? ip.replace('::ffff:', '') : ip;
-  }
-
+const fs = require("fs");
+const path = require("path");
+const threat_intelligence = require("./threatIntelligence");
 
+const getLogsDir = () => {
+  return process.env.Docker_ENV === "True" ? "/var/log/dicomhawk" : path.join(__dirname, "logs");
+};
 
-async function isIpScanned(ip){
-  const ipScanned= await redisClient.lRange("scannedIPs", 0, -1)
-  const result= JSON.stringify(ipScanned).includes(ip.toString())
-  console.log("Ress"+result)
-  return result;
-
+function getClientIp(req) {
+  const ip =
+    req.headers["x-forwarded-for"] ||
+    req.connection.remoteAddress ||
+    req.socket.remoteAddress ||
+    "";
+  return ip.includes("::ffff:") ? ip.replace("::ffff:", "") : ip;
 }
 
-   async function logEvent(event, req, parameter = "N/A") {
-    var ip = ""
-    var r_port = ""
-    rep=parameter
-    if (event==="FileUploaded"){
-        ip = req.headers['x-requestor-ip'];
-        port = req.headers['port']
+async function isIpScanned(ip) {
+  try {
+    const scannedIpsDir = path.join(getLogsDir(), "scanned_ips");
+    if (!fs.existsSync(scannedIpsDir)) {
+      fs.mkdirSync(scannedIpsDir, { recursive: true });
+    }
+    const logPath = path.join(scannedIpsDir, "scanned_ips.log");
+    if (!fs.existsSync(logPath)) {
+      fs.writeFileSync(logPath, "");
+    }
+    const scannedIps = fs.readFileSync(logPath, "utf8");
+    return scannedIps.includes(ip.toString());
+  } catch (error) {
+    console.error("Error reading scanned IPs:", error);
+    return false;
+  }
+}
 
-    }else{
-        ip = getClientIp(req)
-        r_port=req.connection.remotePort
+async function logEvent(event, req, parameter = "N/A") {
+  try {
+    const logsDir = getLogsDir();
+    if (!fs.existsSync(logsDir)) {
+      fs.mkdirSync(logsDir, { recursive: true });
     }
-    let isScanned= await isIpScanned(ip)
 
-    if  ( !isScanned){
-      let repu = await threat_intelligence.getReputationData(ip)
-      redisClient.rPush("reputation",JSON.stringify(repu))
-      redisClient.rPush("scannedIPs",ip)
+    const apiLogsDir = path.join(logsDir, "api_logs");
+    const reputationDir = path.join(logsDir, "reputation");
+    const scannedIpsDir = path.join(logsDir, "scanned_ips");
+
+    if (!fs.existsSync(apiLogsDir)) {
+      fs.mkdirSync(apiLogsDir, { recursive: true });
+    }
+    if (!fs.existsSync(reputationDir)) {
+      fs.mkdirSync(reputationDir, { recursive: true });
+    }
+    if (!fs.existsSync(scannedIpsDir)) {
+      fs.mkdirSync(scannedIpsDir, { recursive: true });
+    }
 
+    var ip = "";
+    var r_port = "";
+    rep = parameter;
+    if (event === "FileUploaded") {
+      ip = req.headers["x-requestor-ip"];
+      port = req.headers["port"];
+    } else {
+      ip = getClientIp(req);
+      r_port = req.connection.remotePort;
     }
-    
-    try {
-  
-      const jsonObject = {
-        ip: ip,
-        timestamp: new Date().toISOString().slice(0, 19),
-        messevent: event,
-        sessionId: process.env.SESSION_SECRET.substring(0, 4) + Date.now().toString().substring(0, 10),
-        port: r_port,
-        report: rep,
-        known_scanner: fs.readFileSync('./blackhole_list.txt', 'utf8').includes('exampleString')
-      };
-
-      const jsonString = JSON.stringify(jsonObject);
-
-      redisClient.rPush('API_logs', jsonString);
-      return;
-    } catch (error) {
-      console.error('Error:', error);
+
+    const isScanned = await isIpScanned(ip);
+
+    if (!isScanned) {
+      try {
+        let repu = await threat_intelligence.getReputationData(ip);
+        if (repu) {
+          fs.appendFileSync(
+            path.join(reputationDir, "reputation.log"),
+            JSON.stringify(repu) + "\n"
+          );
+        }
+      } catch (error) {
+        console.error("Error getting reputation data:", error);
+      }
+      fs.appendFileSync(
+        path.join(scannedIpsDir, "scanned_ips.log"),
+        ip + "\n"
+      );
     }
-    // console.log("")
-  }
-
 
+    const jsonObject = {
+      ip: ip,
+      timestamp: new Date().toISOString().slice(0, 19),
+      messevent: event,
+      sessionId:
+        (process.env.SESSION_SECRET || "default").substring(0, 4) +
+        Date.now().toString().substring(0, 10),
+      port: r_port,
+      report: rep,
+      known_scanner: fs.readFileSync('./blackhole_list.txt', 'utf8').includes('exampleString')
+    };
 
+    const jsonString = JSON.stringify(jsonObject);
+    fs.appendFileSync(
+      path.join(apiLogsDir, "api_logs.log"),
+      jsonString + "\n"
+    );
+  } catch (error) {
+    console.error("Error in logEvent:", error);
+  }
+}
 
-  module.exports = { logEvent };
+module.exports = { logEvent };
+