diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..889f869 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +**/.git +**/node_modules +**/fastAPI +**/src/Deployments \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f502777 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM node:20 + +WORKDIR /safemapsbackend + +COPY package.json ./ + +RUN npm install + +COPY . . + +EXPOSE 3000 + +CMD ["node", "fetchCrimeData.js"] \ No newline at end of file diff --git a/fetchCrimeData.js b/fetchCrimeData.js index a53cc22..38f476c 100644 --- a/fetchCrimeData.js +++ b/fetchCrimeData.js @@ -5,10 +5,16 @@ const express = require("express"); const axios = require("axios"); const bodyParser = require("body-parser"); const cors = require("cors"); +const objectHash = require("object-hash"); const ResponseStatus = require("./ResponseStatus"); const Response = require("./Response"); const ResponseUtils = require("./ResponseUtils"); +const { + initializeRedisClient, + readData, + writeData, +} = require("./src/middleware/redis"); const app = express(); app.use(cors()); @@ -24,10 +30,10 @@ class FetchCrimePostBody { } } +console.log("Setting up API for post") app.post("/fetchCrimeData", async (req, res) => { try { const { latitude, longitude, radius } = req.body; - console.log(req.body); if (!latitude || !longitude) { ResponseUtils.setResponseError( @@ -40,6 +46,21 @@ app.post("/fetchCrimeData", async (req, res) => { const userLocation = new FetchCrimePostBody(latitude, longitude, radius); + const cacheKey = objectHash.sha1([ + userLocation.latitude, + userLocation.longitude, + userLocation.radius, + ]); + console.log('Cached key' + cacheKey) + cachedValue = await readData(cacheKey); + console.log('Cached value found: ' + cachedValue) + if (cachedValue) { + // No need to call API as we have values in cache and can skip it. + const responseStatus = new ResponseStatus(200, "OK", "Success"); + res.json(new Response(cachedValue, responseStatus)); + return; + } + const apiUrl = "https://spotcrime.com/map?lat=${userLocation.latitude}&lon=${userLocation.longitude}"; const response = await axios.get(apiUrl); @@ -55,6 +76,9 @@ app.post("/fetchCrimeData", async (req, res) => { ); if (crimeDataResponse.status == 200) { const responseStatus = new ResponseStatus(200, "OK", "Success"); + + console.log("Saving in cache" + cacheKey + ", value= " + crimeDataResponse.data.crimes) + writeData(cacheKey, crimeDataResponse.data.crimes); res.json(new Response(crimeDataResponse.data.crimes, responseStatus)); } else { ResponseUtils.setResponseError( @@ -77,6 +101,25 @@ app.post("/fetchCrimeData", async (req, res) => { } }); -app.listen(port, () => { - console.log(`Server is running on http://localhost:${port}`); -}); +// app.listen(port, () => { +// console.log(`Server is running on http://localhost:${port}`); +// }); + +async function initializeExpressServer() { + //initialize an Express application + app.use(express.json()); + + //connect to Redis + if (process.env.REDIS_URL) { + await initializeRedisClient(); + } + + // start the server + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); +} + +initializeExpressServer() + .then() + .catch((e) => console.error(e)); diff --git a/package-lock.json b/package-lock.json index 784e0b2..6f889dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,9 +12,66 @@ "axios": "^1.6.7", "body-parser": "^1.20.2", "cors": "^2.8.5", - "express": "^4.18.2" + "express": "^4.18.2", + "object-hash": "^3.0.0", + "redis": "^4.6.13" }, - "devDependencies": {} + "devDependencies": { + "prettier": "3.2.5" + } + }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz", + "integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } }, "node_modules/accepts": { "version": "1.3.8", @@ -97,6 +154,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -360,6 +425,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -544,6 +617,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -576,6 +657,21 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, + "node_modules/prettier": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", + "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -629,6 +725,22 @@ "node": ">= 0.8" } }, + "node_modules/redis": { + "version": "4.6.13", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz", + "integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.14", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -784,6 +896,11 @@ "engines": { "node": ">= 0.8" } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } } diff --git a/package.json b/package.json index 1c7c24f..4c787f9 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "axios": "^1.6.7", "body-parser": "^1.20.2", "cors": "^2.8.5", - "express": "^4.18.2" + "express": "^4.18.2", + "object-hash": "^3.0.0", + "redis": "^4.6.13" }, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -21,5 +23,8 @@ "bugs": { "url": "https://github.com/T-I-P/SafeMapsBackend/issues" }, - "homepage": "https://github.com/T-I-P/SafeMapsBackend#readme" + "homepage": "https://github.com/T-I-P/SafeMapsBackend#readme", + "devDependencies": { + "prettier": "3.2.5" + } } diff --git a/src/middleware/redis.js b/src/middleware/redis.js new file mode 100644 index 0000000..3b878cd --- /dev/null +++ b/src/middleware/redis.js @@ -0,0 +1,138 @@ +const objectHash = require("object-hash"); +const { createClient } = require("redis"); +const zlib = require("zlib"); + +//initialize redis client +let redisClient = undefined; + + +async function initializeRedisClient() { + // read the Redis connection URL from the envs + let redisURL = process.env.REDIS_URL + //let redisURL = 'redis://localhost:6379'; + console.log(redisURL) + if (redisURL) { + // create the Redis client object + redisClient = createClient({ url: redisURL }).on("error", (e) => { + console.error(`Failed to create Redis Client with error:`, e); + }); + } + try { + // connect to the Redis server + await redisClient.connect(); + console.log("Connected to redis successfully"); + } catch (e) { + console.error("Connection to redis failed with error:", e); + } +} + +function requestToKey(req) { + // build a custom object to use as part of the Redis key + const reqDataToHash = { + query: req.query, + body: req.body, + }; + // `${req.path}@...` to make it easier to find + // keys on a Redis client + return `${req.path}@${objectHash.sha1(reqDataToHash)}`; +} + +function isRedisWorking() { + //verify whether there is an active connection to a Redis Server + return redisClient?.isOpen; +} + +async function writeData( + key, + data, + options = { + EX: 21600, //6h + }, + compression = true, // enable compression and decompression by default +) { + if (isRedisWorking()) { + console.log('Attempting write in cache, is redis working: ' + data) + let newdata = JSON.stringify(data); + console.log("New Data",newdata) + let dataToCache = newdata; + if (compression) { + // compress the value with ZLIB to save RAM + dataToCache = zlib.deflateSync(newdata).toString("base64"); + console.log("Compressed cached is being written :", dataToCache) + } + + try { + //write data to th Redis cache + await redisClient.set(key, dataToCache, options); + } catch (e) { + console.error(`Failed to cache data for keys = ${key}`, e); + } + } +} + +async function readData(key, compressed = true) { + let cachedValue = undefined; + if (isRedisWorking()) { + cachedValue = await redisClient.get(key); + console.log("Reading the cached value: ",cachedValue) + if (cachedValue) { + if (compressed) { + // decompress the cached value with ZLIB + console.log("Will return compressed cached value") + return zlib.inflateSync(Buffer.from(cachedValue, "base64")).toString(); + } else { + return cachedValue; + } + } + } + return cachedValue; +} +function redisCaching(key, value) { + return redisCachingMiddleware(key, value); +} + +function redisCachingMiddleware( + key, + value, + options = { + EX: 21600, //6h + }, + compression = true, // enable compression and decompression by default +) { + return async (req, res, next) => { + if (isRedisWorking()) { + const key = requestToKey(req); + // if there is some cached data, retrieve it and return it + const cachedValue = await readData(key, compression); + if (cachedValue) { + try { + // if it is JSON data, then return it + return res.json(JSON.parse(cachedValue)); + } catch { + // if it is not JSON data, then return it + return res.send(cachedValue); + } + } else { + // override how res.send behaves + // to introduce the caching logic + const oldSend = res.send; + res.send = function (data) { + // set the function back to avoid the 'double-send' effect + res.send = oldSend; + + // cache the response only if it is successful + if (res.statusCode.toString().startsWith("2")) { + writeData(key, data, options, compression).then(); + } + return res.send(data); + }; + // continue to the controller function + next(); + } + } else { + //proceed with no caching + next(); + } + }; +} +module.exports = { initializeRedisClient, readData, writeData };