From a35df7259b4413a84dc826f41e85c119431683ca Mon Sep 17 00:00:00 2001 From: itsayushpandey Date: Wed, 10 Apr 2024 22:42:04 -0600 Subject: [PATCH 1/4] Ayush_feature/Adding_Redis_for_caching --- fetchCrimeData.js | 32 ++++++++++ src/middleware/redis.js | 134 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 src/middleware/redis.js diff --git a/fetchCrimeData.js b/fetchCrimeData.js index a53cc22..694f1b1 100644 --- a/fetchCrimeData.js +++ b/fetchCrimeData.js @@ -9,6 +9,7 @@ const cors = require("cors"); const ResponseStatus = require("./ResponseStatus"); const Response = require("./Response"); const ResponseUtils = require("./ResponseUtils"); +const { initializeRedisClient, readData, writeData } = require("./src/middleware/redis"); const app = express(); app.use(cors()); @@ -40,6 +41,16 @@ app.post("/fetchCrimeData", async (req, res) => { const userLocation = new FetchCrimePostBody(latitude, longitude, radius); + const cacheKey = [userLocation.latitude, userLocation.longitude, userLocation.radius] + cachedValue = readData(cacheKey) + if (cachedValue) { + // No need to call API as we have values in cache and can skip it. + const responseStatus = new ResponseStatus(200, "OK", "Success"); + res.json(new Response(cachedValue, responseStatus)); + return + } + + const apiUrl = "https://spotcrime.com/map?lat=${userLocation.latitude}&lon=${userLocation.longitude}"; const response = await axios.get(apiUrl); @@ -75,8 +86,29 @@ app.post("/fetchCrimeData", async (req, res) => { console.error(`Sever error = ${error}`); ResponseUtils.setResponseError(res, 500, "Internal Server Error"); } + writeData(cacheKey, res.crimes); }); app.listen(port, () => { console.log(`Server is running on http://localhost:${port}`); }); + +async function initializeExpressServer() { + //initialize an Express application + const app = express(); + app.use(express.json()); + + //connect to Redis + await initializeRedisClient(); + + + // start the server + const port = 3000; + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); +} + +initializeExpressServer() + .then() + .catch((e) => console.error(e)); \ No newline at end of file diff --git a/src/middleware/redis.js b/src/middleware/redis.js new file mode 100644 index 0000000..a519bdb --- /dev/null +++ b/src/middleware/redis.js @@ -0,0 +1,134 @@ +const objectHash = require("object-hash"); +const { createClient } = require("redis"); + +//initialize redis client +let redisClient = undefined; + +let REDIS_URI = "redis://localhost:6379"; + +async function initializeRedisClient() { + // read the Redis connection URL from the envs + let redisURL = process.env.REDIS_URI + if (redisURL) { + // create the Redis client object + redisClient = createClient({ url: redisURL }).on("error", (e) => { + console.error(`Failed to create Redis Client with error:`, e); + }); + } + try { + // connect to the Redis server + await redisClient.connect(); + console.log('Connected to redis successfully'); + } catch (e) { + console.error("Connection to redis failed with error:", e); + } +} + +function requestToKey(req) { + // build a custom object to use as part of the Redis key + const reqDataToHash = { + query: req.query, + body: req.body + } + // `${req.path}@...` to make it easier to find + // keys on a Redis client + return `${req.path}@${objectHash.sha1(reqDataToHash)}` +} + +function isRedisWorking() { + //verify whether there is an active connection to a Redis Server + return redisClient?.isOpen; + +} + +async function writeData( + key, + data, + options = { + EX: 21600, //6h + }, + compression = true // enable compression and decompression by default +) { + if (isRedisWorking) { + let dataToCache = data; + if (compress) { + // compress the value with ZLIB to save RAM + dataToCache = zlib.deflateSync(data).toString("base64"); + } + + try { + //write data to th Redis cache + await redisClient.set(key, dataToCache, options); + } catch (e) { + console.error(`Failed to cache data for keys = ${key}`, e); + } + } +} + +async function readData(key, compressed = true) { + let cachedValue = undefined; + if (isRedisWorking) { + cachedValue = await redisClient.get(key); + if (cachedValue) { + if (compressed) { + // decompress the cached value with ZLIB + return zlib.inflateSync(Buffer.from(cachedValue, "base64")).toString(); + } + else { + return cachedValue; + } + } + } + return cachedValue; +} +function redisCaching(key, value) { + + return redisCachingMiddleware(key, value) +} + +function redisCachingMiddleware( + key, + value, + options = { + EX: 21600, //6h + }, + compression = true // enable compression and decompression by default +) { + return async (req, res, next) => { + if (isRedisWorking) { + const key = requestToKey(req); + // if there is some cached data, retrieve it and return it + const cachedValue = await readData(key, compression); + if (cachedValue) { + try { + // if it is JSON data, then return it + return res.json(JSON.parse(cachedValue)) + } catch { + // if it is not JSON data, then return it + return res.send(cachedValue); + } + } else { + // override how res.send behaves + // to introduce the caching logic + const oldSend = res.send; + res.send = function (data) { + // set the function back to avoid the 'double-send' effect + res.send = oldSend; + + // cache the response only if it is successful + if (res.statusCode.toString().startsWith('2')) { + writeData(key, data, options, compression).then(); + } + return res.send(data); + }; + // continue to the controller function + next(); + } + } + else { + //proceed with no caching + next(); + } + } +} +module.exports = { initializeRedisClient, readData, writeData }; From 96d0af3e595befcbe8758819a0ffaf9c3400818f Mon Sep 17 00:00:00 2001 From: itsayushpandey Date: Wed, 10 Apr 2024 23:06:46 -0600 Subject: [PATCH 2/4] Ayush_feature/Redis_Cache --- fetchCrimeData.js | 20 +++-- package-lock.json | 19 +++- package.json | 5 +- src/middleware/redis.js | 194 ++++++++++++++++++++-------------------- 4 files changed, 130 insertions(+), 108 deletions(-) diff --git a/fetchCrimeData.js b/fetchCrimeData.js index 694f1b1..9cbd1ab 100644 --- a/fetchCrimeData.js +++ b/fetchCrimeData.js @@ -9,7 +9,11 @@ const cors = require("cors"); const ResponseStatus = require("./ResponseStatus"); const Response = require("./Response"); const ResponseUtils = require("./ResponseUtils"); -const { initializeRedisClient, readData, writeData } = require("./src/middleware/redis"); +const { + initializeRedisClient, + readData, + writeData, +} = require("./src/middleware/redis"); const app = express(); app.use(cors()); @@ -41,16 +45,19 @@ app.post("/fetchCrimeData", async (req, res) => { const userLocation = new FetchCrimePostBody(latitude, longitude, radius); - const cacheKey = [userLocation.latitude, userLocation.longitude, userLocation.radius] - cachedValue = readData(cacheKey) + const cacheKey = [ + userLocation.latitude, + userLocation.longitude, + userLocation.radius, + ]; + cachedValue = readData(cacheKey); if (cachedValue) { // No need to call API as we have values in cache and can skip it. const responseStatus = new ResponseStatus(200, "OK", "Success"); res.json(new Response(cachedValue, responseStatus)); - return + return; } - const apiUrl = "https://spotcrime.com/map?lat=${userLocation.latitude}&lon=${userLocation.longitude}"; const response = await axios.get(apiUrl); @@ -101,7 +108,6 @@ async function initializeExpressServer() { //connect to Redis await initializeRedisClient(); - // start the server const port = 3000; app.listen(port, () => { @@ -111,4 +117,4 @@ async function initializeExpressServer() { initializeExpressServer() .then() - .catch((e) => console.error(e)); \ No newline at end of file + .catch((e) => console.error(e)); diff --git a/package-lock.json b/package-lock.json index 784e0b2..dd194d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "cors": "^2.8.5", "express": "^4.18.2" }, - "devDependencies": {} + "devDependencies": { + "prettier": "3.2.5" + } }, "node_modules/accepts": { "version": "1.3.8", @@ -576,6 +578,21 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, + "node_modules/prettier": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", + "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", diff --git a/package.json b/package.json index 1c7c24f..a575e17 100644 --- a/package.json +++ b/package.json @@ -21,5 +21,8 @@ "bugs": { "url": "https://github.com/T-I-P/SafeMapsBackend/issues" }, - "homepage": "https://github.com/T-I-P/SafeMapsBackend#readme" + "homepage": "https://github.com/T-I-P/SafeMapsBackend#readme", + "devDependencies": { + "prettier": "3.2.5" + } } diff --git a/src/middleware/redis.js b/src/middleware/redis.js index a519bdb..543a148 100644 --- a/src/middleware/redis.js +++ b/src/middleware/redis.js @@ -7,128 +7,124 @@ let redisClient = undefined; let REDIS_URI = "redis://localhost:6379"; async function initializeRedisClient() { - // read the Redis connection URL from the envs - let redisURL = process.env.REDIS_URI - if (redisURL) { - // create the Redis client object - redisClient = createClient({ url: redisURL }).on("error", (e) => { - console.error(`Failed to create Redis Client with error:`, e); - }); - } - try { - // connect to the Redis server - await redisClient.connect(); - console.log('Connected to redis successfully'); - } catch (e) { - console.error("Connection to redis failed with error:", e); - } + // read the Redis connection URL from the envs + let redisURL = process.env.REDIS_URI; + if (redisURL) { + // create the Redis client object + redisClient = createClient({ url: redisURL }).on("error", (e) => { + console.error(`Failed to create Redis Client with error:`, e); + }); + } + try { + // connect to the Redis server + await redisClient.connect(); + console.log("Connected to redis successfully"); + } catch (e) { + console.error("Connection to redis failed with error:", e); + } } function requestToKey(req) { - // build a custom object to use as part of the Redis key - const reqDataToHash = { - query: req.query, - body: req.body - } - // `${req.path}@...` to make it easier to find - // keys on a Redis client - return `${req.path}@${objectHash.sha1(reqDataToHash)}` + // build a custom object to use as part of the Redis key + const reqDataToHash = { + query: req.query, + body: req.body, + }; + // `${req.path}@...` to make it easier to find + // keys on a Redis client + return `${req.path}@${objectHash.sha1(reqDataToHash)}`; } function isRedisWorking() { - //verify whether there is an active connection to a Redis Server - return redisClient?.isOpen; - + //verify whether there is an active connection to a Redis Server + return redisClient?.isOpen; } async function writeData( - key, - data, - options = { - EX: 21600, //6h - }, - compression = true // enable compression and decompression by default + key, + data, + options = { + EX: 21600, //6h + }, + compression = true, // enable compression and decompression by default ) { - if (isRedisWorking) { - let dataToCache = data; - if (compress) { - // compress the value with ZLIB to save RAM - dataToCache = zlib.deflateSync(data).toString("base64"); - } + if (isRedisWorking) { + let dataToCache = data; + if (compress) { + // compress the value with ZLIB to save RAM + dataToCache = zlib.deflateSync(data).toString("base64"); + } - try { - //write data to th Redis cache - await redisClient.set(key, dataToCache, options); - } catch (e) { - console.error(`Failed to cache data for keys = ${key}`, e); - } + try { + //write data to th Redis cache + await redisClient.set(key, dataToCache, options); + } catch (e) { + console.error(`Failed to cache data for keys = ${key}`, e); } + } } async function readData(key, compressed = true) { - let cachedValue = undefined; - if (isRedisWorking) { - cachedValue = await redisClient.get(key); - if (cachedValue) { - if (compressed) { - // decompress the cached value with ZLIB - return zlib.inflateSync(Buffer.from(cachedValue, "base64")).toString(); - } - else { - return cachedValue; - } - } + let cachedValue = undefined; + if (isRedisWorking) { + cachedValue = await redisClient.get(key); + if (cachedValue) { + if (compressed) { + // decompress the cached value with ZLIB + return zlib.inflateSync(Buffer.from(cachedValue, "base64")).toString(); + } else { + return cachedValue; + } } - return cachedValue; + } + return cachedValue; } function redisCaching(key, value) { - - return redisCachingMiddleware(key, value) + return redisCachingMiddleware(key, value); } function redisCachingMiddleware( - key, - value, - options = { - EX: 21600, //6h - }, - compression = true // enable compression and decompression by default + key, + value, + options = { + EX: 21600, //6h + }, + compression = true, // enable compression and decompression by default ) { - return async (req, res, next) => { - if (isRedisWorking) { - const key = requestToKey(req); - // if there is some cached data, retrieve it and return it - const cachedValue = await readData(key, compression); - if (cachedValue) { - try { - // if it is JSON data, then return it - return res.json(JSON.parse(cachedValue)) - } catch { - // if it is not JSON data, then return it - return res.send(cachedValue); - } - } else { - // override how res.send behaves - // to introduce the caching logic - const oldSend = res.send; - res.send = function (data) { - // set the function back to avoid the 'double-send' effect - res.send = oldSend; - - // cache the response only if it is successful - if (res.statusCode.toString().startsWith('2')) { - writeData(key, data, options, compression).then(); - } - return res.send(data); - }; - // continue to the controller function - next(); - } - } - else { - //proceed with no caching - next(); + return async (req, res, next) => { + if (isRedisWorking) { + const key = requestToKey(req); + // if there is some cached data, retrieve it and return it + const cachedValue = await readData(key, compression); + if (cachedValue) { + try { + // if it is JSON data, then return it + return res.json(JSON.parse(cachedValue)); + } catch { + // if it is not JSON data, then return it + return res.send(cachedValue); } + } else { + // override how res.send behaves + // to introduce the caching logic + const oldSend = res.send; + res.send = function (data) { + // set the function back to avoid the 'double-send' effect + res.send = oldSend; + + // cache the response only if it is successful + if (res.statusCode.toString().startsWith("2")) { + writeData(key, data, options, compression).then(); + } + return res.send(data); + }; + // continue to the controller function + next(); + } + } else { + //proceed with no caching + next(); } + }; } module.exports = { initializeRedisClient, readData, writeData }; From 846ccb5bd3766c9be005605b10f6cea1700340fe Mon Sep 17 00:00:00 2001 From: itsayushpandey Date: Fri, 12 Apr 2024 23:06:21 -0600 Subject: [PATCH 3/4] Removed bugs from Redis Code --- fetchCrimeData.js | 27 ++++++----- package-lock.json | 102 +++++++++++++++++++++++++++++++++++++++- package.json | 4 +- src/middleware/redis.js | 24 ++++++---- 4 files changed, 136 insertions(+), 21 deletions(-) diff --git a/fetchCrimeData.js b/fetchCrimeData.js index 9cbd1ab..38f476c 100644 --- a/fetchCrimeData.js +++ b/fetchCrimeData.js @@ -5,6 +5,7 @@ const express = require("express"); const axios = require("axios"); const bodyParser = require("body-parser"); const cors = require("cors"); +const objectHash = require("object-hash"); const ResponseStatus = require("./ResponseStatus"); const Response = require("./Response"); @@ -29,10 +30,10 @@ class FetchCrimePostBody { } } +console.log("Setting up API for post") app.post("/fetchCrimeData", async (req, res) => { try { const { latitude, longitude, radius } = req.body; - console.log(req.body); if (!latitude || !longitude) { ResponseUtils.setResponseError( @@ -45,12 +46,14 @@ app.post("/fetchCrimeData", async (req, res) => { const userLocation = new FetchCrimePostBody(latitude, longitude, radius); - const cacheKey = [ + const cacheKey = objectHash.sha1([ userLocation.latitude, userLocation.longitude, userLocation.radius, - ]; - cachedValue = readData(cacheKey); + ]); + console.log('Cached key' + cacheKey) + cachedValue = await readData(cacheKey); + console.log('Cached value found: ' + cachedValue) if (cachedValue) { // No need to call API as we have values in cache and can skip it. const responseStatus = new ResponseStatus(200, "OK", "Success"); @@ -73,6 +76,9 @@ app.post("/fetchCrimeData", async (req, res) => { ); if (crimeDataResponse.status == 200) { const responseStatus = new ResponseStatus(200, "OK", "Success"); + + console.log("Saving in cache" + cacheKey + ", value= " + crimeDataResponse.data.crimes) + writeData(cacheKey, crimeDataResponse.data.crimes); res.json(new Response(crimeDataResponse.data.crimes, responseStatus)); } else { ResponseUtils.setResponseError( @@ -93,23 +99,22 @@ app.post("/fetchCrimeData", async (req, res) => { console.error(`Sever error = ${error}`); ResponseUtils.setResponseError(res, 500, "Internal Server Error"); } - writeData(cacheKey, res.crimes); }); -app.listen(port, () => { - console.log(`Server is running on http://localhost:${port}`); -}); +// app.listen(port, () => { +// console.log(`Server is running on http://localhost:${port}`); +// }); async function initializeExpressServer() { //initialize an Express application - const app = express(); app.use(express.json()); //connect to Redis - await initializeRedisClient(); + if (process.env.REDIS_URL) { + await initializeRedisClient(); + } // start the server - const port = 3000; app.listen(port, () => { console.log(`Server is running on http://localhost:${port}`); }); diff --git a/package-lock.json b/package-lock.json index dd194d9..6f889dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,67 @@ "axios": "^1.6.7", "body-parser": "^1.20.2", "cors": "^2.8.5", - "express": "^4.18.2" + "express": "^4.18.2", + "object-hash": "^3.0.0", + "redis": "^4.6.13" }, "devDependencies": { "prettier": "3.2.5" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz", + "integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -99,6 +154,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -362,6 +425,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -546,6 +617,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -646,6 +725,22 @@ "node": ">= 0.8" } }, + "node_modules/redis": { + "version": "4.6.13", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz", + "integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.14", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -801,6 +896,11 @@ "engines": { "node": ">= 0.8" } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } } diff --git a/package.json b/package.json index a575e17..4c787f9 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "axios": "^1.6.7", "body-parser": "^1.20.2", "cors": "^2.8.5", - "express": "^4.18.2" + "express": "^4.18.2", + "object-hash": "^3.0.0", + "redis": "^4.6.13" }, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/src/middleware/redis.js b/src/middleware/redis.js index 543a148..3b878cd 100644 --- a/src/middleware/redis.js +++ b/src/middleware/redis.js @@ -1,14 +1,16 @@ const objectHash = require("object-hash"); const { createClient } = require("redis"); +const zlib = require("zlib"); //initialize redis client let redisClient = undefined; -let REDIS_URI = "redis://localhost:6379"; async function initializeRedisClient() { // read the Redis connection URL from the envs - let redisURL = process.env.REDIS_URI; + let redisURL = process.env.REDIS_URL + //let redisURL = 'redis://localhost:6379'; + console.log(redisURL) if (redisURL) { // create the Redis client object redisClient = createClient({ url: redisURL }).on("error", (e) => { @@ -48,11 +50,15 @@ async function writeData( }, compression = true, // enable compression and decompression by default ) { - if (isRedisWorking) { - let dataToCache = data; - if (compress) { + if (isRedisWorking()) { + console.log('Attempting write in cache, is redis working: ' + data) + let newdata = JSON.stringify(data); + console.log("New Data",newdata) + let dataToCache = newdata; + if (compression) { // compress the value with ZLIB to save RAM - dataToCache = zlib.deflateSync(data).toString("base64"); + dataToCache = zlib.deflateSync(newdata).toString("base64"); + console.log("Compressed cached is being written :", dataToCache) } try { @@ -66,11 +72,13 @@ async function writeData( async function readData(key, compressed = true) { let cachedValue = undefined; - if (isRedisWorking) { + if (isRedisWorking()) { cachedValue = await redisClient.get(key); + console.log("Reading the cached value: ",cachedValue) if (cachedValue) { if (compressed) { // decompress the cached value with ZLIB + console.log("Will return compressed cached value") return zlib.inflateSync(Buffer.from(cachedValue, "base64")).toString(); } else { return cachedValue; @@ -92,7 +100,7 @@ function redisCachingMiddleware( compression = true, // enable compression and decompression by default ) { return async (req, res, next) => { - if (isRedisWorking) { + if (isRedisWorking()) { const key = requestToKey(req); // if there is some cached data, retrieve it and return it const cachedValue = await readData(key, compression); From 93703e2a0144fbf84a96ee19addc788a06cdd1d0 Mon Sep 17 00:00:00 2001 From: itsayushpandey Date: Sun, 14 Apr 2024 00:56:11 -0600 Subject: [PATCH 4/4] Added Dockerfile for creating docker Image --- .dockerignore | 4 ++++ Dockerfile | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..889f869 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +**/.git +**/node_modules +**/fastAPI +**/src/Deployments \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f502777 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM node:20 + +WORKDIR /safemapsbackend + +COPY package.json ./ + +RUN npm install + +COPY . . + +EXPOSE 3000 + +CMD ["node", "fetchCrimeData.js"] \ No newline at end of file