redis connection pool implementation

config/redis.conf

@@ -0,0 +1,18 @@
+# Set the maximum allowed number of connected clients.
+# However, ensure your system has adequate resources to handle this.
+maxclients 10000
+
+# Set a timeout to close idle connections after a period (in seconds).
+timeout 60
+
+# Enable append only mode for better data durability.
+appendonly yes
+
+# Set the frequency at which Redis fsyncs the AOF.
+appendfsync everysec
+
+# Slow log logging level (0 = don't log, 1 = log the slower queries, 2 = log all)
+slowlog-log-slower-than 10000
+
+# Maximum length of the slow query log
+slowlog-max-len 128

environment/docker-compose.dev.yml

@@ -59,8 +59,8 @@ services:
       - "$REDIS_PORT"
     env_file: .env
     volumes:
-      - ./config/redis.conf:/redis.conf
-    command: ["redis-server", "/redis.conf", "--port", "${REDIS_PORT}"]
+      - ./config/redis.conf:/usr/local/etc/redis/redis.conf
+    command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--port", "${REDIS_PORT}"]
     networks:
       - rpg-network
 

src/jest/redisV4Mock.ts

@@ -29,5 +29,21 @@ const v4Client = {
   pSetEx: (key: string, ms: number, value: string) => setEx(key, ms / 1000, value),
   on: () => undefined,
   // Add additional functions as needed...
+
+  // IORedis compatible functions
+  hset: promisify(client.hset).bind(client),
+  keys: promisify(client.keys).bind(client),
+  hkeys: promisify(client.hkeys).bind(client),
+  hsetnx: promisify(client.hsetnx).bind(client),
+  hget: promisify(client.hget).bind(client),
+  hdel: promisify(client.hdel).bind(client),
+  hgetall: promisify(client.hgetall).bind(client),
+
+  hexists: promisify(client.hexists).bind(client),
+  flushall: promisify(client.flushall).bind(client),
+  setex: promisify(client.setex).bind(client),
+  mget: promisify(client.mget).bind(client),
+  pttl: promisify(client.pttl).bind(client),
+  psetex: (key: string, ms: number, value: string) => setEx(key, ms / 1000, value),
 };
 export default { ...redis, createClient: () => v4Client };

src/providers/cronjobs/CronJobs.ts

@@ -2,10 +2,11 @@ import { appEnv } from "@providers/config/env";
 import { PM2Helper } from "@providers/server/PM2Helper";
 import { EnvType } from "@rpg-engine/shared";
 import { provide } from "inversify-binding-decorators";
+import { RedisCrons } from "./RedisCrons";
 
 @provide(Cronjob)
 export class Cronjob {
-  constructor(private pm2Helper: PM2Helper) {}
+  constructor(private pm2Helper: PM2Helper, private redisCrons: RedisCrons) {}
 
   public start(): void {
     this.scheduleCrons();
@@ -16,13 +17,13 @@ export class Cronjob {
 
     switch (appEnv.general.ENV) {
       case EnvType.Development:
-        // schedule here
+        this.redisCrons.schedule();
         break;
       case EnvType.Staging:
       case EnvType.Production:
         // make sure it only runs in one instance
         if (process.env.pm_id === this.pm2Helper.pickLastCPUInstance()) {
-          // schedule here
+          this.redisCrons.schedule();
         }
         break;
     }

src/providers/cronjobs/RedisCrons.ts

@@ -0,0 +1,15 @@
+import { RedisManager } from "@providers/database/RedisManager/RedisManager";
+import { provide } from "inversify-binding-decorators";
+import { CronJobScheduler } from "./CronJobScheduler";
+
+@provide(RedisCrons)
+export class RedisCrons {
+  constructor(private cronJobScheduler: CronJobScheduler, private redisManager: RedisManager) {}
+
+  public schedule(): void {
+    // every 5 minutes
+    this.cronJobScheduler.uniqueSchedule("redis-client-connection-count", "* * * * *", async () => {
+      await this.redisManager.getClientCount();
+    });
+  }
+}

src/providers/database/InMemoryHashTable.ts

@@ -1,17 +1,18 @@
 import { appEnv } from "@providers/config/env";
 import { provide } from "inversify-binding-decorators";
-import { RedisManager } from "./RedisManager";
+import { RedisManager } from "./RedisManager/RedisManager";
 
 @provide(InMemoryHashTable)
 export class InMemoryHashTable {
   constructor(private redisManager: RedisManager) {}
 
   public async set(namespace: string, key: string, value: any): Promise<void> {
-    await this.redisManager.client.hSet(namespace?.toString(), key?.toString(), JSON.stringify(value));
+    await this.redisManager.client.hset(namespace?.toString(), key?.toString(), JSON.stringify(value));
   }
 
   public async setNx(namespace: string, key: string, value: any): Promise<boolean> {
-    return await this.redisManager.client.hSetNX(namespace?.toString(), key?.toString(), JSON.stringify(value));
+    const result = await this.redisManager.client.hsetnx(namespace?.toString(), key?.toString(), JSON.stringify(value));
+    return result === 1;
   }
 
   public async expire(key: string, seconds: number, mode: "NX" | "XX" | "GT" | "LT"): Promise<void> {
@@ -25,13 +26,12 @@ export class InMemoryHashTable {
       throw new Error("Namespace is undefined or null.");
     }
 
-    const timeLeft = await this.redisManager.client.pTTL(namespace.toString());
-
+    const timeLeft = await this.redisManager.client.pttl(namespace.toString());
     return timeLeft;
   }
 
   public async getAll<T>(namespace: string): Promise<Record<string, T> | undefined> {
-    const values = await this.redisManager.client.hGetAll(namespace?.toString());
+    const values = await this.redisManager.client.hgetall(namespace?.toString());
 
     if (!values) {
       return;
@@ -41,20 +41,17 @@ export class InMemoryHashTable {
   }
 
   public async has(namespace: string, key: string): Promise<boolean> {
-    const result = await this.redisManager.client.hExists(namespace?.toString(), key?.toString());
-
+    const result = await this.redisManager.client.hexists(namespace?.toString(), key?.toString());
     return Boolean(result);
   }
 
   public async hasAll(namespace: string): Promise<boolean> {
     const result = await this.redisManager.client.exists(namespace?.toString());
-
-    // @ts-ignore
     return result === 1;
   }
 
   public async get(namespace: string, key: string): Promise<Record<string, any> | undefined> {
-    const value = await this.redisManager.client.hGet(namespace?.toString(), key?.toString());
+    const value = await this.redisManager.client.hget(namespace?.toString(), key?.toString());
 
     if (!value) {
       return;
@@ -64,19 +61,17 @@ export class InMemoryHashTable {
   }
 
   public async getAllKeysWithPrefix(prefix: string): Promise<string[]> {
-    const keys = await this.redisManager.client.keys?.(`${prefix}*`);
-
+    const keys = await this.redisManager.client.keys(`${prefix}*`);
     return keys ?? [];
   }
 
   public async getAllKeys(namespace: string): Promise<string[]> {
-    const keys = await this.redisManager.client.hKeys(namespace?.toString());
-
+    const keys = await this.redisManager.client.hkeys(namespace?.toString());
     return keys ?? [];
   }
 
   public async delete(namespace: string, key: string): Promise<void> {
-    await this.redisManager.client.hDel(namespace?.toString(), key?.toString());
+    await this.redisManager.client.hdel(namespace?.toString(), key?.toString());
   }
 
   public async deleteAll(namespace: string): Promise<void> {

src/providers/database/InMemoryRepository.ts

@@ -1,4 +1,4 @@
-import { RedisManager } from "@providers/database/RedisManager";
+import { RedisManager } from "@providers/database/RedisManager/RedisManager";
 import { IResource } from "@rpg-engine/shared";
 import { provide } from "inversify-binding-decorators";
 

src/providers/database/RedisManager/RedisBareClient.ts

@@ -0,0 +1,42 @@
+/* eslint-disable no-async-promise-executor */
+import { appEnv } from "@providers/config/env";
+import { provideSingleton } from "@providers/inversify/provideSingleton";
+import mongoose from "mongoose";
+import { createClient } from "redis";
+import { applySpeedGooseCacheLayer } from "speedgoose";
+
+//! in unit test, just use the traditional redis because its already being mocked by redisV4Mock and changing support to IORedis would be a pain
+
+@provideSingleton(RedisBareClient)
+export class RedisBareClient {
+  public async connect(): Promise<void> {
+    return await new Promise<void>(async (resolve, reject) => {
+      let client;
+
+      try {
+        const redisConnectionUrl = `redis://${appEnv.database.REDIS_CONTAINER}:${appEnv.database.REDIS_PORT}`;
+
+        client = createClient({
+          url: redisConnectionUrl,
+        });
+
+        client.on("error", (err) => {
+          console.log("❌ Redis error:", err);
+          reject(err);
+        });
+
+        await client.connect();
+
+        // @ts-ignore
+        void applySpeedGooseCacheLayer(mongoose, {
+          redisUri: redisConnectionUrl,
+        });
+
+        resolve(client);
+      } catch (error) {
+        console.log("❌ Redis initialization error: ", error);
+        reject(error);
+      }
+    });
+  }
+}