Skip to content

Commit

Permalink
#3667 - Reuse Redis Connections (#3992)
Browse files Browse the repository at this point in the history
During the prior investigation was detected that when all the schedulers
try to execute redis operations at the same time it fails after a
certain amount of concurrent connections.
The possible solutions being investigated are:
1. Allow the connection sharing as mentioned here
https://github.com/OptimalBits/bull/blob/develop/PATTERNS.md#reusing-redis-connections
2. Control the service initialization to prevent multiple concurrent
connections.

This is PR an attempt to test the first options.
  • Loading branch information
andrewsignori-aot committed Nov 25, 2024
1 parent ed10050 commit 232851f
Showing 1 changed file with 51 additions and 10 deletions.
61 changes: 51 additions & 10 deletions sources/packages/backend/libs/services/src/queue/queue.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,34 @@ import { DatabaseModule } from "@sims/sims-db";
})
export class QueueModule {}

/**
* Shared client connection to redis cluster.
* @see https://github.com/OptimalBits/bull/blob/develop/PATTERNS.md#reusing-redis-connections
*/
let sharedClientRedisCluster: Cluster;
/**
* Shared subscriber connection to redis cluster.
* @see https://github.com/OptimalBits/bull/blob/develop/PATTERNS.md#reusing-redis-connections
*/
let sharedSubscriberClientRedisCluster: Cluster;

/**
* Creates a Redis cluster connection.
* @param options the redis connection options.
* @returns a new instance of Redis cluster connection.
*/
function createdRedisClusterConnection(options: RedisOptions): Cluster {
return new Redis.Cluster(
[
{
host: options.host,
port: options.port,
},
],
{ redisOptions: { password: options.password } },
);
}

/**
* Connection factory which returns connection properties
* to connect redis.
Expand All @@ -54,16 +82,29 @@ async function getConnectionFactory(
};
}
return {
createClient: (): Redis | Cluster => {
return new Redis.Cluster(
[
{
host: redisConnectionOptions.host,
port: redisConnectionOptions.port,
},
],
{ redisOptions: { password: redisConnectionOptions.password } },
);
createClient: (
type: "client" | "subscriber" | "bclient",
): Redis | Cluster => {
switch (type) {
case "client":
if (!sharedClientRedisCluster) {
sharedClientRedisCluster = createdRedisClusterConnection(
redisConnectionOptions,
);
}
return sharedClientRedisCluster;
case "subscriber":
if (!sharedSubscriberClientRedisCluster) {
sharedSubscriberClientRedisCluster = createdRedisClusterConnection(
redisConnectionOptions,
);
}
return sharedSubscriberClientRedisCluster;
case "bclient":
// bclient types should always create a new connection.
// @see https://github.com/OptimalBits/bull/blob/develop/PATTERNS.md#reusing-redis-connections
return createdRedisClusterConnection(redisConnectionOptions);
}
},
};
}
Expand Down

0 comments on commit 232851f

Please sign in to comment.