From d8a08957684bd3e473cc6e75ddb443fdd61a1ceb Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 22 Oct 2024 15:25:49 +0200 Subject: [PATCH 1/3] Fix redis connection --- .../cache-storage/cache-storage.module-factory.ts | 3 ++- .../message-queue/message-queue.module-factory.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts index 5a197a98e9df..e85c0549573b 100644 --- a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts @@ -1,6 +1,7 @@ import { CacheModuleOptions } from '@nestjs/common'; import { redisStore } from 'cache-manager-redis-yet'; +import IORedis from 'ioredis'; import { CacheStorageType } from 'src/engine/core-modules/cache-storage/types/cache-storage-type.enum'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; @@ -31,7 +32,7 @@ export const cacheStorageModuleFactory = ( return { ...cacheModuleOptions, store: redisStore, - url: connectionString, + client: new IORedis(connectionString), }; } default: diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts index 2948b2ad9011..875ba1e0f503 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts @@ -1,4 +1,4 @@ -import { ConnectionOptions } from 'tls'; +import IORedis from 'ioredis'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { @@ -48,7 +48,7 @@ export const messageQueueModuleFactory = async ( return { type: MessageQueueDriverType.BullMQ, options: { - connection: connectionString as ConnectionOptions, + connection: new IORedis(connectionString), }, } satisfies BullMQDriverFactoryOptions; } From 8c350557395ce5d640a64875cd3ad3d2c2209559 Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 22 Oct 2024 16:29:32 +0200 Subject: [PATCH 2/3] fix redis client hanging --- .../cache-storage.module-factory.ts | 13 ++------ .../cache-storage/cache-storage.module.ts | 3 +- .../engine/core-modules/core-engine.module.ts | 7 ++-- .../message-queue.module-factory.ts | 14 ++------ .../redis-client/redis-client.module.ts | 12 +++++++ .../redis-client/redis-client.service.ts | 33 +++++++++++++++++++ 6 files changed, 58 insertions(+), 24 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/redis-client/redis-client.module.ts create mode 100644 packages/twenty-server/src/engine/core-modules/redis-client/redis-client.service.ts diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts index e85c0549573b..3a7928e48a50 100644 --- a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts @@ -1,13 +1,14 @@ import { CacheModuleOptions } from '@nestjs/common'; import { redisStore } from 'cache-manager-redis-yet'; -import IORedis from 'ioredis'; import { CacheStorageType } from 'src/engine/core-modules/cache-storage/types/cache-storage-type.enum'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; export const cacheStorageModuleFactory = ( environmentService: EnvironmentService, + redisClientService: RedisClientService, ): CacheModuleOptions => { const cacheStorageType = environmentService.get('CACHE_STORAGE_TYPE'); const cacheStorageTtl = environmentService.get('CACHE_STORAGE_TTL'); @@ -21,18 +22,10 @@ export const cacheStorageModuleFactory = ( return cacheModuleOptions; } case CacheStorageType.Redis: { - const connectionString = environmentService.get('REDIS_URL'); - - if (!connectionString) { - throw new Error( - `${cacheStorageType} cache storage requires REDIS_URL to be defined, check your .env file`, - ); - } - return { ...cacheModuleOptions, store: redisStore, - client: new IORedis(connectionString), + client: redisClientService.getClient(), }; } default: diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module.ts index 3f12d09e6d98..dbefb4284578 100644 --- a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module.ts +++ b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module.ts @@ -7,6 +7,7 @@ import { FlushCacheCommand } from 'src/engine/core-modules/cache-storage/command import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; @Global() @Module({ @@ -15,7 +16,7 @@ import { EnvironmentService } from 'src/engine/core-modules/environment/environm isGlobal: true, imports: [ConfigModule], useFactory: cacheStorageModuleFactory, - inject: [EnvironmentService], + inject: [EnvironmentService, RedisClientService], }), ], providers: [ diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index 00cb30716f39..78f4361f218c 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -32,16 +32,18 @@ import { messageQueueModuleFactory } from 'src/engine/core-modules/message-queue import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timeline-messaging.module'; import { OpenApiModule } from 'src/engine/core-modules/open-api/open-api.module'; import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-credentials/postgres-credentials.module'; +import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; import { serverlessModuleFactory } from 'src/engine/core-modules/serverless/serverless-module.factory'; import { ServerlessModule } from 'src/engine/core-modules/serverless/serverless.module'; +import { WorkspaceSSOModule } from 'src/engine/core-modules/sso/sso.module'; import { TelemetryModule } from 'src/engine/core-modules/telemetry/telemetry.module'; import { UserModule } from 'src/engine/core-modules/user/user.module'; import { WorkflowTriggerApiModule } from 'src/engine/core-modules/workflow/workflow-trigger-api.module'; import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-invitation/workspace-invitation.module'; import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; -import { WorkspaceSSOModule } from 'src/engine/core-modules/sso/sso.module'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; import { AnalyticsModule } from './analytics/analytics.module'; import { ClientConfigModule } from './client-config/client-config.module'; import { FileModule } from './file/file.module'; @@ -69,6 +71,7 @@ import { FileModule } from './file/file.module'; ActorModule, TelemetryModule, EnvironmentModule.forRoot({}), + RedisClientModule, FileStorageModule.forRootAsync({ useFactory: fileStorageModuleFactory, inject: [EnvironmentService], @@ -79,7 +82,7 @@ import { FileModule } from './file/file.module'; }), MessageQueueModule.registerAsync({ useFactory: messageQueueModuleFactory, - inject: [EnvironmentService], + inject: [EnvironmentService, RedisClientService], }), ExceptionHandlerModule.forRootAsync({ useFactory: exceptionHandlerModuleFactory, diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts index 875ba1e0f503..d19b23797e9e 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts @@ -1,5 +1,3 @@ -import IORedis from 'ioredis'; - import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { BullMQDriverFactoryOptions, @@ -8,6 +6,7 @@ import { PgBossDriverFactoryOptions, SyncDriverFactoryOptions, } from 'src/engine/core-modules/message-queue/interfaces'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; /** * MessageQueue Module factory @@ -16,6 +15,7 @@ import { */ export const messageQueueModuleFactory = async ( environmentService: EnvironmentService, + redisClientService: RedisClientService, ): Promise => { const driverType = environmentService.get('MESSAGE_QUEUE_TYPE'); @@ -37,18 +37,10 @@ export const messageQueueModuleFactory = async ( } satisfies PgBossDriverFactoryOptions; } case MessageQueueDriverType.BullMQ: { - const connectionString = environmentService.get('REDIS_URL'); - - if (!connectionString) { - throw new Error( - `${MessageQueueDriverType.BullMQ} message queue requires REDIS_URL to be defined, check your .env file`, - ); - } - return { type: MessageQueueDriverType.BullMQ, options: { - connection: new IORedis(connectionString), + connection: redisClientService.getClient(), }, } satisfies BullMQDriverFactoryOptions; } diff --git a/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.module.ts b/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.module.ts new file mode 100644 index 000000000000..8e62a0db7369 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.module.ts @@ -0,0 +1,12 @@ +import { Global, Module } from '@nestjs/common'; + +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; + +@Global() +@Module({ + imports: [EnvironmentModule], + providers: [RedisClientService], + exports: [RedisClientService], +}) +export class RedisClientModule {} diff --git a/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.service.ts b/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.service.ts new file mode 100644 index 000000000000..f721371b88d1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/redis-client/redis-client.service.ts @@ -0,0 +1,33 @@ +import { Injectable, OnModuleDestroy } from '@nestjs/common'; + +import IORedis from 'ioredis'; + +import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; + +@Injectable() +export class RedisClientService implements OnModuleDestroy { + private redisClient: IORedis | null = null; + + constructor(private readonly environmentService: EnvironmentService) {} + + getClient() { + if (!this.redisClient) { + const redisUrl = this.environmentService.get('REDIS_URL'); + + if (!redisUrl) { + throw new Error('REDIS_URL must be defined'); + } + + this.redisClient = new IORedis(redisUrl); + } + + return this.redisClient; + } + + async onModuleDestroy() { + if (this.redisClient) { + await this.redisClient.quit(); + this.redisClient = null; + } + } +} From 0633bc90fa9d577ba641202e26c1ac54e815b3a2 Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 22 Oct 2024 16:31:58 +0200 Subject: [PATCH 3/3] linter --- .../twenty-server/src/engine/core-modules/core-engine.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index 78f4361f218c..48d1819e1318 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -33,6 +33,7 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel import { OpenApiModule } from 'src/engine/core-modules/open-api/open-api.module'; import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-credentials/postgres-credentials.module'; import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; import { serverlessModuleFactory } from 'src/engine/core-modules/serverless/serverless-module.factory'; import { ServerlessModule } from 'src/engine/core-modules/serverless/serverless.module'; import { WorkspaceSSOModule } from 'src/engine/core-modules/sso/sso.module'; @@ -43,7 +44,6 @@ import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-inv import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; -import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; import { AnalyticsModule } from './analytics/analytics.module'; import { ClientConfigModule } from './client-config/client-config.module'; import { FileModule } from './file/file.module';