5x Fix cache performance issues (#6616)

Calling `getObjectMetadata` from `WorkspaceCacheStorageService` in every query was causing big performance issues. The `objectMetadataCollection` is now part of the `WorkspaceInternalContext` so we only instance it once in the `WorkspaceDatasourceFactory`. Queries are now much faster, for instance for TimelineCalendar, it went from ~450ms to 80ms.
twentyhq · Aug 13, 2024 · 40bbee8 · 40bbee8
1 parent 65a961f
commit 40bbee8
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 42 deletions.
diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
@@ -3,8 +3,8 @@ import { Injectable } from '@nestjs/common';
 import { EntitySchemaRelationOptions } from 'typeorm';
 
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
-import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
 import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util';
+import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
 type EntitySchemaRelationMap = {
@@ -37,11 +37,19 @@ export class EntitySchemaRelationFactory {
         );
       }
 
+      const objectMetadataCollection =
+        await this.workspaceCacheStorageService.getObjectMetadataCollection(
+          workspaceId,
+        );
+
+      if (!objectMetadataCollection) {
+        throw new Error('Object metadata collection not found');
+      }
+
       const relationDetails = await determineRelationDetails(
-        workspaceId,
         fieldMetadata,
         relationMetadata,
-        this.workspaceCacheStorageService,
+        objectMetadataCollection,
       );
 
       entitySchemaRelationMap[fieldMetadata.name] = {

diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
@@ -93,7 +93,7 @@ export class WorkspaceDatasourceFactory {
         const workspaceDataSource = new WorkspaceDataSource(
           {
             workspaceId,
-            workspaceCacheStorage: this.workspaceCacheStorageService,
+            objectMetadataCollection: cachedObjectMetadataCollection,
           },
           {
             url:

diff --git a/...es/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts b/...es/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts
@@ -1,6 +1,6 @@
-import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 
 export interface WorkspaceInternalContext {
   workspaceId: string;
-  workspaceCacheStorage: WorkspaceCacheStorageService;
+  objectMetadataCollection: ObjectMetadataEntity[];
 }
diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts
@@ -18,9 +18,9 @@ import {
   SaveOptions,
   UpdateResult,
 } from 'typeorm';
+import { PickKeysByType } from 'typeorm/common/PickKeysByType';
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
 import { UpsertOptions } from 'typeorm/repository/UpsertOptions';
-import { PickKeysByType } from 'typeorm/common/PickKeysByType';
 
 import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';
 
@@ -235,6 +235,7 @@ export class WorkspaceRepository<
       formattedEntityOrEntities,
       options,
     );
+
     const formattedResult = await this.formatResult(result);
 
     return formattedResult;
@@ -423,6 +424,7 @@ export class WorkspaceRepository<
     entityManager?: EntityManager,
   ): Promise<InsertResult> {
     const manager = entityManager || this.manager;
+
     const formatedEntity = await this.formatData(entity);
     const result = await manager.insert(this.target, formatedEntity);
     const formattedResult = await this.formatResult(result);
@@ -621,22 +623,15 @@ export class WorkspaceRepository<
       throw new Error('Object metadata name is missing');
     }
 
-    const objectMetadata =
-      await this.internalContext.workspaceCacheStorage.getObjectMetadata(
-        this.internalContext.workspaceId,
-        (objectMetadata) => objectMetadata.nameSingular === objectMetadataName,
-      );
+    const objectMetadata = this.internalContext.objectMetadataCollection.find(
+      (objectMetadata) => objectMetadata.nameSingular === objectMetadataName,
+    );
 
     if (!objectMetadata) {
-      const objectMetadataCollection =
-        await this.internalContext.workspaceCacheStorage.getObjectMetadataCollection(
-          this.internalContext.workspaceId,
-        );
-
       throw new Error(
         `Object metadata for object "${objectMetadataName}" is missing ` +
           `in workspace "${this.internalContext.workspaceId}" ` +
-          `with object metadata collection length: ${objectMetadataCollection?.length}`,
+          `with object metadata collection length: ${this.internalContext.objectMetadataCollection.length}`,
       );
     }
 
@@ -806,15 +801,13 @@ export class WorkspaceRepository<
 
       if (relationMetadata) {
         const toObjectMetadata =
-          await this.internalContext.workspaceCacheStorage.getObjectMetadata(
-            relationMetadata.workspaceId,
+          this.internalContext.objectMetadataCollection.find(
             (objectMetadata) =>
               objectMetadata.id === relationMetadata.toObjectMetadataId,
           );
 
         const fromObjectMetadata =
-          await this.internalContext.workspaceCacheStorage.getObjectMetadata(
-            relationMetadata.workspaceId,
+          this.internalContext.objectMetadataCollection.find(
             (objectMetadata) =>
               objectMetadata.id === relationMetadata.fromObjectMetadataId,
           );
@@ -833,6 +826,7 @@ export class WorkspaceRepository<
 
         newData[key] = await this.formatResult(
           value,
+
           relationType === 'one-to-many'
             ? toObjectMetadata
             : fromObjectMetadata,

diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts b/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts
@@ -4,7 +4,6 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
 import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';
-import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
 interface RelationDetails {
   relationType: RelationType;
@@ -14,10 +13,9 @@ interface RelationDetails {
 }
 
 export async function determineRelationDetails(
-  workspaceId: string,
   fieldMetadata: FieldMetadataEntity,
   relationMetadata: RelationMetadataEntity,
-  workspaceCacheStorageService: WorkspaceCacheStorageService,
+  objectMetadataCollection: ObjectMetadataEntity[],
 ): Promise<RelationDetails> {
   const relationType = computeRelationType(fieldMetadata, relationMetadata);
   let fromObjectMetadata: ObjectMetadataEntity | undefined =
@@ -28,8 +26,8 @@ export async function determineRelationDetails(
   // RelationMetadata always store the relation from the perspective of the `from` object, MANY_TO_ONE relations are not stored yet
   if (relationType === 'many-to-one') {
     fromObjectMetadata = fieldMetadata.object;
-    toObjectMetadata = await workspaceCacheStorageService.getObjectMetadata(
-      workspaceId,
+
+    toObjectMetadata = objectMetadataCollection.find(
       (objectMetadata) =>
         objectMetadata.id === relationMetadata.fromObjectMetadataId,
     );

diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts
@@ -63,21 +63,6 @@ export class WorkspaceCacheStorageService {
     );
   }
 
-  async getObjectMetadata(
-    workspaceId: string,
-    predicate: (objectMetadata: ObjectMetadataEntity) => boolean,
-  ): Promise<ObjectMetadataEntity | undefined> {
-    const objectMetadataCollection = await this.workspaceSchemaCache.get<
-      ObjectMetadataEntity[]
-    >(`objectMetadataCollection:${workspaceId}`);
-
-    if (!objectMetadataCollection) {
-      return;
-    }
-
-    return objectMetadataCollection.find(predicate);
-  }
-
   setTypeDefs(workspaceId: string, typeDefs: string): Promise<void> {
     return this.workspaceSchemaCache.set<string>(
       `typeDefs:${workspaceId}`,