Skip to content

Commit 3c41687

Browse files
committed
Refactor metadata caching (#7011)
This PR introduces the following changes: - add the metadataVersion to all our metadata cache keys to ease troubleshooting: <img width="1146" alt="image" src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12"> - introduce a cache recompute lock to avoid overloading the database to recompute the cache many time
1 parent 9b46e8c commit 3c41687

32 files changed

+416
-199
lines changed

packages/twenty-server/src/app.module.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphq
2020
import { RestApiModule } from 'src/engine/api/rest/rest-api.module';
2121
import { MessageQueueDriverType } from 'src/engine/core-modules/message-queue/interfaces';
2222
import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module';
23-
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
2423
import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware';
2524
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
25+
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
2626
import { ModulesModule } from 'src/modules/modules.module';
2727

2828
import { CoreEngineModule } from './engine/core-modules/core-engine.module';
@@ -51,7 +51,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module';
5151
// Modules module, contains all business logic modules
5252
ModulesModule,
5353
// Needed for the user workspace middleware
54-
WorkspaceMetadataVersionModule,
54+
WorkspaceCacheStorageModule,
5555
// Api modules
5656
CoreGraphQLApiModule,
5757
MetadataGraphQLApiModule,

packages/twenty-server/src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module.ts

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
import { Module } from '@nestjs/common';
2+
import { TypeOrmModule } from '@nestjs/typeorm';
23

4+
import { DataSeedDemoWorkspaceService } from 'src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service';
35
import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
6+
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
47
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
5-
import { DataSeedDemoWorkspaceService } from 'src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service';
68

79
@Module({
8-
imports: [WorkspaceManagerModule, EnvironmentModule],
10+
imports: [
11+
WorkspaceManagerModule,
12+
EnvironmentModule,
13+
TypeOrmModule.forFeature([Workspace], 'core'),
14+
],
915
providers: [DataSeedDemoWorkspaceService],
1016
exports: [DataSeedDemoWorkspaceService],
1117
})

packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts

+16-4
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
import { Injectable } from '@nestjs/common';
2+
import { InjectRepository } from '@nestjs/typeorm';
3+
4+
import { Repository } from 'typeorm';
25

3-
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
4-
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
56
import {
67
deleteCoreSchema,
78
seedCoreSchema,
89
} from 'src/database/typeorm-seeds/core/demo';
910
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
11+
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
12+
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
13+
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
1014

1115
@Injectable()
1216
export class DataSeedDemoWorkspaceService {
1317
constructor(
1418
private readonly environmentService: EnvironmentService,
1519
private readonly workspaceManagerService: WorkspaceManagerService,
20+
@InjectRepository(Workspace, 'core')
21+
protected readonly workspaceRepository: Repository<Workspace>,
1622
) {}
1723

1824
async seedDemo(): Promise<void> {
@@ -27,8 +33,14 @@ export class DataSeedDemoWorkspaceService {
2733
);
2834
}
2935
for (const workspaceId of demoWorkspaceIds) {
30-
await deleteCoreSchema(rawDataSource, workspaceId);
31-
await this.workspaceManagerService.delete(workspaceId);
36+
const existingWorkspaces = await this.workspaceRepository.findBy({
37+
id: workspaceId,
38+
});
39+
40+
if (existingWorkspaces.length > 0) {
41+
await this.workspaceManagerService.delete(workspaceId);
42+
await deleteCoreSchema(rawDataSource, workspaceId);
43+
}
3244

3345
await seedCoreSchema(rawDataSource, workspaceId);
3446
await this.workspaceManagerService.initDemo(workspaceId);

packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import {
99
Workspace,
1010
WorkspaceActivationStatus,
1111
} from 'src/engine/core-modules/workspace/workspace.entity';
12-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
12+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
1313
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
1414
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
1515
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';

packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-
66
import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory';
77
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
88
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
9-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
9+
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
1010
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
1111

1212
describe('WorkspaceSchemaFactory', () => {
@@ -41,7 +41,7 @@ describe('WorkspaceSchemaFactory', () => {
4141
useValue: {},
4242
},
4343
{
44-
provide: WorkspaceMetadataVersionService,
44+
provide: WorkspaceMetadataCacheService,
4545
useValue: {},
4646
},
4747
],

packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts

+2-9
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,18 @@ import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-
44
import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module';
55
import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
66
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
7-
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
7+
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
88
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
99

1010
import { WorkspaceSchemaFactory } from './workspace-schema.factory';
1111

1212
@Module({
1313
imports: [
14-
// TODO: Seems like it's breaking /metadata query and mutation arguments
15-
// we should investigate this issue
16-
// GraphQLModule.forRootAsync<YogaDriverConfig>({
17-
// driver: YogaDriver,
18-
// imports: [CoreEngineModule, GraphQLConfigModule],
19-
// useClass: GraphQLConfigService,
20-
// }),
2114
MetadataEngineModule,
2215
WorkspaceSchemaBuilderModule,
2316
WorkspaceResolverBuilderModule,
2417
WorkspaceCacheStorageModule,
25-
WorkspaceMetadataVersionModule,
18+
WorkspaceMetadataCacheModule,
2619
],
2720
providers: [WorkspaceSchemaFactory, ScalarsExplorerService],
2821
exports: [WorkspaceSchemaFactory],

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts

+1
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,5 @@ export enum GraphqlQueryRunnerExceptionCode {
1919
RECORD_NOT_FOUND = 'RECORD_NOT_FOUND',
2020
INVALID_ARGS_FIRST = 'INVALID_ARGS_FIRST',
2121
INVALID_ARGS_LAST = 'INVALID_ARGS_LAST',
22+
METADATA_CACHE_VERSION_NOT_FOUND = 'METADATA_CACHE_VERSION_NOT_FOUND',
2223
}

packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts

+31-19
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
44
import { GraphQLSchema, printSchema } from 'graphql';
55
import { gql } from 'graphql-tag';
66

7+
import {
8+
GraphqlQueryRunnerException,
9+
GraphqlQueryRunnerExceptionCode,
10+
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
711
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
812
import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
913
import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
1014
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
1115
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
1216
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
13-
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
14-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
17+
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
1518
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
1619
@Injectable()
1720
export class WorkspaceSchemaFactory {
1821
constructor(
1922
private readonly dataSourceService: DataSourceService,
20-
private readonly objectMetadataService: ObjectMetadataService,
2123
private readonly scalarsExplorerService: ScalarsExplorerService,
2224
private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory,
2325
private readonly workspaceResolverFactory: WorkspaceResolverFactory,
2426
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
25-
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
27+
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
2628
) {}
2729

2830
async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
@@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory {
3537
authContext.workspace.id,
3638
);
3739

38-
// Can'f find any data sources for this workspace
3940
if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) {
4041
return new GraphQLSchema({});
4142
}
4243

43-
// Validate cache version
44-
await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated(
45-
authContext.workspace.id,
46-
);
44+
const currentCacheVersion =
45+
await this.workspaceCacheStorageService.getMetadataVersion(
46+
authContext.workspace.id,
47+
);
48+
49+
if (currentCacheVersion === undefined) {
50+
await this.workspaceMetadataCacheService.recomputeMetadataCache(
51+
authContext.workspace.id,
52+
);
53+
throw new GraphqlQueryRunnerException(
54+
'Metadata cache version not found',
55+
GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
56+
);
57+
}
4758

48-
// Get object metadata from cache
49-
let objectMetadataCollection =
59+
const objectMetadataCollection =
5060
await this.workspaceCacheStorageService.getObjectMetadataCollection(
5161
authContext.workspace.id,
62+
currentCacheVersion,
5263
);
5364

54-
// If object metadata is not cached, get it from the database
5565
if (!objectMetadataCollection) {
56-
objectMetadataCollection =
57-
await this.objectMetadataService.findManyWithinWorkspace(
58-
authContext.workspace.id,
59-
);
60-
61-
await this.workspaceCacheStorageService.setObjectMetadataCollection(
66+
await this.workspaceMetadataCacheService.recomputeMetadataCache(
6267
authContext.workspace.id,
63-
objectMetadataCollection,
68+
);
69+
throw new GraphqlQueryRunnerException(
70+
'Object metadata collection not found',
71+
GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
6472
);
6573
}
6674

6775
// Get typeDefs from cache
6876
let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
6977
authContext.workspace.id,
78+
currentCacheVersion,
7079
);
7180
let usedScalarNames =
7281
await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
7382
authContext.workspace.id,
83+
currentCacheVersion,
7484
);
7585

7686
// If typeDefs are not cached, generate them
@@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory {
8797

8898
await this.workspaceCacheStorageService.setGraphQLTypeDefs(
8999
authContext.workspace.id,
100+
currentCacheVersion,
90101
typeDefs,
91102
);
92103
await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
93104
authContext.workspace.id,
105+
currentCacheVersion,
94106
usedScalarNames,
95107
);
96108
}

packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity';
1616
import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener';
1717
import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver';
1818
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
19-
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
19+
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
2020
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
2121

2222
import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts';
@@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service';
3232
BillingModule,
3333
FileModule,
3434
FileUploadModule,
35-
WorkspaceMetadataVersionModule,
35+
WorkspaceMetadataCacheModule,
3636
NestjsQueryTypeOrmModule.forFeature(
3737
[User, Workspace, UserWorkspace, FeatureFlagEntity],
3838
'core',

packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts

-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
2626
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
2727
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
2828
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
29-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
3029
import { assert } from 'src/utils/assert';
3130
import { streamToBuffer } from 'src/utils/stream-to-buffer';
3231

@@ -39,7 +38,6 @@ import { WorkspaceService } from './services/workspace.service';
3938
export class WorkspaceResolver {
4039
constructor(
4140
private readonly workspaceService: WorkspaceService,
42-
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
4341
private readonly userWorkspaceService: UserWorkspaceService,
4442
private readonly fileUploadService: FileUploadService,
4543
private readonly fileService: FileService,

packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ import { NameTooLongException } from 'src/engine/metadata-modules/utils/exceptio
4141
import { exceedsDatabaseIdentifierMaximumLength } from 'src/engine/metadata-modules/utils/validate-database-identifier-length.utils';
4242
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
4343
import { validateMetadataNameValidityOrThrow as validateFieldNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
44-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
44+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
4545
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
4646
import {
4747
WorkspaceMigrationColumnActionType,

packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import {
3434
import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete';
3535
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
3636
import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
37-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
37+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
3838
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
3939
import {
4040
WorkspaceMigrationColumnActionType,

packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
import { InvalidStringException } from 'src/engine/metadata-modules/utils/exceptions/invalid-string.exception';
2222
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
2323
import { validateMetadataNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
24-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
24+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
2525
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
2626
import {
2727
WorkspaceMigrationColumnActionType,

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
} from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception';
1212
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util';
1313
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
14-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
14+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
1515
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
1616
import {
1717
ReferencedTable,

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import {
3636
mapUdtNameToFieldType,
3737
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
3838
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
39-
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
39+
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
4040
import {
4141
WorkspaceMigrationColumnAction,
4242
WorkspaceMigrationColumnActionType,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import { CustomException } from 'src/utils/custom-exception';
2+
3+
export class WorkspaceMetadataCacheException extends CustomException {
4+
code: WorkspaceMetadataCacheExceptionCode;
5+
constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) {
6+
super(message, code);
7+
}
8+
}
9+
10+
export enum WorkspaceMetadataCacheExceptionCode {
11+
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
12+
}

0 commit comments

Comments
 (0)