Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix webhook issue #6711

Merged
merged 6 commits into from
Aug 21, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/twenty-front/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,4 @@
}
],
"extends": "../../tsconfig.base.json"
}
}
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
import { Logger } from '@nestjs/common';

import { Like } from 'typeorm';

import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';

import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import {
CallWebhookJob,
CallWebhookJobData,
} from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { WebhookWorkspaceEntity } from 'src/modules/webhook/standard-objects/webhook.workspace-entity';

export enum CallWebhookJobsJobOperation {
create = 'create',
Expand All @@ -32,42 +34,38 @@ export class CallWebhookJobsJob {
private readonly logger = new Logger(CallWebhookJobsJob.name);

constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly dataSourceService: DataSourceService,
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly messageQueueService: MessageQueueService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}

@Process(CallWebhookJobsJob.name)
async handle(data: CallWebhookJobsJobData): Promise<void> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
data.workspaceId,
);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
const webhookRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WebhookWorkspaceEntity>(
data.workspaceId,
'webhook',
);

const nameSingular = data.objectMetadataItem.nameSingular;
const operation = data.operation;
const eventType = `${operation}.${nameSingular}`;
const webhooks: { id: string; targetUrl: string }[] =
await workspaceDataSource?.query(
`
SELECT * FROM ${dataSourceMetadata.schema}."webhook"
WHERE operation LIKE '%${eventType}%'
OR operation LIKE '%*.${nameSingular}%'
OR operation LIKE '%${operation}.*%'
OR operation LIKE '%*.*%'
`,
);
const eventName = `${nameSingular}.${operation}`;

const webhooks = await webhookRepository.find({
where: [
{ operation: Like(`%${eventName}%`) },
{ operation: Like(`%*.${operation}%`) },
{ operation: Like(`%${nameSingular}.*%`) },
{ operation: Like('%*.*%') },
],
Comment on lines +54 to +60
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: Consider using an array of strings for operations and use In operator instead of multiple Like conditions for better performance.

});

webhooks.forEach((webhook) => {
this.messageQueueService.add<CallWebhookJobData>(
CallWebhookJob.name,
{
targetUrl: webhook.targetUrl,
eventType,
eventName,
objectMetadata: {
id: data.objectMetadataItem.id,
nameSingular: data.objectMetadataItem.nameSingular,
Expand All @@ -83,7 +81,7 @@ export class CallWebhookJobsJob {

if (webhooks.length) {
this.logger.log(
`CallWebhookJobsJob on eventType '${eventType}' called on webhooks ids [\n"${webhooks
`CallWebhookJobsJob on eventName '${event}' called on webhooks ids [\n"${webhooks
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logic: 'event' is undefined. Should be 'eventName'.

.map((webhook) => webhook.id)
.join('",\n"')}"\n]`,
);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { Logger } from '@nestjs/common';
import { HttpService } from '@nestjs/axios';
import { Logger } from '@nestjs/common';

import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';

export type CallWebhookJobData = {
targetUrl: string;
eventType: string;
eventName: string;
objectMetadata: { id: string; nameSingular: string };
workspaceId: string;
webhookId: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,24 +40,18 @@ export class UserService extends TypeOrmQueryService<User> {
return null;
}

console.time('loadWorkspaceMember repo');
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
user.defaultWorkspaceId,
'workspaceMember',
);

console.timeEnd('loadWorkspaceMember repo');

console.time('loadWorkspaceMember find');
const workspaceMember = await workspaceMemberRepository.findOne({
where: {
userId: user.id,
},
});

console.timeEnd('loadWorkspaceMember find');

return workspaceMember;
}

Expand All @@ -66,21 +60,14 @@ export class UserService extends TypeOrmQueryService<User> {
return [];
}

console.time('loadWorkspaceMembers repo');
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspace.id,
'workspaceMember',
);

console.timeEnd('loadWorkspaceMembers repo');

console.time('loadWorkspaceMembers find');

const workspaceMembers = workspaceMemberRepository.find();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logic: Missing await keyword for asynchronous operation


console.timeEnd('loadWorkspaceMembers find');

return workspaceMembers;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ export class UserResolver {
nullable: true,
})
async workspaceMember(@Parent() user: User): Promise<WorkspaceMember | null> {
console.time('resolver workspaceMember');
const workspaceMember = await this.userService.loadWorkspaceMember(user);

if (workspaceMember && workspaceMember.avatarUrl) {
Expand All @@ -109,7 +108,6 @@ export class UserResolver {

workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`;
}
console.timeEnd('resolver workspaceMember');

// TODO: Fix typing disrepency between Entity and DTO
return workspaceMember as WorkspaceMember | null;
Expand All @@ -119,7 +117,6 @@ export class UserResolver {
nullable: true,
})
async workspaceMembers(@Parent() user: User): Promise<WorkspaceMember[]> {
console.time('resolver workspaceMembers');
const workspaceMembers = await this.userService.loadWorkspaceMembers(
user.defaultWorkspace,
);
Expand All @@ -135,8 +132,6 @@ export class UserResolver {
}
}

console.timeEnd('resolver workspaceMembers');

// TODO: Fix typing disrepency between Entity and DTO
return workspaceMembers as WorkspaceMember[];
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';

import { EntitySchema, Repository } from 'typeorm';
import { v4 } from 'uuid';

import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
Expand All @@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory {
workspaceId: string,
workspaceMetadataVersion: string | null,
): Promise<WorkspaceDataSource> {
const logId = v4();

console.time(`fetch in datasource factory ${logId}`);

const latestWorkspaceMetadataVersion =
await this.workspaceMetadataVersionService.getMetadataVersion(
workspaceId,
);

console.timeEnd(`fetch in datasource factory ${logId}`);

const desiredWorkspaceMetadataVersion =
workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;

Expand All @@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory {
);
}

let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);

if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});

await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);

cachedObjectMetadataCollection = freshObjectMetadataCollection;
}

const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
async () => {
const logId = v4();
let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);

if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});

await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);

cachedObjectMetadataCollection = freshObjectMetadataCollection;
}

console.log('Creating workspace fresh data source...' + logId);
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
Expand All @@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory {
);
}

console.time('create entity schema' + logId);
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
Expand All @@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory {

cachedEntitySchemas = entitySchemas;
}
console.timeEnd('create entity schema' + logId);

console.time('create workspace data source' + logId);
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
Expand All @@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory {
},
);

console.timeEnd('create workspace data source' + logId);

console.time('initialize workspace data source' + logId);
await workspaceDataSource.initialize();
console.timeEnd('initialize workspace data source' + logId);

return workspaceDataSource;
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,13 @@ export class CacheManager<T> {
return this.cache.get(cacheKey)!;
}

// Remove old entries with the same workspaceId
for (const key of this.cache.keys()) {
if (key.startsWith(`${workspaceId}-`)) {
await onDelete?.(this.cache.get(key)!);
this.cache.delete(key);
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: Consider using this.cache.forEach() instead of for...of loop for better performance when iterating over cache entries.


// Create a new value using the factory callback
const value = await factory();

if (!value) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { Injectable, Type } from '@nestjs/common';

import { ObjectLiteral } from 'typeorm';
import { v4 } from 'uuid';

import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
Expand Down Expand Up @@ -37,16 +36,11 @@ export class TwentyORMGlobalManager {
);
}

const logId = v4();

console.time(`createDataSource in orm ${logId}`);
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
null,
);

console.timeEnd(`createDataSource in orm ${logId}`);

const repository = workspaceDataSource.getRepository<T>(objectMetadataName);

return repository;
Expand Down
Loading