Skip to content

Commit

Permalink
Refactor calendar to use new sync statuses and stages (#6141)
Browse files Browse the repository at this point in the history
- Refactor calendar modules and some messaging modules to better
organize them by business rules and decouple them
- Work toward a common architecture for the different calendar providers
by introducing interfaces for the drivers
- Modify cron job to use the new sync statuses and stages
  • Loading branch information
bosiraphael authored Jul 8, 2024
1 parent 1c3ea9b commit f458322
Show file tree
Hide file tree
Showing 69 changed files with 1,297 additions and 881 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inje
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
import {
CalendarEventsImportJob,
CalendarEventsImportJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';
import {
CalendarChannelWorkspaceEntity,
CalendarChannelVisibility,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import {
CalendarEventsImportJobData,
CalendarEventListFetchJob,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';

@Injectable()
export class GoogleAPIsService {
Expand Down Expand Up @@ -167,13 +167,21 @@ export class GoogleAPIsService {
}

if (isCalendarEnabled) {
await this.calendarQueueService.add<CalendarEventsImportJobData>(
CalendarEventsImportJob.name,
{
workspaceId,
const calendarChannels = await this.calendarChannelRepository.find({
where: {
connectedAccountId: newOrExistingConnectedAccountId,
},
);
});

for (const calendarChannel of calendarChannels) {
await this.calendarQueueService.add<CalendarEventsImportJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,
workspaceId,
},
);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export enum CacheStorageNamespace {
Messaging = 'messaging',
Calendar = 'calendar',
WorkspaceSchema = 'workspaceSchema',
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { AutoCompaniesAndContactsCreationJobModule } from 'src/modules/connected
import { MessagingModule } from 'src/modules/messaging/messaging.module';
import { TimelineJobModule } from 'src/modules/timeline/jobs/timeline-job.module';
import { CalendarModule } from 'src/modules/calendar/calendar.module';
import { CalendarEventParticipantModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant.module';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';

@Module({
imports: [
Expand All @@ -37,7 +37,7 @@ import { CalendarEventParticipantModule } from 'src/modules/calendar/calendar-ev
WorkspaceModule,
MessagingModule,
CalendarModule,
CalendarEventParticipantModule,
CalendarEventParticipantManagerModule,
TimelineActivityModule,
StripeModule,
WorkspaceQueryRunnerJobModule,
Expand Down
72 changes: 68 additions & 4 deletions packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,39 @@ import { Injectable, Optional, Type } from '@nestjs/common';

import { ObjectLiteral } from 'typeorm';

import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { ObjectLiteralStorage } from 'src/engine/twenty-orm/storage/object-literal.storage';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity';
import { ActivityWorkspaceEntity } from 'src/modules/activity/standard-objects/activity.workspace-entity';
import { CommentWorkspaceEntity } from 'src/modules/activity/standard-objects/comment.workspace-entity';
import { ApiKeyWorkspaceEntity } from 'src/modules/api-key/standard-objects/api-key.workspace-entity';
import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity';
import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity';
import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
import { MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
import { OpportunityWorkspaceEntity } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity';
import { BehavioralEventWorkspaceEntity } from 'src/modules/timeline/standard-objects/behavioral-event.workspace-entity';
import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity';
import { ViewFieldWorkspaceEntity } from 'src/modules/view/standard-objects/view-field.workspace-entity';
import { ViewFilterWorkspaceEntity } from 'src/modules/view/standard-objects/view-filter.workspace-entity';
import { ViewSortWorkspaceEntity } from 'src/modules/view/standard-objects/view-sort.workspace-entity';
import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
import { WebhookWorkspaceEntity } from 'src/modules/webhook/standard-objects/webhook.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';

@Injectable()
export class TwentyORMManager {
Expand All @@ -33,7 +61,43 @@ export class TwentyORMManager {
workspaceId: string,
entityClass: Type<T>,
): Promise<WorkspaceRepository<T>> {
const entities = ObjectLiteralStorage.getAllEntitySchemas();
// TODO: This is a temporary solution to get all workspace entities
const workspaceEntities = [
ActivityTargetWorkspaceEntity,
ActivityWorkspaceEntity,
ApiKeyWorkspaceEntity,
AttachmentWorkspaceEntity,
BlocklistWorkspaceEntity,
BehavioralEventWorkspaceEntity,
CalendarChannelEventAssociationWorkspaceEntity,
CalendarChannelWorkspaceEntity,
CalendarEventParticipantWorkspaceEntity,
CalendarEventWorkspaceEntity,
CommentWorkspaceEntity,
CompanyWorkspaceEntity,
ConnectedAccountWorkspaceEntity,
FavoriteWorkspaceEntity,
AuditLogWorkspaceEntity,
MessageChannelMessageAssociationWorkspaceEntity,
MessageChannelWorkspaceEntity,
MessageParticipantWorkspaceEntity,
MessageThreadWorkspaceEntity,
MessageWorkspaceEntity,
OpportunityWorkspaceEntity,
PersonWorkspaceEntity,
TimelineActivityWorkspaceEntity,
ViewFieldWorkspaceEntity,
ViewFilterWorkspaceEntity,
ViewSortWorkspaceEntity,
ViewWorkspaceEntity,
WebhookWorkspaceEntity,
WorkspaceMemberWorkspaceEntity,
];

const entities = workspaceEntities.map((workspaceEntity) =>
this.entitySchemaFactory.create(workspaceEntity as any),
);

const workspaceDataSource = await this.workspaceDataSourceFactory.create(
entities,
workspaceId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/work
import { PersonRepository } from 'src/modules/person/repositories/person.repository';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';

// TODO: Move inside person module and workspace-member module

@Injectable()
export class AddPersonIdAndWorkspaceMemberIdService {
constructor(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util';
import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant-manager/utils/is-email-blocklisted.util';

describe('isEmailBlocklisted', () => {
it('should return true if email is blocklisted', () => {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// TODO: Move inside blocklist module

export const isEmailBlocklisted = (
channelHandle: string,
email: string | null | undefined,
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,62 +1,61 @@
import { Logger, Scope } from '@nestjs/common';
import { Scope } from '@nestjs/common';

import { Any } from 'typeorm';

import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';

export type BlocklistReimportCalendarEventsJobData = {
workspaceId: string;
workspaceMemberId: string;
handle: string;
};

@Processor({
queueName: MessageQueue.calendarQueue,
scope: Scope.REQUEST,
})
export class BlocklistReimportCalendarEventsJob {
private readonly logger = new Logger(BlocklistReimportCalendarEventsJob.name);

constructor(
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly googleCalendarSyncService: CalendarEventsImportService,
@InjectWorkspaceRepository(CalendarChannelWorkspaceEntity)
private readonly calendarChannelRepository: WorkspaceRepository<CalendarChannelWorkspaceEntity>,
) {}

@Process(BlocklistReimportCalendarEventsJob.name)
async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
const { workspaceId, workspaceMemberId, handle } = data;

this.logger.log(
`Reimporting calendar events from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const { workspaceId, workspaceMemberId } = data;

const connectedAccount =
const connectedAccounts =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);

if (!connectedAccount || connectedAccount.length === 0) {
this.logger.error(
`No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`,
);

if (!connectedAccounts || connectedAccounts.length === 0) {
return;
}

await this.googleCalendarSyncService.processCalendarEventsImport(
workspaceId,
connectedAccount[0].id,
handle,
);

this.logger.log(
`Reimporting calendar events from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`,
await this.calendarChannelRepository.update(
{
connectedAccountId: Any(
connectedAccounts.map((connectedAccount) => connectedAccount.id),
),
},
{
syncStage:
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
},
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ export class CalendarBlocklistListener {
{
workspaceId: payload.workspaceId,
workspaceMemberId: payload.properties.before.workspaceMember.id,
handle: payload.properties.before.handle,
},
);
}
Expand All @@ -68,7 +67,6 @@ export class CalendarBlocklistListener {
{
workspaceId: payload.workspaceId,
workspaceMemberId: payload.properties.after.workspaceMember.id,
handle: payload.properties.before.handle,
},
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,16 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-events-import.cron.command';
import { CalendarEventsImportCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job';
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job';
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { CalendarEventsImportJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarEventParticipantModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant.module';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
Expand All @@ -38,20 +42,25 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
PersonWorkspaceEntity,
WorkspaceMemberWorkspaceEntity,
]),
CalendarEventParticipantModule,
CalendarEventParticipantManagerModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
TypeOrmModule.forFeature([DataSourceEntity], 'metadata'),
WorkspaceDataSourceModule,
CalendarEventCleanerModule,
GoogleCalendarDriverModule,
BillingModule,
GoogleAPIRefreshAccessTokenModule,
CalendarCommonModule,
CalendarEventParticipantManagerModule,
],
providers: [
CalendarChannelSyncStatusService,
CalendarEventsImportService,
CalendarEventsImportCronJob,
CalendarEventsImportCronCommand,
CalendarEventsImportJob,
CalendarGetCalendarEventsService,
CalendarSaveEventsService,
CalendarEventListFetchCronJob,
CalendarEventListFetchCronCommand,
CalendarEventListFetchJob,
],
exports: [CalendarEventsImportService],
})
Expand Down
Loading

0 comments on commit f458322

Please sign in to comment.