From 12a657ce297891b3d964b8ac223ffaae5a8a0791 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Mon, 19 Aug 2024 19:19:01 +0200 Subject: [PATCH 01/19] Patch messaging import crons not running (#6688) In 0.23.1, we have introduced a regression by migrating to TwentyORM ; messageChannels were not considered as syncable anymore --- .../crons/jobs/messaging-message-list-fetch.cron.job.ts | 4 +--- .../messaging-message-channel-sync-status-monitoring.cron.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts index 0eb06acc5ec1..3f7334cf7c19 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts @@ -46,9 +46,7 @@ export class MessagingMessageListFetchCronJob { 'messageChannel', ); - const messageChannels = await messageChannelRepository.find({ - select: ['id'], - }); + const messageChannels = await messageChannelRepository.find(); for (const messageChannel of messageChannels) { if ( diff --git a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts index 9d403fa09941..e3a3b5e06c63 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts @@ -46,8 +46,8 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { for (const activeWorkspace of activeWorkspaces) { const messageChannelRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( - 'messageChannel', activeWorkspace.id, + 'messageChannel', ); const messageChannels = await messageChannelRepository.find({ select: ['id', 'syncStatus', 'connectedAccountId'], From be20a690b30243cc80e9b94799c0a00d8cfb9fac Mon Sep 17 00:00:00 2001 From: Syed Md Mihan Chistie <42113027+Mihan786Chistie@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:35:13 +0530 Subject: [PATCH 02/19] added typechecking for all ts files (#6466) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #6436 Changes made: - Added typecheck step before twenty-ui build to check stories TS errors - Added a tsconfig.dev.json to add stories and tests to typecheking when in dev mode - Added tsconfig.dev.json to storybook dev command of twenty-ui to typecheck stories while developing - Fixed twenty-ui stories that were broken - Added a serve command to serve front build - Fixed unit test from another PR --------- Co-authored-by: Félix Malfait Co-authored-by: Lucas Bordeau --- nx.json | 2 +- packages/twenty-front/project.json | 6 + .../components/__stories__/Status.stories.tsx | 1 + .../src/utils/__tests__/title-utils.test.ts | 4 - ...{tsconfig.app.json => tsconfig.build.json} | 0 packages/twenty-front/tsconfig.dev.json | 13 ++ packages/twenty-front/tsconfig.json | 7 +- packages/twenty-front/vite.config.ts | 6 +- packages/twenty-ui/.storybook/main.ts | 15 +++ .../components/__stories__/Banner.stories.tsx | 5 +- .../twenty-ui/src/utilities/config/index.ts | 4 +- ...{tsconfig.lib.json => tsconfig.build.json} | 0 packages/twenty-ui/tsconfig.dev.json | 19 +++ packages/twenty-ui/tsconfig.json | 5 +- packages/twenty-ui/vite.config.ts | 114 ++++++++++-------- 15 files changed, 135 insertions(+), 66 deletions(-) rename packages/twenty-front/{tsconfig.app.json => tsconfig.build.json} (100%) create mode 100644 packages/twenty-front/tsconfig.dev.json rename packages/twenty-ui/{tsconfig.lib.json => tsconfig.build.json} (100%) create mode 100644 packages/twenty-ui/tsconfig.dev.json diff --git a/nx.json b/nx.json index 35b6e7501700..4ec242fd5172 100644 --- a/nx.json +++ b/nx.json @@ -32,7 +32,7 @@ }, "start": { "cache": true, - "dependsOn": ["^build"] + "dependsOn": ["^typecheck","^build"] }, "lint": { "executor": "@nx/eslint:lint", diff --git a/packages/twenty-front/project.json b/packages/twenty-front/project.json index 636fd29af6a6..4dcb7f444a00 100644 --- a/packages/twenty-front/project.json +++ b/packages/twenty-front/project.json @@ -10,6 +10,12 @@ "outputPath": "{projectRoot}/build" } }, + "serve": { + "executor": "nx:run-commands", + "options": { + "command": "npx serve -s {projectRoot}/build" + } + }, "start": { "executor": "@nx/vite:dev-server", "options": { diff --git a/packages/twenty-front/src/modules/ui/display/status/components/__stories__/Status.stories.tsx b/packages/twenty-front/src/modules/ui/display/status/components/__stories__/Status.stories.tsx index 7db5637d0497..cd480d45aae2 100644 --- a/packages/twenty-front/src/modules/ui/display/status/components/__stories__/Status.stories.tsx +++ b/packages/twenty-front/src/modules/ui/display/status/components/__stories__/Status.stories.tsx @@ -15,6 +15,7 @@ const meta: Meta = { component: Status, args: { text: 'Urgent', + weight: 'medium', }, }; diff --git a/packages/twenty-front/src/utils/__tests__/title-utils.test.ts b/packages/twenty-front/src/utils/__tests__/title-utils.test.ts index 6fee91bf0a74..d67c5ecb0c47 100644 --- a/packages/twenty-front/src/utils/__tests__/title-utils.test.ts +++ b/packages/twenty-front/src/utils/__tests__/title-utils.test.ts @@ -9,10 +9,6 @@ describe('title-utils', () => { expect(getPageTitleFromPath('/invite/:workspaceInviteHash')).toBe('Invite'); expect(getPageTitleFromPath('/create/workspace')).toBe('Create Workspace'); expect(getPageTitleFromPath('/create/profile')).toBe('Create Profile'); - expect(getPageTitleFromPath('/tasks')).toBe('Tasks'); - expect(getPageTitleFromPath('/objects/opportunities')).toBe( - 'Opportunities', - ); expect(getPageTitleFromPath('/settings/objects/opportunities')).toBe( SettingsPageTitles.Objects, ); diff --git a/packages/twenty-front/tsconfig.app.json b/packages/twenty-front/tsconfig.build.json similarity index 100% rename from packages/twenty-front/tsconfig.app.json rename to packages/twenty-front/tsconfig.build.json diff --git a/packages/twenty-front/tsconfig.dev.json b/packages/twenty-front/tsconfig.dev.json new file mode 100644 index 000000000000..f90dcae862bb --- /dev/null +++ b/packages/twenty-front/tsconfig.dev.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": [ + "src/**/*.js", + "src/**/*.jsx", + "src/**/*.d.ts", + "src/**/*.ts", + "src/**/*.tsx" + ] +} diff --git a/packages/twenty-front/tsconfig.json b/packages/twenty-front/tsconfig.json index ac1d6e2f997e..f6c268d9c2fd 100644 --- a/packages/twenty-front/tsconfig.json +++ b/packages/twenty-front/tsconfig.json @@ -32,7 +32,10 @@ "include": [], "references": [ { - "path": "./tsconfig.app.json" + "path": "./tsconfig.dev.json" + }, + { + "path": "./tsconfig.build.json" }, { "path": "./tsconfig.spec.json" @@ -42,4 +45,4 @@ } ], "extends": "../../tsconfig.base.json" -} +} \ No newline at end of file diff --git a/packages/twenty-front/vite.config.ts b/packages/twenty-front/vite.config.ts index 924f9c9d7ee0..682b6466ab40 100644 --- a/packages/twenty-front/vite.config.ts +++ b/packages/twenty-front/vite.config.ts @@ -19,9 +19,13 @@ export default defineConfig(({ command, mode }) => { const isBuildCommand = command === 'build'; + const tsConfigPath = isBuildCommand + ? path.resolve(__dirname, './tsconfig.build.json') + : path.resolve(__dirname, './tsconfig.dev.json'); + const checkers: Checkers = { typescript: { - tsconfigPath: path.resolve(__dirname, './tsconfig.app.json'), + tsconfigPath: tsConfigPath, }, overlay: false, }; diff --git a/packages/twenty-ui/.storybook/main.ts b/packages/twenty-ui/.storybook/main.ts index 270259adfdd7..f3f46370d125 100644 --- a/packages/twenty-ui/.storybook/main.ts +++ b/packages/twenty-ui/.storybook/main.ts @@ -1,4 +1,6 @@ import { StorybookConfig } from '@storybook/react-vite'; +import * as path from 'path'; +import checker from 'vite-plugin-checker'; const config: StorybookConfig = { stories: ['../src/**/*.stories.@(js|jsx|ts|tsx|mdx)'], @@ -16,6 +18,19 @@ const config: StorybookConfig = { name: '@storybook/react-vite', options: {}, }, + viteFinal: (config) => { + return { + ...config, + plugins: [ + ...(config.plugins ?? []), + checker({ + typescript: { + tsconfigPath: path.resolve(__dirname, '../tsconfig.dev.json'), + }, + }), + ], + }; + }, }; export default config; diff --git a/packages/twenty-ui/src/display/banner/components/__stories__/Banner.stories.tsx b/packages/twenty-ui/src/display/banner/components/__stories__/Banner.stories.tsx index 425209802564..73e6d6a316e8 100644 --- a/packages/twenty-ui/src/display/banner/components/__stories__/Banner.stories.tsx +++ b/packages/twenty-ui/src/display/banner/components/__stories__/Banner.stories.tsx @@ -13,10 +13,7 @@ const meta: Meta = { Sync lost with mailbox hello@twenty.com. Please reconnect for updates: ), - argTypes: { - as: { control: false }, - theme: { control: false }, - }, + argTypes: {}, }; export default meta; diff --git a/packages/twenty-ui/src/utilities/config/index.ts b/packages/twenty-ui/src/utilities/config/index.ts index 4525e814d61b..5038c501089a 100644 --- a/packages/twenty-ui/src/utilities/config/index.ts +++ b/packages/twenty-ui/src/utilities/config/index.ts @@ -25,6 +25,4 @@ const getDefaultUrl = () => { }; export const REACT_APP_SERVER_BASE_URL = - window._env_?.REACT_APP_SERVER_BASE_URL || - process.env.REACT_APP_SERVER_BASE_URL || - getDefaultUrl(); + window._env_?.REACT_APP_SERVER_BASE_URL || getDefaultUrl(); diff --git a/packages/twenty-ui/tsconfig.lib.json b/packages/twenty-ui/tsconfig.build.json similarity index 100% rename from packages/twenty-ui/tsconfig.lib.json rename to packages/twenty-ui/tsconfig.build.json diff --git a/packages/twenty-ui/tsconfig.dev.json b/packages/twenty-ui/tsconfig.dev.json new file mode 100644 index 000000000000..7cf734bdd02d --- /dev/null +++ b/packages/twenty-ui/tsconfig.dev.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "vite.config.ts", + "jest.config.ts", + "setupTests.ts", + "src/**/*.d.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts", + "src/**/*.stories.tsx", + "src/**/*.tsx", + "src/**/*.ts", + "vite.config.ts" + ] +} diff --git a/packages/twenty-ui/tsconfig.json b/packages/twenty-ui/tsconfig.json index c7c587845c85..cb70a38df1ed 100644 --- a/packages/twenty-ui/tsconfig.json +++ b/packages/twenty-ui/tsconfig.json @@ -17,7 +17,10 @@ "include": [], "references": [ { - "path": "./tsconfig.lib.json" + "path": "./tsconfig.build.json" + }, + { + "path": "./tsconfig.dev.json" }, { "path": "./tsconfig.spec.json" diff --git a/packages/twenty-ui/vite.config.ts b/packages/twenty-ui/vite.config.ts index 3692ff065325..03fc2e351fbd 100644 --- a/packages/twenty-ui/vite.config.ts +++ b/packages/twenty-ui/vite.config.ts @@ -4,64 +4,78 @@ import wyw from '@wyw-in-js/vite'; import * as path from 'path'; import { defineConfig } from 'vite'; import checker from 'vite-plugin-checker'; -import dts from 'vite-plugin-dts'; +import dts, { PluginOptions } from 'vite-plugin-dts'; import svgr from 'vite-plugin-svgr'; import tsconfigPaths from 'vite-tsconfig-paths'; +import { UserPluginConfig } from 'vite-plugin-checker/dist/esm/types'; + // eslint-disable-next-line @nx/enforce-module-boundaries, import/no-relative-packages import packageJson from '../../package.json'; -export default defineConfig({ - root: __dirname, - cacheDir: '../../node_modules/.vite/packages/twenty-ui', +export default defineConfig(({ command }) => { + const isBuildCommand = command === 'build'; - plugins: [ - react({ jsxImportSource: '@emotion/react' }), - tsconfigPaths(), - svgr(), - dts({ - entryRoot: 'src', - tsconfigPath: path.join(__dirname, 'tsconfig.lib.json'), - }), - checker({ - typescript: { - tsconfigPath: path.join(__dirname, 'tsconfig.lib.json'), - }, - }), - wyw({ - include: [ - '**/OverflowingTextWithTooltip.tsx', - '**/Chip.tsx', - '**/Tag.tsx', - '**/Avatar.tsx', - '**/AvatarChip.tsx', - ], - babelOptions: { - presets: ['@babel/preset-typescript', '@babel/preset-react'], - }, - }), - ], + const tsConfigPath = isBuildCommand + ? path.resolve(__dirname, './tsconfig.build.json') + : path.resolve(__dirname, './tsconfig.dev.json'); - // Configuration for building your library. - // See: https://vitejs.dev/guide/build.html#library-mode - build: { - outDir: './dist', - reportCompressedSize: true, - commonjsOptions: { - transformMixedEsModules: true, + const checkersConfig: UserPluginConfig = { + typescript: { + tsconfigPath: tsConfigPath, }, - lib: { - // Could also be a dictionary or array of multiple entry points. - entry: 'src/index.ts', - name: 'twenty-ui', - fileName: 'index', - // Change this to the formats you want to support. - // Don't forget to update your package.json as well. - formats: ['es', 'cjs'], - }, - rollupOptions: { - // External packages that should not be bundled into your library. - external: Object.keys(packageJson.dependencies || {}), + }; + + const dtsConfig: PluginOptions = { + entryRoot: 'src', + tsconfigPath: tsConfigPath, + }; + + return { + root: __dirname, + cacheDir: '../../node_modules/.vite/packages/twenty-ui', + + plugins: [ + react({ jsxImportSource: '@emotion/react' }), + tsconfigPaths(), + svgr(), + dts(dtsConfig), + checker(checkersConfig), + wyw({ + include: [ + '**/OverflowingTextWithTooltip.tsx', + '**/Chip.tsx', + '**/Tag.tsx', + '**/Avatar.tsx', + '**/AvatarChip.tsx', + ], + babelOptions: { + presets: ['@babel/preset-typescript', '@babel/preset-react'], + }, + }), + ], + + // Configuration for building your library. + // See: https://vitejs.dev/guide/build.html#library-mode + build: { + outDir: './dist', + reportCompressedSize: true, + commonjsOptions: { + transformMixedEsModules: true, + }, + lib: { + // Could also be a dictionary or array of multiple entry points. + entry: 'src/index.ts', + name: 'twenty-ui', + fileName: 'index', + // Change this to the formats you want to support. + // Don't forget to update your package.json as well. + formats: ['es', 'cjs'], + }, + rollupOptions: { + // External packages that should not be bundled into your library. + external: Object.keys(packageJson.dependencies || {}), + }, }, - }, + }; }); From 3ae89d15de5c95f13b2c87388de97105bcdba291 Mon Sep 17 00:00:00 2001 From: Lucas Bordeau Date: Tue, 20 Aug 2024 16:45:00 +0200 Subject: [PATCH 03/19] Fix post merge (#6699) Fixed settingsPreviewRecordId state naming. --- .../components/MultiRecordSelect.tsx | 1 + .../components/SettingsDataModelFieldPreview.tsx | 4 ++-- .../SettingsDataModelSetFieldValueEffect.tsx | 6 +++--- .../SettingsDataModelSetRecordEffect.tsx | 16 +++++++++------- .../preview/states/previewRecordIdState.ts | 6 ------ .../states/settingsPreviewRecordIdState.ts | 6 ++++++ 6 files changed, 21 insertions(+), 18 deletions(-) delete mode 100644 packages/twenty-front/src/modules/settings/data-model/fields/preview/states/previewRecordIdState.ts create mode 100644 packages/twenty-front/src/modules/settings/data-model/fields/preview/states/settingsPreviewRecordIdState.ts diff --git a/packages/twenty-front/src/modules/object-record/relation-picker/components/MultiRecordSelect.tsx b/packages/twenty-front/src/modules/object-record/relation-picker/components/MultiRecordSelect.tsx index 0730c05059e3..87c0532208fe 100644 --- a/packages/twenty-front/src/modules/object-record/relation-picker/components/MultiRecordSelect.tsx +++ b/packages/twenty-front/src/modules/object-record/relation-picker/components/MultiRecordSelect.tsx @@ -27,6 +27,7 @@ export const StyledSelectableItem = styled(SelectableItem)` height: 100%; width: 100%; `; + export const MultiRecordSelect = ({ onChange, onSubmit, diff --git a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelFieldPreview.tsx b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelFieldPreview.tsx index 31b78cd0d4eb..a7b69af6a905 100644 --- a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelFieldPreview.tsx +++ b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelFieldPreview.tsx @@ -10,7 +10,7 @@ import { FieldContext } from '@/object-record/record-field/contexts/FieldContext import { BooleanFieldInput } from '@/object-record/record-field/meta-types/input/components/BooleanFieldInput'; import { RatingFieldInput } from '@/object-record/record-field/meta-types/input/components/RatingFieldInput'; import { SettingsDataModelSetFieldValueEffect } from '@/settings/data-model/fields/preview/components/SettingsDataModelSetFieldValueEffect'; -import { SettingsDataModelSetRecordEffect } from '@/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect'; +import { SettingsDataModelSetPreviewRecordEffect } from '@/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect'; import { useFieldPreviewValue } from '@/settings/data-model/fields/preview/hooks/useFieldPreviewValue'; import { usePreviewRecord } from '@/settings/data-model/fields/preview/hooks/usePreviewRecord'; import { FieldMetadataType } from '~/generated-metadata/graphql'; @@ -96,7 +96,7 @@ export const SettingsDataModelFieldPreview = ({ return ( <> {previewRecord ? ( - diff --git a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetFieldValueEffect.tsx b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetFieldValueEffect.tsx index 050ecbd68c5e..110591d3e119 100644 --- a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetFieldValueEffect.tsx +++ b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetFieldValueEffect.tsx @@ -1,7 +1,7 @@ import { useSetRecordFieldValue } from '@/object-record/record-store/contexts/RecordFieldValueSelectorContext'; import { recordStoreFamilyState } from '@/object-record/record-store/states/recordStoreFamilyState'; import { recordStoreFamilySelector } from '@/object-record/record-store/states/selectors/recordStoreFamilySelector'; -import { previewRecordIdState } from '@/settings/data-model/fields/preview/states/previewRecordIdState'; +import { settingsPreviewRecordIdState } from '@/settings/data-model/fields/preview/states/settingsPreviewRecordIdState'; import { useEffect } from 'react'; import { useRecoilValue, useSetRecoilState } from 'recoil'; import { isDefined } from '~/utils/isDefined'; @@ -17,10 +17,10 @@ export const SettingsDataModelSetFieldValueEffect = ({ fieldName, value, }: SettingsDataModelSetFieldValueEffectProps) => { - const previewRecordId = useRecoilValue(previewRecordIdState); + const settingsPreviewRecordId = useRecoilValue(settingsPreviewRecordIdState); const upsertedPreviewRecord = useRecoilValue( - recordStoreFamilyState(previewRecordId ?? ''), + recordStoreFamilyState(settingsPreviewRecordId ?? ''), ); const setFieldValue = useSetRecoilState( diff --git a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect.tsx b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect.tsx index e5ceaf22f831..331f06e19cf3 100644 --- a/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect.tsx +++ b/packages/twenty-front/src/modules/settings/data-model/fields/preview/components/SettingsDataModelSetRecordEffect.tsx @@ -1,34 +1,36 @@ import { useSetRecordFieldValue } from '@/object-record/record-store/contexts/RecordFieldValueSelectorContext'; import { useUpsertRecordsInStore } from '@/object-record/record-store/hooks/useUpsertRecordsInStore'; import { ObjectRecord } from '@/object-record/types/ObjectRecord'; -import { previewRecordIdState } from '@/settings/data-model/fields/preview/states/previewRecordIdState'; +import { settingsPreviewRecordIdState } from '@/settings/data-model/fields/preview/states/settingsPreviewRecordIdState'; import { useEffect } from 'react'; import { useSetRecoilState } from 'recoil'; -type SettingsDataModelSetRecordEffectProps = { +type SettingsDataModelSetPreviewRecordEffectProps = { record: ObjectRecord; fieldName: string; }; -export const SettingsDataModelSetRecordEffect = ({ +export const SettingsDataModelSetPreviewRecordEffect = ({ record, fieldName, -}: SettingsDataModelSetRecordEffectProps) => { +}: SettingsDataModelSetPreviewRecordEffectProps) => { const { upsertRecords: upsertRecordsInStore } = useUpsertRecordsInStore(); const setRecordFieldValue = useSetRecordFieldValue(); - const setPreviewRecordId = useSetRecoilState(previewRecordIdState); + const setSettingsPreviewRecordId = useSetRecoilState( + settingsPreviewRecordIdState, + ); useEffect(() => { upsertRecordsInStore([record]); setRecordFieldValue(record.id, fieldName, record[fieldName]); - setPreviewRecordId(record.id); + setSettingsPreviewRecordId(record.id); }, [ record, upsertRecordsInStore, setRecordFieldValue, fieldName, - setPreviewRecordId, + setSettingsPreviewRecordId, ]); return null; diff --git a/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/previewRecordIdState.ts b/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/previewRecordIdState.ts deleted file mode 100644 index 3d37a853a67a..000000000000 --- a/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/previewRecordIdState.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { createState } from 'twenty-ui'; - -export const previewRecordIdState = createState({ - key: 'previewRecordId', - defaultValue: null, -}); diff --git a/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/settingsPreviewRecordIdState.ts b/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/settingsPreviewRecordIdState.ts new file mode 100644 index 000000000000..68a5bbff9608 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/data-model/fields/preview/states/settingsPreviewRecordIdState.ts @@ -0,0 +1,6 @@ +import { createState } from 'twenty-ui'; + +export const settingsPreviewRecordIdState = createState({ + key: 'settingsPreviewRecordIdState', + defaultValue: null, +}); From 17a1760afd1fdc46765d191f8a851b7d76a07d3e Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Tue, 20 Aug 2024 19:42:02 +0200 Subject: [PATCH 04/19] Improve performance twenty orm (#6691) ## Context As we grow, the messaging scripts are experiencing performance issues forcing us to temporarily disable them on the cloud. While investigating the performance, I have noticed that generating the entity schema (for twentyORM) in the repository is taking ~500ms locally on my Mac M2 so likely more on pods. Caching the entitySchema then! I'm also clarifying naming around schemaVersion and cacheVersions ==> both are renamed workspaceMetadataVersion and migrated to the workspace table (the workspaceCacheVersion table is dropped). --- .../src/generated/graphql.tsx | 2 +- .../src/generated-metadata/graphql.ts | 5 +- .../twenty-front/src/generated/graphql.tsx | 14 +- .../modules/apollo/hooks/useApolloFactory.ts | 8 +- .../auth/states/currentWorkspaceState.ts | 2 +- .../graphql/fragments/userQueryFragment.ts | 2 +- .../src/testing/mock-data/users.ts | 4 +- packages/twenty-server/@types/express.d.ts | 2 +- ...hql-yoga-nestjs-npm-2.1.0-cb509e6047.patch | 12 +- packages/twenty-server/src/app.module.ts | 20 +-- .../data-seed-dev-workspace.command.ts | 10 +- .../commands/database-command.module.ts | 4 +- ...essage-channel-sync-status-enum.command.ts | 8 +- ...user-vars-accounts-to-reconnect.command.ts | 8 +- ...set-workspace-activation-status.command.ts | 8 +- .../0-23/0-23-update-activities.command.ts | 8 +- .../0-23/0-23-upgrade-version.module.ts | 4 +- .../upgrade-version/upgrade-version.module.ts | 4 +- ...MetadataVersionAndDatasourceOnWorkspace.ts | 31 +++++ .../1724173061204-deprecateCacheVersion.ts | 15 ++ .../__tests__/workspace.factory.spec.ts | 9 +- .../api/graphql/core-graphql-api.module.ts | 6 +- .../hooks/use-cached-metadata.ts | 5 +- .../graphql/metadata-graphql-api.module.ts | 16 +-- .../api/graphql/metadata.module-factory.ts | 10 +- .../api/graphql/workspace-schema.factory.ts | 22 +-- .../google-apis-auth.controller.ts | 18 +-- .../auth/services/google-apis.service.ts | 16 ++- .../auth/strategies/google.auth.strategy.ts | 6 +- .../strategies/microsoft.auth.strategy.ts | 2 +- .../auth/types/google-api-request.type.ts | 2 +- .../file/controllers/file.controller.ts | 4 + .../hooks/use-graphql-error-handler.hook.ts | 8 +- .../engine/core-modules/user/user.resolver.ts | 9 +- .../workspace/workspace.entity.ts | 12 ++ .../workspace/workspace.module.ts | 4 +- .../workspace/workspace.resolver.ts | 25 ++-- .../types/cache-storage-namespace.enum.ts | 6 +- .../field-metadata/field-metadata.module.ts | 6 +- .../field-metadata/field-metadata.service.ts | 16 ++- .../metadata-engine.module.ts | 6 +- .../object-metadata/object-metadata.module.ts | 4 +- .../object-metadata.service.ts | 18 ++- .../relation-metadata.module.ts | 4 +- .../relation-metadata.service.ts | 10 +- .../foreign-table/foreign-table.module.ts | 4 +- .../foreign-table/foreign-table.service.ts | 8 +- .../remote-table/remote-table.module.ts | 4 +- .../remote-table/remote-table.service.ts | 60 ++++---- .../workspace-cache-version.entity.ts | 25 ---- .../workspace-cache-version.module.ts | 14 -- .../workspace-cache-version.service.ts | 42 ------ .../workspace-metadata-version.module.ts | 16 +++ .../workspace-metadata-version.service.ts | 79 +++++++++++ ...l-hydrate-request-from-token.middleware.ts | 13 +- .../load-service-with-workspace.context.ts | 44 ------ .../scoped-workspace-context.factory.ts | 8 +- .../factories/workspace-datasource.factory.ts | 77 +++++++--- .../twenty-orm/twenty-orm-core.module.ts | 13 +- .../engine/twenty-orm/twenty-orm.manager.ts | 11 +- .../workspace-cache-storage.module.ts | 2 - .../workspace-cache-storage.service.ts | 131 +++++++++++------- .../commands/delete-workspaces.command.ts | 22 ++- .../workspace-migration-runner.module.ts | 7 +- .../workspace-sync-metadata.module.ts | 4 +- .../workspace-sync-metadata.service.ts | 6 +- .../calendar-event-list-fetch.cron.job.ts | 4 + .../jobs/calendar-event-list-fetch.job.ts | 3 + .../calendar-channel-sync-status.service.ts | 2 +- .../messaging-channel-sync-status.service.ts | 2 +- .../messaging-message-list-fetch.cron.job.ts | 4 + .../messaging-messages-import.cron.job.ts | 3 + ...-single-message-to-cache-for-import.job.ts | 2 +- .../jobs/messaging-clean-cache.ts | 10 +- .../jobs/messaging-message-list-fetch.job.ts | 4 + .../jobs/messaging-messages-import.job.ts | 4 + ...ssaging-full-message-list-fetch.service.ts | 2 +- .../messaging-messages-import.service.ts | 2 +- ...ging-partial-message-list-fetch.service.ts | 2 +- ...age-channel-sync-status-monitoring.cron.ts | 4 + 80 files changed, 584 insertions(+), 469 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts delete mode 100644 packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts diff --git a/packages/twenty-chrome-extension/src/generated/graphql.tsx b/packages/twenty-chrome-extension/src/generated/graphql.tsx index b63bc43e61dd..27d954014743 100644 --- a/packages/twenty-chrome-extension/src/generated/graphql.tsx +++ b/packages/twenty-chrome-extension/src/generated/graphql.tsx @@ -7455,7 +7455,7 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + metadataVersion?: Maybe; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index f5e61e07afd5..ca9941fc60c4 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -1380,7 +1380,9 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']['output']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + currentMetadataVersion: Scalars['Float']['output']; + databaseSchema: Scalars['String']['output']; + databaseUrl: Scalars['String']['output']; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; @@ -1388,6 +1390,7 @@ export type Workspace = { id: Scalars['UUID']['output']; inviteHash?: Maybe; logo?: Maybe; + metadataVersion: Scalars['Float']['output']; updatedAt: Scalars['DateTime']['output']; workspaceMembersCount?: Maybe; }; diff --git a/packages/twenty-front/src/generated/graphql.tsx b/packages/twenty-front/src/generated/graphql.tsx index cb973a45cf52..45bff9ca173c 100644 --- a/packages/twenty-front/src/generated/graphql.tsx +++ b/packages/twenty-front/src/generated/graphql.tsx @@ -1067,7 +1067,8 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + databaseSchema: Scalars['String']; + databaseUrl: Scalars['String']; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; @@ -1075,6 +1076,7 @@ export type Workspace = { id: Scalars['UUID']; inviteHash?: Maybe; logo?: Maybe; + metadataVersion: Scalars['Float']; updatedAt: Scalars['DateTime']; workspaceMembersCount?: Maybe; }; @@ -1360,7 +1362,7 @@ export type ImpersonateMutationVariables = Exact<{ }>; -export type ImpersonateMutation = { __typename?: 'Mutation', impersonate: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; +export type ImpersonateMutation = { __typename?: 'Mutation', impersonate: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; export type RenewTokenMutationVariables = Exact<{ appToken: Scalars['String']; @@ -1392,7 +1394,7 @@ export type VerifyMutationVariables = Exact<{ }>; -export type VerifyMutation = { __typename?: 'Mutation', verify: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; +export type VerifyMutation = { __typename?: 'Mutation', verify: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; export type CheckUserExistsQueryVariables = Exact<{ email: Scalars['String']; @@ -1453,7 +1455,7 @@ export type GetAisqlQueryQueryVariables = Exact<{ export type GetAisqlQueryQuery = { __typename?: 'Query', getAISQLQuery: { __typename?: 'AISQLQueryResult', sqlQuery: string, sqlQueryResult?: string | null, queryFailedErrorMessage?: string | null } }; -export type UserQueryFragmentFragment = { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }; +export type UserQueryFragmentFragment = { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }; export type DeleteUserAccountMutationVariables = Exact<{ [key: string]: never; }>; @@ -1470,7 +1472,7 @@ export type UploadProfilePictureMutation = { __typename?: 'Mutation', uploadProf export type GetCurrentUserQueryVariables = Exact<{ [key: string]: never; }>; -export type GetCurrentUserQuery = { __typename?: 'Query', currentUser: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> } }; +export type GetCurrentUserQuery = { __typename?: 'Query', currentUser: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> } }; export type WorkspaceMemberQueryFragmentFragment = { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }; @@ -1652,7 +1654,7 @@ export const UserQueryFragmentFragmentDoc = gql` value workspaceId } - currentCacheVersion + metadataVersion currentBillingSubscription { id status diff --git a/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts b/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts index 351ac8289350..b70cbdd17cb8 100644 --- a/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts +++ b/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts @@ -1,6 +1,6 @@ +import { InMemoryCache, NormalizedCacheObject } from '@apollo/client'; import { useMemo, useRef } from 'react'; import { useLocation, useNavigate } from 'react-router-dom'; -import { InMemoryCache, NormalizedCacheObject } from '@apollo/client'; import { useRecoilState, useSetRecoilState } from 'recoil'; import { currentUserState } from '@/auth/states/currentUserState'; @@ -49,8 +49,8 @@ export const useApolloFactory = (options: Partial> = {}) => { }, }), headers: { - ...(currentWorkspace?.currentCacheVersion && { - 'X-Schema-Version': currentWorkspace.currentCacheVersion, + ...(currentWorkspace?.metadataVersion && { + 'X-Schema-Version': `${currentWorkspace.metadataVersion}`, }), }, defaultOptions: { @@ -95,7 +95,7 @@ export const useApolloFactory = (options: Partial> = {}) => { setCurrentWorkspace, setWorkspaces, isDebugMode, - currentWorkspace?.currentCacheVersion, + currentWorkspace?.metadataVersion, setPreviousUrl, ]); diff --git a/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts b/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts index a66d7bd1cb06..8dd9ea7bc791 100644 --- a/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts +++ b/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts @@ -13,7 +13,7 @@ export type CurrentWorkspace = Pick< | 'activationStatus' | 'currentBillingSubscription' | 'workspaceMembersCount' - | 'currentCacheVersion' + | 'metadataVersion' >; export const currentWorkspaceState = createState({ diff --git a/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts b/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts index 0d1971e29744..76e5400c34e9 100644 --- a/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts +++ b/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts @@ -29,7 +29,7 @@ export const USER_QUERY_FRAGMENT = gql` value workspaceId } - currentCacheVersion + metadataVersion currentBillingSubscription { id status diff --git a/packages/twenty-front/src/testing/mock-data/users.ts b/packages/twenty-front/src/testing/mock-data/users.ts index 5d6514c42867..f7a4c2727e22 100644 --- a/packages/twenty-front/src/testing/mock-data/users.ts +++ b/packages/twenty-front/src/testing/mock-data/users.ts @@ -63,7 +63,7 @@ export const mockDefaultWorkspace: Workspace = { ], createdAt: '2023-04-26T10:23:42.33625+00:00', updatedAt: '2023-04-26T10:23:42.33625+00:00', - currentCacheVersion: '1', + metadataVersion: 1, currentBillingSubscription: { __typename: 'BillingSubscription', id: '7efbc3f7-6e5e-4128-957e-8d86808cdf6a', @@ -71,6 +71,8 @@ export const mockDefaultWorkspace: Workspace = { status: SubscriptionStatus.Active, }, workspaceMembersCount: 1, + databaseSchema: '', + databaseUrl: '', }; export const mockedWorkspaceMemberData: WorkspaceMember = { diff --git a/packages/twenty-server/@types/express.d.ts b/packages/twenty-server/@types/express.d.ts index 89fea7d77c9f..1e45cf88d079 100644 --- a/packages/twenty-server/@types/express.d.ts +++ b/packages/twenty-server/@types/express.d.ts @@ -8,6 +8,6 @@ declare module 'express-serve-static-core' { apiKey?: ApiKeyWorkspaceEntity | null; workspace?: Workspace; workspaceId?: string; - cacheVersion?: string | null; + workspaceMetadataVersion?: number; } } diff --git a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch b/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch index 70572a4907ee..b15363c16b86 100644 --- a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch +++ b/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch @@ -32,7 +32,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -79,7 +79,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -146,7 +146,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -193,7 +193,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -387,7 +387,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac ...options, + schema: async request => { + const workspaceId = request.req.workspace.id -+ const workspaceCacheVersion = request.req.cacheVersion ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -448,7 +448,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac ...options, + schema: async request => { + const workspaceId = request.req.workspace.id -+ const workspaceCacheVersion = request.req.cacheVersion ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.ts index 914b5b295f6c..6e7b5316e4ee 100644 --- a/packages/twenty-server/src/app.module.ts +++ b/packages/twenty-server/src/app.module.ts @@ -5,27 +5,27 @@ import { RequestMethod, } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; -import { ServeStaticModule } from '@nestjs/serve-static'; import { GraphQLModule } from '@nestjs/graphql'; +import { ServeStaticModule } from '@nestjs/serve-static'; import { existsSync } from 'fs'; import { join } from 'path'; -import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs'; +import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs'; -import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; -import { ModulesModule } from 'src/modules/modules.module'; import { CoreGraphQLApiModule } from 'src/engine/api/graphql/core-graphql-api.module'; -import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module'; import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module'; import { GraphQLConfigService } from 'src/engine/api/graphql/graphql-config/graphql-config.service'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; -import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; -import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module'; +import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module'; +import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; import { MessageQueueDriverType } from 'src/engine/integrations/message-queue/interfaces'; +import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; +import { ModulesModule } from 'src/modules/modules.module'; -import { IntegrationsModule } from './engine/integrations/integrations.module'; import { CoreEngineModule } from './engine/core-modules/core-engine.module'; +import { IntegrationsModule } from './engine/integrations/integrations.module'; @Module({ imports: [ @@ -52,7 +52,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module'; // Modules module, contains all business logic modules ModulesModule, // Needed for the user workspace middleware - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, // Api modules CoreGraphQLApiModule, MetadataGraphQLApiModule, diff --git a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts index 2152c787807a..6e5fef348b0a 100644 --- a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts +++ b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts @@ -1,8 +1,7 @@ import { Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; import { Command, CommandRunner } from 'nest-commander'; -import { EntityManager, Repository } from 'typeorm'; +import { EntityManager } from 'typeorm'; import { seedCoreSchema } from 'src/database/typeorm-seeds/core'; import { @@ -39,7 +38,6 @@ import { DataSourceService } from 'src/engine/metadata-modules/data-source/data- import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { viewPrefillData } from 'src/engine/workspace-manager/standard-objects-prefill-data/view'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; @@ -61,12 +59,9 @@ export class DataSeedWorkspaceCommand extends CommandRunner { private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, private readonly fieldMetadataService: FieldMetadataService, - @InjectRepository(ObjectMetadataEntity, 'metadata') - private readonly objectMetadataRepository: Repository, private readonly objectMetadataService: ObjectMetadataService, - @InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema) + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) private readonly workspaceSchemaCache: CacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, ) { super(); } @@ -75,7 +70,6 @@ export class DataSeedWorkspaceCommand extends CommandRunner { try { for (const workspaceId of this.workspaceIds) { await this.workspaceSchemaCache.flush(); - await this.workspaceCacheVersionService.deleteVersion(workspaceId); await rawDataSource.initialize(); diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index 80d685c955a9..6bf31eb73c2a 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -19,7 +19,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; @@ -45,7 +45,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp ObjectMetadataModule, FieldMetadataModule, DataSeedDemoWorkspaceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, UpgradeTo0_23CommandModule, UpgradeVersionModule, ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts index 47e45fc0a655..5e31d61c2b50 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts @@ -10,7 +10,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service'; import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @@ -34,7 +34,7 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner { private readonly objectMetadataRepository: Repository, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) { super(); } @@ -212,7 +212,9 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner { options: newOptions, }); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts index 875a9fdaa603..726d698eac38 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts @@ -13,7 +13,7 @@ import { Workspace, WorkspaceActivationStatus, } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; @@ -34,7 +34,7 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner { SetUserVarsAccountsToReconnectCommand.name, ); constructor( - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, private readonly accountsToReconnectService: AccountsToReconnectService, @InjectRepository(KeyValuePair, 'core') @@ -149,7 +149,9 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner { throw error; } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts index 7331b7fde219..8021a41830d3 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts @@ -12,7 +12,7 @@ import { WorkspaceActivationStatus, } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; interface SetWorkspaceActivationStatusCommandOptions { workspaceId?: string; @@ -31,7 +31,7 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner { private readonly workspaceRepository: Repository, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly billingSubscriptionService: BillingSubscriptionService, ) { super(); @@ -97,7 +97,9 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner { } } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts index fcc8b7b3cf0d..da4db435a295 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts @@ -9,7 +9,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { notesAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/notes-all.view'; import { tasksAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/tasks-all.view'; @@ -44,7 +44,7 @@ export class UpdateActivitiesCommand extends CommandRunner { private readonly workspaceStatusService: WorkspaceStatusService, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly objectMetadataService: ObjectMetadataService, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) { @@ -400,7 +400,9 @@ export class UpdateActivitiesCommand extends CommandRunner { .execute(); } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts index e62f15b65f15..09fbe2fb9f98 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts @@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; @@ -35,7 +35,7 @@ import { ViewModule } from 'src/modules/view/view.module'; OnboardingModule, TypeORMModule, DataSourceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, FieldMetadataModule, DataSourceModule, WorkspaceStatusModule, diff --git a/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts index d0d869b408ae..2cfefd9d0b0a 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts @@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; @@ -48,7 +48,7 @@ import { ViewModule } from 'src/modules/view/view.module'; WorkspaceStatusModule, ObjectMetadataModule, DataSeedDemoWorkspaceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, FieldMetadataModule, ViewModule, BillingModule, diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts b/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts new file mode 100644 index 000000000000..f53ccd85f410 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043 + implements MigrationInterface +{ + name = 'IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "metadataVersion" integer NOT NULL DEFAULT '1'`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "databaseUrl" character varying NOT NULL DEFAULT ''`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "databaseSchema" character varying NOT NULL DEFAULT ''`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "databaseSchema"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "databaseUrl"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "metadataVersion"`, + ); + } +} diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts new file mode 100644 index 000000000000..37a5d6674a26 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class DeprecateCacheVersion1724173061204 implements MigrationInterface { + name = 'DeprecateCacheVersion1724173061204'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "metadata"."workspaceCacheVersion"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "metadata"."workspaceCacheVersion" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "workspaceId" uuid NOT NULL, "version" character varying NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_1a80ecf2638b477809403cc26ed" UNIQUE ("workspaceId"), CONSTRAINT "PK_5d502f8dbfb5b9a8bf2439320e9" PRIMARY KEY ("id"))`, + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts index f3554cdde0ba..57d051363e3f 100644 --- a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts +++ b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts @@ -1,11 +1,12 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; describe('WorkspaceSchemaFactory', () => { @@ -39,6 +40,10 @@ describe('WorkspaceSchemaFactory', () => { provide: WorkspaceCacheStorageService, useValue: {}, }, + { + provide: WorkspaceMetadataVersionService, + useValue: {}, + }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts index 33c24c621f78..c9b0aa852312 100644 --- a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts @@ -1,10 +1,11 @@ import { Module } from '@nestjs/common'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; -import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module'; +import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceSchemaFactory } from './workspace-schema.factory'; @@ -21,6 +22,7 @@ import { WorkspaceSchemaFactory } from './workspace-schema.factory'; WorkspaceSchemaBuilderModule, WorkspaceResolverBuilderModule, WorkspaceCacheStorageModule, + WorkspaceMetadataVersionModule, ], providers: [WorkspaceSchemaFactory, ScalarsExplorerService], exports: [WorkspaceSchemaFactory], diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts index 3e3be6b1bef5..d8b7de5d67dc 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts @@ -9,10 +9,11 @@ export type CacheMetadataPluginConfig = { export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin { const computeCacheKey = (serverContext: any) => { const workspaceId = serverContext.req.workspace?.id ?? 'anonymous'; - const cacheVersion = serverContext.req.cacheVersion ?? '0'; + const workspaceMetadataVersion = + serverContext.req.workspaceMetadataVersion ?? '0'; const operationName = getOperationName(serverContext); - return `cachedOperations:${operationName}:${workspaceId}:${cacheVersion}`; + return `graphql:operations:${operationName}:${workspaceId}:${workspaceMetadataVersion}`; }; const getOperationName = (serverContext: any) => diff --git a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts index cc41b6e742ff..3d0627ce7336 100644 --- a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts @@ -1,19 +1,18 @@ import { Module } from '@nestjs/common'; import { GraphQLModule } from '@nestjs/graphql'; -import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs'; +import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs'; -import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; -import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; -import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module'; import { metadataModuleFactory } from 'src/engine/api/graphql/metadata.module-factory'; -import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; import { DataloaderModule } from 'src/engine/dataloaders/dataloader.module'; import { DataloaderService } from 'src/engine/dataloaders/dataloader.service'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; -import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache-storage.module'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; +import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; +import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @Module({ imports: [ @@ -25,13 +24,12 @@ import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache- EnvironmentService, ExceptionHandlerService, DataloaderService, - CacheStorageNamespace.WorkspaceSchema, + CacheStorageNamespace.EngineWorkspace, ], }), MetadataEngineModule, WorkspaceMigrationRunnerModule, WorkspaceMigrationModule, - CacheStorageModule, ], }) export class MetadataGraphQLApiModule {} diff --git a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts index 3d2e2b7e1584..5297b747b954 100644 --- a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts @@ -15,7 +15,7 @@ export const metadataModuleFactory = async ( environmentService: EnvironmentService, exceptionHandlerService: ExceptionHandlerService, dataloaderService: DataloaderService, - workspaceSchemaCacheStorage: CacheStorageService, + cacheStorageService: CacheStorageService, ): Promise => { const config: YogaDriverConfig = { autoSchemaFile: true, @@ -36,12 +36,8 @@ export const metadataModuleFactory = async ( exceptionHandlerService, }), useCachedMetadata({ - cacheGetter: workspaceSchemaCacheStorage.get.bind( - workspaceSchemaCacheStorage, - ), - cacheSetter: workspaceSchemaCacheStorage.set.bind( - workspaceSchemaCacheStorage, - ), + cacheGetter: cacheStorageService.get.bind(cacheStorageService), + cacheSetter: cacheStorageService.set.bind(cacheStorageService), operationsToCache: ['ObjectMetadataItems'], }), ], diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index f06c94c20167..c82f3d18fe1d 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -1,17 +1,18 @@ import { Injectable } from '@nestjs/common'; -import { GraphQLSchema, printSchema } from 'graphql'; import { makeExecutableSchema } from '@graphql-tools/schema'; +import { GraphQLSchema, printSchema } from 'graphql'; import { gql } from 'graphql-tag'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; -import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories'; import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; +import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Injectable() export class WorkspaceSchemaFactory { @@ -22,6 +23,7 @@ export class WorkspaceSchemaFactory { private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory, private readonly workspaceResolverFactory: WorkspaceResolverFactory, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) {} async createGraphQLSchema(authContext: AuthContext): Promise { @@ -40,7 +42,7 @@ export class WorkspaceSchemaFactory { } // Validate cache version - await this.workspaceCacheStorageService.validateCacheVersion( + await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated( authContext.workspace.id, ); @@ -64,11 +66,11 @@ export class WorkspaceSchemaFactory { } // Get typeDefs from cache - let typeDefs = await this.workspaceCacheStorageService.getTypeDefs( + let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs( authContext.workspace.id, ); let usedScalarNames = - await this.workspaceCacheStorageService.getUsedScalarNames( + await this.workspaceCacheStorageService.getGraphQLUsedScalarNames( authContext.workspace.id, ); @@ -84,11 +86,11 @@ export class WorkspaceSchemaFactory { this.scalarsExplorerService.getUsedScalarNames(autoGeneratedSchema); typeDefs = printSchema(autoGeneratedSchema); - await this.workspaceCacheStorageService.setTypeDefs( + await this.workspaceCacheStorageService.setGraphQLTypeDefs( authContext.workspace.id, typeDefs, ); - await this.workspaceCacheStorageService.setUsedScalarNames( + await this.workspaceCacheStorageService.setGraphQLUsedScalarNames( authContext.workspace.id, usedScalarNames, ); diff --git a/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts b/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts index de2fe47504b4..a2fe6db4805a 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts @@ -21,7 +21,6 @@ import { TokenService } from 'src/engine/core-modules/auth/services/token.servic import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type'; import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; @Controller('auth/google-apis') @UseFilters(AuthRestApiExceptionFilter) @@ -31,7 +30,6 @@ export class GoogleAPIsAuthController { private readonly tokenService: TokenService, private readonly environmentService: EnvironmentService, private readonly onboardingService: OnboardingService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, ) {} @Get() @@ -80,13 +78,7 @@ export class GoogleAPIsAuthController { const handle = emails[0].value; - const googleAPIsServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.googleAPIsService, - workspaceId, - ); - - await googleAPIsServiceInstance.refreshGoogleRefreshToken({ + await this.googleAPIsService.refreshGoogleRefreshToken({ handle, workspaceMemberId: workspaceMemberId, workspaceId: workspaceId, @@ -97,13 +89,7 @@ export class GoogleAPIsAuthController { }); if (userId) { - const onboardingServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.onboardingService, - workspaceId, - ); - - await onboardingServiceInstance.setOnboardingConnectAccountPending({ + await this.onboardingService.setOnboardingConnectAccountPending({ userId, workspaceId, value: false, diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts index 157c82ab7fd7..dcee921d280a 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts @@ -8,7 +8,7 @@ import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decora import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; -import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { CalendarEventListFetchJob, CalendarEventsImportJobData, @@ -39,7 +39,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta @Injectable() export class GoogleAPIsService { constructor( - private readonly twentyORMManager: TwentyORMManager, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, @InjectMessageQueue(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, @InjectMessageQueue(MessageQueue.calendarQueue) @@ -82,16 +82,19 @@ export class GoogleAPIsService { const newOrExistingConnectedAccountId = existingAccountId ?? v4(); const calendarChannelRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'calendarChannel', ); const messageChannelRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'messageChannel', ); - const workspaceDataSource = await this.twentyORMManager.getDatasource(); + const workspaceDataSource = + await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId); await workspaceDataSource.transaction(async (manager: EntityManager) => { if (!existingAccountId) { @@ -146,7 +149,8 @@ export class GoogleAPIsService { ); const workspaceMemberRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'workspaceMember', ); diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts index ec5669808cc2..6cf51366ffb7 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts @@ -1,14 +1,14 @@ -import { PassportStrategy } from '@nestjs/passport'; import { Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; -import { Strategy, VerifyCallback } from 'passport-google-oauth20'; import { Request } from 'express'; +import { Strategy, VerifyCallback } from 'passport-google-oauth20'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; export type GoogleRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts index f2d002a788c9..8eae65a81d44 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts @@ -12,7 +12,7 @@ import { EnvironmentService } from 'src/engine/integrations/environment/environm export type MicrosoftRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts b/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts index f8d5f609d4d1..4df605ebafa0 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts @@ -5,7 +5,7 @@ import { MessageChannelVisibility } from 'src/modules/messaging/common/standard- export type GoogleAPIsRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts index beb53c4d0407..2124c2b17969 100644 --- a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts @@ -44,6 +44,10 @@ export class FileController { workspaceId, ); + fileStream.on('error', () => { + res.status(500).send({ error: 'Internal server error' }); + }); + fileStream.pipe(res); } catch (error) { if ( diff --git a/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts b/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts index ef47046fe5b8..9785cde8a2db 100644 --- a/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts +++ b/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts @@ -119,13 +119,13 @@ export const useGraphQLErrorHandlerHook = < if (Array.isArray(errors) && errors.length > 0) { const headers = context.req.headers; - const currentSchemaVersion = context.req.cacheVersion; + const currentMetadataVersion = context.req.workspaceMetadataVersion; - const requestSchemaVersion = headers['x-schema-version']; + const requestMetadataVersion = headers['x-schema-version']; if ( - requestSchemaVersion && - requestSchemaVersion !== currentSchemaVersion + requestMetadataVersion && + requestMetadataVersion !== `${currentMetadataVersion}` ) { throw new GraphQLError( `Schema version mismatch, please refresh the page.`, diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index 1d5fb30c5f77..c77bb690baf7 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -33,7 +33,6 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; const getHMACKey = (email?: string, key?: string | null) => { @@ -54,7 +53,6 @@ export class UserResolver { private readonly environmentService: EnvironmentService, private readonly fileUploadService: FileUploadService, private readonly onboardingService: OnboardingService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, private readonly userVarService: UserVarsService, private readonly fileService: FileService, ) {} @@ -189,11 +187,6 @@ export class UserResolver { @ResolveField(() => OnboardingStatus) async onboardingStatus(@Parent() user: User): Promise { - const contextInstance = await this.loadServiceWithWorkspaceContext.load( - this.onboardingService, - user.defaultWorkspaceId, - ); - - return contextInstance.getOnboardingStatus(user); + return this.onboardingService.getOnboardingStatus(user); } } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts index 0b4a3e0488aa..b9948f623b2c 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts @@ -117,4 +117,16 @@ export class Workspace { (postgresCredentials) => postgresCredentials.workspace, ) allPostgresCredentials: Relation; + + @Field() + @Column({ default: 1 }) + metadataVersion: number; + + @Field() + @Column({ default: '' }) + databaseUrl: string; + + @Field() + @Column({ default: '' }) + databaseSchema: string; } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index a8255d8996b8..5c8e832661ec 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener'; import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts'; @@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service'; BillingModule, FileModule, FileUploadModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, NestjsQueryTypeOrmModule.forFeature( [User, Workspace, UserWorkspace, FeatureFlagEntity], 'core', diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts index 1bb019344665..bc995017ba60 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts @@ -26,7 +26,7 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { assert } from 'src/utils/assert'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; @@ -39,7 +39,7 @@ import { WorkspaceService } from './services/workspace.service'; export class WorkspaceResolver { constructor( private readonly workspaceService: WorkspaceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly userWorkspaceService: UserWorkspaceService, private readonly fileUploadService: FileUploadService, private readonly fileService: FileService, @@ -107,13 +107,6 @@ export class WorkspaceResolver { return this.workspaceService.deleteWorkspace(id); } - @ResolveField(() => String, { nullable: true }) - async currentCacheVersion( - @Parent() workspace: Workspace, - ): Promise { - return this.workspaceCacheVersionService.getVersion(workspace.id); - } - @ResolveField(() => BillingSubscription, { nullable: true }) async currentBillingSubscription( @Parent() workspace: Workspace, @@ -133,11 +126,15 @@ export class WorkspaceResolver { @ResolveField(() => String) async logo(@Parent() workspace: Workspace): Promise { if (workspace.logo) { - const workspaceLogoToken = await this.fileService.encodeFileToken({ - workspace_id: workspace.id, - }); - - return `${workspace.logo}?token=${workspaceLogoToken}`; + try { + const workspaceLogoToken = await this.fileService.encodeFileToken({ + workspace_id: workspace.id, + }); + + return `${workspace.logo}?token=${workspaceLogoToken}`; + } catch (e) { + return workspace.logo; + } } return workspace.logo ?? ''; diff --git a/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts b/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts index e89fb00b0396..9296a629c9ec 100644 --- a/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts +++ b/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts @@ -1,5 +1,5 @@ export enum CacheStorageNamespace { - Messaging = 'messaging', - Calendar = 'calendar', - WorkspaceSchema = 'workspaceSchema', + ModuleMessaging = 'module:messaging', + ModuleCalendar = 'module:calendar', + EngineWorkspace = 'engine:workspace', } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts index 7eac8874448d..c8797101d6b6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts @@ -13,15 +13,15 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto'; import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver'; import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor'; +import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook'; import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator'; import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; -import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook'; import { FieldMetadataEntity } from './field-metadata.entity'; import { FieldMetadataService } from './field-metadata.service'; @@ -38,7 +38,7 @@ import { UpdateFieldInput } from './dtos/update-field.input'; WorkspaceStatusModule, TwentyORMModule, WorkspaceMigrationRunnerModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, ObjectMetadataModule, DataSourceModule, TypeORMModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index 9dfe1415c059..8b26733dcc1a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -40,7 +40,7 @@ import { NameTooLongException, validateMetadataNameOrThrow, } from 'src/engine/metadata-modules/utils/validate-metadata-name.utils'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -76,7 +76,7 @@ export class FieldMetadataService extends TypeOrmQueryService >, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly dataSourceService: DataSourceService, private readonly objectMetadataService: ObjectMetadataService, private readonly fieldMetadataService: FieldMetadataService, @@ -250,7 +250,9 @@ export class RemoteTableService { await this.remoteTableRepository.save(remoteTableEntity); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); return { id: remoteTableEntity.id, @@ -435,7 +437,9 @@ export class RemoteTableService { await this.remoteTableRepository.delete(remoteTable.id); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); } private async createRemoteTableMetadata( diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts deleted file mode 100644 index fab0a85d5fb8..000000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm'; - -@Entity('workspaceCacheVersion') -export class WorkspaceCacheVersionEntity { - @PrimaryGeneratedColumn('uuid') - id: string; - - @Column({ unique: true, nullable: false, type: 'uuid' }) - workspaceId: string; - - @Column() - version: string; - - @CreateDateColumn({ type: 'timestamptz' }) - createdAt: Date; - - @UpdateDateColumn({ type: 'timestamptz' }) - updatedAt: Date; -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts deleted file mode 100644 index a0df038e4ee6..000000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; - -import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; - -@Module({ - imports: [ - TypeOrmModule.forFeature([WorkspaceCacheVersionEntity], 'metadata'), - ], - exports: [WorkspaceCacheVersionService], - providers: [WorkspaceCacheVersionService], -}) -export class WorkspaceCacheVersionModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts deleted file mode 100644 index 8b0bfb8c2a10..000000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import { Repository } from 'typeorm'; - -import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity'; - -@Injectable() -export class WorkspaceCacheVersionService { - constructor( - @InjectRepository(WorkspaceCacheVersionEntity, 'metadata') - private readonly workspaceCacheVersionRepository: Repository, - ) {} - - async incrementVersion(workspaceId: string): Promise { - const workspaceCacheVersion = (await this.getVersion(workspaceId)) ?? '0'; - const newVersion = `${+workspaceCacheVersion + 1}`; - - await this.workspaceCacheVersionRepository.upsert( - { - workspaceId, - version: `${+workspaceCacheVersion + 1}`, - }, - ['workspaceId'], - ); - - return newVersion; - } - - async getVersion(workspaceId: string): Promise { - const workspaceCacheVersion = - await this.workspaceCacheVersionRepository.findOne({ - where: { workspaceId }, - }); - - return workspaceCacheVersion?.version ?? null; - } - - async deleteVersion(workspaceId: string): Promise { - await this.workspaceCacheVersionRepository.delete({ workspaceId }); - } -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts new file mode 100644 index 000000000000..4064872a17e2 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + WorkspaceCacheStorageModule, + ], + exports: [WorkspaceMetadataVersionService], + providers: [WorkspaceMetadataVersionService], +}) +export class WorkspaceMetadataVersionModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts new file mode 100644 index 000000000000..8e5c5deaaa6e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts @@ -0,0 +1,79 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; + +@Injectable() +export class WorkspaceMetadataVersionService { + logger = new Logger(WorkspaceMetadataVersionService.name); + + constructor( + @InjectRepository(Workspace, 'core') + private readonly workspaceRepository: Repository, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + ) {} + + async flushCacheIfMetadataVersionIsOutdated( + workspaceId: string, + ): Promise { + const currentVersion = + (await this.workspaceCacheStorageService.getMetadataVersion( + workspaceId, + )) ?? 1; + + let latestVersion = await this.getMetadataVersion(workspaceId); + + if (latestVersion === undefined || currentVersion !== latestVersion) { + this.logger.log( + `Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, + ); + + await this.workspaceCacheStorageService.flush(workspaceId); + + latestVersion = await this.incrementMetadataVersion(workspaceId); + + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + latestVersion, + ); + } + } + + async incrementMetadataVersion(workspaceId: string): Promise { + const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0; + const newMetadataVersion = metadataVersion + 1; + + await this.workspaceRepository.update( + { id: workspaceId }, + { metadataVersion: newMetadataVersion }, + ); + + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + newMetadataVersion, + ); + + return newMetadataVersion; + } + + async getMetadataVersion(workspaceId: string): Promise { + const workspace = await this.workspaceRepository.findOne({ + where: { id: workspaceId }, + }); + + return workspace?.metadataVersion; + } + + async resetMetadataVersion(workspaceId: string): Promise { + await this.workspaceRepository.update( + { id: workspaceId }, + { metadataVersion: 1 }, + ); + + await this.workspaceCacheStorageService.flush(workspaceId); + await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1); + } +} diff --git a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts index a756b8cef472..509e58bb31c7 100644 --- a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts +++ b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts @@ -6,7 +6,7 @@ import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filt import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { handleExceptionAndConvertToGraphQLError } from 'src/engine/utils/global-exception-handler.util'; class GraphqlTokenValidationProxy { @@ -33,7 +33,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware { constructor( private readonly tokenService: TokenService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly exceptionHandlerService: ExceptionHandlerService, ) {} @@ -72,15 +72,16 @@ export class GraphQLHydrateRequestFromTokenMiddleware ); data = await graphqlTokenValidationProxy.validateToken(req); - const cacheVersion = await this.workspaceCacheVersionService.getVersion( - data.workspace.id, - ); + const metadataVersion = + await this.workspaceMetadataVersionService.getMetadataVersion( + data.workspace.id, + ); req.user = data.user; req.apiKey = data.apiKey; req.workspace = data.workspace; req.workspaceId = data.workspace.id; - req.cacheVersion = cacheVersion; + req.workspaceMetadataVersion = metadataVersion; } catch (error) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.write( diff --git a/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts b/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts deleted file mode 100644 index 144f47598fa7..000000000000 --- a/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Inject, Type } from '@nestjs/common'; -import { ModuleRef, createContextId } from '@nestjs/core'; -import { Injector } from '@nestjs/core/injector/injector'; - -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; - -export class LoadServiceWithWorkspaceContext { - private readonly injector = new Injector(); - - constructor( - @Inject(ModuleRef) - private readonly moduleRef: ModuleRef, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, - ) {} - - async load(service: T, workspaceId: string): Promise { - const modules = this.moduleRef['container'].getModules(); - const host = [...modules.values()].find((module) => - module.providers.has((service as Type).constructor), - ); - - if (!host) { - throw new Error('Host module not found for the service'); - } - - const contextId = createContextId(); - const cacheVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); - - if (this.moduleRef.registerRequestByContextId) { - this.moduleRef.registerRequestByContextId( - { req: { workspaceId, cacheVersion } }, - contextId, - ); - } - - return this.injector.loadPerContext( - service, - host, - new Map(host.providers), - contextId, - ); - } -} diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts index 88603807a329..bf70156a94e0 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts @@ -11,16 +11,16 @@ export class ScopedWorkspaceContextFactory { public create(): { workspaceId: string | null; - cacheVersion: string | null; + workspaceMetadataVersion: string | null; } { const workspaceId: string | undefined = this.request?.['req']?.['workspaceId']; - const cacheVersion: string | undefined = - this.request?.['req']?.['cacheVersion']; + const workspaceMetadataVersion: string | undefined = + this.request?.['req']?.['workspaceMetadataVersion']; return { workspaceId: workspaceId ?? null, - cacheVersion: cacheVersion ?? null, + workspaceMetadataVersion: workspaceMetadataVersion ?? null, }; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 1ee50a0b44a3..10daedc7dd77 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { EntitySchema, Repository } from 'typeorm'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module'; @@ -18,7 +18,7 @@ export class WorkspaceDatasourceFactory { private readonly dataSourceService: DataSourceService, private readonly environmentService: EnvironmentService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, private readonly entitySchemaFactory: EntitySchemaFactory, @@ -26,21 +26,29 @@ export class WorkspaceDatasourceFactory { public async create( workspaceId: string, - workspaceSchemaVersion: string | null, + workspaceMetadataVersion: string | null, ): Promise { - const desiredWorkspaceSchemaVersion = - workspaceSchemaVersion ?? - (await this.workspaceCacheVersionService.getVersion(workspaceId)); + const desiredWorkspaceMetadataVersion = + workspaceMetadataVersion ?? + (await this.workspaceMetadataVersionService.getMetadataVersion( + workspaceId, + )); - if (!desiredWorkspaceSchemaVersion) { - throw new Error('Cache version not found'); + if (!desiredWorkspaceMetadataVersion) { + throw new Error( + `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`, + ); } - const latestWorkspaceSchemaVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); + const latestWorkspaceMetadataVersion = + await this.workspaceMetadataVersionService.getMetadataVersion( + workspaceId, + ); - if (latestWorkspaceSchemaVersion !== desiredWorkspaceSchemaVersion) { - throw new Error('Cache version mismatch'); + if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { + throw new Error( + `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, + ); } let cachedObjectMetadataCollection = @@ -70,7 +78,7 @@ export class WorkspaceDatasourceFactory { } const workspaceDataSource = await workspaceDataSourceCacheInstance.execute( - `${workspaceId}-${latestWorkspaceSchemaVersion}`, + `${workspaceId}-${latestWorkspaceMetadataVersion}`, async () => { const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( @@ -78,18 +86,43 @@ export class WorkspaceDatasourceFactory { ); if (!dataSourceMetadata) { - throw new Error('Data source metadata not found'); + throw new Error( + `Data source metadata not found for workspace ${workspaceId}`, + ); } if (!cachedObjectMetadataCollection) { - throw new Error('Object metadata collection not found'); + throw new Error( + `Object metadata collection not found for workspace ${workspaceId}`, + ); + } + + const cachedEntitySchemaOptions = + await this.workspaceCacheStorageService.getORMEntitySchema( + workspaceId, + ); + + let cachedEntitySchemas: EntitySchema[]; + + if (cachedEntitySchemaOptions) { + cachedEntitySchemas = cachedEntitySchemaOptions.map( + (option) => new EntitySchema(option), + ); + } else { + const entitySchemas = await Promise.all( + cachedObjectMetadataCollection.map((objectMetadata) => + this.entitySchemaFactory.create(workspaceId, objectMetadata), + ), + ); + + await this.workspaceCacheStorageService.setORMEntitySchema( + workspaceId, + entitySchemas.map((entitySchema) => entitySchema.options), + ); + + cachedEntitySchemas = entitySchemas; } - const entities = await Promise.all( - cachedObjectMetadataCollection.map((objectMetadata) => - this.entitySchemaFactory.create(workspaceId, objectMetadata), - ), - ); const workspaceDataSource = new WorkspaceDataSource( { workspaceId, @@ -104,7 +137,7 @@ export class WorkspaceDatasourceFactory { ? ['query', 'error'] : ['error'], schema: dataSourceMetadata.schema, - entities, + entities: cachedEntitySchemas, ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED') ? { rejectUnauthorized: false, diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts index ebb929a641d1..122ae0c52c4b 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts @@ -14,8 +14,7 @@ import { import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { entitySchemaFactories } from 'src/engine/twenty-orm/factories'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; @@ -33,21 +32,15 @@ export const workspaceDataSourceCacheInstance = imports: [ TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), DataSourceModule, - WorkspaceCacheVersionModule, WorkspaceCacheStorageModule, + WorkspaceMetadataVersionModule, ], providers: [ ...entitySchemaFactories, TwentyORMManager, TwentyORMGlobalManager, - LoadServiceWithWorkspaceContext, - ], - exports: [ - EntitySchemaFactory, - TwentyORMManager, - LoadServiceWithWorkspaceContext, - TwentyORMGlobalManager, ], + exports: [EntitySchemaFactory, TwentyORMManager, TwentyORMGlobalManager], }) export class TwentyORMCoreModule extends ConfigurableModuleClass diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts index 5ebd65a3c108..6855c705678b 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts @@ -25,7 +25,7 @@ export class TwentyORMManager { async getRepository( workspaceEntityOrobjectMetadataName: Type | string, ): Promise> { - const { workspaceId, cacheVersion } = + const { workspaceId, workspaceMetadataVersion } = this.scopedWorkspaceContextFactory.create(); let objectMetadataName: string; @@ -44,20 +44,23 @@ export class TwentyORMManager { const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, - cacheVersion, + workspaceMetadataVersion, ); return workspaceDataSource.getRepository(objectMetadataName); } async getDatasource() { - const { workspaceId, cacheVersion } = + const { workspaceId, workspaceMetadataVersion } = this.scopedWorkspaceContextFactory.create(); if (!workspaceId) { throw new Error('Workspace not found'); } - return this.workspaceDataSourceFactory.create(workspaceId, cacheVersion); + return this.workspaceDataSourceFactory.create( + workspaceId, + workspaceMetadataVersion, + ); } } diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts index 62d8be6fd6ba..fec5215284bb 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts @@ -1,10 +1,8 @@ import { Module } from '@nestjs/common'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Module({ - imports: [WorkspaceCacheVersionModule], providers: [WorkspaceCacheStorageService], exports: [WorkspaceCacheStorageService], }) diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts index 6160847023a9..18426da5a7bf 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts @@ -1,56 +1,67 @@ import { Injectable, Logger } from '@nestjs/common'; +import { EntitySchemaOptions } from 'typeorm'; + import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; + +enum WorkspaceCacheKeys { + GraphQLTypeDefs = 'graphql:type-defs', + GraphQLUsedScalarNames = 'graphql:used-scalar-names', + GraphQLOperations = 'graphql:operations', + ORMEntitySchemas = 'orm:entity-schemas', + MetadataObjectMetadataCollection = 'metadata:object-metadata-collection', + MetadataVersion = 'metadata:workspace-metadata-version', +} @Injectable() export class WorkspaceCacheStorageService { private readonly logger = new Logger(WorkspaceCacheStorageService.name); constructor( - @InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema) - private readonly workspaceSchemaCache: CacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) + private readonly cacheStorageService: CacheStorageService, ) {} - async validateCacheVersion(workspaceId: string): Promise { - const currentVersion = - (await this.workspaceSchemaCache.get( - `cacheVersion:${workspaceId}`, - )) ?? '0'; - - let latestVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); - - if (!latestVersion || currentVersion !== latestVersion) { - // Invalidate cache if version mismatch is detected" - this.logger.log( - `Cache version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, - ); - - await this.invalidateCache(workspaceId); - - // If the latest version is not found, increment the version - latestVersion ??= - await this.workspaceCacheVersionService.incrementVersion(workspaceId); - - // Update the cache version after invalidation - await this.workspaceSchemaCache.set( - `cacheVersion:${workspaceId}`, - latestVersion, - ); - } + setORMEntitySchema( + workspaceId: string, + entitySchemas: EntitySchemaOptions[], + ) { + return this.cacheStorageService.set[]>( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + entitySchemas, + ); + } + + getORMEntitySchema( + workspaceId: string, + ): Promise[] | undefined> { + return this.cacheStorageService.get[]>( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + ); + } + + setMetadataVersion(workspaceId: string, version: number): Promise { + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + version, + ); + } + + getMetadataVersion(workspaceId: string): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + ); } setObjectMetadataCollection( workspaceId: string, objectMetadataCollection: ObjectMetadataEntity[], ) { - return this.workspaceSchemaCache.set( - `objectMetadataCollection:${workspaceId}`, + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, objectMetadataCollection, ); } @@ -58,43 +69,57 @@ export class WorkspaceCacheStorageService { getObjectMetadataCollection( workspaceId: string, ): Promise { - return this.workspaceSchemaCache.get( - `objectMetadataCollection:${workspaceId}`, + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, ); } - setTypeDefs(workspaceId: string, typeDefs: string): Promise { - return this.workspaceSchemaCache.set( - `typeDefs:${workspaceId}`, + setGraphQLTypeDefs(workspaceId: string, typeDefs: string): Promise { + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, typeDefs, ); } - getTypeDefs(workspaceId: string): Promise { - return this.workspaceSchemaCache.get(`typeDefs:${workspaceId}`); + getGraphQLTypeDefs(workspaceId: string): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + ); } - setUsedScalarNames( + setGraphQLUsedScalarNames( workspaceId: string, - scalarsUsed: string[], + usedScalarNames: string[], ): Promise { - return this.workspaceSchemaCache.set( - `usedScalarNames:${workspaceId}`, - scalarsUsed, + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + usedScalarNames, ); } - getUsedScalarNames(workspaceId: string): Promise { - return this.workspaceSchemaCache.get( - `usedScalarNames:${workspaceId}`, + getGraphQLUsedScalarNames( + workspaceId: string, + ): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, ); } - async invalidateCache(workspaceId: string): Promise { - await this.workspaceSchemaCache.del( - `objectMetadataCollection:${workspaceId}`, + async flush(workspaceId: string): Promise { + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, ); - await this.workspaceSchemaCache.del(`typeDefs:${workspaceId}`); - await this.workspaceSchemaCache.del(`usedScalarNames:${workspaceId}`); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts index 2bbc17b75878..ffa4c387d925 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts @@ -1,14 +1,13 @@ -import { InjectRepository } from '@nestjs/typeorm'; import { Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { Command, CommandRunner, Option } from 'nest-commander'; import { In, Repository } from 'typeorm'; +import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { getDryRunLogHeader } from 'src/utils/get-dry-run-log-header'; -import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; type DeleteWorkspacesCommandOptions = { dryRun?: boolean; @@ -24,7 +23,6 @@ export class DeleteWorkspacesCommand extends CommandRunner { constructor( private readonly workspaceService: WorkspaceService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, @InjectRepository(Workspace, 'core') private readonly workspaceRepository: Repository, private readonly dataSourceService: DataSourceService, @@ -81,15 +79,15 @@ export class DeleteWorkspacesCommand extends CommandRunner { workspace.id } name: '${workspace.displayName}'`, ); - const workspaceServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.workspaceService, - workspace.id, - ); + // const workspaceServiceInstance = + // await this.loadServiceWithWorkspaceContext.load( + // this.workspaceService, + // workspace.id, + // ); - if (!options.dryRun) { - await workspaceServiceInstance.softDeleteWorkspace(workspace.id); - } + // if (!options.dryRun) { + // await workspaceServiceInstance.softDeleteWorkspace(workspace.id); + // } } } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts index 59f0bec2e661..40f6e5d92205 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts @@ -2,7 +2,6 @@ import { Module } from '@nestjs/common'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service'; @@ -10,11 +9,7 @@ import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.se import { WorkspaceMigrationTypeService } from './services/workspace-migration-type.service'; @Module({ - imports: [ - WorkspaceDataSourceModule, - WorkspaceMigrationModule, - WorkspaceCacheVersionModule, - ], + imports: [WorkspaceDataSourceModule, WorkspaceMigrationModule], providers: [ WorkspaceMigrationRunnerService, WorkspaceMigrationEnumService, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts index 447f60e0cf71..76cd7bfe9033 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts @@ -6,7 +6,7 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -35,7 +35,7 @@ import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/works 'metadata', ), TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, ], providers: [ ...workspaceSyncMetadataFactories, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 2a22bac5965f..6dfeb5557823 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -6,7 +6,7 @@ import { DataSource, QueryFailedError } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; import { FeatureFlagFactory } from 'src/engine/core-modules/feature-flag/services/feature-flags.factory'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service'; @@ -32,9 +32,9 @@ export class WorkspaceSyncMetadataService { private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService, private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService, private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService, private readonly workspaceSyncObjectMetadataIdentifiersService: WorkspaceSyncObjectMetadataIdentifiersService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) {} /** @@ -157,7 +157,7 @@ export class WorkspaceSyncMetadataService { await queryRunner.rollbackTransaction(); } finally { await queryRunner.release(); - await this.workspaceCacheVersionService.incrementVersion( + await this.workspaceMetadataVersionService.incrementMetadataVersion( context.workspaceId, ); } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts index 87d58c3f6009..56f9af21099f 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts @@ -32,6 +32,8 @@ export class CalendarEventListFetchCronJob { @Process(CalendarEventListFetchCronJob.name) async handle(): Promise { + console.time('CalendarEventListFetchCronJob time'); + const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -65,5 +67,7 @@ export class CalendarEventListFetchCronJob { ); } } + + console.timeEnd('CalendarEventListFetchCronJob time'); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts index 9006872acc63..402661b93981 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts @@ -33,6 +33,8 @@ export class CalendarEventListFetchJob { @Process(CalendarEventListFetchJob.name) async handle(data: CalendarEventsImportJobData): Promise { + console.time('CalendarEventListFetchJob time'); + const { workspaceId, calendarChannelId } = data; const calendarChannelRepository = @@ -91,5 +93,6 @@ export class CalendarEventListFetchJob { default: break; } + console.timeEnd('CalendarEventListFetchJob time'); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts index 28306cea884b..21ccdaafc047 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts @@ -16,7 +16,7 @@ import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/acc export class CalendarChannelSyncStatusService { constructor( private readonly twentyORMManager: TwentyORMManager, - @InjectCacheStorage(CacheStorageNamespace.Calendar) + @InjectCacheStorage(CacheStorageNamespace.ModuleCalendar) private readonly cacheStorage: CacheStorageService, private readonly accountsToReconnectService: AccountsToReconnectService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts b/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts index 49c52cb8333a..43025a021fb0 100644 --- a/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts @@ -15,7 +15,7 @@ import { @Injectable() export class MessagingChannelSyncStatusService { constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly twentyORMManager: TwentyORMManager, private readonly accountsToReconnectService: AccountsToReconnectService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts index 3f7334cf7c19..2f464188209d 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts @@ -33,6 +33,8 @@ export class MessagingMessageListFetchCronJob { @Process(MessagingMessageListFetchCronJob.name) async handle(): Promise { + console.time('MessagingMessageListFetchCronJob time'); + const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -66,5 +68,7 @@ export class MessagingMessageListFetchCronJob { } } } + + console.timeEnd('MessagingMessageListFetchCronJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts index e3da8b05bbc9..daef69dfa03d 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts @@ -36,6 +36,7 @@ export class MessagingMessagesImportCronJob { @Process(MessagingMessagesImportCronJob.name) async handle(): Promise { + console.time('MessagingMessagesImportCronJob time'); const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -69,5 +70,7 @@ export class MessagingMessagesImportCronJob { } } } + + console.timeEnd('MessagingMessagesImportCronJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts index d81bba9cd096..3f2c2a537901 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts @@ -14,7 +14,7 @@ export type MessagingAddSingleMessageToCacheForImportJobData = { @Processor(MessageQueue.messagingQueue) export class MessagingAddSingleMessageToCacheForImportJob { constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts index 8c2bc681f240..24e048ce01b3 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts @@ -1,11 +1,11 @@ import { Logger } from '@nestjs/common'; -import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; -import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; -import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; -import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; +import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; export type MessagingCleanCacheJobData = { workspaceId: string; @@ -17,7 +17,7 @@ export class MessagingCleanCacheJob { private readonly logger = new Logger(MessagingCleanCacheJob.name); constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts index 09854fbf4f3a..fc0d37413dfd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts @@ -39,6 +39,8 @@ export class MessagingMessageListFetchJob { @Process(MessagingMessageListFetchJob.name) async handle(data: MessagingMessageListFetchJobData): Promise { + console.time('MessagingMessageListFetchJob time'); + const { messageChannelId, workspaceId } = data; await this.messagingTelemetryService.track({ @@ -145,5 +147,7 @@ export class MessagingMessageListFetchJob { default: break; } + + console.timeEnd('MessagingMessageListFetchJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts index 935be741e363..c35243f8e29e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts @@ -35,6 +35,8 @@ export class MessagingMessagesImportJob { @Process(MessagingMessagesImportJob.name) async handle(data: MessagingMessagesImportJobData): Promise { + console.time('MessagingMessagesImportJob time'); + const { messageChannelId, workspaceId } = data; await this.messagingTelemetryService.track({ @@ -95,5 +97,7 @@ export class MessagingMessagesImportJob { connectedAccount, workspaceId, ); + + console.timeEnd('MessagingMessagesImportJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts index dde37698e9c8..eca0aaedf873 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts @@ -29,7 +29,7 @@ export class MessagingFullMessageListFetchService { constructor( private readonly gmailClientProvider: MessagingGmailClientProvider, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, private readonly gmailErrorHandlingService: MessagingErrorHandlingService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts index 89fcedb6d874..968140bc138f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts @@ -31,7 +31,7 @@ export class MessagingMessagesImportService { constructor( private readonly fetchMessagesByBatchesService: MessagingGmailFetchMessagesByBatchesService, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, private readonly saveMessagesAndEnqueueContactCreationService: MessagingSaveMessagesAndEnqueueContactCreationService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts index 7ab5f5c65250..05189ded170a 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts @@ -24,7 +24,7 @@ export class MessagingPartialMessageListFetchService { constructor( private readonly gmailClientProvider: MessagingGmailClientProvider, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly gmailErrorHandlingService: MessagingErrorHandlingService, private readonly gmailGetHistoryService: MessagingGmailHistoryService, diff --git a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts index e3a3b5e06c63..94ac094ffb24 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts @@ -32,6 +32,8 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { async handle(): Promise { this.logger.log('Starting message channel sync status monitoring...'); + console.time('MessagingMessageChannelSyncStatusMonitoringCronJob time'); + await this.messagingTelemetryService.track({ eventName: 'message_channel.monitoring.sync_status.start', message: 'Starting message channel sync status monitoring', @@ -68,5 +70,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { }); } } + + console.timeEnd('MessagingMessageChannelSyncStatusMonitoringCronJob time'); } } From 091c0f83be7bac60f1f5b89bedf000d896d74e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bosi?= <71827178+bosiraphael@users.noreply.github.com> Date: Tue, 20 Aug 2024 19:44:29 +0200 Subject: [PATCH 05/19] 6619 modify event emitter to emit an array of events (#6625) Closes #6619 --------- Co-authored-by: Charles Bochet --- .../listeners/entity-events-to-db.listener.ts | 49 +++-- .../listeners/telemetry.listener.ts | 65 +++--- .../workspace-query-runner.service.ts | 192 ++++++++++-------- .../billing-workspace-member.listener.ts | 5 +- .../engine/core-modules/core-engine.module.ts | 2 + .../user-workspace/user-workspace.service.ts | 16 +- .../services/__tests__/user.service.spec.ts | 10 +- .../user/services/user.service.ts | 13 +- .../workspace-workspace-member.listener.ts | 63 +++--- .../types/object-record-job-data.ts | 11 - .../types/object-record.base.event.ts | 7 +- .../workspace-event-emitter.module.ts | 11 + .../workspace-event-emitter.ts | 21 ++ .../workspace-event.type.ts | 5 + .../listeners/calendar-blocklist.listener.ts | 83 +++++--- .../calendar-event-cleaner.module.ts | 6 +- ...vent-cleaner-connected-account.listener.ts | 40 ++++ .../services/calendar-save-events.service.ts | 13 +- ...endar-event-participant-person.listener.ts | 85 ++++---- ...t-participant-workspace-member.listener.ts | 81 ++++---- .../calendar-event-participant.listener.ts | 83 ++++---- .../calendar-event-participant.service.ts | 2 +- .../listeners/connected-account.listener.ts | 41 ++-- ...ected-account-delete-one.pre-query.hook.ts | 28 +-- .../connected-account-query-hook.module.ts | 4 +- ...acts-creation-calendar-channel.listener.ts | 41 ++-- ...tacts-creation-message-channel.listener.ts | 41 ++-- .../create-company-and-contact.service.ts | 31 +-- .../match-participant.service.ts | 41 +++- .../listeners/messaging-blocklist.listener.ts | 144 +++++++------ ...sage-cleaner-connected-account.listener.ts | 37 ++-- .../messaging-message-cleaner.module.ts | 10 +- ...import-manager-message-channel.listener.ts | 23 ++- .../message-participant-person.listener.ts | 83 ++++---- ...e-participant-workspace-member.listener.ts | 79 +++---- .../listeners/message-participant.listener.ts | 83 ++++---- .../create-audit-log-from-internal-event.ts | 59 +++--- ...meline-activity-from-internal-event.job.ts | 57 +++--- .../services/timeline-activity.service.ts | 16 +- .../database-event-trigger.listener.ts | 38 ++-- .../src/queue-worker/queue-worker.module.ts | 6 +- 41 files changed, 1004 insertions(+), 721 deletions(-) delete mode 100644 packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-job-data.ts create mode 100644 packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.module.ts create mode 100644 packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.ts create mode 100644 packages/twenty-server/src/engine/workspace-event-emitter/workspace-event.type.ts create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts index 6c7d6e066d24..1804901ac1d5 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts @@ -8,6 +8,7 @@ import { objectRecordChangedValues } from 'src/engine/integrations/event-emitter import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { CreateAuditLogFromInternalEvent } from 'src/modules/timeline/jobs/create-audit-log-from-internal-event'; import { UpsertTimelineActivityFromInternalEvent } from 'src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job'; @@ -19,40 +20,46 @@ export class EntityEventsToDbListener { ) {} @OnEvent('*.created') - async handleCreate(payload: ObjectRecordCreateEvent) { + async handleCreate( + payload: WorkspaceEventBatch>, + ) { return this.handle(payload); } @OnEvent('*.updated') - async handleUpdate(payload: ObjectRecordUpdateEvent) { - payload.properties.diff = objectRecordChangedValues( - payload.properties.before, - payload.properties.after, - payload.properties.updatedFields, - payload.objectMetadata, - ); + async handleUpdate( + payload: WorkspaceEventBatch>, + ) { + for (const eventPayload of payload.events) { + eventPayload.properties.diff = objectRecordChangedValues( + eventPayload.properties.before, + eventPayload.properties.after, + eventPayload.properties.updatedFields, + eventPayload.objectMetadata, + ); + } return this.handle(payload); } @OnEvent('*.deleted') - async handleDelete(payload: ObjectRecordUpdateEvent) { + async handleDelete( + payload: WorkspaceEventBatch>, + ) { return this.handle(payload); } - private async handle(payload: ObjectRecordBaseEvent) { - if (!payload.objectMetadata?.isAuditLogged) { - return; - } - - this.messageQueueService.add( - CreateAuditLogFromInternalEvent.name, - payload, + private async handle(payload: WorkspaceEventBatch) { + payload.events = payload.events.filter( + (event) => event.objectMetadata?.isAuditLogged, ); - this.messageQueueService.add( - UpsertTimelineActivityFromInternalEvent.name, - payload, - ); + await this.messageQueueService.add< + WorkspaceEventBatch + >(CreateAuditLogFromInternalEvent.name, payload); + + await this.messageQueueService.add< + WorkspaceEventBatch + >(UpsertTimelineActivityFromInternalEvent.name, payload); } } diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts index 8550a39632b7..f9609794d7ac 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts @@ -4,6 +4,7 @@ import { OnEvent } from '@nestjs/event-emitter'; import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; @Injectable() export class TelemetryListener { @@ -13,36 +14,48 @@ export class TelemetryListener { ) {} @OnEvent('*.created') - async handleAllCreate(payload: ObjectRecordCreateEvent) { - await this.analyticsService.create( - { - type: 'track', - data: { - eventName: payload.name, - }, - }, - payload.userId, - payload.workspaceId, - '', // voluntarely not retrieving this - '', // to avoid slowing down - this.environmentService.get('SERVER_URL'), + async handleAllCreate( + payload: WorkspaceEventBatch>, + ) { + await Promise.all( + payload.events.map((eventPayload) => + this.analyticsService.create( + { + type: 'track', + data: { + eventName: payload.name, + }, + }, + eventPayload.userId, + payload.workspaceId, + '', // voluntarily not retrieving this + '', // to avoid slowing down + this.environmentService.get('SERVER_URL'), + ), + ), ); } @OnEvent('user.signup') - async handleUserSignup(payload: ObjectRecordCreateEvent) { - await this.analyticsService.create( - { - type: 'track', - data: { - eventName: 'user.signup', - }, - }, - payload.userId, - payload.workspaceId, - '', - '', - this.environmentService.get('SERVER_URL'), + async handleUserSignup( + payload: WorkspaceEventBatch>, + ) { + await Promise.all( + payload.events.map((eventPayload) => + this.analyticsService.create( + { + type: 'track', + data: { + eventName: 'user.signup', + }, + }, + eventPayload.userId, + payload.workspaceId, + '', + '', + this.environmentService.get('SERVER_URL'), + ), + ), ); } } diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts index 30f526a345ef..eb91900aea53 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts @@ -1,5 +1,4 @@ import { Injectable, Logger } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; import isEmpty from 'lodash.isempty'; import { DataSource } from 'typeorm'; @@ -55,6 +54,8 @@ import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global. import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; import { isQueryTimeoutError } from 'src/engine/utils/query-timeout.util'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; +import { isDefined } from 'src/utils/is-defined'; import { PGGraphQLMutation, @@ -78,7 +79,7 @@ export class WorkspaceQueryRunnerService { private readonly queryResultGettersFactory: QueryResultGettersFactory, @InjectMessageQueue(MessageQueue.webhookQueue) private readonly messageQueueService: MessageQueueService, - private readonly eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, private readonly workspaceQueryHookService: WorkspaceQueryHookService, private readonly environmentService: EnvironmentService, private readonly duplicateService: DuplicateService, @@ -304,18 +305,21 @@ export class WorkspaceQueryRunnerService { options, ); - parsedResults.forEach((record) => { - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.created`, { - name: `${objectMetadataItem.nameSingular}.created`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: record.id, - objectMetadata: objectMetadataItem, - properties: { - after: record, - }, - } satisfies ObjectRecordCreateEvent); - }); + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.created`, + parsedResults.map( + (record) => + ({ + userId: authContext.user?.id, + recordId: record.id, + objectMetadata: objectMetadataItem, + properties: { + after: record, + }, + }) satisfies ObjectRecordCreateEvent, + ), + authContext.workspace.id, + ); return parsedResults; } @@ -440,18 +444,22 @@ export class WorkspaceQueryRunnerService { options, ); - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.updated`, { - name: `${objectMetadataItem.nameSingular}.updated`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: existingRecord.id, - objectMetadata: objectMetadataItem, - properties: { - updatedFields: Object.keys(args.data), - before: this.removeNestedProperties(existingRecord as Record), - after: this.removeNestedProperties(parsedResults?.[0]), - }, - } satisfies ObjectRecordUpdateEvent); + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.updated`, + [ + { + userId: authContext.user?.id, + recordId: existingRecord.id, + objectMetadata: objectMetadataItem, + properties: { + updatedFields: Object.keys(args.data), + before: this.removeNestedProperties(existingRecord as Record), + after: this.removeNestedProperties(parsedResults?.[0]), + }, + } satisfies ObjectRecordUpdateEvent, + ], + authContext.workspace.id, + ); return parsedResults?.[0]; } @@ -513,30 +521,36 @@ export class WorkspaceQueryRunnerService { options, ); - parsedResults.forEach((record) => { - const existingRecord = mappedRecords.get(record.id); + const eventsToEmit: ObjectRecordUpdateEvent[] = parsedResults + .map((record) => { + const existingRecord = mappedRecords.get(record.id); - if (!existingRecord) { - this.logger.warn( - `Record with id ${record.id} not found in the database`, - ); + if (!existingRecord) { + this.logger.warn( + `Record with id ${record.id} not found in the database`, + ); - return; - } + return; + } - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.updated`, { - name: `${objectMetadataItem.nameSingular}.updated`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: existingRecord.id, - objectMetadata: objectMetadataItem, - properties: { - updatedFields: Object.keys(args.data), - before: this.removeNestedProperties(existingRecord as Record), - after: this.removeNestedProperties(record), - }, - } satisfies ObjectRecordUpdateEvent); - }); + return { + userId: authContext.user?.id, + recordId: existingRecord.id, + objectMetadata: objectMetadataItem, + properties: { + updatedFields: Object.keys(args.data), + before: this.removeNestedProperties(existingRecord as Record), + after: this.removeNestedProperties(record), + }, + }; + }) + .filter(isDefined); + + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.updated`, + eventsToEmit, + authContext.workspace.id, + ); return parsedResults; } @@ -602,18 +616,21 @@ export class WorkspaceQueryRunnerService { options, ); - parsedResults.forEach((record) => { - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, { - name: `${objectMetadataItem.nameSingular}.deleted`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: record.id, - objectMetadata: objectMetadataItem, - properties: { - before: this.removeNestedProperties(record), - }, - } satisfies ObjectRecordDeleteEvent); - }); + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.deleted`, + parsedResults.map( + (record) => + ({ + userId: authContext.user?.id, + recordId: record.id, + objectMetadata: objectMetadataItem, + properties: { + before: this.removeNestedProperties(record), + }, + }) satisfies ObjectRecordDeleteEvent, + ), + authContext.workspace.id, + ); return parsedResults; } @@ -744,18 +761,21 @@ export class WorkspaceQueryRunnerService { options, ); - parsedResults.forEach((record) => { - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.created`, { - name: `${objectMetadataItem.nameSingular}.created`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: record.id, - objectMetadata: objectMetadataItem, - properties: { - after: this.removeNestedProperties(record), - }, - } satisfies ObjectRecordCreateEvent); - }); + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.created`, + parsedResults.map( + (record) => + ({ + userId: authContext.user?.id, + recordId: record.id, + objectMetadata: objectMetadataItem, + properties: { + after: this.removeNestedProperties(record), + }, + }) satisfies ObjectRecordCreateEvent, + ), + authContext.workspace.id, + ); return parsedResults; } @@ -821,19 +841,23 @@ export class WorkspaceQueryRunnerService { options, ); - this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, { - name: `${objectMetadataItem.nameSingular}.deleted`, - workspaceId: authContext.workspace.id, - userId: authContext.user?.id, - recordId: args.id, - objectMetadata: objectMetadataItem, - properties: { - before: { - ...(existingRecord ?? {}), - ...this.removeNestedProperties(parsedResults?.[0]), - }, - }, - } satisfies ObjectRecordDeleteEvent); + this.workspaceEventEmitter.emit( + `${objectMetadataItem.nameSingular}.deleted`, + [ + { + userId: authContext.user?.id, + recordId: args.id, + objectMetadata: objectMetadataItem, + properties: { + before: { + ...(existingRecord ?? {}), + ...this.removeNestedProperties(parsedResults?.[0]), + }, + }, + } satisfies ObjectRecordDeleteEvent, + ], + authContext.workspace.id, + ); return parsedResults?.[0]; } diff --git a/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts b/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts index d318b7277da7..0d6f03ef49a8 100644 --- a/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts +++ b/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts @@ -10,6 +10,7 @@ import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/t import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @Injectable() @@ -23,7 +24,9 @@ export class BillingWorkspaceMemberListener { @OnEvent('workspaceMember.created') @OnEvent('workspaceMember.deleted') async handleCreateOrDeleteEvent( - payload: ObjectRecordCreateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { if (!this.environmentService.get('IS_BILLING_ENABLED')) { return; diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index 11e07c91da4d..37b9ac882052 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -13,6 +13,7 @@ import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-cred import { UserModule } from 'src/engine/core-modules/user/user.module'; import { WorkflowTriggerCoreModule } from 'src/engine/core-modules/workflow/core-workflow-trigger.module'; import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; +import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; import { AnalyticsModule } from './analytics/analytics.module'; import { ClientConfigModule } from './client-config/client-config.module'; @@ -36,6 +37,7 @@ import { FileModule } from './file/file.module'; AISQLQueryModule, PostgresCredentialsModule, WorkflowTriggerCoreModule, + WorkspaceEventEmitterModule, ], exports: [ AnalyticsModule, diff --git a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts index 671977148d15..cc2727b3a118 100644 --- a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts @@ -1,5 +1,4 @@ /* eslint-disable @nx/workspace-inject-workspace-repository */ -import { EventEmitter2 } from '@nestjs/event-emitter'; import { InjectRepository } from '@nestjs/typeorm'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; @@ -11,6 +10,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; import { assert } from 'src/utils/assert'; @@ -22,7 +22,7 @@ export class UserWorkspaceService extends TypeOrmQueryService { private readonly userRepository: Repository, private readonly dataSourceService: DataSourceService, private readonly typeORMService: TypeORMService, - private eventEmitter: EventEmitter2, + private workspaceEventEmitter: WorkspaceEventEmitter, ) { super(userWorkspaceRepository); } @@ -35,11 +35,9 @@ export class UserWorkspaceService extends TypeOrmQueryService { const payload = new ObjectRecordCreateEvent(); - payload.workspaceId = workspaceId; payload.userId = userId; - payload.name = 'user.signup'; - this.eventEmitter.emit('user.signup', payload); + this.workspaceEventEmitter.emit('user.signup', [payload], workspaceId); return this.userWorkspaceRepository.save(userWorkspace); } @@ -76,14 +74,16 @@ export class UserWorkspaceService extends TypeOrmQueryService { const payload = new ObjectRecordCreateEvent(); - payload.workspaceId = workspaceId; payload.properties = { after: workspaceMember[0], }; payload.recordId = workspaceMember[0].id; - payload.name = 'workspaceMember.created'; - this.eventEmitter.emit('workspaceMember.created', payload); + this.workspaceEventEmitter.emit( + 'workspaceMember.created', + [payload], + workspaceId, + ); } async addUserToWorkspace(user: User, workspace: Workspace) { diff --git a/packages/twenty-server/src/engine/core-modules/user/services/__tests__/user.service.spec.ts b/packages/twenty-server/src/engine/core-modules/user/services/__tests__/user.service.spec.ts index 1773f249c7e4..24b5b23bfed9 100644 --- a/packages/twenty-server/src/engine/core-modules/user/services/__tests__/user.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/user/services/__tests__/user.service.spec.ts @@ -1,14 +1,14 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { UserService } from 'src/engine/core-modules/user/services/user.service'; -import { User } from 'src/engine/core-modules/user/user.entity'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; +import { UserService } from 'src/engine/core-modules/user/services/user.service'; +import { User } from 'src/engine/core-modules/user/user.entity'; import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; describe('UserService', () => { let service: UserService; @@ -34,7 +34,7 @@ describe('UserService', () => { useValue: {}, }, { - provide: EventEmitter2, + provide: WorkspaceEventEmitter, useValue: {}, }, { diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts index e8aef7fe899b..3ee3636ee270 100644 --- a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts @@ -1,4 +1,3 @@ -import { EventEmitter2 } from '@nestjs/event-emitter'; import { InjectRepository } from '@nestjs/typeorm'; import assert from 'assert'; @@ -16,6 +15,7 @@ import { import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; // eslint-disable-next-line @nx/workspace-inject-workspace-repository @@ -25,7 +25,7 @@ export class UserService extends TypeOrmQueryService { private readonly userRepository: Repository, private readonly dataSourceService: DataSourceService, private readonly typeORMService: TypeORMService, - private readonly eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, private readonly workspaceService: WorkspaceService, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) { @@ -110,15 +110,16 @@ export class UserService extends TypeOrmQueryService { const payload = new ObjectRecordDeleteEvent(); - payload.workspaceId = workspaceId; payload.properties = { before: workspaceMember, }; - payload.name = 'workspaceMember.deleted'; payload.recordId = workspaceMember.id; - payload.name = 'workspaceMember.deleted'; - this.eventEmitter.emit('workspaceMember.deleted', payload); + this.workspaceEventEmitter.emit( + 'workspaceMember.deleted', + [payload], + workspaceId, + ); return user; } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace-workspace-member.listener.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace-workspace-member.listener.ts index 4236e3a3fb79..fbc01935d467 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace-workspace-member.listener.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace-workspace-member.listener.ts @@ -11,6 +11,7 @@ import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/t import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @Injectable() @@ -23,39 +24,51 @@ export class WorkspaceWorkspaceMemberListener { @OnEvent('workspaceMember.updated') async handleUpdateEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, ) { - const { firstName: firstNameAfter, lastName: lastNameAfter } = - payload.properties.after.name; - - if (firstNameAfter === '' && lastNameAfter === '') { - return; - } - - if (!payload.userId) { - return; - } - - await this.onboardingService.setOnboardingCreateProfilePending({ - userId: payload.userId, - workspaceId: payload.workspaceId, - value: false, - }); + await Promise.all( + payload.events.map((eventPayload) => { + const { firstName: firstNameAfter, lastName: lastNameAfter } = + eventPayload.properties.after.name; + + if (firstNameAfter === '' && lastNameAfter === '') { + return; + } + + if (!eventPayload.userId) { + return; + } + + return this.onboardingService.setOnboardingCreateProfilePending({ + userId: eventPayload.userId, + workspaceId: payload.workspaceId, + value: false, + }); + }), + ); } @OnEvent('workspaceMember.deleted') async handleDeleteEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - const userId = payload.properties.before.userId; + await Promise.all( + payload.events.map((eventPayload) => { + const userId = eventPayload.properties.before.userId; - if (!userId) { - return; - } + if (!userId) { + return; + } - await this.messageQueueService.add( - HandleWorkspaceMemberDeletedJob.name, - { workspaceId: payload.workspaceId, userId }, + return this.messageQueueService.add( + HandleWorkspaceMemberDeletedJob.name, + { workspaceId: payload.workspaceId, userId }, + ); + }), ); } } diff --git a/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-job-data.ts b/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-job-data.ts deleted file mode 100644 index d1f7aa5ec0ef..000000000000 --- a/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-job-data.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event'; - -export class ObjectRecordJobData extends ObjectRecordBaseEvent { - getOperation() { - return this.name.split('.')[1]; - } - - getObjectName() { - return this.name.split('.')[0]; - } -} diff --git a/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record.base.event.ts b/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record.base.event.ts index d34fbd1afcc5..e515c130928f 100644 --- a/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record.base.event.ts +++ b/packages/twenty-server/src/engine/integrations/event-emitter/types/object-record.base.event.ts @@ -1,11 +1,14 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; export class ObjectRecordBaseEvent { - name: string; - workspaceId: string; recordId: string; userId?: string; workspaceMemberId?: string; objectMetadata: ObjectMetadataInterface; properties: any; } + +export class ObjectRecordBaseEventWithNameAndWorkspaceId extends ObjectRecordBaseEvent { + name: string; + workspaceId: string; +} diff --git a/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.module.ts b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.module.ts new file mode 100644 index 000000000000..f81fe0fee4ad --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.module.ts @@ -0,0 +1,11 @@ +import { Global, Module } from '@nestjs/common'; + +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; + +@Global() +@Module({ + imports: [], + providers: [WorkspaceEventEmitter], + exports: [WorkspaceEventEmitter], +}) +export class WorkspaceEventEmitterModule {} diff --git a/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.ts b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.ts new file mode 100644 index 000000000000..374bc07c54cb --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event-emitter.ts @@ -0,0 +1,21 @@ +import { Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; + +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; + +@Injectable() +export class WorkspaceEventEmitter { + constructor(private readonly eventEmitter: EventEmitter2) {} + + public emit(eventName: string, events: any[], workspaceId: string) { + if (!events.length) { + return; + } + + return this.eventEmitter.emit(eventName, { + name: eventName, + workspaceId, + events, + } satisfies WorkspaceEventBatch); + } +} diff --git a/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event.type.ts b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event.type.ts new file mode 100644 index 000000000000..623c523f280c --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-event-emitter/workspace-event.type.ts @@ -0,0 +1,5 @@ +export type WorkspaceEventBatch = { + name: string; + workspaceId: string; + events: WorkspaceEvent[]; +}; diff --git a/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts b/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts index 9c1c54c5fab3..7a2b7e0e8389 100644 --- a/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts @@ -7,6 +7,7 @@ import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/t import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { BlocklistItemDeleteCalendarEventsJob, @@ -26,48 +27,74 @@ export class CalendarBlocklistListener { @OnEvent('blocklist.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { - await this.messageQueueService.add( - BlocklistItemDeleteCalendarEventsJob.name, - { - workspaceId: payload.workspaceId, - blocklistItemId: payload.recordId, - }, + await Promise.all( + payload.events.map((eventPayload) => + this.messageQueueService.add( + BlocklistItemDeleteCalendarEventsJob.name, + { + workspaceId: payload.workspaceId, + blocklistItemId: eventPayload.recordId, + }, + ), + ), ); } @OnEvent('blocklist.deleted') async handleDeletedEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - await this.messageQueueService.add( - BlocklistReimportCalendarEventsJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.before.workspaceMember.id, - }, + await Promise.all( + payload.events.map((eventPayload) => + this.messageQueueService.add( + BlocklistReimportCalendarEventsJob.name, + { + workspaceId: payload.workspaceId, + workspaceMemberId: + eventPayload.properties.before.workspaceMember.id, + }, + ), + ), ); } @OnEvent('blocklist.updated') async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, ) { - await this.messageQueueService.add( - BlocklistItemDeleteCalendarEventsJob.name, - { - workspaceId: payload.workspaceId, - blocklistItemId: payload.recordId, - }, - ); + await Promise.all( + payload.events.reduce((acc: Promise[], eventPayload) => { + acc.push( + this.messageQueueService.add( + BlocklistItemDeleteCalendarEventsJob.name, + { + workspaceId: payload.workspaceId, + blocklistItemId: eventPayload.recordId, + }, + ), + ); + + acc.push( + this.messageQueueService.add( + BlocklistReimportCalendarEventsJob.name, + { + workspaceId: payload.workspaceId, + workspaceMemberId: + eventPayload.properties.after.workspaceMember.id, + }, + ), + ); - await this.messageQueueService.add( - BlocklistReimportCalendarEventsJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.after.workspaceMember.id, - }, + return acc; + }, []), ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module.ts index 50da2766148f..bb9af960eb44 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module.ts @@ -1,15 +1,15 @@ import { Module } from '@nestjs/common'; -import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/calendar/calendar-event-cleaner/jobs/delete-connected-account-associated-calendar-data.job'; +import { CalendarEventCleanerConnectedAccountListener } from 'src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener'; import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service'; -import { CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity'; @Module({ - imports: [TwentyORMModule.forFeature([CalendarEventWorkspaceEntity])], + imports: [], providers: [ CalendarEventCleanerService, DeleteConnectedAccountAssociatedCalendarDataJob, + CalendarEventCleanerConnectedAccountListener, ], exports: [CalendarEventCleanerService], }) diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts new file mode 100644 index 000000000000..0c14b6b60655 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts @@ -0,0 +1,40 @@ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; + +import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; +import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; +import { + DeleteConnectedAccountAssociatedCalendarDataJob, + DeleteConnectedAccountAssociatedCalendarDataJobData, +} from 'src/modules/calendar/calendar-event-cleaner/jobs/delete-connected-account-associated-calendar-data.job'; +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; + +@Injectable() +export class CalendarEventCleanerConnectedAccountListener { + constructor( + @InjectMessageQueue(MessageQueue.calendarQueue) + private readonly calendarQueueService: MessageQueueService, + ) {} + + @OnEvent('connectedAccount.deleted') + async handleDeletedEvent( + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, + ) { + await Promise.all( + payload.events.map((eventPayload) => + this.calendarQueueService.add( + DeleteConnectedAccountAssociatedCalendarDataJob.name, + { + workspaceId: payload.workspaceId, + connectedAccountId: eventPayload.recordId, + }, + ), + ), + ); + } +} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts index e67d2c498394..3f22e0c781ac 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts @@ -1,12 +1,13 @@ import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; import { Any } from 'typeorm'; import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { injectIdsInCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/inject-ids-in-calendar-events.util'; import { CalendarEventParticipantService } from 'src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service'; import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity'; @@ -19,7 +20,6 @@ import { CreateCompanyAndContactJob, CreateCompanyAndContactJobData, } from 'src/modules/contact-creation-manager/jobs/create-company-and-contact.job'; -import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type'; @Injectable() export class CalendarSaveEventsService { @@ -28,7 +28,7 @@ export class CalendarSaveEventsService { private readonly calendarEventParticipantService: CalendarEventParticipantService, @InjectMessageQueue(MessageQueue.contactCreationQueue) private readonly messageQueueService: MessageQueueService, - private readonly eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, ) {} public async saveCalendarEventsAndEnqueueContactCreationJob( @@ -140,13 +140,6 @@ export class CalendarSaveEventsService { ); }); - this.eventEmitter.emit(`calendarEventParticipant.matched`, { - workspaceId, - name: 'calendarEventParticipant.matched', - workspaceMemberId: connectedAccount.accountOwnerId, - calendarEventParticipants: savedCalendarEventParticipantsToEmit, - }); - if (calendarChannel.isContactAutoCreationEnabled) { await this.messageQueueService.add( CreateCompanyAndContactJob.name, diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts index 7b369af572d6..a48b842b94cd 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts @@ -7,13 +7,14 @@ import { objectRecordChangedProperties as objectRecordUpdateEventChangedProperti import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { - CalendarEventParticipantMatchParticipantJobData, CalendarEventParticipantMatchParticipantJob, + CalendarEventParticipantMatchParticipantJobData, } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-event-participant-match-participant.job'; import { - CalendarEventParticipantUnmatchParticipantJobData, CalendarEventParticipantUnmatchParticipantJob, + CalendarEventParticipantUnmatchParticipantJobData, } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-event-participant-unmatch-participant.job'; import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity'; @@ -26,49 +27,59 @@ export class CalendarEventParticipantPersonListener { @OnEvent('person.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, - ) { - if (payload.properties.after.email === null) { - return; - } - - await this.messageQueueService.add( - CalendarEventParticipantMatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.after.email, - personId: payload.recordId, - }, - ); - } - - @OnEvent('person.updated') - async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { - if ( - objectRecordUpdateEventChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('email') - ) { - await this.messageQueueService.add( - CalendarEventParticipantUnmatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.before.email, - personId: payload.recordId, - }, - ); + for (const eventPayload of payload.events) { + if (eventPayload.properties.after.email === null) { + continue; + } + // TODO: modify this job to take an array of participants to match await this.messageQueueService.add( CalendarEventParticipantMatchParticipantJob.name, { workspaceId: payload.workspaceId, - email: payload.properties.after.email, - personId: payload.recordId, + email: eventPayload.properties.after.email, + personId: eventPayload.recordId, }, ); } } + + @OnEvent('person.updated') + async handleUpdatedEvent( + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, + ) { + for (const eventPayload of payload.events) { + if ( + objectRecordUpdateEventChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('email') + ) { + // TODO: modify this job to take an array of participants to match + await this.messageQueueService.add( + CalendarEventParticipantUnmatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.before.email, + personId: eventPayload.recordId, + }, + ); + + await this.messageQueueService.add( + CalendarEventParticipantMatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.after.email, + personId: eventPayload.recordId, + }, + ); + } + } + } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts index 1f84fce0dc2f..ab02392c52c5 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts @@ -7,13 +7,14 @@ import { objectRecordChangedProperties as objectRecordUpdateEventChangedProperti import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { CalendarEventParticipantMatchParticipantJob, CalendarEventParticipantMatchParticipantJobData, } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-event-participant-match-participant.job'; import { - CalendarEventParticipantUnmatchParticipantJobData, CalendarEventParticipantUnmatchParticipantJob, + CalendarEventParticipantUnmatchParticipantJobData, } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-event-participant-unmatch-participant.job'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -26,49 +27,57 @@ export class CalendarEventParticipantWorkspaceMemberListener { @OnEvent('workspaceMember.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, - ) { - if (payload.properties.after.userEmail === null) { - return; - } - - await this.messageQueueService.add( - CalendarEventParticipantMatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.after.userEmail, - workspaceMemberId: payload.properties.after.id, - }, - ); - } - - @OnEvent('workspaceMember.updated') - async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { - if ( - objectRecordUpdateEventChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('userEmail') - ) { - await this.messageQueueService.add( - CalendarEventParticipantUnmatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.before.userEmail, - personId: payload.recordId, - }, - ); + for (const eventPayload of payload.events) { + if (eventPayload.properties.after.userEmail === null) { + continue; + } await this.messageQueueService.add( CalendarEventParticipantMatchParticipantJob.name, { workspaceId: payload.workspaceId, - email: payload.properties.after.userEmail, - workspaceMemberId: payload.recordId, + email: eventPayload.properties.after.userEmail, + workspaceMemberId: eventPayload.recordId, }, ); } } + + @OnEvent('workspaceMember.updated') + async handleUpdatedEvent( + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, + ) { + for (const eventPayload of payload.events) { + if ( + objectRecordUpdateEventChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('userEmail') + ) { + await this.messageQueueService.add( + CalendarEventParticipantUnmatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.before.userEmail, + personId: eventPayload.recordId, + }, + ); + + await this.messageQueueService.add( + CalendarEventParticipantMatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.after.userEmail, + workspaceMemberId: eventPayload.recordId, + }, + ); + } + } + } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant.listener.ts index d75d400ecee6..72152cdc36f2 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant.listener.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant.listener.ts @@ -7,6 +7,7 @@ import { Repository } from 'typeorm'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; import { TimelineActivityRepository } from 'src/modules/timeline/repositiories/timeline-activity.repository'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; @@ -22,49 +23,55 @@ export class CalendarEventParticipantListener { ) {} @OnEvent('calendarEventParticipant.matched') - public async handleCalendarEventParticipantMatchedEvent(payload: { - workspaceId: string; - workspaceMemberId: string; - participants: CalendarEventParticipantWorkspaceEntity[]; - }): Promise { - const calendarEventParticipants = payload.participants ?? []; + public async handleCalendarEventParticipantMatchedEvent( + payload: WorkspaceEventBatch<{ + workspaceMemberId: string; + participants: CalendarEventParticipantWorkspaceEntity[]; + }>, + ): Promise { + const workspaceId = payload.workspaceId; - // TODO: move to a job? + // TODO: Refactor to insertTimelineActivitiesForObject once + for (const eventPayload of payload.events) { + const calendarEventParticipants = eventPayload.participants; + const workspaceMemberId = eventPayload.workspaceMemberId; - const dataSourceSchema = this.workspaceDataSourceService.getSchemaName( - payload.workspaceId, - ); + // TODO: move to a job? - const calendarEventObjectMetadata = - await this.objectMetadataRepository.findOneOrFail({ - where: { - nameSingular: 'calendarEvent', - workspaceId: payload.workspaceId, - }, - }); + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); - const calendarEventParticipantsWithPersonId = - calendarEventParticipants.filter((participant) => participant.personId); + const calendarEventObjectMetadata = + await this.objectMetadataRepository.findOneOrFail({ + where: { + nameSingular: 'calendarEvent', + workspaceId, + }, + }); - if (calendarEventParticipantsWithPersonId.length === 0) { - return; - } + const calendarEventParticipantsWithPersonId = + calendarEventParticipants.filter((participant) => participant.personId); + + if (calendarEventParticipantsWithPersonId.length === 0) { + continue; + } - await this.timelineActivityRepository.insertTimelineActivitiesForObject( - 'person', - calendarEventParticipantsWithPersonId.map((participant) => ({ - dataSourceSchema, - name: 'calendarEvent.linked', - properties: null, - objectName: 'calendarEvent', - recordId: participant.personId, - workspaceMemberId: payload.workspaceMemberId, - workspaceId: payload.workspaceId, - linkedObjectMetadataId: calendarEventObjectMetadata.id, - linkedRecordId: participant.calendarEventId, - linkedRecordCachedName: '', - })), - payload.workspaceId, - ); + await this.timelineActivityRepository.insertTimelineActivitiesForObject( + 'person', + calendarEventParticipantsWithPersonId.map((participant) => ({ + dataSourceSchema, + name: 'calendarEvent.linked', + properties: null, + objectName: 'calendarEvent', + recordId: participant.personId, + workspaceMemberId, + workspaceId, + linkedObjectMetadataId: calendarEventObjectMetadata.id, + linkedRecordId: participant.calendarEventId, + linkedRecordCachedName: '', + })), + workspaceId, + ); + } } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts index a704e6d29295..cc71ebf8282a 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts @@ -111,7 +111,7 @@ export class CalendarEventParticipantService { await this.matchParticipantService.matchParticipants( savedParticipants, - 'messageParticipant', + 'calendarEventParticipant', transactionManager, ); } diff --git a/packages/twenty-server/src/modules/connected-account/listeners/connected-account.listener.ts b/packages/twenty-server/src/modules/connected-account/listeners/connected-account.listener.ts index 0c9b78843ce1..455721d817f6 100644 --- a/packages/twenty-server/src/modules/connected-account/listeners/connected-account.listener.ts +++ b/packages/twenty-server/src/modules/connected-account/listeners/connected-account.listener.ts @@ -3,6 +3,7 @@ import { OnEvent } from '@nestjs/event-emitter'; import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -16,27 +17,31 @@ export class ConnectedAccountListener { @OnEvent('connectedAccount.deleted') async handleDeletedEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - const workspaceMemberId = payload.properties.before.accountOwnerId; - const workspaceId = payload.workspaceId; - const workspaceMemberRepository = - await this.twentyORMGlobalManager.getRepositoryForWorkspace( - workspaceId, - 'workspaceMember', - ); - const workspaceMember = await workspaceMemberRepository.findOneOrFail({ - where: { id: workspaceMemberId }, - }); + for (const eventPayload of payload.events) { + const workspaceMemberId = eventPayload.properties.before.accountOwnerId; + const workspaceId = payload.workspaceId; + const workspaceMemberRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'workspaceMember', + ); + const workspaceMember = await workspaceMemberRepository.findOneOrFail({ + where: { id: workspaceMemberId }, + }); - const userId = workspaceMember.userId; + const userId = workspaceMember.userId; - const connectedAccountId = payload.properties.before.id; + const connectedAccountId = eventPayload.properties.before.id; - await this.accountsToReconnectService.removeAccountToReconnect( - userId, - workspaceId, - connectedAccountId, - ); + await this.accountsToReconnectService.removeAccountToReconnect( + userId, + workspaceId, + connectedAccountId, + ); + } } } diff --git a/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-delete-one.pre-query.hook.ts b/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-delete-one.pre-query.hook.ts index 7a730431acf7..0812c0e33ea0 100644 --- a/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-delete-one.pre-query.hook.ts +++ b/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-delete-one.pre-query.hook.ts @@ -1,5 +1,3 @@ -import { EventEmitter2 } from '@nestjs/event-emitter'; - import { WorkspaceQueryHookInstance } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/interfaces/workspace-query-hook.interface'; import { DeleteOneResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; @@ -7,6 +5,7 @@ import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runne import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @WorkspaceQueryHook(`connectedAccount.deleteOne`) @@ -15,7 +14,7 @@ export class ConnectedAccountDeleteOnePreQueryHook { constructor( private readonly twentyORMManager: TwentyORMManager, - private eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, ) {} async execute( @@ -34,16 +33,19 @@ export class ConnectedAccountDeleteOnePreQueryHook connectedAccountId, }); - messageChannels.forEach((messageChannel) => { - this.eventEmitter.emit('messageChannel.deleted', { - workspaceId: authContext.workspace.id, - name: 'messageChannel.deleted', - recordId: messageChannel.id, - } satisfies Pick< - ObjectRecordDeleteEvent, - 'workspaceId' | 'recordId' | 'name' - >); - }); + this.workspaceEventEmitter.emit( + 'messageChannel.deleted', + messageChannels.map( + (messageChannel) => + ({ + recordId: messageChannel.id, + }) satisfies Pick< + ObjectRecordDeleteEvent, + 'recordId' + >, + ), + authContext.workspace.id, + ); return payload; } diff --git a/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-query-hook.module.ts b/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-query-hook.module.ts index 57aaa6408be0..225a1c3bebbf 100644 --- a/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-query-hook.module.ts +++ b/packages/twenty-server/src/modules/connected-account/query-hooks/connected-account-query-hook.module.ts @@ -1,11 +1,9 @@ import { Module } from '@nestjs/common'; -import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; import { ConnectedAccountDeleteOnePreQueryHook } from 'src/modules/connected-account/query-hooks/connected-account-delete-one.pre-query.hook'; -import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @Module({ - imports: [TwentyORMModule.forFeature([MessageChannelWorkspaceEntity])], + imports: [], providers: [ConnectedAccountDeleteOnePreQueryHook], }) export class ConnectedAccountQueryHookModule {} diff --git a/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener.ts b/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener.ts index 51104de273c4..5dd7497d009b 100644 --- a/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener.ts +++ b/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener.ts @@ -6,9 +6,10 @@ import { objectRecordChangedProperties } from 'src/engine/integrations/event-emi import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { - CalendarCreateCompanyAndContactAfterSyncJobData, CalendarCreateCompanyAndContactAfterSyncJob, + CalendarCreateCompanyAndContactAfterSyncJobData, } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job'; import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @@ -21,22 +22,28 @@ export class AutoCompaniesAndContactsCreationCalendarChannelListener { @OnEvent('calendarChannel.updated') async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, ) { - if ( - objectRecordChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('isContactAutoCreationEnabled') && - payload.properties.after.isContactAutoCreationEnabled - ) { - await this.messageQueueService.add( - CalendarCreateCompanyAndContactAfterSyncJob.name, - { - workspaceId: payload.workspaceId, - calendarChannelId: payload.recordId, - }, - ); - } + await Promise.all( + payload.events.map((eventPayload) => { + if ( + objectRecordChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('isContactAutoCreationEnabled') && + eventPayload.properties.after.isContactAutoCreationEnabled + ) { + return this.messageQueueService.add( + CalendarCreateCompanyAndContactAfterSyncJob.name, + { + workspaceId: payload.workspaceId, + calendarChannelId: eventPayload.recordId, + }, + ); + } + }), + ); } } diff --git a/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts b/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts index f1d6362d45a9..ac2511f745c0 100644 --- a/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts +++ b/packages/twenty-server/src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts @@ -6,10 +6,11 @@ import { objectRecordChangedProperties } from 'src/engine/integrations/event-emi import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { - MessagingCreateCompanyAndContactAfterSyncJobData, MessagingCreateCompanyAndContactAfterSyncJob, + MessagingCreateCompanyAndContactAfterSyncJobData, } from 'src/modules/messaging/message-participant-manager/jobs/messaging-create-company-and-contact-after-sync.job'; @Injectable() @@ -21,22 +22,28 @@ export class AutoCompaniesAndContactsCreationMessageChannelListener { @OnEvent('messageChannel.updated') async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, ) { - if ( - objectRecordChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('isContactAutoCreationEnabled') && - payload.properties.after.isContactAutoCreationEnabled - ) { - await this.messageQueueService.add( - MessagingCreateCompanyAndContactAfterSyncJob.name, - { - workspaceId: payload.workspaceId, - messageChannelId: payload.recordId, - }, - ); - } + await Promise.all( + payload.events.map((eventPayload) => { + if ( + objectRecordChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('isContactAutoCreationEnabled') && + eventPayload.properties.after.isContactAutoCreationEnabled + ) { + return this.messageQueueService.add( + MessagingCreateCompanyAndContactAfterSyncJob.name, + { + workspaceId: payload.workspaceId, + messageChannelId: eventPayload.recordId, + }, + ); + } + }), + ); } } diff --git a/packages/twenty-server/src/modules/contact-creation-manager/services/create-company-and-contact.service.ts b/packages/twenty-server/src/modules/contact-creation-manager/services/create-company-and-contact.service.ts index 6435f0d65327..db6a6328bb29 100644 --- a/packages/twenty-server/src/modules/contact-creation-manager/services/create-company-and-contact.service.ts +++ b/packages/twenty-server/src/modules/contact-creation-manager/services/create-company-and-contact.service.ts @@ -1,5 +1,4 @@ import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; import { InjectRepository } from '@nestjs/typeorm'; import chunk from 'lodash.chunk'; @@ -11,6 +10,7 @@ import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/com import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/contact-creation-manager/constants/contacts-creation-batch-size.constant'; @@ -32,7 +32,7 @@ export class CreateCompanyAndContactService { private readonly createCompaniesService: CreateCompanyService, @InjectObjectMetadataRepository(WorkspaceMemberWorkspaceEntity) private readonly workspaceMemberRepository: WorkspaceMemberRepository, - private readonly eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, @@ -191,18 +191,21 @@ export class CreateCompanyAndContactService { source, ); - for (const createdPerson of createdPeople) { - this.eventEmitter.emit('person.created', { - name: 'person.created', - workspaceId, - // FixMe: TypeORM typing issue... id is always returned when using save - recordId: createdPerson.id as string, - objectMetadata, - properties: { - after: createdPerson, - }, - } satisfies ObjectRecordCreateEvent); - } + this.workspaceEventEmitter.emit( + 'person.created', + createdPeople.map( + (createdPerson) => + ({ + // FixMe: TypeORM typing issue... id is always returned when using save + recordId: createdPerson.id as string, + objectMetadata, + properties: { + after: createdPerson, + }, + }) satisfies ObjectRecordCreateEvent, + ), + workspaceId, + ); } } } diff --git a/packages/twenty-server/src/modules/match-participant/match-participant.service.ts b/packages/twenty-server/src/modules/match-participant/match-participant.service.ts index d7bba2e4b8e2..c5cd06f301f9 100644 --- a/packages/twenty-server/src/modules/match-participant/match-participant.service.ts +++ b/packages/twenty-server/src/modules/match-participant/match-participant.service.ts @@ -1,10 +1,10 @@ import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; import { Any } from 'typeorm'; import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity'; import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity'; @@ -17,7 +17,7 @@ export class MatchParticipantService< | MessageParticipantWorkspaceEntity, > { constructor( - private readonly eventEmitter: EventEmitter2, + private readonly workspaceEventEmitter: WorkspaceEventEmitter, private readonly twentyORMManager: TwentyORMManager, private readonly scopedWorkspaceContextFactory: ScopedWorkspaceContextFactory, ) {} @@ -46,6 +46,10 @@ export class MatchParticipantService< const workspaceId = this.scopedWorkspaceContextFactory.create().workspaceId; + if (!workspaceId) { + throw new Error('Workspace ID is required'); + } + const participantIds = participants.map((participant) => participant.id); const uniqueParticipantsHandles = [ ...new Set(participants.map((participant) => participant.handle)), @@ -109,11 +113,16 @@ export class MatchParticipantService< transactionManager, ); - this.eventEmitter.emit(`${objectMetadataName}.matched`, { + this.workspaceEventEmitter.emit( + `${objectMetadataName}.matched`, + [ + { + workspaceMemberId: null, + participants: matchedParticipants, + }, + ], workspaceId, - workspaceMemberId: null, - participants: matchedParticipants, - }); + ); } public async matchParticipantsAfterPersonOrWorkspaceMemberCreation( @@ -127,6 +136,10 @@ export class MatchParticipantService< const workspaceId = this.scopedWorkspaceContextFactory.create().workspaceId; + if (!workspaceId) { + throw new Error('Workspace ID is required'); + } + const participantsToUpdate = await participantRepository.find({ where: { handle, @@ -155,12 +168,18 @@ export class MatchParticipantService< }, }); - this.eventEmitter.emit(`${objectMetadataName}.matched`, { + this.workspaceEventEmitter.emit( + `${objectMetadataName}.matched`, + [ + { + workspaceId, + name: `${objectMetadataName}.matched`, + workspaceMemberId: null, + participants: updatedParticipants, + }, + ], workspaceId, - name: `${objectMetadataName}.matched`, - workspaceMemberId: null, - participants: updatedParticipants, - }); + ); } if (workspaceMemberId) { diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 996aca092466..c34bbe192f12 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -9,6 +9,7 @@ import { MessageQueue } from 'src/engine/integrations/message-queue/message-queu import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; @@ -32,90 +33,109 @@ export class MessagingBlocklistListener { @OnEvent('blocklist.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { - await this.messageQueueService.add( - BlocklistItemDeleteMessagesJob.name, - { - workspaceId: payload.workspaceId, - blocklistItemId: payload.recordId, - }, + await Promise.all( + payload.events.map((eventPayload) => + // TODO: modify to pass an array of blocklist items + this.messageQueueService.add( + BlocklistItemDeleteMessagesJob.name, + { + workspaceId: payload.workspaceId, + blocklistItemId: eventPayload.recordId, + }, + ), + ), ); } @OnEvent('blocklist.deleted') async handleDeletedEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - const workspaceMemberId = payload.properties.before.workspaceMember.id; const workspaceId = payload.workspaceId; - const connectedAccount = - await this.connectedAccountRepository.getAllByWorkspaceMemberId( - workspaceMemberId, + for (const eventPayload of payload.events) { + const workspaceMemberId = + eventPayload.properties.before.workspaceMember.id; + + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannelRepository = + await this.twentyORMManager.getRepository( + 'messageChannel', + ); + + const messageChannel = await messageChannelRepository.findOneOrFail({ + where: { + connectedAccountId: connectedAccount[0].id, + }, + }); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel.id, workspaceId, ); - - if (!connectedAccount || connectedAccount.length === 0) { - return; } - - const messageChannelRepository = - await this.twentyORMManager.getRepository( - 'messageChannel', - ); - - const messageChannel = await messageChannelRepository.findOneOrFail({ - where: { - connectedAccountId: connectedAccount[0].id, - }, - }); - - await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( - messageChannel.id, - workspaceId, - ); } @OnEvent('blocklist.updated') async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, ) { - const workspaceMemberId = payload.properties.before.workspaceMember.id; const workspaceId = payload.workspaceId; - await this.messageQueueService.add( - BlocklistItemDeleteMessagesJob.name, - { - workspaceId, - blocklistItemId: payload.recordId, - }, - ); + for (const eventPayload of payload.events) { + const workspaceMemberId = + eventPayload.properties.before.workspaceMember.id; - const connectedAccount = - await this.connectedAccountRepository.getAllByWorkspaceMemberId( - workspaceMemberId, - workspaceId, + await this.messageQueueService.add( + BlocklistItemDeleteMessagesJob.name, + { + workspaceId, + blocklistItemId: eventPayload.recordId, + }, ); - if (!connectedAccount || connectedAccount.length === 0) { - return; - } - - const messageChannelRepository = - await this.twentyORMManager.getRepository( - 'messageChannel', + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + continue; + } + + const messageChannelRepository = + await this.twentyORMManager.getRepository( + 'messageChannel', + ); + + const messageChannel = await messageChannelRepository.findOneOrFail({ + where: { + connectedAccountId: connectedAccount[0].id, + }, + }); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel.id, + workspaceId, ); - - const messageChannel = await messageChannelRepository.findOneOrFail({ - where: { - connectedAccountId: connectedAccount[0].id, - }, - }); - - await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( - messageChannel.id, - workspaceId, - ); + } } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts index 8880ad5d10d4..8caf65c58857 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts @@ -2,46 +2,39 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; +import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { MessagingConnectedAccountDeletionCleanupJob, MessagingConnectedAccountDeletionCleanupJobData, } from 'src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job'; -import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; -import { - DeleteConnectedAccountAssociatedCalendarDataJobData, - DeleteConnectedAccountAssociatedCalendarDataJob, -} from 'src/modules/calendar/calendar-event-cleaner/jobs/delete-connected-account-associated-calendar-data.job'; @Injectable() export class MessagingMessageCleanerConnectedAccountListener { constructor( @InjectMessageQueue(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, - @InjectMessageQueue(MessageQueue.calendarQueue) - private readonly calendarQueueService: MessageQueueService, ) {} @OnEvent('connectedAccount.deleted') async handleDeletedEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - await this.messageQueueService.add( - MessagingConnectedAccountDeletionCleanupJob.name, - { - workspaceId: payload.workspaceId, - connectedAccountId: payload.recordId, - }, - ); - - await this.calendarQueueService.add( - DeleteConnectedAccountAssociatedCalendarDataJob.name, - { - workspaceId: payload.workspaceId, - connectedAccountId: payload.recordId, - }, + await Promise.all( + payload.events.map((eventPayload) => + this.messageQueueService.add( + MessagingConnectedAccountDeletionCleanupJob.name, + { + workspaceId: payload.workspaceId, + connectedAccountId: eventPayload.recordId, + }, + ), + ), ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts index ae4221a85e85..7b25fa456ae3 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts @@ -1,19 +1,11 @@ import { Module } from '@nestjs/common'; -import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; -import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity'; -import { MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity'; import { MessagingConnectedAccountDeletionCleanupJob } from 'src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job'; import { MessagingMessageCleanerConnectedAccountListener } from 'src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; @Module({ - imports: [ - TwentyORMModule.forFeature([ - MessageWorkspaceEntity, - MessageThreadWorkspaceEntity, - ]), - ], + imports: [], providers: [ MessagingMessageCleanerService, MessagingConnectedAccountDeletionCleanupJob, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener.ts index d3f00c95fc42..b81118d3a20f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener.ts @@ -2,9 +2,10 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event'; +import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; -import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { MessagingCleanCacheJob, @@ -20,14 +21,20 @@ export class MessagingMessageImportManagerMessageChannelListener { @OnEvent('messageChannel.deleted') async handleDeletedEvent( - payload: ObjectRecordDeleteEvent, + payload: WorkspaceEventBatch< + ObjectRecordDeleteEvent + >, ) { - await this.messageQueueService.add( - MessagingCleanCacheJob.name, - { - workspaceId: payload.workspaceId, - messageChannelId: payload.recordId, - }, + await Promise.all( + payload.events.map((eventPayload) => + this.messageQueueService.add( + MessagingCleanCacheJob.name, + { + workspaceId: payload.workspaceId, + messageChannelId: eventPayload.recordId, + }, + ), + ), ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts index 521c92585054..2765c661129d 100644 --- a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts +++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts @@ -7,13 +7,14 @@ import { objectRecordChangedProperties as objectRecordUpdateEventChangedProperti import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { - MessageParticipantMatchParticipantJobData, MessageParticipantMatchParticipantJob, + MessageParticipantMatchParticipantJobData, } from 'src/modules/messaging/message-participant-manager/jobs/message-participant-match-participant.job'; import { - MessageParticipantUnmatchParticipantJobData, MessageParticipantUnmatchParticipantJob, + MessageParticipantUnmatchParticipantJobData, } from 'src/modules/messaging/message-participant-manager/jobs/message-participant-unmatch-participant.job'; import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity'; @@ -26,49 +27,57 @@ export class MessageParticipantPersonListener { @OnEvent('person.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, - ) { - if (payload.properties.after.email === null) { - return; - } - - await this.messageQueueService.add( - MessageParticipantMatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.after.email, - personId: payload.recordId, - }, - ); - } - - @OnEvent('person.updated') - async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { - if ( - objectRecordUpdateEventChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('email') - ) { - await this.messageQueueService.add( - MessageParticipantUnmatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.before.email, - personId: payload.recordId, - }, - ); + for (const eventPayload of payload.events) { + if (eventPayload.properties.after.email === null) { + continue; + } await this.messageQueueService.add( MessageParticipantMatchParticipantJob.name, { workspaceId: payload.workspaceId, - email: payload.properties.after.email, - personId: payload.recordId, + email: eventPayload.properties.after.email, + personId: eventPayload.recordId, }, ); } } + + @OnEvent('person.updated') + async handleUpdatedEvent( + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, + ) { + for (const eventPayload of payload.events) { + if ( + objectRecordUpdateEventChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('email') + ) { + await this.messageQueueService.add( + MessageParticipantUnmatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.before.email, + personId: eventPayload.recordId, + }, + ); + + await this.messageQueueService.add( + MessageParticipantMatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.after.email, + personId: eventPayload.recordId, + }, + ); + } + } + } } diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts index 506bf3d5aa18..07968f379cec 100644 --- a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts +++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts @@ -14,6 +14,7 @@ import { objectRecordChangedProperties as objectRecordUpdateEventChangedProperti import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { MessageParticipantMatchParticipantJob, MessageParticipantMatchParticipantJobData, @@ -35,7 +36,9 @@ export class MessageParticipantWorkspaceMemberListener { @OnEvent('workspaceMember.created') async handleCreatedEvent( - payload: ObjectRecordCreateEvent, + payload: WorkspaceEventBatch< + ObjectRecordCreateEvent + >, ) { const workspace = await this.workspaceRepository.findOneBy({ id: payload.workspaceId, @@ -48,47 +51,53 @@ export class MessageParticipantWorkspaceMemberListener { return; } - if (payload.properties.after.userEmail === null) { - return; - } - - await this.messageQueueService.add( - MessageParticipantMatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.after.userEmail, - workspaceMemberId: payload.properties.after.id, - }, - ); - } - - @OnEvent('workspaceMember.updated') - async handleUpdatedEvent( - payload: ObjectRecordUpdateEvent, - ) { - if ( - objectRecordUpdateEventChangedProperties( - payload.properties.before, - payload.properties.after, - ).includes('userEmail') - ) { - await this.messageQueueService.add( - MessageParticipantUnmatchParticipantJob.name, - { - workspaceId: payload.workspaceId, - email: payload.properties.before.userEmail, - personId: payload.recordId, - }, - ); + for (const eventPayload of payload.events) { + if (eventPayload.properties.after.userEmail === null) { + continue; + } await this.messageQueueService.add( MessageParticipantMatchParticipantJob.name, { workspaceId: payload.workspaceId, - email: payload.properties.after.userEmail, - workspaceMemberId: payload.recordId, + email: eventPayload.properties.after.userEmail, + workspaceMemberId: eventPayload.recordId, }, ); } } + + @OnEvent('workspaceMember.updated') + async handleUpdatedEvent( + payload: WorkspaceEventBatch< + ObjectRecordUpdateEvent + >, + ) { + for (const eventPayload of payload.events) { + if ( + objectRecordUpdateEventChangedProperties( + eventPayload.properties.before, + eventPayload.properties.after, + ).includes('userEmail') + ) { + await this.messageQueueService.add( + MessageParticipantUnmatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.before.userEmail, + personId: eventPayload.recordId, + }, + ); + + await this.messageQueueService.add( + MessageParticipantMatchParticipantJob.name, + { + workspaceId: payload.workspaceId, + email: eventPayload.properties.after.userEmail, + workspaceMemberId: eventPayload.recordId, + }, + ); + } + } + } } diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant.listener.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant.listener.ts index f7a21ae898b3..d57ebdc4d513 100644 --- a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant.listener.ts +++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant.listener.ts @@ -7,6 +7,7 @@ import { Repository } from 'typeorm'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity'; import { TimelineActivityRepository } from 'src/modules/timeline/repositiories/timeline-activity.repository'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; @@ -22,50 +23,54 @@ export class MessageParticipantListener { ) {} @OnEvent('messageParticipant.matched') - public async handleMessageParticipantMatched(payload: { - workspaceId: string; - workspaceMemberId: string; - participants: MessageParticipantWorkspaceEntity[]; - }): Promise { - const messageParticipants = payload.participants ?? []; + public async handleMessageParticipantMatched( + payload: WorkspaceEventBatch<{ + workspaceMemberId: string; + participants: MessageParticipantWorkspaceEntity[]; + }>, + ): Promise { + // TODO: Refactor to insertTimelineActivitiesForObject once + for (const eventPayload of payload.events) { + const messageParticipants = eventPayload.participants ?? []; - // TODO: move to a job? + // TODO: move to a job? - const dataSourceSchema = this.workspaceDataSourceService.getSchemaName( - payload.workspaceId, - ); + const dataSourceSchema = this.workspaceDataSourceService.getSchemaName( + payload.workspaceId, + ); - const messageObjectMetadata = - await this.objectMetadataRepository.findOneOrFail({ - where: { - nameSingular: 'message', - workspaceId: payload.workspaceId, - }, - }); + const messageObjectMetadata = + await this.objectMetadataRepository.findOneOrFail({ + where: { + nameSingular: 'message', + workspaceId: payload.workspaceId, + }, + }); - const messageParticipantsWithPersonId = messageParticipants.filter( - (participant) => participant.personId, - ); + const messageParticipantsWithPersonId = messageParticipants.filter( + (participant) => participant.personId, + ); - if (messageParticipantsWithPersonId.length === 0) { - return; - } + if (messageParticipantsWithPersonId.length === 0) { + return; + } - await this.timelineActivityRepository.insertTimelineActivitiesForObject( - 'person', - messageParticipantsWithPersonId.map((participant) => ({ - dataSourceSchema, - name: 'message.linked', - properties: null, - objectName: 'message', - recordId: participant.personId, - workspaceMemberId: payload.workspaceMemberId, - workspaceId: payload.workspaceId, - linkedObjectMetadataId: messageObjectMetadata.id, - linkedRecordId: participant.messageId, - linkedRecordCachedName: '', - })), - payload.workspaceId, - ); + await this.timelineActivityRepository.insertTimelineActivitiesForObject( + 'person', + messageParticipantsWithPersonId.map((participant) => ({ + dataSourceSchema, + name: 'message.linked', + properties: null, + objectName: 'message', + recordId: participant.personId, + workspaceMemberId: eventPayload.workspaceMemberId, + workspaceId: payload.workspaceId, + linkedObjectMetadataId: messageObjectMetadata.id, + linkedRecordId: participant.messageId, + linkedRecordCachedName: '', + })), + payload.workspaceId, + ); + } } } diff --git a/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts b/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts index edf4c3d9e488..d820020d3efc 100644 --- a/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts +++ b/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts @@ -1,12 +1,13 @@ import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event'; +import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { AuditLogRepository } from 'src/modules/timeline/repositiories/audit-log.repository'; import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity'; import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; -import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; -import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; -import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; @Processor(MessageQueue.entityEventsToDbQueue) export class CreateAuditLogFromInternalEvent { @@ -18,33 +19,37 @@ export class CreateAuditLogFromInternalEvent { ) {} @Process(CreateAuditLogFromInternalEvent.name) - async handle(data: ObjectRecordBaseEvent): Promise { - let workspaceMemberId: string | null = null; + async handle( + data: WorkspaceEventBatch, + ): Promise { + for (const eventData of data.events) { + let workspaceMemberId: string | null = null; - if (data.userId) { - const workspaceMember = await this.workspaceMemberService.getByIdOrFail( - data.userId, - data.workspaceId, - ); + if (eventData.userId) { + const workspaceMember = await this.workspaceMemberService.getByIdOrFail( + eventData.userId, + data.workspaceId, + ); - workspaceMemberId = workspaceMember.id; - } + workspaceMemberId = workspaceMember.id; + } - if (data.properties.diff) { - // we remove "before" and "after" property for a cleaner/slimmer event payload - data.properties = { - diff: data.properties.diff, - }; - } + if (eventData.properties.diff) { + // we remove "before" and "after" property for a cleaner/slimmer event payload + eventData.properties = { + diff: eventData.properties.diff, + }; + } - await this.auditLogRepository.insert( - data.name, - data.properties, - workspaceMemberId, - data.name.split('.')[0], - data.objectMetadata.id, - data.recordId, - data.workspaceId, - ); + await this.auditLogRepository.insert( + data.name, + eventData.properties, + workspaceMemberId, + data.name.split('.')[0], + eventData.objectMetadata.id, + eventData.recordId, + data.workspaceId, + ); + } } } diff --git a/packages/twenty-server/src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job.ts b/packages/twenty-server/src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job.ts index ada03bf13c8f..c73479745c7c 100644 --- a/packages/twenty-server/src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job.ts +++ b/packages/twenty-server/src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job.ts @@ -3,6 +3,7 @@ import { Process } from 'src/engine/integrations/message-queue/decorators/proces import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { TimelineActivityService } from 'src/modules/timeline/services/timeline-activity.service'; import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -16,33 +17,41 @@ export class UpsertTimelineActivityFromInternalEvent { ) {} @Process(UpsertTimelineActivityFromInternalEvent.name) - async handle(data: ObjectRecordBaseEvent): Promise { - if (data.userId) { - const workspaceMember = await this.workspaceMemberService.getByIdOrFail( - data.userId, - data.workspaceId, - ); + async handle( + data: WorkspaceEventBatch, + ): Promise { + for (const eventData of data.events) { + if (eventData.userId) { + const workspaceMember = await this.workspaceMemberService.getByIdOrFail( + eventData.userId, + data.workspaceId, + ); - data.workspaceMemberId = workspaceMember.id; - } + eventData.workspaceMemberId = workspaceMember.id; + } - if (data.properties.diff) { - // we remove "before" and "after" property for a cleaner/slimmer event payload - data.properties = { - diff: data.properties.diff, - }; - } + if (eventData.properties.diff) { + // we remove "before" and "after" property for a cleaner/slimmer event payload + eventData.properties = { + diff: eventData.properties.diff, + }; + } - // Temporary - // We ignore every that is not a LinkedObject or a Business Object - if ( - data.objectMetadata.isSystem && - data.objectMetadata.nameSingular !== 'noteTarget' && - data.objectMetadata.nameSingular !== 'taskTarget' - ) { - return; - } + // Temporary + // We ignore every that is not a LinkedObject or a Business Object + if ( + eventData.objectMetadata.isSystem && + eventData.objectMetadata.nameSingular !== 'noteTarget' && + eventData.objectMetadata.nameSingular !== 'taskTarget' + ) { + continue; + } - await this.timelineActivityService.upsertEvent(data); + await this.timelineActivityService.upsertEvent({ + ...eventData, + workspaceId: data.workspaceId, + name: data.name, + }); + } } } diff --git a/packages/twenty-server/src/modules/timeline/services/timeline-activity.service.ts b/packages/twenty-server/src/modules/timeline/services/timeline-activity.service.ts index b3b32ef3b522..963a65d5f35f 100644 --- a/packages/twenty-server/src/modules/timeline/services/timeline-activity.service.ts +++ b/packages/twenty-server/src/modules/timeline/services/timeline-activity.service.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event'; +import { ObjectRecordBaseEventWithNameAndWorkspaceId } from 'src/engine/integrations/event-emitter/types/object-record.base.event'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { TimelineActivityRepository } from 'src/modules/timeline/repositiories/timeline-activity.repository'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; -type TransformedEvent = ObjectRecordBaseEvent & { +type TransformedEvent = ObjectRecordBaseEventWithNameAndWorkspaceId & { objectName?: string; linkedRecordCachedName?: string; linkedRecordId?: string; @@ -26,7 +26,7 @@ export class TimelineActivityService { task: 'taskTarget', }; - async upsertEvent(event: ObjectRecordBaseEvent) { + async upsertEvent(event: ObjectRecordBaseEventWithNameAndWorkspaceId) { const events = await this.transformEvent(event); if (!events || events.length === 0) return; @@ -47,7 +47,7 @@ export class TimelineActivityService { } private async transformEvent( - event: ObjectRecordBaseEvent, + event: ObjectRecordBaseEventWithNameAndWorkspaceId, ): Promise { if (['note', 'task'].includes(event.objectMetadata.nameSingular)) { const linkedObjects = await this.handleLinkedObjects(event); @@ -69,7 +69,9 @@ export class TimelineActivityService { return [event]; } - private async handleLinkedObjects(event: ObjectRecordBaseEvent) { + private async handleLinkedObjects( + event: ObjectRecordBaseEventWithNameAndWorkspaceId, + ) { const dataSourceSchema = this.workspaceDataSourceService.getSchemaName( event.workspaceId, ); @@ -92,7 +94,7 @@ export class TimelineActivityService { } private async processActivity( - event: ObjectRecordBaseEvent, + event: ObjectRecordBaseEventWithNameAndWorkspaceId, dataSourceSchema: string, activityType: string, ) { @@ -145,7 +147,7 @@ export class TimelineActivityService { } private async processActivityTarget( - event: ObjectRecordBaseEvent, + event: ObjectRecordBaseEventWithNameAndWorkspaceId, dataSourceSchema: string, activityType: string, ) { diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener.ts index 3bf1eaa7e416..5291b8094ba9 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener.ts @@ -10,6 +10,7 @@ import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decora import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { WorkflowEventListenerWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-event-listener.workspace-entity'; import { WorkflowEventTriggerJob, @@ -28,25 +29,32 @@ export class DatabaseEventTriggerListener { ) {} @OnEvent('*.created') - async handleObjectRecordCreateEvent(payload: ObjectRecordCreateEvent) { + async handleObjectRecordCreateEvent( + payload: WorkspaceEventBatch>, + ) { await this.handleEvent(payload); } @OnEvent('*.updated') - async handleObjectRecordUpdateEvent(payload: ObjectRecordUpdateEvent) { + async handleObjectRecordUpdateEvent( + payload: WorkspaceEventBatch>, + ) { await this.handleEvent(payload); } @OnEvent('*.deleted') - async handleObjectRecordDeleteEvent(payload: ObjectRecordDeleteEvent) { + async handleObjectRecordDeleteEvent( + payload: WorkspaceEventBatch>, + ) { await this.handleEvent(payload); } private async handleEvent( - payload: + payload: WorkspaceEventBatch< | ObjectRecordCreateEvent | ObjectRecordUpdateEvent - | ObjectRecordDeleteEvent, + | ObjectRecordDeleteEvent + >, ) { const workspaceId = payload.workspaceId; const eventName = payload.name; @@ -84,15 +92,17 @@ export class DatabaseEventTriggerListener { }); for (const eventListener of eventListeners) { - this.messageQueueService.add( - WorkflowEventTriggerJob.name, - { - workspaceId, - workflowId: eventListener.workflowId, - payload, - }, - { retryLimit: 3 }, - ); + for (const eventPayload of payload.events) { + this.messageQueueService.add( + WorkflowEventTriggerJob.name, + { + workspaceId, + workflowId: eventListener.workflowId, + payload: eventPayload, + }, + { retryLimit: 3 }, + ); + } } } } diff --git a/packages/twenty-server/src/queue-worker/queue-worker.module.ts b/packages/twenty-server/src/queue-worker/queue-worker.module.ts index 6f3374a1c79c..c6eb553d2219 100644 --- a/packages/twenty-server/src/queue-worker/queue-worker.module.ts +++ b/packages/twenty-server/src/queue-worker/queue-worker.module.ts @@ -1,15 +1,17 @@ import { Module } from '@nestjs/common'; -import { JobsModule } from 'src/engine/integrations/message-queue/jobs.module'; import { IntegrationsModule } from 'src/engine/integrations/integrations.module'; -import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; +import { JobsModule } from 'src/engine/integrations/message-queue/jobs.module'; import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module'; +import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; +import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; @Module({ imports: [ TwentyORMModule.register({}), IntegrationsModule, MessageQueueModule.registerExplorer(), + WorkspaceEventEmitterModule, JobsModule, ], }) From 08e07ac2d5c3338f9f9d1c9b9c8e46122c0828b7 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Tue, 20 Aug 2024 23:58:11 +0200 Subject: [PATCH 06/19] Update graphql-yoga patch in yarn.lock --- .../workspace-metadata-version.service.ts | 2 ++ yarn.lock | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts index 8e5c5deaaa6e..d3986e8db2d5 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts @@ -51,6 +51,8 @@ export class WorkspaceMetadataVersionService { { metadataVersion: newMetadataVersion }, ); + await this.workspaceCacheStorageService.flush(workspaceId); + await this.workspaceCacheStorageService.setMetadataVersion( workspaceId, newMetadataVersion, diff --git a/yarn.lock b/yarn.lock index bac1580e3bc0..7fa46500801a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6506,14 +6506,14 @@ __metadata: "@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch::locator=twenty-server%40workspace%3Apackages%2Ftwenty-server": version: 2.1.0 - resolution: "@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@npm%3A2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch::version=2.1.0&hash=194cbb&locator=twenty-server%40workspace%3Apackages%2Ftwenty-server" + resolution: "@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@npm%3A2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch::version=2.1.0&hash=96a960&locator=twenty-server%40workspace%3Apackages%2Ftwenty-server" peerDependencies: "@nestjs/common": ^10.0.0 "@nestjs/core": ^10.0.0 "@nestjs/graphql": ^12.0.0 graphql: ^15.0.0 || ^16.0.0 graphql-yoga: ^4.0.4 - checksum: 10c0/f0529bfae125d22569dd5dbd7d06ac8e70315d720df5a315a644d9acf2d81d7d0d845967336b358dd11b29bab9920f38eff4c9ee774dd838aefc78dba6c91546 + checksum: 10c0/d7a9f1ff65642cf64e942ef1caf2596b0b2866118bd95200fc2112902e71d4aeb00fe436d5d253f2415f51cf79c48417f3859fae8ed4beaf2ac7e52d49eef3a1 languageName: node linkType: hard From 6caa78008f79a890d2171581d581e629da24cd09 Mon Sep 17 00:00:00 2001 From: "gitstart-app[bot]" <57568882+gitstart-app[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:19:21 +0200 Subject: [PATCH 07/19] If an object is disabled, then the relationships to that object should be disabled (#6690) This PR was created by [GitStart](https://gitstart.com/) to address the requirements from this ticket: [TWNTY-5370](https://clients.gitstart.com/twenty/5449/tickets/TWNTY-5370). This ticket was imported from: [TWNTY-5370](https://github.com/twentyhq/twenty/issues/5370) --- ### Description - We updated the logic in packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts Test cases: 1. Ensure that when an object is disabled, all related relationships are also disabled. a. Example disable the people object b. Check the company object and verify that the people field has been disabled too c. Check the opportunity object and check that the point of contact field has been disabled too 2. Verify that when a previously disabled object is restored, the relationships are also restored. 3. Ensure that previously disabled relationships remain disabled when the object is disabled and later restored. 4. Verify that relationships of a disabled object are not visible in the UI. 5. Ensure that relationships to a disabled object are marked as inactive in the data models screen ### Refs #5370 ### Demo Fixes #5370 --------- Co-authored-by: gitstart-twenty Co-authored-by: Marie Stoppa --- .../object-metadata.service.ts | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index 973a5cf35be5..53df8a651a24 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -5,7 +5,7 @@ import console from 'console'; import { Query, QueryOptions } from '@ptc-org/nestjs-query-core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; -import { FindManyOptions, FindOneOptions, Repository } from 'typeorm'; +import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; @@ -369,6 +369,10 @@ export class ObjectMetadataService extends TypeOrmQueryService { + acc.push(fromFieldMetadataId, toFieldMetadataId); + + return acc; + }, + [] as string[], + ); + + if (affectedFieldIds.length > 0) { + await this.fieldMetadataRepository.update( + { id: In(affectedFieldIds) }, + { isActive: isActive }, + ); + } + } } From 614a81860f4ac01c52d3673a791db6e372ab714a Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 21 Aug 2024 11:48:20 +0200 Subject: [PATCH 08/19] Add logging on currentWorkspaceMember query (#6706) We are experiencing slow GetCurrentUser endpoint, this is helping us troubleshoot --- .../database/typeorm/core/core.datasource.ts | 2 +- .../user/services/user.service.ts | 17 +++++++++- .../engine/core-modules/user/user.resolver.ts | 5 +++ .../entity-manager/entity.manager.ts | 1 + .../factories/workspace-datasource.factory.ts | 32 +++++++++++++------ .../twenty-orm/twenty-orm-global.manager.ts | 10 +++++- 6 files changed, 55 insertions(+), 12 deletions(-) diff --git a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts index 6733370d004c..1260f9bcf01d 100644 --- a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts +++ b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts @@ -1,7 +1,7 @@ import { TypeOrmModuleOptions } from '@nestjs/typeorm'; -import { DataSource, DataSourceOptions } from 'typeorm'; import { config } from 'dotenv'; +import { DataSource, DataSourceOptions } from 'typeorm'; config(); export const typeORMCoreModuleOptions: TypeOrmModuleOptions = { diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts index 3ee3636ee270..6c42f93b2042 100644 --- a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts @@ -40,18 +40,24 @@ export class UserService extends TypeOrmQueryService { return null; } + console.time('loadWorkspaceMember repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( user.defaultWorkspaceId, 'workspaceMember', ); + console.timeEnd('loadWorkspaceMember repo'); + + console.time('loadWorkspaceMember find'); const workspaceMember = await workspaceMemberRepository.findOne({ where: { userId: user.id, }, }); + console.timeEnd('loadWorkspaceMember find'); + return workspaceMember; } @@ -60,13 +66,22 @@ export class UserService extends TypeOrmQueryService { return []; } + console.time('loadWorkspaceMembers repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( workspace.id, 'workspaceMember', ); - return workspaceMemberRepository.find(); + console.timeEnd('loadWorkspaceMembers repo'); + + console.time('loadWorkspaceMembers find'); + + const workspaceMembers = workspaceMemberRepository.find(); + + console.timeEnd('loadWorkspaceMembers find'); + + return workspaceMembers; } async deleteUser(userId: string): Promise { diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index c77bb690baf7..b9114062151b 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -98,6 +98,7 @@ export class UserResolver { nullable: true, }) async workspaceMember(@Parent() user: User): Promise { + console.time('resolver workspaceMember'); const workspaceMember = await this.userService.loadWorkspaceMember(user); if (workspaceMember && workspaceMember.avatarUrl) { @@ -108,6 +109,7 @@ export class UserResolver { workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`; } + console.timeEnd('resolver workspaceMember'); // TODO: Fix typing disrepency between Entity and DTO return workspaceMember as WorkspaceMember | null; @@ -117,6 +119,7 @@ export class UserResolver { nullable: true, }) async workspaceMembers(@Parent() user: User): Promise { + console.time('resolver workspaceMembers'); const workspaceMembers = await this.userService.loadWorkspaceMembers( user.defaultWorkspace, ); @@ -132,6 +135,8 @@ export class UserResolver { } } + console.timeEnd('resolver workspaceMembers'); + // TODO: Fix typing disrepency between Entity and DTO return workspaceMembers as WorkspaceMember[]; } diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/entity.manager.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/entity.manager.ts index 4be629755c13..66f03915fe76 100644 --- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/entity.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/entity.manager.ts @@ -26,6 +26,7 @@ export class WorkspaceEntityManager extends EntityManager { target: EntityTarget, ): WorkspaceRepository { // find already created repository instance and return it if found + const repoFromMap = this.repositories.get(target); if (repoFromMap) { diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 10daedc7dd77..1a1b92a28bdb 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { EntitySchema, Repository } from 'typeorm'; +import { v4 } from 'uuid'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; @@ -28,11 +29,19 @@ export class WorkspaceDatasourceFactory { workspaceId: string, workspaceMetadataVersion: string | null, ): Promise { - const desiredWorkspaceMetadataVersion = - workspaceMetadataVersion ?? - (await this.workspaceMetadataVersionService.getMetadataVersion( + const logId = v4(); + + console.time(`fetch in datasource factory ${logId}`); + + const latestWorkspaceMetadataVersion = + await this.workspaceMetadataVersionService.getMetadataVersion( workspaceId, - )); + ); + + console.timeEnd(`fetch in datasource factory ${logId}`); + + const desiredWorkspaceMetadataVersion = + workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; if (!desiredWorkspaceMetadataVersion) { throw new Error( @@ -40,11 +49,6 @@ export class WorkspaceDatasourceFactory { ); } - const latestWorkspaceMetadataVersion = - await this.workspaceMetadataVersionService.getMetadataVersion( - workspaceId, - ); - if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { throw new Error( `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, @@ -80,6 +84,9 @@ export class WorkspaceDatasourceFactory { const workspaceDataSource = await workspaceDataSourceCacheInstance.execute( `${workspaceId}-${latestWorkspaceMetadataVersion}`, async () => { + const logId = v4(); + + console.log('Creating workspace fresh data source...' + logId); const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( workspaceId, @@ -97,6 +104,7 @@ export class WorkspaceDatasourceFactory { ); } + console.time('create entity schema' + logId); const cachedEntitySchemaOptions = await this.workspaceCacheStorageService.getORMEntitySchema( workspaceId, @@ -122,7 +130,9 @@ export class WorkspaceDatasourceFactory { cachedEntitySchemas = entitySchemas; } + console.timeEnd('create entity schema' + logId); + console.time('create workspace data source' + logId); const workspaceDataSource = new WorkspaceDataSource( { workspaceId, @@ -146,7 +156,11 @@ export class WorkspaceDatasourceFactory { }, ); + console.timeEnd('create workspace data source' + logId); + + console.time('initialize workspace data source' + logId); await workspaceDataSource.initialize(); + console.timeEnd('initialize workspace data source' + logId); return workspaceDataSource; }, diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts index add735169cf0..6d24e6f8e562 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts @@ -1,6 +1,7 @@ import { Injectable, Type } from '@nestjs/common'; import { ObjectLiteral } from 'typeorm'; +import { v4 } from 'uuid'; import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory'; import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; @@ -36,12 +37,19 @@ export class TwentyORMGlobalManager { ); } + const logId = v4(); + + console.time(`createDataSource in orm ${logId}`); const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, null, ); - return workspaceDataSource.getRepository(objectMetadataName); + console.timeEnd(`createDataSource in orm ${logId}`); + + const repository = workspaceDataSource.getRepository(objectMetadataName); + + return repository; } async getDataSourceForWorkspace(workspaceId: string) { From aa4ae53fb497d292f9b2319012482b942cbecfa5 Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:08:57 +0200 Subject: [PATCH 09/19] [Fix] field does not appear directly after creation (#6708) --- .../metadata-modules/field-metadata/field-metadata.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index 8b26733dcc1a..6cb1cdebefad 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -274,7 +274,7 @@ export class FieldMetadataService extends TypeOrmQueryService Date: Wed, 21 Aug 2024 14:17:11 +0200 Subject: [PATCH 10/19] Remove performance logs (#6709) We have found the root cause of the issue: - when using a datasource (including the cached ones), we are fetching ObjectMetadataCollection from cache (700kB). Datasource usage is happening any time we are using twentyORM, which is everywhere in the jobs and in some resolvers (including the GetCurrentUser one). This is leading to a high load on redis and leading to the performance issues we are seeing. - we actually don't need to fetch this objectMetadataCollection while using a cached datasource, only when we instantiate a new one --- .../user/services/user.service.ts | 13 ---- .../engine/core-modules/user/user.resolver.ts | 5 -- .../factories/workspace-datasource.factory.ts | 67 +++++++------------ .../storage/cache-manager.storage.ts | 2 - .../twenty-orm/twenty-orm-global.manager.ts | 6 -- 5 files changed, 25 insertions(+), 68 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts index 6c42f93b2042..aec142d2655a 100644 --- a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts @@ -40,24 +40,18 @@ export class UserService extends TypeOrmQueryService { return null; } - console.time('loadWorkspaceMember repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( user.defaultWorkspaceId, 'workspaceMember', ); - console.timeEnd('loadWorkspaceMember repo'); - - console.time('loadWorkspaceMember find'); const workspaceMember = await workspaceMemberRepository.findOne({ where: { userId: user.id, }, }); - console.timeEnd('loadWorkspaceMember find'); - return workspaceMember; } @@ -66,21 +60,14 @@ export class UserService extends TypeOrmQueryService { return []; } - console.time('loadWorkspaceMembers repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( workspace.id, 'workspaceMember', ); - console.timeEnd('loadWorkspaceMembers repo'); - - console.time('loadWorkspaceMembers find'); - const workspaceMembers = workspaceMemberRepository.find(); - console.timeEnd('loadWorkspaceMembers find'); - return workspaceMembers; } diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index b9114062151b..c77bb690baf7 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -98,7 +98,6 @@ export class UserResolver { nullable: true, }) async workspaceMember(@Parent() user: User): Promise { - console.time('resolver workspaceMember'); const workspaceMember = await this.userService.loadWorkspaceMember(user); if (workspaceMember && workspaceMember.avatarUrl) { @@ -109,7 +108,6 @@ export class UserResolver { workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`; } - console.timeEnd('resolver workspaceMember'); // TODO: Fix typing disrepency between Entity and DTO return workspaceMember as WorkspaceMember | null; @@ -119,7 +117,6 @@ export class UserResolver { nullable: true, }) async workspaceMembers(@Parent() user: User): Promise { - console.time('resolver workspaceMembers'); const workspaceMembers = await this.userService.loadWorkspaceMembers( user.defaultWorkspace, ); @@ -135,8 +132,6 @@ export class UserResolver { } } - console.timeEnd('resolver workspaceMembers'); - // TODO: Fix typing disrepency between Entity and DTO return workspaceMembers as WorkspaceMember[]; } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 1a1b92a28bdb..c133010204a8 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { EntitySchema, Repository } from 'typeorm'; -import { v4 } from 'uuid'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; @@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory { workspaceId: string, workspaceMetadataVersion: string | null, ): Promise { - const logId = v4(); - - console.time(`fetch in datasource factory ${logId}`); - const latestWorkspaceMetadataVersion = await this.workspaceMetadataVersionService.getMetadataVersion( workspaceId, ); - console.timeEnd(`fetch in datasource factory ${logId}`); - const desiredWorkspaceMetadataVersion = workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; @@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory { ); } - let cachedObjectMetadataCollection = - await this.workspaceCacheStorageService.getObjectMetadataCollection( - workspaceId, - ); - - if (!cachedObjectMetadataCollection) { - const freshObjectMetadataCollection = - await this.objectMetadataRepository.find({ - where: { workspaceId }, - relations: [ - 'fields.object', - 'fields', - 'fields.fromRelationMetadata', - 'fields.toRelationMetadata', - 'fields.fromRelationMetadata.toObjectMetadata', - ], - }); - - await this.workspaceCacheStorageService.setObjectMetadataCollection( - workspaceId, - freshObjectMetadataCollection, - ); - - cachedObjectMetadataCollection = freshObjectMetadataCollection; - } - const workspaceDataSource = await workspaceDataSourceCacheInstance.execute( `${workspaceId}-${latestWorkspaceMetadataVersion}`, async () => { - const logId = v4(); + let cachedObjectMetadataCollection = + await this.workspaceCacheStorageService.getObjectMetadataCollection( + workspaceId, + ); + + if (!cachedObjectMetadataCollection) { + const freshObjectMetadataCollection = + await this.objectMetadataRepository.find({ + where: { workspaceId }, + relations: [ + 'fields.object', + 'fields', + 'fields.fromRelationMetadata', + 'fields.toRelationMetadata', + 'fields.fromRelationMetadata.toObjectMetadata', + ], + }); + + await this.workspaceCacheStorageService.setObjectMetadataCollection( + workspaceId, + freshObjectMetadataCollection, + ); + + cachedObjectMetadataCollection = freshObjectMetadataCollection; + } - console.log('Creating workspace fresh data source...' + logId); const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( workspaceId, @@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory { ); } - console.time('create entity schema' + logId); const cachedEntitySchemaOptions = await this.workspaceCacheStorageService.getORMEntitySchema( workspaceId, @@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory { cachedEntitySchemas = entitySchemas; } - console.timeEnd('create entity schema' + logId); - console.time('create workspace data source' + logId); const workspaceDataSource = new WorkspaceDataSource( { workspaceId, @@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory { }, ); - console.timeEnd('create workspace data source' + logId); - - console.time('initialize workspace data source' + logId); await workspaceDataSource.initialize(); - console.timeEnd('initialize workspace data source' + logId); return workspaceDataSource; }, diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts index 6dee55b501fc..fcd40d6f38ee 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts @@ -16,7 +16,6 @@ export class CacheManager { return this.cache.get(cacheKey)!; } - // Remove old entries with the same workspaceId for (const key of this.cache.keys()) { if (key.startsWith(`${workspaceId}-`)) { await onDelete?.(this.cache.get(key)!); @@ -24,7 +23,6 @@ export class CacheManager { } } - // Create a new value using the factory callback const value = await factory(); if (!value) { diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts index 6d24e6f8e562..f4d8190c556d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts @@ -1,7 +1,6 @@ import { Injectable, Type } from '@nestjs/common'; import { ObjectLiteral } from 'typeorm'; -import { v4 } from 'uuid'; import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory'; import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; @@ -37,16 +36,11 @@ export class TwentyORMGlobalManager { ); } - const logId = v4(); - - console.time(`createDataSource in orm ${logId}`); const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, null, ); - console.timeEnd(`createDataSource in orm ${logId}`); - const repository = workspaceDataSource.getRepository(objectMetadataName); return repository; From 79641327f3935d24b1b90ea17f3caba579f3caa9 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 21 Aug 2024 15:33:38 +0200 Subject: [PATCH 11/19] Fix webhook issue (#6711) Fix [#web](https://github.com/orgs/twentyhq/projects/1/views/3?pane=issue&itemId=75329194) This PR does 2 things: - migrate webhooks to TwentyORM - Fix inversion between objectNameSingular and operation in webhook eventName. It is stored as {objectNameSingular}.{operation} and we were querying {operation}.{objectNameSingular} --- packages/twenty-front/tsconfig.json | 2 +- .../jobs/call-webhook-jobs.job.ts | 50 +++++++++---------- .../jobs/call-webhook.job.ts | 6 +-- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/packages/twenty-front/tsconfig.json b/packages/twenty-front/tsconfig.json index f6c268d9c2fd..ec14700eb407 100644 --- a/packages/twenty-front/tsconfig.json +++ b/packages/twenty-front/tsconfig.json @@ -45,4 +45,4 @@ } ], "extends": "../../tsconfig.base.json" -} \ No newline at end of file +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts index 818be2a60d65..f4f6f9dca930 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts @@ -1,18 +1,20 @@ import { Logger } from '@nestjs/common'; +import { Like } from 'typeorm'; + import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; -import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; -import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { CallWebhookJob, CallWebhookJobData, } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job'; import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; -import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WebhookWorkspaceEntity } from 'src/modules/webhook/standard-objects/webhook.workspace-entity'; export enum CallWebhookJobsJobOperation { create = 'create', @@ -32,42 +34,38 @@ export class CallWebhookJobsJob { private readonly logger = new Logger(CallWebhookJobsJob.name); constructor( - private readonly workspaceDataSourceService: WorkspaceDataSourceService, - private readonly dataSourceService: DataSourceService, @InjectMessageQueue(MessageQueue.webhookQueue) private readonly messageQueueService: MessageQueueService, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) {} @Process(CallWebhookJobsJob.name) async handle(data: CallWebhookJobsJobData): Promise { - const dataSourceMetadata = - await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( - data.workspaceId, - ); - const workspaceDataSource = - await this.workspaceDataSourceService.connectToWorkspaceDataSource( + const webhookRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( data.workspaceId, + 'webhook', ); + const nameSingular = data.objectMetadataItem.nameSingular; const operation = data.operation; - const eventType = `${operation}.${nameSingular}`; - const webhooks: { id: string; targetUrl: string }[] = - await workspaceDataSource?.query( - ` - SELECT * FROM ${dataSourceMetadata.schema}."webhook" - WHERE operation LIKE '%${eventType}%' - OR operation LIKE '%*.${nameSingular}%' - OR operation LIKE '%${operation}.*%' - OR operation LIKE '%*.*%' - `, - ); + const eventName = `${nameSingular}.${operation}`; + + const webhooks = await webhookRepository.find({ + where: [ + { operation: Like(`%${eventName}%`) }, + { operation: Like(`%*.${operation}%`) }, + { operation: Like(`%${nameSingular}.*%`) }, + { operation: Like('%*.*%') }, + ], + }); webhooks.forEach((webhook) => { this.messageQueueService.add( CallWebhookJob.name, { targetUrl: webhook.targetUrl, - eventType, + eventName, objectMetadata: { id: data.objectMetadataItem.id, nameSingular: data.objectMetadataItem.nameSingular, @@ -83,7 +81,7 @@ export class CallWebhookJobsJob { if (webhooks.length) { this.logger.log( - `CallWebhookJobsJob on eventType '${eventType}' called on webhooks ids [\n"${webhooks + `CallWebhookJobsJob on eventName '${event}' called on webhooks ids [\n"${webhooks .map((webhook) => webhook.id) .join('",\n"')}"\n]`, ); diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job.ts index d18686e96aba..6e0d7af5240b 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job.ts @@ -1,13 +1,13 @@ -import { Logger } from '@nestjs/common'; import { HttpService } from '@nestjs/axios'; +import { Logger } from '@nestjs/common'; +import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; -import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; export type CallWebhookJobData = { targetUrl: string; - eventType: string; + eventName: string; objectMetadata: { id: string; nameSingular: string }; workspaceId: string; webhookId: string; From c2cf8b45548b2080bb63ea98f9f070491212a295 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 21 Aug 2024 15:39:55 +0200 Subject: [PATCH 12/19] Bump version to 0.23.2 --- packages/twenty-emails/package.json | 2 +- packages/twenty-front/package.json | 2 +- packages/twenty-server/package.json | 2 +- packages/twenty-ui/package.json | 2 +- packages/twenty-website/package.json | 2 +- .../src/content/developers/self-hosting/upgrade-guide.mdx | 1 - 6 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/twenty-emails/package.json b/packages/twenty-emails/package.json index 3ba73e7b0c1a..8c34817e3a26 100644 --- a/packages/twenty-emails/package.json +++ b/packages/twenty-emails/package.json @@ -1,6 +1,6 @@ { "name": "twenty-emails", - "version": "0.23.1", + "version": "0.23.2", "description": "", "author": "", "private": true, diff --git a/packages/twenty-front/package.json b/packages/twenty-front/package.json index 1d955d63b26a..3dd492559b75 100644 --- a/packages/twenty-front/package.json +++ b/packages/twenty-front/package.json @@ -1,6 +1,6 @@ { "name": "twenty-front", - "version": "0.23.1", + "version": "0.23.2", "private": true, "type": "module", "scripts": { diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 4b80f49e6871..e6cd012e0ffc 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -1,6 +1,6 @@ { "name": "twenty-server", - "version": "0.23.1", + "version": "0.23.2", "description": "", "author": "", "private": true, diff --git a/packages/twenty-ui/package.json b/packages/twenty-ui/package.json index d01372f77523..dc46ba46c4a2 100644 --- a/packages/twenty-ui/package.json +++ b/packages/twenty-ui/package.json @@ -1,6 +1,6 @@ { "name": "twenty-ui", - "version": "0.23.1", + "version": "0.23.2", "type": "module", "main": "./src/index.ts", "exports": { diff --git a/packages/twenty-website/package.json b/packages/twenty-website/package.json index e95d65df4213..04c1678f737a 100644 --- a/packages/twenty-website/package.json +++ b/packages/twenty-website/package.json @@ -1,6 +1,6 @@ { "name": "twenty-website", - "version": "0.23.1", + "version": "0.23.2", "private": true, "scripts": { "nx": "NX_DEFAULT_PROJECT=twenty-website node ../../node_modules/nx/bin/nx.js", diff --git a/packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx b/packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx index a82bf8998e1e..cd121f7357f9 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx @@ -45,7 +45,6 @@ yarn command:prod upgrade-0.23 ``` The `yarn database:migrate:prod` command will apply the migrations to the Database. -The `yarn command:prod workspace:sync-metadata -f` command will sync the definition of standard objects to the metadata tables and apply to required migrations to existing workspaces. We're introducing Task and Notes as standard object and will be deprecating the Activity object. The `yarn command:prod upgrade-0.23` takes care of the data migration, including transferring activities to tasks/notes. From da5dfb7a5b7914c9ef72e1c31f1d47ce82f63dac Mon Sep 17 00:00:00 2001 From: nitin <142569587+ehconitin@users.noreply.github.com> Date: Wed, 21 Aug 2024 20:06:04 +0530 Subject: [PATCH 13/19] corrected targetableobject being undefined when clicked on create task in command menu (#6635) Issue #6630 It seems this bug is caused by `targetableObjects` being assigned an empty array, which then leads to an error due to it being undefined. I've made some changes that should address the issue, but I would appreciate any feedback or suggestions on alternative solutions. Please let me know if there is a better approach to resolving this. Thank you! https://github.com/user-attachments/assets/d6409798-3320-49b3-834f-2b6888847ed8 --- .../effect-components/PageChangeEffect.tsx | 7 ++- .../hooks/useOpenCreateActivityDrawer.ts | 44 +++++++++++++------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/packages/twenty-front/src/effect-components/PageChangeEffect.tsx b/packages/twenty-front/src/effect-components/PageChangeEffect.tsx index 5dd9c9b2845e..bf083a157e5a 100644 --- a/packages/twenty-front/src/effect-components/PageChangeEffect.tsx +++ b/packages/twenty-front/src/effect-components/PageChangeEffect.tsx @@ -16,10 +16,10 @@ import { AppPath } from '@/types/AppPath'; import { PageHotkeyScope } from '@/types/PageHotkeyScope'; import { SettingsPath } from '@/types/SettingsPath'; import { useSetHotkeyScope } from '@/ui/utilities/hotkey/hooks/useSetHotkeyScope'; +import { useCleanRecoilState } from '~/hooks/useCleanRecoilState'; import { useIsMatchingLocation } from '~/hooks/useIsMatchingLocation'; import { usePageChangeEffectNavigateLocation } from '~/hooks/usePageChangeEffectNavigateLocation'; import { isDefined } from '~/utils/isDefined'; -import { useCleanRecoilState } from '~/hooks/useCleanRecoilState'; // TODO: break down into smaller functions and / or hooks // - moved usePageChangeEffectNavigateLocation into dedicated hook @@ -153,7 +153,10 @@ export const PageChangeEffect = () => { label: 'Create Task', type: CommandType.Create, Icon: IconCheckbox, - onCommandClick: () => openCreateActivity({ targetableObjects: [] }), + onCommandClick: () => + openCreateActivity({ + targetableObjects: [], + }), }, ]); }, [addToCommandMenu, setToInitialCommandMenu, openCreateActivity]); diff --git a/packages/twenty-front/src/modules/activities/hooks/useOpenCreateActivityDrawer.ts b/packages/twenty-front/src/modules/activities/hooks/useOpenCreateActivityDrawer.ts index 74781c52cecd..400c1f398a95 100644 --- a/packages/twenty-front/src/modules/activities/hooks/useOpenCreateActivityDrawer.ts +++ b/packages/twenty-front/src/modules/activities/hooks/useOpenCreateActivityDrawer.ts @@ -68,24 +68,40 @@ export const useOpenCreateActivityDrawer = ({ assigneeId: customAssignee?.id, }); - const targetableObjectRelationIdName = `${targetableObjects[0].targetObjectNameSingular}Id`; - - await createOneActivityTarget({ - taskId: - activityObjectNameSingular === CoreObjectNameSingular.Task - ? activity.id - : undefined, - noteId: - activityObjectNameSingular === CoreObjectNameSingular.Note - ? activity.id - : undefined, - [targetableObjectRelationIdName]: targetableObjects[0].id, - }); + if (targetableObjects.length > 0) { + const targetableObjectRelationIdName = `${targetableObjects[0].targetObjectNameSingular}Id`; + + await createOneActivityTarget({ + taskId: + activityObjectNameSingular === CoreObjectNameSingular.Task + ? activity.id + : undefined, + noteId: + activityObjectNameSingular === CoreObjectNameSingular.Note + ? activity.id + : undefined, + [targetableObjectRelationIdName]: targetableObjects[0].id, + }); + + setActivityTargetableEntityArray(targetableObjects); + } else { + await createOneActivityTarget({ + taskId: + activityObjectNameSingular === CoreObjectNameSingular.Task + ? activity.id + : undefined, + noteId: + activityObjectNameSingular === CoreObjectNameSingular.Note + ? activity.id + : undefined, + }); + + setActivityTargetableEntityArray([]); + } setHotkeyScope(RightDrawerHotkeyScope.RightDrawer, { goto: false }); setViewableRecordId(activity.id); setViewableRecordNameSingular(activityObjectNameSingular); - setActivityTargetableEntityArray(targetableObjects ?? []); openRightDrawer(RightDrawerPages.ViewRecord); setIsUpsertingActivityInDB(false); From 663acd56e4a092a0787066b526801453d515e9ee Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Wed, 21 Aug 2024 17:41:26 +0200 Subject: [PATCH 14/19] Trigger workflow run manually (#6696) Fix https://github.com/twentyhq/twenty/issues/6669 - create a commun function `startWorkflowRun` that both create the run object and the job for executing the workflow - use it in both the `workflowEventJob` and the `runWorkflowVersion` endpoint Bonus: - use filtering for exceptions instead of a util. It avoids doing a try catch in all endpoint --- packages/twenty-server/@types/express.d.ts | 1 + .../engine/core-modules/actor/actor.module.ts | 12 +++ .../query-hooks/created-by.pre-query-hook.ts | 13 ++- ...d-created-by-from-workspace-member.util.ts | 11 ++ .../auth/services/token.service.spec.ts | 5 + .../auth/services/token.service.ts | 34 +++++- .../auth/strategies/jwt.auth.strategy.ts | 12 ++- .../auth/types/auth-context.type.ts | 1 + .../engine/core-modules/core-engine.module.ts | 6 +- .../workflow/core-workflow-trigger.module.ts | 12 --- .../workflow/dtos/workflow-run.dto.ts | 9 ++ .../dtos/workflow-trigger-result.dto.ts | 14 --- ...w-trigger-graphql-api-exception.filter.ts} | 19 ++-- .../workflow/workflow-trigger-api.module.ts | 10 ++ .../workflow/workflow-trigger.resolver.ts | 42 ++++---- .../auth-workspace-member-id.decorator.ts | 11 ++ .../field-metadata/field-metadata.module.ts | 9 +- ...l-hydrate-request-from-token.middleware.ts | 1 + .../workflow-action-executor.exception.ts | 12 +++ .../workflow-action-executor.factory.ts | 23 ++++ .../workflow-action-executor.interface.ts} | 2 +- .../workflow-action-executor.module.ts | 17 +++ .../code-workflow-action-executor.ts} | 14 +-- .../workflow-action-runner.exception.ts | 12 --- .../workflow-action-runner.factory.ts | 23 ---- .../workflow-action-runner.module.ts | 17 --- .../workflow-executor.exception.ts} | 4 +- .../workflow-executor.module.ts | 12 +++ .../workflow-executor.workspace-service.ts | 84 +++++++++++++++ .../run-workflow.job.ts} | 20 ++-- .../workflow-run/workflow-run.exception.ts | 13 +++ .../workflow-run/workflow-run.module.ts | 9 ++ .../workflow-run.workspace-service.ts} | 24 ++--- .../workflow-runner/workflow-runner.module.ts | 14 +-- .../workflow-runner.workspace-service.ts | 101 ++++++------------ .../workflow-status.exception.ts | 13 --- .../workflow-status/workflow-status.module.ts | 9 -- .../jobs/workflow-event-trigger.job.ts | 43 ++++---- .../jobs/workflow-trigger-job.module.ts | 11 -- .../workflow-trigger-listener.module.ts | 10 -- .../workflow-trigger.module.ts | 18 +++- .../workflow-trigger.workspace-service.ts | 38 +++++-- .../src/modules/workflow/workflow.module.ts | 3 +- 43 files changed, 452 insertions(+), 316 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/actor/actor.module.ts rename packages/twenty-server/src/engine/{metadata-modules/field-metadata => core-modules/actor}/query-hooks/created-by.pre-query-hook.ts (84%) create mode 100644 packages/twenty-server/src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/workflow/core-workflow-trigger.module.ts create mode 100644 packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-run.dto.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto.ts rename packages/twenty-server/src/engine/core-modules/workflow/{utils/workflow-trigger-graphql-api-exception-handler.util.ts => filters/workflow-trigger-graphql-api-exception.filter.ts} (59%) create mode 100644 packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger-api.module.ts create mode 100644 packages/twenty-server/src/engine/decorators/auth/auth-workspace-member-id.decorator.ts create mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.exception.ts create mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.factory.ts rename packages/twenty-server/src/modules/workflow/{workflow-action-runner/workflow-action-runner.interface.ts => workflow-action-executor/workflow-action-executor.interface.ts} (87%) create mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.module.ts rename packages/twenty-server/src/modules/workflow/{workflow-action-runner/workflow-action-runners/code-workflow-action-runner.ts => workflow-action-executor/workflow-action-executors/code-workflow-action-executor.ts} (71%) delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.exception.ts delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.factory.ts delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.module.ts rename packages/twenty-server/src/modules/workflow/{workflow-runner/workflow-runner.exception.ts => workflow-executor/workflow-executor.exception.ts} (62%) create mode 100644 packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts create mode 100644 packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.workspace-service.ts rename packages/twenty-server/src/modules/workflow/workflow-runner/{workflow-runner.job.ts => jobs/run-workflow.job.ts} (65%) create mode 100644 packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.exception.ts create mode 100644 packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts rename packages/twenty-server/src/modules/workflow/{workflow-status/workflow-status.workspace-service.ts => workflow-runner/workflow-run/workflow-run.workspace-service.ts} (79%) delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.exception.ts delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.module.ts delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-trigger-job.module.ts delete mode 100644 packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/workflow-trigger-listener.module.ts diff --git a/packages/twenty-server/@types/express.d.ts b/packages/twenty-server/@types/express.d.ts index 1e45cf88d079..f30a99a198d7 100644 --- a/packages/twenty-server/@types/express.d.ts +++ b/packages/twenty-server/@types/express.d.ts @@ -9,5 +9,6 @@ declare module 'express-serve-static-core' { workspace?: Workspace; workspaceId?: string; workspaceMetadataVersion?: number; + workspaceMemberId?: string; } } diff --git a/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts b/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts new file mode 100644 index 000000000000..d69cc69a23f4 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { CreatedByPreQueryHook } from 'src/engine/core-modules/actor/query-hooks/created-by.pre-query-hook'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([FieldMetadataEntity], 'metadata')], + providers: [CreatedByPreQueryHook], + exports: [CreatedByPreQueryHook], +}) +export class ActorModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook.ts b/packages/twenty-server/src/engine/core-modules/actor/query-hooks/created-by.pre-query-hook.ts similarity index 84% rename from packages/twenty-server/src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook.ts rename to packages/twenty-server/src/engine/core-modules/actor/query-hooks/created-by.pre-query-hook.ts index 1e33aa850510..2a90843f1d69 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook.ts +++ b/packages/twenty-server/src/engine/core-modules/actor/query-hooks/created-by.pre-query-hook.ts @@ -1,3 +1,4 @@ +import { Logger } from '@nestjs/common/services/logger.service'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; @@ -6,6 +7,7 @@ import { WorkspaceQueryHookInstance } from 'src/engine/api/graphql/workspace-que import { CreateManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator'; +import { buildCreatedByFromWorkspaceMember } from 'src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { ActorMetadata, @@ -26,6 +28,8 @@ type CustomWorkspaceItem = Omit< @WorkspaceQueryHook(`*.createMany`) export class CreatedByPreQueryHook implements WorkspaceQueryHookInstance { + private readonly logger = new Logger(CreatedByPreQueryHook.name); + constructor( private readonly twentyORMGlobalManager: TwentyORMGlobalManager, @InjectRepository(FieldMetadataEntity, 'metadata') @@ -55,7 +59,14 @@ export class CreatedByPreQueryHook implements WorkspaceQueryHookInstance { } // If user is logged in, we use the workspace member - if (authContext.user) { + if (authContext.workspaceMemberId && authContext.user) { + createdBy = buildCreatedByFromWorkspaceMember( + authContext.workspaceMemberId, + authContext.user, + ); + // TODO: remove that code once we have the workspace member id in all tokens + } else if (authContext.user) { + this.logger.warn("User doesn't have a workspace member id in the token"); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( authContext.workspace.id, diff --git a/packages/twenty-server/src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util.ts b/packages/twenty-server/src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util.ts new file mode 100644 index 000000000000..a553f9970c60 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util.ts @@ -0,0 +1,11 @@ +import { User } from 'src/engine/core-modules/user/user.entity'; +import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type'; + +export const buildCreatedByFromWorkspaceMember = ( + workspaceMemberId: string, + user: User, +) => ({ + workspaceMemberId, + source: FieldActorSource.MANUAL, + name: `${user.firstName} ${user.lastName}`, +}); diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.spec.ts b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.spec.ts index 0d807db41bff..29e2df741b82 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.spec.ts @@ -16,6 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { EmailService } from 'src/engine/integrations/email/email.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { TokenService } from './token.service'; @@ -66,6 +67,10 @@ describe('TokenService', () => { provide: getRepositoryToken(Workspace, 'core'), useValue: {}, }, + { + provide: TwentyORMGlobalManager, + useValue: {}, + }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts index da571fbe5184..200430de73b0 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts @@ -41,6 +41,8 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { EmailService } from 'src/engine/integrations/email/email.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @Injectable() export class TokenService { @@ -55,6 +57,7 @@ export class TokenService { @InjectRepository(Workspace, 'core') private readonly workspaceRepository: Repository, private readonly emailService: EmailService, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) {} async generateAccessToken( @@ -91,9 +94,33 @@ export class TokenService { ); } + const workspaceIdNonNullable = workspaceId + ? workspaceId + : user.defaultWorkspace.id; + + const workspaceMemberRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceIdNonNullable, + 'workspaceMember', + ); + + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { + userId: user.id, + }, + }); + + if (!workspaceMember) { + throw new AuthException( + 'User is not a member of the workspace', + AuthExceptionCode.FORBIDDEN_EXCEPTION, + ); + } + const jwtPayload: JwtPayload = { sub: user.id, workspaceId: workspaceId ? workspaceId : user.defaultWorkspace.id, + workspaceMemberId: workspaceMember.id, }; return { @@ -247,11 +274,10 @@ export class TokenService { this.environmentService.get('ACCESS_TOKEN_SECRET'), ); - const { user, apiKey, workspace } = await this.jwtStrategy.validate( - decoded as JwtPayload, - ); + const { user, apiKey, workspace, workspaceMemberId } = + await this.jwtStrategy.validate(decoded as JwtPayload); - return { user, apiKey, workspace }; + return { user, apiKey, workspace, workspaceMemberId }; } async verifyLoginToken(loginToken: string): Promise { diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts index 2807232fc9c6..5764e967815f 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts @@ -17,7 +17,12 @@ import { EnvironmentService } from 'src/engine/integrations/environment/environm import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ApiKeyWorkspaceEntity } from 'src/modules/api-key/standard-objects/api-key.workspace-entity'; -export type JwtPayload = { sub: string; workspaceId: string; jti?: string }; +export type JwtPayload = { + sub: string; + workspaceId: string; + workspaceMemberId: string; + jti?: string; +}; @Injectable() export class JwtAuthStrategy extends PassportStrategy(Strategy, 'jwt') { @@ -95,6 +100,9 @@ export class JwtAuthStrategy extends PassportStrategy(Strategy, 'jwt') { } } - return { user, apiKey, workspace }; + // We don't check if the user is a member of the workspace yet + const workspaceMemberId = payload.workspaceMemberId; + + return { user, apiKey, workspace, workspaceMemberId }; } } diff --git a/packages/twenty-server/src/engine/core-modules/auth/types/auth-context.type.ts b/packages/twenty-server/src/engine/core-modules/auth/types/auth-context.type.ts index cf8faca181bc..80c223f4e926 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/types/auth-context.type.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/types/auth-context.type.ts @@ -5,5 +5,6 @@ import { ApiKeyWorkspaceEntity } from 'src/modules/api-key/standard-objects/api- export type AuthContext = { user?: User | null | undefined; apiKey?: ApiKeyWorkspaceEntity | null | undefined; + workspaceMemberId?: string; workspace: Workspace; }; diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index 37b9ac882052..27312a15159d 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; +import { ActorModule } from 'src/engine/core-modules/actor/actor.module'; import { AISQLQueryModule } from 'src/engine/core-modules/ai-sql-query/ai-sql-query.module'; import { AppTokenModule } from 'src/engine/core-modules/app-token/app-token.module'; import { AuthModule } from 'src/engine/core-modules/auth/auth.module'; @@ -11,7 +12,7 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel import { OpenApiModule } from 'src/engine/core-modules/open-api/open-api.module'; import { PostgresCredentialsModule } from 'src/engine/core-modules/postgres-credentials/postgres-credentials.module'; import { UserModule } from 'src/engine/core-modules/user/user.module'; -import { WorkflowTriggerCoreModule } from 'src/engine/core-modules/workflow/core-workflow-trigger.module'; +import { WorkflowTriggerApiModule } from 'src/engine/core-modules/workflow/workflow-trigger-api.module'; import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; @@ -36,8 +37,9 @@ import { FileModule } from './file/file.module'; WorkspaceModule, AISQLQueryModule, PostgresCredentialsModule, - WorkflowTriggerCoreModule, + WorkflowTriggerApiModule, WorkspaceEventEmitterModule, + ActorModule, ], exports: [ AnalyticsModule, diff --git a/packages/twenty-server/src/engine/core-modules/workflow/core-workflow-trigger.module.ts b/packages/twenty-server/src/engine/core-modules/workflow/core-workflow-trigger.module.ts deleted file mode 100644 index 76cefa494ab1..000000000000 --- a/packages/twenty-server/src/engine/core-modules/workflow/core-workflow-trigger.module.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { WorkflowTriggerResolver } from 'src/engine/core-modules/workflow/workflow-trigger.resolver'; -import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module'; -import { WorkflowRunnerModule } from 'src/modules/workflow/workflow-runner/workflow-runner.module'; -import { WorkflowTriggerWorkspaceService } from 'src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service'; - -@Module({ - imports: [WorkflowCommonModule, WorkflowRunnerModule], - providers: [WorkflowTriggerWorkspaceService, WorkflowTriggerResolver], -}) -export class WorkflowTriggerCoreModule {} diff --git a/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-run.dto.ts b/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-run.dto.ts new file mode 100644 index 000000000000..7516eca15e46 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-run.dto.ts @@ -0,0 +1,9 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars'; + +@ObjectType('WorkflowRun') +export class WorkflowRunDTO { + @Field(() => UUIDScalarType) + workflowRunId: string; +} diff --git a/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto.ts b/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto.ts deleted file mode 100644 index d5433e5b1a7e..000000000000 --- a/packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Field, ObjectType } from '@nestjs/graphql'; - -import { IsObject } from 'class-validator'; -import graphqlTypeJson from 'graphql-type-json'; - -@ObjectType('WorkflowTriggerResult') -export class WorkflowTriggerResultDTO { - @IsObject() - @Field(() => graphqlTypeJson, { - description: 'Execution result in JSON format', - nullable: true, - }) - result?: JSON; -} diff --git a/packages/twenty-server/src/engine/core-modules/workflow/utils/workflow-trigger-graphql-api-exception-handler.util.ts b/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts similarity index 59% rename from packages/twenty-server/src/engine/core-modules/workflow/utils/workflow-trigger-graphql-api-exception-handler.util.ts rename to packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts index e3ac10a136c5..13edf52a99ee 100644 --- a/packages/twenty-server/src/engine/core-modules/workflow/utils/workflow-trigger-graphql-api-exception-handler.util.ts +++ b/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts @@ -1,3 +1,5 @@ +import { Catch, ExceptionFilter } from '@nestjs/common'; + import { InternalServerError, UserInputError, @@ -7,18 +9,19 @@ import { WorkflowTriggerExceptionCode, } from 'src/modules/workflow/workflow-trigger/workflow-trigger.exception'; -export const workflowTriggerGraphqlApiExceptionHandler = (error: Error) => { - if (error instanceof WorkflowTriggerException) { - switch (error.code) { +@Catch(WorkflowTriggerException) +export class WorkflowTriggerGraphqlApiExceptionFilter + implements ExceptionFilter +{ + catch(exception: WorkflowTriggerException) { + switch (exception.code) { case WorkflowTriggerExceptionCode.INVALID_INPUT: - throw new UserInputError(error.message); + throw new UserInputError(exception.message); case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER: case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION: case WorkflowTriggerExceptionCode.INVALID_ACTION_TYPE: default: - throw new InternalServerError(error.message); + throw new InternalServerError(exception.message); } } - - throw error; -}; +} diff --git a/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger-api.module.ts b/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger-api.module.ts new file mode 100644 index 000000000000..8c469833ce29 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger-api.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { WorkflowTriggerResolver } from 'src/engine/core-modules/workflow/workflow-trigger.resolver'; +import { WorkflowTriggerModule } from 'src/modules/workflow/workflow-trigger/workflow-trigger.module'; + +@Module({ + imports: [WorkflowTriggerModule], + providers: [WorkflowTriggerResolver], +}) +export class WorkflowTriggerApiModule {} diff --git a/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.resolver.ts b/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.resolver.ts index 35c8a40882ba..cdce3b954a9b 100644 --- a/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.resolver.ts @@ -1,14 +1,18 @@ -import { UseGuards } from '@nestjs/common'; +import { UseFilters, UseGuards } from '@nestjs/common'; import { Args, Mutation, Resolver } from '@nestjs/graphql'; +import { User } from 'src/engine/core-modules/user/user.entity'; import { RunWorkflowVersionInput } from 'src/engine/core-modules/workflow/dtos/run-workflow-version-input.dto'; -import { WorkflowTriggerResultDTO } from 'src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto'; -import { workflowTriggerGraphqlApiExceptionHandler } from 'src/engine/core-modules/workflow/utils/workflow-trigger-graphql-api-exception-handler.util'; +import { WorkflowRunDTO } from 'src/engine/core-modules/workflow/dtos/workflow-run.dto'; +import { WorkflowTriggerGraphqlApiExceptionFilter } from 'src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter'; +import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; +import { AuthWorkspaceMemberId } from 'src/engine/decorators/auth/auth-workspace-member-id.decorator'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { WorkflowTriggerWorkspaceService } from 'src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service'; -@UseGuards(JwtAuthGuard) @Resolver() +@UseGuards(JwtAuthGuard) +@UseFilters(WorkflowTriggerGraphqlApiExceptionFilter) export class WorkflowTriggerResolver { constructor( private readonly workflowTriggerWorkspaceService: WorkflowTriggerWorkspaceService, @@ -18,28 +22,22 @@ export class WorkflowTriggerResolver { async enableWorkflowTrigger( @Args('workflowVersionId') workflowVersionId: string, ) { - try { - return await this.workflowTriggerWorkspaceService.enableWorkflowTrigger( - workflowVersionId, - ); - } catch (error) { - workflowTriggerGraphqlApiExceptionHandler(error); - } + return await this.workflowTriggerWorkspaceService.enableWorkflowTrigger( + workflowVersionId, + ); } - @Mutation(() => WorkflowTriggerResultDTO) + @Mutation(() => WorkflowRunDTO) async runWorkflowVersion( + @AuthWorkspaceMemberId() workspaceMemberId: string, + @AuthUser() user: User, @Args('input') { workflowVersionId, payload }: RunWorkflowVersionInput, ) { - try { - return { - result: await this.workflowTriggerWorkspaceService.runWorkflowVersion( - workflowVersionId, - payload ?? {}, - ), - }; - } catch (error) { - workflowTriggerGraphqlApiExceptionHandler(error); - } + return await this.workflowTriggerWorkspaceService.runWorkflowVersion( + workflowVersionId, + payload ?? {}, + workspaceMemberId, + user, + ); } } diff --git a/packages/twenty-server/src/engine/decorators/auth/auth-workspace-member-id.decorator.ts b/packages/twenty-server/src/engine/decorators/auth/auth-workspace-member-id.decorator.ts new file mode 100644 index 000000000000..17900a327c7a --- /dev/null +++ b/packages/twenty-server/src/engine/decorators/auth/auth-workspace-member-id.decorator.ts @@ -0,0 +1,11 @@ +import { ExecutionContext, createParamDecorator } from '@nestjs/common'; + +import { getRequest } from 'src/utils/extract-request'; + +export const AuthWorkspaceMemberId = createParamDecorator( + (data: unknown, ctx: ExecutionContext) => { + const request = getRequest(ctx); + + return request.workspaceMemberId; + }, +); diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts index c8797101d6b6..933729c437be 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts @@ -8,12 +8,12 @@ import { import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; +import { ActorModule } from 'src/engine/core-modules/actor/actor.module'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto'; import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver'; import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor'; -import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook'; import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator'; import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; @@ -42,12 +42,9 @@ import { UpdateFieldInput } from './dtos/update-field.input'; ObjectMetadataModule, DataSourceModule, TypeORMModule, + ActorModule, ], - services: [ - IsFieldMetadataDefaultValue, - FieldMetadataService, - CreatedByPreQueryHook, - ], + services: [IsFieldMetadataDefaultValue, FieldMetadataService], resolvers: [ { EntityClass: FieldMetadataEntity, diff --git a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts index 509e58bb31c7..c510f3e4a3b7 100644 --- a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts +++ b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts @@ -82,6 +82,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware req.workspace = data.workspace; req.workspaceId = data.workspace.id; req.workspaceMetadataVersion = metadataVersion; + req.workspaceMemberId = data.workspaceMemberId; } catch (error) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.write( diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.exception.ts b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.exception.ts new file mode 100644 index 000000000000..83d6150586df --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.exception.ts @@ -0,0 +1,12 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class WorkflowActionExecutorException extends CustomException { + code: WorkflowActionExecutorExceptionCode; + constructor(message: string, code: WorkflowActionExecutorExceptionCode) { + super(message, code); + } +} + +export enum WorkflowActionExecutorExceptionCode { + SCOPED_WORKSPACE_NOT_FOUND = 'SCOPED_WORKSPACE_NOT_FOUND', +} diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.factory.ts b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.factory.ts new file mode 100644 index 000000000000..485bb8133235 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.factory.ts @@ -0,0 +1,23 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkflowActionType } from 'src/modules/workflow/common/types/workflow-action.type'; +import { WorkflowActionExecutor } from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.interface'; +import { CodeWorkflowActionExecutor } from 'src/modules/workflow/workflow-action-executor/workflow-action-executors/code-workflow-action-executor'; + +@Injectable() +export class WorkflowActionExecutorFactory { + constructor( + private readonly codeWorkflowActionExecutor: CodeWorkflowActionExecutor, + ) {} + + get(actionType: WorkflowActionType): WorkflowActionExecutor { + switch (actionType) { + case WorkflowActionType.CODE: + return this.codeWorkflowActionExecutor; + default: + throw new Error( + `Workflow action executor not found for action type '${actionType}'`, + ); + } + } +} diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.interface.ts b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.interface.ts similarity index 87% rename from packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.interface.ts rename to packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.interface.ts index 1591af06c027..ea2adfdcf03b 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.interface.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.interface.ts @@ -1,7 +1,7 @@ import { WorkflowAction } from 'src/modules/workflow/common/types/workflow-action.type'; import { WorkflowResult } from 'src/modules/workflow/common/types/workflow-result.type'; -export interface WorkflowActionRunner { +export interface WorkflowActionExecutor { execute({ action, payload, diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.module.ts b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.module.ts new file mode 100644 index 000000000000..c827d1a05f54 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executor.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; + +import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module'; +import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; +import { WorkflowActionExecutorFactory } from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.factory'; +import { CodeWorkflowActionExecutor } from 'src/modules/workflow/workflow-action-executor/workflow-action-executors/code-workflow-action-executor'; + +@Module({ + imports: [ServerlessFunctionModule], + providers: [ + WorkflowActionExecutorFactory, + CodeWorkflowActionExecutor, + ScopedWorkspaceContextFactory, + ], + exports: [WorkflowActionExecutorFactory], +}) +export class WorkflowActionExecutorModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runners/code-workflow-action-runner.ts b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executors/code-workflow-action-executor.ts similarity index 71% rename from packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runners/code-workflow-action-runner.ts rename to packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executors/code-workflow-action-executor.ts index 2afedcb21402..cd11dedbf967 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runners/code-workflow-action-runner.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-action-executor/workflow-action-executors/code-workflow-action-executor.ts @@ -5,13 +5,13 @@ import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/s import { WorkflowAction } from 'src/modules/workflow/common/types/workflow-action.type'; import { WorkflowResult } from 'src/modules/workflow/common/types/workflow-result.type'; import { - WorkflowActionRunnerException, - WorkflowActionRunnerExceptionCode, -} from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.exception'; -import { WorkflowActionRunner } from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.interface'; + WorkflowActionExecutorException, + WorkflowActionExecutorExceptionCode, +} from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.exception'; +import { WorkflowActionExecutor } from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.interface'; @Injectable() -export class CodeWorkflowActionRunner implements WorkflowActionRunner { +export class CodeWorkflowActionExecutor implements WorkflowActionExecutor { constructor( private readonly serverlessFunctionService: ServerlessFunctionService, private readonly scopedWorkspaceContextFactory: ScopedWorkspaceContextFactory, @@ -27,9 +27,9 @@ export class CodeWorkflowActionRunner implements WorkflowActionRunner { const { workspaceId } = this.scopedWorkspaceContextFactory.create(); if (!workspaceId) { - throw new WorkflowActionRunnerException( + throw new WorkflowActionExecutorException( 'Scoped workspace not found', - WorkflowActionRunnerExceptionCode.SCOPED_WORKSPACE_NOT_FOUND, + WorkflowActionExecutorExceptionCode.SCOPED_WORKSPACE_NOT_FOUND, ); } diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.exception.ts b/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.exception.ts deleted file mode 100644 index 9284ce454dab..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.exception.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { CustomException } from 'src/utils/custom-exception'; - -export class WorkflowActionRunnerException extends CustomException { - code: WorkflowActionRunnerExceptionCode; - constructor(message: string, code: WorkflowActionRunnerExceptionCode) { - super(message, code); - } -} - -export enum WorkflowActionRunnerExceptionCode { - SCOPED_WORKSPACE_NOT_FOUND = 'SCOPED_WORKSPACE_NOT_FOUND', -} diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.factory.ts b/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.factory.ts deleted file mode 100644 index 5ab4d2a42fe6..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.factory.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { CodeWorkflowActionRunner } from 'src/modules/workflow/workflow-action-runner/workflow-action-runners/code-workflow-action-runner'; -import { WorkflowActionType } from 'src/modules/workflow/common/types/workflow-action.type'; -import { WorkflowActionRunner } from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.interface'; - -@Injectable() -export class WorkflowActionRunnerFactory { - constructor( - private readonly codeWorkflowActionRunner: CodeWorkflowActionRunner, - ) {} - - get(actionType: WorkflowActionType): WorkflowActionRunner { - switch (actionType) { - case WorkflowActionType.CODE: - return this.codeWorkflowActionRunner; - default: - throw new Error( - `Workflow action executor not found for action type '${actionType}'`, - ); - } - } -} diff --git a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.module.ts b/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.module.ts deleted file mode 100644 index 1bafd6cfe396..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-action-runner/workflow-action-runner.module.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module'; -import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; -import { WorkflowActionRunnerFactory } from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.factory'; -import { CodeWorkflowActionRunner } from 'src/modules/workflow/workflow-action-runner/workflow-action-runners/code-workflow-action-runner'; - -@Module({ - imports: [ServerlessFunctionModule], - providers: [ - WorkflowActionRunnerFactory, - CodeWorkflowActionRunner, - ScopedWorkspaceContextFactory, - ], - exports: [WorkflowActionRunnerFactory], -}) -export class WorkflowActionRunnerModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.exception.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.exception.ts similarity index 62% rename from packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.exception.ts rename to packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.exception.ts index 1bab0da515ee..cdba717b8f67 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.exception.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.exception.ts @@ -1,11 +1,11 @@ import { CustomException } from 'src/utils/custom-exception'; -export class WorkflowRunnerException extends CustomException { +export class WorkflowExecutorException extends CustomException { constructor(message: string, code: string) { super(message, code); } } -export enum WorkflowRunnerExceptionCode { +export enum WorkflowExecutorExceptionCode { WORKFLOW_FAILED = 'WORKFLOW_FAILED', } diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts new file mode 100644 index 000000000000..43d9a6ad6af5 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; + +import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module'; +import { WorkflowActionExecutorModule } from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.module'; +import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workflow-executor.workspace-service'; + +@Module({ + imports: [WorkflowCommonModule, WorkflowActionExecutorModule], + providers: [WorkflowExecutorWorkspaceService], + exports: [WorkflowExecutorWorkspaceService], +}) +export class WorkflowExecutorModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.workspace-service.ts new file mode 100644 index 000000000000..0408056865b1 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.workspace-service.ts @@ -0,0 +1,84 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkflowAction } from 'src/modules/workflow/common/types/workflow-action.type'; +import { WorkflowActionExecutorFactory } from 'src/modules/workflow/workflow-action-executor/workflow-action-executor.factory'; +import { + WorkflowExecutorException, + WorkflowExecutorExceptionCode, +} from 'src/modules/workflow/workflow-executor/workflow-executor.exception'; + +const MAX_RETRIES_ON_FAILURE = 3; + +export type WorkflowExecutionOutput = { + data?: object; + error?: object; +}; + +@Injectable() +export class WorkflowExecutorWorkspaceService { + constructor( + private readonly workflowActionExecutorFactory: WorkflowActionExecutorFactory, + ) {} + + async execute({ + action, + payload, + attemptCount = 1, + }: { + action?: WorkflowAction; + payload?: object; + attemptCount?: number; + }): Promise { + if (!action) { + return { + data: payload, + }; + } + + const workflowActionExecutor = this.workflowActionExecutorFactory.get( + action.type, + ); + + const result = await workflowActionExecutor.execute({ + action, + payload, + }); + + if (result.data) { + return await this.execute({ + action: action.nextAction, + payload: result.data, + }); + } + + if (!result.error) { + throw new WorkflowExecutorException( + 'Execution result error, no data or error', + WorkflowExecutorExceptionCode.WORKFLOW_FAILED, + ); + } + + if (action.settings.errorHandlingOptions.continueOnFailure.value) { + return await this.execute({ + action: action.nextAction, + payload, + }); + } + + if ( + action.settings.errorHandlingOptions.retryOnFailure.value && + attemptCount < MAX_RETRIES_ON_FAILURE + ) { + return await this.execute({ + action, + payload, + attemptCount: attemptCount + 1, + }); + } + + throw new WorkflowExecutorException( + `Workflow failed: ${result.error}`, + WorkflowExecutorExceptionCode.WORKFLOW_FAILED, + ); + } +} diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts similarity index 65% rename from packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.job.ts rename to packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts index 64f1d6984fa5..619d4babffd7 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.job.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts @@ -5,8 +5,8 @@ import { Processor } from 'src/engine/integrations/message-queue/decorators/proc import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workflow-common.workspace-service'; -import { WorkflowRunnerWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-runner.workspace-service'; -import { WorkflowStatusWorkspaceService } from 'src/modules/workflow/workflow-status/workflow-status.workspace-service'; +import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workflow-executor.workspace-service'; +import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; export type RunWorkflowJobData = { workspaceId: string; @@ -16,20 +16,20 @@ export type RunWorkflowJobData = { }; @Processor({ queueName: MessageQueue.workflowQueue, scope: Scope.REQUEST }) -export class WorkflowRunnerJob { +export class RunWorkflowJob { constructor( private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService, - private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService, - private readonly workflowStatusWorkspaceService: WorkflowStatusWorkspaceService, + private readonly workflowExecutorWorkspaceService: WorkflowExecutorWorkspaceService, + private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService, ) {} - @Process(WorkflowRunnerJob.name) + @Process(RunWorkflowJob.name) async handle({ workflowVersionId, workflowRunId, payload, }: RunWorkflowJobData): Promise { - await this.workflowStatusWorkspaceService.startWorkflowRun(workflowRunId); + await this.workflowRunWorkspaceService.startWorkflowRun(workflowRunId); const workflowVersion = await this.workflowCommonWorkspaceService.getWorkflowVersion( @@ -37,17 +37,17 @@ export class WorkflowRunnerJob { ); try { - await this.workflowRunnerWorkspaceService.run({ + await this.workflowExecutorWorkspaceService.execute({ action: workflowVersion.trigger.nextAction, payload, }); - await this.workflowStatusWorkspaceService.endWorkflowRun( + await this.workflowRunWorkspaceService.endWorkflowRun( workflowRunId, WorkflowRunStatus.COMPLETED, ); } catch (error) { - await this.workflowStatusWorkspaceService.endWorkflowRun( + await this.workflowRunWorkspaceService.endWorkflowRun( workflowRunId, WorkflowRunStatus.FAILED, ); diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.exception.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.exception.ts new file mode 100644 index 000000000000..e1668045ac0a --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.exception.ts @@ -0,0 +1,13 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class WorkflowRunException extends CustomException { + code: WorkflowRunExceptionCode; + constructor(message: string, code: WorkflowRunExceptionCode) { + super(message, code); + } +} + +export enum WorkflowRunExceptionCode { + WORKFLOW_RUN_NOT_FOUND = 'WORKFLOW_RUN_NOT_FOUND', + INVALID_OPERATION = 'INVALID_OPERATION', +} diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts new file mode 100644 index 000000000000..27ec554daf5f --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; + +import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; + +@Module({ + providers: [WorkflowRunWorkspaceService], + exports: [WorkflowRunWorkspaceService], +}) +export class WorkflowRunModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts similarity index 79% rename from packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.workspace-service.ts rename to packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts index 949ae0c27e21..a772d9a6fb28 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts @@ -7,12 +7,12 @@ import { WorkflowRunWorkspaceEntity, } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; import { - WorkflowStatusException, - WorkflowStatusExceptionCode, -} from 'src/modules/workflow/workflow-status/workflow-status.exception'; + WorkflowRunException, + WorkflowRunExceptionCode, +} from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.exception'; @Injectable() -export class WorkflowStatusWorkspaceService { +export class WorkflowRunWorkspaceService { constructor(private readonly twentyORMManager: TwentyORMManager) {} async createWorkflowRun(workflowVersionId: string, createdBy: ActorMetadata) { @@ -41,16 +41,16 @@ export class WorkflowStatusWorkspaceService { }); if (!workflowRunToUpdate) { - throw new WorkflowStatusException( + throw new WorkflowRunException( 'No workflow run to start', - WorkflowStatusExceptionCode.WORKFLOW_RUN_NOT_FOUND, + WorkflowRunExceptionCode.WORKFLOW_RUN_NOT_FOUND, ); } if (workflowRunToUpdate.status !== WorkflowRunStatus.NOT_STARTED) { - throw new WorkflowStatusException( + throw new WorkflowRunException( 'Workflow run already started', - WorkflowStatusExceptionCode.INVALID_OPERATION, + WorkflowRunExceptionCode.INVALID_OPERATION, ); } @@ -71,16 +71,16 @@ export class WorkflowStatusWorkspaceService { }); if (!workflowRunToUpdate) { - throw new WorkflowStatusException( + throw new WorkflowRunException( 'No workflow run to end', - WorkflowStatusExceptionCode.WORKFLOW_RUN_NOT_FOUND, + WorkflowRunExceptionCode.WORKFLOW_RUN_NOT_FOUND, ); } if (workflowRunToUpdate.status !== WorkflowRunStatus.RUNNING) { - throw new WorkflowStatusException( + throw new WorkflowRunException( 'Workflow cannot be ended as it is not running', - WorkflowStatusExceptionCode.INVALID_OPERATION, + WorkflowRunExceptionCode.INVALID_OPERATION, ); } diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.module.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.module.ts index 9bd7e69d3876..bf33e5b21f02 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.module.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.module.ts @@ -1,18 +1,14 @@ import { Module } from '@nestjs/common'; import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module'; -import { WorkflowActionRunnerModule } from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.module'; -import { WorkflowRunnerJob } from 'src/modules/workflow/workflow-runner/workflow-runner.job'; +import { WorkflowExecutorModule } from 'src/modules/workflow/workflow-executor/workflow-executor.module'; +import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.module'; import { WorkflowRunnerWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-runner.workspace-service'; -import { WorkflowStatusModule } from 'src/modules/workflow/workflow-status/workflow-status.module'; @Module({ - imports: [ - WorkflowCommonModule, - WorkflowActionRunnerModule, - WorkflowStatusModule, - ], - providers: [WorkflowRunnerWorkspaceService, WorkflowRunnerJob], + imports: [WorkflowRunModule, WorkflowCommonModule, WorkflowExecutorModule], + providers: [WorkflowRunnerWorkspaceService, RunWorkflowJob], exports: [WorkflowRunnerWorkspaceService], }) export class WorkflowRunnerModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.workspace-service.ts index 0822bc5d2f34..bc43bd1b5a0d 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-runner.workspace-service.ts @@ -1,84 +1,45 @@ import { Injectable } from '@nestjs/common'; -import { WorkflowAction } from 'src/modules/workflow/common/types/workflow-action.type'; -import { WorkflowActionRunnerFactory } from 'src/modules/workflow/workflow-action-runner/workflow-action-runner.factory'; +import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; +import { ActorMetadata } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type'; import { - WorkflowRunnerException, - WorkflowRunnerExceptionCode, -} from 'src/modules/workflow/workflow-runner/workflow-runner.exception'; - -const MAX_RETRIES_ON_FAILURE = 3; - -export type WorkflowRunOutput = { - data?: object; - error?: object; -}; + RunWorkflowJob, + RunWorkflowJobData, +} from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; @Injectable() export class WorkflowRunnerWorkspaceService { constructor( - private readonly workflowActionRunnerFactory: WorkflowActionRunnerFactory, + private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService, + @InjectMessageQueue(MessageQueue.workflowQueue) + private readonly messageQueueService: MessageQueueService, ) {} - async run({ - action, - payload, - attemptCount = 1, - }: { - action?: WorkflowAction; - payload?: object; - attemptCount?: number; - }): Promise { - if (!action) { - return { - data: payload, - }; - } - - const workflowActionRunner = this.workflowActionRunnerFactory.get( - action.type, - ); - - const result = await workflowActionRunner.execute({ - action, - payload, - }); - - if (result.data) { - return await this.run({ - action: action.nextAction, - payload: result.data, - }); - } - - if (!result.error) { - throw new WorkflowRunnerException( - 'Execution result error, no data or error', - WorkflowRunnerExceptionCode.WORKFLOW_FAILED, + async run( + workspaceId: string, + workflowVersionId: string, + payload: object, + source: ActorMetadata, + ) { + const workflowRunId = + await this.workflowRunWorkspaceService.createWorkflowRun( + workflowVersionId, + source, ); - } - - if (action.settings.errorHandlingOptions.continueOnFailure.value) { - return await this.run({ - action: action.nextAction, - payload, - }); - } - if ( - action.settings.errorHandlingOptions.retryOnFailure.value && - attemptCount < MAX_RETRIES_ON_FAILURE - ) { - return await this.run({ - action, - payload, - attemptCount: attemptCount + 1, - }); - } - - throw new WorkflowRunnerException( - `Workflow failed: ${result.error}`, - WorkflowRunnerExceptionCode.WORKFLOW_FAILED, + await this.messageQueueService.add( + RunWorkflowJob.name, + { + workspaceId, + workflowVersionId, + payload: payload, + workflowRunId, + }, ); + + return { workflowRunId }; } } diff --git a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.exception.ts b/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.exception.ts deleted file mode 100644 index 6510815f090c..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.exception.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { CustomException } from 'src/utils/custom-exception'; - -export class WorkflowStatusException extends CustomException { - code: WorkflowStatusExceptionCode; - constructor(message: string, code: WorkflowStatusExceptionCode) { - super(message, code); - } -} - -export enum WorkflowStatusExceptionCode { - WORKFLOW_RUN_NOT_FOUND = 'WORKFLOW_RUN_NOT_FOUND', - INVALID_OPERATION = 'INVALID_OPERATION', -} diff --git a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.module.ts b/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.module.ts deleted file mode 100644 index 14eec4fb10c7..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-status/workflow-status.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { WorkflowStatusWorkspaceService } from 'src/modules/workflow/workflow-status/workflow-status.workspace-service'; - -@Module({ - providers: [WorkflowStatusWorkspaceService], - exports: [WorkflowStatusWorkspaceService], -}) -export class WorkflowStatusModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job.ts index f96e91b14e4f..0137dd57a3cd 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job.ts @@ -1,18 +1,16 @@ import { Scope } from '@nestjs/common'; -import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; -import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity'; +import { WorkflowRunnerWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-runner.workspace-service'; import { - RunWorkflowJobData, - WorkflowRunnerJob, -} from 'src/modules/workflow/workflow-runner/workflow-runner.job'; -import { WorkflowStatusWorkspaceService } from 'src/modules/workflow/workflow-status/workflow-status.workspace-service'; + WorkflowTriggerException, + WorkflowTriggerExceptionCode, +} from 'src/modules/workflow/workflow-trigger/workflow-trigger.exception'; export type WorkflowEventTriggerJobData = { workspaceId: string; @@ -23,10 +21,8 @@ export type WorkflowEventTriggerJobData = { @Processor({ queueName: MessageQueue.workflowQueue, scope: Scope.REQUEST }) export class WorkflowEventTriggerJob { constructor( - @InjectMessageQueue(MessageQueue.workflowQueue) - private readonly messageQueueService: MessageQueueService, private readonly twentyORMManager: TwentyORMManager, - private readonly workflowStatusWorkspaceService: WorkflowStatusWorkspaceService, + private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService, ) {} @Process(WorkflowEventTriggerJob.name) @@ -41,23 +37,20 @@ export class WorkflowEventTriggerJob { }); if (!workflow.publishedVersionId) { - throw new Error('Workflow has no published version'); - } - - const workflowRunId = - await this.workflowStatusWorkspaceService.createWorkflowRun( - workflow.publishedVersionId, - { - source: FieldActorSource.WORKFLOW, - name: workflow.name, - }, + throw new WorkflowTriggerException( + 'Workflow has no published version', + WorkflowTriggerExceptionCode.INTERNAL_ERROR, ); + } - this.messageQueueService.add(WorkflowRunnerJob.name, { - workspaceId: data.workspaceId, - workflowVersionId: workflow.publishedVersionId, - payload: data.payload, - workflowRunId, - }); + await this.workflowRunnerWorkspaceService.run( + data.workspaceId, + workflow.publishedVersionId, + data.payload, + { + source: FieldActorSource.WORKFLOW, + name: workflow.name, + }, + ); } } diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-trigger-job.module.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-trigger-job.module.ts deleted file mode 100644 index 4f03c12fd958..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/jobs/workflow-trigger-job.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { WorkflowRunnerModule } from 'src/modules/workflow/workflow-runner/workflow-runner.module'; -import { WorkflowStatusModule } from 'src/modules/workflow/workflow-status/workflow-status.module'; -import { WorkflowEventTriggerJob } from 'src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job'; - -@Module({ - imports: [WorkflowRunnerModule, WorkflowStatusModule], - providers: [WorkflowEventTriggerJob], -}) -export class WorkflowTriggerJobModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/workflow-trigger-listener.module.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/workflow-trigger-listener.module.ts deleted file mode 100644 index 0c8b8afec9c3..000000000000 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/listeners/workflow-trigger-listener.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; -import { DatabaseEventTriggerListener } from 'src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener'; - -@Module({ - imports: [FeatureFlagModule], - providers: [DatabaseEventTriggerListener], -}) -export class WorkflowTriggerListenerModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.module.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.module.ts index 6ffdefb0e13c..ca8a6c83d71b 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.module.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.module.ts @@ -1,9 +1,21 @@ import { Module } from '@nestjs/common'; -import { WorkflowTriggerJobModule } from 'src/modules/workflow/workflow-trigger/jobs/workflow-trigger-job.module'; -import { WorkflowTriggerListenerModule } from 'src/modules/workflow/workflow-trigger/listeners/workflow-trigger-listener.module'; +import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; +import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module'; +import { WorkflowRunnerModule } from 'src/modules/workflow/workflow-runner/workflow-runner.module'; +import { WorkflowEventTriggerJob } from 'src/modules/workflow/workflow-trigger/jobs/workflow-event-trigger.job'; +import { DatabaseEventTriggerListener } from 'src/modules/workflow/workflow-trigger/listeners/database-event-trigger.listener'; +import { WorkflowTriggerWorkspaceService } from 'src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service'; @Module({ - imports: [WorkflowTriggerJobModule, WorkflowTriggerListenerModule], + imports: [WorkflowCommonModule, WorkflowRunnerModule, FeatureFlagModule], + providers: [ + WorkflowTriggerWorkspaceService, + ScopedWorkspaceContextFactory, + DatabaseEventTriggerListener, + WorkflowEventTriggerJob, + ], + exports: [WorkflowTriggerWorkspaceService], }) export class WorkflowTriggerModule {} diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts index ea661862dd98..54079003ea7d 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts @@ -1,5 +1,8 @@ import { Injectable } from '@nestjs/common'; +import { buildCreatedByFromWorkspaceMember } from 'src/engine/core-modules/actor/utils/build-created-by-from-workspace-member.util'; +import { User } from 'src/engine/core-modules/user/user.entity'; +import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { WorkflowEventListenerWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-event-listener.workspace-entity'; import { WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity'; @@ -19,26 +22,43 @@ export class WorkflowTriggerWorkspaceService { constructor( private readonly twentyORMManager: TwentyORMManager, private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService, + private readonly scopedWorkspaceContextFactory: ScopedWorkspaceContextFactory, private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService, ) {} - async runWorkflowVersion(workflowVersionId: string, payload: object) { + async runWorkflowVersion( + workflowVersionId: string, + payload: object, + workspaceMemberId: string, + user: User, + ) { + const workspaceId = this.scopedWorkspaceContextFactory.create().workspaceId; + + if (!workspaceId) { + throw new WorkflowTriggerException( + 'No workspace id found', + WorkflowTriggerExceptionCode.INTERNAL_ERROR, + ); + } + const workflowVersion = await this.workflowCommonWorkspaceService.getWorkflowVersion( workflowVersionId, ); - try { - return await this.workflowRunnerWorkspaceService.run({ - action: workflowVersion.trigger.nextAction, - payload, - }); - } catch (error) { + if (!workflowVersion) { throw new WorkflowTriggerException( - `Error running workflow version ${error}`, - WorkflowTriggerExceptionCode.INTERNAL_ERROR, + 'No workflow version found', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION, ); } + + return await this.workflowRunnerWorkspaceService.run( + workspaceId, + workflowVersionId, + payload, + buildCreatedByFromWorkspaceMember(workspaceMemberId, user), + ); } async enableWorkflowTrigger(workflowVersionId: string) { diff --git a/packages/twenty-server/src/modules/workflow/workflow.module.ts b/packages/twenty-server/src/modules/workflow/workflow.module.ts index 97536cf9045c..5f794b972a4d 100644 --- a/packages/twenty-server/src/modules/workflow/workflow.module.ts +++ b/packages/twenty-server/src/modules/workflow/workflow.module.ts @@ -1,9 +1,8 @@ import { Module } from '@nestjs/common'; -import { WorkflowRunnerModule } from 'src/modules/workflow/workflow-runner/workflow-runner.module'; import { WorkflowTriggerModule } from 'src/modules/workflow/workflow-trigger/workflow-trigger.module'; @Module({ - imports: [WorkflowRunnerModule, WorkflowTriggerModule], + imports: [WorkflowTriggerModule], }) export class WorkflowModule {} From be50a6256ff5e1909ee9d5f0cea5256df30a20be Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Wed, 21 Aug 2024 17:56:33 +0200 Subject: [PATCH 15/19] Check workflow version is valid before publishing (#6702) Fix https://github.com/twentyhq/twenty/issues/6670 --- ...ow-trigger-graphql-api-exception.filter.ts | 4 +- .../workflow-common.workspace-service.ts | 2 +- .../utils/assert-workflow-version-is-valid.ts | 66 +++++++++++++++++++ .../workflow-trigger.workspace-service.ts | 12 ++-- 4 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 packages/twenty-server/src/modules/workflow/workflow-trigger/utils/assert-workflow-version-is-valid.ts diff --git a/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts b/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts index 13edf52a99ee..bc707d5500ce 100644 --- a/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts +++ b/packages/twenty-server/src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter.ts @@ -16,10 +16,10 @@ export class WorkflowTriggerGraphqlApiExceptionFilter catch(exception: WorkflowTriggerException) { switch (exception.code) { case WorkflowTriggerExceptionCode.INVALID_INPUT: - throw new UserInputError(exception.message); - case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER: case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION: case WorkflowTriggerExceptionCode.INVALID_ACTION_TYPE: + case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER: + throw new UserInputError(exception.message); default: throw new InternalServerError(exception.message); } diff --git a/packages/twenty-server/src/modules/workflow/common/workflow-common.workspace-service.ts b/packages/twenty-server/src/modules/workflow/common/workflow-common.workspace-service.ts index 3db70e89c0d5..a975ecd42afe 100644 --- a/packages/twenty-server/src/modules/workflow/common/workflow-common.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/common/workflow-common.workspace-service.ts @@ -35,7 +35,7 @@ export class WorkflowCommonWorkspaceService { ); } - if (!workflowVersion.trigger || !workflowVersion.trigger?.type) { + if (!workflowVersion.trigger) { throw new WorkflowTriggerException( 'Workflow version does not contains trigger', WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION, diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/utils/assert-workflow-version-is-valid.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/utils/assert-workflow-version-is-valid.ts new file mode 100644 index 000000000000..d38cc8034460 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/utils/assert-workflow-version-is-valid.ts @@ -0,0 +1,66 @@ +import { WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity'; +import { + WorkflowTrigger, + WorkflowTriggerType, +} from 'src/modules/workflow/common/types/workflow-trigger.type'; +import { + WorkflowTriggerException, + WorkflowTriggerExceptionCode, +} from 'src/modules/workflow/workflow-trigger/workflow-trigger.exception'; + +export function assertWorkflowVersionIsValid( + workflowVersion: Omit & { + trigger: WorkflowTrigger; + }, +) { + if (!workflowVersion.trigger) { + throw new WorkflowTriggerException( + 'Workflow version does not contain trigger', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION, + ); + } + + if (!workflowVersion.trigger.type) { + throw new WorkflowTriggerException( + 'No trigger type provided', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER, + ); + } + + if (!workflowVersion.trigger.nextAction) { + throw new WorkflowTriggerException( + 'No next action provided in trigger', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER, + ); + } + + assertTriggerSettingsAreValid( + workflowVersion.trigger.type, + workflowVersion.trigger.settings, + ); +} + +function assertTriggerSettingsAreValid( + triggerType: WorkflowTriggerType, + settings: any, +) { + switch (triggerType) { + case WorkflowTriggerType.DATABASE_EVENT: + assertDatabaseEventTriggerSettingsAreValid(settings); + break; + default: + throw new WorkflowTriggerException( + 'Invalid trigger type for enabling workflow trigger', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER, + ); + } +} + +function assertDatabaseEventTriggerSettingsAreValid(settings: any) { + if (!settings?.eventName) { + throw new WorkflowTriggerException( + 'No event name provided in database event trigger', + WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER, + ); + } +} diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts index 54079003ea7d..a7782753028e 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/workflow-trigger.workspace-service.ts @@ -12,6 +12,7 @@ import { } from 'src/modules/workflow/common/types/workflow-trigger.type'; import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workflow-common.workspace-service'; import { WorkflowRunnerWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-runner.workspace-service'; +import { assertWorkflowVersionIsValid } from 'src/modules/workflow/workflow-trigger/utils/assert-workflow-version-is-valid'; import { WorkflowTriggerException, WorkflowTriggerExceptionCode, @@ -67,6 +68,8 @@ export class WorkflowTriggerWorkspaceService { workflowVersionId, ); + assertWorkflowVersionIsValid(workflowVersion); + switch (workflowVersion.trigger.type) { case WorkflowTriggerType.DATABASE_EVENT: await this.upsertEventListenerAndPublishVersion( @@ -87,14 +90,7 @@ export class WorkflowTriggerWorkspaceService { workflowVersionId: string, trigger: WorkflowDatabaseEventTrigger, ) { - const eventName = trigger?.settings?.eventName; - - if (!eventName) { - throw new WorkflowTriggerException( - 'No event name provided in database event trigger', - WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER, - ); - } + const eventName = trigger.settings.eventName; const workflowEventListenerRepository = await this.twentyORMManager.getRepository( From da4bd73881a607d71a31763dc97a631b1fb1107e Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 21 Aug 2024 20:12:14 +0200 Subject: [PATCH 16/19] Fix logging error in webhook system --- .../workspace-query-runner/jobs/call-webhook-jobs.job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts index f4f6f9dca930..74e955093e6d 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts @@ -81,7 +81,7 @@ export class CallWebhookJobsJob { if (webhooks.length) { this.logger.log( - `CallWebhookJobsJob on eventName '${event}' called on webhooks ids [\n"${webhooks + `CallWebhookJobsJob on eventName '${eventName}' called on webhooks ids [\n"${webhooks .map((webhook) => webhook.id) .join('",\n"')}"\n]`, ); From eab202f107963c3ada12d4b3aec4edf3de04f342 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Thu, 22 Aug 2024 00:28:19 +0200 Subject: [PATCH 17/19] Make workspaceMemberId optional in JWT for workspaces that are not ACTIVE (#6714) WorkspaceMemberId is mandatory in the jwt token generated for a given user on a given workspace. However, when a user signs up, it does not have a workspaceMemberId yet. --- .../auth/services/token.service.ts | 49 +++++++++++-------- .../auth/strategies/jwt.auth.strategy.ts | 2 +- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts index 200430de73b0..4cf18c206060 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/token.service.ts @@ -38,7 +38,10 @@ import { import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service'; import { User } from 'src/engine/core-modules/user/user.entity'; -import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { + Workspace, + WorkspaceActivationStatus, +} from 'src/engine/core-modules/workspace/workspace.entity'; import { EmailService } from 'src/engine/integrations/email/email.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; @@ -94,33 +97,39 @@ export class TokenService { ); } - const workspaceIdNonNullable = workspaceId - ? workspaceId - : user.defaultWorkspace.id; + const tokenWorkspaceId = workspaceId ?? user.defaultWorkspace.id; + let tokenWorkspaceMemberId: string | undefined; - const workspaceMemberRepository = - await this.twentyORMGlobalManager.getRepositoryForWorkspace( - workspaceIdNonNullable, - 'workspaceMember', - ); + if ( + user.defaultWorkspace.activationStatus === + WorkspaceActivationStatus.ACTIVE + ) { + const workspaceMemberRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + tokenWorkspaceId, + 'workspaceMember', + ); - const workspaceMember = await workspaceMemberRepository.findOne({ - where: { - userId: user.id, - }, - }); + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { + userId: user.id, + }, + }); - if (!workspaceMember) { - throw new AuthException( - 'User is not a member of the workspace', - AuthExceptionCode.FORBIDDEN_EXCEPTION, - ); + if (!workspaceMember) { + throw new AuthException( + 'User is not a member of the workspace', + AuthExceptionCode.FORBIDDEN_EXCEPTION, + ); + } + + tokenWorkspaceMemberId = workspaceMember.id; } const jwtPayload: JwtPayload = { sub: user.id, workspaceId: workspaceId ? workspaceId : user.defaultWorkspace.id, - workspaceMemberId: workspaceMember.id, + workspaceMemberId: tokenWorkspaceMemberId, }; return { diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts index 5764e967815f..b6a0954a268f 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/jwt.auth.strategy.ts @@ -20,7 +20,7 @@ import { ApiKeyWorkspaceEntity } from 'src/modules/api-key/standard-objects/api- export type JwtPayload = { sub: string; workspaceId: string; - workspaceMemberId: string; + workspaceMemberId?: string; jti?: string; }; From 9898ca3e53b6ea4be3514b274938b63c2f679a50 Mon Sep 17 00:00:00 2001 From: "gitstart-app[bot]" <57568882+gitstart-app[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:42:49 +0200 Subject: [PATCH 18/19] TWNTY-6135 - Improve Data Importer Select Matching (#6338) ### Description: - we move all logic about the unmatchedOptions to a new component called UnmatchColumn, because as it will be a full line in the table, it was better to update where the component will be rendered - In the latest changes to keep the columns when we change the step to step 3 and go back to step 2, we added a fallback state initialComputedColumnsState that saves the columns and only reverts the updates when we go back to step 1 or close by clicking the X button ### Refs: #6135 ``` It was necessary to add references and floating styles to the generic component to fix the bug when the last option was open and the dropdown was being hidden in the next row of the spreadsheet table. We fixed the same problem that occurs in the companies table as well ``` we used this approach mentioned on this documentation to be able to use the hook without calling it on each component, we are calling only once, on the shared component \ before: ![](https://assets-service.gitstart.com/25493/2c994e0f-6548-4a9e-8b22-2c6eccb73b2e.png) now: ![](https://assets-service.gitstart.com/25493/f56fd516-7e95-4616-b1ed-c9ea5195a8ae.png)### Demo: Fixes #6135 NOTES: the enter key are not working on main branch too --------- Co-authored-by: gitstart-twenty Co-authored-by: Lucas Bordeau --- .../input/components/SelectFieldInput.tsx | 168 +++-------- .../hooks/useBuildAvailableFieldsForImport.ts | 32 +++ .../types/AvailableFieldForImport.ts | 9 +- .../spreadsheet-import/components/Heading.tsx | 2 +- .../components/ModalWrapper.tsx | 3 +- ...ReactSpreadsheetImportContextProvider.tsx} | 6 +- .../components/StepNavigationButton.tsx | 35 +-- .../__tests__/useSpreadsheetImport.test.tsx | 4 +- .../useSpreadsheetImportInitialStep.test.ts | 16 +- .../useSpreadsheetImportInternal.test.tsx | 6 +- .../hooks/useSpreadsheetImportInitialStep.ts | 17 +- .../hooks/useSpreadsheetImportInternal.ts | 2 +- .../provider/components/SpreadsheetImport.tsx | 10 +- .../components/SpreadsheetImportProvider.tsx | 7 +- .../MatchColumnsStep/MatchColumnsStep.tsx | 107 +++++-- .../components/ColumnGrid.tsx | 48 +++- .../components/SubMatchingSelect.tsx | 99 +++++-- .../components/TemplateColumn.tsx | 103 +------ .../components/UnmatchColumn.tsx | 111 ++++++++ .../states/initialComputedColumnsState.ts | 41 +++ .../SelectHeaderStep/SelectHeaderStep.tsx | 47 +++- .../SelectSheetStep/SelectSheetStep.tsx | 58 +++- .../components/SpreadsheetImportStepper.tsx | 146 ++++++++++ ... => SpreadsheetImportStepperContainer.tsx} | 12 +- .../steps/components/UploadFlow.tsx | 260 ------------------ .../components/UploadStep/UploadStep.tsx | 85 +++++- .../UploadStep/components/DropZone.tsx | 2 +- .../ValidationStep/ValidationStep.tsx | 22 +- .../__stories__/MatchColumns.stories.tsx | 13 +- .../__stories__/SelectHeader.stories.tsx | 16 +- .../__stories__/SelectSheet.stories.tsx | 34 ++- .../components/__stories__/Steps.stories.tsx | 8 +- .../components/__stories__/Upload.stories.tsx | 18 +- .../__stories__/Validation.stories.tsx | 7 +- .../steps/types/SpreadsheetImportStep.ts | 30 ++ .../steps/types/SpreadsheetImportStepType.ts | 8 + .../modules/spreadsheet-import/types/index.ts | 48 ++-- .../utils/getMatchedColumns.ts | 2 +- .../spreadsheet-import/utils/setColumn.ts | 2 +- .../spreadsheet-import/utils/uniqueEntries.ts | 2 +- .../modules/ui/input/components/Select.tsx | 4 +- .../ui/input/components/SelectInput.tsx | 180 ++++++++++++ .../navigation/step-bar/components/Step.tsx | 34 ++- .../src/display/tag/components/Tag.tsx | 2 +- 44 files changed, 1209 insertions(+), 657 deletions(-) rename packages/twenty-front/src/modules/spreadsheet-import/components/{Providers.tsx => ReactSpreadsheetImportContextProvider.tsx} (72%) create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState.ts create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepper.tsx rename packages/twenty-front/src/modules/spreadsheet-import/steps/components/{Steps.tsx => SpreadsheetImportStepperContainer.tsx} (81%) delete mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStep.ts create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStepType.ts create mode 100644 packages/twenty-front/src/modules/ui/input/components/SelectInput.tsx diff --git a/packages/twenty-front/src/modules/object-record/record-field/meta-types/input/components/SelectFieldInput.tsx b/packages/twenty-front/src/modules/object-record/record-field/meta-types/input/components/SelectFieldInput.tsx index 7def066ed26d..2d9b4072e059 100644 --- a/packages/twenty-front/src/modules/object-record/record-field/meta-types/input/components/SelectFieldInput.tsx +++ b/packages/twenty-front/src/modules/object-record/record-field/meta-types/input/components/SelectFieldInput.tsx @@ -1,29 +1,15 @@ -import styled from '@emotion/styled'; -import { useRef, useState } from 'react'; -import { useRecoilValue } from 'recoil'; -import { Key } from 'ts-key-enum'; - import { useClearField } from '@/object-record/record-field/hooks/useClearField'; import { useSelectField } from '@/object-record/record-field/meta-types/hooks/useSelectField'; import { FieldInputEvent } from '@/object-record/record-field/types/FieldInputEvent'; import { SINGLE_ENTITY_SELECT_BASE_LIST } from '@/object-record/relation-picker/constants/SingleEntitySelectBaseList'; -import { DropdownMenu } from '@/ui/layout/dropdown/components/DropdownMenu'; -import { DropdownMenuItemsContainer } from '@/ui/layout/dropdown/components/DropdownMenuItemsContainer'; -import { DropdownMenuSearchInput } from '@/ui/layout/dropdown/components/DropdownMenuSearchInput'; -import { DropdownMenuSeparator } from '@/ui/layout/dropdown/components/DropdownMenuSeparator'; +import { SelectOption } from '@/spreadsheet-import/types'; +import { SelectInput } from '@/ui/input/components/SelectInput'; import { SelectableList } from '@/ui/layout/selectable-list/components/SelectableList'; -import { useSelectableListStates } from '@/ui/layout/selectable-list/hooks/internal/useSelectableListStates'; import { useSelectableList } from '@/ui/layout/selectable-list/hooks/useSelectableList'; -import { MenuItemSelectTag } from '@/ui/navigation/menu-item/components/MenuItemSelectTag'; import { useScopedHotkeys } from '@/ui/utilities/hotkey/hooks/useScopedHotkeys'; -import { useListenClickOutside } from '@/ui/utilities/pointer-event/hooks/useListenClickOutside'; -import { isDefined } from '~/utils/isDefined'; - -const StyledRelationPickerContainer = styled.div` - left: -1px; - position: absolute; - top: -1px; -`; +import { useState } from 'react'; +import { Key } from 'ts-key-enum'; +import { isDefined } from 'twenty-ui'; type SelectFieldInputProps = { onSubmit?: FieldInputEvent; @@ -36,55 +22,30 @@ export const SelectFieldInput = ({ }: SelectFieldInputProps) => { const { persistField, fieldDefinition, fieldValue, hotkeyScope } = useSelectField(); - const { selectedItemIdState } = useSelectableListStates({ - selectableListScopeId: SINGLE_ENTITY_SELECT_BASE_LIST, - }); + const [selectWrapperRef, setSelectWrapperRef] = + useState(null); + + const [filteredOptions, setFilteredOptions] = useState([]); + const { handleResetSelectedPosition } = useSelectableList( SINGLE_ENTITY_SELECT_BASE_LIST, ); const clearField = useClearField(); - const selectedItemId = useRecoilValue(selectedItemIdState); - const [searchFilter, setSearchFilter] = useState(''); - const containerRef = useRef(null); - const selectedOption = fieldDefinition.metadata.options.find( (option) => option.value === fieldValue, ); - - const optionsToSelect = - fieldDefinition.metadata.options.filter((option) => { - return ( - option.value !== fieldValue && - option.label.toLowerCase().includes(searchFilter.toLowerCase()) - ); - }) || []; - - const optionsInDropDown = selectedOption - ? [selectedOption, ...optionsToSelect] - : optionsToSelect; - // handlers const handleClearField = () => { clearField(); onCancel?.(); }; - useListenClickOutside({ - refs: [containerRef], - callback: (event) => { - event.stopImmediatePropagation(); + const handleSubmit = (option: SelectOption) => { + onSubmit?.(() => persistField(option?.value)); - const weAreNotInAnHTMLInput = !( - event.target instanceof HTMLInputElement && - event.target.tagName === 'INPUT' - ); - if (weAreNotInAnHTMLInput && isDefined(onCancel)) { - onCancel(); - handleResetSelectedPosition(); - } - }, - }); + handleResetSelectedPosition(); + }; useScopedHotkeys( Key.Escape, @@ -96,81 +57,40 @@ export const SelectFieldInput = ({ [onCancel, handleResetSelectedPosition], ); - useScopedHotkeys( - Key.Enter, - () => { - const selectedOption = optionsInDropDown.find((option) => - option.label.toLowerCase().includes(searchFilter.toLowerCase()), - ); - - if (isDefined(selectedOption)) { - onSubmit?.(() => persistField(selectedOption.value)); - } - handleResetSelectedPosition(); - }, - hotkeyScope, - ); - const optionIds = [ `No ${fieldDefinition.label}`, - ...optionsInDropDown.map((option) => option.value), + ...filteredOptions.map((option) => option.value), ]; return ( - { - const option = optionsInDropDown.find( - (option) => option.value === itemId, - ); - if (isDefined(option)) { - onSubmit?.(() => persistField(option.value)); - handleResetSelectedPosition(); - } - }} - > - - - setSearchFilter(event.currentTarget.value)} - autoFocus - /> - - - - {fieldDefinition.metadata.isNullable ?? ( - - )} - - {optionsInDropDown.map((option) => { - return ( - { - onSubmit?.(() => persistField(option.value)); - handleResetSelectedPosition(); - }} - isKeySelected={selectedItemId === option.value} - /> - ); - })} - - - - +
+ { + const option = filteredOptions.find( + (option) => option.value === itemId, + ); + if (isDefined(option)) { + onSubmit?.(() => persistField(option.value)); + handleResetSelectedPosition(); + } + }} + > + + +
); }; diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts index 7128e6129844..7e4edf83926a 100644 --- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts +++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts @@ -123,6 +123,38 @@ export const useBuildAvailableFieldsForImport = () => { ), }); }); + } else if (fieldMetadataItem.type === FieldMetadataType.Select) { + availableFieldsForImport.push({ + icon: getIcon(fieldMetadataItem.icon), + label: fieldMetadataItem.label, + key: fieldMetadataItem.name, + fieldType: { + type: 'select', + options: + fieldMetadataItem.options?.map((option) => ({ + label: option.label, + value: option.value, + color: option.color, + })) || [], + }, + fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions( + fieldMetadataItem.type, + fieldMetadataItem.label + ' (ID)', + ), + }); + } else if (fieldMetadataItem.type === FieldMetadataType.Boolean) { + availableFieldsForImport.push({ + icon: getIcon(fieldMetadataItem.icon), + label: fieldMetadataItem.label, + key: fieldMetadataItem.name, + fieldType: { + type: 'checkbox', + }, + fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions( + fieldMetadataItem.type, + fieldMetadataItem.label, + ), + }); } else { availableFieldsForImport.push({ icon: getIcon(fieldMetadataItem.icon), diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/types/AvailableFieldForImport.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/types/AvailableFieldForImport.ts index 0716f0acf516..d6cf50ca9aca 100644 --- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/types/AvailableFieldForImport.ts +++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/types/AvailableFieldForImport.ts @@ -1,12 +1,13 @@ -import { FieldValidationDefinition } from '@/spreadsheet-import/types'; +import { + FieldValidationDefinition, + SpreadsheetImportFieldType, +} from '@/spreadsheet-import/types'; import { IconComponent } from 'twenty-ui'; export type AvailableFieldForImport = { icon: IconComponent; label: string; key: string; - fieldType: { - type: 'input' | 'checkbox'; - }; + fieldType: SpreadsheetImportFieldType; fieldValidationDefinitions?: FieldValidationDefinition[]; }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/components/Heading.tsx b/packages/twenty-front/src/modules/spreadsheet-import/components/Heading.tsx index e58e452c9ab3..26b7fccd7092 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/components/Heading.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/components/Heading.tsx @@ -20,7 +20,7 @@ const StyledTitle = styled.span` `; const StyledDescription = styled.span` - color: ${({ theme }) => theme.font.color.primary}; + color: ${({ theme }) => theme.font.color.secondary}; font-size: ${({ theme }) => theme.font.size.sm}; font-weight: ${({ theme }) => theme.font.weight.regular}; margin-top: ${({ theme }) => theme.spacing(3)}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/components/ModalWrapper.tsx b/packages/twenty-front/src/modules/spreadsheet-import/components/ModalWrapper.tsx index 1b69206fd819..bb7f3d3b2528 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/components/ModalWrapper.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/components/ModalWrapper.tsx @@ -10,6 +10,7 @@ const StyledModal = styled(Modal)` height: 61%; min-height: 600px; min-width: 800px; + padding: 0; position: relative; width: 63%; @media (max-width: ${MOBILE_VIEWPORT}px) { @@ -42,7 +43,7 @@ export const ModalWrapper = ({ return ( <> {isOpen && ( - + {children} diff --git a/packages/twenty-front/src/modules/spreadsheet-import/components/Providers.tsx b/packages/twenty-front/src/modules/spreadsheet-import/components/ReactSpreadsheetImportContextProvider.tsx similarity index 72% rename from packages/twenty-front/src/modules/spreadsheet-import/components/Providers.tsx rename to packages/twenty-front/src/modules/spreadsheet-import/components/ReactSpreadsheetImportContextProvider.tsx index b8b548c5a222..e2b182ae3594 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/components/Providers.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/components/ReactSpreadsheetImportContextProvider.tsx @@ -5,15 +5,15 @@ import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull'; export const RsiContext = createContext({} as any); -type ProvidersProps = { +type ReactSpreadsheetImportContextProviderProps = { children: React.ReactNode; values: SpreadsheetImportDialogOptions; }; -export const Providers = ({ +export const ReactSpreadsheetImportContextProvider = ({ children, values, -}: ProvidersProps) => { +}: ReactSpreadsheetImportContextProviderProps) => { if (isUndefinedOrNull(values.fields)) { throw new Error('Fields must be provided to spreadsheet-import'); } diff --git a/packages/twenty-front/src/modules/spreadsheet-import/components/StepNavigationButton.tsx b/packages/twenty-front/src/modules/spreadsheet-import/components/StepNavigationButton.tsx index 2f09b9f41083..6462fcd8c666 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/components/StepNavigationButton.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/components/StepNavigationButton.tsx @@ -7,8 +7,9 @@ import { Modal } from '@/ui/layout/modal/components/Modal'; import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull'; const StyledFooter = styled(Modal.Footer)` - gap: ${({ theme }) => theme.spacing(2)}; + gap: ${({ theme }) => theme.spacing(2.5)}; justify-content: space-between; + padding: ${({ theme }) => theme.spacing(6)} ${({ theme }) => theme.spacing(8)}; `; type StepNavigationButtonProps = { @@ -23,21 +24,23 @@ export const StepNavigationButton = ({ title, isLoading, onBack, -}: StepNavigationButtonProps) => ( - - {!isUndefinedOrNull(onBack) && ( +}: StepNavigationButtonProps) => { + return ( + + {!isUndefinedOrNull(onBack) && ( + + )} - )} - - -); + + ); +}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImport.test.tsx b/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImport.test.tsx index 1ae9cd6f7c19..dc1d67124a72 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImport.test.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImport.test.tsx @@ -3,7 +3,7 @@ import { RecoilRoot, useRecoilState } from 'recoil'; import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog'; import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState'; -import { StepType } from '@/spreadsheet-import/steps/components/UploadFlow'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { ImportedRow, SpreadsheetImportDialogOptions, @@ -38,7 +38,7 @@ export const mockedSpreadsheetOptions: SpreadsheetImportDialogOptions { it('should return correct number for each step type', async () => { const { result } = renderHook(() => { - const [step, setStep] = useState(); + const [step, setStep] = useState(); const { initialStep } = useSpreadsheetImportInitialStep(step); return { initialStep, setStep }; }); @@ -15,31 +15,31 @@ describe('useSpreadsheetImportInitialStep', () => { expect(result.current.initialStep).toBe(-1); act(() => { - result.current.setStep(StepType.upload); + result.current.setStep(SpreadsheetImportStepType.upload); }); expect(result.current.initialStep).toBe(0); act(() => { - result.current.setStep(StepType.selectSheet); + result.current.setStep(SpreadsheetImportStepType.selectSheet); }); expect(result.current.initialStep).toBe(0); act(() => { - result.current.setStep(StepType.selectHeader); + result.current.setStep(SpreadsheetImportStepType.selectHeader); }); expect(result.current.initialStep).toBe(0); act(() => { - result.current.setStep(StepType.matchColumns); + result.current.setStep(SpreadsheetImportStepType.matchColumns); }); expect(result.current.initialStep).toBe(2); act(() => { - result.current.setStep(StepType.validateData); + result.current.setStep(SpreadsheetImportStepType.validateData); }); expect(result.current.initialStep).toBe(3); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImportInternal.test.tsx b/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImportInternal.test.tsx index 2e8e8f0396cf..5053c2104ae3 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImportInternal.test.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/hooks/__tests__/useSpreadsheetImportInternal.test.tsx @@ -1,11 +1,13 @@ import { renderHook } from '@testing-library/react'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { mockedSpreadsheetOptions } from '@/spreadsheet-import/hooks/__tests__/useSpreadsheetImport.test'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; const Wrapper = ({ children }: { children: React.ReactNode }) => ( - {children} + + {children} + ); describe('useSpreadsheetImportInternal', () => { diff --git a/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInitialStep.ts b/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInitialStep.ts index f0b52b8b1e31..e645fac42334 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInitialStep.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInitialStep.ts @@ -1,21 +1,22 @@ +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { useMemo } from 'react'; -import { StepType } from '@/spreadsheet-import/steps/components/UploadFlow'; - -export const useSpreadsheetImportInitialStep = (initialStep?: StepType) => { +export const useSpreadsheetImportInitialStep = ( + initialStep?: SpreadsheetImportStepType, +) => { const steps = ['uploadStep', 'matchColumnsStep', 'validationStep'] as const; const initialStepNumber = useMemo(() => { switch (initialStep) { - case StepType.upload: + case SpreadsheetImportStepType.upload: return 0; - case StepType.selectSheet: + case SpreadsheetImportStepType.selectSheet: return 0; - case StepType.selectHeader: + case SpreadsheetImportStepType.selectHeader: return 0; - case StepType.matchColumns: + case SpreadsheetImportStepType.matchColumns: return 2; - case StepType.validateData: + case SpreadsheetImportStepType.validateData: return 3; default: return -1; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInternal.ts b/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInternal.ts index 0cf87b8138eb..fd5aec6c3b03 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInternal.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/hooks/useSpreadsheetImportInternal.ts @@ -1,7 +1,7 @@ import { useContext } from 'react'; import { SetRequired } from 'type-fest'; -import { RsiContext } from '@/spreadsheet-import/components/Providers'; +import { RsiContext } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { defaultSpreadsheetImportProps } from '@/spreadsheet-import/provider/components/SpreadsheetImport'; import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types'; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImport.tsx b/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImport.tsx index 8237c146409d..50bc89f4358a 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImport.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImport.tsx @@ -1,6 +1,6 @@ import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; -import { Steps } from '@/spreadsheet-import/steps/components/Steps'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; +import { SpreadsheetImportStepperContainer } from '@/spreadsheet-import/steps/components/SpreadsheetImportStepperContainer'; import { SpreadsheetImportDialogOptions as SpreadsheetImportProps } from '@/spreadsheet-import/types'; export const defaultSpreadsheetImportProps: Partial< @@ -25,11 +25,11 @@ export const SpreadsheetImport = ( props: SpreadsheetImportProps, ) => { return ( - + - + - + ); }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImportProvider.tsx b/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImportProvider.tsx index 88041e2718d1..6e93d3dfb650 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImportProvider.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/provider/components/SpreadsheetImportProvider.tsx @@ -1,8 +1,9 @@ import React from 'react'; -import { useRecoilState } from 'recoil'; +import { useRecoilState, useSetRecoilState } from 'recoil'; import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState'; +import { matchColumnsState } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState'; import { SpreadsheetImport } from './SpreadsheetImport'; type SpreadsheetImportProviderProps = React.PropsWithChildren; @@ -14,11 +15,15 @@ export const SpreadsheetImportProvider = ( spreadsheetImportDialogState, ); + const setMatchColumnsState = useSetRecoilState(matchColumnsState); + const handleClose = () => { setSpreadsheetImportDialog({ isOpen: false, options: null, }); + + setMatchColumnsState([]); }; return ( diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx index 666af5c1f43b..20b0051c9147 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx @@ -4,7 +4,11 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { Heading } from '@/spreadsheet-import/components/Heading'; import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; -import { Field, ImportedRow } from '@/spreadsheet-import/types'; +import { + Field, + ImportedRow, + ImportedStructuredRow, +} from '@/spreadsheet-import/types'; import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields'; import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns'; import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData'; @@ -16,6 +20,12 @@ import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/Snac import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar'; import { Modal } from '@/ui/layout/modal/components/Modal'; + +import { UnmatchColumn } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn'; +import { initialComputedColumnsState } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; +import { useRecoilState } from 'recoil'; import { ColumnGrid } from './components/ColumnGrid'; import { TemplateColumn } from './components/TemplateColumn'; import { UserTableColumn } from './components/UserTableColumn'; @@ -45,15 +55,15 @@ const StyledColumn = styled.span` font-weight: ${({ theme }) => theme.font.weight.regular}; `; -export type MatchColumnsStepProps = { +export type MatchColumnsStepProps = { data: ImportedRow[]; headerValues: ImportedRow; - onContinue: ( - data: any[], - rawData: ImportedRow[], - columns: Columns, - ) => void; - onBack: () => void; + onBack?: () => void; + setCurrentStepState: (currentStepState: SpreadsheetImportStep) => void; + setPreviousStepState: (currentStepState: SpreadsheetImportStep) => void; + currentStepState: SpreadsheetImportStep; + nextStep: () => void; + errorToast: (message: string) => void; }; export enum ColumnType { @@ -121,28 +131,30 @@ export type Columns = Column[]; export const MatchColumnsStep = ({ data, headerValues, - onContinue, onBack, -}: MatchColumnsStepProps) => { + setCurrentStepState, + setPreviousStepState, + currentStepState, + nextStep, + errorToast, +}: MatchColumnsStepProps) => { const { enqueueDialog } = useDialogManager(); const { enqueueSnackBar } = useSnackBar(); const dataExample = data.slice(0, 2); const { fields, autoMapHeaders, autoMapDistance } = useSpreadsheetImportInternal(); const [isLoading, setIsLoading] = useState(false); - const [columns, setColumns] = useState>( - // Do not remove spread, it indexes empty array elements, otherwise map() skips over them - ([...headerValues] as string[]).map((value, index) => ({ - type: ColumnType.empty, - index, - header: value ?? '', - })), + const [columns, setColumns] = useRecoilState( + initialComputedColumnsState(headerValues), ); + + const { matchColumnsStepHook } = useSpreadsheetImportInternal(); + const onIgnore = useCallback( (columnIndex: number) => { setColumns( columns.map((column, index) => - columnIndex === index ? setIgnoreColumn(column) : column, + columnIndex === index ? setIgnoreColumn(column) : column, ), ); }, @@ -176,7 +188,7 @@ export const MatchColumnsStep = ({ (column) => 'value' in column && column.value === field.key, ); setColumns( - columns.map>((column, index) => { + columns.map>((column, index) => { if (columnIndex === index) { return setColumn(column, field, data); } else if (index === existingFieldIndex) { @@ -192,7 +204,44 @@ export const MatchColumnsStep = ({ ); } }, - [columns, onRevertIgnore, onIgnore, fields, data, enqueueSnackBar], + [ + columns, + onRevertIgnore, + onIgnore, + fields, + setColumns, + data, + enqueueSnackBar, + ], + ); + + const onContinue = useCallback( + async ( + values: ImportedStructuredRow[], + rawData: ImportedRow[], + columns: Columns, + ) => { + try { + const data = await matchColumnsStepHook(values, rawData, columns); + setCurrentStepState({ + type: SpreadsheetImportStepType.validateData, + data, + importedColumns: columns, + }); + setPreviousStepState(currentStepState); + nextStep(); + } catch (e) { + errorToast((e as Error).message); + } + }, + [ + errorToast, + matchColumnsStepHook, + nextStep, + setPreviousStepState, + setCurrentStepState, + currentStepState, + ], ); const onSubChange = useCallback( @@ -262,7 +311,10 @@ export const MatchColumnsStep = ({ ]); useEffect(() => { - if (autoMapHeaders) { + const isInitialColumnsState = columns.every( + (column) => column.type === ColumnType.empty, + ); + if (autoMapHeaders && isInitialColumnsState) { setColumns(getMatchedColumns(columns, fields, data, autoMapDistance)); } // eslint-disable-next-line react-hooks/exhaustive-deps @@ -290,16 +342,25 @@ export const MatchColumnsStep = ({ columns={columns} columnIndex={columnIndex} onChange={onChange} + /> + )} + renderUnmatchedColumn={(columns, columnIndex) => ( + )} /> { + onBack?.(); + setColumns([]); + }} /> ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/ColumnGrid.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/ColumnGrid.tsx index ea053f2f29ef..3d1ad999af7e 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/ColumnGrid.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/ColumnGrid.tsx @@ -1,5 +1,5 @@ -import React from 'react'; import styled from '@emotion/styled'; +import React from 'react'; import { Columns } from '../MatchColumnsStep'; @@ -24,9 +24,12 @@ const StyledGrid = styled.div` type HeightProps = { height?: `${number}px`; + withBorder?: boolean; }; const StyledGridRow = styled.div` + border-bottom: ${({ withBorder, theme }) => + withBorder && `1px solid ${theme.border.color.medium}`}; box-sizing: border-box; display: flex; flex-direction: row; @@ -34,7 +37,7 @@ const StyledGridRow = styled.div` `; type PositionProps = { - position: 'left' | 'right'; + position: 'left' | 'right' | 'full-line'; }; const StyledGridCell = styled.div` @@ -50,11 +53,21 @@ const StyledGridCell = styled.div` return ` padding-left: ${theme.spacing(4)}; padding-right: ${theme.spacing(2)}; + padding-top: ${theme.spacing(4)}; + `; + } + if (position === 'full-line') { + return ` + padding-left: ${theme.spacing(2)}; + padding-right: ${theme.spacing(4)}; + padding-top: ${theme.spacing(0)}; + width: 100%; `; } return ` padding-left: ${theme.spacing(2)}; padding-right: ${theme.spacing(4)}; + padding-top: ${theme.spacing(4)}; `; }}; `; @@ -89,12 +102,17 @@ type ColumnGridProps = { columns: Columns, columnIndex: number, ) => React.ReactNode; + renderUnmatchedColumn: ( + columns: Columns, + columnIndex: number, + ) => React.ReactNode; }; export const ColumnGrid = ({ columns, renderUserColumn, renderTemplateColumn, + renderUnmatchedColumn, }: ColumnGridProps) => { return ( <> @@ -107,15 +125,29 @@ export const ColumnGrid = ({ {columns.map((column, index) => { const userColumn = renderUserColumn(columns, index); const templateColumn = renderTemplateColumn(columns, index); + const unmatchedColumn = renderUnmatchedColumn(columns, index); + const isSelect = 'matchedOptions' in columns[index]; + const isLast = index === columns.length - 1; if (React.isValidElement(userColumn)) { return ( - - {userColumn} - - {templateColumn} - - +
+ + + {userColumn} + + + {templateColumn} + + + {isSelect && ( + + + {unmatchedColumn} + + + )} +
); } diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect.tsx index 7a4abd135f32..9a7e7e5f556c 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect.tsx @@ -1,10 +1,14 @@ +import { useTheme } from '@emotion/react'; import styled from '@emotion/styled'; -import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; import { SelectOption } from '@/spreadsheet-import/types'; + import { getFieldOptions } from '@/spreadsheet-import/utils/getFieldOptions'; +import { SelectInput } from '@/ui/input/components/SelectInput'; +import { useState } from 'react'; +import { IconChevronDown, Tag, TagColor } from 'twenty-ui'; import { MatchedOptions, MatchedSelectColumn, @@ -12,45 +16,106 @@ import { } from '../MatchColumnsStep'; const StyledContainer = styled.div` + align-items: center; + display: flex; + gap: ${({ theme }) => theme.spacing(4)}; + justify-content: space-between; padding-bottom: ${({ theme }) => theme.spacing(1)}; - padding-left: ${({ theme }) => theme.spacing(2)}; `; -const StyledSelectLabel = styled.span` +const StyledControlContainer = styled.div<{ cursor: string }>` + align-items: center; + background-color: ${({ theme }) => theme.background.transparent.lighter}; + border: 1px solid ${({ theme }) => theme.border.color.medium}; + box-sizing: border-box; + border-radius: ${({ theme }) => theme.border.radius.sm}; + color: ${({ theme }) => theme.font.color.primary}; + cursor: ${({ cursor }) => cursor}; + display: flex; + gap: ${({ theme }) => theme.spacing(1)}; + height: ${({ theme }) => theme.spacing(8)}; + justify-content: space-between; + padding: 0 ${({ theme }) => theme.spacing(2)}; + width: 100%; +`; + +const StyledLabel = styled.span` color: ${({ theme }) => theme.font.color.primary}; - font-size: ${({ theme }) => theme.font.size.sm}; - font-weight: ${({ theme }) => theme.font.weight.medium}; - padding-bottom: ${({ theme }) => theme.spacing(2)}; - padding-top: ${({ theme }) => theme.spacing(1)}; + font-weight: ${({ theme }) => theme.font.weight.regular}; + font-size: ${({ theme }) => theme.font.size.md}; +`; + +const StyledControlLabel = styled.div` + align-items: center; + display: flex; + gap: ${({ theme }) => theme.spacing(1)}; +`; + +const StyledIconChevronDown = styled(IconChevronDown)` + color: ${({ theme }) => theme.font.color.tertiary}; `; interface SubMatchingSelectProps { option: MatchedOptions | Partial>; column: MatchedSelectColumn | MatchedSelectOptionsColumn; onSubChange: (val: T, index: number, option: string) => void; + placeholder: string; + selectedOption?: MatchedOptions | Partial>; } export const SubMatchingSelect = ({ option, column, onSubChange, + placeholder, }: SubMatchingSelectProps) => { const { fields } = useSpreadsheetImportInternal(); const options = getFieldOptions(fields, column.value) as SelectOption[]; const value = options.find((opt) => opt.value === option.value); + const [isOpen, setIsOpen] = useState(false); + const [selectWrapperRef, setSelectWrapperRef] = + useState(null); + + const theme = useTheme(); + + const handleSelect = (selectedOption: SelectOption) => { + onSubChange(selectedOption.value as T, column.index, option.entry ?? ''); + setIsOpen(false); + }; return ( - {option.entry} - - onSubChange(value?.value as T, column.index, option.entry ?? '') - } - options={options} - name={option.entry} - /> + + + {option.entry} + + + + setIsOpen(!isOpen)} + id="control" + ref={setSelectWrapperRef} + > + + + + {isOpen && ( + setIsOpen(false)} + /> + )} + ); }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/TemplateColumn.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/TemplateColumn.tsx index 6190742aadf0..265ead428a09 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/TemplateColumn.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/TemplateColumn.tsx @@ -1,22 +1,10 @@ -// TODO: We should create our own accordion component -import { - Accordion, - AccordionIcon, - AccordionItem, - AccordionPanel, - AccordionButton as ChakraAccordionButton, -} from '@chakra-ui/accordion'; import styled from '@emotion/styled'; -import { IconChevronDown, IconForbid } from 'twenty-ui'; +import { IconForbid } from 'twenty-ui'; import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; -import { Fields } from '@/spreadsheet-import/types'; -import { Column, Columns, ColumnType } from '../MatchColumnsStep'; - -import { isDefined } from '~/utils/isDefined'; -import { SubMatchingSelect } from './SubMatchingSelect'; +import { Columns, ColumnType } from '../MatchColumnsStep'; const StyledContainer = styled.div` display: flex; @@ -25,89 +13,38 @@ const StyledContainer = styled.div` width: 100%; `; -const StyledAccordionButton = styled(ChakraAccordionButton)` - align-items: center; - background-color: ${({ theme }) => theme.accent.secondary}; - border: none; - border-radius: ${({ theme }) => theme.border.radius.sm}; - box-sizing: border-box; - color: ${({ theme }) => theme.font.color.primary}; - display: flex; - flex-direction: row; - margin-top: ${({ theme }) => theme.spacing(2)}; - padding-bottom: ${({ theme }) => theme.spacing(1)}; - padding-left: ${({ theme }) => theme.spacing(2)}; - padding-right: ${({ theme }) => theme.spacing(2)}; - padding-top: ${({ theme }) => theme.spacing(1)}; - width: 100%; - - &:hover { - background-color: ${({ theme }) => theme.accent.primary}; - } -`; - -const StyledAccordionContainer = styled.div` - display: flex; - width: 100%; -`; - -const StyledAccordionLabel = styled.span` - color: ${({ theme }) => theme.font.color.primary}; - display: flex; - flex: 1; - font-size: ${({ theme }) => theme.font.size.sm}; - padding-left: ${({ theme }) => theme.spacing(1)}; - text-align: left; -`; - -const getAccordionTitle = ( - fields: Fields, - column: Column, -) => { - const fieldLabel = fields.find( - (field) => 'value' in column && field.key === column.value, - )?.label; - - return `Match ${fieldLabel} (${ - 'matchedOptions' in column && - column.matchedOptions.filter((option) => !isDefined(option.value)).length - } Unmatched)`; -}; - type TemplateColumnProps = { - columns: Columns; + columns: Columns; columnIndex: number; onChange: (val: T, index: number) => void; - onSubChange: (val: T, index: number, option: string) => void; }; export const TemplateColumn = ({ columns, columnIndex, onChange, - onSubChange, }: TemplateColumnProps) => { const { fields } = useSpreadsheetImportInternal(); const column = columns[columnIndex]; const isIgnored = column.type === ColumnType.ignored; - const isSelect = 'matchedOptions' in column; + const fieldOptions = fields.map(({ icon, label, key }) => { const isSelected = columns.findIndex((column) => { if ('value' in column) { return column.value === key; } - return false; }) !== -1; return { - icon, + icon: icon, value: key, - label, + label: label, disabled: isSelected, } as const; }); + const selectOptions = [ { icon: IconForbid, @@ -116,9 +53,11 @@ export const TemplateColumn = ({ }, ...fieldOptions, ]; + const selectValue = fieldOptions.find( ({ value }) => 'value' in column && column.value === value, ); + const ignoreValue = selectOptions.find( ({ value }) => value === 'do-not-import', ); @@ -132,30 +71,6 @@ export const TemplateColumn = ({ options={selectOptions} name={column.header} /> - {isSelect && ( - - - - - - {getAccordionTitle(fields, column)} - - - - - {column.matchedOptions.map((option) => ( - - ))} - - - - - )} ); }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx new file mode 100644 index 000000000000..3e1cd3abb2b2 --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx @@ -0,0 +1,111 @@ +import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; +import { SubMatchingSelect } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect'; +import { Column } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; +import { Fields } from '@/spreadsheet-import/types'; +import { + Accordion, + AccordionIcon, + AccordionItem, + AccordionPanel, + AccordionButton as ChakraAccordionButton, +} from '@chakra-ui/accordion'; +import styled from '@emotion/styled'; +import { IconChevronDown, IconInfoCircle, isDefined } from 'twenty-ui'; + +const StyledAccordionButton = styled(ChakraAccordionButton)` + align-items: center; + background-color: ${({ theme }) => theme.accent.secondary}; + border: none; + border-radius: ${({ theme }) => theme.border.radius.md}; + box-sizing: border-box; + color: ${({ theme }) => theme.font.color.primary}; + display: flex; + flex-direction: row; + padding: ${({ theme }) => theme.spacing(2)}; + width: 100%; + height: 40px; + + &:hover { + background-color: ${({ theme }) => theme.accent.primary}; + } +`; + +const StyledAccordionContainer = styled.div` + display: flex; + width: 100%; + height: auto; +`; + +const StyledAccordionLabel = styled.span` + color: ${({ theme }) => theme.color.blue}; + display: flex; + flex: 1; + font-size: ${({ theme }) => theme.font.size.sm}; + align-items: center; + gap: ${({ theme }) => theme.spacing(2)}; + text-align: left; +`; + +const StyledIconChevronDown = styled(IconChevronDown)` + color: ${({ theme }) => theme.color.blue} !important; +`; + +const getAccordionTitle = ( + fields: Fields, + column: Column, +) => { + const fieldLabel = fields.find( + (field) => 'value' in column && field.key === column.value, + )?.label; + + return `Match ${fieldLabel} (${ + 'matchedOptions' in column && + column.matchedOptions.filter((option) => !isDefined(option.value)).length + } Unmatched)`; +}; + +type UnmatchColumnProps = { + columns: Column[]; + columnIndex: number; + onSubChange: (val: T, index: number, option: string) => void; +}; + +export const UnmatchColumn = ({ + columns, + columnIndex, + onSubChange, +}: UnmatchColumnProps) => { + const { fields } = useSpreadsheetImportInternal(); + + const column = columns[columnIndex]; + const isSelect = 'matchedOptions' in column; + + return ( + isSelect && ( + + + + + + + {getAccordionTitle(fields, column)} + + + + + {column.matchedOptions.map((option) => ( + + ))} + + + + + ) + ); +}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState.ts b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState.ts new file mode 100644 index 000000000000..ed147aadd6ae --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/states/initialComputedColumnsState.ts @@ -0,0 +1,41 @@ +import { + Columns, + ColumnType, +} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; +import { ImportedRow } from '@/spreadsheet-import/types'; +import { atom, selectorFamily } from 'recoil'; + +export const matchColumnsState = atom({ + key: 'MatchColumnsState', + default: [] as Columns, +}); + +export const initialComputedColumnsState = selectorFamily< + Columns, + ImportedRow +>({ + key: 'InitialComputedColumnsState', + get: + (headerValues: ImportedRow) => + ({ get }) => { + const currentState = get(matchColumnsState) as Columns; + if (currentState.length === 0) { + // Do not remove spread, it indexes empty array elements, otherwise map() skips over them + const initialState = ([...headerValues] as string[]).map( + (value, index) => ({ + type: ColumnType.empty, + index, + header: value ?? '', + }), + ); + return initialState as Columns; + } else { + return currentState; + } + }, + set: + () => + ({ set }, newValue) => { + set(matchColumnsState, newValue as Columns); + }, +}); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx index a7107cb71b89..55c46e433457 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx @@ -6,6 +6,10 @@ import { StepNavigationButton } from '@/spreadsheet-import/components/StepNaviga import { ImportedRow } from '@/spreadsheet-import/types'; import { Modal } from '@/ui/layout/modal/components/Modal'; + +import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { SelectHeaderTable } from './components/SelectHeaderTable'; const StyledHeading = styled(Heading)` @@ -20,17 +24,22 @@ const StyledTableContainer = styled.div` type SelectHeaderStepProps = { importedRows: ImportedRow[]; - onContinue: ( - headerValues: ImportedRow, - importedRows: ImportedRow[], - ) => Promise; + setCurrentStepState: (currentStepState: SpreadsheetImportStep) => void; + nextStep: () => void; + setPreviousStepState: (currentStepState: SpreadsheetImportStep) => void; + errorToast: (message: string) => void; onBack: () => void; + currentStepState: SpreadsheetImportStep; }; export const SelectHeaderStep = ({ importedRows, - onContinue, + setCurrentStepState, + nextStep, + setPreviousStepState, + errorToast, onBack, + currentStepState, }: SelectHeaderStepProps) => { const [selectedRowIndexes, setSelectedRowIndexes] = useState< ReadonlySet @@ -38,6 +47,34 @@ export const SelectHeaderStep = ({ const [isLoading, setIsLoading] = useState(false); + const { selectHeaderStepHook } = useSpreadsheetImportInternal(); + + const onContinue = useCallback( + async (...args: Parameters) => { + try { + const { importedRows: data, headerRow: headerValues } = + await selectHeaderStepHook(...args); + setCurrentStepState({ + type: SpreadsheetImportStepType.matchColumns, + data, + headerValues, + }); + setPreviousStepState(currentStepState); + nextStep(); + } catch (e) { + errorToast((e as Error).message); + } + }, + [ + errorToast, + nextStep, + selectHeaderStepHook, + setPreviousStepState, + setCurrentStepState, + currentStepState, + ], + ); + const handleContinue = useCallback(async () => { const [selectedRowIndex] = Array.from(new Set(selectedRowIndexes)); // We consider data above header to be redundant diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectSheetStep/SelectSheetStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectSheetStep/SelectSheetStep.tsx index 58e61ba9723c..ba4e3bc3202b 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectSheetStep/SelectSheetStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectSheetStep/SelectSheetStep.tsx @@ -3,10 +3,16 @@ import { useCallback, useState } from 'react'; import { Heading } from '@/spreadsheet-import/components/Heading'; import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton'; +import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; +import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords'; +import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook'; import { Radio } from '@/ui/input/components/Radio'; import { RadioGroup } from '@/ui/input/components/RadioGroup'; import { Modal } from '@/ui/layout/modal/components/Modal'; +import { WorkBook } from 'xlsx-ugnis'; const StyledContent = styled(Modal.Content)` align-items: center; @@ -27,19 +33,65 @@ const StyledRadioContainer = styled.div` type SelectSheetStepProps = { sheetNames: string[]; - onContinue: (sheetName: string) => Promise; onBack: () => void; + setCurrentStepState: (data: SpreadsheetImportStep) => void; + errorToast: (message: string) => void; + setPreviousStepState: (data: SpreadsheetImportStep) => void; + currentStepState: { + type: SpreadsheetImportStepType.selectSheet; + workbook: WorkBook; + }; }; export const SelectSheetStep = ({ sheetNames, - onContinue, + setCurrentStepState, + errorToast, + setPreviousStepState, onBack, + currentStepState, }: SelectSheetStepProps) => { const [isLoading, setIsLoading] = useState(false); const [value, setValue] = useState(sheetNames[0]); + const { maxRecords, uploadStepHook } = useSpreadsheetImportInternal(); + + const onContinue = useCallback( + async (sheetName: string) => { + if ( + maxRecords > 0 && + exceedsMaxRecords( + currentStepState.workbook.Sheets[sheetName], + maxRecords, + ) + ) { + errorToast(`Too many records. Up to ${maxRecords.toString()} allowed`); + return; + } + try { + const mappedWorkbook = await uploadStepHook( + mapWorkbook(currentStepState.workbook, sheetName), + ); + setCurrentStepState({ + type: SpreadsheetImportStepType.selectHeader, + data: mappedWorkbook, + }); + setPreviousStepState(currentStepState); + } catch (e) { + errorToast((e as Error).message); + } + }, + [ + errorToast, + maxRecords, + currentStepState, + setPreviousStepState, + setCurrentStepState, + uploadStepHook, + ], + ); + const handleOnContinue = useCallback( async (data: typeof value) => { setIsLoading(true); @@ -65,7 +117,7 @@ export const SelectSheetStep = ({ onClick={() => handleOnContinue(value)} onBack={onBack} isLoading={isLoading} - title="Continue" + title="Next Step" /> ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepper.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepper.tsx new file mode 100644 index 000000000000..7b85b17cdd05 --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepper.tsx @@ -0,0 +1,146 @@ +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { useCallback, useState } from 'react'; + +import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; +import { CircularProgressBar } from '@/ui/feedback/progress-bar/components/CircularProgressBar'; +import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar'; +import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar'; +import { Modal } from '@/ui/layout/modal/components/Modal'; + +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; +import { MatchColumnsStep } from './MatchColumnsStep/MatchColumnsStep'; +import { SelectHeaderStep } from './SelectHeaderStep/SelectHeaderStep'; +import { SelectSheetStep } from './SelectSheetStep/SelectSheetStep'; +import { UploadStep } from './UploadStep/UploadStep'; +import { ValidationStep } from './ValidationStep/ValidationStep'; + +const StyledProgressBarContainer = styled(Modal.Content)` + align-items: center; + display: flex; + justify-content: center; +`; + +type SpreadsheetImportStepperProps = { + nextStep: () => void; + prevStep: () => void; +}; + +export const SpreadsheetImportStepper = ({ + nextStep, + prevStep, +}: SpreadsheetImportStepperProps) => { + const theme = useTheme(); + + const { initialStepState } = useSpreadsheetImportInternal(); + + const [currentStepState, setCurrentStepState] = + useState( + initialStepState || { type: SpreadsheetImportStepType.upload }, + ); + const [previousStepState, setPreviousStepState] = + useState( + initialStepState || { type: SpreadsheetImportStepType.upload }, + ); + + const [uploadedFile, setUploadedFile] = useState(null); + + const { enqueueSnackBar } = useSnackBar(); + + const errorToast = useCallback( + (description: string) => { + enqueueSnackBar(description, { + title: 'Error', + variant: SnackBarVariant.Error, + }); + }, + [enqueueSnackBar], + ); + + const onBack = useCallback(() => { + setCurrentStepState(previousStepState); + prevStep(); + }, [prevStep, previousStepState]); + + switch (currentStepState.type) { + case SpreadsheetImportStepType.upload: + return ( + + ); + case SpreadsheetImportStepType.selectSheet: + return ( + + ); + case SpreadsheetImportStepType.selectHeader: + return ( + + ); + case SpreadsheetImportStepType.matchColumns: + return ( + { + onBack(); + }} + errorToast={errorToast} + /> + ); + case SpreadsheetImportStepType.validateData: + if (!uploadedFile) { + throw new Error('File not found'); + } + return ( + { + onBack(); + setPreviousStepState( + initialStepState || { type: SpreadsheetImportStepType.upload }, + ); + }} + /> + ); + case SpreadsheetImportStepType.loading: + default: + return ( + + + + ); + } +}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/Steps.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepperContainer.tsx similarity index 81% rename from packages/twenty-front/src/modules/spreadsheet-import/steps/components/Steps.tsx rename to packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepperContainer.tsx index 1b21a981bec7..11d5e6a6caf8 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/Steps.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SpreadsheetImportStepperContainer.tsx @@ -8,7 +8,7 @@ import { StepBar } from '@/ui/navigation/step-bar/components/StepBar'; import { useStepBar } from '@/ui/navigation/step-bar/hooks/useStepBar'; import { Modal } from '@/ui/layout/modal/components/Modal'; -import { UploadFlow } from './UploadFlow'; +import { SpreadsheetImportStepper } from './SpreadsheetImportStepper'; const StyledHeader = styled(Modal.Header)` background-color: ${({ theme }) => theme.background.secondary}; @@ -29,7 +29,7 @@ const stepTitles = { validationStep: 'Validate data', } as const; -export const Steps = () => { +export const SpreadsheetImportStepperContainer = () => { const { initialStepState } = useSpreadsheetImportInternal(); const { steps, initialStep } = useSpreadsheetImportInitialStep( @@ -45,11 +45,15 @@ export const Steps = () => { {steps.map((key) => ( - + ))} - + ); }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx deleted file mode 100644 index a74495da4fbc..000000000000 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx +++ /dev/null @@ -1,260 +0,0 @@ -import { useTheme } from '@emotion/react'; -import styled from '@emotion/styled'; -import { useCallback, useState } from 'react'; -import { WorkBook } from 'xlsx-ugnis'; - -import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; -import { ImportedRow } from '@/spreadsheet-import/types'; -import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords'; -import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook'; -import { CircularProgressBar } from '@/ui/feedback/progress-bar/components/CircularProgressBar'; -import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar'; -import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar'; - -import { Modal } from '@/ui/layout/modal/components/Modal'; -import { Columns, MatchColumnsStep } from './MatchColumnsStep/MatchColumnsStep'; -import { SelectHeaderStep } from './SelectHeaderStep/SelectHeaderStep'; -import { SelectSheetStep } from './SelectSheetStep/SelectSheetStep'; -import { UploadStep } from './UploadStep/UploadStep'; -import { ValidationStep } from './ValidationStep/ValidationStep'; - -const StyledProgressBarContainer = styled(Modal.Content)` - align-items: center; - display: flex; - justify-content: center; -`; - -export enum StepType { - upload = 'upload', - selectSheet = 'selectSheet', - selectHeader = 'selectHeader', - matchColumns = 'matchColumns', - validateData = 'validateData', - loading = 'loading', -} -export type StepState = - | { - type: StepType.upload; - } - | { - type: StepType.selectSheet; - workbook: WorkBook; - } - | { - type: StepType.selectHeader; - data: ImportedRow[]; - } - | { - type: StepType.matchColumns; - data: ImportedRow[]; - headerValues: ImportedRow; - } - | { - type: StepType.validateData; - data: any[]; - importedColumns: Columns; - } - | { - type: StepType.loading; - }; - -interface UploadFlowProps { - nextStep: () => void; - prevStep: () => void; -} - -export const UploadFlow = ({ nextStep, prevStep }: UploadFlowProps) => { - const theme = useTheme(); - const { initialStepState } = useSpreadsheetImportInternal(); - const [state, setState] = useState( - initialStepState || { type: StepType.upload }, - ); - const [previousState, setPreviousState] = useState( - initialStepState || { type: StepType.upload }, - ); - const [uploadedFile, setUploadedFile] = useState(null); - const { - maxRecords, - uploadStepHook, - selectHeaderStepHook, - matchColumnsStepHook, - selectHeader, - } = useSpreadsheetImportInternal(); - const { enqueueSnackBar } = useSnackBar(); - - const errorToast = useCallback( - (description: string) => { - enqueueSnackBar(description, { - title: 'Error', - variant: SnackBarVariant.Error, - }); - }, - [enqueueSnackBar], - ); - - const onBack = useCallback(() => { - setState(previousState); - prevStep(); - }, [prevStep, previousState]); - - switch (state.type) { - case StepType.upload: - return ( - { - setUploadedFile(file); - const isSingleSheet = workbook.SheetNames.length === 1; - if (isSingleSheet) { - if ( - maxRecords > 0 && - exceedsMaxRecords( - workbook.Sheets[workbook.SheetNames[0]], - maxRecords, - ) - ) { - errorToast( - `Too many records. Up to ${maxRecords.toString()} allowed`, - ); - return; - } - try { - const mappedWorkbook = await uploadStepHook( - mapWorkbook(workbook), - ); - - if (selectHeader) { - setState({ - type: StepType.selectHeader, - data: mappedWorkbook, - }); - } else { - // Automatically select first row as header - const trimmedData = mappedWorkbook.slice(1); - - const { importedRows: data, headerRow: headerValues } = - await selectHeaderStepHook(mappedWorkbook[0], trimmedData); - - setState({ - type: StepType.matchColumns, - data, - headerValues, - }); - } - } catch (e) { - errorToast((e as Error).message); - } - } else { - setState({ type: StepType.selectSheet, workbook }); - } - setPreviousState(state); - nextStep(); - }} - /> - ); - case StepType.selectSheet: - return ( - { - if ( - maxRecords > 0 && - exceedsMaxRecords(state.workbook.Sheets[sheetName], maxRecords) - ) { - errorToast( - `Too many records. Up to ${maxRecords.toString()} allowed`, - ); - return; - } - try { - const mappedWorkbook = await uploadStepHook( - mapWorkbook(state.workbook, sheetName), - ); - setState({ - type: StepType.selectHeader, - data: mappedWorkbook, - }); - setPreviousState(state); - } catch (e) { - errorToast((e as Error).message); - } - }} - onBack={onBack} - /> - ); - case StepType.selectHeader: - return ( - { - try { - const { importedRows: data, headerRow: headerValues } = - await selectHeaderStepHook(...args); - setState({ - type: StepType.matchColumns, - data, - headerValues, - }); - setPreviousState(state); - nextStep(); - } catch (e) { - errorToast((e as Error).message); - } - }} - onBack={onBack} - /> - ); - case StepType.matchColumns: - return ( - { - try { - const data = await matchColumnsStepHook(values, rawData, columns); - setState({ - type: StepType.validateData, - data, - importedColumns: columns, - }); - setPreviousState(state); - nextStep(); - } catch (e) { - errorToast((e as Error).message); - } - }} - onBack={onBack} - /> - ); - case StepType.validateData: - if (!uploadedFile) { - throw new Error('File not found'); - } - return ( - - setState({ - type: StepType.loading, - }) - } - onBack={() => { - onBack(); - setPreviousState(initialStepState || { type: StepType.upload }); - }} - /> - ); - case StepType.loading: - default: - return ( - - - - ); - } -}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/UploadStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/UploadStep.tsx index 54f1a0403711..9d3109601152 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/UploadStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/UploadStep.tsx @@ -3,6 +3,12 @@ import { useCallback, useState } from 'react'; import { WorkBook } from 'xlsx-ugnis'; import { Modal } from '@/ui/layout/modal/components/Modal'; + +import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; +import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords'; +import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook'; import { DropZone } from './components/DropZone'; const StyledContent = styled(Modal.Content)` @@ -10,11 +16,86 @@ const StyledContent = styled(Modal.Content)` `; type UploadStepProps = { - onContinue: (data: WorkBook, file: File) => Promise; + setUploadedFile: (file: File) => void; + setCurrentStepState: (data: any) => void; + errorToast: (message: string) => void; + nextStep: () => void; + setPreviousStepState: (data: any) => void; + currentStepState: SpreadsheetImportStep; }; -export const UploadStep = ({ onContinue }: UploadStepProps) => { +export const UploadStep = ({ + setUploadedFile, + setCurrentStepState, + errorToast, + nextStep, + setPreviousStepState, + currentStepState, +}: UploadStepProps) => { const [isLoading, setIsLoading] = useState(false); + const { maxRecords, uploadStepHook, selectHeaderStepHook, selectHeader } = + useSpreadsheetImportInternal(); + + const onContinue = useCallback( + async (workbook: WorkBook, file: File) => { + setUploadedFile(file); + const isSingleSheet = workbook.SheetNames.length === 1; + if (isSingleSheet) { + if ( + maxRecords > 0 && + exceedsMaxRecords(workbook.Sheets[workbook.SheetNames[0]], maxRecords) + ) { + errorToast( + `Too many records. Up to ${maxRecords.toString()} allowed`, + ); + return; + } + try { + const mappedWorkbook = await uploadStepHook(mapWorkbook(workbook)); + + if (selectHeader) { + setCurrentStepState({ + type: SpreadsheetImportStepType.selectHeader, + data: mappedWorkbook, + }); + } else { + // Automatically select first row as header + const trimmedData = mappedWorkbook.slice(1); + + const { importedRows: data, headerRow: headerValues } = + await selectHeaderStepHook(mappedWorkbook[0], trimmedData); + + setCurrentStepState({ + type: SpreadsheetImportStepType.matchColumns, + data, + headerValues, + }); + } + } catch (e) { + errorToast((e as Error).message); + } + } else { + setCurrentStepState({ + type: SpreadsheetImportStepType.selectSheet, + workbook, + }); + } + setPreviousStepState(currentStepState); + nextStep(); + }, + [ + errorToast, + maxRecords, + nextStep, + selectHeader, + selectHeaderStepHook, + setPreviousStepState, + setCurrentStepState, + setUploadedFile, + currentStepState, + uploadStepHook, + ], + ); const handleOnContinue = useCallback( async (data: WorkBook, file: File) => { diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx index c368cdd66b59..534deb1e0b8a 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx @@ -79,7 +79,7 @@ const StyledText = styled.span` font-size: ${({ theme }) => theme.font.size.sm}; font-weight: ${({ theme }) => theme.font.weight.medium}; text-align: center; - padding: 15px; + padding: 16px; `; type DropZoneProps = { diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx index bcd0405becaf..21b6d034923c 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx @@ -1,6 +1,12 @@ import styled from '@emotion/styled'; -import { useCallback, useMemo, useState } from 'react'; -// @ts-expect-error Todo: remove usage of react-data-grid +import { + Dispatch, + SetStateAction, + useCallback, + useMemo, + useState, +} from 'react'; +// @ts-expect-error Todo: remove usage of react-data-grid` import { RowsChangeData } from 'react-data-grid'; import { IconTrash } from 'twenty-ui'; @@ -22,6 +28,8 @@ import { Button } from '@/ui/input/button/components/Button'; import { Toggle } from '@/ui/input/components/Toggle'; import { isDefined } from '~/utils/isDefined'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { Modal } from '@/ui/layout/modal/components/Modal'; import { generateColumns } from './components/columns'; import { ImportedStructuredRowMetadata } from './types'; @@ -71,15 +79,15 @@ type ValidationStepProps = { initialData: ImportedStructuredRow[]; importedColumns: Columns; file: File; - onSubmitStart?: () => void; onBack: () => void; + setCurrentStepState: Dispatch>; }; export const ValidationStep = ({ initialData, importedColumns, file, - onSubmitStart, + setCurrentStepState, onBack, }: ValidationStepProps) => { const { enqueueDialog } = useDialogManager(); @@ -209,7 +217,11 @@ export const ValidationStep = ({ allStructuredRows: data, } satisfies ImportValidationResult, ); - onSubmitStart?.(); + + setCurrentStepState({ + type: SpreadsheetImportStepType.loading, + }); + await onSubmit(calculatedData, file); onClose(); }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/MatchColumns.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/MatchColumns.stories.tsx index d7eaff8699dd..e3a8499646ce 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/MatchColumns.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/MatchColumns.stories.tsx @@ -1,8 +1,9 @@ import { Meta } from '@storybook/react'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { MatchColumnsStep } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; import { mockRsiValues } from '@/spreadsheet-import/tests/mockRsiValues'; import { DialogManagerScope } from '@/ui/feedback/dialog-manager/scopes/DialogManagerScope'; import { SnackBarDecorator } from '~/testing/decorators/SnackBarDecorator'; @@ -61,15 +62,19 @@ const mockData = [ export const Default = () => ( - + null}> null} onBack={() => null} + setCurrentStepState={() => null} + setPreviousStepState={() => null} + currentStepState={{} as SpreadsheetImportStep} + nextStep={() => null} + errorToast={() => null} /> - + ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectHeader.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectHeader.stories.tsx index c5b5f05242b9..d6de08e0d1d1 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectHeader.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectHeader.stories.tsx @@ -1,8 +1,9 @@ import { Meta } from '@storybook/react'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { SelectHeaderStep } from '@/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { headerSelectionTableFields, mockRsiValues, @@ -21,14 +22,21 @@ export default meta; export const Default = () => ( - + null}> Promise.resolve()} + setCurrentStepState={() => null} + nextStep={() => Promise.resolve()} + setPreviousStepState={() => null} + errorToast={() => null} onBack={() => Promise.resolve()} + currentStepState={{ + type: SpreadsheetImportStepType.selectHeader, + data: headerSelectionTableFields, + }} /> - + ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectSheet.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectSheet.stories.tsx index 30b8e48731b4..3c37538e472a 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectSheet.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/SelectSheet.stories.tsx @@ -1,8 +1,9 @@ import { Meta } from '@storybook/react'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { SelectSheetStep } from '@/spreadsheet-import/steps/components/SelectSheetStep/SelectSheetStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { mockRsiValues } from '@/spreadsheet-import/tests/mockRsiValues'; import { DialogManagerScope } from '@/ui/feedback/dialog-manager/scopes/DialogManagerScope'; @@ -20,14 +21,39 @@ const sheetNames = ['Sheet1', 'Sheet2', 'Sheet3']; export const Default = () => ( - + null}> Promise.resolve()} + setCurrentStepState={() => {}} + setPreviousStepState={() => {}} + currentStepState={{ + type: SpreadsheetImportStepType.selectSheet, + workbook: { + SheetNames: sheetNames, + Sheets: { + Sheet1: { + A1: 1, + A2: 2, + A3: 3, + }, + Sheet2: { + A1: 1, + A2: 2, + A3: 3, + }, + Sheet3: { + A1: 1, + A2: 2, + A3: 3, + }, + }, + }, + }} + errorToast={() => null} onBack={() => Promise.resolve()} /> - + ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Steps.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Steps.stories.tsx index 420fccde9d3c..e2427d6175a7 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Steps.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Steps.stories.tsx @@ -5,16 +5,16 @@ import { within } from '@storybook/test'; import { ComponentWithRecoilScopeDecorator } from '~/testing/decorators/ComponentWithRecoilScopeDecorator'; import { SnackBarDecorator } from '~/testing/decorators/SnackBarDecorator'; -import { Steps } from '../Steps'; +import { SpreadsheetImportStepperContainer } from '../SpreadsheetImportStepperContainer'; -const meta: Meta = { +const meta: Meta = { title: 'Modules/SpreadsheetImport/Steps', - component: Steps, + component: SpreadsheetImportStepperContainer, decorators: [ComponentWithRecoilScopeDecorator, SnackBarDecorator], }; export default meta; -type Story = StoryObj; +type Story = StoryObj; export const Default: Story = { play: async () => { diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Upload.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Upload.stories.tsx index 0b469fea0f31..fb7e9d78d6e4 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Upload.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Upload.stories.tsx @@ -1,8 +1,9 @@ import { Meta } from '@storybook/react'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { UploadStep } from '@/spreadsheet-import/steps/components/UploadStep/UploadStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; import { mockRsiValues } from '@/spreadsheet-import/tests/mockRsiValues'; import { DialogManagerScope } from '@/ui/feedback/dialog-manager/scopes/DialogManagerScope'; import { SnackBarDecorator } from '~/testing/decorators/SnackBarDecorator'; @@ -20,10 +21,19 @@ export default meta; export const Default = () => ( - + null}> - Promise.resolve()} /> + null} + setCurrentStepState={() => null} + errorToast={() => null} + nextStep={() => null} + setPreviousStepState={() => null} + currentStepState={{ + type: SpreadsheetImportStepType.upload, + }} + /> - + ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Validation.stories.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Validation.stories.tsx index 1a5adabc0346..9126371d1dc2 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Validation.stories.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/__stories__/Validation.stories.tsx @@ -1,7 +1,7 @@ import { Meta } from '@storybook/react'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; -import { Providers } from '@/spreadsheet-import/components/Providers'; +import { ReactSpreadsheetImportContextProvider } from '@/spreadsheet-import/components/ReactSpreadsheetImportContextProvider'; import { ValidationStep } from '@/spreadsheet-import/steps/components/ValidationStep/ValidationStep'; import { editableTableInitialData, @@ -24,15 +24,16 @@ const file = new File([''], 'file.csv'); export const Default = () => ( - + null}> Promise.resolve()} + setCurrentStepState={() => null} /> - + ); diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStep.ts b/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStep.ts new file mode 100644 index 000000000000..ad04f1512648 --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStep.ts @@ -0,0 +1,30 @@ +import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; +import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType'; +import { ImportedRow } from '@/spreadsheet-import/types'; +import { WorkBook } from 'xlsx-ugnis'; + +export type SpreadsheetImportStep = + | { + type: SpreadsheetImportStepType.upload; + } + | { + type: SpreadsheetImportStepType.selectSheet; + workbook: WorkBook; + } + | { + type: SpreadsheetImportStepType.selectHeader; + data: ImportedRow[]; + } + | { + type: SpreadsheetImportStepType.matchColumns; + data: ImportedRow[]; + headerValues: ImportedRow; + } + | { + type: SpreadsheetImportStepType.validateData; + data: any[]; + importedColumns: Columns; + } + | { + type: SpreadsheetImportStepType.loading; + }; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStepType.ts b/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStepType.ts new file mode 100644 index 000000000000..9c2bf555dd25 --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/types/SpreadsheetImportStepType.ts @@ -0,0 +1,8 @@ +export enum SpreadsheetImportStepType { + upload = 'upload', + selectSheet = 'selectSheet', + selectHeader = 'selectHeader', + matchColumns = 'matchColumns', + validateData = 'validateData', + loading = 'loading', +} diff --git a/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts b/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts index fa5cf6d97586..d63692460300 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts @@ -1,9 +1,9 @@ -import { IconComponent } from 'twenty-ui'; +import { IconComponent, ThemeColor } from 'twenty-ui'; import { ReadonlyDeep } from 'type-fest'; import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; -import { StepState } from '@/spreadsheet-import/steps/components/UploadFlow'; import { ImportedStructuredRowMetadata } from '@/spreadsheet-import/steps/components/ValidationStep/types'; +import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep'; export type SpreadsheetImportDialogOptions = { // Is modal visible. @@ -47,7 +47,7 @@ export type SpreadsheetImportDialogOptions = { // Headers matching accuracy: 1 for strict and up for more flexible matching autoMapDistance?: number; // Initial Step state to be rendered on load - initialStepState?: StepState; + initialStepState?: SpreadsheetImportStep; // Sets SheetJS dateNF option. If date parsing is applied, date will be formatted e.g. "yyyy-mm-dd hh:mm:ss", "m/d/yy h:mm", 'mmm-yy', etc. dateFormat?: string; // Sets SheetJS "raw" option. If true, parsing will only be applied to xlsx date fields. @@ -67,25 +67,6 @@ export type ImportedStructuredRow = { // Data model RSI uses for spreadsheet imports export type Fields = ReadonlyDeep[]>; -export type Field = { - // Icon - icon: IconComponent | null | undefined; - // UI-facing field label - label: string; - // Field's unique identifier - key: T; - // UI-facing additional information displayed via tooltip and ? icon - description?: string; - // Alternate labels used for fields' auto-matching, e.g. "fname" -> "firstName" - alternateMatches?: string[]; - // Validations used for field entries - fieldValidationDefinitions?: FieldValidationDefinition[]; - // Field entry component, default: Input - fieldType: Checkbox | Select | Input; - // UI-facing values shown to user as field examples pre-upload phase - example?: string; -}; - export type Checkbox = { type: 'checkbox'; // Alternate values to be treated as booleans, e.g. {yes: true, no: false} @@ -107,12 +88,35 @@ export type SelectOption = { value: string; // Disabled option when already select disabled?: boolean; + // Option color + color?: ThemeColor; }; export type Input = { type: 'input'; }; +export type SpreadsheetImportFieldType = Checkbox | Select | Input; + +export type Field = { + // Icon + icon: IconComponent | null | undefined; + // UI-facing field label + label: string; + // Field's unique identifier + key: T; + // UI-facing additional information displayed via tooltip and ? icon + description?: string; + // Alternate labels used for fields' auto-matching, e.g. "fname" -> "firstName" + alternateMatches?: string[]; + // Validations used for field entries + fieldValidationDefinitions?: FieldValidationDefinition[]; + // Field entry component, default: Input + fieldType: SpreadsheetImportFieldType; + // UI-facing values shown to user as field examples pre-upload phase + example?: string; +}; + export type FieldValidationDefinition = | RequiredValidation | UniqueValidation diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumns.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumns.ts index 661466154ceb..4397231640a8 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumns.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumns.ts @@ -14,7 +14,7 @@ import { setColumn } from './setColumn'; export const getMatchedColumns = ( columns: Columns, fields: Fields, - data: MatchColumnsStepProps['data'], + data: MatchColumnsStepProps['data'], autoMapDistance: number, ) => columns.reduce[]>((arr, column) => { diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts index 191cdf2081cc..ceb7d205884a 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts @@ -11,7 +11,7 @@ import { uniqueEntries } from './uniqueEntries'; export const setColumn = ( oldColumn: Column, field?: Field, - data?: MatchColumnsStepProps['data'], + data?: MatchColumnsStepProps['data'], ): Column => { if (field?.fieldType.type === 'select') { const fieldOptions = field.fieldType.options; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/uniqueEntries.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/uniqueEntries.ts index 0e82bc44fe41..803f37c5af88 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/utils/uniqueEntries.ts +++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/uniqueEntries.ts @@ -6,7 +6,7 @@ import { } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; export const uniqueEntries = ( - data: MatchColumnsStepProps['data'], + data: MatchColumnsStepProps['data'], index: number, ): Partial>[] => uniqBy( diff --git a/packages/twenty-front/src/modules/ui/input/components/Select.tsx b/packages/twenty-front/src/modules/ui/input/components/Select.tsx index 6a3964769d45..c31a48a198ea 100644 --- a/packages/twenty-front/src/modules/ui/input/components/Select.tsx +++ b/packages/twenty-front/src/modules/ui/input/components/Select.tsx @@ -68,7 +68,9 @@ const StyledControlLabel = styled.div` gap: ${({ theme }) => theme.spacing(1)}; `; -const StyledIconChevronDown = styled(IconChevronDown)<{ disabled?: boolean }>` +const StyledIconChevronDown = styled(IconChevronDown)<{ + disabled?: boolean; +}>` color: ${({ disabled, theme }) => disabled ? theme.font.color.extraLight : theme.font.color.tertiary}; `; diff --git a/packages/twenty-front/src/modules/ui/input/components/SelectInput.tsx b/packages/twenty-front/src/modules/ui/input/components/SelectInput.tsx new file mode 100644 index 000000000000..e08131d9a147 --- /dev/null +++ b/packages/twenty-front/src/modules/ui/input/components/SelectInput.tsx @@ -0,0 +1,180 @@ +import styled from '@emotion/styled'; + +import { SelectOption } from '@/spreadsheet-import/types'; + +import { SelectFieldHotkeyScope } from '@/object-record/select/types/SelectFieldHotkeyScope'; +import { DropdownMenu } from '@/ui/layout/dropdown/components/DropdownMenu'; +import { DropdownMenuItemsContainer } from '@/ui/layout/dropdown/components/DropdownMenuItemsContainer'; +import { DropdownMenuSearchInput } from '@/ui/layout/dropdown/components/DropdownMenuSearchInput'; +import { DropdownMenuSeparator } from '@/ui/layout/dropdown/components/DropdownMenuSeparator'; +import { MenuItemSelectTag } from '@/ui/navigation/menu-item/components/MenuItemSelectTag'; +import { useScopedHotkeys } from '@/ui/utilities/hotkey/hooks/useScopedHotkeys'; +import { useSetHotkeyScope } from '@/ui/utilities/hotkey/hooks/useSetHotkeyScope'; +import { useListenClickOutside } from '@/ui/utilities/pointer-event/hooks/useListenClickOutside'; +import { useTheme } from '@emotion/react'; +import { + ReferenceType, + autoUpdate, + flip, + offset, + size, + useFloating, +} from '@floating-ui/react'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import { Key } from 'ts-key-enum'; +import { TagColor, isDefined } from 'twenty-ui'; + +const StyledRelationPickerContainer = styled.div` + left: -1px; + position: absolute; + top: -1px; + z-index: ${({ theme }) => theme.lastLayerZIndex}; +`; + +interface SelectInputProps { + onOptionSelected: (selectedOption: SelectOption) => void; + options: SelectOption[]; + onCancel?: () => void; + defaultOption?: SelectOption; + parentRef?: ReferenceType | null | undefined; + onFilterChange?: (filteredOptions: SelectOption[]) => void; + onClear?: () => void; + clearLabel?: string; +} + +export const SelectInput = ({ + onOptionSelected, + onClear, + clearLabel, + options, + onCancel, + defaultOption, + parentRef, + onFilterChange, +}: SelectInputProps) => { + const containerRef = useRef(null); + + const theme = useTheme(); + const [searchFilter, setSearchFilter] = useState(''); + const [selectedOption, setSelectedOption] = useState< + SelectOption | undefined + >(defaultOption); + + const optionsToSelect = useMemo( + () => + options.filter((option) => { + return ( + option.value !== selectedOption?.value && + option.label.toLowerCase().includes(searchFilter.toLowerCase()) + ); + }) || [], + [options, searchFilter, selectedOption?.value], + ); + + const optionsInDropDown = useMemo( + () => + selectedOption ? [selectedOption, ...optionsToSelect] : optionsToSelect, + [optionsToSelect, selectedOption], + ); + + const handleOptionChange = (option: SelectOption) => { + setSelectedOption(option); + onOptionSelected(option); + }; + + const { refs, floatingStyles } = useFloating({ + elements: { reference: parentRef }, + strategy: 'absolute', + middleware: [ + offset(() => { + return parseInt(theme.spacing(2), 10); + }), + flip(), + size(), + ], + whileElementsMounted: autoUpdate, + open: true, + placement: 'bottom-start', + }); + + const setHotkeyScope = useSetHotkeyScope(); + + useEffect(() => { + setHotkeyScope(SelectFieldHotkeyScope.SelectField); + }, [setHotkeyScope]); + + useEffect(() => { + onFilterChange?.(optionsInDropDown); + }, [onFilterChange, optionsInDropDown]); + + useListenClickOutside({ + refs: [refs.floating], + callback: (event) => { + event.stopImmediatePropagation(); + + const weAreNotInAnHTMLInput = !( + event.target instanceof HTMLInputElement && + event.target.tagName === 'INPUT' + ); + if (weAreNotInAnHTMLInput && isDefined(onCancel)) { + onCancel(); + } + }, + }); + + useScopedHotkeys( + Key.Enter, + () => { + const selectedOption = optionsInDropDown.find((option) => + option.label.toLowerCase().includes(searchFilter.toLowerCase()), + ); + if (isDefined(selectedOption)) { + handleOptionChange(selectedOption); + } + }, + SelectFieldHotkeyScope.SelectField, + [searchFilter, optionsInDropDown], + ); + + return ( + + + setSearchFilter(e.target.value)} + autoFocus + /> + + + {onClear && clearLabel && ( + { + setSelectedOption(undefined); + onClear(); + }} + /> + )} + {optionsInDropDown.map((option) => { + return ( + handleOptionChange(option)} + /> + ); + })} + + + + ); +}; diff --git a/packages/twenty-front/src/modules/ui/navigation/step-bar/components/Step.tsx b/packages/twenty-front/src/modules/ui/navigation/step-bar/components/Step.tsx index 01a93c9ac5f6..1b08f142b28e 100644 --- a/packages/twenty-front/src/modules/ui/navigation/step-bar/components/Step.tsx +++ b/packages/twenty-front/src/modules/ui/navigation/step-bar/components/Step.tsx @@ -14,11 +14,15 @@ const StyledContainer = styled.div<{ isLast: boolean }>` } `; -const StyledStepCircle = styled(motion.div)` +const StyledStepCircle = styled(motion.div)<{ isNextStep: boolean }>` align-items: center; border-radius: 50%; border-style: solid; border-width: 1px; + border-color: ${({ theme, isNextStep }) => + isNextStep + ? theme.border.color.inverted + : theme.border.color.medium} !important; display: flex; flex-basis: auto; flex-shrink: 0; @@ -29,17 +33,20 @@ const StyledStepCircle = styled(motion.div)` width: 20px; `; -const StyledStepIndex = styled.span` - color: ${({ theme }) => theme.font.color.tertiary}; +const StyledStepIndex = styled.span<{ isNextStep: boolean }>` + color: ${({ theme, isNextStep }) => + isNextStep ? theme.font.color.secondary : theme.font.color.tertiary}; font-size: ${({ theme }) => theme.font.size.md}; font-weight: ${({ theme }) => theme.font.weight.medium}; `; -const StyledStepLabel = styled.span<{ isActive: boolean }>` - color: ${({ theme, isActive }) => - isActive ? theme.font.color.primary : theme.font.color.tertiary}; +const StyledStepLabel = styled.span<{ isActive: boolean; isNextStep: boolean }>` + color: ${({ theme, isActive, isNextStep }) => + isActive || isNextStep + ? theme.font.color.primary + : theme.font.color.tertiary}; font-size: ${({ theme }) => theme.font.size.md}; - font-weight: ${({ theme }) => theme.font.weight.medium}; + font-weight: ${({ theme }) => theme.font.weight.semiBold}; margin-left: ${({ theme }) => theme.spacing(2)}; white-space: nowrap; `; @@ -58,6 +65,7 @@ export type StepProps = React.PropsWithChildren & isLast?: boolean; index?: number; label: string; + activeStep?: number; }; export const Step = ({ @@ -66,6 +74,7 @@ export const Step = ({ index = 0, label, children, + activeStep = 0, }: StepProps) => { const theme = useTheme(); const isMobile = useIsMobile(); @@ -94,11 +103,14 @@ export const Step = ({ }, }; + const isNextStep = activeStep + 1 === index; + return ( {isActive && ( )} - {!isActive && {index + 1}} + {!isActive && ( + {index + 1} + )} - {label} + + {label} + {!isLast && !isMobile && ( Date: Thu, 22 Aug 2024 17:51:08 +0200 Subject: [PATCH 19/19] 6687 change messaging import cron job to run every minute (#6704) Closes #6687 --- .../messaging-messages-import.cron.command.ts | 4 ++- .../messaging-messages-import.cron.job.ts | 28 ++++++++----------- ...-users-messages-get-batch-size.constant.ts | 2 +- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/commands/messaging-messages-import.cron.command.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/commands/messaging-messages-import.cron.command.ts index 206f2d47815b..11ef7211d76a 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/commands/messaging-messages-import.cron.command.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/commands/messaging-messages-import.cron.command.ts @@ -5,6 +5,8 @@ import { MessageQueue } from 'src/engine/integrations/message-queue/message-queu import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessagingMessagesImportCronJob } from 'src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job'; +const MESSAGING_MESSAGES_IMPORT_CRON_PATTERN = '*/1 * * * *'; + @Command({ name: 'cron:messaging:messages-import', description: 'Starts a cron job to fetch all messages from cache', @@ -23,7 +25,7 @@ export class MessagingMessagesImportCronCommand extends CommandRunner { undefined, { repeat: { - every: 30000, + pattern: MESSAGING_MESSAGES_IMPORT_CRON_PATTERN, }, }, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts index daef69dfa03d..a90c09f89d08 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts @@ -1,4 +1,3 @@ -import { Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; @@ -24,8 +23,6 @@ import { @Processor(MessageQueue.cronQueue) export class MessagingMessagesImportCronJob { - private readonly logger = new Logger(MessagingMessagesImportCronJob.name); - constructor( @InjectRepository(Workspace, 'core') private readonly workspaceRepository: Repository, @@ -51,23 +48,20 @@ export class MessagingMessagesImportCronJob { ); const messageChannels = await messageChannelRepository.find({ - select: ['id', 'isSyncEnabled', 'syncStage'], + where: { + isSyncEnabled: true, + syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING, + }, }); for (const messageChannel of messageChannels) { - if ( - messageChannel.isSyncEnabled && - messageChannel.syncStage === - MessageChannelSyncStage.MESSAGES_IMPORT_PENDING - ) { - await this.messageQueueService.add( - MessagingMessagesImportJob.name, - { - workspaceId: activeWorkspace.id, - messageChannelId: messageChannel.id, - }, - ); - } + await this.messageQueueService.add( + MessagingMessagesImportJob.name, + { + workspaceId: activeWorkspace.id, + messageChannelId: messageChannel.id, + }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts index fdf015b35a78..dc89d1a32bcb 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts @@ -1 +1 @@ -export const MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 100; +export const MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 200;