Skip to content

Commit af5d946

Browse files
thomtrpthomtrp
authored
Add backfill position job by workspace (twentyhq#5725)
- Removing existing listener that was backfilling created records without position - Switch to a job that backfill all objects within workspace - Adapting `FIND_BY_POSITION` so it can fetch objects without position. Currently we needed to input a number
1 parent fc71172 commit af5d946

File tree

9 files changed

+307
-104
lines changed

9 files changed

+307
-104
lines changed

‎packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/__tests__/record-position-query.factory.spec.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,22 @@ describe('RecordPositionQueryFactory', () => {
1919
it('should return query and params for FIND_BY_POSITION', async () => {
2020
const positionValue = 1;
2121
const queryType = RecordPositionQueryType.FIND_BY_POSITION;
22-
const [query, params] = await factory.create(
22+
const [query, params] = factory.create(
2323
{ positionValue, recordPositionQueryType: queryType },
2424
objectMetadataItem,
2525
dataSourceSchema,
2626
);
2727

2828
expect(query).toEqual(
29-
`SELECT position FROM ${dataSourceSchema}."${objectMetadataItem.nameSingular}"
29+
`SELECT id, position FROM ${dataSourceSchema}."${objectMetadataItem.nameSingular}"
3030
WHERE "position" = $1`,
3131
);
3232
expect(params).toEqual([positionValue]);
3333
});
3434

3535
it('should return query and params for FIND_MIN_POSITION', async () => {
3636
const queryType = RecordPositionQueryType.FIND_MIN_POSITION;
37-
const [query, params] = await factory.create(
37+
const [query, params] = factory.create(
3838
{ recordPositionQueryType: queryType },
3939
objectMetadataItem,
4040
dataSourceSchema,
@@ -48,7 +48,7 @@ describe('RecordPositionQueryFactory', () => {
4848

4949
it('should return query and params for FIND_MAX_POSITION', async () => {
5050
const queryType = RecordPositionQueryType.FIND_MAX_POSITION;
51-
const [query, params] = await factory.create(
51+
const [query, params] = factory.create(
5252
{ recordPositionQueryType: queryType },
5353
objectMetadataItem,
5454
dataSourceSchema,
@@ -64,7 +64,7 @@ describe('RecordPositionQueryFactory', () => {
6464
const positionValue = 1;
6565
const recordId = '1';
6666
const queryType = RecordPositionQueryType.UPDATE_POSITION;
67-
const [query, params] = await factory.create(
67+
const [query, params] = factory.create(
6868
{ positionValue, recordId, recordPositionQueryType: queryType },
6969
objectMetadataItem,
7070
dataSourceSchema,

‎packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory.ts

+6-4
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export enum RecordPositionQueryType {
1010
}
1111

1212
type FindByPositionQueryArgs = {
13-
positionValue: number;
13+
positionValue: number | null;
1414
recordPositionQueryType: RecordPositionQueryType.FIND_BY_POSITION;
1515
};
1616

@@ -77,10 +77,12 @@ export class RecordPositionQueryFactory {
7777
name: string,
7878
dataSourceSchema: string,
7979
): [RecordPositionQuery, RecordPositionQueryParams] {
80+
const positionStringParam = positionValue ? '= $1' : 'IS NULL';
81+
8082
return [
81-
`SELECT position FROM ${dataSourceSchema}."${name}"
82-
WHERE "position" = $1`,
83-
[positionValue],
83+
`SELECT id, position FROM ${dataSourceSchema}."${name}"
84+
WHERE "position" ${positionStringParam}`,
85+
positionValue ? [positionValue] : [],
8486
];
8587
}
8688

‎packages/twenty-server/src/engine/api/graphql/workspace-query-runner/commands/0-20-record-position-backfill.command.ts

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { Inject } from '@nestjs/common';
2+
3+
import { Command, CommandRunner, Option } from 'nest-commander';
4+
5+
import {
6+
RecordPositionBackfillJob,
7+
RecordPositionBackfillJobData,
8+
} from 'src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job';
9+
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
10+
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
11+
12+
export type RecordPositionBackfillCommandOptions = {
13+
workspaceId: string;
14+
dryRun?: boolean;
15+
};
16+
17+
@Command({
18+
name: 'migrate-0.20:backfill-record-position',
19+
description: 'Backfill record position',
20+
})
21+
export class RecordPositionBackfillCommand extends CommandRunner {
22+
constructor(
23+
@Inject(MessageQueue.recordPositionBackfillQueue)
24+
private readonly messageQueueService: MessageQueueService,
25+
) {
26+
super();
27+
}
28+
29+
@Option({
30+
flags: '-w, --workspace-id [workspace_id]',
31+
description: 'workspace id',
32+
required: true,
33+
})
34+
parseWorkspaceId(value: string): string {
35+
return value;
36+
}
37+
38+
@Option({
39+
flags: '-d, --dry-run [dry run]',
40+
description: 'Dry run: Log backfill actions.',
41+
required: false,
42+
})
43+
dryRun(value: string): boolean {
44+
return Boolean(value);
45+
}
46+
47+
async run(
48+
_passedParam: string[],
49+
options: RecordPositionBackfillCommandOptions,
50+
): Promise<void> {
51+
this.messageQueueService.add<RecordPositionBackfillJobData>(
52+
RecordPositionBackfillJob.name,
53+
{ workspaceId: options.workspaceId, dryRun: options.dryRun ?? false },
54+
{ retryLimit: 3 },
55+
);
56+
}
57+
}

‎packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-
66

77
export type RecordPositionBackfillJobData = {
88
workspaceId: string;
9-
objectMetadata: { nameSingular: string; isCustom: boolean };
10-
recordId: string;
9+
dryRun: boolean;
1110
};
1211

1312
@Injectable()
@@ -19,10 +18,6 @@ export class RecordPositionBackfillJob
1918
) {}
2019

2120
async handle(data: RecordPositionBackfillJobData): Promise<void> {
22-
this.recordPositionBackfillService.backfill(
23-
data.workspaceId,
24-
data.objectMetadata,
25-
data.recordId,
26-
);
21+
this.recordPositionBackfillService.backfill(data.workspaceId, data.dryRun);
2722
}
2823
}

‎packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/record-position.listener.ts

-59
This file was deleted.

‎packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/__tests__/record-position-backfill-service.spec.ts

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import { TestingModule, Test } from '@nestjs/testing';
2+
3+
import { RecordPositionQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
4+
import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
5+
import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service';
6+
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
7+
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
8+
9+
describe('RecordPositionBackfillService', () => {
10+
let recordPositionQueryFactory;
11+
let recordPositionFactory;
12+
let objectMetadataService;
13+
let workspaceDataSourceService;
14+
15+
let service: RecordPositionBackfillService;
16+
17+
beforeEach(async () => {
18+
recordPositionQueryFactory = {
19+
create: jest.fn().mockReturnValue(['query', []]),
20+
};
21+
22+
recordPositionFactory = {
23+
create: jest.fn().mockResolvedValue([
24+
{
25+
position: 1,
26+
},
27+
]),
28+
};
29+
30+
objectMetadataService = {
31+
findManyWithinWorkspace: jest.fn().mockReturnValue([]),
32+
};
33+
34+
workspaceDataSourceService = {
35+
getSchemaName: jest.fn().mockReturnValue('schemaName'),
36+
executeRawQuery: jest.fn().mockResolvedValue([]),
37+
};
38+
const module: TestingModule = await Test.createTestingModule({
39+
providers: [
40+
RecordPositionBackfillService,
41+
{
42+
provide: RecordPositionQueryFactory,
43+
useValue: recordPositionQueryFactory,
44+
},
45+
{
46+
provide: RecordPositionFactory,
47+
useValue: recordPositionFactory,
48+
},
49+
{
50+
provide: WorkspaceDataSourceService,
51+
useValue: workspaceDataSourceService,
52+
},
53+
{
54+
provide: ObjectMetadataService,
55+
useValue: objectMetadataService,
56+
},
57+
],
58+
}).compile();
59+
60+
service = module.get<RecordPositionBackfillService>(
61+
RecordPositionBackfillService,
62+
);
63+
});
64+
65+
afterEach(() => {
66+
jest.clearAllMocks();
67+
});
68+
69+
it('should be defined', () => {
70+
expect(service).toBeDefined();
71+
});
72+
73+
it('when no object metadata found, should do nothing', async () => {
74+
await service.backfill('workspaceId', false);
75+
expect(workspaceDataSourceService.executeRawQuery).not.toHaveBeenCalled();
76+
});
77+
78+
it('when objectMetadata without position, should do nothing', async () => {
79+
objectMetadataService.findManyWithinWorkspace.mockReturnValue([
80+
{
81+
id: '1',
82+
nameSingular: 'name',
83+
fields: [],
84+
},
85+
]);
86+
await service.backfill('workspaceId', false);
87+
expect(workspaceDataSourceService.executeRawQuery).not.toHaveBeenCalled();
88+
});
89+
90+
it('when objectMetadata but all record with position, should create and run query once', async () => {
91+
objectMetadataService.findManyWithinWorkspace.mockReturnValue([
92+
{
93+
id: '1',
94+
nameSingular: 'company',
95+
fields: [],
96+
},
97+
]);
98+
await service.backfill('workspaceId', false);
99+
expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(1);
100+
});
101+
102+
it('when record without position, should create and run query twice', async () => {
103+
objectMetadataService.findManyWithinWorkspace.mockReturnValue([
104+
{
105+
id: '1',
106+
nameSingular: 'company',
107+
fields: [],
108+
},
109+
]);
110+
workspaceDataSourceService.executeRawQuery.mockResolvedValueOnce([
111+
{
112+
id: '1',
113+
},
114+
]);
115+
await service.backfill('workspaceId', false);
116+
expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(2);
117+
expect(recordPositionFactory.create).toHaveBeenCalledTimes(1);
118+
expect(recordPositionQueryFactory.create).toHaveBeenCalledTimes(2);
119+
});
120+
121+
it('when dryRun is true, should not update position', async () => {
122+
objectMetadataService.findManyWithinWorkspace.mockReturnValue([
123+
{
124+
id: '1',
125+
nameSingular: 'company',
126+
fields: [],
127+
},
128+
]);
129+
workspaceDataSourceService.executeRawQuery.mockResolvedValueOnce([
130+
{
131+
id: '1',
132+
},
133+
]);
134+
await service.backfill('workspaceId', true);
135+
expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(1);
136+
expect(recordPositionFactory.create).toHaveBeenCalledTimes(1);
137+
expect(recordPositionQueryFactory.create).toHaveBeenCalledTimes(1);
138+
});
139+
});

‎packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
44
import { RecordPositionQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
55
import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
66
import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service';
7+
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
78

89
@Module({
9-
imports: [WorkspaceDataSourceModule],
10+
imports: [WorkspaceDataSourceModule, ObjectMetadataModule],
1011
providers: [
1112
RecordPositionFactory,
1213
RecordPositionQueryFactory,

0 commit comments

Comments
 (0)