[POC] add graphql query runner (#6747)

## Context
The goal is to replace pg_graphql with our own ORM wrapper (TwentyORM).
This PR tries to add some parsing logic to convert graphql requests to
send to the ORM to replace pg_graphql implementation.

---------

Co-authored-by: Charles Bochet <[email protected]>

Author: Weiko

packages/twenty-front/vite.config.ts

@@ -32,11 +32,19 @@ export default defineConfig(({ command, mode }) => {
     overlay: false,
   };
 
-  console.log(
-    `VITE_DISABLE_TYPESCRIPT_CHECKER: ${VITE_DISABLE_TYPESCRIPT_CHECKER}`,
-  );
-  console.log(`VITE_DISABLE_ESLINT_CHECKER: ${VITE_DISABLE_ESLINT_CHECKER}`);
-  console.log(`VITE_BUILD_SOURCEMAP: ${VITE_BUILD_SOURCEMAP}`);
+  if (VITE_DISABLE_TYPESCRIPT_CHECKER === 'true') {
+    console.log(
+      `VITE_DISABLE_TYPESCRIPT_CHECKER: ${VITE_DISABLE_TYPESCRIPT_CHECKER}`,
+    );
+  }
+
+  if (VITE_DISABLE_ESLINT_CHECKER === 'true') {
+    console.log(`VITE_DISABLE_ESLINT_CHECKER: ${VITE_DISABLE_ESLINT_CHECKER}`);
+  }
+
+  if (VITE_BUILD_SOURCEMAP === 'true') {
+    console.log(`VITE_BUILD_SOURCEMAP: ${VITE_BUILD_SOURCEMAP}`);
+  }
 
   if (VITE_DISABLE_TYPESCRIPT_CHECKER !== 'true') {
     checkers['typescript'] = {

packages/twenty-server/src/app.module.ts

@@ -22,6 +22,7 @@ import { MessageQueueDriverType } from 'src/engine/integrations/message-queue/in
 import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module';
 import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
 import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware';
+import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
 import { ModulesModule } from 'src/modules/modules.module';
 
 import { CoreEngineModule } from './engine/core-modules/core-engine.module';
@@ -45,6 +46,7 @@ import { IntegrationsModule } from './engine/integrations/integrations.module';
       imports: [CoreEngineModule, GraphQLConfigModule],
       useClass: GraphQLConfigService,
     }),
+    TwentyORMModule,
     // Integrations module, contains all the integrations with other services
     IntegrationsModule,
     // Core engine module, contains all the core modules

packages/twenty-server/src/database/typeorm/typeorm.module.ts

@@ -3,7 +3,6 @@ import { TypeOrmModule, TypeOrmModuleOptions } from '@nestjs/typeorm';
 
 import { typeORMCoreModuleOptions } from 'src/database/typeorm/core/core.datasource';
 import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
-import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
 
 import { TypeORMService } from './typeorm.service';
 
@@ -29,7 +28,6 @@ const coreTypeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
       useFactory: coreTypeORMFactory,
       name: 'core',
     }),
-    TwentyORMModule.register({}),
     EnvironmentModule,
   ],
   providers: [TypeORMService],

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/constants/connection-max-depth.constant.ts

@@ -0,0 +1 @@
+export const CONNECTION_MAX_DEPTH = 5;

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant.ts

@@ -0,0 +1 @@
+export const QUERY_MAX_RECORDS = 60;

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts

@@ -0,0 +1,17 @@
+import { CustomException } from 'src/utils/custom-exception';
+
+export class GraphqlQueryRunnerException extends CustomException {
+  code: GraphqlQueryRunnerExceptionCode;
+  constructor(message: string, code: GraphqlQueryRunnerExceptionCode) {
+    super(message, code);
+  }
+}
+
+export enum GraphqlQueryRunnerExceptionCode {
+  MAX_DEPTH_REACHED = 'MAX_DEPTH_REACHED',
+  INVALID_CURSOR = 'INVALID_CURSOR',
+  INVALID_DIRECTION = 'INVALID_DIRECTION',
+  UNSUPPORTED_OPERATOR = 'UNSUPPORTED_OPERATOR',
+  ARGS_CONFLICT = 'ARGS_CONFLICT',
+  FIELD_NOT_FOUND = 'FIELD_NOT_FOUND',
+}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+
+import { GraphqlQueryRunnerService } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-runner.service';
+
+@Module({
+  providers: [GraphqlQueryRunnerService],
+  exports: [GraphqlQueryRunnerService],
+})
+export class GraphqlQueryRunnerModule {}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.service.ts

@@ -0,0 +1,130 @@
+import { Injectable } from '@nestjs/common';
+
+import graphqlFields from 'graphql-fields';
+import { FindManyOptions, ObjectLiteral } from 'typeorm';
+
+import {
+  Record as IRecord,
+  RecordFilter,
+  RecordOrderBy,
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { IConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/connection.interface';
+import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
+import { FindManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
+
+import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant';
+import {
+  GraphqlQueryRunnerException,
+  GraphqlQueryRunnerExceptionCode,
+} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
+import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query.parser';
+import { applyRangeFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/apply-range-filter.util';
+import {
+  createConnection,
+  decodeCursor,
+} from 'src/engine/api/graphql/graphql-query-runner/utils/connection.util';
+import { convertObjectMetadataToMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
+import { LogExecutionTime } from 'src/engine/decorators/observability/log-execution-time.decorator';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+
+@Injectable()
+export class GraphqlQueryRunnerService {
+  constructor(
+    private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+  ) {}
+
+  @LogExecutionTime()
+  async findManyWithTwentyOrm<
+    ObjectRecord extends IRecord = IRecord,
+    Filter extends RecordFilter = RecordFilter,
+    OrderBy extends RecordOrderBy = RecordOrderBy,
+  >(
+    args: FindManyResolverArgs<Filter, OrderBy>,
+    options: WorkspaceQueryRunnerOptions,
+  ): Promise<IConnection<ObjectRecord>> {
+    const { authContext, objectMetadataItem, info, objectMetadataCollection } =
+      options;
+
+    const repository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        authContext.workspace.id,
+        objectMetadataItem.nameSingular,
+      );
+
+    const selectedFields = graphqlFields(info);
+
+    const objectMetadataMap = convertObjectMetadataToMap(
+      objectMetadataCollection,
+    );
+
+    const objectMetadata = objectMetadataMap[objectMetadataItem.nameSingular];
+
+    if (!objectMetadata) {
+      throw new Error(
+        `Object metadata for ${objectMetadataItem.nameSingular} not found`,
+      );
+    }
+
+    const fieldMetadataMap = objectMetadata.fields;
+
+    const graphqlQueryParser = new GraphqlQueryParser(
+      fieldMetadataMap,
+      objectMetadataMap,
+    );
+
+    const { select, relations } = graphqlQueryParser.parseSelectedFields(
+      objectMetadataItem,
+      selectedFields,
+    );
+
+    const order = args.orderBy
+      ? graphqlQueryParser.parseOrder(args.orderBy)
+      : undefined;
+
+    const where = args.filter
+      ? graphqlQueryParser.parseFilter(args.filter)
+      : {};
+
+    let cursor: Record<string, any> | undefined;
+
+    if (args.after) {
+      cursor = decodeCursor(args.after);
+    } else if (args.before) {
+      cursor = decodeCursor(args.before);
+    }
+
+    if (args.first && args.last) {
+      throw new GraphqlQueryRunnerException(
+        'Cannot provide both first and last',
+        GraphqlQueryRunnerExceptionCode.ARGS_CONFLICT,
+      );
+    }
+
+    const take = args.first ?? args.last ?? QUERY_MAX_RECORDS;
+
+    const findOptions: FindManyOptions<ObjectLiteral> = {
+      where,
+      order,
+      select,
+      relations,
+      take,
+    };
+
+    const totalCount = await repository.count({
+      where,
+    });
+
+    if (cursor) {
+      applyRangeFilter(where, order, cursor);
+    }
+
+    const objectRecords = await repository.find(findOptions);
+
+    return createConnection(
+      (objectRecords as ObjectRecord[]) ?? [],
+      take,
+      totalCount,
+      order,
+    );
+  }
+}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/__tests__/graphql-query-filter-field.parser.spec.ts

@@ -0,0 +1,43 @@
+import { FindOperator, Not } from 'typeorm';
+
+import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
+
+import { GraphqlQueryFilterFieldParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/graphql-query-filter-field.parser';
+import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+
+describe('GraphqlQueryFilterFieldParser', () => {
+  let parser: GraphqlQueryFilterFieldParser;
+  let mockFieldMetadataMap: Record<string, FieldMetadataInterface>;
+
+  beforeEach(() => {
+    mockFieldMetadataMap = {
+      simpleField: {
+        id: '1',
+        name: 'simpleField',
+        type: FieldMetadataType.TEXT,
+        label: 'Simple Field',
+        objectMetadataId: 'obj1',
+      },
+    };
+    parser = new GraphqlQueryFilterFieldParser(mockFieldMetadataMap);
+  });
+  it('should parse simple field correctly', () => {
+    const result = parser.parse('simpleField', 'value', false);
+
+    expect(result).toEqual({ simpleField: 'value' });
+  });
+
+  it('should negate simple field correctly', () => {
+    const result = parser.parse('simpleField', 'value', true);
+
+    expect(result).toEqual({ simpleField: Not('value') });
+  });
+
+  it('should parse object value using operator parser', () => {
+    const result = parser.parse('simpleField', { like: '%value%' }, false);
+
+    expect(result).toEqual({
+      simpleField: new FindOperator('like', '%%value%%'),
+    });
+  });
+});

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/__tests__/graphql-query-filter-operator.parser.spec.ts

@@ -0,0 +1,130 @@
+import {
+  FindOperator,
+  ILike,
+  In,
+  IsNull,
+  LessThan,
+  LessThanOrEqual,
+  Like,
+  MoreThan,
+  MoreThanOrEqual,
+  Not,
+} from 'typeorm';
+
+import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
+import { GraphqlQueryFilterOperatorParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/graphql-query-filter-operator.parser';
+
+describe('GraphqlQueryFilterOperatorParser', () => {
+  let parser: GraphqlQueryFilterOperatorParser;
+
+  beforeEach(() => {
+    parser = new GraphqlQueryFilterOperatorParser();
+  });
+
+  describe('parseOperator', () => {
+    it('should parse eq operator correctly', () => {
+      const result = parser.parseOperator({ eq: 'value' }, false);
+
+      expect(result).toBe('value');
+    });
+
+    it('should parse neq operator correctly', () => {
+      const result = parser.parseOperator({ neq: 'value' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(Not('value'));
+    });
+
+    it('should parse gt operator correctly', () => {
+      const result = parser.parseOperator({ gt: 5 }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(MoreThan(5));
+    });
+
+    it('should parse gte operator correctly', () => {
+      const result = parser.parseOperator({ gte: 5 }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(MoreThanOrEqual(5));
+    });
+
+    it('should parse lt operator correctly', () => {
+      const result = parser.parseOperator({ lt: 5 }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(LessThan(5));
+    });
+
+    it('should parse lte operator correctly', () => {
+      const result = parser.parseOperator({ lte: 5 }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(LessThanOrEqual(5));
+    });
+
+    it('should parse in operator correctly', () => {
+      const result = parser.parseOperator({ in: [1, 2, 3] }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(In([1, 2, 3]));
+    });
+
+    it('should parse is operator with NULL correctly', () => {
+      const result = parser.parseOperator({ is: 'NULL' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(IsNull());
+    });
+
+    it('should parse is operator with non-NULL value correctly', () => {
+      const result = parser.parseOperator({ is: 'NOT_NULL' }, false);
+
+      expect(result).toBe('NOT_NULL');
+    });
+
+    it('should parse like operator correctly', () => {
+      const result = parser.parseOperator({ like: 'test' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(Like('%test%'));
+    });
+
+    it('should parse ilike operator correctly', () => {
+      const result = parser.parseOperator({ ilike: 'test' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(ILike('%test%'));
+    });
+
+    it('should parse startsWith operator correctly', () => {
+      const result = parser.parseOperator({ startsWith: 'test' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(ILike('test%'));
+    });
+
+    it('should parse endsWith operator correctly', () => {
+      const result = parser.parseOperator({ endsWith: 'test' }, false);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(ILike('%test'));
+    });
+
+    it('should negate the operator when isNegated is true', () => {
+      const result = parser.parseOperator({ eq: 'value' }, true);
+
+      expect(result).toBeInstanceOf(FindOperator);
+      expect(result).toEqual(Not('value'));
+    });
+
+    it('should throw an exception for unsupported operator', () => {
+      expect(() =>
+        parser.parseOperator({ unsupported: 'value' }, false),
+      ).toThrow(GraphqlQueryRunnerException);
+      expect(() =>
+        parser.parseOperator({ unsupported: 'value' }, false),
+      ).toThrow('Operator "unsupported" is not supported');
+    });
+  });
+});