Aggregated queries #1 (#8345)

First step of #6868

Adds min.., max.. queries for DATETIME fields
adds min.., max.., avg.., sum.. queries for NUMBER fields 

(count distinct operation and composite fields such as CURRENCY handling
will be dealt with in a future PR)

<img width="1422" alt="Capture d’écran 2024-11-06 à 15 48 46"
src="https://github.com/user-attachments/assets/4bcdece0-ad3e-4536-9720-fe4044a36719">

---------

Co-authored-by: Charles Bochet <[email protected]>
Co-authored-by: Weiko <[email protected]>

Author: 3 people

.github/workflows/ci-tinybird.yaml

@@ -3,33 +3,24 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'package.json'
+      - 'packages/twenty-tinybird/**'
 
   pull_request:
+    paths:
+      - 'package.json'
+      - 'packages/twenty-tinybird/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   ci:
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
     uses: tinybirdco/ci/.github/workflows/ci.yml@main
-    steps:
-      - name: Check for changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v11
-        with:
-          files: |
-            package.json
-            packages/twenty-tinybird/**
-
-      - name: Skip if no relevant changes
-        if: steps.changed-files.outputs.any_changed == 'false'
-        run: echo "No relevant changes. Skipping CI."
-
-      - name: Check twenty-tinybird package
-        with:
-          data_project_dir: packages/twenty-tinybird
-          tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}
-          tb_host: https://api.eu-central-1.aws.tinybird.co
+    with:
+      data_project_dir: packages/twenty-tinybird
+    secrets:
+      tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}
+      tb_host: https://api.eu-central-1.aws.tinybird.co

packages/twenty-front/src/modules/navigation/components/MainNavigationDrawerItems.tsx

@@ -13,7 +13,6 @@ import { isNavigationDrawerExpandedState } from '@/ui/navigation/states/isNaviga
 import { navigationDrawerExpandedMemorizedState } from '@/ui/navigation/states/navigationDrawerExpandedMemorizedState';
 import { navigationMemorizedUrlState } from '@/ui/navigation/states/navigationMemorizedUrlState';
 import { useIsMobile } from '@/ui/utilities/responsive/hooks/useIsMobile';
-import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled';
 import styled from '@emotion/styled';
 
 const StyledMainSection = styled(NavigationDrawerSection)`
@@ -27,9 +26,7 @@ export const MainNavigationDrawerItems = () => {
   const setNavigationMemorizedUrl = useSetRecoilState(
     navigationMemorizedUrlState,
   );
-  const isWorkspaceFavoriteEnabled = useIsFeatureEnabled(
-    'IS_WORKSPACE_FAVORITE_ENABLED',
-  );
+
   const [isNavigationDrawerExpanded, setIsNavigationDrawerExpanded] =
     useRecoilState(isNavigationDrawerExpandedState);
   const setNavigationDrawerExpandedMemorized = useSetRecoilState(
@@ -58,18 +55,9 @@ export const MainNavigationDrawerItems = () => {
           />
         </StyledMainSection>
       )}
-
-      {isWorkspaceFavoriteEnabled && <NavigationDrawerOpenedSection />}
-
+      <NavigationDrawerOpenedSection />
       <CurrentWorkspaceMemberFavorites />
-
-      {isWorkspaceFavoriteEnabled ? (
-        <WorkspaceFavorites />
-      ) : (
-        <NavigationDrawerSectionForObjectMetadataItemsWrapper
-          isRemote={false}
-        />
-      )}
+      <WorkspaceFavorites />
       <NavigationDrawerSectionForObjectMetadataItemsWrapper isRemote={true} />
     </>
   );

packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts

@@ -9,12 +9,11 @@ export type FeatureFlagKey =
   | 'IS_FREE_ACCESS_ENABLED'
   | 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED'
   | 'IS_WORKFLOW_ENABLED'
-  | 'IS_WORKSPACE_FAVORITE_ENABLED'
-  | 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED'
   | 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED'
   | 'IS_ANALYTICS_V2_ENABLED'
   | 'IS_SSO_ENABLED'
   | 'IS_UNIQUE_INDEXES_ENABLED'
   | 'IS_ARRAY_AND_JSON_FILTER_ENABLED'
   | 'IS_MICROSOFT_SYNC_ENABLED'
-  | 'IS_ADVANCED_FILTERS_ENABLED';
+  | 'IS_ADVANCED_FILTERS_ENABLED'
+  | 'IS_AGGREGATE_QUERY_ENABLED';

packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts

@@ -50,11 +50,6 @@ export const seedFeatureFlags = async (
         workspaceId: workspaceId,
         value: false,
       },
-      {
-        key: FeatureFlagKey.IsWorkspaceFavoriteEnabled,
-        workspaceId: workspaceId,
-        value: true,
-      },
       {
         key: FeatureFlagKey.IsAnalyticsV2Enabled,
         workspaceId: workspaceId,
@@ -85,6 +80,11 @@ export const seedFeatureFlags = async (
         workspaceId: workspaceId,
         value: false,
       },
+      {
+        key: FeatureFlagKey.IsAggregateQueryEnabled,
+        workspaceId: workspaceId,
+        value: false,
+      },
     ])
     .execute();
 };

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-condition.parser.ts

@@ -5,27 +5,27 @@ import {
   WhereExpressionBuilder,
 } from 'typeorm';
 
-import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 
 import { GraphqlQueryFilterFieldParser } from './graphql-query-filter-field.parser';
 
 export class GraphqlQueryFilterConditionParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
   private queryFilterFieldParser: GraphqlQueryFilterFieldParser;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
     this.queryFilterFieldParser = new GraphqlQueryFilterFieldParser(
-      this.fieldMetadataMap,
+      this.fieldMetadataMapByName,
     );
   }
 
   public parse(
     queryBuilder: SelectQueryBuilder<any>,
     objectNameSingular: string,
-    filter: Partial<RecordFilter>,
+    filter: Partial<ObjectRecordFilter>,
   ): SelectQueryBuilder<any> {
     if (!filter || Object.keys(filter).length === 0) {
       return queryBuilder;
@@ -50,7 +50,7 @@ export class GraphqlQueryFilterConditionParser {
     switch (key) {
       case 'and': {
         const andWhereCondition = new Brackets((qb) => {
-          value.forEach((filter: RecordFilter, index: number) => {
+          value.forEach((filter: ObjectRecordFilter, index: number) => {
             const whereCondition = new Brackets((qb2) => {
               Object.entries(filter).forEach(
                 ([subFilterkey, subFilterValue], index) => {
@@ -82,7 +82,7 @@ export class GraphqlQueryFilterConditionParser {
       }
       case 'or': {
         const orWhereCondition = new Brackets((qb) => {
-          value.forEach((filter: RecordFilter, index: number) => {
+          value.forEach((filter: ObjectRecordFilter, index: number) => {
             const whereCondition = new Brackets((qb2) => {
               Object.entries(filter).forEach(
                 ([subFilterkey, subFilterValue], index) => {

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts

@@ -9,17 +9,17 @@ import {
 import { computeWhereConditionParts } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts';
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { capitalize } from 'src/utils/capitalize';
 
 const ARRAY_OPERATORS = ['in', 'contains', 'not_contains'];
 
 export class GraphqlQueryFilterFieldParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
   }
 
   public parse(
@@ -29,7 +29,7 @@ export class GraphqlQueryFilterFieldParser {
     filterValue: any,
     isFirst = false,
   ): void {
-    const fieldMetadata = this.fieldMetadataMap[`${key}`];
+    const fieldMetadata = this.fieldMetadataMapByName[`${key}`];
 
     if (!fieldMetadata) {
       throw new Error(`Field metadata not found for field: ${key}`);

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-order/graphql-query-order.parser.ts

@@ -1,7 +1,7 @@
 import {
+  ObjectRecordOrderBy,
   OrderByDirection,
-  RecordOrderBy,
-} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
 import {
@@ -10,25 +10,25 @@ import {
 } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { capitalize } from 'src/utils/capitalize';
 export class GraphqlQueryOrderFieldParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
   }
 
   parse(
-    orderBy: RecordOrderBy,
+    orderBy: ObjectRecordOrderBy,
     objectNameSingular: string,
     isForwardPagination = true,
   ): Record<string, string> {
     return orderBy.reduce(
       (acc, item) => {
         Object.entries(item).forEach(([key, value]) => {
-          const fieldMetadata = this.fieldMetadataMap[key];
+          const fieldMetadata = this.fieldMetadataMapByName[key];
 
           if (!fieldMetadata || value === undefined) {
             throw new GraphqlQueryRunnerException(

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-aggregate.parser.ts

@@ -0,0 +1,30 @@
+import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
+
+import { GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
+import {
+  AggregationField,
+  getAvailableAggregationsFromObjectFields,
+} from 'src/engine/api/graphql/workspace-schema-builder/utils/get-available-aggregations-from-object-fields.util';
+
+export class GraphqlQuerySelectedFieldsAggregateParser {
+  parse(
+    graphqlSelectedFields: Partial<Record<string, any>>,
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
+    accumulator: GraphqlQuerySelectedFieldsResult,
+  ): void {
+    const availableAggregations: Record<string, AggregationField> =
+      getAvailableAggregationsFromObjectFields(
+        Object.values(fieldMetadataMapByName),
+      );
+
+    for (const selectedField of Object.keys(graphqlSelectedFields)) {
+      const selectedAggregation = availableAggregations[selectedField];
+
+      if (!selectedAggregation) {
+        continue;
+      }
+
+      accumulator.aggregate[selectedField] = selectedAggregation;
+    }
+  }
+}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-relation.parser.ts

@@ -1,43 +1,47 @@
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
-import { GraphqlQuerySelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
+import {
+  GraphqlQuerySelectedFieldsParser,
+  GraphqlQuerySelectedFieldsResult,
+} from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
 import { getRelationObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util';
-import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 
 export class GraphqlQuerySelectedFieldsRelationParser {
-  private objectMetadataMap: ObjectMetadataMap;
+  private objectMetadataMaps: ObjectMetadataMaps;
 
-  constructor(objectMetadataMap: ObjectMetadataMap) {
-    this.objectMetadataMap = objectMetadataMap;
+  constructor(objectMetadataMaps: ObjectMetadataMaps) {
+    this.objectMetadataMaps = objectMetadataMaps;
   }
 
   parseRelationField(
     fieldMetadata: FieldMetadataInterface,
     fieldKey: string,
     fieldValue: any,
-    result: { select: Record<string, any>; relations: Record<string, any> },
+    accumulator: GraphqlQuerySelectedFieldsResult,
   ): void {
     if (!fieldValue || typeof fieldValue !== 'object') {
       return;
     }
 
-    result.relations[fieldKey] = true;
+    accumulator.relations[fieldKey] = true;
 
     const referencedObjectMetadata = getRelationObjectMetadata(
       fieldMetadata,
-      this.objectMetadataMap,
+      this.objectMetadataMaps,
     );
 
-    const relationFields = referencedObjectMetadata.fields;
+    const relationFields = referencedObjectMetadata.fieldsByName;
     const fieldParser = new GraphqlQuerySelectedFieldsParser(
-      this.objectMetadataMap,
+      this.objectMetadataMaps,
     );
-    const subResult = fieldParser.parse(fieldValue, relationFields);
+    const relationAccumulator = fieldParser.parse(fieldValue, relationFields);
 
-    result.select[fieldKey] = {
+    accumulator.select[fieldKey] = {
       id: true,
-      ...subResult.select,
+      ...relationAccumulator.select,
     };
-    result.relations[fieldKey] = subResult.relations;
+    accumulator.relations[fieldKey] = relationAccumulator.relations;
+    accumulator.aggregate[fieldKey] = relationAccumulator.aggregate;
   }
 }