Skip to content

Commit

Permalink
Aggregated queries twentyhq#1 (twentyhq#8345)
Browse files Browse the repository at this point in the history
First step of twentyhq#6868

Adds min.., max.. queries for DATETIME fields
adds min.., max.., avg.., sum.. queries for NUMBER fields 

(count distinct operation and composite fields such as CURRENCY handling
will be dealt with in a future PR)

<img width="1422" alt="Capture d’écran 2024-11-06 à 15 48 46"
src="https://github.com/user-attachments/assets/4bcdece0-ad3e-4536-9720-fe4044a36719">

---------

Co-authored-by: Charles Bochet <[email protected]>
Co-authored-by: Weiko <[email protected]>
  • Loading branch information
3 people authored and khuddite committed Nov 18, 2024
1 parent 5ba1155 commit 25020b8
Show file tree
Hide file tree
Showing 93 changed files with 1,584 additions and 1,172 deletions.
31 changes: 11 additions & 20 deletions .github/workflows/ci-tinybird.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,24 @@ on:
push:
branches:
- main
paths:
- 'package.json'
- 'packages/twenty-tinybird/**'

pull_request:
paths:
- 'package.json'
- 'packages/twenty-tinybird/**'

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
ci:
timeout-minutes: 10
runs-on: ubuntu-latest
uses: tinybirdco/ci/.github/workflows/ci.yml@main
steps:
- name: Check for changed files
id: changed-files
uses: tj-actions/changed-files@v11
with:
files: |
package.json
packages/twenty-tinybird/**
- name: Skip if no relevant changes
if: steps.changed-files.outputs.any_changed == 'false'
run: echo "No relevant changes. Skipping CI."

- name: Check twenty-tinybird package
with:
data_project_dir: packages/twenty-tinybird
tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}
tb_host: https://api.eu-central-1.aws.tinybird.co
with:
data_project_dir: packages/twenty-tinybird
secrets:
tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}
tb_host: https://api.eu-central-1.aws.tinybird.co
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { isNavigationDrawerExpandedState } from '@/ui/navigation/states/isNaviga
import { navigationDrawerExpandedMemorizedState } from '@/ui/navigation/states/navigationDrawerExpandedMemorizedState';
import { navigationMemorizedUrlState } from '@/ui/navigation/states/navigationMemorizedUrlState';
import { useIsMobile } from '@/ui/utilities/responsive/hooks/useIsMobile';
import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled';
import styled from '@emotion/styled';

const StyledMainSection = styled(NavigationDrawerSection)`
Expand All @@ -27,9 +26,7 @@ export const MainNavigationDrawerItems = () => {
const setNavigationMemorizedUrl = useSetRecoilState(
navigationMemorizedUrlState,
);
const isWorkspaceFavoriteEnabled = useIsFeatureEnabled(
'IS_WORKSPACE_FAVORITE_ENABLED',
);

const [isNavigationDrawerExpanded, setIsNavigationDrawerExpanded] =
useRecoilState(isNavigationDrawerExpandedState);
const setNavigationDrawerExpandedMemorized = useSetRecoilState(
Expand Down Expand Up @@ -58,18 +55,9 @@ export const MainNavigationDrawerItems = () => {
/>
</StyledMainSection>
)}

{isWorkspaceFavoriteEnabled && <NavigationDrawerOpenedSection />}

<NavigationDrawerOpenedSection />
<CurrentWorkspaceMemberFavorites />

{isWorkspaceFavoriteEnabled ? (
<WorkspaceFavorites />
) : (
<NavigationDrawerSectionForObjectMetadataItemsWrapper
isRemote={false}
/>
)}
<WorkspaceFavorites />
<NavigationDrawerSectionForObjectMetadataItemsWrapper isRemote={true} />
</>
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,11 @@ export type FeatureFlagKey =
| 'IS_FREE_ACCESS_ENABLED'
| 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED'
| 'IS_WORKFLOW_ENABLED'
| 'IS_WORKSPACE_FAVORITE_ENABLED'
| 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED'
| 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED'
| 'IS_ANALYTICS_V2_ENABLED'
| 'IS_SSO_ENABLED'
| 'IS_UNIQUE_INDEXES_ENABLED'
| 'IS_ARRAY_AND_JSON_FILTER_ENABLED'
| 'IS_MICROSOFT_SYNC_ENABLED'
| 'IS_ADVANCED_FILTERS_ENABLED';
| 'IS_ADVANCED_FILTERS_ENABLED'
| 'IS_AGGREGATE_QUERY_ENABLED';
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ export const seedFeatureFlags = async (
workspaceId: workspaceId,
value: false,
},
{
key: FeatureFlagKey.IsWorkspaceFavoriteEnabled,
workspaceId: workspaceId,
value: true,
},
{
key: FeatureFlagKey.IsAnalyticsV2Enabled,
workspaceId: workspaceId,
Expand Down Expand Up @@ -85,6 +80,11 @@ export const seedFeatureFlags = async (
workspaceId: workspaceId,
value: false,
},
{
key: FeatureFlagKey.IsAggregateQueryEnabled,
workspaceId: workspaceId,
value: false,
},
])
.execute();
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,27 @@ import {
WhereExpressionBuilder,
} from 'typeorm';

import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';

import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';

import { GraphqlQueryFilterFieldParser } from './graphql-query-filter-field.parser';

export class GraphqlQueryFilterConditionParser {
private fieldMetadataMap: FieldMetadataMap;
private fieldMetadataMapByName: FieldMetadataMap;
private queryFilterFieldParser: GraphqlQueryFilterFieldParser;

constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
constructor(fieldMetadataMapByName: FieldMetadataMap) {
this.fieldMetadataMapByName = fieldMetadataMapByName;
this.queryFilterFieldParser = new GraphqlQueryFilterFieldParser(
this.fieldMetadataMap,
this.fieldMetadataMapByName,
);
}

public parse(
queryBuilder: SelectQueryBuilder<any>,
objectNameSingular: string,
filter: Partial<RecordFilter>,
filter: Partial<ObjectRecordFilter>,
): SelectQueryBuilder<any> {
if (!filter || Object.keys(filter).length === 0) {
return queryBuilder;
Expand All @@ -50,7 +50,7 @@ export class GraphqlQueryFilterConditionParser {
switch (key) {
case 'and': {
const andWhereCondition = new Brackets((qb) => {
value.forEach((filter: RecordFilter, index: number) => {
value.forEach((filter: ObjectRecordFilter, index: number) => {
const whereCondition = new Brackets((qb2) => {
Object.entries(filter).forEach(
([subFilterkey, subFilterValue], index) => {
Expand Down Expand Up @@ -82,7 +82,7 @@ export class GraphqlQueryFilterConditionParser {
}
case 'or': {
const orWhereCondition = new Brackets((qb) => {
value.forEach((filter: RecordFilter, index: number) => {
value.forEach((filter: ObjectRecordFilter, index: number) => {
const whereCondition = new Brackets((qb2) => {
Object.entries(filter).forEach(
([subFilterkey, subFilterValue], index) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ import {
import { computeWhereConditionParts } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { capitalize } from 'src/utils/capitalize';

const ARRAY_OPERATORS = ['in', 'contains', 'not_contains'];

export class GraphqlQueryFilterFieldParser {
private fieldMetadataMap: FieldMetadataMap;
private fieldMetadataMapByName: FieldMetadataMap;

constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
constructor(fieldMetadataMapByName: FieldMetadataMap) {
this.fieldMetadataMapByName = fieldMetadataMapByName;
}

public parse(
Expand All @@ -29,7 +29,7 @@ export class GraphqlQueryFilterFieldParser {
filterValue: any,
isFirst = false,
): void {
const fieldMetadata = this.fieldMetadataMap[`${key}`];
const fieldMetadata = this.fieldMetadataMapByName[`${key}`];

if (!fieldMetadata) {
throw new Error(`Field metadata not found for field: ${key}`);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {
ObjectRecordOrderBy,
OrderByDirection,
RecordOrderBy,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';

import {
Expand All @@ -10,25 +10,25 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { capitalize } from 'src/utils/capitalize';
export class GraphqlQueryOrderFieldParser {
private fieldMetadataMap: FieldMetadataMap;
private fieldMetadataMapByName: FieldMetadataMap;

constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
constructor(fieldMetadataMapByName: FieldMetadataMap) {
this.fieldMetadataMapByName = fieldMetadataMapByName;
}

parse(
orderBy: RecordOrderBy,
orderBy: ObjectRecordOrderBy,
objectNameSingular: string,
isForwardPagination = true,
): Record<string, string> {
return orderBy.reduce(
(acc, item) => {
Object.entries(item).forEach(([key, value]) => {
const fieldMetadata = this.fieldMetadataMap[key];
const fieldMetadata = this.fieldMetadataMapByName[key];

if (!fieldMetadata || value === undefined) {
throw new GraphqlQueryRunnerException(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';

import { GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
import {
AggregationField,
getAvailableAggregationsFromObjectFields,
} from 'src/engine/api/graphql/workspace-schema-builder/utils/get-available-aggregations-from-object-fields.util';

export class GraphqlQuerySelectedFieldsAggregateParser {
parse(
graphqlSelectedFields: Partial<Record<string, any>>,
fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
accumulator: GraphqlQuerySelectedFieldsResult,
): void {
const availableAggregations: Record<string, AggregationField> =
getAvailableAggregationsFromObjectFields(
Object.values(fieldMetadataMapByName),
);

for (const selectedField of Object.keys(graphqlSelectedFields)) {
const selectedAggregation = availableAggregations[selectedField];

if (!selectedAggregation) {
continue;
}

accumulator.aggregate[selectedField] = selectedAggregation;
}
}
}
Original file line number Diff line number Diff line change
@@ -1,43 +1,47 @@
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';

import { GraphqlQuerySelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
import {
GraphqlQuerySelectedFieldsParser,
GraphqlQuerySelectedFieldsResult,
} from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
import { getRelationObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util';
import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';

export class GraphqlQuerySelectedFieldsRelationParser {
private objectMetadataMap: ObjectMetadataMap;
private objectMetadataMaps: ObjectMetadataMaps;

constructor(objectMetadataMap: ObjectMetadataMap) {
this.objectMetadataMap = objectMetadataMap;
constructor(objectMetadataMaps: ObjectMetadataMaps) {
this.objectMetadataMaps = objectMetadataMaps;
}

parseRelationField(
fieldMetadata: FieldMetadataInterface,
fieldKey: string,
fieldValue: any,
result: { select: Record<string, any>; relations: Record<string, any> },
accumulator: GraphqlQuerySelectedFieldsResult,
): void {
if (!fieldValue || typeof fieldValue !== 'object') {
return;
}

result.relations[fieldKey] = true;
accumulator.relations[fieldKey] = true;

const referencedObjectMetadata = getRelationObjectMetadata(
fieldMetadata,
this.objectMetadataMap,
this.objectMetadataMaps,
);

const relationFields = referencedObjectMetadata.fields;
const relationFields = referencedObjectMetadata.fieldsByName;
const fieldParser = new GraphqlQuerySelectedFieldsParser(
this.objectMetadataMap,
this.objectMetadataMaps,
);
const subResult = fieldParser.parse(fieldValue, relationFields);
const relationAccumulator = fieldParser.parse(fieldValue, relationFields);

result.select[fieldKey] = {
accumulator.select[fieldKey] = {
id: true,
...subResult.select,
...relationAccumulator.select,
};
result.relations[fieldKey] = subResult.relations;
accumulator.relations[fieldKey] = relationAccumulator.relations;
accumulator.aggregate[fieldKey] = relationAccumulator.aggregate;
}
}
Loading

0 comments on commit 25020b8

Please sign in to comment.