feat(wren-ui): Support record type in BigQuery

wren-ui/migrations/20240928165009_create_model_nested_column.js

@@ -0,0 +1,48 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function (knex) {
+  return knex.schema.createTable('model_nested_column', (table) => {
+    table.increments('id').comment('ID');
+    table.integer('model_id').comment('Reference to model ID');
+    table.integer('column_id').comment('Reference to column ID');
+    table
+      .string('column_path')
+      .comment(
+        'The path of the nested column, array of strings, [sourceColumnName..sourceColumnName(n)]',
+      );
+
+    table.string('display_name').comment('Display name of the nested column');
+    table
+      .string('source_column_name')
+      .comment('the nested column name in the datasource');
+    table
+      .string('reference_name')
+      .comment('The name used in the MDL structure and query');
+    table
+      .string('type')
+      .comment('Data type, refer to the nested column type in the datasource')
+      .nullable();
+    table
+      .text('properties')
+      .comment(
+        'nested column properties, a json string, the description should be stored here',
+      )
+      .nullable();
+
+    table
+      .foreign('column_id')
+      .references('model_column.id')
+      .onDelete('CASCADE');
+    table.timestamps(true, true);
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function (knex) {
+  return knex.schema.dropTable('model_nested_column');
+};

wren-ui/src/apollo/client/graphql/__types__.ts

@@ -174,6 +174,7 @@ export type DetailedColumn = {
   __typename?: 'DetailedColumn';
   displayName: Scalars['String'];
   isCalculated: Scalars['Boolean'];
+  nestedColumns?: Maybe<Array<DetailedNestedColumn>>;
   notNull: Scalars['Boolean'];
   properties: Scalars['JSON'];
   referenceName: Scalars['String'];
@@ -197,6 +198,16 @@ export type DetailedModel = {
   sourceTableName: Scalars['String'];
 };
 
+export type DetailedNestedColumn = {
+  __typename?: 'DetailedNestedColumn';
+  displayName: Scalars['String'];
+  id: Scalars['Int'];
+  properties?: Maybe<Scalars['JSON']>;
+  referenceName: Scalars['String'];
+  sourceColumnName: Scalars['String'];
+  type?: Maybe<Scalars['String']>;
+};
+
 export type DetailedRelation = {
   __typename?: 'DetailedRelation';
   fromColumnId: Scalars['Int'];
@@ -250,11 +261,23 @@ export type DiagramModelField = {
   id: Scalars['String'];
   isPrimaryKey: Scalars['Boolean'];
   lineage?: Maybe<Array<Scalars['Int']>>;
+  nestedFields?: Maybe<Array<DiagramModelNestedField>>;
   nodeType: NodeType;
   referenceName: Scalars['String'];
   type: Scalars['String'];
 };
 
+export type DiagramModelNestedField = {
+  __typename?: 'DiagramModelNestedField';
+  columnPath: Array<Scalars['String']>;
+  description?: Maybe<Scalars['String']>;
+  displayName: Scalars['String'];
+  id: Scalars['String'];
+  nestedColumnId: Scalars['Int'];
+  referenceName: Scalars['String'];
+  type: Scalars['String'];
+};
+
 export type DiagramModelRelationField = {
   __typename?: 'DiagramModelRelationField';
   description?: Maybe<Scalars['String']>;
@@ -880,9 +903,16 @@ export type UpdateModelMetadataInput = {
   columns?: InputMaybe<Array<UpdateColumnMetadataInput>>;
   description?: InputMaybe<Scalars['String']>;
   displayName?: InputMaybe<Scalars['String']>;
+  nestedColumns?: InputMaybe<Array<UpdateNestedColumnMetadataInput>>;
   relationships?: InputMaybe<Array<UpdateRelationshipMetadataInput>>;
 };
 
+export type UpdateNestedColumnMetadataInput = {
+  description?: InputMaybe<Scalars['String']>;
+  displayName?: InputMaybe<Scalars['String']>;
+  id: Scalars['Int'];
+};
+
 export type UpdateRelationInput = {
   type: RelationType;
 };

wren-ui/src/apollo/client/graphql/diagram.generated.ts

@@ -7,12 +7,14 @@ export type ViewFieldFragment = { __typename?: 'DiagramViewField', id: string, d
 
 export type RelationFieldFragment = { __typename?: 'DiagramModelRelationField', id: string, relationId: number, type: Types.RelationType, nodeType: Types.NodeType, displayName: string, referenceName: string, fromModelId: number, fromModelName: string, fromModelDisplayName: string, fromColumnId: number, fromColumnName: string, fromColumnDisplayName: string, toModelId: number, toModelName: string, toModelDisplayName: string, toColumnId: number, toColumnName: string, toColumnDisplayName: string, description?: string | null };
 
-export type FieldFragment = { __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null };
+export type NestedFieldFragment = { __typename?: 'DiagramModelNestedField', id: string, nestedColumnId: number, columnPath: Array<string>, type: string, displayName: string, referenceName: string, description?: string | null };
+
+export type FieldFragment = { __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null, nestedFields?: Array<{ __typename?: 'DiagramModelNestedField', id: string, nestedColumnId: number, columnPath: Array<string>, type: string, displayName: string, referenceName: string, description?: string | null }> | null };
 
 export type DiagramQueryVariables = Types.Exact<{ [key: string]: never; }>;
 
 
-export type DiagramQuery = { __typename?: 'Query', diagram: { __typename?: 'Diagram', models: Array<{ __typename?: 'DiagramModel', id: string, modelId: number, nodeType: Types.NodeType, displayName: string, referenceName: string, sourceTableName: string, refSql?: string | null, cached: boolean, refreshTime?: string | null, description?: string | null, fields: Array<{ __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null } | null>, calculatedFields: Array<{ __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null } | null>, relationFields: Array<{ __typename?: 'DiagramModelRelationField', id: string, relationId: number, type: Types.RelationType, nodeType: Types.NodeType, displayName: string, referenceName: string, fromModelId: number, fromModelName: string, fromModelDisplayName: string, fromColumnId: number, fromColumnName: string, fromColumnDisplayName: string, toModelId: number, toModelName: string, toModelDisplayName: string, toColumnId: number, toColumnName: string, toColumnDisplayName: string, description?: string | null } | null> } | null>, views: Array<{ __typename?: 'DiagramView', id: string, viewId: number, nodeType: Types.NodeType, displayName: string, description?: string | null, referenceName: string, statement: string, fields: Array<{ __typename?: 'DiagramViewField', id: string, displayName: string, referenceName: string, type: string, nodeType: Types.NodeType, description?: string | null } | null> } | null> } };
+export type DiagramQuery = { __typename?: 'Query', diagram: { __typename?: 'Diagram', models: Array<{ __typename?: 'DiagramModel', id: string, modelId: number, nodeType: Types.NodeType, displayName: string, referenceName: string, sourceTableName: string, refSql?: string | null, cached: boolean, refreshTime?: string | null, description?: string | null, fields: Array<{ __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null, nestedFields?: Array<{ __typename?: 'DiagramModelNestedField', id: string, nestedColumnId: number, columnPath: Array<string>, type: string, displayName: string, referenceName: string, description?: string | null }> | null } | null>, calculatedFields: Array<{ __typename?: 'DiagramModelField', id: string, columnId: number, type: string, nodeType: Types.NodeType, displayName: string, referenceName: string, description?: string | null, isPrimaryKey: boolean, expression?: string | null, aggregation?: string | null, lineage?: Array<number> | null, nestedFields?: Array<{ __typename?: 'DiagramModelNestedField', id: string, nestedColumnId: number, columnPath: Array<string>, type: string, displayName: string, referenceName: string, description?: string | null }> | null } | null>, relationFields: Array<{ __typename?: 'DiagramModelRelationField', id: string, relationId: number, type: Types.RelationType, nodeType: Types.NodeType, displayName: string, referenceName: string, fromModelId: number, fromModelName: string, fromModelDisplayName: string, fromColumnId: number, fromColumnName: string, fromColumnDisplayName: string, toModelId: number, toModelName: string, toModelDisplayName: string, toColumnId: number, toColumnName: string, toColumnDisplayName: string, description?: string | null } | null> } | null>, views: Array<{ __typename?: 'DiagramView', id: string, viewId: number, nodeType: Types.NodeType, displayName: string, description?: string | null, referenceName: string, statement: string, fields: Array<{ __typename?: 'DiagramViewField', id: string, displayName: string, referenceName: string, type: string, nodeType: Types.NodeType, description?: string | null } | null> } | null> } };
 
 export const ViewFieldFragmentDoc = gql`
     fragment ViewField on DiagramViewField {
@@ -47,6 +49,17 @@ export const RelationFieldFragmentDoc = gql`
   description
 }
     `;
+export const NestedFieldFragmentDoc = gql`
+    fragment NestedField on DiagramModelNestedField {
+  id
+  nestedColumnId
+  columnPath
+  type
+  displayName
+  referenceName
+  description
+}
+    `;
 export const FieldFragmentDoc = gql`
     fragment Field on DiagramModelField {
   id
@@ -60,8 +73,11 @@ export const FieldFragmentDoc = gql`
   expression
   aggregation
   lineage
+  nestedFields {
+    ...NestedField
+  }
 }
-    `;
+    ${NestedFieldFragmentDoc}`;
 export const DiagramDocument = gql`
     query Diagram {
   diagram {

wren-ui/src/apollo/client/graphql/diagram.ts

@@ -35,6 +35,18 @@ const RELATION_FIELD = gql`
   }
 `;
 
+const NESTED_FIELD = gql`
+  fragment NestedField on DiagramModelNestedField {
+    id
+    nestedColumnId
+    columnPath
+    type
+    displayName
+    referenceName
+    description
+  }
+`;
+
 const FIELD = gql`
   fragment Field on DiagramModelField {
     id
@@ -48,7 +60,11 @@ const FIELD = gql`
     expression
     aggregation
     lineage
+    nestedFields {
+      ...NestedField
+    }
   }
+  ${NESTED_FIELD}
 `;
 
 export const DIAGRAM = gql`

wren-ui/src/apollo/server/mdl/mdlBuilder.ts

@@ -2,6 +2,7 @@ import { isEmpty, isNil, pickBy } from 'lodash';
 import {
   Model,
   ModelColumn,
+  ModelNestedColumn,
   Project,
   RelationInfo,
   View,
@@ -16,6 +17,7 @@ export interface MDLBuilderBuildFromOptions {
   project: Project;
   models: Model[];
   columns?: ModelColumn[];
+  nestedColumns?: ModelNestedColumn[];
   relations?: RelationInfo[];
   views: View[];
   relatedModels?: Model[];
@@ -34,6 +36,7 @@ export class MDLBuilder implements IMDLBuilder {
   private project: Project;
   private readonly models: Model[];
   private readonly columns: ModelColumn[];
+  private readonly nestedColumns: ModelNestedColumn[];
   private readonly relations: RelationInfo[];
   private readonly views: View[];
 
@@ -50,6 +53,7 @@ export class MDLBuilder implements IMDLBuilder {
       project,
       models,
       columns,
+      nestedColumns,
       relations,
       views,
       relatedModels,
@@ -59,6 +63,7 @@ export class MDLBuilder implements IMDLBuilder {
     this.project = project;
     this.models = models;
     this.columns = columns;
+    this.nestedColumns = nestedColumns;
     this.relations = relations;
     this.views = views || [];
     this.relatedModels = relatedModels;
@@ -181,6 +186,24 @@ export class MDLBuilder implements IMDLBuilder {
         if (column.displayName) {
           properties.displayName = column.displayName;
         }
+        // put nested columns in properties
+        if (column.type.includes('STRUCT')) {
+          const nestedColumns = this.nestedColumns.filter(
+            (nestedColumn) => nestedColumn.columnId === column.id,
+          );
+          nestedColumns.forEach((column) => {
+            if (column.displayName) {
+              properties[
+                `nestedColumn.${column.sourceColumnName}.displayName`
+              ] = column.displayName;
+            }
+            if (column.properties?.description) {
+              properties[
+                `nestedColumn.${column.sourceColumnName}.description`
+              ] = column.properties.description;
+            }
+          }, {});
+        }
         const expression = this.getColumnExpression(column, model);
         model.columns.push({
           name: column.referenceName,