Update foreign table to distant table schema (#5508)

Closes #5069 back-end part

And:
- do not display schemaPendingUpdates status on remote server lists as
this call will become too costly if there are dozens of servers
- (refacto) create foreignTableService

After this is merged we will be able to delete remoteTable's
availableTables column

Author: ijreilly

packages/twenty-front/src/generated-metadata/graphql.ts

@@ -338,6 +338,8 @@ export enum FileFolder {
 export type FindManyRemoteTablesInput = {
   /** The id of the remote server. */
   id: Scalars['ID']['input'];
+  /** Indicates if pending schema updates status should be computed. */
+  shouldFetchPendingSchemaUpdates?: InputMaybe<Scalars['Boolean']['input']>;
 };
 
 export type FullName = {

packages/twenty-front/src/modules/databases/hooks/useGetDatabaseConnectionTables.ts

@@ -10,11 +10,13 @@ import {
 type UseGetDatabaseConnectionTablesParams = {
   connectionId: string;
   skip?: boolean;
+  shouldFetchPendingSchemaUpdates?: boolean;
 };
 
 export const useGetDatabaseConnectionTables = ({
   connectionId,
   skip,
+  shouldFetchPendingSchemaUpdates,
 }: UseGetDatabaseConnectionTablesParams) => {
   const apolloMetadataClient = useApolloMetadataClient();
 
@@ -27,6 +29,7 @@ export const useGetDatabaseConnectionTables = ({
     variables: {
       input: {
         id: connectionId,
+        shouldFetchPendingSchemaUpdates,
       },
     },
   });

packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSummaryCard.tsx

@@ -53,6 +53,7 @@ export const SettingsIntegrationDatabaseConnectionSummaryCard = ({
         <>
           <SettingsIntegrationDatabaseConnectionSyncStatus
             connectionId={connectionId}
+            shouldFetchPendingSchemaUpdates
           />
           <Dropdown
             dropdownId={dropdownId}

packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSyncStatus.tsx

@@ -6,15 +6,18 @@ import { isDefined } from '~/utils/isDefined';
 type SettingsIntegrationDatabaseConnectionSyncStatusProps = {
   connectionId: string;
   skip?: boolean;
+  shouldFetchPendingSchemaUpdates?: boolean;
 };
 
 export const SettingsIntegrationDatabaseConnectionSyncStatus = ({
   connectionId,
   skip,
+  shouldFetchPendingSchemaUpdates,
 }: SettingsIntegrationDatabaseConnectionSyncStatusProps) => {
   const { tables, error } = useGetDatabaseConnectionTables({
     connectionId,
     skip,
+    shouldFetchPendingSchemaUpdates,
   });
 
   if (isDefined(error)) {

packages/twenty-front/src/modules/settings/integrations/database-connection/hooks/useDatabaseConnection.ts

@@ -42,6 +42,7 @@ export const useDatabaseConnection = () => {
   const { tables } = useGetDatabaseConnectionTables({
     connectionId,
     skip: !connection,
+    shouldFetchPendingSchemaUpdates: true,
   });
 
   return { connection, integration, databaseKey, tables };

packages/twenty-server/src/database/typeorm/metadata/migrations/1716310822694-removeAvailableTables.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class RemoveAvailableTables1716310822694 implements MigrationInterface {
+  name = 'RemoveAvailableTables1716310822694';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."remoteServer" DROP COLUMN "availableTables"`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."remoteServer" ADD "availableTables" jsonb`,
+    );
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts

@@ -11,7 +11,6 @@ import {
 
 import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
 import { UserMappingOptions } from 'src/engine/metadata-modules/remote-server/types/user-mapping-options';
-import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table';
 
 export enum RemoteServerType {
   POSTGRES_FDW = 'postgres_fdw',
@@ -59,9 +58,6 @@ export class RemoteServerEntity<T extends RemoteServerType> {
   @Column({ nullable: false, type: 'uuid' })
   workspaceId: string;
 
-  @Column({ type: 'jsonb', nullable: true })
-  availableTables: DistantTables;
-
   @OneToMany(() => RemoteTableEntity, (table) => table.server, {
     cascade: true,
   })

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts

@@ -23,43 +23,40 @@ export class DistantTableService {
     >,
   ) {}
 
-  public getDistantTableColumns(
-    remoteServer: RemoteServerEntity<RemoteServerType>,
-    tableName: string,
-  ): PostgresTableSchemaColumn[] {
-    if (!remoteServer.availableTables) {
-      throw new BadRequestException(
-        'Remote server available tables are not defined',
-      );
-    }
-
-    return remoteServer.availableTables[tableName];
-  }
-
   public async fetchDistantTables(
     remoteServer: RemoteServerEntity<RemoteServerType>,
     workspaceId: string,
-  ): Promise<DistantTables> {
-    return this.createAvailableTables(remoteServer, workspaceId);
-  }
-
-  private async createAvailableTables(
-    remoteServer: RemoteServerEntity<RemoteServerType>,
-    workspaceId: string,
+    tableName?: string,
   ): Promise<DistantTables> {
     if (remoteServer.schema) {
-      return this.createAvailableTablesFromDynamicSchema(
+      return this.getDistantTablesFromDynamicSchema(
         remoteServer,
         workspaceId,
+        tableName,
       );
     }
 
-    return this.createAvailableTablesFromStaticSchema(remoteServer);
+    return this.getDistantTablesFromStaticSchema(remoteServer);
   }
 
-  private async createAvailableTablesFromDynamicSchema(
+  public async getDistantTableColumns(
     remoteServer: RemoteServerEntity<RemoteServerType>,
     workspaceId: string,
+    tableName: string,
+  ): Promise<PostgresTableSchemaColumn[]> {
+    const distantTables = await this.fetchDistantTables(
+      remoteServer,
+      workspaceId,
+      tableName,
+    );
+
+    return distantTables[tableName] || [];
+  }
+
+  private async getDistantTablesFromDynamicSchema(
+    remoteServer: RemoteServerEntity<RemoteServerType>,
+    workspaceId: string,
+    tableName?: string,
   ): Promise<DistantTables> {
     if (!remoteServer.schema) {
       throw new BadRequestException('Remote server schema is not defined');
@@ -73,12 +70,16 @@ export class DistantTableService {
         workspaceId,
       );
 
-    const availableTables = await workspaceDataSource.transaction(
+    const distantTables = await workspaceDataSource.transaction(
       async (entityManager: EntityManager) => {
         await entityManager.query(`CREATE SCHEMA "${tmpSchemaName}"`);
 
+        const tableLimitationsOptions = tableName
+          ? ` LIMIT TO (${tableName})`
+          : '';
+
         await entityManager.query(
-          `IMPORT FOREIGN SCHEMA "${remoteServer.schema}" FROM SERVER "${remoteServer.foreignDataWrapperId}" INTO "${tmpSchemaName}"`,
+          `IMPORT FOREIGN SCHEMA "${remoteServer.schema}"${tableLimitationsOptions} FROM SERVER "${remoteServer.foreignDataWrapperId}" INTO "${tmpSchemaName}"`,
         );
 
         const createdForeignTableNames = await entityManager.query(
@@ -106,22 +107,14 @@ export class DistantTableService {
       },
     );
 
-    await this.remoteServerRepository.update(remoteServer.id, {
-      availableTables,
-    });
-
-    return availableTables;
+    return distantTables;
   }
 
-  private async createAvailableTablesFromStaticSchema(
+  private async getDistantTablesFromStaticSchema(
     remoteServer: RemoteServerEntity<RemoteServerType>,
   ): Promise<DistantTables> {
     switch (remoteServer.foreignDataWrapperType) {
       case RemoteServerType.STRIPE_FDW:
-        this.remoteServerRepository.update(remoteServer.id, {
-          availableTables: STRIPE_DISTANT_TABLES,
-        });
-
         return STRIPE_DISTANT_TABLES;
       default:
         throw new BadRequestException(

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts

@@ -1,5 +1,5 @@
 import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
 
 export type DistantTables = {
-  [tableName: string]: PostgresTableSchemaColumn[];
+  [distantTableName: string]: PostgresTableSchemaColumn[];
 };

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts

@@ -1,9 +1,18 @@
-import { InputType, ID } from '@nestjs/graphql';
+import { InputType, ID, Field } from '@nestjs/graphql';
 
 import { IDField } from '@ptc-org/nestjs-query-graphql';
+import { IsOptional } from 'class-validator';
 
 @InputType()
 export class FindManyRemoteTablesInput {
   @IDField(() => ID, { description: 'The id of the remote server.' })
   id!: string;
+
+  @IsOptional()
+  @Field(() => Boolean, {
+    description:
+      'Indicates if pending schema updates status should be computed.',
+    nullable: true,
+  })
+  shouldFetchPendingSchemaUpdates?: boolean;
 }

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+
+import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service';
+import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
+import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
+
+@Module({
+  imports: [
+    WorkspaceMigrationModule,
+    WorkspaceMigrationRunnerModule,
+    WorkspaceDataSourceModule,
+    WorkspaceCacheVersionModule,
+  ],
+  providers: [ForeignTableService],
+  exports: [ForeignTableService],
+})
+export class ForeignTableModule {}