Serverless function improvements (#6769)

- add layer for lambda execution
- add layer for local execution
- add package resolve for the monaco editor
- add route to get installed package for serverless functions
- add layer versioning

Author: martmull

package.json

@@ -94,7 +94,6 @@
     "facepaint": "^1.2.1",
     "file-type": "16.5.4",
     "framer-motion": "^10.12.17",
-    "fs-extra": "^11.2.0",
     "googleapis": "105",
     "graphiql": "^3.1.1",
     "graphql": "16.8.0",

packages/twenty-front/src/generated-metadata/gql.ts

@@ -39,6 +39,7 @@ const documents = {
     "\n  mutation ExecuteOneServerlessFunction(\n    $input: ExecuteServerlessFunctionInput!\n  ) {\n    executeOneServerlessFunction(input: $input) {\n      data\n      duration\n      status\n      error\n    }\n  }\n": types.ExecuteOneServerlessFunctionDocument,
     "\n  \n  mutation PublishOneServerlessFunction(\n    $input: PublishServerlessFunctionInput!\n  ) {\n    publishServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n": types.PublishOneServerlessFunctionDocument,
     "\n  \n  mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n    updateOneServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n": types.UpdateOneServerlessFunctionDocument,
+    "\n  query FindManyAvailablePackages {\n    getAvailablePackages\n  }\n": types.FindManyAvailablePackagesDocument,
     "\n  \n  query GetManyServerlessFunctions {\n    serverlessFunctions(paging: { first: 100 }) {\n      edges {\n        node {\n          ...ServerlessFunctionFields\n        }\n      }\n    }\n  }\n": types.GetManyServerlessFunctionsDocument,
     "\n  \n  query GetOneServerlessFunction($id: UUID!) {\n    serverlessFunction(id: $id) {\n      ...ServerlessFunctionFields\n    }\n  }\n": types.GetOneServerlessFunctionDocument,
     "\n  query FindOneServerlessFunctionSourceCode(\n    $input: GetServerlessFunctionSourceCodeInput!\n  ) {\n    getServerlessFunctionSourceCode(input: $input)\n  }\n": types.FindOneServerlessFunctionSourceCodeDocument,
@@ -162,6 +163,10 @@ export function graphql(source: "\n  \n  mutation PublishOneServerlessFunction(\
  * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
  */
 export function graphql(source: "\n  \n  mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n    updateOneServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n"): (typeof documents)["\n  \n  mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n    updateOneServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n"];
+/**
+ * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
+ */
+export function graphql(source: "\n  query FindManyAvailablePackages {\n    getAvailablePackages\n  }\n"): (typeof documents)["\n  query FindManyAvailablePackages {\n    getAvailablePackages\n  }\n"];
 /**
  * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
  */

packages/twenty-front/src/generated-metadata/graphql.ts

@@ -314,7 +314,7 @@ export type ExecuteServerlessFunctionInput = {
   /** Id of the serverless function to execute */
   id: Scalars['UUID']['input'];
   /** Payload in JSON format */
-  payload?: InputMaybe<Scalars['JSON']['input']>;
+  payload: Scalars['JSON']['input'];
   /** Version of the serverless function to execute */
   version?: Scalars['String']['input'];
 };
@@ -814,9 +814,10 @@ export type Query = {
   findOneRemoteServerById: RemoteServer;
   findWorkspaceFromInviteHash: Workspace;
   getAISQLQuery: AisqlQueryResult;
+  getAvailablePackages: Scalars['JSON']['output'];
   getPostgresCredentials?: Maybe<PostgresCredentials>;
   getProductPrices: ProductPricesEntity;
-  getServerlessFunctionSourceCode: Scalars['String']['output'];
+  getServerlessFunctionSourceCode?: Maybe<Scalars['String']['output']>;
   getTimelineCalendarEventsFromCompanyId: TimelineCalendarEventsWithTotal;
   getTimelineCalendarEventsFromPersonId: TimelineCalendarEventsWithTotal;
   getTimelineThreadsFromCompanyId: TimelineThreadsWithTotal;
@@ -1761,6 +1762,11 @@ export type UpdateOneServerlessFunctionMutationVariables = Exact<{
 
 export type UpdateOneServerlessFunctionMutation = { __typename?: 'Mutation', updateOneServerlessFunction: { __typename?: 'ServerlessFunction', id: any, name: string, description?: string | null, sourceCodeHash: string, runtime: string, syncStatus: ServerlessFunctionSyncStatus, latestVersion?: string | null, createdAt: any, updatedAt: any } };
 
+export type FindManyAvailablePackagesQueryVariables = Exact<{ [key: string]: never; }>;
+
+
+export type FindManyAvailablePackagesQuery = { __typename?: 'Query', getAvailablePackages: any };
+
 export type GetManyServerlessFunctionsQueryVariables = Exact<{ [key: string]: never; }>;
 
 
@@ -1778,7 +1784,7 @@ export type FindOneServerlessFunctionSourceCodeQueryVariables = Exact<{
 }>;
 
 
-export type FindOneServerlessFunctionSourceCodeQuery = { __typename?: 'Query', getServerlessFunctionSourceCode: string };
+export type FindOneServerlessFunctionSourceCodeQuery = { __typename?: 'Query', getServerlessFunctionSourceCode?: string | null };
 
 export const RemoteServerFieldsFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"RemoteServerFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"RemoteServer"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperId"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperOptions"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperType"}},{"kind":"Field","name":{"kind":"Name","value":"userMappingOptions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"user"}}]}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"schema"}},{"kind":"Field","name":{"kind":"Name","value":"label"}}]}}]} as unknown as DocumentNode<RemoteServerFieldsFragment, unknown>;
 export const RemoteTableFieldsFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"RemoteTableFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"RemoteTable"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"schema"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"schemaPendingUpdates"}}]}}]} as unknown as DocumentNode<RemoteTableFieldsFragment, unknown>;
@@ -1806,6 +1812,7 @@ export const DeleteOneServerlessFunctionDocument = {"kind":"Document","definitio
 export const ExecuteOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ExecuteOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ExecuteServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"executeOneServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"data"}},{"kind":"Field","name":{"kind":"Name","value":"duration"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"error"}}]}}]}}]} as unknown as DocumentNode<ExecuteOneServerlessFunctionMutation, ExecuteOneServerlessFunctionMutationVariables>;
 export const PublishOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"PublishOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"PublishServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"publishServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<PublishOneServerlessFunctionMutation, PublishOneServerlessFunctionMutationVariables>;
 export const UpdateOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UpdateServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateOneServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<UpdateOneServerlessFunctionMutation, UpdateOneServerlessFunctionMutationVariables>;
+export const FindManyAvailablePackagesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindManyAvailablePackages"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getAvailablePackages"}}]}}]} as unknown as DocumentNode<FindManyAvailablePackagesQuery, FindManyAvailablePackagesQueryVariables>;
 export const GetManyServerlessFunctionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetManyServerlessFunctions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverlessFunctions"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"paging"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"first"},"value":{"kind":"IntValue","value":"100"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"edges"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"node"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetManyServerlessFunctionsQuery, GetManyServerlessFunctionsQueryVariables>;
 export const GetOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UUID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetOneServerlessFunctionQuery, GetOneServerlessFunctionQueryVariables>;
 export const FindOneServerlessFunctionSourceCodeDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindOneServerlessFunctionSourceCode"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"GetServerlessFunctionSourceCodeInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getServerlessFunctionSourceCode"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode<FindOneServerlessFunctionSourceCodeQuery, FindOneServerlessFunctionSourceCodeQueryVariables>;

packages/twenty-front/src/hooks/usePreventOverlapCallback.ts

@@ -0,0 +1,32 @@
+import { useEffect, useState } from 'react';
+import { useDebouncedCallback } from 'use-debounce';
+
+export const usePreventOverlapCallback = (
+  callback: () => Promise<void>,
+  wait?: number,
+) => {
+  const [isRunning, setIsRunning] = useState(false);
+  const [pendingRun, setPendingRun] = useState(false);
+
+  const handleCallback = async () => {
+    if (isRunning) {
+      setPendingRun(true);
+      return;
+    }
+    setIsRunning(true);
+    try {
+      await callback();
+    } finally {
+      setIsRunning(false);
+    }
+  };
+
+  useEffect(() => {
+    if (!isRunning && pendingRun) {
+      setPendingRun(false);
+      callback();
+    }
+  }, [callback, isRunning, pendingRun, setPendingRun]);
+
+  return useDebouncedCallback(handleCallback, wait);
+};

packages/twenty-front/src/modules/settings/components/SettingsNavigationDrawerItems.tsx

@@ -108,6 +108,7 @@ export const SettingsNavigationDrawerItems = () => {
           />
           {accountSubSettings.map((navigationItem, index) => (
             <SettingsNavigationDrawerItem
+              key={index}
               label={navigationItem.label}
               path={navigationItem.path}
               Icon={navigationItem.Icon}

packages/twenty-front/src/modules/settings/serverless-functions/graphql/queries/findManyAvailablePackages.ts

@@ -0,0 +1,7 @@
+import { gql } from '@apollo/client';
+
+export const FIND_MANY_AVAILABLE_PACKAGES = gql`
+  query FindManyAvailablePackages {
+    getAvailablePackages
+  }
+`;

packages/twenty-front/src/modules/settings/serverless-functions/hooks/useGetAvailablePackages.ts

@@ -0,0 +1,20 @@
+import { useApolloMetadataClient } from '@/object-metadata/hooks/useApolloMetadataClient';
+import { useQuery } from '@apollo/client';
+import { FIND_MANY_AVAILABLE_PACKAGES } from '@/settings/serverless-functions/graphql/queries/findManyAvailablePackages';
+import {
+  FindManyAvailablePackagesQuery,
+  FindManyAvailablePackagesQueryVariables,
+} from '~/generated-metadata/graphql';
+
+export const useGetAvailablePackages = () => {
+  const apolloMetadataClient = useApolloMetadataClient();
+  const { data } = useQuery<
+    FindManyAvailablePackagesQuery,
+    FindManyAvailablePackagesQueryVariables
+  >(FIND_MANY_AVAILABLE_PACKAGES, {
+    client: apolloMetadataClient ?? undefined,
+  });
+  return {
+    availablePackages: data?.getAvailablePackages || null,
+  };
+};