From 31aaefad71d5c5c1986f6161e2849a47600cba58 Mon Sep 17 00:00:00 2001 From: AmitChauhan63390 Date: Sat, 11 Jan 2025 19:00:35 +0530 Subject: [PATCH 1/2] schema for airtable added --- server/src/models/Robot.ts | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 3b2717d64..307da7685 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -25,6 +25,10 @@ interface RobotAttributes { google_sheet_id?: string | null; google_access_token?: string | null; google_refresh_token?: string | null; + airtable_base_id?: string | null; // Airtable Base ID + airtable_table_name?: string | null; // Airtable Table Name + airtable_api_key?: string | null; // Airtable API Key + airtable_access_token?: string | null; // Airtable OAuth Access Token schedule?: ScheduleConfig | null; } @@ -41,7 +45,7 @@ interface ScheduleConfig { cronExpression?: string; } -interface RobotCreationAttributes extends Optional { } +interface RobotCreationAttributes extends Optional {} class Robot extends Model implements RobotAttributes { public id!: string; @@ -53,6 +57,10 @@ class Robot extends Model implements R public google_sheet_id?: string | null; public google_access_token!: string | null; public google_refresh_token!: string | null; + public airtable_base_id!: string | null; + public airtable_table_name!: string | null; + public airtable_api_key!: string | null; + public airtable_access_token!: string | null; public schedule!: ScheduleConfig | null; } @@ -95,6 +103,22 @@ Robot.init( type: DataTypes.STRING, allowNull: true, }, + airtable_base_id: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_table_name: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_api_key: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_access_token: { + type: DataTypes.STRING, + allowNull: true, + }, schedule: { type: DataTypes.JSONB, allowNull: true, @@ -107,9 +131,10 @@ Robot.init( } ); +// Uncomment and define relationships if needed // Robot.hasMany(Run, { // foreignKey: 'robotId', // as: 'runs', // Alias for the relation // }); -export default Robot; \ No newline at end of file +export default Robot; From 722b97e13003a551b2deadaa34cc97a82114fc7e Mon Sep 17 00:00:00 2001 From: AmitChauhan63390 Date: Thu, 23 Jan 2025 18:53:55 +0530 Subject: [PATCH 2/2] great progress --- config/config.json | 11 ++ ...0111140925-add-airtable-fields-to-robot.js | 35 +++++ package.json | 1 + server/src/models/Robot.ts | 20 +-- .../integrations/airtableintegration.ts | 139 ++++++++++++++++++ .../workflow-management/scheduler/index.ts | 14 +- .../integration/IntegrationSettings.tsx | 126 ++++++++++++++-- 7 files changed, 322 insertions(+), 24 deletions(-) create mode 100644 config/config.json create mode 100644 migrations/20250111140925-add-airtable-fields-to-robot.js create mode 100644 server/src/workflow-management/integrations/airtableintegration.ts diff --git a/config/config.json b/config/config.json new file mode 100644 index 000000000..9fe88bf0b --- /dev/null +++ b/config/config.json @@ -0,0 +1,11 @@ +{ + "development": { + "username": "postgres", + "password": "postgres", + "database": "maxun", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + } + } + \ No newline at end of file diff --git a/migrations/20250111140925-add-airtable-fields-to-robot.js b/migrations/20250111140925-add-airtable-fields-to-robot.js new file mode 100644 index 000000000..abbfbdc18 --- /dev/null +++ b/migrations/20250111140925-add-airtable-fields-to-robot.js @@ -0,0 +1,35 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // Add new Airtable-related columns to the 'robot' table + await queryInterface.addColumn('robot', 'airtable_base_id', { + type: Sequelize.STRING, + allowNull: true, + }); + + await queryInterface.addColumn('robot', 'airtable_table_name', { + type: Sequelize.STRING, + allowNull: true, + }); + + await queryInterface.addColumn('robot', 'airtable_api_key', { + type: Sequelize.STRING, + allowNull: true, + }); + + await queryInterface.addColumn('robot', 'airtable_access_token', { + type: Sequelize.STRING, + allowNull: true, + }); + }, + + async down(queryInterface, Sequelize) { + // Remove Airtable-related columns from the 'robot' table + await queryInterface.removeColumn('robot', 'airtable_base_id'); + await queryInterface.removeColumn('robot', 'airtable_table_name'); + await queryInterface.removeColumn('robot', 'airtable_api_key'); + await queryInterface.removeColumn('robot', 'airtable_access_token'); + }, +}; diff --git a/package.json b/package.json index 5dae78a29..eed8264b1 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@types/react": "^18.0.5", "@types/react-dom": "^18.0.1", "@types/uuid": "^8.3.4", + "airtable": "^0.12.2", "axios": "^0.26.0", "bcrypt": "^5.1.1", "body-parser": "^1.20.3", diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 307da7685..b731d4a76 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -27,8 +27,8 @@ interface RobotAttributes { google_refresh_token?: string | null; airtable_base_id?: string | null; // Airtable Base ID airtable_table_name?: string | null; // Airtable Table Name - airtable_api_key?: string | null; // Airtable API Key - airtable_access_token?: string | null; // Airtable OAuth Access Token + airtable_personal_access_token?: string | null; // Airtable Personal Access Token + airtable_access_token?: string | null; // Airtable OAuth Access Token (if using OAuth) schedule?: ScheduleConfig | null; } @@ -53,14 +53,14 @@ class Robot extends Model implements R public recording_meta!: RobotMeta; public recording!: RobotWorkflow; public google_sheet_email!: string | null; - public google_sheet_name?: string | null; - public google_sheet_id?: string | null; + public google_sheet_name!: string | null; + public google_sheet_id!: string | null; public google_access_token!: string | null; public google_refresh_token!: string | null; - public airtable_base_id!: string | null; - public airtable_table_name!: string | null; - public airtable_api_key!: string | null; - public airtable_access_token!: string | null; + public airtable_base_id!: string | null; // Airtable Base ID + public airtable_table_name!: string | null; // Airtable Table Name + public airtable_personal_access_token!: string | null; // Airtable Personal Access Token + public airtable_access_token!: string | null; // Airtable OAuth Access Token public schedule!: ScheduleConfig | null; } @@ -111,7 +111,7 @@ Robot.init( type: DataTypes.STRING, allowNull: true, }, - airtable_api_key: { + airtable_personal_access_token: { type: DataTypes.STRING, allowNull: true, }, @@ -137,4 +137,4 @@ Robot.init( // as: 'runs', // Alias for the relation // }); -export default Robot; +export default Robot; \ No newline at end of file diff --git a/server/src/workflow-management/integrations/airtableintegration.ts b/server/src/workflow-management/integrations/airtableintegration.ts new file mode 100644 index 000000000..378ed6799 --- /dev/null +++ b/server/src/workflow-management/integrations/airtableintegration.ts @@ -0,0 +1,139 @@ +import Airtable from 'airtable'; +import logger from '../../logger'; +import Run from '../../models/Run'; +import Robot from '../../models/Robot'; + +interface AirtableUpdateTask { + robotId: string; + runId: string; + status: 'pending' | 'completed' | 'failed'; + retries: number; +} + +const MAX_RETRIES = 5; + +export let airtableUpdateTasks: { [runId: string]: AirtableUpdateTask } = {}; + +/** + * Updates Airtable with data from a successful run. + * @param robotId - The ID of the robot. + * @param runId - The ID of the run. + */ +export async function updateAirtable(robotId: string, runId: string) { + try { + const run = await Run.findOne({ where: { runId } }); + + if (!run) { + throw new Error(`Run not found for runId: ${runId}`); + } + + const plainRun = run.toJSON(); + + if (plainRun.status === 'success') { + let data: { [key: string]: any }[] = []; + if (plainRun.serializableOutput && Object.keys(plainRun.serializableOutput).length > 0) { + data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[]; + } else if (plainRun.binaryOutput && plainRun.binaryOutput['item-0']) { + const binaryUrl = plainRun.binaryOutput['item-0'] as string; + data = [{ "Screenshot URL": binaryUrl }]; + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + throw new Error(`Robot not found for robotId: ${robotId}`); + } + + const plainRobot = robot.toJSON(); + + const tableName = plainRobot.airtable_table_name; + const baseId = plainRobot.airtable_base_id; + const personalAccessToken = plainRobot.airtable_personal_access_token; + + if (tableName && baseId && personalAccessToken) { + console.log(`Preparing to write data to Airtable for robot: ${robotId}, table: ${tableName}`); + + await writeDataToAirtable(baseId, tableName, personalAccessToken, data); + console.log(`Data written to Airtable successfully for Robot: ${robotId} and Run: ${runId}`); + } else { + console.log('Airtable integration not configured.'); + } + } else { + console.log('Run status is not success or serializableOutput is missing.'); + } + } catch (error: any) { + console.error(`Failed to write data to Airtable for Robot: ${robotId} and Run: ${runId}: ${error.message}`); + } +} + +/** + * Writes data to Airtable. + * @param baseId - The ID of the Airtable base. + * @param tableName - The name of the Airtable table. + * @param personalAccessToken - The Airtable Personal Access Token. + * @param data - The data to write to Airtable. + */ +export async function writeDataToAirtable(baseId: string, tableName: string, personalAccessToken: string, data: any[]) { + try { + // Initialize Airtable with Personal Access Token + const base = new Airtable({ apiKey: personalAccessToken }).base(baseId); + + const table = base(tableName); + + // Prepare records for Airtable + const records = data.map((row) => ({ fields: row })); + + // Write data to Airtable + const response = await table.create(records); + + if (response) { + console.log('Data successfully appended to Airtable.'); + } else { + console.error('Airtable append failed:', response); + } + + logger.log(`info`, `Data written to Airtable: ${tableName}`); + } catch (error: any) { + logger.log(`error`, `Error writing data to Airtable: ${error.message}`); + throw error; + } +} + +/** + * Processes pending Airtable update tasks. + */ +export const processAirtableUpdates = async () => { + while (true) { + let hasPendingTasks = false; + for (const runId in airtableUpdateTasks) { + const task = airtableUpdateTasks[runId]; + console.log(`Processing task for runId: ${runId}, status: ${task.status}`); + + if (task.status === 'pending') { + hasPendingTasks = true; + try { + await updateAirtable(task.robotId, task.runId); + console.log(`Successfully updated Airtable for runId: ${runId}`); + delete airtableUpdateTasks[runId]; + } catch (error: any) { + console.error(`Failed to update Airtable for run ${task.runId}:`, error); + if (task.retries < MAX_RETRIES) { + airtableUpdateTasks[runId].retries += 1; + console.log(`Retrying task for runId: ${runId}, attempt: ${task.retries}`); + } else { + airtableUpdateTasks[runId].status = 'failed'; + console.log(`Max retries reached for runId: ${runId}. Marking task as failed.`); + } + } + } + } + + if (!hasPendingTasks) { + console.log('No pending tasks. Exiting loop.'); + break; + } + + console.log('Waiting for 5 seconds before checking again...'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } +}; \ No newline at end of file diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index ade7d9699..4089caf4a 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -6,6 +6,7 @@ import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-m import logger from '../../logger'; import { browserPool } from "../../server"; import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integrations/gsheet"; +import { airtableUpdateTasks, processAirtableUpdates } from "../integrations/airtableintegration"; // Import Airtable functions import Robot from "../../models/Robot"; import Run from "../../models/Run"; import { getDecryptedProxyConfig } from "../../routes/proxy"; @@ -44,7 +45,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { }; } - const browserId = createRemoteBrowserForRun( userId); + const browserId = createRemoteBrowserForRun(userId); const runId = uuid(); const run = await Run.create({ @@ -177,6 +178,7 @@ async function executeRun(id: string) { } ); + // Add task for Google Sheets update googleSheetUpdateTasks[id] = { robotId: plainRun.robotMetaId, runId: id, @@ -184,6 +186,16 @@ async function executeRun(id: string) { retries: 5, }; processGoogleSheetUpdates(); + + // Add task for Airtable update + airtableUpdateTasks[id] = { + robotId: plainRun.robotMetaId, + runId: id, + status: 'pending', + retries: 5, + }; + processAirtableUpdates(); + return true; } catch (error: any) { logger.log('info', `Error while running a robot with id: ${id} - ${error.message}`); diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index f9c397aed..6583ca5c0 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -17,7 +17,6 @@ import { apiUrl } from "../../apiConfig.js"; import Cookies from 'js-cookie'; import { useTranslation } from "react-i18next"; - interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; @@ -27,6 +26,8 @@ interface IntegrationProps { export interface IntegrationSettings { spreadsheetId: string; spreadsheetName: string; + airtableBaseId: string; // New field for Airtable Base ID + airtableTableName: string; // New field for Airtable Table Name data: string; } @@ -53,22 +54,28 @@ export const IntegrationSettingsModal = ({ const [settings, setSettings] = useState({ spreadsheetId: "", spreadsheetName: "", + airtableBaseId: "", // Initialize Airtable fields + airtableTableName: "", data: "", }); - const [spreadsheets, setSpreadsheets] = useState< - { id: string; name: string }[] - >([]); + const [spreadsheets, setSpreadsheets] = useState<{ id: string; name: string }[]>([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const { recordingId, notify } = useGlobalInfoStore(); const [recording, setRecording] = useState(null); + // Google Sheets Authentication const authenticateWithGoogle = () => { window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; }; + // Airtable Authentication + const authenticateWithAirtable = () => { + window.location.href = `${apiUrl}/auth/airtable?robotId=${recordingId}`; + }; + const handleOAuthCallback = async () => { try { const response = await axios.get(`${apiUrl}/auth/google/callback`); @@ -82,9 +89,7 @@ export const IntegrationSettingsModal = ({ try { const response = await axios.get( `${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, - { - withCredentials: true, - } + { withCredentials: true } ); setSpreadsheets(response.data); } catch (error: any) { @@ -135,20 +140,47 @@ export const IntegrationSettingsModal = ({ } }; - const removeIntegration = async () => { + const updateAirtableConfig = async () => { + try { + const response = await axios.post( + `${apiUrl}/auth/airtable/update`, + { + airtableBaseId: settings.airtableBaseId, + airtableTableName: settings.airtableTableName, + robotId: recordingId, + }, + { withCredentials: true } + ); + notify(`success`, t('integration_settings.notifications.airtable_updated')); + console.log("Airtable configuration updated:", response.data); + } catch (error: any) { + console.error( + "Error updating Airtable configuration:", + error.response?.data?.message || error.message + ); + } + }; + + const removeIntegration = async (integrationType: 'google' | 'airtable') => { try { await axios.post( - `${apiUrl}/auth/gsheets/remove`, + `${apiUrl}/auth/${integrationType}/remove`, { robotId: recordingId }, { withCredentials: true } ); setRecording(null); setSpreadsheets([]); - setSettings({ spreadsheetId: "", spreadsheetName: "", data: "" }); + setSettings({ + spreadsheetId: "", + spreadsheetName: "", + airtableBaseId: "", + airtableTableName: "", + data: "", + }); } catch (error: any) { console.error( - "Error removing Google Sheets integration:", + `Error removing ${integrationType} integration:`, error.response?.data?.message || error.message ); } @@ -196,6 +228,7 @@ export const IntegrationSettingsModal = ({ {t('integration_settings.title')} + {/* Google Sheets Integration */} {recording && recording.google_sheet_id ? ( <> @@ -212,7 +245,7 @@ export const IntegrationSettingsModal = ({ @@ -309,6 +342,73 @@ export const IntegrationSettingsModal = ({ )} )} + + {/* Airtable Integration */} + + {t('integration_settings.airtable.title')} + + + {recording && recording.airtable_base_id ? ( + <> + + {t('integration_settings.airtable.alerts.success.title')} + {t('integration_settings.airtable.alerts.success.content', { + baseId: recording.airtable_base_id, + tableName: recording.airtable_table_name, + })} + + + + ) : ( + <> +

{t('integration_settings.airtable.descriptions.sync_info')}

+ + + {recording?.airtable_access_token && ( + <> + setSettings({ ...settings, airtableBaseId: e.target.value })} + fullWidth + /> + setSettings({ ...settings, airtableTableName: e.target.value })} + fullWidth + /> + + + )} + + )} );