diff --git a/apps/docs/content/docs/en/tools/airtable.mdx b/apps/docs/content/docs/en/tools/airtable.mdx index 0cabaca3a0..b0cc7d38e3 100644 --- a/apps/docs/content/docs/en/tools/airtable.mdx +++ b/apps/docs/content/docs/en/tools/airtable.mdx @@ -57,7 +57,7 @@ In Sim, the Airtable integration enables your agents to interact with your Airta ## Usage Instructions -Integrates Airtable into the workflow. Can create, get, list, or update Airtable records. Requires OAuth. Can be used in trigger mode to trigger a workflow when an update is made to an Airtable table. +Integrates Airtable into the workflow. Can create, get, list, or update Airtable records. Can be used in trigger mode to trigger a workflow when an update is made to an Airtable table. diff --git a/apps/docs/content/docs/en/tools/browser_use.mdx b/apps/docs/content/docs/en/tools/browser_use.mdx index 7795433f9c..70f10b0df3 100644 --- a/apps/docs/content/docs/en/tools/browser_use.mdx +++ b/apps/docs/content/docs/en/tools/browser_use.mdx @@ -57,7 +57,7 @@ In Sim, the BrowserUse integration allows your agents to interact with the web a ## Usage Instructions -Integrate Browser Use into the workflow. Can navigate the web and perform actions as if a real user was interacting with the browser. Requires API Key. +Integrate Browser Use into the workflow. Can navigate the web and perform actions as if a real user was interacting with the browser. diff --git a/apps/docs/content/docs/en/tools/clay.mdx b/apps/docs/content/docs/en/tools/clay.mdx index fa06fa911c..2b9510d20a 100644 --- a/apps/docs/content/docs/en/tools/clay.mdx +++ b/apps/docs/content/docs/en/tools/clay.mdx @@ -198,7 +198,7 @@ In Sim, the Clay integration allows your agents to push structured data into Cla ## Usage Instructions -Integrate Clay into the workflow. Can populate a table with data. Requires an API Key. +Integrate Clay into the workflow. Can populate a table with data. diff --git a/apps/docs/content/docs/en/tools/confluence.mdx b/apps/docs/content/docs/en/tools/confluence.mdx index 6a1c40949c..e7286f9193 100644 --- a/apps/docs/content/docs/en/tools/confluence.mdx +++ b/apps/docs/content/docs/en/tools/confluence.mdx @@ -43,7 +43,7 @@ In Sim, the Confluence integration enables your agents to access and leverage yo ## Usage Instructions -Integrate Confluence into the workflow. Can read and update a page. Requires OAuth. +Integrate Confluence into the workflow. Can read and update a page. diff --git a/apps/docs/content/docs/en/tools/discord.mdx b/apps/docs/content/docs/en/tools/discord.mdx index bb3e768241..b65aae4337 100644 --- a/apps/docs/content/docs/en/tools/discord.mdx +++ b/apps/docs/content/docs/en/tools/discord.mdx @@ -57,7 +57,7 @@ Discord components in Sim use efficient lazy loading, only fetching data when ne ## Usage Instructions -Integrate Discord into the workflow. Can send and get messages, get server information, and get a user’s information. Requires bot API key. +Integrate Discord into the workflow. Can send and get messages, get server information, and get a user’s information. diff --git a/apps/docs/content/docs/en/tools/elevenlabs.mdx b/apps/docs/content/docs/en/tools/elevenlabs.mdx index bb109e602a..1431427363 100644 --- a/apps/docs/content/docs/en/tools/elevenlabs.mdx +++ b/apps/docs/content/docs/en/tools/elevenlabs.mdx @@ -39,7 +39,7 @@ In Sim, the ElevenLabs integration enables your agents to convert text to lifeli ## Usage Instructions -Integrate ElevenLabs into the workflow. Can convert text to speech. Requires API key. +Integrate ElevenLabs into the workflow. Can convert text to speech. diff --git a/apps/docs/content/docs/en/tools/exa.mdx b/apps/docs/content/docs/en/tools/exa.mdx index 8517146972..7727f5645e 100644 --- a/apps/docs/content/docs/en/tools/exa.mdx +++ b/apps/docs/content/docs/en/tools/exa.mdx @@ -44,7 +44,7 @@ In Sim, the Exa integration allows your agents to search the web for information ## Usage Instructions -Integrate Exa into the workflow. Can search, get contents, find similar links, answer a question, and perform research. Requires API Key. +Integrate Exa into the workflow. Can search, get contents, find similar links, answer a question, and perform research. diff --git a/apps/docs/content/docs/en/tools/firecrawl.mdx b/apps/docs/content/docs/en/tools/firecrawl.mdx index f135cf18e8..4aa221f9fb 100644 --- a/apps/docs/content/docs/en/tools/firecrawl.mdx +++ b/apps/docs/content/docs/en/tools/firecrawl.mdx @@ -59,7 +59,7 @@ This allows your agents to gather information from websites, extract structured ## Usage Instructions -Integrate Firecrawl into the workflow. Can search, scrape, or crawl websites. Requires API Key. +Integrate Firecrawl into the workflow. Can search, scrape, or crawl websites. diff --git a/apps/docs/content/docs/en/tools/github.mdx b/apps/docs/content/docs/en/tools/github.mdx index 9f5fcf1513..f63b381472 100644 --- a/apps/docs/content/docs/en/tools/github.mdx +++ b/apps/docs/content/docs/en/tools/github.mdx @@ -35,7 +35,7 @@ In Sim, the GitHub integration enables your agents to interact directly with Git ## Usage Instructions -Integrate Github into the workflow. Can get get PR details, create PR comment, get repository info, and get latest commit. Requires github token API Key. Can be used in trigger mode to trigger a workflow when a PR is created, commented on, or a commit is pushed. +Integrate Github into the workflow. Can get get PR details, create PR comment, get repository info, and get latest commit. Can be used in trigger mode to trigger a workflow when a PR is created, commented on, or a commit is pushed. diff --git a/apps/docs/content/docs/en/tools/gmail.mdx b/apps/docs/content/docs/en/tools/gmail.mdx index e0913ad02b..2d71786c2d 100644 --- a/apps/docs/content/docs/en/tools/gmail.mdx +++ b/apps/docs/content/docs/en/tools/gmail.mdx @@ -51,7 +51,7 @@ In Sim, the Gmail integration enables your agents to send, read, and search emai ## Usage Instructions -Integrate Gmail into the workflow. Can send, read, and search emails. Requires OAuth. Can be used in trigger mode to trigger a workflow when a new email is received. +Integrate Gmail into the workflow. Can send, read, and search emails. Can be used in trigger mode to trigger a workflow when a new email is received. diff --git a/apps/docs/content/docs/en/tools/google_calendar.mdx b/apps/docs/content/docs/en/tools/google_calendar.mdx index 4a7eb3dd78..8e8228824a 100644 --- a/apps/docs/content/docs/en/tools/google_calendar.mdx +++ b/apps/docs/content/docs/en/tools/google_calendar.mdx @@ -90,7 +90,7 @@ In Sim, the Google Calendar integration enables your agents to programmatically ## Usage Instructions -Integrate Google Calendar into the workflow. Can create, read, update, and list calendar events. Requires OAuth. +Integrate Google Calendar into the workflow. Can create, read, update, and list calendar events. diff --git a/apps/docs/content/docs/en/tools/google_docs.mdx b/apps/docs/content/docs/en/tools/google_docs.mdx index 5bdf21d89b..df0e394de0 100644 --- a/apps/docs/content/docs/en/tools/google_docs.mdx +++ b/apps/docs/content/docs/en/tools/google_docs.mdx @@ -81,7 +81,7 @@ In Sim, the Google Docs integration enables your agents to interact directly wit ## Usage Instructions -Integrate Google Docs into the workflow. Can read, write, and create documents. Requires OAuth. +Integrate Google Docs into the workflow. Can read, write, and create documents. diff --git a/apps/docs/content/docs/en/tools/google_drive.mdx b/apps/docs/content/docs/en/tools/google_drive.mdx index 164dd67adc..f2d7f2ee64 100644 --- a/apps/docs/content/docs/en/tools/google_drive.mdx +++ b/apps/docs/content/docs/en/tools/google_drive.mdx @@ -73,7 +73,7 @@ In Sim, the Google Drive integration enables your agents to interact directly wi ## Usage Instructions -Integrate Google Drive into the workflow. Can create, upload, and list files. Requires OAuth. +Integrate Google Drive into the workflow. Can create, upload, and list files. diff --git a/apps/docs/content/docs/en/tools/google_forms.mdx b/apps/docs/content/docs/en/tools/google_forms.mdx index 02ba3c73f4..59510b4ed6 100644 --- a/apps/docs/content/docs/en/tools/google_forms.mdx +++ b/apps/docs/content/docs/en/tools/google_forms.mdx @@ -69,9 +69,6 @@ Integrate Google Forms into your workflow. Provide a Form ID to list responses, | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| formId | string | Yes | The ID of the Google Form | -| responseId | string | No | If provided, returns this specific response | -| pageSize | number | No | Max responses to return (service may return fewer). Defaults to 5000 | #### Output diff --git a/apps/docs/content/docs/en/tools/google_search.mdx b/apps/docs/content/docs/en/tools/google_search.mdx index 260f3da8f5..69c13b6e86 100644 --- a/apps/docs/content/docs/en/tools/google_search.mdx +++ b/apps/docs/content/docs/en/tools/google_search.mdx @@ -58,7 +58,7 @@ In Sim, the Google Search integration enables your agents to search the web prog ## Usage Instructions -Integrate Google Search into the workflow. Can search the web. Requires API Key. +Integrate Google Search into the workflow. Can search the web. diff --git a/apps/docs/content/docs/en/tools/google_sheets.mdx b/apps/docs/content/docs/en/tools/google_sheets.mdx index 5e4688e0a9..85bac385e0 100644 --- a/apps/docs/content/docs/en/tools/google_sheets.mdx +++ b/apps/docs/content/docs/en/tools/google_sheets.mdx @@ -96,7 +96,7 @@ In Sim, the Google Sheets integration enables your agents to interact directly w ## Usage Instructions -Integrate Google Sheets into the workflow. Can read, write, append, and update data. Requires OAuth. +Integrate Google Sheets into the workflow. Can read, write, append, and update data. diff --git a/apps/docs/content/docs/en/tools/huggingface.mdx b/apps/docs/content/docs/en/tools/huggingface.mdx index d40703259c..0c37a78eab 100644 --- a/apps/docs/content/docs/en/tools/huggingface.mdx +++ b/apps/docs/content/docs/en/tools/huggingface.mdx @@ -66,7 +66,7 @@ In Sim, the HuggingFace integration enables your agents to programmatically gene ## Usage Instructions -Integrate Hugging Face into the workflow. Can generate completions using the Hugging Face Inference API. Requires API Key. +Integrate Hugging Face into the workflow. Can generate completions using the Hugging Face Inference API. diff --git a/apps/docs/content/docs/en/tools/hunter.mdx b/apps/docs/content/docs/en/tools/hunter.mdx index 95c2f8b0d3..c67f575c64 100644 --- a/apps/docs/content/docs/en/tools/hunter.mdx +++ b/apps/docs/content/docs/en/tools/hunter.mdx @@ -41,7 +41,7 @@ In Sim, the Hunter.io integration enables your agents to programmatically search ## Usage Instructions -Integrate Hunter into the workflow. Can search domains, find email addresses, verify email addresses, discover companies, find companies, and count email addresses. Requires API Key. +Integrate Hunter into the workflow. Can search domains, find email addresses, verify email addresses, discover companies, find companies, and count email addresses. diff --git a/apps/docs/content/docs/en/tools/image_generator.mdx b/apps/docs/content/docs/en/tools/image_generator.mdx index ebda407134..d559b0f615 100644 --- a/apps/docs/content/docs/en/tools/image_generator.mdx +++ b/apps/docs/content/docs/en/tools/image_generator.mdx @@ -46,7 +46,7 @@ In Sim, the DALL-E integration enables your agents to generate images programmat ## Usage Instructions -Integrate Image Generator into the workflow. Can generate images using DALL-E 3 or GPT Image. Requires API Key. +Integrate Image Generator into the workflow. Can generate images using DALL-E 3 or GPT Image. diff --git a/apps/docs/content/docs/en/tools/jina.mdx b/apps/docs/content/docs/en/tools/jina.mdx index 7d81fbd37d..665f2ebe2a 100644 --- a/apps/docs/content/docs/en/tools/jina.mdx +++ b/apps/docs/content/docs/en/tools/jina.mdx @@ -63,7 +63,7 @@ This integration is particularly valuable for building agents that need to gathe ## Usage Instructions -Integrate Jina into the workflow. Extracts content from websites. Requires API Key. +Integrate Jina into the workflow. Extracts content from websites. diff --git a/apps/docs/content/docs/en/tools/jira.mdx b/apps/docs/content/docs/en/tools/jira.mdx index 7985784372..5a01a1cf84 100644 --- a/apps/docs/content/docs/en/tools/jira.mdx +++ b/apps/docs/content/docs/en/tools/jira.mdx @@ -43,7 +43,7 @@ In Sim, the Jira integration allows your agents to seamlessly interact with your ## Usage Instructions -Integrate Jira into the workflow. Can read, write, and update issues. Requires OAuth. +Integrate Jira into the workflow. Can read, write, and update issues. diff --git a/apps/docs/content/docs/en/tools/linear.mdx b/apps/docs/content/docs/en/tools/linear.mdx index 8db1841190..be2fd68940 100644 --- a/apps/docs/content/docs/en/tools/linear.mdx +++ b/apps/docs/content/docs/en/tools/linear.mdx @@ -10,8 +10,11 @@ import { BlockInfoCard } from "@/components/ui/block-info-card" color="#5E6AD2" icon={true} iconSvg={` ({ - cookies: () => ({ - get: vi.fn().mockReturnValue({ value: 'test-session-token' }), - }), - headers: () => ({ - get: vi.fn().mockReturnValue('test-value'), - }), -})) - -vi.mock('@/lib/auth/session', () => ({ - getSession: vi.fn().mockResolvedValue({ - user: { - id: 'user-id', - email: 'test@example.com', - }, - sessionToken: 'test-session-token', - }), -})) - -beforeEach(() => { - vi.clearAllMocks() -}) - -afterEach(() => { - vi.restoreAllMocks() -}) diff --git a/apps/sim/app/api/__test-utils__/utils.ts b/apps/sim/app/api/__test-utils__/utils.ts index da96feedd8..e7a91b2020 100644 --- a/apps/sim/app/api/__test-utils__/utils.ts +++ b/apps/sim/app/api/__test-utils__/utils.ts @@ -1364,24 +1364,6 @@ export function setupKnowledgeApiMocks( } } -// Legacy functions for backward compatibility (DO NOT REMOVE - still used in tests) - -/** - * @deprecated Use mockAuth instead - provides same functionality with improved interface - */ -export function mockAuthSession(isAuthenticated = true, user: MockUser = mockUser) { - const authMocks = mockAuth(user) - if (isAuthenticated) { - authMocks.setAuthenticated(user) - } else { - authMocks.setUnauthenticated() - } - return authMocks -} - -/** - * @deprecated Use setupComprehensiveTestMocks instead - provides better organization and features - */ export function setupApiTestMocks( options: { authenticated?: boolean @@ -1412,9 +1394,6 @@ export function setupApiTestMocks( }) } -/** - * @deprecated Use createStorageProviderMocks instead - */ export function mockUploadUtils( options: { isCloudStorage?: boolean; uploadResult?: any; uploadError?: boolean } = {} ) { @@ -1452,10 +1431,6 @@ export function mockUploadUtils( })) } -/** - * Create a mock transaction function for database testing - * @deprecated Use createMockDatabase instead - */ export function createMockTransaction( mockData: { selectData?: DatabaseSelectResult[] diff --git a/apps/sim/app/api/auth/oauth/microsoft/file/route.ts b/apps/sim/app/api/auth/oauth/microsoft/file/route.ts index da3e30314d..b1f7411dac 100644 --- a/apps/sim/app/api/auth/oauth/microsoft/file/route.ts +++ b/apps/sim/app/api/auth/oauth/microsoft/file/route.ts @@ -4,6 +4,7 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateMicrosoftGraphId } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' @@ -11,21 +12,15 @@ export const dynamic = 'force-dynamic' const logger = createLogger('MicrosoftFileAPI') -/** - * Get a single file from Microsoft OneDrive - */ export async function GET(request: NextRequest) { const requestId = generateRequestId() try { - // Get the session const session = await getSession() - // Check if the user is authenticated if (!session?.user?.id) { return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) } - // Get the credential ID and file ID from the query params const { searchParams } = new URL(request.url) const credentialId = searchParams.get('credentialId') const fileId = searchParams.get('fileId') @@ -34,7 +29,12 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and File ID are required' }, { status: 400 }) } - // Get the credential from the database + const fileIdValidation = validateMicrosoftGraphId(fileId, 'fileId') + if (!fileIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid file ID: ${fileIdValidation.error}`) + return NextResponse.json({ error: fileIdValidation.error }, { status: 400 }) + } + const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { @@ -43,12 +43,10 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Check if the credential belongs to the user if (credential.userId !== session.user.id) { return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) } - // Refresh access token if needed using the utility function const accessToken = await refreshAccessTokenIfNeeded(credentialId, session.user.id, requestId) if (!accessToken) { @@ -80,7 +78,6 @@ export async function GET(request: NextRequest) { const file = await response.json() - // Transform the response to match expected format const transformedFile = { id: file.id, name: file.name, diff --git a/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts b/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts index 5a8d9f0845..8cead83d0f 100644 --- a/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts +++ b/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts @@ -4,6 +4,7 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateEnum, validatePathSegment } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' @@ -38,16 +39,24 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and Item ID are required' }, { status: 400 }) } - // Validate item type - only handle contacts now - if (type !== 'contact') { - logger.warn(`[${requestId}] Invalid item type: ${type}`) - return NextResponse.json( - { error: 'Invalid item type. Only contact is supported.' }, - { status: 400 } - ) + const typeValidation = validateEnum(type, ['contact'] as const, 'type') + if (!typeValidation.isValid) { + logger.warn(`[${requestId}] Invalid item type: ${typeValidation.error}`) + return NextResponse.json({ error: typeValidation.error }, { status: 400 }) + } + + const itemIdValidation = validatePathSegment(itemId, { + paramName: 'itemId', + maxLength: 100, + allowHyphens: true, + allowUnderscores: true, + allowDots: false, + }) + if (!itemIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid item ID: ${itemIdValidation.error}`) + return NextResponse.json({ error: itemIdValidation.error }, { status: 400 }) } - // Get the credential from the database const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { @@ -57,7 +66,6 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Check if the credential belongs to the user if (credential.userId !== session.user.id) { logger.warn(`[${requestId}] Unauthorized credential access attempt`, { credentialUserId: credential.userId, @@ -66,7 +74,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) } - // Refresh access token if needed const accessToken = await refreshAccessTokenIfNeeded(credentialId, session.user.id, requestId) if (!accessToken) { @@ -74,7 +81,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Determine the endpoint based on item type - only contacts const endpoints = { contact: 'contacts', } @@ -82,7 +88,6 @@ export async function GET(request: NextRequest) { logger.info(`[${requestId}] Fetching ${type} ${itemId} from Wealthbox`) - // Make request to Wealthbox API const response = await fetch(`https://api.crmworkspace.com/v1/${endpoint}/${itemId}`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -120,13 +125,10 @@ export async function GET(request: NextRequest) { totalCount: data.meta?.total_count, }) - // Transform the response to match our expected format let items: any[] = [] if (type === 'contact') { - // Handle single contact response - API returns contact data directly when fetching by ID if (data?.id) { - // Single contact response const item = { id: data.id?.toString() || '', name: `${data.first_name || ''} ${data.last_name || ''}`.trim() || `Contact ${data.id}`, diff --git a/apps/sim/app/api/copilot/checkpoints/revert/route.test.ts b/apps/sim/app/api/copilot/checkpoints/revert/route.test.ts index d1fdb6d526..1ec2ef5399 100644 --- a/apps/sim/app/api/copilot/checkpoints/revert/route.test.ts +++ b/apps/sim/app/api/copilot/checkpoints/revert/route.test.ts @@ -168,7 +168,7 @@ describe('Copilot Checkpoints Revert API Route', () => { // Mock checkpoint found but workflow not found const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'a1b2c3d4-e5f6-4a78-b9c0-d1e2f3a4b5c6', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } @@ -196,13 +196,13 @@ describe('Copilot Checkpoints Revert API Route', () => { // Mock checkpoint found but workflow belongs to different user const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'b2c3d4e5-f6a7-4b89-a0d1-e2f3a4b5c6d7', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } const mockWorkflow = { - id: 'workflow-456', + id: 'b2c3d4e5-f6a7-4b89-a0d1-e2f3a4b5c6d7', userId: 'different-user', } @@ -228,7 +228,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'c3d4e5f6-a7b8-4c09-a1e2-f3a4b5c6d7e8', userId: 'user-123', workflowState: { blocks: { block1: { type: 'start' } }, @@ -241,7 +241,7 @@ describe('Copilot Checkpoints Revert API Route', () => { } const mockWorkflow = { - id: 'workflow-456', + id: 'c3d4e5f6-a7b8-4c09-a1e2-f3a4b5c6d7e8', userId: 'user-123', } @@ -274,7 +274,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const responseData = await response.json() expect(responseData).toEqual({ success: true, - workflowId: 'workflow-456', + workflowId: 'c3d4e5f6-a7b8-4c09-a1e2-f3a4b5c6d7e8', checkpointId: 'checkpoint-123', revertedAt: '2024-01-01T00:00:00.000Z', checkpoint: { @@ -293,7 +293,7 @@ describe('Copilot Checkpoints Revert API Route', () => { // Verify fetch was called with correct parameters expect(global.fetch).toHaveBeenCalledWith( - 'http://localhost:3000/api/workflows/workflow-456/state', + 'http://localhost:3000/api/workflows/c3d4e5f6-a7b8-4c09-a1e2-f3a4b5c6d7e8/state', { method: 'PUT', headers: { @@ -319,7 +319,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-with-date', - workflowId: 'workflow-456', + workflowId: 'd4e5f6a7-b8c9-4d10-a2e3-a4b5c6d7e8f9', userId: 'user-123', workflowState: { blocks: {}, @@ -330,7 +330,7 @@ describe('Copilot Checkpoints Revert API Route', () => { } const mockWorkflow = { - id: 'workflow-456', + id: 'd4e5f6a7-b8c9-4d10-a2e3-a4b5c6d7e8f9', userId: 'user-123', } @@ -360,7 +360,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-invalid-date', - workflowId: 'workflow-456', + workflowId: 'e5f6a7b8-c9d0-4e11-a3f4-b5c6d7e8f9a0', userId: 'user-123', workflowState: { blocks: {}, @@ -371,7 +371,7 @@ describe('Copilot Checkpoints Revert API Route', () => { } const mockWorkflow = { - id: 'workflow-456', + id: 'e5f6a7b8-c9d0-4e11-a3f4-b5c6d7e8f9a0', userId: 'user-123', } @@ -401,7 +401,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-null-values', - workflowId: 'workflow-456', + workflowId: 'f6a7b8c9-d0e1-4f23-a4b5-c6d7e8f9a0b1', userId: 'user-123', workflowState: { blocks: null, @@ -413,7 +413,7 @@ describe('Copilot Checkpoints Revert API Route', () => { } const mockWorkflow = { - id: 'workflow-456', + id: 'f6a7b8c9-d0e1-4f23-a4b5-c6d7e8f9a0b1', userId: 'user-123', } @@ -452,13 +452,13 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'a7b8c9d0-e1f2-4a34-b5c6-d7e8f9a0b1c2', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } const mockWorkflow = { - id: 'workflow-456', + id: 'a7b8c9d0-e1f2-4a34-b5c6-d7e8f9a0b1c2', userId: 'user-123', } @@ -510,7 +510,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'b8c9d0e1-f2a3-4b45-a6d7-e8f9a0b1c2d3', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } @@ -537,13 +537,13 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'c9d0e1f2-a3b4-4c56-a7e8-f9a0b1c2d3e4', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } const mockWorkflow = { - id: 'workflow-456', + id: 'c9d0e1f2-a3b4-4c56-a7e8-f9a0b1c2d3e4', userId: 'user-123', } @@ -594,13 +594,13 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'd0e1f2a3-b4c5-4d67-a8f9-a0b1c2d3e4f5', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } const mockWorkflow = { - id: 'workflow-456', + id: 'd0e1f2a3-b4c5-4d67-a8f9-a0b1c2d3e4f5', userId: 'user-123', } @@ -626,7 +626,7 @@ describe('Copilot Checkpoints Revert API Route', () => { await POST(req) expect(global.fetch).toHaveBeenCalledWith( - 'http://localhost:3000/api/workflows/workflow-456/state', + 'http://localhost:3000/api/workflows/d0e1f2a3-b4c5-4d67-a8f9-a0b1c2d3e4f5/state', { method: 'PUT', headers: { @@ -644,13 +644,13 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-123', - workflowId: 'workflow-456', + workflowId: 'e1f2a3b4-c5d6-4e78-a9a0-b1c2d3e4f5a6', userId: 'user-123', workflowState: { blocks: {}, edges: [] }, } const mockWorkflow = { - id: 'workflow-456', + id: 'e1f2a3b4-c5d6-4e78-a9a0-b1c2d3e4f5a6', userId: 'user-123', } @@ -677,7 +677,7 @@ describe('Copilot Checkpoints Revert API Route', () => { expect(response.status).toBe(200) expect(global.fetch).toHaveBeenCalledWith( - 'http://localhost:3000/api/workflows/workflow-456/state', + 'http://localhost:3000/api/workflows/e1f2a3b4-c5d6-4e78-a9a0-b1c2d3e4f5a6/state', { method: 'PUT', headers: { @@ -695,7 +695,7 @@ describe('Copilot Checkpoints Revert API Route', () => { const mockCheckpoint = { id: 'checkpoint-complex', - workflowId: 'workflow-456', + workflowId: 'f2a3b4c5-d6e7-4f89-a0b1-c2d3e4f5a6b7', userId: 'user-123', workflowState: { blocks: { @@ -723,7 +723,7 @@ describe('Copilot Checkpoints Revert API Route', () => { } const mockWorkflow = { - id: 'workflow-456', + id: 'f2a3b4c5-d6e7-4f89-a0b1-c2d3e4f5a6b7', userId: 'user-123', } diff --git a/apps/sim/app/api/copilot/checkpoints/revert/route.ts b/apps/sim/app/api/copilot/checkpoints/revert/route.ts index c3e5f0fb70..16162a96a2 100644 --- a/apps/sim/app/api/copilot/checkpoints/revert/route.ts +++ b/apps/sim/app/api/copilot/checkpoints/revert/route.ts @@ -11,6 +11,7 @@ import { createUnauthorizedResponse, } from '@/lib/copilot/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateUUID } from '@/lib/security/input-validation' const logger = createLogger('CheckpointRevertAPI') @@ -36,7 +37,6 @@ export async function POST(request: NextRequest) { logger.info(`[${tracker.requestId}] Reverting to checkpoint ${checkpointId}`) - // Get the checkpoint and verify ownership const checkpoint = await db .select() .from(workflowCheckpoints) @@ -47,7 +47,6 @@ export async function POST(request: NextRequest) { return createNotFoundResponse('Checkpoint not found or access denied') } - // Verify user still has access to the workflow const workflowData = await db .select() .from(workflowTable) @@ -62,10 +61,8 @@ export async function POST(request: NextRequest) { return createUnauthorizedResponse() } - // Apply the checkpoint state to the workflow using the existing state endpoint const checkpointState = checkpoint.workflowState as any // Cast to any for property access - // Clean the checkpoint state to remove any null/undefined values that could cause validation errors const cleanedState = { blocks: checkpointState?.blocks || {}, edges: checkpointState?.edges || [], @@ -74,7 +71,6 @@ export async function POST(request: NextRequest) { isDeployed: checkpointState?.isDeployed || false, deploymentStatuses: checkpointState?.deploymentStatuses || {}, lastSaved: Date.now(), - // Only include deployedAt if it's a valid date string that can be converted ...(checkpointState?.deployedAt && checkpointState.deployedAt !== null && checkpointState.deployedAt !== undefined && @@ -90,13 +86,19 @@ export async function POST(request: NextRequest) { isDeployed: cleanedState.isDeployed, }) + const workflowIdValidation = validateUUID(checkpoint.workflowId, 'workflowId') + if (!workflowIdValidation.isValid) { + logger.error(`[${tracker.requestId}] Invalid workflow ID: ${workflowIdValidation.error}`) + return NextResponse.json({ error: 'Invalid workflow ID format' }, { status: 400 }) + } + const stateResponse = await fetch( `${request.nextUrl.origin}/api/workflows/${checkpoint.workflowId}/state`, { method: 'PUT', headers: { 'Content-Type': 'application/json', - Cookie: request.headers.get('Cookie') || '', // Forward auth cookies + Cookie: request.headers.get('Cookie') || '', }, body: JSON.stringify(cleanedState), } @@ -123,7 +125,7 @@ export async function POST(request: NextRequest) { revertedAt: new Date().toISOString(), checkpoint: { id: checkpoint.id, - workflowState: cleanedState, // Return the reverted state for frontend use + workflowState: cleanedState, }, }) } catch (error) { diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts index f87eba7927..a8c8b74371 100644 --- a/apps/sim/app/api/files/parse/route.ts +++ b/apps/sim/app/api/files/parse/route.ts @@ -6,6 +6,7 @@ import binaryExtensionsList from 'binary-extensions' import { type NextRequest, NextResponse } from 'next/server' import { isSupportedFileType, parseFile } from '@/lib/file-parsers' import { createLogger } from '@/lib/logs/console/logger' +import { validateExternalUrl } from '@/lib/security/input-validation' import { downloadFile, isUsingCloudStorage } from '@/lib/uploads' import { UPLOAD_DIR_SERVER } from '@/lib/uploads/setup.server' import '@/lib/uploads/setup.server' @@ -220,6 +221,16 @@ async function handleExternalUrl(url: string, fileType?: string): Promise = { // Text formats txt: 'text/plain', @@ -79,9 +70,6 @@ export const contentTypeMap: Record = { googleFolder: 'application/vnd.google-apps.folder', } -/** - * List of binary file extensions - */ export const binaryExtensions = [ 'doc', 'docx', @@ -97,38 +85,23 @@ export const binaryExtensions = [ 'pdf', ] -/** - * Determine content type from file extension - */ export function getContentType(filename: string): string { const extension = filename.split('.').pop()?.toLowerCase() || '' return contentTypeMap[extension] || 'application/octet-stream' } -/** - * Check if a path is an S3 path - */ export function isS3Path(path: string): boolean { return path.includes('/api/files/serve/s3/') } -/** - * Check if a path is a Blob path - */ export function isBlobPath(path: string): boolean { return path.includes('/api/files/serve/blob/') } -/** - * Check if a path points to cloud storage (S3, Blob, or generic cloud) - */ export function isCloudPath(path: string): boolean { return isS3Path(path) || isBlobPath(path) } -/** - * Generic function to extract storage key from a path - */ export function extractStorageKey(path: string, storageType: 's3' | 'blob'): string { const prefix = `/api/files/serve/${storageType}/` if (path.includes(prefix)) { @@ -137,23 +110,14 @@ export function extractStorageKey(path: string, storageType: 's3' | 'blob'): str return path } -/** - * Extract S3 key from a path - */ export function extractS3Key(path: string): string { return extractStorageKey(path, 's3') } -/** - * Extract Blob key from a path - */ export function extractBlobKey(path: string): string { return extractStorageKey(path, 'blob') } -/** - * Extract filename from a serve path - */ export function extractFilename(path: string): string { let filename: string @@ -168,25 +132,20 @@ export function extractFilename(path: string): string { .replace(/\/\.\./g, '') .replace(/\.\.\//g, '') - // Handle cloud storage paths (s3/key, blob/key) - preserve forward slashes for these if (filename.startsWith('s3/') || filename.startsWith('blob/')) { - // For cloud paths, only sanitize the key portion after the prefix const parts = filename.split('/') const prefix = parts[0] // 's3' or 'blob' const keyParts = parts.slice(1) - // Sanitize each part of the key to prevent traversal const sanitizedKeyParts = keyParts .map((part) => part.replace(/\.\./g, '').replace(/^\./g, '').trim()) .filter((part) => part.length > 0) filename = `${prefix}/${sanitizedKeyParts.join('/')}` } else { - // For regular filenames, remove any remaining path separators filename = filename.replace(/[/\\]/g, '') } - // Additional validation: ensure filename is not empty after sanitization if (!filename || filename.trim().length === 0) { throw new Error('Invalid or empty filename after sanitization') } @@ -194,19 +153,12 @@ export function extractFilename(path: string): string { return filename } -/** - * Sanitize filename to prevent path traversal attacks - */ function sanitizeFilename(filename: string): string { if (!filename || typeof filename !== 'string') { throw new Error('Invalid filename provided') } - const sanitized = filename - .replace(/\.\./g, '') // Remove .. sequences - .replace(/[/\\]/g, '') // Remove path separators - .replace(/^\./g, '') // Remove leading dots - .trim() + const sanitized = filename.replace(/\.\./g, '').replace(/[/\\]/g, '').replace(/^\./g, '').trim() if (!sanitized || sanitized.length === 0) { throw new Error('Invalid or empty filename after sanitization') @@ -217,8 +169,8 @@ function sanitizeFilename(filename: string): string { sanitized.includes('|') || sanitized.includes('?') || sanitized.includes('*') || - sanitized.includes('\x00') || // Null bytes - /[\x00-\x1F\x7F]/.test(sanitized) // Control characters + sanitized.includes('\x00') || + /[\x00-\x1F\x7F]/.test(sanitized) ) { throw new Error('Filename contains invalid characters') } @@ -226,9 +178,6 @@ function sanitizeFilename(filename: string): string { return sanitized } -/** - * Find a file in possible local storage locations with proper path validation - */ export function findLocalFile(filename: string): string | null { try { const sanitizedFilename = sanitizeFilename(filename) @@ -247,7 +196,7 @@ export function findLocalFile(filename: string): string | null { ) if (!isWithinAllowedDir) { - continue // Skip this path as it's outside allowed directories + continue } if (existsSync(resolvedPath)) { @@ -273,32 +222,24 @@ const SAFE_INLINE_TYPES = new Set([ 'application/json', ]) -// File extensions that should always be served as attachment for security const FORCE_ATTACHMENT_EXTENSIONS = new Set(['html', 'htm', 'svg', 'js', 'css', 'xml']) -/** - * Determines safe content type and disposition for file serving - */ function getSecureFileHeaders(filename: string, originalContentType: string) { const extension = filename.split('.').pop()?.toLowerCase() || '' - // Force attachment for potentially dangerous file types if (FORCE_ATTACHMENT_EXTENSIONS.has(extension)) { return { - contentType: 'application/octet-stream', // Force download + contentType: 'application/octet-stream', disposition: 'attachment', } } - // Override content type for safety while preserving legitimate use cases let safeContentType = originalContentType - // Handle potentially dangerous content types if (originalContentType === 'text/html' || originalContentType === 'image/svg+xml') { - safeContentType = 'text/plain' // Prevent browser rendering + safeContentType = 'text/plain' } - // Use inline only for verified safe content types const disposition = SAFE_INLINE_TYPES.has(safeContentType) ? 'inline' : 'attachment' return { @@ -307,10 +248,6 @@ function getSecureFileHeaders(filename: string, originalContentType: string) { } } -/** - * Encode filename for Content-Disposition header to support non-ASCII characters - * Uses RFC 5987 encoding for international characters - */ function encodeFilenameForHeader(filename: string): string { const hasNonAscii = /[^\x00-\x7F]/.test(filename) @@ -323,9 +260,6 @@ function encodeFilenameForHeader(filename: string): string { return `filename="${asciiSafe}"; filename*=UTF-8''${encodedFilename}` } -/** - * Create a file response with appropriate security headers - */ export function createFileResponse(file: FileResponse): NextResponse { const { contentType, disposition } = getSecureFileHeaders(file.filename, file.contentType) @@ -334,18 +268,14 @@ export function createFileResponse(file: FileResponse): NextResponse { headers: { 'Content-Type': contentType, 'Content-Disposition': `${disposition}; ${encodeFilenameForHeader(file.filename)}`, - 'Cache-Control': 'public, max-age=31536000', // Cache for 1 year + 'Cache-Control': 'public, max-age=31536000', 'X-Content-Type-Options': 'nosniff', 'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; sandbox;", }, }) } -/** - * Create a standardized error response - */ export function createErrorResponse(error: Error, status = 500): NextResponse { - // Map error types to appropriate status codes const statusCode = error instanceof FileNotFoundError ? 404 : error instanceof InvalidRequestError ? 400 : status @@ -358,16 +288,10 @@ export function createErrorResponse(error: Error, status = 500): NextResponse { ) } -/** - * Create a standardized success response - */ export function createSuccessResponse(data: ApiSuccessResponse): NextResponse { return NextResponse.json(data) } -/** - * Handle CORS preflight requests - */ export function createOptionsResponse(): NextResponse { return new NextResponse(null, { status: 204, diff --git a/apps/sim/app/api/function/execute/route.test.ts b/apps/sim/app/api/function/execute/route.test.ts index 8e32ae9cc0..5733608b24 100644 --- a/apps/sim/app/api/function/execute/route.test.ts +++ b/apps/sim/app/api/function/execute/route.test.ts @@ -67,7 +67,7 @@ describe('Function Execute API Route', () => { }) it.concurrent('should block SSRF attacks through secure fetch wrapper', async () => { - const { validateProxyUrl } = await import('@/lib/security/url-validation') + const { validateProxyUrl } = await import('@/lib/security/input-validation') expect(validateProxyUrl('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) expect(validateProxyUrl('http://127.0.0.1:8080/admin').isValid).toBe(false) @@ -76,15 +76,15 @@ describe('Function Execute API Route', () => { }) it.concurrent('should allow legitimate external URLs', async () => { - const { validateProxyUrl } = await import('@/lib/security/url-validation') + const { validateProxyUrl } = await import('@/lib/security/input-validation') expect(validateProxyUrl('https://api.github.com/user').isValid).toBe(true) expect(validateProxyUrl('https://httpbin.org/get').isValid).toBe(true) - expect(validateProxyUrl('http://example.com/api').isValid).toBe(true) + expect(validateProxyUrl('https://example.com/api').isValid).toBe(true) }) it.concurrent('should block dangerous protocols', async () => { - const { validateProxyUrl } = await import('@/lib/security/url-validation') + const { validateProxyUrl } = await import('@/lib/security/input-validation') expect(validateProxyUrl('file:///etc/passwd').isValid).toBe(false) expect(validateProxyUrl('ftp://internal.server/files').isValid).toBe(false) diff --git a/apps/sim/app/api/function/execute/route.ts b/apps/sim/app/api/function/execute/route.ts index 49e53cd8dc..31a5340d46 100644 --- a/apps/sim/app/api/function/execute/route.ts +++ b/apps/sim/app/api/function/execute/route.ts @@ -4,7 +4,7 @@ import { env, isTruthy } from '@/lib/env' import { executeInE2B } from '@/lib/execution/e2b' import { CodeLanguage, DEFAULT_CODE_LANGUAGE, isValidCodeLanguage } from '@/lib/execution/languages' import { createLogger } from '@/lib/logs/console/logger' -import { validateProxyUrl } from '@/lib/security/url-validation' +import { validateProxyUrl } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' export const dynamic = 'force-dynamic' export const runtime = 'nodejs' diff --git a/apps/sim/app/api/proxy/image/route.ts b/apps/sim/app/api/proxy/image/route.ts index dba9958ea9..fc7717b671 100644 --- a/apps/sim/app/api/proxy/image/route.ts +++ b/apps/sim/app/api/proxy/image/route.ts @@ -1,6 +1,6 @@ import { type NextRequest, NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' -import { validateImageUrl } from '@/lib/security/url-validation' +import { validateImageUrl } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' const logger = createLogger('ImageProxyAPI') diff --git a/apps/sim/app/api/proxy/route.ts b/apps/sim/app/api/proxy/route.ts index c207c7bc55..f4699e7234 100644 --- a/apps/sim/app/api/proxy/route.ts +++ b/apps/sim/app/api/proxy/route.ts @@ -2,7 +2,7 @@ import { NextResponse } from 'next/server' import { generateInternalToken } from '@/lib/auth/internal' import { isDev } from '@/lib/environment' import { createLogger } from '@/lib/logs/console/logger' -import { validateProxyUrl } from '@/lib/security/url-validation' +import { validateProxyUrl } from '@/lib/security/input-validation' import { getBaseUrl } from '@/lib/urls/utils' import { generateRequestId } from '@/lib/utils' import { executeTool } from '@/tools' diff --git a/apps/sim/app/api/proxy/tts/route.ts b/apps/sim/app/api/proxy/tts/route.ts index a54071e722..609ea53e4f 100644 --- a/apps/sim/app/api/proxy/tts/route.ts +++ b/apps/sim/app/api/proxy/tts/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId } from '@/lib/security/input-validation' import { uploadFile } from '@/lib/uploads/storage-client' import { getBaseUrl } from '@/lib/urls/utils' @@ -14,6 +15,12 @@ export async function POST(request: Request) { return new NextResponse('Missing required parameters', { status: 400 }) } + const voiceIdValidation = validateAlphanumericId(voiceId, 'voiceId', 255) + if (!voiceIdValidation.isValid) { + logger.error(`Invalid voice ID: ${voiceIdValidation.error}`) + return new NextResponse(voiceIdValidation.error, { status: 400 }) + } + logger.info('Proxying TTS request for voice:', voiceId) const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}` @@ -46,13 +53,11 @@ export async function POST(request: Request) { return new NextResponse('Empty audio received', { status: 422 }) } - // Upload the audio file to storage and return multiple URL options const audioBuffer = Buffer.from(await audioBlob.arrayBuffer()) const timestamp = Date.now() const fileName = `elevenlabs-tts-${timestamp}.mp3` const fileInfo = await uploadFile(audioBuffer, fileName, 'audio/mpeg') - // Generate the full URL for external use using the configured base URL const audioUrl = `${getBaseUrl()}${fileInfo.path}` return NextResponse.json({ diff --git a/apps/sim/app/api/proxy/tts/stream/route.ts b/apps/sim/app/api/proxy/tts/stream/route.ts index 2d8f3c6c67..a2910d3738 100644 --- a/apps/sim/app/api/proxy/tts/stream/route.ts +++ b/apps/sim/app/api/proxy/tts/stream/route.ts @@ -1,6 +1,7 @@ import type { NextRequest } from 'next/server' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId } from '@/lib/security/input-validation' const logger = createLogger('ProxyTTSStreamAPI') @@ -13,6 +14,12 @@ export async function POST(request: NextRequest) { return new Response('Missing required parameters', { status: 400 }) } + const voiceIdValidation = validateAlphanumericId(voiceId, 'voiceId', 255) + if (!voiceIdValidation.isValid) { + logger.error(`Invalid voice ID: ${voiceIdValidation.error}`) + return new Response(voiceIdValidation.error, { status: 400 }) + } + const apiKey = env.ELEVENLABS_API_KEY if (!apiKey) { logger.error('ELEVENLABS_API_KEY not configured on server') @@ -31,7 +38,6 @@ export async function POST(request: NextRequest) { body: JSON.stringify({ text, model_id: modelId, - // Maximum performance settings optimize_streaming_latency: 4, output_format: 'mp3_22050_32', // Fastest format voice_settings: { @@ -42,9 +48,7 @@ export async function POST(request: NextRequest) { }, enable_ssml_parsing: false, apply_text_normalization: 'off', - // Use auto mode for fastest possible streaming - // Note: This may sacrifice some quality for speed - use_pvc_as_ivc: false, // Use fastest voice processing + use_pvc_as_ivc: false, }), }) @@ -60,14 +64,11 @@ export async function POST(request: NextRequest) { return new Response('No audio stream received', { status: 422 }) } - // Create optimized streaming response const { readable, writable } = new TransformStream({ transform(chunk, controller) { - // Pass through chunks immediately without buffering controller.enqueue(chunk) }, flush(controller) { - // Ensure all data is flushed immediately controller.terminate() }, }) @@ -83,7 +84,6 @@ export async function POST(request: NextRequest) { await writer.close() break } - // Write immediately without waiting writer.write(value).catch(logger.error) } } catch (error) { @@ -102,19 +102,15 @@ export async function POST(request: NextRequest) { 'X-Content-Type-Options': 'nosniff', 'Access-Control-Allow-Origin': '*', Connection: 'keep-alive', - // Stream headers for better streaming - 'X-Accel-Buffering': 'no', // Disable nginx buffering + 'X-Accel-Buffering': 'no', 'X-Stream-Type': 'real-time', }, }) } catch (error) { logger.error('Error in Stream TTS:', error) - return new Response( - `Internal Server Error: ${error instanceof Error ? error.message : 'Unknown error'}`, - { - status: 500, - } - ) + return new Response('Internal Server Error', { + status: 500, + }) } } diff --git a/apps/sim/app/api/tools/confluence/page/route.ts b/apps/sim/app/api/tools/confluence/page/route.ts index 7a49b25bb1..1fa3c5b1ff 100644 --- a/apps/sim/app/api/tools/confluence/page/route.ts +++ b/apps/sim/app/api/tools/confluence/page/route.ts @@ -1,4 +1,5 @@ import { NextResponse } from 'next/server' +import { validateAlphanumericId, validateJiraCloudId } from '@/lib/security/input-validation' import { getConfluenceCloudId } from '@/tools/confluence/utils' export const dynamic = 'force-dynamic' @@ -19,13 +20,20 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) } - // Use provided cloudId or fetch it if not provided + const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) + if (!pageIdValidation.isValid) { + return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) + } + const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - // Build the URL for the Confluence API + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}?expand=body.storage,body.view,body.atlas_doc_format` - // Make the request to Confluence API const response = await fetch(url, { method: 'GET', headers: { @@ -52,7 +60,6 @@ export async function POST(request: Request) { const data = await response.json() - // If body is empty, try to provide a minimal valid response return NextResponse.json({ id: data.id, title: data.title, @@ -103,9 +110,18 @@ export async function PUT(request: Request) { return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) } + const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) + if (!pageIdValidation.isValid) { + return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) + } + const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - // First, get the current page to check its version + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + const currentPageUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}` const currentPageResponse = await fetch(currentPageUrl, { headers: { @@ -121,7 +137,6 @@ export async function PUT(request: Request) { const currentPage = await currentPageResponse.json() const currentVersion = currentPage.version.number - // Build the update body with incremented version const updateBody: any = { id: pageId, version: { diff --git a/apps/sim/app/api/tools/discord/channels/route.ts b/apps/sim/app/api/tools/discord/channels/route.ts index 8b6bb2872f..643860d2fc 100644 --- a/apps/sim/app/api/tools/discord/channels/route.ts +++ b/apps/sim/app/api/tools/discord/channels/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateNumericId } from '@/lib/security/input-validation' interface DiscordChannel { id: string @@ -26,11 +27,21 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Server ID is required' }, { status: 400 }) } - // If channelId is provided, we'll fetch just that specific channel + const serverIdValidation = validateNumericId(serverId, 'serverId') + if (!serverIdValidation.isValid) { + logger.error(`Invalid server ID: ${serverIdValidation.error}`) + return NextResponse.json({ error: serverIdValidation.error }, { status: 400 }) + } + if (channelId) { + const channelIdValidation = validateNumericId(channelId, 'channelId') + if (!channelIdValidation.isValid) { + logger.error(`Invalid channel ID: ${channelIdValidation.error}`) + return NextResponse.json({ error: channelIdValidation.error }, { status: 400 }) + } + logger.info(`Fetching single Discord channel: ${channelId}`) - // Fetch a specific channel by ID const response = await fetch(`https://discord.com/api/v10/channels/${channelId}`, { method: 'GET', headers: { @@ -58,7 +69,6 @@ export async function POST(request: Request) { const channel = (await response.json()) as DiscordChannel - // Verify this is a text channel and belongs to the requested server if (channel.guild_id !== serverId) { logger.error('Channel does not belong to the specified server') return NextResponse.json( @@ -85,8 +95,6 @@ export async function POST(request: Request) { logger.info(`Fetching all Discord channels for server: ${serverId}`) - // Listing guild channels with a bot token is allowed if the bot is in the guild. - // Keep the request, but if unauthorized, return an empty list so the selector doesn't hard fail. const response = await fetch(`https://discord.com/api/v10/guilds/${serverId}/channels`, { method: 'GET', headers: { @@ -108,7 +116,6 @@ export async function POST(request: Request) { const channels = (await response.json()) as DiscordChannel[] - // Filter to just text channels (type 0) const textChannels = channels.filter((channel: DiscordChannel) => channel.type === 0) logger.info(`Successfully fetched ${textChannels.length} text channels`) diff --git a/apps/sim/app/api/tools/discord/servers/route.ts b/apps/sim/app/api/tools/discord/servers/route.ts index 38db70a694..9588a5a88c 100644 --- a/apps/sim/app/api/tools/discord/servers/route.ts +++ b/apps/sim/app/api/tools/discord/servers/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateNumericId } from '@/lib/security/input-validation' interface DiscordServer { id: string @@ -20,11 +21,15 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Bot token is required' }, { status: 400 }) } - // If serverId is provided, we'll fetch just that server if (serverId) { + const serverIdValidation = validateNumericId(serverId, 'serverId') + if (!serverIdValidation.isValid) { + logger.error(`Invalid server ID: ${serverIdValidation.error}`) + return NextResponse.json({ error: serverIdValidation.error }, { status: 400 }) + } + logger.info(`Fetching single Discord server: ${serverId}`) - // Fetch a specific server by ID const response = await fetch(`https://discord.com/api/v10/guilds/${serverId}`, { method: 'GET', headers: { @@ -64,10 +69,6 @@ export async function POST(request: Request) { }) } - // Listing guilds via REST requires a user OAuth2 access token with the 'guilds' scope. - // A bot token cannot call /users/@me/guilds and will return 401. - // Since this selector only has a bot token, return an empty list instead of erroring - // and let users provide a Server ID in advanced mode. logger.info( 'Skipping guild listing: bot token cannot list /users/@me/guilds; returning empty list' ) diff --git a/apps/sim/app/api/tools/drive/file/route.ts b/apps/sim/app/api/tools/drive/file/route.ts index 71dc57f97b..02e116288e 100644 --- a/apps/sim/app/api/tools/drive/file/route.ts +++ b/apps/sim/app/api/tools/drive/file/route.ts @@ -1,6 +1,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' @@ -25,6 +26,12 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and File ID are required' }, { status: 400 }) } + const fileIdValidation = validateAlphanumericId(fileId, 'fileId', 255) + if (!fileIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid file ID: ${fileIdValidation.error}`) + return NextResponse.json({ error: fileIdValidation.error }, { status: 400 }) + } + const authz = await authorizeCredentialUse(request, { credentialId: credentialId, workflowId }) if (!authz.ok || !authz.credentialOwnerUserId) { return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) @@ -67,10 +74,10 @@ export async function GET(request: NextRequest) { const file = await response.json() const exportFormats: { [key: string]: string } = { - 'application/vnd.google-apps.document': 'application/pdf', // Google Docs to PDF + 'application/vnd.google-apps.document': 'application/pdf', 'application/vnd.google-apps.spreadsheet': - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // Google Sheets to XLSX - 'application/vnd.google-apps.presentation': 'application/pdf', // Google Slides to PDF + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.google-apps.presentation': 'application/pdf', } if ( diff --git a/apps/sim/app/api/tools/drive/files/route.ts b/apps/sim/app/api/tools/drive/files/route.ts index 70d08108ad..74481fefde 100644 --- a/apps/sim/app/api/tools/drive/files/route.ts +++ b/apps/sim/app/api/tools/drive/files/route.ts @@ -8,9 +8,10 @@ export const dynamic = 'force-dynamic' const logger = createLogger('GoogleDriveFilesAPI') -/** - * Get files from Google Drive - */ +function escapeForDriveQuery(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") +} + export async function GET(request: NextRequest) { const requestId = generateRequestId() logger.info(`[${requestId}] Google Drive files request received`) @@ -53,13 +54,13 @@ export async function GET(request: NextRequest) { const qParts: string[] = ['trashed = false'] if (folderId) { - qParts.push(`'${folderId.replace(/'/g, "\\'")}' in parents`) + qParts.push(`'${escapeForDriveQuery(folderId)}' in parents`) } if (mimeType) { - qParts.push(`mimeType = '${mimeType.replace(/'/g, "\\'")}'`) + qParts.push(`mimeType = '${escapeForDriveQuery(mimeType)}'`) } if (query) { - qParts.push(`name contains '${query.replace(/'/g, "\\'")}'`) + qParts.push(`name contains '${escapeForDriveQuery(query)}'`) } const q = encodeURIComponent(qParts.join(' and ')) diff --git a/apps/sim/app/api/tools/gmail/label/route.ts b/apps/sim/app/api/tools/gmail/label/route.ts index 5a0e8aa6d9..2da053cfe2 100644 --- a/apps/sim/app/api/tools/gmail/label/route.ts +++ b/apps/sim/app/api/tools/gmail/label/route.ts @@ -4,6 +4,7 @@ import { and, eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' @@ -15,10 +16,8 @@ export async function GET(request: NextRequest) { const requestId = generateRequestId() try { - // Get the session const session = await getSession() - // Check if the user is authenticated if (!session?.user?.id) { logger.warn(`[${requestId}] Unauthenticated label request rejected`) return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) @@ -36,7 +35,12 @@ export async function GET(request: NextRequest) { ) } - // Get the credential from the database + const labelIdValidation = validateAlphanumericId(labelId, 'labelId', 255) + if (!labelIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid label ID: ${labelIdValidation.error}`) + return NextResponse.json({ error: labelIdValidation.error }, { status: 400 }) + } + const credentials = await db .select() .from(account) @@ -50,19 +54,16 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Log the credential info (without exposing sensitive data) logger.info( `[${requestId}] Using credential: ${credential.id}, provider: ${credential.providerId}` ) - // Refresh access token if needed using the utility function const accessToken = await refreshAccessTokenIfNeeded(credentialId, session.user.id, requestId) if (!accessToken) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Fetch specific label from Gmail API logger.info(`[${requestId}] Fetching label ${labelId} from Gmail API`) const response = await fetch( `https://gmail.googleapis.com/gmail/v1/users/me/labels/${labelId}`, @@ -73,7 +74,6 @@ export async function GET(request: NextRequest) { } ) - // Log the response status logger.info(`[${requestId}] Gmail API response status: ${response.status}`) if (!response.ok) { @@ -90,13 +90,9 @@ export async function GET(request: NextRequest) { const label = await response.json() - // Transform the label to a more usable format - // Format the label name with proper capitalization let formattedName = label.name - // Handle system labels (INBOX, SENT, etc.) if (label.type === 'system') { - // Convert to title case (first letter uppercase, rest lowercase) formattedName = label.name.charAt(0).toUpperCase() + label.name.slice(1).toLowerCase() } diff --git a/apps/sim/app/api/tools/gmail/labels/route.ts b/apps/sim/app/api/tools/gmail/labels/route.ts index 3b177d4f4e..70a62c7952 100644 --- a/apps/sim/app/api/tools/gmail/labels/route.ts +++ b/apps/sim/app/api/tools/gmail/labels/route.ts @@ -22,10 +22,8 @@ export async function GET(request: NextRequest) { const requestId = generateRequestId() try { - // Get the session const session = await getSession() - // Check if the user is authenticated if (!session?.user?.id) { logger.warn(`[${requestId}] Unauthenticated labels request rejected`) return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) @@ -40,8 +38,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID is required' }, { status: 400 }) } - // Get the credential from the database. Prefer session-owned credential, but - // if not found, resolve by credential ID to support collaborator-owned credentials. let credentials = await db .select() .from(account) @@ -58,26 +54,22 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Log the credential info (without exposing sensitive data) logger.info( `[${requestId}] Using credential: ${credential.id}, provider: ${credential.providerId}` ) - // Refresh access token if needed using the utility function const accessToken = await refreshAccessTokenIfNeeded(credentialId, credential.userId, requestId) if (!accessToken) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Fetch labels from Gmail API const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/labels', { headers: { Authorization: `Bearer ${accessToken}`, }, }) - // Log the response status logger.info(`[${requestId}] Gmail API response status: ${response.status}`) if (!response.ok) { @@ -98,14 +90,10 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Invalid labels response' }, { status: 500 }) } - // Transform the labels to a more usable format const labels = data.labels.map((label: GmailLabel) => { - // Format the label name with proper capitalization let formattedName = label.name - // Handle system labels (INBOX, SENT, etc.) if (label.type === 'system') { - // Convert to title case (first letter uppercase, rest lowercase) formattedName = label.name.charAt(0).toUpperCase() + label.name.slice(1).toLowerCase() } @@ -118,7 +106,6 @@ export async function GET(request: NextRequest) { } }) - // Filter labels if a query is provided const filteredLabels = query ? labels.filter((label: GmailLabel) => label.name.toLowerCase().includes((query as string).toLowerCase()) diff --git a/apps/sim/app/api/tools/jira/issue/route.ts b/apps/sim/app/api/tools/jira/issue/route.ts index 1aae5c1fad..60ba1a1365 100644 --- a/apps/sim/app/api/tools/jira/issue/route.ts +++ b/apps/sim/app/api/tools/jira/issue/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/security/input-validation' import { getJiraCloudId } from '@/tools/jira/utils' export const dynamic = 'force-dynamic' @@ -9,7 +10,6 @@ const logger = createLogger('JiraIssueAPI') export async function POST(request: Request) { try { const { domain, accessToken, issueId, cloudId: providedCloudId } = await request.json() - // Add detailed request logging if (!domain) { logger.error('Missing domain in request') return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) @@ -25,16 +25,23 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Issue ID is required' }, { status: 400 }) } - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) logger.info('Using cloud ID:', cloudId) - // Build the URL using cloudId for Jira API + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + + const issueIdValidation = validateJiraIssueKey(issueId, 'issueId') + if (!issueIdValidation.isValid) { + return NextResponse.json({ error: issueIdValidation.error }, { status: 400 }) + } + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${issueId}` logger.info('Fetching Jira issue from:', url) - // Make the request to Jira API const response = await fetch(url, { method: 'GET', headers: { @@ -63,7 +70,6 @@ export async function POST(request: Request) { const data = await response.json() logger.info('Successfully fetched issue:', data.key) - // Transform the Jira issue data into our expected format const issueInfo: any = { id: data.key, name: data.fields.summary, @@ -71,7 +77,6 @@ export async function POST(request: Request) { url: `https://${domain}/browse/${data.key}`, modifiedTime: data.fields.updated, webViewLink: `https://${domain}/browse/${data.key}`, - // Add additional fields that might be needed for the workflow status: data.fields.status?.name, description: data.fields.description, priority: data.fields.priority?.name, @@ -85,11 +90,10 @@ export async function POST(request: Request) { return NextResponse.json({ issue: issueInfo, - cloudId, // Return the cloudId so it can be cached + cloudId, }) } catch (error) { logger.error('Error processing request:', error) - // Add more context to the error response return NextResponse.json( { error: 'Failed to retrieve Jira issue', diff --git a/apps/sim/app/api/tools/jira/issues/route.ts b/apps/sim/app/api/tools/jira/issues/route.ts index 7310dd95de..ea11563381 100644 --- a/apps/sim/app/api/tools/jira/issues/route.ts +++ b/apps/sim/app/api/tools/jira/issues/route.ts @@ -1,12 +1,12 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId, validateJiraCloudId } from '@/lib/security/input-validation' import { getJiraCloudId } from '@/tools/jira/utils' export const dynamic = 'force-dynamic' const logger = createLogger('JiraIssuesAPI') -// Helper functions const createErrorResponse = async (response: Response, defaultMessage: string) => { try { const errorData = await response.json() @@ -38,13 +38,15 @@ export async function POST(request: Request) { return NextResponse.json({ issues: [] }) } - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain!, accessToken!)) - // Build the URL using cloudId for Jira API + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/bulkfetch` - // Prepare the request body for bulk fetch const requestBody = { expand: ['names'], fields: ['summary', 'status', 'assignee', 'updated', 'project'], @@ -53,7 +55,6 @@ export async function POST(request: Request) { properties: [], } - // Make the request to Jira API with OAuth Bearer token const requestConfig = { method: 'POST', headers: { @@ -112,6 +113,29 @@ export async function GET(request: Request) { if (validationError) return validationError const cloudId = providedCloudId || (await getJiraCloudId(domain!, accessToken!)) + + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + + if (projectId) { + const projectIdValidation = validateAlphanumericId(projectId, 'projectId', 100) + if (!projectIdValidation.isValid) { + return NextResponse.json({ error: projectIdValidation.error }, { status: 400 }) + } + } + if (manualProjectId) { + const manualProjectIdValidation = validateAlphanumericId( + manualProjectId, + 'manualProjectId', + 100 + ) + if (!manualProjectIdValidation.isValid) { + return NextResponse.json({ error: manualProjectIdValidation.error }, { status: 400 }) + } + } + let data: any if (query) { diff --git a/apps/sim/app/api/tools/jira/projects/route.ts b/apps/sim/app/api/tools/jira/projects/route.ts index da2ce3aaae..6f7b856caf 100644 --- a/apps/sim/app/api/tools/jira/projects/route.ts +++ b/apps/sim/app/api/tools/jira/projects/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId, validateJiraCloudId } from '@/lib/security/input-validation' import { getJiraCloudId } from '@/tools/jira/utils' export const dynamic = 'force-dynamic' @@ -22,19 +23,20 @@ export async function GET(request: Request) { return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) } - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) logger.info(`Using cloud ID: ${cloudId}`) - // Build the URL for the Jira API projects endpoint + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + const apiUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/project/search` - // Add query parameters if searching const queryParams = new URLSearchParams() if (query) { queryParams.append('query', query) } - // Add other useful parameters queryParams.append('orderBy', 'name') queryParams.append('expand', 'description,lead,url,projectKeys') @@ -66,18 +68,16 @@ export async function GET(request: Request) { const data = await response.json() - // Add detailed logging logger.info(`Jira API Response Status: ${response.status}`) logger.info(`Found projects: ${data.values?.length || 0}`) - // Transform the response to match our expected format const projects = data.values?.map((project: any) => ({ id: project.id, key: project.key, name: project.name, url: project.self, - avatarUrl: project.avatarUrls?.['48x48'], // Use the medium size avatar + avatarUrl: project.avatarUrls?.['48x48'], description: project.description, projectTypeKey: project.projectTypeKey, simplified: project.simplified, @@ -87,7 +87,7 @@ export async function GET(request: Request) { return NextResponse.json({ projects, - cloudId, // Return the cloudId so it can be cached + cloudId, }) } catch (error) { logger.error('Error fetching Jira projects:', error) @@ -98,7 +98,6 @@ export async function GET(request: Request) { } } -// For individual project retrieval if needed export async function POST(request: Request) { try { const { domain, accessToken, projectId, cloudId: providedCloudId } = await request.json() @@ -115,9 +114,18 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Project ID is required' }, { status: 400 }) } - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + + const projectIdValidation = validateAlphanumericId(projectId, 'projectId', 100) + if (!projectIdValidation.isValid) { + return NextResponse.json({ error: projectIdValidation.error }, { status: 400 }) + } + const apiUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/project/${projectId}` const response = await fetch(apiUrl, { diff --git a/apps/sim/app/api/tools/jira/update/route.ts b/apps/sim/app/api/tools/jira/update/route.ts index 0657bb57fc..68921a90f8 100644 --- a/apps/sim/app/api/tools/jira/update/route.ts +++ b/apps/sim/app/api/tools/jira/update/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/security/input-validation' import { getJiraCloudId } from '@/tools/jira/utils' export const dynamic = 'force-dynamic' @@ -13,7 +14,7 @@ export async function PUT(request: Request) { accessToken, issueKey, summary, - title, // Support both summary and title for backwards compatibility + title, description, status, priority, @@ -21,7 +22,6 @@ export async function PUT(request: Request) { cloudId: providedCloudId, } = await request.json() - // Validate required parameters if (!domain) { logger.error('Missing domain in request') return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) @@ -37,16 +37,23 @@ export async function PUT(request: Request) { return NextResponse.json({ error: 'Issue key is required' }, { status: 400 }) } - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) logger.info('Using cloud ID:', cloudId) - // Build the URL using cloudId for Jira API + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + + const issueKeyValidation = validateJiraIssueKey(issueKey, 'issueKey') + if (!issueKeyValidation.isValid) { + return NextResponse.json({ error: issueKeyValidation.error }, { status: 400 }) + } + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${issueKey}` logger.info('Updating Jira issue at:', url) - // Map the summary from either summary or title field const summaryValue = summary || title const fields: Record = {} @@ -92,7 +99,6 @@ export async function PUT(request: Request) { const body = { fields } - // Make the request to Jira API const response = await fetch(url, { method: 'PUT', headers: { @@ -117,7 +123,6 @@ export async function PUT(request: Request) { ) } - // Note: Jira update API typically returns 204 No Content on success const responseData = response.status === 204 ? {} : await response.json() logger.info('Successfully updated Jira issue:', issueKey) diff --git a/apps/sim/app/api/tools/jira/write/route.ts b/apps/sim/app/api/tools/jira/write/route.ts index fc4eab419b..96f4813a52 100644 --- a/apps/sim/app/api/tools/jira/write/route.ts +++ b/apps/sim/app/api/tools/jira/write/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server' import { createLogger } from '@/lib/logs/console/logger' +import { validateAlphanumericId, validateJiraCloudId } from '@/lib/security/input-validation' import { getJiraCloudId } from '@/tools/jira/utils' export const dynamic = 'force-dynamic' @@ -21,7 +22,6 @@ export async function POST(request: Request) { parent, } = await request.json() - // Validate required parameters if (!domain) { logger.error('Missing domain in request') return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) @@ -44,16 +44,23 @@ export async function POST(request: Request) { const normalizedIssueType = issueType || 'Task' - // Use provided cloudId or fetch it if not provided const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) logger.info('Using cloud ID:', cloudId) - // Build the URL using cloudId for Jira API + const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') + if (!cloudIdValidation.isValid) { + return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) + } + + const projectIdValidation = validateAlphanumericId(projectId, 'projectId', 100) + if (!projectIdValidation.isValid) { + return NextResponse.json({ error: projectIdValidation.error }, { status: 400 }) + } + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue` logger.info('Creating Jira issue at:', url) - // Construct fields object with only the necessary fields const fields: Record = { project: { id: projectId, @@ -64,7 +71,6 @@ export async function POST(request: Request) { summary: summary, } - // Only add description if it exists if (description) { fields.description = { type: 'doc', @@ -83,19 +89,16 @@ export async function POST(request: Request) { } } - // Only add parent if it exists if (parent) { fields.parent = parent } - // Only add priority if it exists if (priority) { fields.priority = { name: priority, } } - // Only add assignee if it exists if (assignee) { fields.assignee = { id: assignee, @@ -104,7 +107,6 @@ export async function POST(request: Request) { const body = { fields } - // Make the request to Jira API const response = await fetch(url, { method: 'POST', headers: { diff --git a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts index c898e03383..82f90022b2 100644 --- a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts +++ b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateMicrosoftGraphId } from '@/lib/security/input-validation' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { PlannerTask } from '@/tools/microsoft_planner/types' @@ -35,7 +36,12 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Plan ID is required' }, { status: 400 }) } - // Get the credential from the database + const planIdValidation = validateMicrosoftGraphId(planId, 'planId') + if (!planIdValidation.isValid) { + logger.error(`[${requestId}] Invalid planId: ${planIdValidation.error}`) + return NextResponse.json({ error: planIdValidation.error }, { status: 400 }) + } + const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { @@ -45,7 +51,6 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Check if the credential belongs to the user if (credential.userId !== session.user.id) { logger.warn(`[${requestId}] Unauthorized credential access attempt`, { credentialUserId: credential.userId, @@ -54,7 +59,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) } - // Refresh access token if needed const accessToken = await refreshAccessTokenIfNeeded(credentialId, session.user.id, requestId) if (!accessToken) { @@ -62,7 +66,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Fetch tasks directly from Microsoft Graph API const response = await fetch(`https://graph.microsoft.com/v1.0/planner/plans/${planId}/tasks`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -81,7 +84,6 @@ export async function GET(request: NextRequest) { const data = await response.json() const tasks = data.value || [] - // Filter tasks to only include useful fields (matching our read_task tool) const filteredTasks = tasks.map((task: PlannerTask) => ({ id: task.id, title: task.title, diff --git a/apps/sim/app/api/tools/onedrive/folder/route.ts b/apps/sim/app/api/tools/onedrive/folder/route.ts index 49d2befb6a..503c1fbf68 100644 --- a/apps/sim/app/api/tools/onedrive/folder/route.ts +++ b/apps/sim/app/api/tools/onedrive/folder/route.ts @@ -5,15 +5,13 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateMicrosoftGraphId } from '@/lib/security/input-validation' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('OneDriveFolderAPI') -/** - * Get a single folder from Microsoft OneDrive - */ export async function GET(request: NextRequest) { const requestId = randomUUID().slice(0, 8) @@ -31,6 +29,11 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and File ID are required' }, { status: 400 }) } + const fileIdValidation = validateMicrosoftGraphId(fileId, 'fileId') + if (!fileIdValidation.isValid) { + return NextResponse.json({ error: fileIdValidation.error }, { status: 400 }) + } + const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) @@ -65,7 +68,6 @@ export async function GET(request: NextRequest) { const folder = await response.json() - // Transform the response to match expected format const transformedFolder = { id: folder.id, name: folder.name, diff --git a/apps/sim/app/api/tools/sharepoint/site/route.ts b/apps/sim/app/api/tools/sharepoint/site/route.ts index a31a8ed94e..803ba2c820 100644 --- a/apps/sim/app/api/tools/sharepoint/site/route.ts +++ b/apps/sim/app/api/tools/sharepoint/site/route.ts @@ -5,15 +5,13 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateMicrosoftGraphId } from '@/lib/security/input-validation' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('SharePointSiteAPI') -/** - * Get a single SharePoint site from Microsoft Graph API - */ export async function GET(request: NextRequest) { const requestId = randomUUID().slice(0, 8) @@ -31,6 +29,11 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and Site ID are required' }, { status: 400 }) } + const siteIdValidation = validateMicrosoftGraphId(siteId, 'siteId') + if (!siteIdValidation.isValid) { + return NextResponse.json({ error: siteIdValidation.error }, { status: 400 }) + } + const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) @@ -46,24 +49,14 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Handle different ways to access SharePoint sites: - // 1. Site ID: sites/{site-id} - // 2. Root site: sites/root - // 3. Hostname: sites/{hostname} - // 4. Server-relative URL: sites/{hostname}:/{server-relative-path} - // 5. Group team site: groups/{group-id}/sites/root - let endpoint: string if (siteId === 'root') { endpoint = 'sites/root' } else if (siteId.includes(':')) { - // Server-relative URL format endpoint = `sites/${siteId}` } else if (siteId.includes('groups/')) { - // Group team site format endpoint = siteId } else { - // Standard site ID or hostname endpoint = `sites/${siteId}` } @@ -86,7 +79,6 @@ export async function GET(request: NextRequest) { const site = await response.json() - // Transform the response to match expected format const transformedSite = { id: site.id, name: site.displayName || site.name, diff --git a/apps/sim/app/api/tools/wealthbox/item/route.ts b/apps/sim/app/api/tools/wealthbox/item/route.ts index a57d405441..57ecac5baf 100644 --- a/apps/sim/app/api/tools/wealthbox/item/route.ts +++ b/apps/sim/app/api/tools/wealthbox/item/route.ts @@ -4,6 +4,7 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { createLogger } from '@/lib/logs/console/logger' +import { validateEnum, validatePathSegment } from '@/lib/security/input-validation' import { generateRequestId } from '@/lib/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' @@ -11,23 +12,17 @@ export const dynamic = 'force-dynamic' const logger = createLogger('WealthboxItemAPI') -/** - * Get a single item (note, contact, task) from Wealthbox - */ export async function GET(request: NextRequest) { const requestId = generateRequestId() try { - // Get the session const session = await getSession() - // Check if the user is authenticated if (!session?.user?.id) { logger.warn(`[${requestId}] Unauthenticated request rejected`) return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) } - // Get parameters from query const { searchParams } = new URL(request.url) const credentialId = searchParams.get('credentialId') const itemId = searchParams.get('itemId') @@ -38,13 +33,37 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Credential ID and Item ID are required' }, { status: 400 }) } - // Validate item type - if (!['note', 'contact', 'task'].includes(type)) { + const ALLOWED_TYPES = ['note', 'contact', 'task'] as const + const typeValidation = validateEnum(type, ALLOWED_TYPES, 'type') + if (!typeValidation.isValid) { logger.warn(`[${requestId}] Invalid item type: ${type}`) - return NextResponse.json({ error: 'Invalid item type' }, { status: 400 }) + return NextResponse.json({ error: typeValidation.error }, { status: 400 }) + } + + const itemIdValidation = validatePathSegment(itemId, { + paramName: 'itemId', + maxLength: 100, + allowHyphens: true, + allowUnderscores: true, + allowDots: false, + }) + if (!itemIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid itemId format: ${itemId}`) + return NextResponse.json({ error: itemIdValidation.error }, { status: 400 }) + } + + const credentialIdValidation = validatePathSegment(credentialId, { + paramName: 'credentialId', + maxLength: 100, + allowHyphens: true, + allowUnderscores: true, + allowDots: false, + }) + if (!credentialIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid credentialId format: ${credentialId}`) + return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) } - // Get the credential from the database const credentials = await db.select().from(account).where(eq(account.id, credentialId)).limit(1) if (!credentials.length) { @@ -54,7 +73,6 @@ export async function GET(request: NextRequest) { const credential = credentials[0] - // Check if the credential belongs to the user if (credential.userId !== session.user.id) { logger.warn(`[${requestId}] Unauthorized credential access attempt`, { credentialUserId: credential.userId, @@ -63,7 +81,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) } - // Refresh access token if needed const accessToken = await refreshAccessTokenIfNeeded(credentialId, session.user.id, requestId) if (!accessToken) { @@ -71,7 +88,6 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Determine the endpoint based on item type const endpoints = { note: 'notes', contact: 'contacts', @@ -81,7 +97,6 @@ export async function GET(request: NextRequest) { logger.info(`[${requestId}] Fetching ${type} ${itemId} from Wealthbox`) - // Make request to Wealthbox API const response = await fetch(`https://api.crmworkspace.com/v1/${endpoint}/${itemId}`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -112,7 +127,6 @@ export async function GET(request: NextRequest) { const data = await response.json() - // Transform the response to match our expected format const item = { id: data.id?.toString() || itemId, name: diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx index 9ecbc00426..aec9495f3c 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/chat/chat.tsx @@ -694,7 +694,13 @@ export function Chat({ chatMessage, setChatMessage }: ChatProps) {
{chatFiles.map((file) => { const isImage = file.type.startsWith('image/') - const previewUrl = isImage ? URL.createObjectURL(file.file) : null + let previewUrl: string | null = null + if (isImage) { + const blobUrl = URL.createObjectURL(file.file) + if (blobUrl.startsWith('blob:')) { + previewUrl = blobUrl + } + } const getFileIcon = (type: string) => { if (type.includes('pdf')) return diff --git a/apps/sim/blocks/blocks/confluence.ts b/apps/sim/blocks/blocks/confluence.ts index 9f98983b84..dcb040f4e3 100644 --- a/apps/sim/blocks/blocks/confluence.ts +++ b/apps/sim/blocks/blocks/confluence.ts @@ -14,7 +14,6 @@ export const ConfluenceBlock: BlockConfig = { bgColor: '#E0E0E0', icon: ConfluenceIcon, subBlocks: [ - // Operation selector { id: 'operation', title: 'Operation', @@ -50,7 +49,6 @@ export const ConfluenceBlock: BlockConfig = { placeholder: 'Select Confluence account', required: true, }, - // Page selector (basic mode) { id: 'pageId', title: 'Select Page', @@ -63,7 +61,6 @@ export const ConfluenceBlock: BlockConfig = { dependsOn: ['credential', 'domain'], mode: 'basic', }, - // Manual page ID input (advanced mode) { id: 'manualPageId', title: 'Page ID', @@ -73,7 +70,6 @@ export const ConfluenceBlock: BlockConfig = { placeholder: 'Enter Confluence page ID', mode: 'advanced', }, - // Update page fields { id: 'title', title: 'New Title', @@ -107,7 +103,6 @@ export const ConfluenceBlock: BlockConfig = { params: (params) => { const { credential, pageId, manualPageId, ...rest } = params - // Use the selected page ID or the manually entered one const effectivePageId = (pageId || manualPageId || '').trim() if (!effectivePageId) { @@ -128,7 +123,6 @@ export const ConfluenceBlock: BlockConfig = { credential: { type: 'string', description: 'Confluence access token' }, pageId: { type: 'string', description: 'Page identifier' }, manualPageId: { type: 'string', description: 'Manual page identifier' }, - // Update operation inputs title: { type: 'string', description: 'New page title' }, content: { type: 'string', description: 'New page content' }, }, diff --git a/apps/sim/blocks/blocks/google_docs.ts b/apps/sim/blocks/blocks/google_docs.ts index f023bdfe30..ca7ec792ab 100644 --- a/apps/sim/blocks/blocks/google_docs.ts +++ b/apps/sim/blocks/blocks/google_docs.ts @@ -145,7 +145,6 @@ export const GoogleDocsBlock: BlockConfig = { const { credential, documentId, manualDocumentId, folderSelector, folderId, ...rest } = params - // Handle both selector and manual inputs const effectiveDocumentId = (documentId || manualDocumentId || '').trim() const effectiveFolderId = (folderSelector || folderId || '').trim() diff --git a/apps/sim/blocks/blocks/google_sheets.ts b/apps/sim/blocks/blocks/google_sheets.ts index 51dff49095..5b2fe61bcb 100644 --- a/apps/sim/blocks/blocks/google_sheets.ts +++ b/apps/sim/blocks/blocks/google_sheets.ts @@ -180,7 +180,6 @@ export const GoogleSheetsBlock: BlockConfig = { const parsedValues = values ? JSON.parse(values as string) : undefined - // Handle both selector and manual input const effectiveSpreadsheetId = (spreadsheetId || manualSpreadsheetId || '').trim() if (!effectiveSpreadsheetId) { diff --git a/apps/sim/blocks/blocks/linear.ts b/apps/sim/blocks/blocks/linear.ts index f64c446494..86d575834c 100644 --- a/apps/sim/blocks/blocks/linear.ts +++ b/apps/sim/blocks/blocks/linear.ts @@ -1,10 +1,8 @@ import { LinearIcon } from '@/components/icons' -import type { BlockConfig, BlockIcon } from '@/blocks/types' +import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' import type { LinearResponse } from '@/tools/linear/types' -const LinearBlockIcon: BlockIcon = (props) => LinearIcon(props as any) - export const LinearBlock: BlockConfig = { type: 'linear', name: 'Linear', @@ -12,7 +10,7 @@ export const LinearBlock: BlockConfig = { authMode: AuthMode.OAuth, longDescription: 'Integrate Linear into the workflow. Can read and create issues.', category: 'tools', - icon: LinearBlockIcon, + icon: LinearIcon, bgColor: '#5E6AD2', subBlocks: [ { diff --git a/apps/sim/blocks/blocks/microsoft_excel.ts b/apps/sim/blocks/blocks/microsoft_excel.ts index 3f9a5ca692..5e0aebf605 100644 --- a/apps/sim/blocks/blocks/microsoft_excel.ts +++ b/apps/sim/blocks/blocks/microsoft_excel.ts @@ -151,10 +151,8 @@ export const MicrosoftExcelBlock: BlockConfig = { const { credential, values, spreadsheetId, manualSpreadsheetId, tableName, ...rest } = params - // Handle both selector and manual input const effectiveSpreadsheetId = (spreadsheetId || manualSpreadsheetId || '').trim() - // Parse values from JSON string to array if it exists let parsedValues try { parsedValues = values ? JSON.parse(values as string) : undefined @@ -166,7 +164,6 @@ export const MicrosoftExcelBlock: BlockConfig = { throw new Error('Spreadsheet ID is required.') } - // For table operations, ensure tableName is provided if (params.operation === 'table_add' && !tableName) { throw new Error('Table name is required for table operations.') } @@ -178,7 +175,6 @@ export const MicrosoftExcelBlock: BlockConfig = { credential, } - // Add table-specific parameters if (params.operation === 'table_add') { return { ...baseParams, diff --git a/apps/sim/blocks/blocks/microsoft_teams.ts b/apps/sim/blocks/blocks/microsoft_teams.ts index 8944a7225a..cafbaf4a95 100644 --- a/apps/sim/blocks/blocks/microsoft_teams.ts +++ b/apps/sim/blocks/blocks/microsoft_teams.ts @@ -127,7 +127,6 @@ export const MicrosoftTeamsBlock: BlockConfig = { mode: 'advanced', condition: { field: 'operation', value: ['read_channel', 'write_channel'] }, }, - // Create-specific Fields { id: 'content', title: 'Message', @@ -181,7 +180,6 @@ export const MicrosoftTeamsBlock: BlockConfig = { ...rest } = params - // Use the selected IDs or the manually entered ones const effectiveTeamId = (teamId || manualTeamId || '').trim() const effectiveChatId = (chatId || manualChatId || '').trim() const effectiveChannelId = (channelId || manualChannelId || '').trim() @@ -192,7 +190,6 @@ export const MicrosoftTeamsBlock: BlockConfig = { } if (operation === 'read_chat' || operation === 'write_chat') { - // Don't pass empty chatId - let the tool handle the error if (!effectiveChatId) { throw new Error('Chat ID is required. Please select a chat or enter a chat ID.') } @@ -226,14 +223,12 @@ export const MicrosoftTeamsBlock: BlockConfig = { content: { type: 'string', description: 'Message content' }, }, outputs: { - // Read operation outputs content: { type: 'string', description: 'Formatted message content from chat/channel' }, metadata: { type: 'json', description: 'Message metadata with full details' }, messageCount: { type: 'number', description: 'Number of messages retrieved' }, messages: { type: 'json', description: 'Array of message objects' }, totalAttachments: { type: 'number', description: 'Total number of attachments' }, attachmentTypes: { type: 'json', description: 'Array of attachment content types' }, - // Write operation outputs updatedContent: { type: 'boolean', description: 'Whether content was successfully updated/sent', @@ -241,14 +236,12 @@ export const MicrosoftTeamsBlock: BlockConfig = { messageId: { type: 'string', description: 'ID of the created/sent message' }, createdTime: { type: 'string', description: 'Timestamp when message was created' }, url: { type: 'string', description: 'Web URL to the message' }, - // Individual message fields (from read operations) sender: { type: 'string', description: 'Message sender display name' }, messageTimestamp: { type: 'string', description: 'Individual message timestamp' }, messageType: { type: 'string', description: 'Type of message (message, systemEventMessage, etc.)', }, - // Trigger outputs type: { type: 'string', description: 'Type of Teams message' }, id: { type: 'string', description: 'Unique message identifier' }, timestamp: { type: 'string', description: 'Message timestamp' }, diff --git a/apps/sim/blocks/blocks/youtube.ts b/apps/sim/blocks/blocks/youtube.ts index 982f924c0b..145b1ff9d6 100644 --- a/apps/sim/blocks/blocks/youtube.ts +++ b/apps/sim/blocks/blocks/youtube.ts @@ -1,10 +1,8 @@ import { YouTubeIcon } from '@/components/icons' -import type { BlockConfig, BlockIcon } from '@/blocks/types' +import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' import type { YouTubeSearchResponse } from '@/tools/youtube/types' -const YouTubeBlockIcon: BlockIcon = (props) => YouTubeIcon(props as any) - export const YouTubeBlock: BlockConfig = { type: 'youtube', name: 'YouTube', @@ -14,7 +12,7 @@ export const YouTubeBlock: BlockConfig = { docsLink: 'https://docs.sim.ai/tools/youtube', category: 'tools', bgColor: '#FF0000', - icon: YouTubeBlockIcon, + icon: YouTubeIcon, subBlocks: [ { id: 'query', diff --git a/apps/sim/executor/resolver/resolver.ts b/apps/sim/executor/resolver/resolver.ts index bad316c65a..4bebd6d0fc 100644 --- a/apps/sim/executor/resolver/resolver.ts +++ b/apps/sim/executor/resolver/resolver.ts @@ -1,4 +1,3 @@ -import { BlockPathCalculator } from '@/lib/block-path-calculator' import { createLogger } from '@/lib/logs/console/logger' import { VariableManager } from '@/lib/variables/variable-manager' import { extractReferencePrefixes, SYSTEM_REFERENCE_PREFIXES } from '@/lib/workflows/references' @@ -11,16 +10,10 @@ import { normalizeBlockName } from '@/stores/workflows/utils' const logger = createLogger('InputResolver') -/** - * Helper function to resolve property access - */ function resolvePropertyAccess(obj: any, property: string): any { return obj[property] } -/** - * Resolves input values for blocks by handling references and variable substitution. - */ export class InputResolver { private blockById: Map private blockByNormalizedName: Map @@ -947,7 +940,12 @@ export class InputResolver { */ private stringifyForCondition(value: any): string { if (typeof value === 'string') { - return `"${value.replace(/"/g, '\\"').replace(/\n/g, '\\n')}"` + const sanitized = value + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + return `"${sanitized}"` } if (value === null) { return 'null' @@ -1098,45 +1096,6 @@ export class InputResolver { return accessibleBlocks } - /** - * Gets block names that the current block can reference for helpful error messages. - * Uses shared utility when pre-calculated data is available. - * - * @param currentBlockId - ID of the block requesting references - * @returns Array of accessible block names and aliases - */ - private getAccessibleBlockNames(currentBlockId: string): string[] { - // Use shared utility if pre-calculated data is available - if (this.accessibleBlocksMap) { - return BlockPathCalculator.getAccessibleBlockNames( - currentBlockId, - this.workflow, - this.accessibleBlocksMap - ) - } - - // Fallback to legacy calculation - const accessibleBlockIds = this.getAccessibleBlocks(currentBlockId) - const names: string[] = [] - - for (const blockId of accessibleBlockIds) { - const block = this.blockById.get(blockId) - if (block) { - // Add both the actual name and the normalized name - if (block.metadata?.name) { - names.push(block.metadata.name) - names.push(this.normalizeBlockName(block.metadata.name)) - } - names.push(blockId) - } - } - - // Add special aliases - names.push('start') // Always allow start alias - - return [...new Set(names)] // Remove duplicates - } - /** * Gets user-friendly block names for error messages. * Only returns the actual block names that users see in the UI. diff --git a/apps/sim/lib/copilot/tools/server/other/make-api-request.ts b/apps/sim/lib/copilot/tools/server/other/make-api-request.ts index 546151f526..2491907797 100644 --- a/apps/sim/lib/copilot/tools/server/other/make-api-request.ts +++ b/apps/sim/lib/copilot/tools/server/other/make-api-request.ts @@ -48,14 +48,21 @@ export const makeApiRequestServerTool: BaseServerTool return String(val) } } + const stripHtml = (html: string): string => { try { - return html - .replace(//gi, '') - .replace(//gi, '') - .replace(/<[^>]+>/g, ' ') - .replace(/\s+/g, ' ') - .trim() + let text = html + let previous: string + + do { + previous = text + text = text.replace(//gi, '') + text = text.replace(//gi, '') + text = text.replace(/<[^>]*>/g, ' ') + text = text.replace(/[<>]/g, ' ') + } while (text !== previous) + + return text.replace(/\s+/g, ' ').trim() } catch { return html } diff --git a/apps/sim/lib/security/input-validation.test.ts b/apps/sim/lib/security/input-validation.test.ts new file mode 100644 index 0000000000..b34bd462dd --- /dev/null +++ b/apps/sim/lib/security/input-validation.test.ts @@ -0,0 +1,590 @@ +import { describe, expect, it } from 'vitest' +import { + sanitizeForLogging, + validateAlphanumericId, + validateEnum, + validateFileExtension, + validateHostname, + validateNumericId, + validatePathSegment, + validateUUID, +} from './input-validation' + +describe('validatePathSegment', () => { + describe('valid inputs', () => { + it.concurrent('should accept alphanumeric strings', () => { + const result = validatePathSegment('abc123') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('abc123') + }) + + it.concurrent('should accept strings with hyphens', () => { + const result = validatePathSegment('test-item-123') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept strings with underscores', () => { + const result = validatePathSegment('test_item_123') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept strings with hyphens and underscores', () => { + const result = validatePathSegment('test-item_123') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept dots when allowDots is true', () => { + const result = validatePathSegment('file.name.txt', { allowDots: true }) + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept custom patterns', () => { + const result = validatePathSegment('v1.2.3', { + customPattern: /^v\d+\.\d+\.\d+$/, + }) + expect(result.isValid).toBe(true) + }) + }) + + describe('invalid inputs - null/empty', () => { + it.concurrent('should reject null', () => { + const result = validatePathSegment(null) + expect(result.isValid).toBe(false) + expect(result.error).toContain('required') + }) + + it.concurrent('should reject undefined', () => { + const result = validatePathSegment(undefined) + expect(result.isValid).toBe(false) + expect(result.error).toContain('required') + }) + + it.concurrent('should reject empty string', () => { + const result = validatePathSegment('') + expect(result.isValid).toBe(false) + expect(result.error).toContain('required') + }) + }) + + describe('invalid inputs - path traversal', () => { + it.concurrent('should reject path traversal with ../', () => { + const result = validatePathSegment('../etc/passwd') + expect(result.isValid).toBe(false) + expect(result.error).toContain('path traversal') + }) + + it.concurrent('should reject path traversal with ..\\', () => { + const result = validatePathSegment('..\\windows\\system32') + expect(result.isValid).toBe(false) + expect(result.error).toContain('path traversal') + }) + + it.concurrent('should reject URL-encoded path traversal %2e%2e', () => { + const result = validatePathSegment('%2e%2e%2f') + expect(result.isValid).toBe(false) + expect(result.error).toContain('path traversal') + }) + + it.concurrent('should reject double URL-encoded path traversal', () => { + const result = validatePathSegment('%252e%252e') + expect(result.isValid).toBe(false) + expect(result.error).toContain('path traversal') + }) + + it.concurrent('should reject mixed case path traversal attempts', () => { + const result = validatePathSegment('..%2F') + expect(result.isValid).toBe(false) + expect(result.error).toContain('path traversal') + }) + + it.concurrent('should reject dots in path by default', () => { + const result = validatePathSegment('..') + expect(result.isValid).toBe(false) + }) + }) + + describe('invalid inputs - directory separators', () => { + it.concurrent('should reject forward slashes', () => { + const result = validatePathSegment('path/to/file') + expect(result.isValid).toBe(false) + expect(result.error).toContain('directory separator') + }) + + it.concurrent('should reject backslashes', () => { + const result = validatePathSegment('path\\to\\file') + expect(result.isValid).toBe(false) + expect(result.error).toContain('directory separator') + }) + }) + + describe('invalid inputs - null bytes', () => { + it.concurrent('should reject null bytes', () => { + const result = validatePathSegment('file\0name') + expect(result.isValid).toBe(false) + expect(result.error).toContain('invalid characters') + }) + + it.concurrent('should reject URL-encoded null bytes', () => { + const result = validatePathSegment('file%00name') + expect(result.isValid).toBe(false) + expect(result.error).toContain('invalid characters') + }) + }) + + describe('invalid inputs - special characters', () => { + it.concurrent('should reject special characters by default', () => { + const result = validatePathSegment('file@name') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject dots by default', () => { + const result = validatePathSegment('file.txt') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject spaces', () => { + const result = validatePathSegment('file name') + expect(result.isValid).toBe(false) + }) + }) + + describe('options', () => { + it.concurrent('should reject strings exceeding maxLength', () => { + const longString = 'a'.repeat(300) + const result = validatePathSegment(longString, { maxLength: 255 }) + expect(result.isValid).toBe(false) + expect(result.error).toContain('exceeds maximum length') + }) + + it.concurrent('should use custom param name in errors', () => { + const result = validatePathSegment('', { paramName: 'itemId' }) + expect(result.isValid).toBe(false) + expect(result.error).toContain('itemId') + }) + + it.concurrent('should reject hyphens when allowHyphens is false', () => { + const result = validatePathSegment('test-item', { allowHyphens: false }) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject underscores when allowUnderscores is false', () => { + const result = validatePathSegment('test_item', { + allowUnderscores: false, + }) + expect(result.isValid).toBe(false) + }) + }) + + describe('custom patterns', () => { + it.concurrent('should validate against custom pattern', () => { + const result = validatePathSegment('ABC-123', { + customPattern: /^[A-Z]{3}-\d{3}$/, + }) + expect(result.isValid).toBe(true) + }) + + it.concurrent('should reject when custom pattern does not match', () => { + const result = validatePathSegment('ABC123', { + customPattern: /^[A-Z]{3}-\d{3}$/, + }) + expect(result.isValid).toBe(false) + }) + }) +}) + +describe('validateUUID', () => { + describe('valid UUIDs', () => { + it.concurrent('should accept valid UUID v4', () => { + const result = validateUUID('550e8400-e29b-41d4-a716-446655440000') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept UUID with uppercase letters', () => { + const result = validateUUID('550E8400-E29B-41D4-A716-446655440000') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('550e8400-e29b-41d4-a716-446655440000') + }) + + it.concurrent('should normalize UUID to lowercase', () => { + const result = validateUUID('550E8400-E29B-41D4-A716-446655440000') + expect(result.sanitized).toBe('550e8400-e29b-41d4-a716-446655440000') + }) + }) + + describe('invalid UUIDs', () => { + it.concurrent('should reject non-UUID strings', () => { + const result = validateUUID('not-a-uuid') + expect(result.isValid).toBe(false) + expect(result.error).toContain('valid UUID') + }) + + it.concurrent('should reject UUID with wrong version', () => { + const result = validateUUID('550e8400-e29b-31d4-a716-446655440000') // version 3 + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject UUID with wrong variant', () => { + const result = validateUUID('550e8400-e29b-41d4-1716-446655440000') // wrong variant + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateUUID('') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject null', () => { + const result = validateUUID(null) + expect(result.isValid).toBe(false) + }) + }) +}) + +describe('validateAlphanumericId', () => { + it.concurrent('should accept alphanumeric IDs', () => { + const result = validateAlphanumericId('user123') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept IDs with hyphens and underscores', () => { + const result = validateAlphanumericId('user-id_123') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should reject IDs with special characters', () => { + const result = validateAlphanumericId('user@123') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IDs exceeding maxLength', () => { + const longId = 'a'.repeat(150) + const result = validateAlphanumericId(longId, 'userId', 100) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should use custom param name in errors', () => { + const result = validateAlphanumericId('', 'customId') + expect(result.error).toContain('customId') + }) +}) + +describe('validateNumericId', () => { + describe('valid numeric IDs', () => { + it.concurrent('should accept numeric strings', () => { + const result = validateNumericId('123') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('123') + }) + + it.concurrent('should accept numbers', () => { + const result = validateNumericId(456) + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('456') + }) + + it.concurrent('should accept zero', () => { + const result = validateNumericId(0) + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept negative numbers', () => { + const result = validateNumericId(-5) + expect(result.isValid).toBe(true) + }) + }) + + describe('invalid numeric IDs', () => { + it.concurrent('should reject non-numeric strings', () => { + const result = validateNumericId('abc') + expect(result.isValid).toBe(false) + expect(result.error).toContain('valid number') + }) + + it.concurrent('should reject null', () => { + const result = validateNumericId(null) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateNumericId('') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject NaN', () => { + const result = validateNumericId(Number.NaN) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject Infinity', () => { + const result = validateNumericId(Number.POSITIVE_INFINITY) + expect(result.isValid).toBe(false) + }) + }) + + describe('min/max constraints', () => { + it.concurrent('should accept values within range', () => { + const result = validateNumericId(50, 'value', { min: 1, max: 100 }) + expect(result.isValid).toBe(true) + }) + + it.concurrent('should reject values below min', () => { + const result = validateNumericId(0, 'value', { min: 1 }) + expect(result.isValid).toBe(false) + expect(result.error).toContain('at least 1') + }) + + it.concurrent('should reject values above max', () => { + const result = validateNumericId(101, 'value', { max: 100 }) + expect(result.isValid).toBe(false) + expect(result.error).toContain('at most 100') + }) + + it.concurrent('should accept value equal to min', () => { + const result = validateNumericId(1, 'value', { min: 1 }) + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept value equal to max', () => { + const result = validateNumericId(100, 'value', { max: 100 }) + expect(result.isValid).toBe(true) + }) + }) +}) + +describe('validateEnum', () => { + const allowedTypes = ['note', 'contact', 'task'] as const + + describe('valid enum values', () => { + it.concurrent('should accept values in the allowed list', () => { + const result = validateEnum('note', allowedTypes, 'type') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('note') + }) + + it.concurrent('should accept all values in the list', () => { + for (const type of allowedTypes) { + const result = validateEnum(type, allowedTypes) + expect(result.isValid).toBe(true) + } + }) + }) + + describe('invalid enum values', () => { + it.concurrent('should reject values not in the allowed list', () => { + const result = validateEnum('invalid', allowedTypes, 'type') + expect(result.isValid).toBe(false) + expect(result.error).toContain('note, contact, task') + }) + + it.concurrent('should reject case-mismatched values', () => { + const result = validateEnum('Note', allowedTypes, 'type') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject null', () => { + const result = validateEnum(null, allowedTypes) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateEnum('', allowedTypes) + expect(result.isValid).toBe(false) + }) + }) + + describe('error messages', () => { + it.concurrent('should include param name in error', () => { + const result = validateEnum('invalid', allowedTypes, 'itemType') + expect(result.error).toContain('itemType') + }) + + it.concurrent('should list all allowed values in error', () => { + const result = validateEnum('invalid', allowedTypes) + expect(result.error).toContain('note') + expect(result.error).toContain('contact') + expect(result.error).toContain('task') + }) + }) +}) + +describe('validateHostname', () => { + describe('valid hostnames', () => { + it.concurrent('should accept valid domain names', () => { + const result = validateHostname('example.com') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept subdomains', () => { + const result = validateHostname('api.example.com') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept domains with hyphens', () => { + const result = validateHostname('my-domain.com') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept multi-level domains', () => { + const result = validateHostname('api.v2.example.co.uk') + expect(result.isValid).toBe(true) + }) + }) + + describe('invalid hostnames - private IPs', () => { + it.concurrent('should reject localhost', () => { + const result = validateHostname('localhost') + expect(result.isValid).toBe(false) + expect(result.error).toContain('private IP') + }) + + it.concurrent('should reject 127.0.0.1', () => { + const result = validateHostname('127.0.0.1') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject 10.x.x.x private range', () => { + const result = validateHostname('10.0.0.1') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject 192.168.x.x private range', () => { + const result = validateHostname('192.168.1.1') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject 172.16-31.x.x private range', () => { + const result = validateHostname('172.16.0.1') + expect(result.isValid).toBe(false) + const result2 = validateHostname('172.31.255.255') + expect(result2.isValid).toBe(false) + }) + + it.concurrent('should reject link-local addresses', () => { + const result = validateHostname('169.254.169.254') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IPv6 loopback', () => { + const result = validateHostname('::1') + expect(result.isValid).toBe(false) + }) + }) + + describe('invalid hostnames - format', () => { + it.concurrent('should reject invalid characters', () => { + const result = validateHostname('example_domain.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject hostnames starting with hyphen', () => { + const result = validateHostname('-example.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject hostnames ending with hyphen', () => { + const result = validateHostname('example-.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateHostname('') + expect(result.isValid).toBe(false) + }) + }) +}) + +describe('validateFileExtension', () => { + const allowedExtensions = ['jpg', 'png', 'gif', 'pdf'] as const + + describe('valid extensions', () => { + it.concurrent('should accept allowed extensions', () => { + const result = validateFileExtension('jpg', allowedExtensions) + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('jpg') + }) + + it.concurrent('should accept extensions with leading dot', () => { + const result = validateFileExtension('.png', allowedExtensions) + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('png') + }) + + it.concurrent('should normalize to lowercase', () => { + const result = validateFileExtension('JPG', allowedExtensions) + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('jpg') + }) + + it.concurrent('should accept all allowed extensions', () => { + for (const ext of allowedExtensions) { + const result = validateFileExtension(ext, allowedExtensions) + expect(result.isValid).toBe(true) + } + }) + }) + + describe('invalid extensions', () => { + it.concurrent('should reject extensions not in allowed list', () => { + const result = validateFileExtension('exe', allowedExtensions) + expect(result.isValid).toBe(false) + expect(result.error).toContain('jpg, png, gif, pdf') + }) + + it.concurrent('should reject empty string', () => { + const result = validateFileExtension('', allowedExtensions) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject null', () => { + const result = validateFileExtension(null, allowedExtensions) + expect(result.isValid).toBe(false) + }) + }) +}) + +describe('sanitizeForLogging', () => { + it.concurrent('should truncate long strings', () => { + const longString = 'a'.repeat(200) + const result = sanitizeForLogging(longString, 50) + expect(result.length).toBe(50) + }) + + it.concurrent('should mask Bearer tokens', () => { + const input = 'Authorization: Bearer abc123xyz' + const result = sanitizeForLogging(input) + expect(result).toContain('[REDACTED]') + expect(result).not.toContain('abc123xyz') + }) + + it.concurrent('should mask password fields', () => { + const input = 'password: "secret123"' + const result = sanitizeForLogging(input) + expect(result).toContain('[REDACTED]') + expect(result).not.toContain('secret123') + }) + + it.concurrent('should mask token fields', () => { + const input = 'token: "tokenvalue"' + const result = sanitizeForLogging(input) + expect(result).toContain('[REDACTED]') + expect(result).not.toContain('tokenvalue') + }) + + it.concurrent('should mask API keys', () => { + const input = 'api_key: "key123"' + const result = sanitizeForLogging(input) + expect(result).toContain('[REDACTED]') + expect(result).not.toContain('key123') + }) + + it.concurrent('should handle empty strings', () => { + const result = sanitizeForLogging('') + expect(result).toBe('') + }) + + it.concurrent('should not modify safe strings', () => { + const input = 'This is a safe string' + const result = sanitizeForLogging(input) + expect(result).toBe(input) + }) +}) diff --git a/apps/sim/lib/security/input-validation.ts b/apps/sim/lib/security/input-validation.ts new file mode 100644 index 0000000000..35e1b9afa8 --- /dev/null +++ b/apps/sim/lib/security/input-validation.ts @@ -0,0 +1,778 @@ +import { createLogger } from '@/lib/logs/console/logger' + +const logger = createLogger('InputValidation') + +/** + * Result type for validation functions + */ +export interface ValidationResult { + isValid: boolean + error?: string + sanitized?: string +} + +/** + * Options for path segment validation + */ +export interface PathSegmentOptions { + /** Name of the parameter for error messages */ + paramName?: string + /** Maximum length allowed (default: 255) */ + maxLength?: number + /** Allow hyphens (default: true) */ + allowHyphens?: boolean + /** Allow underscores (default: true) */ + allowUnderscores?: boolean + /** Allow dots (default: false, to prevent directory traversal) */ + allowDots?: boolean + /** Custom regex pattern to match */ + customPattern?: RegExp +} + +/** + * Validates a path segment to prevent path traversal and SSRF attacks + * + * This function ensures that user-provided input used in URL paths or file paths + * cannot be used for directory traversal attacks or SSRF. + * + * Default behavior: + * - Allows: letters (a-z, A-Z), numbers (0-9), hyphens (-), underscores (_) + * - Blocks: dots (.), slashes (/, \), null bytes, URL encoding, and special characters + * + * @param value - The path segment to validate + * @param options - Validation options + * @returns ValidationResult with isValid flag and optional error message + * + * @example + * ```typescript + * const result = validatePathSegment(itemId, { paramName: 'itemId' }) + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validatePathSegment( + value: string | null | undefined, + options: PathSegmentOptions = {} +): ValidationResult { + const { + paramName = 'path segment', + maxLength = 255, + allowHyphens = true, + allowUnderscores = true, + allowDots = false, + customPattern, + } = options + + // Check for null/undefined + if (value === null || value === undefined || value === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + // Check length + if (value.length > maxLength) { + logger.warn('Path segment exceeds maximum length', { + paramName, + length: value.length, + maxLength, + }) + return { + isValid: false, + error: `${paramName} exceeds maximum length of ${maxLength} characters`, + } + } + + // Check for null bytes (potential for bypass attacks) + if (value.includes('\0') || value.includes('%00')) { + logger.warn('Path segment contains null bytes', { paramName }) + return { + isValid: false, + error: `${paramName} contains invalid characters`, + } + } + + // Check for path traversal patterns + const pathTraversalPatterns = [ + '..', + './', + '.\\.', // Windows path traversal + '%2e%2e', // URL encoded .. + '%252e%252e', // Double URL encoded .. + '..%2f', + '..%5c', + '%2e%2e%2f', + '%2e%2e/', + '..%252f', + ] + + const lowerValue = value.toLowerCase() + for (const pattern of pathTraversalPatterns) { + if (lowerValue.includes(pattern.toLowerCase())) { + logger.warn('Path traversal attempt detected', { + paramName, + pattern, + value: value.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} contains invalid path traversal sequences`, + } + } + } + + // Check for directory separators + if (value.includes('/') || value.includes('\\')) { + logger.warn('Path segment contains directory separators', { paramName }) + return { + isValid: false, + error: `${paramName} cannot contain directory separators`, + } + } + + // Use custom pattern if provided + if (customPattern) { + if (!customPattern.test(value)) { + logger.warn('Path segment failed custom pattern validation', { + paramName, + pattern: customPattern.toString(), + }) + return { + isValid: false, + error: `${paramName} format is invalid`, + } + } + return { isValid: true, sanitized: value } + } + + // Build allowed character pattern + let pattern = '^[a-zA-Z0-9' + if (allowHyphens) pattern += '\\-' + if (allowUnderscores) pattern += '_' + if (allowDots) pattern += '\\.' + pattern += ']+$' + + const regex = new RegExp(pattern) + + if (!regex.test(value)) { + logger.warn('Path segment contains disallowed characters', { + paramName, + value: value.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} can only contain alphanumeric characters${allowHyphens ? ', hyphens' : ''}${allowUnderscores ? ', underscores' : ''}${allowDots ? ', dots' : ''}`, + } + } + + return { isValid: true, sanitized: value } +} + +/** + * Validates a UUID (v4 format) + * + * @param value - The UUID to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateUUID(workflowId, 'workflowId') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateUUID( + value: string | null | undefined, + paramName = 'UUID' +): ValidationResult { + if (value === null || value === undefined || value === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + // UUID v4 pattern + const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + + if (!uuidPattern.test(value)) { + logger.warn('Invalid UUID format', { paramName, value: value.substring(0, 50) }) + return { + isValid: false, + error: `${paramName} must be a valid UUID`, + } + } + + return { isValid: true, sanitized: value.toLowerCase() } +} + +/** + * Validates an alphanumeric ID (letters, numbers, hyphens, underscores only) + * + * @param value - The ID to validate + * @param paramName - Name of the parameter for error messages + * @param maxLength - Maximum length (default: 100) + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateAlphanumericId(userId, 'userId') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateAlphanumericId( + value: string | null | undefined, + paramName = 'ID', + maxLength = 100 +): ValidationResult { + return validatePathSegment(value, { + paramName, + maxLength, + allowHyphens: true, + allowUnderscores: true, + allowDots: false, + }) +} + +/** + * Validates a numeric ID + * + * @param value - The ID to validate + * @param paramName - Name of the parameter for error messages + * @param options - Additional options (min, max) + * @returns ValidationResult with sanitized number as string + * + * @example + * ```typescript + * const result = validateNumericId(pageNumber, 'pageNumber', { min: 1, max: 1000 }) + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateNumericId( + value: string | number | null | undefined, + paramName = 'ID', + options: { min?: number; max?: number } = {} +): ValidationResult { + if (value === null || value === undefined || value === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + const num = typeof value === 'number' ? value : Number(value) + + if (Number.isNaN(num) || !Number.isFinite(num)) { + logger.warn('Invalid numeric ID', { paramName, value }) + return { + isValid: false, + error: `${paramName} must be a valid number`, + } + } + + if (options.min !== undefined && num < options.min) { + return { + isValid: false, + error: `${paramName} must be at least ${options.min}`, + } + } + + if (options.max !== undefined && num > options.max) { + return { + isValid: false, + error: `${paramName} must be at most ${options.max}`, + } + } + + return { isValid: true, sanitized: num.toString() } +} + +/** + * Validates that a value is in an allowed list (enum validation) + * + * @param value - The value to validate + * @param allowedValues - Array of allowed values + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateEnum(type, ['note', 'contact', 'task'], 'type') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateEnum( + value: string | null | undefined, + allowedValues: readonly T[], + paramName = 'value' +): ValidationResult { + if (value === null || value === undefined || value === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + if (!allowedValues.includes(value as T)) { + logger.warn('Value not in allowed list', { + paramName, + value, + allowedValues, + }) + return { + isValid: false, + error: `${paramName} must be one of: ${allowedValues.join(', ')}`, + } + } + + return { isValid: true, sanitized: value } +} + +/** + * Validates a hostname to prevent SSRF attacks + * + * This function checks that a hostname is not a private IP, localhost, or other reserved address. + * It complements the validateProxyUrl function by providing hostname-specific validation. + * + * @param hostname - The hostname to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateHostname(webhookDomain, 'webhook domain') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateHostname( + hostname: string | null | undefined, + paramName = 'hostname' +): ValidationResult { + if (hostname === null || hostname === undefined || hostname === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + // Import the blocked IP ranges from url-validation + const BLOCKED_IP_RANGES = [ + // Private IPv4 ranges (RFC 1918) + /^10\./, + /^172\.(1[6-9]|2[0-9]|3[01])\./, + /^192\.168\./, + + // Loopback addresses + /^127\./, + /^localhost$/i, + + // Link-local addresses (RFC 3927) + /^169\.254\./, + + // Cloud metadata endpoints + /^169\.254\.169\.254$/, + + // Broadcast and other reserved ranges + /^0\./, + /^224\./, + /^240\./, + /^255\./, + + // IPv6 loopback and link-local + /^::1$/, + /^fe80:/i, + /^::ffff:127\./i, + /^::ffff:10\./i, + /^::ffff:172\.(1[6-9]|2[0-9]|3[01])\./i, + /^::ffff:192\.168\./i, + ] + + const lowerHostname = hostname.toLowerCase() + + for (const pattern of BLOCKED_IP_RANGES) { + if (pattern.test(lowerHostname)) { + logger.warn('Hostname matches blocked IP range', { + paramName, + hostname: hostname.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} cannot be a private IP address or localhost`, + } + } + } + + // Basic hostname format validation + const hostnamePattern = + /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i + + if (!hostnamePattern.test(hostname)) { + logger.warn('Invalid hostname format', { + paramName, + hostname: hostname.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} is not a valid hostname`, + } + } + + return { isValid: true, sanitized: hostname } +} + +/** + * Validates a file extension + * + * @param extension - The file extension (with or without leading dot) + * @param allowedExtensions - Array of allowed extensions (without dots) + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateFileExtension(ext, ['jpg', 'png', 'gif'], 'file extension') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateFileExtension( + extension: string | null | undefined, + allowedExtensions: readonly string[], + paramName = 'file extension' +): ValidationResult { + if (extension === null || extension === undefined || extension === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + // Remove leading dot if present + const ext = extension.startsWith('.') ? extension.slice(1) : extension + + // Normalize to lowercase + const normalizedExt = ext.toLowerCase() + + if (!allowedExtensions.map((e) => e.toLowerCase()).includes(normalizedExt)) { + logger.warn('File extension not in allowed list', { + paramName, + extension: ext, + allowedExtensions, + }) + return { + isValid: false, + error: `${paramName} must be one of: ${allowedExtensions.join(', ')}`, + } + } + + return { isValid: true, sanitized: normalizedExt } +} + +/** + * Sanitizes a string for safe logging (removes potential sensitive data patterns) + * + * @param value - The value to sanitize + * @param maxLength - Maximum length to return (default: 100) + * @returns Sanitized string safe for logging + */ +export function sanitizeForLogging(value: string, maxLength = 100): string { + if (!value) return '' + + // Truncate long values + let sanitized = value.substring(0, maxLength) + + // Mask common sensitive patterns + sanitized = sanitized + .replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]') + .replace(/password['":\s]*['"]\w+['"]/gi, 'password: "[REDACTED]"') + .replace(/token['":\s]*['"]\w+['"]/gi, 'token: "[REDACTED]"') + .replace(/api[_-]?key['":\s]*['"]\w+['"]/gi, 'api_key: "[REDACTED]"') + + return sanitized +} + +/** + * Validates Microsoft Graph API resource IDs + * + * Microsoft Graph IDs can be complex - for example, SharePoint site IDs can include: + * - "root" (literal string) + * - GUIDs + * - Hostnames with colons and slashes (e.g., "hostname:/sites/sitename") + * - Group paths (e.g., "groups/{guid}/sites/root") + * + * This function allows these legitimate patterns while blocking path traversal. + * + * @param value - The Microsoft Graph ID to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateMicrosoftGraphId(siteId, 'siteId') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateMicrosoftGraphId( + value: string | null | undefined, + paramName = 'ID' +): ValidationResult { + if (value === null || value === undefined || value === '') { + return { + isValid: false, + error: `${paramName} is required`, + } + } + + // Check for path traversal patterns (../) + const pathTraversalPatterns = [ + '../', + '..\\', + '%2e%2e%2f', + '%2e%2e/', + '..%2f', + '%2e%2e%5c', + '%2e%2e\\', + '..%5c', + '%252e%252e%252f', // double encoded + ] + + const lowerValue = value.toLowerCase() + for (const pattern of pathTraversalPatterns) { + if (lowerValue.includes(pattern)) { + logger.warn('Path traversal attempt in Microsoft Graph ID', { + paramName, + value: value.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} contains invalid path traversal sequence`, + } + } + } + + // Check for control characters and null bytes + if (/[\x00-\x1f\x7f]/.test(value) || value.includes('%00')) { + logger.warn('Control characters in Microsoft Graph ID', { paramName }) + return { + isValid: false, + error: `${paramName} contains invalid control characters`, + } + } + + // Check for newlines (which could be used for header injection) + if (value.includes('\n') || value.includes('\r')) { + return { + isValid: false, + error: `${paramName} contains invalid newline characters`, + } + } + + // Microsoft Graph IDs can contain many characters, but not suspicious patterns + // We've blocked path traversal, so allow the rest + return { isValid: true, sanitized: value } +} + +/** + * Validates Jira Cloud IDs (typically UUID format) + * + * @param value - The Jira Cloud ID to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateJiraCloudId(cloudId, 'cloudId') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateJiraCloudId( + value: string | null | undefined, + paramName = 'cloudId' +): ValidationResult { + // Jira cloud IDs are alphanumeric with hyphens (UUID-like) + return validatePathSegment(value, { + paramName, + allowHyphens: true, + allowUnderscores: false, + allowDots: false, + maxLength: 100, + }) +} + +/** + * Validates Jira issue keys (format: PROJECT-123 or PROJECT-KEY-123) + * + * @param value - The Jira issue key to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateJiraIssueKey(issueKey, 'issueKey') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateJiraIssueKey( + value: string | null | undefined, + paramName = 'issueKey' +): ValidationResult { + // Jira issue keys: letters, numbers, hyphens (PROJECT-123 format) + return validatePathSegment(value, { + paramName, + allowHyphens: true, + allowUnderscores: false, + allowDots: false, + maxLength: 255, + }) +} + +/** + * Validates a URL to prevent SSRF attacks + * + * This function checks that URLs: + * - Use https:// protocol only + * - Do not point to private IP ranges or localhost + * - Do not use suspicious ports + * + * @param url - The URL to validate + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult + * + * @example + * ```typescript + * const result = validateExternalUrl(url, 'fileUrl') + * if (!result.isValid) { + * return NextResponse.json({ error: result.error }, { status: 400 }) + * } + * ``` + */ +export function validateExternalUrl( + url: string | null | undefined, + paramName = 'url' +): ValidationResult { + if (!url || typeof url !== 'string') { + return { + isValid: false, + error: `${paramName} is required and must be a string`, + } + } + + // Must be a valid URL + let parsedUrl: URL + try { + parsedUrl = new URL(url) + } catch { + return { + isValid: false, + error: `${paramName} must be a valid URL`, + } + } + + // Only allow https protocol + if (parsedUrl.protocol !== 'https:') { + return { + isValid: false, + error: `${paramName} must use https:// protocol`, + } + } + + // Block private IP ranges and localhost + const hostname = parsedUrl.hostname.toLowerCase() + + // Block localhost variations + if ( + hostname === 'localhost' || + hostname === '127.0.0.1' || + hostname === '::1' || + hostname.startsWith('127.') || + hostname === '0.0.0.0' + ) { + return { + isValid: false, + error: `${paramName} cannot point to localhost`, + } + } + + // Block private IP ranges + const privateIpPatterns = [ + /^10\./, + /^172\.(1[6-9]|2[0-9]|3[0-1])\./, + /^192\.168\./, + /^169\.254\./, // Link-local + /^fe80:/i, // IPv6 link-local + /^fc00:/i, // IPv6 unique local + /^fd00:/i, // IPv6 unique local + ] + + for (const pattern of privateIpPatterns) { + if (pattern.test(hostname)) { + return { + isValid: false, + error: `${paramName} cannot point to private IP addresses`, + } + } + } + + // Block suspicious ports commonly used for internal services + const port = parsedUrl.port + const blockedPorts = [ + '22', // SSH + '23', // Telnet + '25', // SMTP + '3306', // MySQL + '5432', // PostgreSQL + '6379', // Redis + '27017', // MongoDB + '9200', // Elasticsearch + ] + + if (port && blockedPorts.includes(port)) { + return { + isValid: false, + error: `${paramName} uses a blocked port`, + } + } + + return { isValid: true } +} + +/** + * Validates an image URL to prevent SSRF attacks + * Alias for validateExternalUrl for backward compatibility + */ +export function validateImageUrl( + url: string | null | undefined, + paramName = 'imageUrl' +): ValidationResult { + return validateExternalUrl(url, paramName) +} + +/** + * Validates a proxy URL to prevent SSRF attacks + * Alias for validateExternalUrl for backward compatibility + */ +export function validateProxyUrl( + url: string | null | undefined, + paramName = 'proxyUrl' +): ValidationResult { + return validateExternalUrl(url, paramName) +} diff --git a/apps/sim/lib/security/url-validation.test.ts b/apps/sim/lib/security/url-validation.test.ts deleted file mode 100644 index e65852aea6..0000000000 --- a/apps/sim/lib/security/url-validation.test.ts +++ /dev/null @@ -1,712 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { isPrivateHostname, validateImageUrl, validateProxyUrl } from './url-validation' - -describe('validateProxyUrl', () => { - describe('legitimate external APIs should pass', () => { - it.concurrent('should allow HTTPS APIs', () => { - const result = validateProxyUrl('https://api.openai.com/v1/chat/completions') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should allow HTTP APIs', () => { - const result = validateProxyUrl('http://api.example.com/data') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should allow various legitimate external APIs', () => { - const validUrls = [ - 'https://api.github.com/user', - 'https://graph.microsoft.com/v1.0/me', - 'https://api.notion.com/v1/databases', - 'https://api.airtable.com/v0/appXXX', - 'https://hooks.zapier.com/hooks/catch/123/abc', - 'https://discord.com/api/webhooks/123/abc', - 'https://api.twilio.com/2010-04-01/Accounts', - 'https://api.sendgrid.com/v3/mail/send', - 'https://api.stripe.com/v1/charges', - 'http://httpbin.org/get', - ] - - validUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('SSRF attacks should be blocked', () => { - it.concurrent('should block localhost addresses', () => { - const maliciousUrls = [ - 'http://localhost:3000/api/users', - 'http://127.0.0.1:8080/admin', - 'https://127.0.0.1/internal', - 'http://127.1:9999', - ] - - maliciousUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - }) - - it.concurrent('should block private IP ranges', () => { - const privateIps = [ - 'http://10.0.0.1/secret', - 'http://172.16.0.1:9999/admin', - 'http://192.168.1.1/config', - 'http://172.31.255.255/internal', - ] - - privateIps.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - }) - - it.concurrent('should block cloud metadata endpoints', () => { - const metadataUrls = [ - 'http://169.254.169.254/latest/meta-data/', - 'http://169.254.169.254/computeMetadata/v1/', - 'https://169.254.169.254/metadata/instance', - ] - - metadataUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - }) - - it.concurrent('should block dangerous protocols', () => { - const dangerousUrls = [ - 'file:///etc/passwd', - 'ftp://internal.server.com/files', - 'gopher://localhost:70/secret', - 'ldap://internal.ad.com/', - 'dict://localhost:2628/show:db', - 'data:text/html,', - ] - - dangerousUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toMatch(/Protocol .* is (not allowed|blocked)/) - }) - }) - - it.concurrent('should handle URL encoding bypass attempts', () => { - const encodedUrls = [ - 'http://127.0.0.1%2F@example.com/', - 'http://%31%32%37%2e%30%2e%30%2e%31/', // 127.0.0.1 encoded - 'http://localhost%2F@example.com/', - ] - - encodedUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - - it.concurrent('should reject invalid URL formats', () => { - const invalidUrls = ['not-a-url', 'http://', 'https://host..com', ''] - - invalidUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('Invalid') - }) - }) - }) - - describe('edge cases', () => { - it.concurrent('should handle mixed case protocols', () => { - const result = validateProxyUrl('HTTP://api.example.com') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should handle non-standard ports on external hosts', () => { - const result = validateProxyUrl('https://api.example.com:8443/webhook') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should block broadcast and reserved addresses', () => { - const reservedUrls = ['http://0.0.0.0:80', 'http://255.255.255.255', 'http://224.0.0.1'] - - reservedUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - }) -}) - -describe('validateImageUrl', () => { - it.concurrent('should pass standard proxy validation first', () => { - const result = validateImageUrl('http://localhost/image.jpg') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should allow legitimate image URLs', () => { - const validImageUrls = [ - 'https://cdn.example.com/images/photo.jpg', - 'https://storage.googleapis.com/bucket/image.png', - 'https://example.s3.amazonaws.com/images/avatar.webp', - ] - - validImageUrls.forEach((url) => { - const result = validateImageUrl(url) - expect(result.isValid).toBe(true) - }) - }) -}) - -describe('isPrivateHostname', () => { - it.concurrent('should identify private hostnames', () => { - expect(isPrivateHostname('127.0.0.1')).toBe(true) - expect(isPrivateHostname('10.0.0.1')).toBe(true) - expect(isPrivateHostname('192.168.1.1')).toBe(true) - expect(isPrivateHostname('localhost')).toBe(true) - }) - - it.concurrent('should not flag public hostnames', () => { - expect(isPrivateHostname('api.openai.com')).toBe(false) - expect(isPrivateHostname('8.8.8.8')).toBe(false) - expect(isPrivateHostname('github.com')).toBe(false) - }) -}) - -describe('Real-world API URL validation', () => { - describe('All production APIs used by the system should pass', () => { - it.concurrent('should allow OpenAI APIs', () => { - const openaiUrls = [ - 'https://api.openai.com/v1/chat/completions', - 'https://api.openai.com/v1/images/generations', - 'https://api.openai.com/v1/embeddings', - ] - - openaiUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow Google APIs', () => { - const googleUrls = [ - 'https://www.googleapis.com/drive/v3/files', - 'https://www.googleapis.com/auth/userinfo.email', - 'https://www.googleapis.com/auth/calendar', - 'https://graph.googleapis.com/v1.0/me', - 'https://accounts.google.com/.well-known/openid-configuration', - ] - - googleUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow Microsoft APIs', () => { - const microsoftUrls = [ - 'https://graph.microsoft.com/v1.0/me', - 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize', - 'https://login.microsoftonline.com/common/oauth2/v2.0/token', - ] - - microsoftUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow GitHub APIs', () => { - const githubUrls = [ - 'https://api.github.com/user', - 'https://api.github.com/user/emails', - 'https://github.com/login/oauth/authorize', - 'https://github.com/login/oauth/access_token', - ] - - githubUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow third-party service APIs', () => { - const thirdPartyUrls = [ - 'https://api.notion.com/v1/databases', - 'https://api.linear.app/graphql', - 'https://api.airtable.com/v0/appXXX', - 'https://api.twilio.com/2010-04-01/Accounts', - 'https://api.sendgrid.com/v3/mail/send', - 'https://api.stripe.com/v1/charges', - 'https://hooks.zapier.com/hooks/catch/123/abc', - 'https://discord.com/api/webhooks/123/abc', - 'https://api.firecrawl.dev/v1/crawl', - 'https://api.mistral.ai/v1/ocr', - 'https://api.tavily.com/search', - 'https://api.exa.ai/search', - 'https://api.perplexity.ai/chat/completions', - 'https://google.serper.dev/search', - 'https://api.linkup.so/v1/search', - 'https://api.pinecone.io/embed', - 'https://api.crmworkspace.com/v1/contacts', - 'https://slack.com/api/conversations.history', - 'https://api.atlassian.com/ex/jira/123/rest/api/3/issue/bulkfetch', - 'https://api.browser-use.com/api/v1/task/123', - ] - - thirdPartyUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow webhook URLs (Clay example)', () => { - const webhookUrls = [ - 'https://clay.com/webhooks/123/abc', - 'https://hooks.clay.com/webhook/xyz789', - 'https://api.clay.com/v1/populate/webhook', - ] - - webhookUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow dynamic URLs with parameters', () => { - const dynamicUrls = [ - 'https://google.serper.dev/search', - 'https://api.example.com/users/123/posts/456', - 'https://api.service.com/endpoint?param1=value1¶m2=value2', - 'https://cdn.example.com/files/document.pdf', - ] - - dynamicUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow custom QDrant instances on external hosts', () => { - const qdrantUrls = [ - 'https://my-qdrant.cloud.qdrant.io/collections/test/points', - 'https://qdrant.example.com/collections/docs/points', - ] - - qdrantUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Image proxy validation with real examples', () => { - it.concurrent('should allow legitimate image hosting services', () => { - const imageUrls = [ - 'https://cdn.openai.com/generated/image123.png', - 'https://storage.googleapis.com/bucket/images/photo.jpg', - 'https://example.s3.amazonaws.com/uploads/avatar.webp', - 'https://cdn.example.com/assets/logo.svg', - 'https://images.unsplash.com/photo-123?w=800', - 'https://avatars.githubusercontent.com/u/123456', - ] - - imageUrls.forEach((url) => { - const result = validateImageUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Edge cases that might be problematic but should still work', () => { - it.concurrent('should allow non-standard ports on external hosts', () => { - const customPortUrls = [ - 'https://api.example.com:8443/webhook', - 'https://custom-service.com:9000/api/v1/data', - 'http://external-service.com:8080/callback', - ] - - customPortUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow subdomains and complex domain structures', () => { - const complexDomainUrls = [ - 'https://api-staging.service.example.com/v1/test', - 'https://user123.cloud.provider.com/api', - 'https://region-us-east-1.service.aws.example.com/endpoint', - ] - - complexDomainUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should handle URLs with various query parameters and fragments', () => { - const complexUrls = [ - 'https://api.example.com/search?q=test&filter=active&sort=desc#results', - 'https://service.com/oauth/callback?code=abc123&state=xyz789', - 'https://api.service.com/v1/data?include[]=profile&include[]=settings', - ] - - complexUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Security tests - attack vectors should be blocked', () => { - it.concurrent('should block all SSRF attack patterns from the vulnerability report', () => { - const attackUrls = [ - 'http://172.17.0.1:9999', - 'file:///etc/passwd', - 'file:///proc/self/environ', - 'http://169.254.169.254/latest/meta-data/', - 'http://localhost:3000/internal', - 'http://127.0.0.1:8080/admin', - ] - - attackUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toBeDefined() - }) - }) - - it.concurrent('should block attempts to bypass with URL encoding', () => { - const encodedAttackUrls = [ - 'http://localhost%2F@example.com/', - 'http://%31%32%37%2e%30%2e%30%2e%31/', // 127.0.0.1 encoded - 'file%3A///etc/passwd', - ] - - encodedAttackUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - }) -}) - -describe('SSRF Vulnerability Resolution Verification', () => { - describe('Attack vectors should be blocked', () => { - it.concurrent('should block access to internal network endpoints (172.17.0.1:9999)', () => { - const result = validateProxyUrl('http://172.17.0.1:9999') - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - - it.concurrent('should block file:// protocol access to /proc/self/environ', () => { - const result = validateProxyUrl('file:///proc/self/environ') - expect(result.isValid).toBe(false) - expect(result.error).toContain('not allowed') - }) - - it.concurrent('should block file:// protocol access to /etc/passwd', () => { - const result = validateProxyUrl('file:///etc/passwd') - expect(result.isValid).toBe(false) - expect(result.error).toContain('not allowed') - }) - - it.concurrent('should block cloud metadata endpoint access', () => { - const result = validateProxyUrl('http://169.254.169.254/latest/meta-data/') - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - - it.concurrent('should block localhost access on various ports', () => { - const localhostUrls = [ - 'http://localhost:3000', - 'http://127.0.0.1:8080', - 'http://127.0.0.1:9999', - ] - - localhostUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - }) - }) - - describe('Both proxy endpoints are protected', () => { - it.concurrent('should protect /api/proxy/route.ts endpoint', () => { - const attackUrls = [ - 'http://172.17.0.1:9999', - 'file:///etc/passwd', - 'http://localhost:3000/admin', - ] - - attackUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - - it.concurrent('should protect /api/proxy/image/route.ts endpoint', () => { - const attackUrls = [ - 'http://172.17.0.1:9999/image.jpg', - 'file:///etc/passwd.jpg', - 'http://localhost:3000/internal/image.png', - ] - - attackUrls.forEach((url) => { - const result = validateImageUrl(url) - expect(result.isValid).toBe(false) - }) - }) - }) - - describe('All legitimate use cases still work', () => { - it.concurrent('should allow all external API calls the system makes', () => { - const legitimateUrls = [ - 'https://api.openai.com/v1/chat/completions', - 'https://api.github.com/user', - 'https://www.googleapis.com/drive/v3/files', - 'https://graph.microsoft.com/v1.0/me', - 'https://api.notion.com/v1/pages', - 'https://api.linear.app/graphql', - 'https://hooks.zapier.com/hooks/catch/123/abc', - 'https://discord.com/api/webhooks/123/token', - 'https://api.mistral.ai/v1/ocr', - 'https://api.twilio.com/2010-04-01/Accounts', - ] - - legitimateUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow legitimate image URLs for OpenAI image tool', () => { - const imageUrls = [ - 'https://cdn.openai.com/generated/image123.png', - 'https://storage.googleapis.com/bucket/image.jpg', - 'https://example.s3.amazonaws.com/images/photo.webp', - ] - - imageUrls.forEach((url) => { - const result = validateImageUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should allow user-provided webhook URLs', () => { - const webhookUrls = [ - 'https://webhook.site/unique-id', - 'https://my-app.herokuapp.com/webhook', - 'https://api.company.com/webhook/receive', - ] - - webhookUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Comprehensive attack prevention', () => { - it.concurrent('should block all private IP ranges', () => { - const privateIpUrls = [ - 'http://10.0.0.1/secret', - 'http://172.16.0.1/admin', - 'http://192.168.1.1/config', - 'http://169.254.169.254/metadata', - ] - - privateIpUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toContain('private networks') - }) - }) - - it.concurrent('should block all dangerous protocols', () => { - const dangerousProtocols = [ - 'file:///etc/passwd', - 'ftp://internal.server.com/files', - 'gopher://localhost:70/secret', - 'ldap://internal.ad.com/', - 'dict://localhost:2628/show', - ] - - dangerousProtocols.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - expect(result.error).toMatch(/Protocol .* is (not allowed|blocked)/) - }) - }) - }) -}) - -describe('User-provided URL validation scenarios', () => { - describe('HTTP Request tool with user URLs', () => { - it.concurrent('should allow legitimate user-provided API endpoints', () => { - const userApiUrls = [ - 'https://my-company-api.com/webhook', - 'https://api.my-service.io/v1/data', - 'https://webhook.site/unique-id', - 'https://httpbin.org/post', - 'https://postman-echo.com/post', - 'https://my-custom-domain.org/api/callback', - 'https://user123.ngrok.io/webhook', // Common tunneling service for dev - ] - - userApiUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Mistral parser with user PDF URLs', () => { - it.concurrent('should allow legitimate PDF hosting services', () => { - const pdfUrls = [ - 'https://example.com/documents/report.pdf', - 'https://cdn.company.com/files/manual.pdf', - 'https://storage.cloud.google.com/bucket/document.pdf', - 'https://s3.amazonaws.com/bucket/files/doc.pdf', - 'https://assets.website.com/pdfs/guide.pdf', - ] - - pdfUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should block attempts to use PDF URLs for SSRF', () => { - const maliciousPdfUrls = [ - 'http://localhost:3000/admin/report.pdf', - 'http://127.0.0.1:8080/internal/secret.pdf', - 'http://192.168.1.1/config/backup.pdf', - 'file:///etc/passwd.pdf', - ] - - maliciousPdfUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - }) - - describe('Clay webhooks and custom services', () => { - it.concurrent('should allow legitimate webhook services', () => { - const webhookUrls = [ - 'https://clay.com/webhooks/abc123', - 'https://hooks.zapier.com/hooks/catch/123/xyz', - 'https://maker.ifttt.com/trigger/event/with/key/abc123', - 'https://webhook.site/unique-uuid-here', - 'https://discord.com/api/webhooks/123/token', - 'https://slack.com/api/webhook/incoming', - 'https://my-app.herokuapp.com/webhook', - 'https://api.custom-service.com/webhook/receive', - ] - - webhookUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Custom QDrant/vector database instances', () => { - it.concurrent('should allow external vector database services', () => { - const vectorDbUrls = [ - 'https://my-qdrant.cloud.provider.com/collections/docs/points', - 'https://vector-db.company.com/api/v1/search', - 'https://pinecone-index.pinecone.io/vectors/query', - 'https://weaviate.company-cluster.com/v1/objects', - ] - - vectorDbUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should block internal vector database instances', () => { - const internalDbUrls = [ - 'http://localhost:6333/collections/sensitive/points', - 'http://127.0.0.1:8080/qdrant/search', - 'http://192.168.1.100:6333/collections/admin/points', - ] - - internalDbUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(false) - }) - }) - }) - - describe('Development and testing scenarios', () => { - it.concurrent('should allow common development tools and services', () => { - const devUrls = [ - 'https://postman-echo.com/get', - 'https://httpbin.org/anything', - 'https://jsonplaceholder.typicode.com/posts', - 'https://reqres.in/api/users', - 'https://api.github.com/repos/owner/repo', - 'https://raw.githubusercontent.com/owner/repo/main/file.json', - ] - - devUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should handle tunneling services used in development', () => { - const tunnelUrls = [ - 'https://abc123.ngrok.io/webhook', - 'https://random-string.loca.lt/api', - 'https://subdomain.serveo.net/callback', - ] - - tunnelUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - - it.concurrent('should block attempts to tunnel to localhost', () => { - const maliciousTunnelUrls = [ - 'https://tunnel.com/proxy?url=http://localhost:3000', - 'https://proxy.service.com/?target=http://127.0.0.1:8080', - ] - - maliciousTunnelUrls.forEach((url) => { - const result = validateProxyUrl(url) - // These URLs themselves are valid, but they can't contain localhost in the main URL - // The actual attack prevention happens at the parameter level - expect(result.isValid).toBe(true) - }) - }) - }) - - describe('Enterprise and custom domain scenarios', () => { - it.concurrent('should allow corporate domains and custom TLDs', () => { - const enterpriseUrls = [ - 'https://api.company.internal/v1/data', // .internal TLD - 'https://webhook.corp/receive', // .corp TLD - 'https://api.organization.local/webhook', // .local TLD - 'https://service.company.co.uk/api', // Country code TLD - 'https://api.startup.io/v2/callback', // Modern TLD - 'https://webhook.company.ai/receive', // AI TLD - ] - - enterpriseUrls.forEach((url) => { - const result = validateProxyUrl(url) - expect(result.isValid).toBe(true) - }) - }) - }) -}) diff --git a/apps/sim/lib/security/url-validation.ts b/apps/sim/lib/security/url-validation.ts deleted file mode 100644 index 1197cb746d..0000000000 --- a/apps/sim/lib/security/url-validation.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { createLogger } from '@/lib/logs/console/logger' - -const logger = createLogger('URLValidation') - -/** - * Validates URLs for proxy requests to prevent SSRF attacks - * while preserving legitimate external API functionality - */ - -const BLOCKED_IP_RANGES = [ - // Private IPv4 ranges (RFC 1918) - /^10\./, - /^172\.(1[6-9]|2[0-9]|3[01])\./, - /^192\.168\./, - - // Loopback addresses - /^127\./, - /^localhost$/i, - - // Link-local addresses (RFC 3927) - /^169\.254\./, - - // Cloud metadata endpoints - /^169\.254\.169\.254$/, - - // Broadcast and other reserved ranges - /^0\./, - /^224\./, - /^240\./, - /^255\./, - - // IPv6 loopback and link-local - /^::1$/, - /^fe80:/i, - /^::ffff:127\./i, - /^::ffff:10\./i, - /^::ffff:172\.(1[6-9]|2[0-9]|3[01])\./i, - /^::ffff:192\.168\./i, -] - -const ALLOWED_PROTOCOLS = ['http:', 'https:'] - -const BLOCKED_PROTOCOLS = [ - 'file:', - 'ftp:', - 'ftps:', - 'gopher:', - 'ldap:', - 'ldaps:', - 'dict:', - 'sftp:', - 'ssh:', - 'jar:', - 'netdoc:', - 'data:', -] - -/** - * Validates a URL to prevent SSRF attacks - * @param url - The URL to validate - * @returns Object with isValid boolean and error message if invalid - */ -export function validateProxyUrl(url: string): { isValid: boolean; error?: string } { - try { - // Parse the URL - const parsedUrl = new URL(url) - - // Check protocol - if (!ALLOWED_PROTOCOLS.includes(parsedUrl.protocol)) { - logger.warn('Blocked request with disallowed protocol', { - url: url.substring(0, 100), - protocol: parsedUrl.protocol, - }) - return { - isValid: false, - error: `Protocol '${parsedUrl.protocol}' is not allowed. Only HTTP and HTTPS are permitted.`, - } - } - - // Check for explicitly blocked protocols - if (BLOCKED_PROTOCOLS.includes(parsedUrl.protocol)) { - logger.warn('Blocked request with dangerous protocol', { - url: url.substring(0, 100), - protocol: parsedUrl.protocol, - }) - return { - isValid: false, - error: `Protocol '${parsedUrl.protocol}' is blocked for security reasons.`, - } - } - - // Get hostname for validation - const hostname = parsedUrl.hostname.toLowerCase() - - // Check if hostname matches blocked patterns - for (const pattern of BLOCKED_IP_RANGES) { - if (pattern.test(hostname)) { - logger.warn('Blocked request to private/reserved IP range', { - hostname, - url: url.substring(0, 100), - }) - return { - isValid: false, - error: 'Access to private networks, localhost, and reserved IP ranges is not allowed.', - } - } - } - - // Additional hostname validation - if (hostname === '' || hostname.includes('..')) { - return { - isValid: false, - error: 'Invalid hostname format.', - } - } - - // Check for URL encoding attempts to bypass validation - const decodedUrl = decodeURIComponent(url) - if (decodedUrl !== url) { - // Recursively validate the decoded URL - return validateProxyUrl(decodedUrl) - } - - logger.debug('URL validation passed', { - hostname, - protocol: parsedUrl.protocol, - url: url.substring(0, 100), - }) - - return { isValid: true } - } catch (error) { - logger.warn('URL parsing failed during validation', { - url: url.substring(0, 100), - error: error instanceof Error ? error.message : String(error), - }) - return { - isValid: false, - error: 'Invalid URL format.', - } - } -} - -/** - * Enhanced validation specifically for image URLs with additional checks - * @param url - The image URL to validate - * @returns Object with isValid boolean and error message if invalid - */ -export function validateImageUrl(url: string): { isValid: boolean; error?: string } { - // First run standard proxy URL validation - const baseValidation = validateProxyUrl(url) - if (!baseValidation.isValid) { - return baseValidation - } - - try { - const parsedUrl = new URL(url) - - // Additional checks for image URLs - // Ensure it's not trying to access internal services via common ports - if (parsedUrl.port) { - const port = Number.parseInt(parsedUrl.port, 10) - const dangerousPorts = [ - 22, - 23, - 25, - 53, - 80, - 110, - 143, - 443, - 993, - 995, // Common service ports - 3000, - 3001, - 5000, - 8000, - 8080, - 8443, - 9000, // Common dev ports - ] - - // Only block if hostname suggests internal access - if ( - BLOCKED_IP_RANGES.some((pattern) => pattern.test(parsedUrl.hostname)) && - dangerousPorts.includes(port) - ) { - return { - isValid: false, - error: 'Access to internal services on common ports is not allowed.', - } - } - } - - return { isValid: true } - } catch (error) { - return { - isValid: false, - error: 'Invalid image URL format.', - } - } -} - -/** - * Helper function to check if a hostname resolves to a private IP - * Note: This is a basic check and doesn't perform actual DNS resolution - * which could be added for enhanced security if needed - */ -export function isPrivateHostname(hostname: string): boolean { - return BLOCKED_IP_RANGES.some((pattern) => pattern.test(hostname)) -} diff --git a/apps/sim/lib/workflows/streaming.ts b/apps/sim/lib/workflows/streaming.ts index 6d3e72c062..4498d84be8 100644 --- a/apps/sim/lib/workflows/streaming.ts +++ b/apps/sim/lib/workflows/streaming.ts @@ -97,7 +97,6 @@ export async function createStreamingResponse( for (const outputId of matchingOutputs) { const path = extractPathFromOutputId(outputId, blockId) - // Response blocks have their data nested under 'response' let outputValue = traverseObjectPath(output, path) if (outputValue === undefined && output.response) { outputValue = traverseObjectPath(output.response, path) @@ -159,7 +158,6 @@ export async function createStreamingResponse( output: {} as any, } - // If there are selected outputs, only include those specific fields if (streamConfig.selectedOutputs?.length && result.output) { const { extractBlockIdFromOutputId, extractPathFromOutputId, traverseObjectPath } = await import('@/lib/response-format') @@ -168,19 +166,28 @@ export async function createStreamingResponse( const blockId = extractBlockIdFromOutputId(outputId) const path = extractPathFromOutputId(outputId, blockId) - // Find the output value from the result if (result.logs) { const blockLog = result.logs.find((log: any) => log.blockId === blockId) if (blockLog?.output) { - // Response blocks have their data nested under 'response' let value = traverseObjectPath(blockLog.output, path) if (value === undefined && blockLog.output.response) { value = traverseObjectPath(blockLog.output.response, path) } if (value !== undefined) { - // Store it in a structured way + const dangerousKeys = ['__proto__', 'constructor', 'prototype'] + if (dangerousKeys.includes(blockId) || dangerousKeys.includes(path)) { + logger.warn( + `[${requestId}] Blocked potentially dangerous property assignment`, + { + blockId, + path, + } + ) + continue + } + if (!minimalResult.output[blockId]) { - minimalResult.output[blockId] = {} + minimalResult.output[blockId] = Object.create(null) } minimalResult.output[blockId][path] = value } @@ -188,7 +195,6 @@ export async function createStreamingResponse( } } } else if (!streamConfig.selectedOutputs?.length) { - // No selected outputs means include the full output (but still filtered) minimalResult.output = result.output } diff --git a/apps/sim/tools/confluence/retrieve.ts b/apps/sim/tools/confluence/retrieve.ts index 962e10171b..234fafb7d1 100644 --- a/apps/sim/tools/confluence/retrieve.ts +++ b/apps/sim/tools/confluence/retrieve.ts @@ -46,7 +46,6 @@ export const confluenceRetrieveTool: ToolConfig< request: { url: (params: ConfluenceRetrieveParams) => { - // Instead of calling Confluence API directly, use your API route return '/api/tools/confluence/page' }, method: 'POST', diff --git a/apps/sim/tools/confluence/utils.ts b/apps/sim/tools/confluence/utils.ts index 8a070dfc8e..e58ee842a0 100644 --- a/apps/sim/tools/confluence/utils.ts +++ b/apps/sim/tools/confluence/utils.ts @@ -9,7 +9,6 @@ export async function getConfluenceCloudId(domain: string, accessToken: string): const resources = await response.json() - // If we have resources, find the matching one if (Array.isArray(resources) && resources.length > 0) { const normalizedInput = `https://${domain}`.toLowerCase() const matchedResource = resources.find((r) => r.url.toLowerCase() === normalizedInput) @@ -19,8 +18,6 @@ export async function getConfluenceCloudId(domain: string, accessToken: string): } } - // If we couldn't find a match, return the first resource's ID - // This is a fallback in case the URL matching fails if (Array.isArray(resources) && resources.length > 0) { return resources[0].id } @@ -28,8 +25,38 @@ export async function getConfluenceCloudId(domain: string, accessToken: string): throw new Error('No Confluence resources found') } +function decodeHtmlEntities(text: string): string { + let decoded = text + let previous: string + + do { + previous = decoded + decoded = decoded + .replace(/ /g, ' ') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + decoded = decoded.replace(/&/g, '&') + } while (decoded !== previous) + + return decoded +} + +function stripHtmlTags(html: string): string { + let text = html + let previous: string + + do { + previous = text + text = text.replace(/<[^>]*>/g, '') + text = text.replace(/[<>]/g, '') + } while (text !== previous) + + return text.trim() +} + export function transformPageData(data: any) { - // Get content from wherever we can find it const content = data.body?.view?.value || data.body?.storage?.value || @@ -38,14 +65,9 @@ export function transformPageData(data: any) { data.description || `Content for page ${data.title || 'Unknown'}` - const cleanContent = content - .replace(/<[^>]*>/g, '') - .replace(/ /g, ' ') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/\s+/g, ' ') - .trim() + let cleanContent = stripHtmlTags(content) + cleanContent = decodeHtmlEntities(cleanContent) + cleanContent = cleanContent.replace(/\s+/g, ' ').trim() return { success: true, diff --git a/apps/sim/tools/sharepoint/utils.ts b/apps/sim/tools/sharepoint/utils.ts index 7ed3169f22..8dfc32f601 100644 --- a/apps/sim/tools/sharepoint/utils.ts +++ b/apps/sim/tools/sharepoint/utils.ts @@ -3,7 +3,19 @@ import type { CanvasLayout } from '@/tools/sharepoint/types' const logger = createLogger('SharepointUtils') -// Extract readable text from SharePoint canvas layout +function stripHtmlTags(html: string): string { + let text = html + let previous: string + + do { + previous = text + text = text.replace(/<[^>]*>/g, '') + text = text.replace(/[<>]/g, '') + } while (text !== previous) + + return text.trim() +} + export function extractTextFromCanvasLayout(canvasLayout: CanvasLayout | null | undefined): string { logger.info('Extracting text from canvas layout', { hasCanvasLayout: !!canvasLayout, @@ -37,8 +49,7 @@ export function extractTextFromCanvasLayout(canvasLayout: CanvasLayout | null | }) if (webpart.innerHtml) { - // Extract text from HTML, removing tags - const text = webpart.innerHtml.replace(/<[^>]*>/g, '').trim() + const text = stripHtmlTags(webpart.innerHtml) if (text) { textParts.push(text) logger.info('Extracted text', { text }) @@ -50,7 +61,7 @@ export function extractTextFromCanvasLayout(canvasLayout: CanvasLayout | null | } else if (section.webparts) { for (const webpart of section.webparts) { if (webpart.innerHtml) { - const text = webpart.innerHtml.replace(/<[^>]*>/g, '').trim() + const text = stripHtmlTags(webpart.innerHtml) if (text) textParts.push(text) } } @@ -67,7 +78,6 @@ export function extractTextFromCanvasLayout(canvasLayout: CanvasLayout | null | return finalContent } -// Remove OData metadata from objects export function cleanODataMetadata(obj: T): T { if (!obj || typeof obj !== 'object') return obj @@ -77,7 +87,6 @@ export function cleanODataMetadata(obj: T): T { const cleaned: Record = {} for (const [key, value] of Object.entries(obj as Record)) { - // Skip OData metadata keys if (key.includes('@odata')) continue cleaned[key] = cleanODataMetadata(value) diff --git a/bun.lock b/bun.lock index e064472bab..14a26cbed0 100644 --- a/bun.lock +++ b/bun.lock @@ -3943,4 +3943,4 @@ "lint-staged/listr2/log-update/cli-cursor/restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], } -} +} \ No newline at end of file diff --git a/packages/ts-sdk/src/index.ts b/packages/ts-sdk/src/index.ts index 0514ae4f58..559f04cfeb 100644 --- a/packages/ts-sdk/src/index.ts +++ b/packages/ts-sdk/src/index.ts @@ -94,6 +94,20 @@ export class SimStudioError extends Error { } } +/** + * Remove trailing slashes from a URL + * Uses string operations instead of regex to prevent ReDoS attacks + * @param url - The URL to normalize + * @returns URL without trailing slashes + */ +function normalizeBaseUrl(url: string): string { + let normalized = url + while (normalized.endsWith('/')) { + normalized = normalized.slice(0, -1) + } + return normalized +} + export class SimStudioClient { private apiKey: string private baseUrl: string @@ -101,7 +115,7 @@ export class SimStudioClient { constructor(config: SimStudioConfig) { this.apiKey = config.apiKey - this.baseUrl = (config.baseUrl || 'https://sim.ai').replace(/\/+$/, '') + this.baseUrl = normalizeBaseUrl(config.baseUrl || 'https://sim.ai') } /** @@ -306,7 +320,7 @@ export class SimStudioClient { * Set a new base URL */ setBaseUrl(baseUrl: string): void { - this.baseUrl = baseUrl.replace(/\/+$/, '') + this.baseUrl = normalizeBaseUrl(baseUrl) } /** diff --git a/scripts/README.md b/scripts/README.md index 9b607ec433..0c0f139411 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -54,7 +54,7 @@ The documentation generator runs automatically as part of the CI/CD pipeline whe ## Adding Support for New Block Properties -If you add new properties to block definitions that should be included in the documentation, update the `generateMarkdownForBlock` function in `scripts/generate-block-docs.ts`. +If you add new properties to block definitions that should be included in the documentation, update the `generateMarkdownForBlock` function in `scripts/generate-docs.ts`. ## Preserving Manual Content diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh deleted file mode 100755 index 01c07fed08..0000000000 --- a/scripts/generate-docs.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Set error handling -set -e - -# Enable debug mode if DEBUG env var is set -if [ ! -z "$DEBUG" ]; then - set -x - echo "Debug mode enabled" -fi - -# Get script directories -SCRIPTS_DIR=$(dirname "$0") -ROOT_DIR=$(cd "$SCRIPTS_DIR/.." && pwd) -echo "Scripts directory: $SCRIPTS_DIR" -echo "Root directory: $ROOT_DIR" - -# Check if dependencies are installed in scripts directory -if [ ! -d "$SCRIPTS_DIR/node_modules" ]; then - echo "Required dependencies not found. Installing now..." - bash "$SCRIPTS_DIR/setup-doc-generator.sh" -fi - -# Generate documentation -echo "Generating block documentation..." - -# Check if necessary files exist -if [ ! -f "$SCRIPTS_DIR/generate-block-docs.ts" ]; then - echo "Error: Could not find generate-block-docs.ts script" - ls -la "$SCRIPTS_DIR" - exit 1 -fi - -if [ ! -f "$SCRIPTS_DIR/tsconfig.json" ]; then - echo "Error: Could not find tsconfig.json in scripts directory" - exit 1 -fi - -# Check if npx is available -if ! command -v npx &> /dev/null; then - echo "Error: npx is not installed. Please install Node.js first." - exit 1 -fi - -# Change to scripts directory to use local dependencies -cd "$SCRIPTS_DIR" -echo "Executing: npx tsx ./generate-block-docs.ts" - -# Run the generator with tsx using local dependencies -if ! npx tsx ./generate-block-docs.ts; then - echo "" - echo "Error running documentation generator." - echo "" - echo "For more detailed debugging, run with DEBUG=1:" - echo "DEBUG=1 ./scripts/generate-docs.sh" - exit 1 -fi - -echo "Documentation generation complete!" diff --git a/scripts/generate-block-docs.ts b/scripts/generate-docs.ts similarity index 73% rename from scripts/generate-block-docs.ts rename to scripts/generate-docs.ts index f4abbd5548..4c744e578f 100755 --- a/scripts/generate-block-docs.ts +++ b/scripts/generate-docs.ts @@ -6,22 +6,18 @@ import { glob } from 'glob' console.log('Starting documentation generator...') -// Define directory paths const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) const rootDir = path.resolve(__dirname, '..') -// Paths configuration const BLOCKS_PATH = path.join(rootDir, 'apps/sim/blocks/blocks') const DOCS_OUTPUT_PATH = path.join(rootDir, 'apps/docs/content/docs/en/tools') const ICONS_PATH = path.join(rootDir, 'apps/sim/components/icons.tsx') -// Make sure the output directory exists if (!fs.existsSync(DOCS_OUTPUT_PATH)) { fs.mkdirSync(DOCS_OUTPUT_PATH, { recursive: true }) } -// Basic interface for BlockConfig to avoid import issues interface BlockConfig { type: string name: string @@ -36,39 +32,32 @@ interface BlockConfig { [key: string]: any } -// Function to extract SVG icons from icons.tsx file function extractIcons(): Record { try { const iconsContent = fs.readFileSync(ICONS_PATH, 'utf-8') const icons: Record = {} - // Match both function declaration and arrow function export patterns const functionDeclarationRegex = /export\s+function\s+(\w+Icon)\s*\([^)]*\)\s*{[\s\S]*?return\s*\(\s*\s*\)/g const arrowFunctionRegex = /export\s+const\s+(\w+Icon)\s*=\s*\([^)]*\)\s*=>\s*(\(?\s*\s*\)?)/g - // Extract function declaration style icons const functionMatches = Array.from(iconsContent.matchAll(functionDeclarationRegex)) for (const match of functionMatches) { const iconName = match[1] const svgMatch = match[0].match(//) if (iconName && svgMatch) { - // Clean the SVG to remove {...props} and standardize size let svgContent = svgMatch[0] svgContent = svgContent.replace(/{\.\.\.props}/g, '') svgContent = svgContent.replace(/{\.\.\.(props|rest)}/g, '') - // Remove any existing width/height attributes to let CSS handle sizing svgContent = svgContent.replace(/width=["'][^"']*["']/g, '') svgContent = svgContent.replace(/height=["'][^"']*["']/g, '') - // Add className for styling svgContent = svgContent.replace(/ { const svgMatch = svgContent.match(//) if (iconName && svgMatch) { - // Clean the SVG to remove {...props} and standardize size let cleanedSvg = svgMatch[0] cleanedSvg = cleanedSvg.replace(/{\.\.\.props}/g, '') cleanedSvg = cleanedSvg.replace(/{\.\.\.(props|rest)}/g, '') - // Remove any existing width/height attributes to let CSS handle sizing cleanedSvg = cleanedSvg.replace(/width=["'][^"']*["']/g, '') cleanedSvg = cleanedSvg.replace(/height=["'][^"']*["']/g, '') - // Add className for styling cleanedSvg = cleanedSvg.replace(/ { } } -// Function to extract block configuration from file content function extractBlockConfig(fileContent: string): BlockConfig | null { try { - // Extract the block name from export statement const exportMatch = fileContent.match(/export\s+const\s+(\w+)Block\s*:/) if (!exportMatch) { @@ -109,7 +93,6 @@ function extractBlockConfig(fileContent: string): BlockConfig | null { const blockName = exportMatch[1] const blockType = findBlockType(fileContent, blockName) - // Extract individual properties with more robust regex const name = extractStringProperty(fileContent, 'name') || `${blockName} Block` const description = extractStringProperty(fileContent, 'description') || '' const longDescription = extractStringProperty(fileContent, 'longDescription') || '' @@ -117,10 +100,8 @@ function extractBlockConfig(fileContent: string): BlockConfig | null { const bgColor = extractStringProperty(fileContent, 'bgColor') || '#F5F5F5' const iconName = extractIconName(fileContent) || '' - // Extract outputs object with better handling const outputs = extractOutputs(fileContent) - // Extract tools access array const toolsAccess = extractToolsAccess(fileContent) return { @@ -142,10 +123,7 @@ function extractBlockConfig(fileContent: string): BlockConfig | null { } } -// Helper function to find the block type function findBlockType(content: string, blockName: string): string { - // Try to find the type within the main block export - // Look for the pattern: export const [BlockName]Block: BlockConfig = { ... type: 'value' ... } const blockExportRegex = new RegExp( `export\\s+const\\s+${blockName}Block\\s*:[^{]*{[\\s\\S]*?type\\s*:\\s*['"]([^'"]+)['"][\\s\\S]*?}`, 'i' @@ -153,18 +131,14 @@ function findBlockType(content: string, blockName: string): string { const blockExportMatch = content.match(blockExportRegex) if (blockExportMatch) return blockExportMatch[1] - // Fallback: try to find type within a block config object that comes after the export const exportMatch = content.match(new RegExp(`export\\s+const\\s+${blockName}Block\\s*:`)) if (exportMatch) { - // Find the content after the export statement const afterExport = content.substring(exportMatch.index! + exportMatch[0].length) - // Look for the first opening brace and then find type within that block const blockStartMatch = afterExport.match(/{/) if (blockStartMatch) { const blockStart = blockStartMatch.index! - // Find the matching closing brace by counting braces let braceCount = 1 let blockEnd = blockStart + 1 @@ -174,47 +148,37 @@ function findBlockType(content: string, blockName: string): string { blockEnd++ } - // Extract the block content and look for type const blockContent = afterExport.substring(blockStart, blockEnd) const typeMatch = blockContent.match(/type\s*:\s*['"]([^'"]+)['"]/) if (typeMatch) return typeMatch[1] } } - // Convert CamelCase to snake_case as fallback return blockName .replace(/([A-Z])/g, '_$1') .toLowerCase() .replace(/^_/, '') } -// Helper to extract a string property from content function extractStringProperty(content: string, propName: string): string | null { - // Try single quotes first - more permissive approach const singleQuoteMatch = content.match(new RegExp(`${propName}\\s*:\\s*'(.*?)'`, 'm')) if (singleQuoteMatch) return singleQuoteMatch[1] - // Try double quotes const doubleQuoteMatch = content.match(new RegExp(`${propName}\\s*:\\s*"(.*?)"`, 'm')) if (doubleQuoteMatch) return doubleQuoteMatch[1] - // Try to match multi-line string with template literals const templateMatch = content.match(new RegExp(`${propName}\\s*:\\s*\`([^\`]+)\``, 's')) if (templateMatch) { let templateContent = templateMatch[1] - // Handle template literals with expressions by replacing them with reasonable defaults - // This is a simple approach - we'll replace common variable references with sensible defaults templateContent = templateContent.replace( /\$\{[^}]*shouldEnableURLInput[^}]*\?[^:]*:[^}]*\}/g, 'Upload files directly. ' ) templateContent = templateContent.replace(/\$\{[^}]*shouldEnableURLInput[^}]*\}/g, 'false') - // Remove any remaining template expressions that we can't safely evaluate templateContent = templateContent.replace(/\$\{[^}]+\}/g, '') - // Clean up any extra whitespace templateContent = templateContent.replace(/\s+/g, ' ').trim() return templateContent @@ -223,23 +187,18 @@ function extractStringProperty(content: string, propName: string): string | null return null } -// Helper to extract icon name from content function extractIconName(content: string): string | null { const iconMatch = content.match(/icon\s*:\s*(\w+Icon)/) return iconMatch ? iconMatch[1] : null } -// Updated function to extract outputs with a simpler and more reliable approach function extractOutputs(content: string): Record { - // Look for the outputs section using balanced brace matching const outputsStart = content.search(/outputs\s*:\s*{/) if (outputsStart === -1) return {} - // Find the opening brace position const openBracePos = content.indexOf('{', outputsStart) if (openBracePos === -1) return {} - // Use balanced brace counting to find the complete outputs section let braceCount = 1 let pos = openBracePos + 1 @@ -256,27 +215,22 @@ function extractOutputs(content: string): Record { const outputsContent = content.substring(openBracePos + 1, pos - 1).trim() const outputs: Record = {} - // First try to handle the new object format: fieldName: { type: 'type', description: 'desc' } - // Use a more robust approach to extract field definitions const fieldRegex = /(\w+)\s*:\s*{/g let match const fieldPositions: Array<{ name: string; start: number }> = [] - // Find all field starting positions while ((match = fieldRegex.exec(outputsContent)) !== null) { fieldPositions.push({ name: match[1], - start: match.index + match[0].length - 1, // Position of the opening brace + start: match.index + match[0].length - 1, }) } - // Extract each field's content by finding balanced braces fieldPositions.forEach((field) => { const startPos = field.start let braceCount = 1 let endPos = startPos + 1 - // Find the matching closing brace while (endPos < outputsContent.length && braceCount > 0) { if (outputsContent[endPos] === '{') { braceCount++ @@ -287,10 +241,8 @@ function extractOutputs(content: string): Record { } if (braceCount === 0) { - // Extract the content between braces const fieldContent = outputsContent.substring(startPos + 1, endPos - 1).trim() - // Extract type and description from the object const typeMatch = fieldContent.match(/type\s*:\s*['"](.*?)['"]/) const descriptionMatch = fieldContent.match(/description\s*:\s*['"](.*?)['"]/) @@ -305,12 +257,10 @@ function extractOutputs(content: string): Record { } }) - // If we found object fields, return them if (Object.keys(outputs).length > 0) { return outputs } - // Fallback: try to handle the old flat format: fieldName: 'type' const flatFieldMatches = outputsContent.match(/(\w+)\s*:\s*['"](.*?)['"]/g) if (flatFieldMatches && flatFieldMatches.length > 0) { @@ -327,7 +277,6 @@ function extractOutputs(content: string): Record { } }) - // If we found flat fields, return them if (Object.keys(outputs).length > 0) { return outputs } @@ -337,9 +286,8 @@ function extractOutputs(content: string): Record { return {} } -// Helper to extract tools access array function extractToolsAccess(content: string): string[] { - const accessMatch = content.match(/access\s*:\s*\[\s*((?:['"][^'"]+['"](?:\s*,\s*)?)+)\s*\]/) + const accessMatch = content.match(/access\s*:\s*\[\s*([^\]]+)\s*\]/) if (!accessMatch) return [] const accessContent = accessMatch[1] @@ -358,7 +306,6 @@ function extractToolsAccess(content: string): string[] { return tools } -// Function to extract tool information from file content function extractToolInfo( toolName: string, fileContent: string @@ -368,33 +315,27 @@ function extractToolInfo( outputs: Record } | null { try { - // Extract tool config section - Match params until the next top-level property const toolConfigRegex = /params\s*:\s*{([\s\S]*?)},?\s*(?:outputs|oauth|request|directExecution|postProcess|transformResponse)/ const toolConfigMatch = fileContent.match(toolConfigRegex) - // Extract description const descriptionRegex = /description\s*:\s*['"](.*?)['"].*/ const descriptionMatch = fileContent.match(descriptionRegex) const description = descriptionMatch ? descriptionMatch[1] : 'No description available' - // Parse parameters const params: Array<{ name: string; type: string; required: boolean; description: string }> = [] if (toolConfigMatch) { const paramsContent = toolConfigMatch[1] - // More robust approach to extract parameters with balanced brace matching - // Extract each parameter block completely const paramBlocksRegex = /(\w+)\s*:\s*{/g let paramMatch const paramPositions: Array<{ name: string; start: number; content: string }> = [] while ((paramMatch = paramBlocksRegex.exec(paramsContent)) !== null) { const paramName = paramMatch[1] - const startPos = paramMatch.index + paramMatch[0].length - 1 // Position of opening brace + const startPos = paramMatch.index + paramMatch[0].length - 1 - // Find matching closing brace using balanced counting let braceCount = 1 let endPos = startPos + 1 @@ -417,27 +358,21 @@ function extractToolInfo( const paramName = param.name const paramBlock = param.content - // Skip the accessToken parameter as it's handled automatically by the OAuth flow - // Also skip any params parameter which isn't a real input if (paramName === 'accessToken' || paramName === 'params' || paramName === 'tools') { continue } - // Extract param details with more robust patterns const typeMatch = paramBlock.match(/type\s*:\s*['"]([^'"]+)['"]/) const requiredMatch = paramBlock.match(/required\s*:\s*(true|false)/) - // More careful extraction of description with handling for multiline descriptions let descriptionMatch = paramBlock.match(/description\s*:\s*'(.*?)'(?=\s*[,}])/s) if (!descriptionMatch) { descriptionMatch = paramBlock.match(/description\s*:\s*"(.*?)"(?=\s*[,}])/s) } if (!descriptionMatch) { - // Try for template literals if the description uses backticks descriptionMatch = paramBlock.match(/description\s*:\s*`([^`]+)`/s) } if (!descriptionMatch) { - // Handle multi-line descriptions without ending quote on same line descriptionMatch = paramBlock.match( /description\s*:\s*['"]([^'"]*(?:\n[^'"]*)*?)['"](?=\s*[,}])/s ) @@ -452,7 +387,6 @@ function extractToolInfo( } } - // First priority: Extract outputs from the new outputs field in ToolConfig let outputs: Record = {} const outputsFieldRegex = /outputs\s*:\s*{([\s\S]*?)}\s*,?\s*(?:oauth|params|request|directExecution|postProcess|transformResponse|$|\})/ @@ -475,7 +409,6 @@ function extractToolInfo( } } -// Helper function to recursively format output structure for documentation function formatOutputStructure(outputs: Record, indentLevel = 0): string { let result = '' @@ -493,7 +426,6 @@ function formatOutputStructure(outputs: Record, indentLevel = 0): s } } - // Escape special characters in the description const escapedDescription = description .replace(/\|/g, '\\|') .replace(/\{/g, '\\{') @@ -505,28 +437,21 @@ function formatOutputStructure(outputs: Record, indentLevel = 0): s .replace(//g, '>') - // Create prefix based on nesting level with visual hierarchy let prefix = '' if (indentLevel === 1) { prefix = '↳ ' } else if (indentLevel >= 2) { - // For deeper nesting (like array items), use indented arrows prefix = ' ↳ ' } - // For arrays, expand nested items if (typeof output === 'object' && output !== null && output.type === 'array') { result += `| ${prefix}\`${key}\` | ${type} | ${escapedDescription} |\n` - // Handle array items with properties (nested TWO more levels to show it's inside the array) if (output.items?.properties) { - // Create a visual separator to show these are array item properties const arrayItemsResult = formatOutputStructure(output.items.properties, indentLevel + 2) result += arrayItemsResult } - } - // For objects, expand properties - else if ( + } else if ( typeof output === 'object' && output !== null && output.properties && @@ -536,9 +461,7 @@ function formatOutputStructure(outputs: Record, indentLevel = 0): s const nestedResult = formatOutputStructure(output.properties, indentLevel + 1) result += nestedResult - } - // For simple types, show with prefix if nested - else { + } else { result += `| ${prefix}\`${key}\` | ${type} | ${escapedDescription} |\n` } } @@ -546,11 +469,9 @@ function formatOutputStructure(outputs: Record, indentLevel = 0): s return result } -// New function to parse the structured outputs field from ToolConfig function parseToolOutputsField(outputsContent: string): Record { const outputs: Record = {} - // Calculate nesting levels for all braces first const braces: Array<{ type: 'open' | 'close'; pos: number; level: number }> = [] for (let i = 0; i < outputsContent.length; i++) { if (outputsContent[i] === '{') { @@ -560,7 +481,6 @@ function parseToolOutputsField(outputsContent: string): Record { } } - // Calculate actual nesting levels let currentLevel = 0 for (const brace of braces) { if (brace.type === 'open') { @@ -572,7 +492,6 @@ function parseToolOutputsField(outputsContent: string): Record { } } - // Find field definitions and their nesting levels const fieldStartRegex = /(\w+)\s*:\s*{/g let match const fieldPositions: Array<{ name: string; start: number; end: number; level: number }> = [] @@ -581,10 +500,8 @@ function parseToolOutputsField(outputsContent: string): Record { const fieldName = match[1] const bracePos = match.index + match[0].length - 1 - // Find the corresponding opening brace to determine nesting level const openBrace = braces.find((b) => b.type === 'open' && b.pos === bracePos) if (openBrace) { - // Find the matching closing brace let braceCount = 1 let endPos = bracePos + 1 @@ -606,13 +523,11 @@ function parseToolOutputsField(outputsContent: string): Record { } } - // Only process level 0 fields (top-level outputs) const topLevelFields = fieldPositions.filter((f) => f.level === 0) topLevelFields.forEach((field) => { const fieldContent = outputsContent.substring(field.start + 1, field.end - 1).trim() - // Parse the field content const parsedField = parseFieldContent(fieldContent) if (parsedField) { outputs[field.name] = parsedField @@ -622,9 +537,7 @@ function parseToolOutputsField(outputsContent: string): Record { return outputs } -// Helper function to parse individual field content with support for nested structures function parseFieldContent(fieldContent: string): any { - // Extract type and description const typeMatch = fieldContent.match(/type\s*:\s*['"]([^'"]+)['"]/) const descMatch = fieldContent.match(/description\s*:\s*['"`]([^'"`\n]+)['"`]/) @@ -638,7 +551,6 @@ function parseFieldContent(fieldContent: string): any { description: description, } - // Check for properties (nested objects) - only for object types, not arrays if (fieldType === 'object' || fieldType === 'json') { const propertiesRegex = /properties\s*:\s*{/ const propertiesStart = fieldContent.search(propertiesRegex) @@ -648,7 +560,6 @@ function parseFieldContent(fieldContent: string): any { let braceCount = 1 let braceEnd = braceStart + 1 - // Find matching closing brace while (braceEnd < fieldContent.length && braceCount > 0) { if (fieldContent[braceEnd] === '{') braceCount++ else if (fieldContent[braceEnd] === '}') braceCount-- @@ -662,7 +573,6 @@ function parseFieldContent(fieldContent: string): any { } } - // Check for items (array items) - ensure balanced brace matching const itemsRegex = /items\s*:\s*{/ const itemsStart = fieldContent.search(itemsRegex) @@ -671,7 +581,6 @@ function parseFieldContent(fieldContent: string): any { let braceCount = 1 let braceEnd = braceStart + 1 - // Find matching closing brace while (braceEnd < fieldContent.length && braceCount > 0) { if (fieldContent[braceEnd] === '{') braceCount++ else if (fieldContent[braceEnd] === '}') braceCount-- @@ -682,7 +591,6 @@ function parseFieldContent(fieldContent: string): any { const itemsContent = fieldContent.substring(braceStart + 1, braceEnd - 1).trim() const itemsType = itemsContent.match(/type\s*:\s*['"]([^'"]+)['"]/) - // Only look for description before any properties block to avoid picking up nested property descriptions const propertiesStart = itemsContent.search(/properties\s*:\s*{/) const searchContent = propertiesStart >= 0 ? itemsContent.substring(0, propertiesStart) : itemsContent @@ -693,7 +601,6 @@ function parseFieldContent(fieldContent: string): any { description: itemsDesc ? itemsDesc[1] : '', } - // Check if items have properties const itemsPropertiesRegex = /properties\s*:\s*{/ const itemsPropsStart = itemsContent.search(itemsPropertiesRegex) @@ -721,11 +628,9 @@ function parseFieldContent(fieldContent: string): any { return result } -// Helper function to parse properties content recursively function parsePropertiesContent(propertiesContent: string): Record { const properties: Record = {} - // Find property definitions using balanced brace matching, but exclude type-only definitions const propStartRegex = /(\w+)\s*:\s*{/g let match const propPositions: Array<{ name: string; start: number; content: string }> = [] @@ -733,14 +638,12 @@ function parsePropertiesContent(propertiesContent: string): Record while ((match = propStartRegex.exec(propertiesContent)) !== null) { const propName = match[1] - // Skip structural keywords that should never be treated as property names if (propName === 'items' || propName === 'properties') { continue } - const startPos = match.index + match[0].length - 1 // Position of opening brace + const startPos = match.index + match[0].length - 1 - // Find the matching closing brace let braceCount = 1 let endPos = startPos + 1 @@ -756,9 +659,6 @@ function parsePropertiesContent(propertiesContent: string): Record if (braceCount === 0) { const propContent = propertiesContent.substring(startPos + 1, endPos - 1).trim() - // Skip if this is just a type definition (contains only 'type' field) rather than a real property - // This happens with array items definitions like: items: { type: 'string' } - // More precise check: only skip if it ONLY has 'type' and nothing else meaningful const hasDescription = /description\s*:\s*/.test(propContent) const hasProperties = /properties\s*:\s*{/.test(propContent) const hasItems = /items\s*:\s*{/.test(propContent) @@ -778,7 +678,6 @@ function parsePropertiesContent(propertiesContent: string): Record } } - // Process the actual property definitions propPositions.forEach((prop) => { const parsedProp = parseFieldContent(prop.content) if (parsedProp) { @@ -789,26 +688,21 @@ function parsePropertiesContent(propertiesContent: string): Record return properties } -// Find and extract information about a tool async function getToolInfo(toolName: string): Promise<{ description: string params: Array<{ name: string; type: string; required: boolean; description: string }> outputs: Record } | null> { try { - // Split the tool name into parts const parts = toolName.split('_') - // Try to find the correct split point by checking if directories exist let toolPrefix = '' let toolSuffix = '' - // Start from the longest possible prefix and work backwards for (let i = parts.length - 1; i >= 1; i--) { const possiblePrefix = parts.slice(0, i).join('_') const possibleSuffix = parts.slice(i).join('_') - // Check if a directory exists for this prefix const toolDirPath = path.join(rootDir, `apps/sim/tools/${possiblePrefix}`) if (fs.existsSync(toolDirPath) && fs.statSync(toolDirPath).isDirectory()) { @@ -818,29 +712,23 @@ async function getToolInfo(toolName: string): Promise<{ } } - // If no directory was found, fall back to single-part prefix if (!toolPrefix) { toolPrefix = parts[0] toolSuffix = parts.slice(1).join('_') } - // Simplify the file search strategy const possibleLocations = [] - // Most common pattern: suffix.ts file in the prefix directory possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/${toolSuffix}.ts`)) - // Try camelCase version of suffix const camelCaseSuffix = toolSuffix .split('_') .map((part, i) => (i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1))) .join('') possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/${camelCaseSuffix}.ts`)) - // Also check the index.ts file in the tool directory possibleLocations.push(path.join(rootDir, `apps/sim/tools/${toolPrefix}/index.ts`)) - // Try to find the tool definition file let toolFileContent = '' for (const location of possibleLocations) { @@ -855,7 +743,6 @@ async function getToolInfo(toolName: string): Promise<{ return null } - // Extract tool information from the file return extractToolInfo(toolName, toolFileContent) } catch (error) { console.error(`Error getting info for tool ${toolName}:`, error) @@ -863,10 +750,8 @@ async function getToolInfo(toolName: string): Promise<{ } } -// Function to extract content between manual content markers function extractManualContent(existingContent: string): Record { const manualSections: Record = {} - // Improved regex to better handle MDX comments const manualContentRegex = /\{\/\*\s*MANUAL-CONTENT-START:(\w+)\s*\*\/\}([\s\S]*?)\{\/\*\s*MANUAL-CONTENT-END\s*\*\/\}/g @@ -881,7 +766,6 @@ function extractManualContent(existingContent: string): Record { return manualSections } -// Function to merge generated markdown with manual content function mergeWithManualContent( generatedMarkdown: string, existingContent: string | null, @@ -893,18 +777,14 @@ function mergeWithManualContent( console.log('Merging manual content with generated markdown') - // Log what we found for debugging console.log(`Found ${Object.keys(manualSections).length} manual sections`) Object.keys(manualSections).forEach((section) => { console.log(` - ${section}: ${manualSections[section].substring(0, 20)}...`) }) - // Replace placeholders in generated markdown with manual content let mergedContent = generatedMarkdown - // Add manual content for each section we found Object.entries(manualSections).forEach(([sectionName, content]) => { - // Define insertion points for different section types with improved patterns const insertionPoints: Record = { intro: { regex: /`}\s*\/>/, @@ -920,15 +800,12 @@ function mergeWithManualContent( }, } - // Find the appropriate insertion point const insertionPoint = insertionPoints[sectionName] if (insertionPoint) { - // Use regex to find the insertion point const match = mergedContent.match(insertionPoint.regex) if (match && match.index !== undefined) { - // Insert after the matched content const insertPosition = match.index + match[0].length console.log(`Inserting ${sectionName} content after position ${insertPosition}`) mergedContent = `${mergedContent.slice(0, insertPosition)}\n\n{/* MANUAL-CONTENT-START:${sectionName} */}\n${content}\n{/* MANUAL-CONTENT-END */}\n${mergedContent.slice(insertPosition)}` @@ -945,19 +822,15 @@ function mergeWithManualContent( return mergedContent } -// Function to generate documentation for a block async function generateBlockDoc(blockPath: string, icons: Record) { try { - // Extract the block name from the file path const blockFileName = path.basename(blockPath, '.ts') if (blockFileName.endsWith('.test')) { - return // Skip test files + return } - // Read the file content const fileContent = fs.readFileSync(blockPath, 'utf-8') - // Extract block configuration from the file content const blockConfig = extractBlockConfig(fileContent) if (!blockConfig || !blockConfig.type) { @@ -965,7 +838,11 @@ async function generateBlockDoc(blockPath: string, icons: Record return } - // Skip blocks with category 'blocks' (except memory type), and skip specific blocks + if (blockConfig.type.includes('_trigger')) { + console.log(`Skipping ${blockConfig.type} - contains '_trigger'`) + return + } + if ( (blockConfig.category === 'blocks' && blockConfig.type !== 'memory' && @@ -976,23 +853,18 @@ async function generateBlockDoc(blockPath: string, icons: Record return } - // Output file path const outputFilePath = path.join(DOCS_OUTPUT_PATH, `${blockConfig.type}.mdx`) - // IMPORTANT: Check if file already exists and read its content FIRST let existingContent: string | null = null if (fs.existsSync(outputFilePath)) { existingContent = fs.readFileSync(outputFilePath, 'utf-8') console.log(`Existing file found for ${blockConfig.type}.mdx, checking for manual content...`) } - // Extract manual content from existing file before generating new content const manualSections = existingContent ? extractManualContent(existingContent) : {} - // Create the markdown content - now async const markdown = await generateMarkdownForBlock(blockConfig, icons) - // Merge with manual content if we found any let finalContent = markdown if (Object.keys(manualSections).length > 0) { console.log(`Found manual content in ${blockConfig.type}.mdx, merging...`) @@ -1001,7 +873,6 @@ async function generateBlockDoc(blockPath: string, icons: Record console.log(`No manual content found in ${blockConfig.type}.mdx`) } - // Write the markdown file fs.writeFileSync(outputFilePath, finalContent) console.log(`Generated documentation for ${blockConfig.type}`) } catch (error) { @@ -1009,7 +880,6 @@ async function generateBlockDoc(blockPath: string, icons: Record } } -// Update generateMarkdownForBlock to remove placeholders async function generateMarkdownForBlock( blockConfig: BlockConfig, icons: Record @@ -1026,48 +896,39 @@ async function generateMarkdownForBlock( tools = { access: [] }, } = blockConfig - // Get SVG icon if available const iconSvg = iconName && icons[iconName] ? icons[iconName] : null - // Generate the outputs section let outputsSection = '' if (outputs && Object.keys(outputs).length > 0) { outputsSection = '## Outputs\n\n' - // Create the base outputs table outputsSection += '| Output | Type | Description |\n' outputsSection += '| ------ | ---- | ----------- |\n' - // Process each output field for (const outputKey in outputs) { const output = outputs[outputKey] - // Escape special characters in the description that could break markdown tables const escapedDescription = output.description ? output.description - .replace(/\|/g, '\\|') // Escape pipe characters - .replace(/\{/g, '\\{') // Escape curly braces - .replace(/\}/g, '\\}') // Escape curly braces - .replace(/\(/g, '\\(') // Escape opening parentheses - .replace(/\)/g, '\\)') // Escape closing parentheses - .replace(/\[/g, '\\[') // Escape opening brackets - .replace(/\]/g, '\\]') // Escape closing brackets - .replace(//g, '>') // Convert greater than to HTML entity + .replace(/\|/g, '\\|') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(//g, '>') : `Output from ${outputKey}` if (typeof output.type === 'string') { - // Simple output with explicit type outputsSection += `| \`${outputKey}\` | ${output.type} | ${escapedDescription} |\n` } else if (output.type && typeof output.type === 'object') { - // For cases where output.type is an object containing field types outputsSection += `| \`${outputKey}\` | object | ${escapedDescription} |\n` - // Add properties directly to the main table with indentation for (const propName in output.type) { const propType = output.type[propName] - // Get description from comments if available const commentMatch = propName && output.type[propName]._comment ? output.type[propName]._comment @@ -1076,24 +937,21 @@ async function generateMarkdownForBlock( outputsSection += `| ↳ \`${propName}\` | ${propType} | ${commentMatch} |\n` } } else if (output.properties) { - // Complex output with properties outputsSection += `| \`${outputKey}\` | object | ${escapedDescription} |\n` - // Add properties directly to the main table with indentation for (const propName in output.properties) { const prop = output.properties[propName] - // Escape special characters in the description const escapedPropertyDescription = prop.description ? prop.description - .replace(/\|/g, '\\|') // Escape pipe characters - .replace(/\{/g, '\\{') // Escape curly braces - .replace(/\}/g, '\\}') // Escape curly braces - .replace(/\(/g, '\\(') // Escape opening parentheses - .replace(/\)/g, '\\)') // Escape closing parentheses - .replace(/\[/g, '\\[') // Escape opening brackets - .replace(/\]/g, '\\]') // Escape closing brackets - .replace(//g, '>') // Convert greater than to HTML entity + .replace(/\|/g, '\\|') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(//g, '>') : `The ${propName} of the ${outputKey}` outputsSection += `| ↳ \`${propName}\` | ${prop.type} | ${escapedPropertyDescription} |\n` @@ -1104,16 +962,14 @@ async function generateMarkdownForBlock( outputsSection = 'This block does not produce any outputs.' } - // Create tools section with more details let toolsSection = '' if (tools.access?.length) { toolsSection = '## Tools\n\n' - // For each tool, try to find its definition and extract parameter information for (const tool of tools.access) { toolsSection += `### \`${tool}\`\n\n` - // Get dynamic tool information + console.log(`Getting info for tool: ${tool}`) const toolInfo = await getToolInfo(tool) if (toolInfo) { @@ -1121,45 +977,37 @@ async function generateMarkdownForBlock( toolsSection += `${toolInfo.description}\n\n` } - // Add Input Parameters section for the tool toolsSection += '#### Input\n\n' toolsSection += '| Parameter | Type | Required | Description |\n' toolsSection += '| --------- | ---- | -------- | ----------- |\n' if (toolInfo.params.length > 0) { - // Use dynamically extracted parameters for (const param of toolInfo.params) { - // Escape special characters in the description that could break markdown tables const escapedDescription = param.description ? param.description - .replace(/\|/g, '\\|') // Escape pipe characters - .replace(/\{/g, '\\{') // Escape curly braces - .replace(/\}/g, '\\}') // Escape curly braces - .replace(/\(/g, '\\(') // Escape opening parentheses - .replace(/\)/g, '\\)') // Escape closing parentheses - .replace(/\[/g, '\\[') // Escape opening brackets - .replace(/\]/g, '\\]') // Escape closing brackets - .replace(//g, '>') // Convert greater than to HTML entity + .replace(/\|/g, '\\|') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(//g, '>') : 'No description' toolsSection += `| \`${param.name}\` | ${param.type} | ${param.required ? 'Yes' : 'No'} | ${escapedDescription} |\n` } } - // Add Output Parameters section for the tool toolsSection += '\n#### Output\n\n' - // Always prefer tool-specific outputs over block outputs for accuracy if (Object.keys(toolInfo.outputs).length > 0) { - // Use tool-specific outputs (most accurate) toolsSection += '| Parameter | Type | Description |\n' toolsSection += '| --------- | ---- | ----------- |\n' - // Use the enhanced formatOutputStructure function to handle nested structures toolsSection += formatOutputStructure(toolInfo.outputs) } else if (Object.keys(outputs).length > 0) { - // Fallback to block outputs only if no tool outputs are available toolsSection += '| Parameter | Type | Description |\n' toolsSection += '| --------- | ---- | ----------- |\n' @@ -1178,7 +1026,6 @@ async function generateMarkdownForBlock( } } - // Escape special characters in the description const escapedDescription = description .replace(/\|/g, '\\|') .replace(/\{/g, '\\{') @@ -1201,13 +1048,11 @@ async function generateMarkdownForBlock( } } - // Add usage instructions if available in block config let usageInstructions = '' if (longDescription) { usageInstructions = `## Usage Instructions\n\n${longDescription}\n\n` } - // Generate the markdown content without any placeholders return `--- title: ${name} description: ${description} @@ -1233,21 +1078,16 @@ ${toolsSection} ` } -// Main function to generate all block docs async function generateAllBlockDocs() { try { - // Extract icons first const icons = extractIcons() - // Get all block files const blockFiles = await glob(`${BLOCKS_PATH}/*.ts`) - // Generate docs for each block for (const blockFile of blockFiles) { await generateBlockDoc(blockFile, icons) } - // Update the meta.json file updateMetaJson() return true @@ -1257,18 +1097,14 @@ async function generateAllBlockDocs() { } } -// Function to update the meta.json file with all blocks function updateMetaJson() { const metaJsonPath = path.join(DOCS_OUTPUT_PATH, 'meta.json') - // Get all MDX files in the tools directory const blockFiles = fs .readdirSync(DOCS_OUTPUT_PATH) .filter((file: string) => file.endsWith('.mdx')) .map((file: string) => path.basename(file, '.mdx')) - // Create meta.json structure - // Keep "index" as the first item if it exists const items = [ ...(blockFiles.includes('index') ? ['index'] : []), ...blockFiles.filter((file: string) => file !== 'index').sort(), @@ -1278,11 +1114,9 @@ function updateMetaJson() { items, } - // Write the meta.json file fs.writeFileSync(metaJsonPath, JSON.stringify(metaJson, null, 2)) } -// Run the script generateAllBlockDocs() .then((success) => { if (success) {