feat(dashboard): media providers pages + Web Fetch category

open-sse/executors/firecrawl-fetch.ts

@@ -0,0 +1,143 @@
+/**
+ * Firecrawl Web Fetch Executor
+ *
+ * Fetches content from a URL using the Firecrawl scrape API.
+ * POST https://api.firecrawl.dev/v1/scrape
+ *
+ * Free tier: 500 fetches/month, no credit card required.
+ * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape
+ */
+
+import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
+import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
+
+const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1";
+const FIRECRAWL_TIMEOUT_MS = 30_000;
+
+function mapFormat(format: WebFetchFormat): string {
+  switch (format) {
+    case "html":
+      return "html";
+    case "links":
+      return "links";
+    case "screenshot":
+      return "screenshot";
+    case "markdown":
+    default:
+      return "markdown";
+  }
+}
+
+interface FirecrawlScrapeOptions {
+  url: string;
+  format: WebFetchFormat;
+  depth: number;
+  waitForSelector?: string;
+  includeMetadata: boolean;
+  credentials: WebFetchCredentials;
+}
+
+/**
+ * Execute a Firecrawl scrape request.
+ */
+export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebFetchResult> {
+  const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts;
+
+  if (!credentials.apiKey) {
+    const body = buildErrorBody(401, "Firecrawl API key required");
+    return { success: false, status: 401, error: body.error.message };
+  }
+
+  const formats = [mapFormat(format)];
+
+  const requestBody: Record<string, unknown> = {
+    url,
+    formats,
+  };
+
+  if (includeMetadata) {
+    requestBody.includeTags = ["title", "description", "og:title", "og:description"];
+  }
+
+  if (depth > 0) {
+    requestBody.maxDepth = depth;
+  }
+
+  if (waitForSelector) {
+    requestBody.waitFor = waitForSelector;
+  }
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${credentials.apiKey}`,
+      },
+      body: JSON.stringify(requestBody),
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      const rawError = await response.text().catch(() => `HTTP ${response.status}`);
+      const msg = sanitizeErrorMessage(`Firecrawl error ${response.status}: ${rawError}`);
+      const body = buildErrorBody(response.status, msg);
+      return { success: false, status: response.status, error: body.error.message };
+    }
+
+    const data = (await response.json()) as Record<string, unknown>;
+
+    const scraped = (data.data as Record<string, unknown> | null) ?? {};
+
+    const content =
+      format === "html"
+        ? String(scraped.html ?? "")
+        : format === "links"
+          ? JSON.stringify(scraped.links ?? [])
+          : String(scraped.markdown ?? scraped.content ?? "");
+
+    const rawLinks = scraped.links;
+    const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
+
+    const rawMeta = scraped.metadata as Record<string, unknown> | null | undefined;
+    const metadata = includeMetadata
+      ? {
+          title: rawMeta?.title != null ? String(rawMeta.title) : null,
+          description: rawMeta?.description != null ? String(rawMeta.description) : null,
+        }
+      : null;
+
+    const screenshotUrl =
+      format === "screenshot"
+        ? scraped.screenshot != null
+          ? String(scraped.screenshot)
+          : null
+        : null;
+
+    return {
+      success: true,
+      data: {
+        provider: "firecrawl",
+        url,
+        content,
+        links,
+        metadata,
+        screenshot_url: screenshotUrl,
+      },
+    };
+  } catch (err: unknown) {
+    if (err instanceof Error && err.name === "AbortError") {
+      const body = buildErrorBody(504, "Firecrawl request timed out");
+      return { success: false, status: 504, error: body.error.message };
+    }
+    const msg =
+      err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
+    const body = buildErrorBody(502, msg);
+    return { success: false, status: 502, error: body.error.message };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}

open-sse/executors/jina-reader-fetch.ts

@@ -0,0 +1,119 @@
+/**
+ * Jina Reader Web Fetch Executor
+ *
+ * Fetches content from a URL using the Jina Reader API.
+ * GET https://r.jina.ai/{url}
+ *
+ * Free tier: 1M fetches/month.
+ * Docs: https://jina.ai/reader/
+ */
+
+import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
+import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
+
+const JINA_READER_BASE = "https://r.jina.ai";
+const JINA_TIMEOUT_MS = 30_000;
+
+interface JinaReaderFetchOptions {
+  url: string;
+  format: WebFetchFormat;
+  includeMetadata: boolean;
+  credentials: WebFetchCredentials;
+}
+
+/**
+ * Execute a Jina Reader fetch request.
+ * Jina Reader uses a URL-based approach: GET https://r.jina.ai/<url>
+ */
+export async function jinaReaderFetch(opts: JinaReaderFetchOptions): Promise<WebFetchResult> {
+  const { url, format, includeMetadata, credentials } = opts;
+
+  if (!credentials.apiKey) {
+    const body = buildErrorBody(401, "Jina Reader API key required");
+    return { success: false, status: 401, error: body.error.message };
+  }
+
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${credentials.apiKey}`,
+    Accept: "application/json",
+    "X-Return-Format": format === "html" ? "html" : "markdown",
+  };
+
+  if (includeMetadata) {
+    headers["X-With-Generated-Alt"] = "true";
+  }
+
+  if (format === "links") {
+    headers["X-Gather-All-Links-At-The-End"] = "true";
+  }
+
+  const encodedUrl = encodeURIComponent(url);
+  const requestUrl = `${JINA_READER_BASE}/${encodedUrl}`;
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), JINA_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(requestUrl, {
+      method: "GET",
+      headers,
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      const rawError = await response.text().catch(() => `HTTP ${response.status}`);
+      const msg = sanitizeErrorMessage(`Jina Reader error ${response.status}: ${rawError}`);
+      const body = buildErrorBody(response.status, msg);
+      return { success: false, status: response.status, error: body.error.message };
+    }
+
+    // Jina Reader returns JSON with data.content or plain text
+    const contentType = response.headers.get("content-type") ?? "";
+    let content = "";
+    let links: string[] = [];
+    let metadata: { title: string | null; description: string | null } | null = null;
+
+    if (contentType.includes("application/json")) {
+      const json = (await response.json()) as Record<string, unknown>;
+      const responseData = (json.data as Record<string, unknown> | null) ?? {};
+      content = String(responseData.content ?? responseData.text ?? "");
+
+      if (includeMetadata) {
+        metadata = {
+          title: responseData.title != null ? String(responseData.title) : null,
+          description: responseData.description != null ? String(responseData.description) : null,
+        };
+      }
+
+      if (format === "links") {
+        const rawLinks = responseData.links;
+        links = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
+      }
+    } else {
+      content = await response.text();
+    }
+
+    return {
+      success: true,
+      data: {
+        provider: "jina-reader",
+        url,
+        content,
+        links,
+        metadata,
+        screenshot_url: null,
+      },
+    };
+  } catch (err: unknown) {
+    if (err instanceof Error && err.name === "AbortError") {
+      const body = buildErrorBody(504, "Jina Reader request timed out");
+      return { success: false, status: 504, error: body.error.message };
+    }
+    const msg =
+      err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
+    const body = buildErrorBody(502, msg);
+    return { success: false, status: 502, error: body.error.message };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}

open-sse/executors/tavily-fetch.ts

@@ -0,0 +1,103 @@
+/**
+ * Tavily Web Fetch Executor
+ *
+ * Fetches content from a URL using the Tavily Extract API.
+ * POST https://api.tavily.com/extract
+ *
+ * Free tier: included in Tavily plan.
+ * Docs: https://docs.tavily.com/documentation/api-reference/endpoint/post-extract
+ */
+
+import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
+import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
+
+const TAVILY_EXTRACT_URL = "https://api.tavily.com/extract";
+const TAVILY_TIMEOUT_MS = 30_000;
+
+interface TavilyFetchOptions {
+  url: string;
+  format: WebFetchFormat;
+  includeMetadata: boolean;
+  credentials: WebFetchCredentials;
+}
+
+/**
+ * Execute a Tavily extract request.
+ * Tavily Extract returns the raw content of a given URL.
+ */
+export async function tavilyFetch(opts: TavilyFetchOptions): Promise<WebFetchResult> {
+  const { url, includeMetadata, credentials } = opts;
+
+  if (!credentials.apiKey) {
+    const body = buildErrorBody(401, "Tavily API key required");
+    return { success: false, status: 401, error: body.error.message };
+  }
+
+  const requestBody: Record<string, unknown> = {
+    api_key: credentials.apiKey,
+    urls: [url],
+    extract_depth: "basic",
+  };
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), TAVILY_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(TAVILY_EXTRACT_URL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${credentials.apiKey}`,
+      },
+      body: JSON.stringify(requestBody),
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      const rawError = await response.text().catch(() => `HTTP ${response.status}`);
+      const msg = sanitizeErrorMessage(`Tavily error ${response.status}: ${rawError}`);
+      const body = buildErrorBody(response.status, msg);
+      return { success: false, status: response.status, error: body.error.message };
+    }
+
+    const data = (await response.json()) as Record<string, unknown>;
+
+    const results = data.results as Array<Record<string, unknown>> | null;
+    const firstResult = results?.[0] ?? {};
+
+    const content = String(firstResult.raw_content ?? firstResult.content ?? "");
+
+    const rawLinks = firstResult.links;
+    const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
+
+    const metadata = includeMetadata
+      ? {
+          title: firstResult.title != null ? String(firstResult.title) : null,
+          description: null,
+        }
+      : null;
+
+    return {
+      success: true,
+      data: {
+        provider: "tavily-search",
+        url,
+        content,
+        links,
+        metadata,
+        screenshot_url: null,
+      },
+    };
+  } catch (err: unknown) {
+    if (err instanceof Error && err.name === "AbortError") {
+      const body = buildErrorBody(504, "Tavily request timed out");
+      return { success: false, status: 504, error: body.error.message };
+    }
+    const msg =
+      err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
+    const body = buildErrorBody(502, msg);
+    return { success: false, status: 502, error: body.error.message };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}