feat(search): add Firecrawl search provider

changelog.d/features/8814-firecrawl-search.md

@@ -0,0 +1 @@
+- **feat(search):** add Firecrawl to `POST /v1/search` supported providers with two sources (web|news) ([#8814](https://github.com/diegosouzapw/OmniRoute/pull/8814)) — thanks @allanvb

open-sse/config/searchRegistry.ts

@@ -113,6 +113,22 @@ export const SEARCH_PROVIDERS: Record<string, SearchProviderConfig> = {
     cacheTTLMs: 5 * 60 * 1000,
   },
 
+  firecrawl: {
+    id: "firecrawl",
+    name: "Firecrawl",
+    baseUrl: "https://api.firecrawl.dev/v2/search",
+    method: "POST",
+    authType: "apikey",
+    authHeader: "bearer",
+    costPerQuery: 0.002,
+    freeMonthlyQuota: 1000,
+    searchTypes: ["web", "news"],
+    defaultMaxResults: 5,
+    maxMaxResults: 100,
+    timeoutMs: 30_000,
+    cacheTTLMs: 5 * 60 * 1000,
+  },
+
   "google-pse-search": {
     id: "google-pse-search",
     name: "Google Programmable Search",

open-sse/executors/firecrawl-fetch.ts

@@ -4,7 +4,7 @@
  * Fetches content from a URL using the Firecrawl scrape API.
  * POST https://api.firecrawl.dev/v1/scrape
  *
- * Free tier: 500 fetches/month, no credit card required.
+ * Free tier: 1,000 credits/month, no credit card required.
  * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape
  *
  * Self-hosted: set FIRECRAWL_BASE_URL to point at a self-hosted Firecrawl

open-sse/handlers/search.ts

@@ -3,9 +3,10 @@ import { randomUUID } from "crypto";
  * Search Handler
  *
  * Handles POST /v1/search requests.
- * Routes to 11 search providers with automatic failover:
+ * Routes to search providers with automatic failover:
  *   serper-search, brave-search, perplexity-search, exa-search, tavily-search,
- *   google-pse-search, linkup-search, searchapi-search, youcom-search, searxng-search, ollama-search, zai-search
+ *   firecrawl, google-pse-search, linkup-search, searchapi-search,
+ *   youcom-search, searxng-search, ollama-search, zai-search, duckduckgo-free
  *
  * Request format:
  * {
@@ -18,6 +19,7 @@ import { randomUUID } from "crypto";
 
 import { getSearchProvider, type SearchProviderConfig } from "../config/searchRegistry.ts";
 import { buildPerplexityRequest, parsePerplexitySearchOptions } from "./search/perplexitySearch.ts";
+import * as fcSearch from "./search/firecrawlSearch.ts";
 import { freeWebSearch } from "../services/freeWebSearch.ts";
 import { saveCallLog } from "@/lib/usageDb";
 import { safeOutboundFetch } from "@/shared/network/safeOutboundFetch";
@@ -26,8 +28,6 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/
 import { z } from "zod";
 import { sanitizeErrorMessage } from "../utils/error.ts";
 
-// ── Types ────────────────────────────────────────────────────────────────
-
 export interface SearchResult {
   title: string;
   url: string;
@@ -600,6 +600,7 @@ function buildRequest(
   if (config.id === "perplexity-search") return buildPerplexityRequest(config, params);
   if (config.id === "exa-search") return buildExaRequest(config, params);
   if (config.id === "tavily-search") return buildTavilyRequest(config, params);
+  if (config.id === "firecrawl") return fcSearch.buildFirecrawlSearchRequest(config, params);
   if (config.id === "google-pse-search") return buildGooglePseRequest(config, params);
   if (config.id === "linkup-search") return buildLinkupRequest(config, params);
   if (config.id === "searchapi-search") return buildSearchApiRequest(config, params);
@@ -1166,6 +1167,8 @@ function normalizeResponse(
     return normalizePerplexityResponse(data, query, searchType);
   if (providerId === "exa-search") return normalizeExaResponse(data, query, searchType);
   if (providerId === "tavily-search") return normalizeTavilyResponse(data, query, searchType);
+  if (providerId === "firecrawl")
+    return fcSearch.normalizeFirecrawlSearchResponse(data, searchType, makeResult);
   if (providerId === "google-pse-search")
     return normalizeGooglePseResponse(data, query, searchType);
   if (providerId === "linkup-search") return normalizeLinkupResponse(data, query, searchType);
@@ -1175,9 +1178,6 @@ function normalizeResponse(
   if (providerId === "ollama-search") return normalizeOllamaResponse(data, query, searchType);
   return { results: [], totalResults: null };
 }
-
-// ── Main Handler ────────────────────────────────────────────────────────
-
 export async function handleSearch(options: SearchHandlerOptions): Promise<SearchHandlerResult> {
   const {
     query,

open-sse/handlers/search/firecrawlSearch.ts

@@ -0,0 +1,142 @@
+import type { SearchProviderConfig } from "../../config/searchRegistry.ts";
+
+export interface FirecrawlSearchParams {
+  query: string;
+  searchType: string;
+  maxResults: number;
+  token?: string;
+  country?: string;
+  language?: string;
+  timeRange?: string;
+  domainFilter?: string[];
+}
+
+export type FirecrawlNormalizedHit = {
+  title?: string;
+  url?: string;
+  snippet?: string;
+  published_at?: string | null;
+  image_url?: string;
+  source_type?: string;
+};
+
+type FirecrawlSearchHit = {
+  title?: string;
+  url?: string;
+  link?: string;
+  description?: string;
+  snippet?: string;
+  markdown?: string;
+  content?: string;
+  date?: string | null;
+  published_at?: string | null;
+  imageUrl?: string | null;
+  metadata?: {
+    title?: string;
+    sourceURL?: string;
+    publishedTime?: string | null;
+  };
+};
+
+export type FirecrawlSearchEnvelope = {
+  data?: {
+    web?: FirecrawlSearchHit[];
+    news?: FirecrawlSearchHit[];
+  };
+};
+
+function parseDomainFilter(domainFilter?: string[]): { includes: string[]; excludes: string[] } {
+  if (!domainFilter?.length) return { includes: [], excludes: [] };
+  const includes = domainFilter.filter((d) => !d.startsWith("-"));
+  const excludes = domainFilter.filter((d) => d.startsWith("-")).map((d) => d.slice(1));
+  return { includes, excludes };
+}
+
+function firecrawlSearchTbs(timeRange?: string): string | undefined {
+  if (!timeRange || timeRange === "any") return undefined;
+  const map: Record<string, string> = {
+    day: "qdr:d",
+    week: "qdr:w",
+    month: "qdr:m",
+    year: "qdr:y",
+  };
+  return map[timeRange];
+}
+
+export function buildFirecrawlSearchRequest(
+  config: SearchProviderConfig,
+  params: FirecrawlSearchParams
+): { url: string; init: RequestInit } {
+  const envBase = process.env.FIRECRAWL_BASE_URL?.trim().replace(/\/+$/, "");
+  const url = envBase ? `${envBase}/v2/search` : config.baseUrl;
+  const { includes, excludes } = parseDomainFilter(params.domainFilter);
+  const source = params.searchType === "news" ? "news" : "web";
+
+  const body: Record<string, unknown> = {
+    query: params.query,
+    limit: params.maxResults,
+    sources: [source],
+  };
+  if (params.country) body.country = params.country.toLowerCase();
+  if (params.language) body.lang = params.language;
+  const tbs = firecrawlSearchTbs(params.timeRange);
+  if (tbs) body.tbs = tbs;
+  if (includes.length) body.includeDomains = includes.map((d) => d.toLowerCase());
+  if (excludes.length) body.excludeDomains = excludes.map((d) => d.toLowerCase());
+
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (params.token) {
+    headers.Authorization = `Bearer ${params.token}`;
+  }
+
+  return {
+    url,
+    init: {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+    },
+  };
+}
+
+function pickFirecrawlSearchItems(
+  data: FirecrawlSearchEnvelope,
+  searchType: string
+): FirecrawlSearchHit[] {
+  const buckets = data?.data;
+  if (!buckets || Array.isArray(buckets)) return [];
+  const items = searchType === "news" ? buckets.news : buckets.web;
+  return Array.isArray(items) ? items : [];
+}
+
+export function collectFirecrawlSearchHits(
+  data: FirecrawlSearchEnvelope,
+  searchType: string
+): FirecrawlNormalizedHit[] {
+  const isNews = searchType === "news";
+  return pickFirecrawlSearchItems(data, searchType).map((item) => ({
+    title: item.title || item.metadata?.title || "",
+    url: item.url || item.metadata?.sourceURL || item.link || "",
+    snippet:
+      item.description ||
+      item.snippet ||
+      item.markdown?.slice(0, 300) ||
+      item.content?.slice(0, 300) ||
+      "",
+    published_at: item.date || item.published_at || item.metadata?.publishedTime || null,
+    image_url: item.imageUrl || undefined,
+    source_type: isNews ? "news" : undefined,
+  }));
+}
+
+export function normalizeFirecrawlSearchResponse<T>(
+  data: FirecrawlSearchEnvelope,
+  searchType: string,
+  makeResult: (providerId: string, item: FirecrawlNormalizedHit, idx: number, now: string) => T
+): { results: T[]; totalResults: number | null } {
+  const now = new Date().toISOString();
+  const results = collectFirecrawlSearchHits(data, searchType).map((item, idx) =>
+    makeResult("firecrawl", item, idx, now)
+  );
+  return { results, totalResults: results.length };
+}

open-sse/services/firecrawlQuotaFetcher.ts

@@ -1,7 +1,7 @@
 /**
  * firecrawlQuotaFetcher.ts — Firecrawl team credit quota
  *
- * Live credit pool for the `firecrawl` connection (web fetch + firecrawl-search).
+ * Live credit pool for the `firecrawl` connection (web fetch + search).
  *
  * Endpoint:
  *   GET https://api.firecrawl.dev/v2/team/credit-usage

src/app/api/search/providers/route.ts

@@ -30,7 +30,7 @@ const FETCH_PROVIDERS: FetchProviderDef[] = [
     id: "firecrawl",
     name: "Firecrawl",
     costPerQuery: 0.002,
-    freeMonthlyQuota: 500,
+    freeMonthlyQuota: 1000,
     fetchFormats: ["markdown", "html", "links", "screenshot"],
   },
   {

src/shared/constants/providers/apikey/specialty-media.ts

@@ -257,21 +257,6 @@ export const APIKEY_PROVIDERS_SPECIALTY = {
     freeNote: "Free-tier API key via signup, no credit card required.",
     authHint: "Bearer API key for the Mixedbread embeddings API.",
   },
-  firecrawl: {
-    id: "firecrawl",
-    alias: "fc",
-    name: "Firecrawl",
-    icon: "language",
-    color: "#FB923C",
-    textIcon: "FC",
-    website: "https://firecrawl.dev",
-    hasFree: true,
-    notice: {
-      text: "Free tier: 500 fetches/month, no credit card needed.",
-      apiKeyUrl: "https://firecrawl.dev/app/api-keys",
-    },
-    serviceKinds: ["webFetch"],
-  },
   "jina-reader": {
     id: "jina-reader",
     alias: "jr",

src/shared/constants/providers/search.ts

@@ -60,6 +60,21 @@ export const SEARCH_PROVIDERS = {
     authHint: "API key from app.tavily.com (format: tvly-...)",
     serviceKinds: ["webSearch", "webFetch"],
   },
+  firecrawl: {
+    id: "firecrawl",
+    alias: "fc",
+    name: "Firecrawl",
+    icon: "language",
+    color: "#FB923C",
+    textIcon: "FC",
+    website: "https://firecrawl.dev",
+    hasFree: true,
+    notice: {
+      text: "Free tier: 1,000 credits/month. Powers /v1/web/fetch and /v1/search.",
+      apiKeyUrl: "https://firecrawl.dev/app/api-keys",
+    },
+    serviceKinds: ["webSearch", "webFetch"],
+  },
   "google-pse-search": {
     id: "google-pse-search",
     alias: "google-pse",

src/shared/validation/schemas/apiV1.ts

@@ -283,6 +283,7 @@ export const v1SearchSchema = z
         "perplexity-search",
         "exa-search",
         "tavily-search",
+        "firecrawl",
         "google-pse-search",
         "linkup-search",
         "ollama-search",

stryker.conf.json

@@ -204,6 +204,7 @@
       "tests/unit/headroom-codex-quota-snapshot-6379.test.ts",
       "tests/unit/headroom-proxy-lifecycle.test.ts",
       "tests/unit/idempotency-fusion-collision.test.ts",
+      "tests/unit/isLocalStreamLifecycleError-abort-shape.test.ts",
       "tests/unit/issue-6343-v0-web-alias-collision.test.ts",
       "tests/unit/issue-6638-ollama-quota.test.ts",
       "tests/unit/issue-6686-quota-preflight-coverage.test.ts",

tests/integration/search-providers-catalog.test.ts

@@ -2,7 +2,7 @@
  * Integration tests for GET /api/search/providers — extended catalog (F4).
  *
  * Tests:
- * - Returns 16 items total (13 search + 3 fetch providers).
+ * - Returns 18 items total (14 search + 4 fetch providers).
  * - Each item carries the correct `kind` field.
  * - Status reflects actual DB credential state:
  *   - "configured"  when an active, non-rate-limited connection exists.
@@ -48,10 +48,10 @@ const route = await import("../../src/app/api/search/providers/route.ts");
 // Constants
 // ---------------------------------------------------------------------------
 
-// 13 search-kind providers: serper, brave, perplexity, exa, tavily, google-pse,
-// linkup, searchapi, youcom, searxng, ollama, zai + duckduckgo-free (added in the
-// v3.8.27 cycle, registry open-sse/config/searchRegistry.ts).
-const EXPECTED_SEARCH_COUNT = 13;
+// 14 search-kind providers: serper, brave, perplexity, exa, tavily, firecrawl,
+// google-pse, linkup, searchapi, youcom, searxng, ollama, zai + duckduckgo-free
+// (registry open-sse/config/searchRegistry.ts).
+const EXPECTED_SEARCH_COUNT = 14;
 const EXPECTED_FETCH_COUNT = 4;
 const EXPECTED_TOTAL = EXPECTED_SEARCH_COUNT + EXPECTED_FETCH_COUNT;
 
@@ -295,7 +295,7 @@ test("search-providers-catalog: fetch providers have correct metadata", async ()
   const firecrawl = fetchProviders.find((p: { id: string }) => p.id === "firecrawl");
   assert.equal(firecrawl.name, "Firecrawl");
   assert.equal(firecrawl.costPerQuery, 0.002);
-  assert.equal(firecrawl.freeMonthlyQuota, 500);
+  assert.equal(firecrawl.freeMonthlyQuota, 1000);
   assert.ok(Array.isArray(firecrawl.fetchFormats), "fetchFormats must be an array");
   assert.ok(
     firecrawl.fetchFormats.includes("markdown"),
@@ -348,6 +348,14 @@ test("search-providers-catalog: search providers have correct fields", async ()
   assert.ok(serper, "serper-search must be in search providers");
   assert.ok(serper.searchTypes.includes("web"), "serper must support web search");
   assert.equal(serper.kind, "search");
+
+  const firecrawlSearch = searchProviders.find((p: { id: string }) => p.id === "firecrawl");
+  assert.ok(firecrawlSearch, "firecrawl must be in search providers");
+  assert.equal(firecrawlSearch.kind, "search");
+  assert.equal(firecrawlSearch.costPerQuery, 0.002);
+  assert.equal(firecrawlSearch.freeMonthlyQuota, 1000);
+  assert.ok(firecrawlSearch.searchTypes.includes("web"), "firecrawl must support web");
+  assert.ok(firecrawlSearch.searchTypes.includes("news"), "firecrawl must support news");
 });
 
 test("search-providers-catalog: response validates against SearchProviderCatalogResponseSchema", async () => {