diff --git a/.env.example b/.env.example index afac229a5ca..26c1e78ba97 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,13 @@ CLOUD_URL=https://9router.com NEXT_PUBLIC_BASE_URL=http://localhost:20128 NEXT_PUBLIC_CLOUD_URL=https://9router.com +# Firecrawl web fetch provider +# Leave FIRECRAWL_API_KEY empty to route to a self-hosted instance. +FIRECRAWL_BASE_URL=https://api.firecrawl.dev +FIRECRAWL_API_KEY= +FIRECRAWL_TIMEOUT_MS=30000 +FIRECRAWL_DEFAULT_FORMAT=markdown + # Optional outbound proxy variables for upstream provider calls # Lowercase variants are also supported: http_proxy, https_proxy, all_proxy, no_proxy # HTTP_PROXY=http://127.0.0.1:7890 diff --git a/open-sse/handlers/fetch/index.js b/open-sse/handlers/fetch/index.js index da1c2303ac8..1e7741c5b39 100644 --- a/open-sse/handlers/fetch/index.js +++ b/open-sse/handlers/fetch/index.js @@ -4,6 +4,17 @@ const DEFAULT_TIMEOUT_MS = 15000; const DEFAULT_FORMAT = "markdown"; +function getDefaultTimeoutMs() { + const env = process.env.FIRECRAWL_TIMEOUT_MS; + if (!env) return DEFAULT_TIMEOUT_MS; + const n = Number(env); + return Number.isFinite(n) && n > 0 ? n : DEFAULT_TIMEOUT_MS; +} + +function getDefaultFormat() { + return process.env.FIRECRAWL_DEFAULT_FORMAT || DEFAULT_FORMAT; +} + /** * @typedef {Object} FetchResult * @property {boolean} success @@ -93,15 +104,15 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr return { success: false, status: 400, error: "provider is required" }; } - const fmt = format || DEFAULT_FORMAT; - const timeoutMs = providerConfig?.timeoutMs || DEFAULT_TIMEOUT_MS; + const fmt = format || getDefaultFormat(); + const timeoutMs = providerConfig?.timeoutMs || getDefaultTimeoutMs(); const apiKey = credentials?.apiKey || credentials?.key || credentials?.token || ""; const costPerQuery = providerConfig?.costPerQuery ?? null; const startedAt = Date.now(); try { - if (provider === "firecrawl") { - return await runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }); + if (provider === "firecrawl" || provider === "firecrawl_custom") { + return await runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, provider }); } if (provider === "jina-reader") { return await runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }); @@ -119,19 +130,34 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr } } -async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) { +async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, provider }) { + const isCustom = provider === "firecrawl_custom"; + + if (!isCustom && !apiKey) { + return { success: false, status: 400, error: "FIRECRAWL_API_KEY is required for the official Firecrawl provider" }; + } + + const baseUrl = isCustom + ? (process.env.FIRECRAWL_BASE_URL || "http://127.0.0.1:3002") + : (process.env.FIRECRAWL_BASE_URL || "https://api.firecrawl.dev"); + const endpoint = isCustom ? "/v2/scrape" : "/v1/scrape"; + + const headers = { "content-type": "application/json" }; + if (!isCustom && apiKey) { + headers.authorization = `Bearer ${apiKey}`; + } + const upstreamStart = Date.now(); - const r = await tryFetch("https://api.firecrawl.dev/v1/scrape", { + const r = await tryFetch(`${baseUrl}${endpoint}`, { method: "POST", - headers: { - "content-type": "application/json", - ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}) - }, + headers, body: JSON.stringify({ url, formats: [fmt] }) }, timeoutMs); if (!r.ok) { - return { success: false, status: r.timeout ? 504 : 502, error: r.error }; + const status = r.timeout ? 504 : 502; + const error = isCustom ? `Custom Firecrawl instance unreachable: ${r.error}` : r.error; + return { success: false, status, error }; } const upstreamMs = Date.now() - upstreamStart; const { json } = await readJsonOrText(r.res); @@ -144,7 +170,7 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe return { success: true, data: buildData({ - provider: "firecrawl", url, title, format: fmt, text, + provider, url, title, format: fmt, text, costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs }) }; diff --git a/open-sse/providers/registry/firecrawl_custom.js b/open-sse/providers/registry/firecrawl_custom.js new file mode 100644 index 00000000000..7d70d479173 --- /dev/null +++ b/open-sse/providers/registry/firecrawl_custom.js @@ -0,0 +1,27 @@ +export default { + id: "firecrawl_custom", + alias: "firecrawl_custom", + display: { + name: "Firecrawl (Custom/Local)", + icon: "local_fire_department", + color: "#F59E0B", + textIcon: "FC", + website: "https://firecrawl.dev" + }, + category: "apikey", + noAuth: true, + serviceKinds: [ + "webFetch" + ], + fetchConfig: { + baseUrl: "http://127.0.0.1:3002/v2/scrape", + method: "POST", + formats: [ + "markdown", + "html", + "text" + ], + maxCharacters: 200000, + timeoutMs: 30000 + } +}; diff --git a/open-sse/providers/registry/index.js b/open-sse/providers/registry/index.js index 64edcf329e7..5603311502f 100644 --- a/open-sse/providers/registry/index.js +++ b/open-sse/providers/registry/index.js @@ -30,71 +30,72 @@ import p27 from "./elevenlabs.js"; import p28 from "./exa.js"; import p29 from "./fal-ai.js"; import p30 from "./firecrawl.js"; -import p31 from "./fireworks.js"; -import p32 from "./gemini-cli.js"; -import p33 from "./gemini.js"; -import p34 from "./github.js"; -import p35 from "./gitlab.js"; -import p36 from "./glm-cn.js"; -import p37 from "./glm.js"; -import p38 from "./google-pse.js"; -import p39 from "./google-tts.js"; -import p40 from "./grok-web.js"; -import p41 from "./groq.js"; -import p42 from "./huggingface.js"; -import p43 from "./hyperbolic.js"; -import p44 from "./iflow.js"; -import p45 from "./inworld.js"; -import p46 from "./jina-ai.js"; -import p47 from "./jina-reader.js"; -import p48 from "./kilocode.js"; -import p49 from "./kimchi.js"; -import p50 from "./kimi-coding.js"; -import p51 from "./kimi.js"; -import p52 from "./kiro.js"; -import p53 from "./linkup.js"; -import p54 from "./local-device.js"; -import p55 from "./mimo-free.js"; -import p56 from "./minimax-cn.js"; -import p57 from "./minimax.js"; -import p58 from "./mistral.js"; -import p59 from "./mmf.js"; -import p60 from "./nanobanana.js"; -import p61 from "./nebius.js"; -import p62 from "./nvidia.js"; -import p63 from "./ollama-local.js"; -import p64 from "./ollama.js"; -import p65 from "./openai.js"; -import p66 from "./opencode-go.js"; -import p67 from "./opencode.js"; -import p68 from "./openrouter.js"; -import p69 from "./perplexity-web.js"; -import p70 from "./perplexity.js"; -import p71 from "./playht.js"; -import p72 from "./qoder.js"; -import p73 from "./qwen.js"; -import p74 from "./recraft.js"; -import p75 from "./runwayml.js"; -import p76 from "./sdwebui.js"; -import p77 from "./searchapi.js"; -import p78 from "./searxng.js"; -import p79 from "./serper.js"; -import p80 from "./siliconflow.js"; -import p81 from "./stability-ai.js"; -import p82 from "./tavily.js"; -import p83 from "./together.js"; -import p84 from "./topaz.js"; -import p85 from "./tortoise.js"; -import p86 from "./venice.js"; -import p87 from "./vercel-ai-gateway.js"; -import p88 from "./vertex-partner.js"; -import p89 from "./vertex.js"; -import p90 from "./volcengine-ark.js"; -import p91 from "./voyage-ai.js"; -import p92 from "./xai.js"; -import p93 from "./xiaomi-mimo.js"; -import p94 from "./xiaomi-tokenplan.js"; -import p95 from "./youcom.js"; +import p31 from "./firecrawl_custom.js"; +import p32 from "./fireworks.js"; +import p33 from "./gemini-cli.js"; +import p34 from "./gemini.js"; +import p35 from "./github.js"; +import p36 from "./gitlab.js"; +import p37 from "./glm-cn.js"; +import p38 from "./glm.js"; +import p39 from "./google-pse.js"; +import p40 from "./google-tts.js"; +import p41 from "./grok-web.js"; +import p42 from "./groq.js"; +import p43 from "./huggingface.js"; +import p44 from "./hyperbolic.js"; +import p45 from "./iflow.js"; +import p46 from "./inworld.js"; +import p47 from "./jina-ai.js"; +import p48 from "./jina-reader.js"; +import p49 from "./kilocode.js"; +import p50 from "./kimchi.js"; +import p51 from "./kimi-coding.js"; +import p52 from "./kimi.js"; +import p53 from "./kiro.js"; +import p54 from "./linkup.js"; +import p55 from "./local-device.js"; +import p56 from "./mimo-free.js"; +import p57 from "./minimax-cn.js"; +import p58 from "./minimax.js"; +import p59 from "./mistral.js"; +import p60 from "./mmf.js"; +import p61 from "./nanobanana.js"; +import p62 from "./nebius.js"; +import p63 from "./nvidia.js"; +import p64 from "./ollama-local.js"; +import p65 from "./ollama.js"; +import p66 from "./openai.js"; +import p67 from "./opencode-go.js"; +import p68 from "./opencode.js"; +import p69 from "./openrouter.js"; +import p70 from "./perplexity-web.js"; +import p71 from "./perplexity.js"; +import p72 from "./playht.js"; +import p73 from "./qoder.js"; +import p74 from "./qwen.js"; +import p75 from "./recraft.js"; +import p76 from "./runwayml.js"; +import p77 from "./sdwebui.js"; +import p78 from "./searchapi.js"; +import p79 from "./searxng.js"; +import p80 from "./serper.js"; +import p81 from "./siliconflow.js"; +import p82 from "./stability-ai.js"; +import p83 from "./tavily.js"; +import p84 from "./together.js"; +import p85 from "./topaz.js"; +import p86 from "./tortoise.js"; +import p87 from "./venice.js"; +import p88 from "./vercel-ai-gateway.js"; +import p89 from "./vertex-partner.js"; +import p90 from "./vertex.js"; +import p91 from "./volcengine-ark.js"; +import p92 from "./voyage-ai.js"; +import p93 from "./xai.js"; +import p94 from "./xiaomi-mimo.js"; +import p95 from "./xiaomi-tokenplan.js"; +import p96 from "./youcom.js"; export default [ p0, @@ -192,5 +193,6 @@ export default [ p92, p93, p94, - p95 + p95, + p96 ]; diff --git a/tests/unit/firecrawl-selfhosted.test.js b/tests/unit/firecrawl-selfhosted.test.js new file mode 100644 index 00000000000..3c099fab668 --- /dev/null +++ b/tests/unit/firecrawl-selfhosted.test.js @@ -0,0 +1,216 @@ +/** + * Unit tests for Firecrawl official and self-hosted (custom) providers. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { handleFetchCore } from "open-sse/handlers/fetch/index.js"; + +const originalFetch = global.fetch; +const originalEnv = { ...process.env }; + +describe("Firecrawl providers", () => { + beforeEach(() => { + vi.clearAllMocks(); + global.fetch = originalFetch; + }); + + afterEach(() => { + global.fetch = originalFetch; + process.env = { ...originalEnv }; + }); + + describe("official firecrawl provider", () => { + it("rejects when API key is missing", async () => { + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(false); + expect(res.status).toBe(400); + expect(res.error).toBe("FIRECRAWL_API_KEY is required for the official Firecrawl provider"); + }); + + it("uses cloud URL and sends Authorization: Bearer ", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { markdown: "# Hello" } }) + }); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl", + providerConfig: {}, + credentials: { apiKey: "fc-cloud-key" } + }); + + expect(res.success).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + "https://api.firecrawl.dev/v1/scrape", + expect.objectContaining({ + method: "POST", + headers: expect.objectContaining({ + "content-type": "application/json", + authorization: "Bearer fc-cloud-key" + }) + }) + ); + }); + }); + + describe("firecrawl_custom provider", () => { + it("uses FIRECRAWL_BASE_URL and hits /v2/scrape", async () => { + process.env.FIRECRAWL_BASE_URL = "http://my-local-firecrawl:3002"; + + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { markdown: "# Local" } }) + }); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + "http://my-local-firecrawl:3002/v2/scrape", + expect.anything() + ); + }); + + it("falls back to the default local URL when FIRECRAWL_BASE_URL is not set", async () => { + delete process.env.FIRECRAWL_BASE_URL; + + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { markdown: "# Local" } }) + }); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + "http://127.0.0.1:3002/v2/scrape", + expect.anything() + ); + }); + + it("does not send an Authorization header", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { markdown: "# Local" } }) + }); + + await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: { apiKey: "should-be-ignored" } + }); + + const [, init] = global.fetch.mock.calls[0]; + expect(init.headers).not.toHaveProperty("authorization"); + }); + + it("returns a clear error when the instance is unreachable", async () => { + global.fetch = vi.fn().mockRejectedValue(new Error("Connection refused")); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(false); + expect(res.status).toBe(502); + expect(res.error).toBe("Custom Firecrawl instance unreachable: Connection refused"); + }); + + it("returns a clear error when the instance returns a non-2xx response", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: false, + status: 500, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ error: "Internal error" }) + }); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(false); + expect(res.status).toBe(500); + expect(res.error).toBe("Internal error"); + }); + }); + + describe("environment variable overrides", () => { + it("respects FIRECRAWL_TIMEOUT_MS", async () => { + process.env.FIRECRAWL_TIMEOUT_MS = "60000"; + + global.fetch = vi.fn().mockImplementation((_url, init) => { + expect(init.signal).toBeDefined(); + return Promise.resolve({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { markdown: "# OK" } }) + }); + }); + + await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(global.fetch).toHaveBeenCalled(); + }); + + it("respects FIRECRAWL_DEFAULT_FORMAT", async () => { + process.env.FIRECRAWL_DEFAULT_FORMAT = "html"; + + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ data: { html: "

Hello

" } }) + }); + + const res = await handleFetchCore({ + url: "https://example.com", + provider: "firecrawl_custom", + providerConfig: {}, + credentials: {} + }); + + expect(res.success).toBe(true); + const [, init] = global.fetch.mock.calls[0]; + const body = JSON.parse(init.body); + expect(body.formats).toEqual(["html"]); + }); + }); +});