Skip to content

feat: add self-hosted FireCrawl provider #2253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bloodf wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: add self-hosted FireCrawl provider #2253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ CLOUD_URL=https://9router.com
NEXT_PUBLIC_BASE_URL=http://localhost:20128
NEXT_PUBLIC_CLOUD_URL=https://9router.com

# Firecrawl web fetch provider
# Leave FIRECRAWL_API_KEY empty to route to a self-hosted instance.
FIRECRAWL_BASE_URL=https://api.firecrawl.dev
FIRECRAWL_API_KEY=
FIRECRAWL_TIMEOUT_MS=30000
FIRECRAWL_DEFAULT_FORMAT=markdown

# Optional outbound proxy variables for upstream provider calls
# Lowercase variants are also supported: http_proxy, https_proxy, all_proxy, no_proxy
# HTTP_PROXY=http://127.0.0.1:7890
Expand Down
50 changes: 38 additions & 12 deletions open-sse/handlers/fetch/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,17 @@
const DEFAULT_TIMEOUT_MS = 15000;
const DEFAULT_FORMAT = "markdown";

function getDefaultTimeoutMs() {
const env = process.env.FIRECRAWL_TIMEOUT_MS;
if (!env) return DEFAULT_TIMEOUT_MS;
const n = Number(env);
return Number.isFinite(n) && n > 0 ? n : DEFAULT_TIMEOUT_MS;
}

function getDefaultFormat() {
return process.env.FIRECRAWL_DEFAULT_FORMAT || DEFAULT_FORMAT;
}

/**
* @typedef {Object} FetchResult
* @property {boolean} success
Expand Down Expand Up @@ -93,15 +104,15 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr
return { success: false, status: 400, error: "provider is required" };
}

const fmt = format || DEFAULT_FORMAT;
const timeoutMs = providerConfig?.timeoutMs || DEFAULT_TIMEOUT_MS;
const fmt = format || getDefaultFormat();
const timeoutMs = providerConfig?.timeoutMs || getDefaultTimeoutMs();
const apiKey = credentials?.apiKey || credentials?.key || credentials?.token || "";
const costPerQuery = providerConfig?.costPerQuery ?? null;
const startedAt = Date.now();

try {
if (provider === "firecrawl") {
return await runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
if (provider === "firecrawl" || provider === "firecrawl_custom") {
return await runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, provider });
}
if (provider === "jina-reader") {
return await runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
Expand All @@ -119,19 +130,34 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr
}
}

async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, provider }) {
const isCustom = provider === "firecrawl_custom";

if (!isCustom && !apiKey) {
return { success: false, status: 400, error: "FIRECRAWL_API_KEY is required for the official Firecrawl provider" };
}

const baseUrl = isCustom
? (process.env.FIRECRAWL_BASE_URL || "http://127.0.0.1:3002")
: (process.env.FIRECRAWL_BASE_URL || "https://api.firecrawl.dev");
const endpoint = isCustom ? "/v2/scrape" : "/v1/scrape";

const headers = { "content-type": "application/json" };
if (!isCustom && apiKey) {
headers.authorization = `Bearer ${apiKey}`;
}

const upstreamStart = Date.now();
const r = await tryFetch("https://api.firecrawl.dev/v1/scrape", {
const r = await tryFetch(`${baseUrl}${endpoint}`, {
method: "POST",
headers: {
"content-type": "application/json",
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
},
headers,
body: JSON.stringify({ url, formats: [fmt] })
}, timeoutMs);

if (!r.ok) {
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
const status = r.timeout ? 504 : 502;
const error = isCustom ? `Custom Firecrawl instance unreachable: ${r.error}` : r.error;
return { success: false, status, error };
}
const upstreamMs = Date.now() - upstreamStart;
const { json } = await readJsonOrText(r.res);
Expand All @@ -144,7 +170,7 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe
return {
success: true,
data: buildData({
provider: "firecrawl", url, title, format: fmt, text,
provider, url, title, format: fmt, text,
costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs
})
};
Expand Down
27 changes: 27 additions & 0 deletions open-sse/providers/registry/firecrawl_custom.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
export default {
id: "firecrawl_custom",
alias: "firecrawl_custom",
display: {
name: "Firecrawl (Custom/Local)",
icon: "local_fire_department",
color: "#F59E0B",
textIcon: "FC",
website: "https://firecrawl.dev"
},
category: "apikey",
noAuth: true,
serviceKinds: [
"webFetch"
],
fetchConfig: {
baseUrl: "http://127.0.0.1:3002/v2/scrape",
method: "POST",
formats: [
"markdown",
"html",
"text"
],
maxCharacters: 200000,
timeoutMs: 30000
}
};
134 changes: 68 additions & 66 deletions open-sse/providers/registry/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,71 +30,72 @@ import p27 from "./elevenlabs.js";
import p28 from "./exa.js";
import p29 from "./fal-ai.js";
import p30 from "./firecrawl.js";
import p31 from "./fireworks.js";
import p32 from "./gemini-cli.js";
import p33 from "./gemini.js";
import p34 from "./github.js";
import p35 from "./gitlab.js";
import p36 from "./glm-cn.js";
import p37 from "./glm.js";
import p38 from "./google-pse.js";
import p39 from "./google-tts.js";
import p40 from "./grok-web.js";
import p41 from "./groq.js";
import p42 from "./huggingface.js";
import p43 from "./hyperbolic.js";
import p44 from "./iflow.js";
import p45 from "./inworld.js";
import p46 from "./jina-ai.js";
import p47 from "./jina-reader.js";
import p48 from "./kilocode.js";
import p49 from "./kimchi.js";
import p50 from "./kimi-coding.js";
import p51 from "./kimi.js";
import p52 from "./kiro.js";
import p53 from "./linkup.js";
import p54 from "./local-device.js";
import p55 from "./mimo-free.js";
import p56 from "./minimax-cn.js";
import p57 from "./minimax.js";
import p58 from "./mistral.js";
import p59 from "./mmf.js";
import p60 from "./nanobanana.js";
import p61 from "./nebius.js";
import p62 from "./nvidia.js";
import p63 from "./ollama-local.js";
import p64 from "./ollama.js";
import p65 from "./openai.js";
import p66 from "./opencode-go.js";
import p67 from "./opencode.js";
import p68 from "./openrouter.js";
import p69 from "./perplexity-web.js";
import p70 from "./perplexity.js";
import p71 from "./playht.js";
import p72 from "./qoder.js";
import p73 from "./qwen.js";
import p74 from "./recraft.js";
import p75 from "./runwayml.js";
import p76 from "./sdwebui.js";
import p77 from "./searchapi.js";
import p78 from "./searxng.js";
import p79 from "./serper.js";
import p80 from "./siliconflow.js";
import p81 from "./stability-ai.js";
import p82 from "./tavily.js";
import p83 from "./together.js";
import p84 from "./topaz.js";
import p85 from "./tortoise.js";
import p86 from "./venice.js";
import p87 from "./vercel-ai-gateway.js";
import p88 from "./vertex-partner.js";
import p89 from "./vertex.js";
import p90 from "./volcengine-ark.js";
import p91 from "./voyage-ai.js";
import p92 from "./xai.js";
import p93 from "./xiaomi-mimo.js";
import p94 from "./xiaomi-tokenplan.js";
import p95 from "./youcom.js";
import p31 from "./firecrawl_custom.js";
import p32 from "./fireworks.js";
import p33 from "./gemini-cli.js";
import p34 from "./gemini.js";
import p35 from "./github.js";
import p36 from "./gitlab.js";
import p37 from "./glm-cn.js";
import p38 from "./glm.js";
import p39 from "./google-pse.js";
import p40 from "./google-tts.js";
import p41 from "./grok-web.js";
import p42 from "./groq.js";
import p43 from "./huggingface.js";
import p44 from "./hyperbolic.js";
import p45 from "./iflow.js";
import p46 from "./inworld.js";
import p47 from "./jina-ai.js";
import p48 from "./jina-reader.js";
import p49 from "./kilocode.js";
import p50 from "./kimchi.js";
import p51 from "./kimi-coding.js";
import p52 from "./kimi.js";
import p53 from "./kiro.js";
import p54 from "./linkup.js";
import p55 from "./local-device.js";
import p56 from "./mimo-free.js";
import p57 from "./minimax-cn.js";
import p58 from "./minimax.js";
import p59 from "./mistral.js";
import p60 from "./mmf.js";
import p61 from "./nanobanana.js";
import p62 from "./nebius.js";
import p63 from "./nvidia.js";
import p64 from "./ollama-local.js";
import p65 from "./ollama.js";
import p66 from "./openai.js";
import p67 from "./opencode-go.js";
import p68 from "./opencode.js";
import p69 from "./openrouter.js";
import p70 from "./perplexity-web.js";
import p71 from "./perplexity.js";
import p72 from "./playht.js";
import p73 from "./qoder.js";
import p74 from "./qwen.js";
import p75 from "./recraft.js";
import p76 from "./runwayml.js";
import p77 from "./sdwebui.js";
import p78 from "./searchapi.js";
import p79 from "./searxng.js";
import p80 from "./serper.js";
import p81 from "./siliconflow.js";
import p82 from "./stability-ai.js";
import p83 from "./tavily.js";
import p84 from "./together.js";
import p85 from "./topaz.js";
import p86 from "./tortoise.js";
import p87 from "./venice.js";
import p88 from "./vercel-ai-gateway.js";
import p89 from "./vertex-partner.js";
import p90 from "./vertex.js";
import p91 from "./volcengine-ark.js";
import p92 from "./voyage-ai.js";
import p93 from "./xai.js";
import p94 from "./xiaomi-mimo.js";
import p95 from "./xiaomi-tokenplan.js";
import p96 from "./youcom.js";

export default [
p0,
Expand Down Expand Up @@ -192,5 +193,6 @@ export default [
p92,
p93,
p94,
p95
p95,
p96
];
Loading