From 88d0e45f31ef148b71acd97240b15999c39a8719 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Wed, 17 Jun 2026 23:30:26 -0300 Subject: [PATCH 1/6] feat(search): add free DuckDuckGo lite parser + fetch (no API key) --- open-sse/services/freeWebSearch.ts | 118 +++++++++++++++++++++++++++++ tests/unit/free-web-search.test.ts | 76 +++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 open-sse/services/freeWebSearch.ts create mode 100644 tests/unit/free-web-search.test.ts diff --git a/open-sse/services/freeWebSearch.ts b/open-sse/services/freeWebSearch.ts new file mode 100644 index 000000000000..f5c2c96f2345 --- /dev/null +++ b/open-sse/services/freeWebSearch.ts @@ -0,0 +1,118 @@ +/** + * Free, no-API-key web search via DuckDuckGo's HTML "lite" endpoint + * (free-claude-code port). Used as a LAST-RESORT fallback search provider + * (`duckduckgo-free`, `fallbackOnly`) when no credentialed search provider is + * configured — see open-sse/config/searchRegistry.ts. + * + * Best-effort HTML scraping: the lite endpoint's markup can drift, so the parser + * is tolerant (quote styles, attribute order, `` highlights, entities) and the + * unit test pins the contract against a real captured response. The network call + * goes through `safeOutboundFetch` with the public-only SSRF guard. + */ +import { safeOutboundFetch } from "@/shared/network/safeOutboundFetch"; + +export interface FreeSearchResult { + title: string; + url: string; + snippet: string; +} + +const DUCKDUCKGO_LITE_URL = "https://lite.duckduckgo.com/lite/"; +// A browser-like UA — the lite endpoint rejects obvious bot agents. +const DDG_USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"; + +// Real lite shape: `Title` (href usually +// before class; quotes may be single or double) and `…`. +const ANCHOR_RE = /]*?class=['"][^'"]*result-link[^'"]*['"][^>]*)>([\s\S]*?)<\/a>/gi; +const HREF_RE = /href=['"]([^'"]+)['"]/i; +const SNIPPET_RE = + /]*?class=['"][^'"]*result-snippet[^'"]*['"][^>]*>([\s\S]*?)<\/td>/gi; + +function decodeEntities(text: string): string { + return text + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/"/gi, '"') + .replace(/�*39;|�*27;|'/gi, "'"); +} + +function stripTags(html: string): string { + return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim(); +} + +/** Resolve a lite result href to a real absolute URL. */ +function resolveResultUrl(href: string): string { + // Older/HTML endpoints wrap the target in a redirect: //duckduckgo.com/l/?uddg=&… + const redirect = href.match(/[?&]uddg=([^&]+)/); + if (redirect) { + try { + return decodeURIComponent(redirect[1]); + } catch { + // malformed encoding — fall through to the raw href + } + } + if (href.startsWith("//")) return `https:${href}`; + return href; +} + +/** + * Parse the DuckDuckGo lite HTML into ordered results. Pure (no network) so it is + * fully unit-testable. Result link N aligns with snippet N (the lite layout emits + * them 1:1); a missing snippet yields an empty string rather than a crash. + */ +export function parseDuckDuckGoLite(html: string): FreeSearchResult[] { + if (!html) return []; + + const snippets = [...html.matchAll(SNIPPET_RE)].map((m) => stripTags(m[1])); + const results: FreeSearchResult[] = []; + let index = 0; + + for (const match of html.matchAll(ANCHOR_RE)) { + const attrs = match[1]; + const inner = match[2]; + const hrefMatch = attrs.match(HREF_RE); + const title = stripTags(inner); + if (hrefMatch && title) { + const url = resolveResultUrl(hrefMatch[1]); + if (url) results.push({ url, title, snippet: snippets[index] ?? "" }); + } + index += 1; + } + + return results; +} + +/** + * Run a free DuckDuckGo lite search and return up to `maxResults` parsed results. + * Throws on a non-2xx upstream so the search handler can record the failure and + * fall through. The URL is fixed, but the call still goes through the public-only + * SSRF guard for defense in depth. + */ +export async function freeWebSearch( + query: string, + maxResults = 5, + timeoutMs = 10_000 +): Promise { + const response = await safeOutboundFetch(DUCKDUCKGO_LITE_URL, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": DDG_USER_AGENT, + Accept: "text/html", + }, + body: new URLSearchParams({ q: query }).toString(), + guard: "public-only", + allowRedirect: true, + timeoutMs, + }); + + if (!response.ok) { + throw new Error(`DuckDuckGo lite search returned HTTP ${response.status}`); + } + + const html = await response.text(); + return parseDuckDuckGoLite(html).slice(0, Math.max(1, maxResults)); +} diff --git a/tests/unit/free-web-search.test.ts b/tests/unit/free-web-search.test.ts new file mode 100644 index 000000000000..178b1c877c8c --- /dev/null +++ b/tests/unit/free-web-search.test.ts @@ -0,0 +1,76 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { parseDuckDuckGoLite } from "../../open-sse/services/freeWebSearch.ts"; + +// Real DuckDuckGo lite shape (captured from lite.duckduckgo.com/lite/): +// double-quoted href BEFORE single-quoted class='result-link', direct URLs, +// highlight tags inside titles/snippets, link[i] aligns with snippet[i]. +const REAL_LITE_HTML = ` + + + + + + + + + + + + + + + + + +
1.  + API overview - Claude API Docs - Anthropic +
   Claude API Documentation. New to Claude?
2.  + Claude Platform | Claude - Anthropic +
   Build with the most capable models.
`; + +test("parseDuckDuckGoLite extracts aligned title/url/snippet from real lite HTML", () => { + const results = parseDuckDuckGoLite(REAL_LITE_HTML); + assert.equal(results.length, 2); + + assert.equal(results[0].url, "https://platform.claude.com/docs/en/api/overview"); + assert.equal(results[0].title, "API overview - Claude API Docs - Anthropic"); + assert.ok(results[0].snippet.includes("Claude API Documentation")); + + assert.equal(results[1].url, "https://www.anthropic.com/api"); + assert.equal(results[1].title, "Claude Platform | Claude - Anthropic"); + assert.ok(results[1].snippet.includes("most capable models")); +}); + +test("parseDuckDuckGoLite strips highlight tags and collapses whitespace", () => { + const results = parseDuckDuckGoLite(REAL_LITE_HTML); + assert.doesNotMatch(results[0].title, /|<\/b>/); + assert.doesNotMatch(results[0].snippet, /|<\/b>/); +}); + +test("parseDuckDuckGoLite decodes a uddg redirect href (defensive)", () => { + const html = `Example + snippet`; + const results = parseDuckDuckGoLite(html); + assert.equal(results[0].url, "https://example.com/a?b=1"); +}); + +test("parseDuckDuckGoLite upgrades protocol-relative hrefs to https", () => { + const html = `Proto rel + s`; + const results = parseDuckDuckGoLite(html); + assert.equal(results[0].url, "https://example.org/path"); +}); + +test("parseDuckDuckGoLite returns [] when there are no result links", () => { + assert.deepEqual(parseDuckDuckGoLite("no results"), []); + assert.deepEqual(parseDuckDuckGoLite(""), []); +}); + +test("parseDuckDuckGoLite tolerates a missing snippet (empty string, not crash)", () => { + const html = `Only a link`; + const results = parseDuckDuckGoLite(html); + assert.equal(results.length, 1); + assert.equal(results[0].snippet, ""); +}); From 2ab1e8daa1c336c9e72c6beba2b9cfbe8a12f233 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Wed, 17 Jun 2026 23:33:16 -0300 Subject: [PATCH 2/6] feat(search): register duckduckgo-free as fallbackOnly provider --- open-sse/config/searchRegistry.ts | 34 ++++++++++++++++++++++++-- src/shared/validation/schemas/apiV1.ts | 1 + tests/unit/search-registry.test.ts | 33 +++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/open-sse/config/searchRegistry.ts b/open-sse/config/searchRegistry.ts index 724d349525c0..b8534134df39 100644 --- a/open-sse/config/searchRegistry.ts +++ b/open-sse/config/searchRegistry.ts @@ -24,6 +24,12 @@ export interface SearchProviderConfig { maxMaxResults: number; timeoutMs: number; cacheTTLMs: number; + /** + * Last-resort provider: excluded from automatic (cost-based) selection so a + * cost-0 free provider never overrides a configured paid one. Only used when no + * credentialed provider is available, or when requested explicitly by id. + */ + fallbackOnly?: boolean; } export const SEARCH_PROVIDERS: Record = { @@ -218,6 +224,26 @@ export const SEARCH_PROVIDERS: Record = { timeoutMs: 10_000, cacheTTLMs: 5 * 60 * 1000, }, + + // Free, no-API-key DuckDuckGo lite scraping (free-claude-code port). Last-resort + // only (fallbackOnly): never auto-selected over a configured provider; served by + // the dedicated HTML path in open-sse/handlers/search.ts (not the generic JSON one). + "duckduckgo-free": { + id: "duckduckgo-free", + name: "DuckDuckGo (free)", + baseUrl: "https://lite.duckduckgo.com/lite/", + method: "POST", + authType: "none", + authHeader: "none", + costPerQuery: 0, + freeMonthlyQuota: 999999, + searchTypes: ["web"], + defaultMaxResults: 5, + maxMaxResults: 25, + timeoutMs: 10_000, + cacheTTLMs: 5 * 60 * 1000, + fallbackOnly: true, + }, }; /** @@ -278,8 +304,12 @@ export function selectProvider( return provider; } - const providers = Object.values(SEARCH_PROVIDERS).filter((provider) => - searchType ? supportsSearchType(provider, searchType) : true + // Auto-selection excludes fallbackOnly providers so a free cost-0 provider never + // overrides a configured paid one — they are reached only via explicit id or the + // route handler's last-resort step. + const providers = Object.values(SEARCH_PROVIDERS).filter( + (provider) => + !provider.fallbackOnly && (searchType ? supportsSearchType(provider, searchType) : true) ); if (providers.length === 0) return null; diff --git a/src/shared/validation/schemas/apiV1.ts b/src/shared/validation/schemas/apiV1.ts index fc0ab50b419a..0f4bf85475e6 100644 --- a/src/shared/validation/schemas/apiV1.ts +++ b/src/shared/validation/schemas/apiV1.ts @@ -146,6 +146,7 @@ export const v1SearchSchema = z "youcom-search", "searxng-search", "zai-search", + "duckduckgo-free", ]) .optional(), max_results: z.coerce.number().int().min(1).max(100).default(5), diff --git a/tests/unit/search-registry.test.ts b/tests/unit/search-registry.test.ts index 8c21f0792652..59804d75117e 100644 --- a/tests/unit/search-registry.test.ts +++ b/tests/unit/search-registry.test.ts @@ -32,7 +32,18 @@ test("SEARCH_PROVIDERS has all 11 providers", () => { assert.ok(SEARCH_PROVIDERS["searxng-search"], "searxng should exist"); assert.ok(SEARCH_PROVIDERS["ollama-search"], "ollama-search should exist"); assert.ok(SEARCH_PROVIDERS["zai-search"], "zai should exist"); - assert.equal(Object.keys(SEARCH_PROVIDERS).length, 12); + assert.ok(SEARCH_PROVIDERS["duckduckgo-free"], "duckduckgo-free should exist"); + assert.equal(Object.keys(SEARCH_PROVIDERS).length, 13); +}); + +test("duckduckgo-free config is a no-key, fallback-only provider", () => { + const d = SEARCH_PROVIDERS["duckduckgo-free"]; + assert.equal(d.id, "duckduckgo-free"); + assert.equal(d.method, "POST"); + assert.equal(d.authType, "none"); + assert.equal(d.costPerQuery, 0); + assert.equal(d.fallbackOnly, true, "must only be used as a last resort"); + assert.deepEqual(d.searchTypes, ["web"]); }); test("serper-search config is correct", () => { @@ -154,7 +165,8 @@ test("zai-search config is correct", () => { test("getAllSearchProviders returns flat list", () => { const all = getAllSearchProviders(); - assert.equal(all.length, 12); + assert.equal(all.length, 13); + assert.ok(all.some((p) => p.id === "duckduckgo-free")); assert.ok(all.some((p) => p.id === "serper-search")); assert.ok(all.some((p) => p.id === "brave-search")); assert.ok(all.some((p) => p.id === "perplexity-search")); @@ -191,6 +203,22 @@ test("selectProvider without argument returns cheapest provider", () => { assert.equal(config.id, "searxng-search"); }); +test("selectProvider auto-selection never returns a fallbackOnly provider", () => { + // duckduckgo-free is cost 0 (ties searxng) but must be excluded from auto-select. + const auto = selectProvider(); + assert.ok(auto); + assert.notEqual(auto.fallbackOnly, true); + const autoWeb = selectProvider(undefined, "web"); + assert.ok(autoWeb); + assert.notEqual(autoWeb.fallbackOnly, true); +}); + +test("selectProvider still honors an explicit fallbackOnly provider", () => { + const config = selectProvider("duckduckgo-free", "web"); + assert.ok(config); + assert.equal(config.id, "duckduckgo-free"); +}); + test("selectProvider filters by search type support", () => { const config = selectProvider(undefined, "news"); assert.ok(config); @@ -364,6 +392,7 @@ test("v1SearchSchema accepts new search providers", async () => { "youcom-search", "searxng-search", "ollama-search", + "duckduckgo-free", ] as const; for (const provider of providers) { From 04dc304212b96a8db58892b3c0fb979a98683d60 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Wed, 17 Jun 2026 23:39:51 -0300 Subject: [PATCH 3/6] feat(search): wire duckduckgo-free as last-resort failover + HTML handler --- open-sse/handlers/search.ts | 101 +++++++++++++++++++ src/app/api/v1/search/route.ts | 26 ++++- tests/unit/search-handler-duckduckgo.test.ts | 83 +++++++++++++++ 3 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 tests/unit/search-handler-duckduckgo.test.ts diff --git a/open-sse/handlers/search.ts b/open-sse/handlers/search.ts index 218705149301..3d87f8bab64c 100644 --- a/open-sse/handlers/search.ts +++ b/open-sse/handlers/search.ts @@ -17,6 +17,7 @@ import { randomUUID } from "crypto"; */ import { getSearchProvider, type SearchProviderConfig } from "../config/searchRegistry.ts"; +import { freeWebSearch } from "../services/freeWebSearch.ts"; import { saveCallLog } from "@/lib/usageDb"; import { safeOutboundFetch } from "@/shared/network/safeOutboundFetch"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; @@ -1279,6 +1280,102 @@ export async function handleSearch(options: SearchHandlerOptions): Promise, + startTime: number, + globalStartTime: number, + log?: any +): Promise { + const { query, searchType, maxResults } = params; + const remainingGlobal = GLOBAL_TIMEOUT_MS - (Date.now() - globalStartTime); + const timeout = Math.min(config.timeoutMs, Math.max(remainingGlobal, 1000)); + + if (log) { + log.info("SEARCH", `${config.id} | query: "${query.slice(0, 80)}" | type: ${searchType}`); + } + + const requestBody = { + query: query.slice(0, 200), + search_type: searchType, + max_results: maxResults, + }; + + try { + const freeResults = await freeWebSearch(query, maxResults, timeout); + const now = new Date().toISOString(); + const results = freeResults + .slice(0, maxResults) + .map((r, idx) => + makeResult(config.id, { title: r.title, url: r.url, snippet: r.snippet }, idx, now) + ); + const duration = Date.now() - startTime; + + saveCallLog({ + method: config.method, + path: "/v1/search", + status: 200, + model: config.id, + provider: config.id, + duration, + requestType: "search", + tokens: { prompt_tokens: 0, completion_tokens: 0 }, + requestBody, + responseBody: { results_count: results.length, cached: false }, + }).catch(() => { + /* non-critical — logging must not block search response */ + }); + + return { + success: true, + data: { + provider: config.id, + query, + results, + answer: null, + usage: { queries_used: 1, search_cost_usd: 0 }, + metrics: { + response_time_ms: duration, + upstream_latency_ms: duration, + total_results_available: results.length, + }, + errors: [], + }, + }; + } catch (err: any) { + const duration = Date.now() - startTime; + const message = sanitizeErrorMessage(err); + if (log) { + log.error("SEARCH", `${config.id} error: ${message}`); + } + + saveCallLog({ + method: config.method, + path: "/v1/search", + status: 502, + model: config.id, + provider: config.id, + duration, + requestType: "search", + error: message.slice(0, 500), + requestBody, + }).catch(() => { + /* non-critical */ + }); + + return { + success: false, + status: 502, + error: `DuckDuckGo free search failed: ${message}`, + }; + } +} + async function tryProvider( config: SearchProviderConfig, params: Omit, @@ -1303,6 +1400,10 @@ async function tryProvider( const { query, searchType, maxResults } = params; + if (config.id === "duckduckgo-free") { + return tryDuckDuckGoFreeProvider(config, params, startTime, globalStartTime, log); + } + if (config.id === "zai-search" && token) { return tryZaiMCPProvider( config, diff --git a/src/app/api/v1/search/route.ts b/src/app/api/v1/search/route.ts index 22b59fa9aba2..40f5fa569494 100644 --- a/src/app/api/v1/search/route.ts +++ b/src/app/api/v1/search/route.ts @@ -177,9 +177,10 @@ async function postHandler(request: Request, context: unknown) { } if (!credentials) { - // Sort by cost to find cheapest with credentials + // Sort by cost to find cheapest with credentials (fallback-only providers + // are reached via the last-resort step below, never the primary pick). const sortedIds = Object.values(SEARCH_PROVIDERS) - .filter((provider) => supportsSearchType(provider, body.search_type)) + .filter((provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type)) .sort((a, b) => a.costPerQuery - b.costPerQuery) .map((p) => p.id); @@ -212,9 +213,10 @@ async function postHandler(request: Request, context: unknown) { ); } - // Find alternate for failover — must bind credentials to the matched provider + // Find alternate for failover — must bind credentials to the matched provider. + // Exclude fallback-only providers; they are only used by the last-resort step. const otherIds = Object.values(SEARCH_PROVIDERS) - .filter((provider) => supportsSearchType(provider, body.search_type)) + .filter((provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type)) .sort((a, b) => a.costPerQuery - b.costPerQuery) .map((p) => p.id) .filter((id) => id !== providerConfig.id); @@ -229,6 +231,22 @@ async function postHandler(request: Request, context: unknown) { break; } } + + // Last-resort: guarantee a free no-key fallback (e.g. duckduckgo-free) as the + // failover so out-of-the-box search still works when no credentialed provider + // is configured. Only used when no real alternate was found above. + if (!alternateProviderId) { + for (const provider of Object.values(SEARCH_PROVIDERS)) { + if (!provider.fallbackOnly || provider.id === providerConfig.id) continue; + if (!supportsSearchType(provider, body.search_type)) continue; + const fallbackCreds = await resolveSearchExecutionCredentials(provider); + if (fallbackCreds && !isAllRateLimitedCredentials(fallbackCreds)) { + alternateProviderId = provider.id; + alternateCredentials = fallbackCreds; + break; + } + } + } } // Clamp max_results to provider limit diff --git a/tests/unit/search-handler-duckduckgo.test.ts b/tests/unit/search-handler-duckduckgo.test.ts new file mode 100644 index 000000000000..0090d0985442 --- /dev/null +++ b/tests/unit/search-handler-duckduckgo.test.ts @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-ddg-")); + +const { handleSearch } = await import("../../open-sse/handlers/search.ts"); + +// Real lite shape: single-quoted class='result-link', direct https href, highlights. +const DDG_HTML = ` + + + + +
Claude API Docs
The Claude API reference.
Anthropic
AI safety company.
`; + +test("handleSearch fulfills duckduckgo-free via the HTML scraping path (no API key)", async () => { + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + globalThis.fetch = (async (url: string | URL) => { + capturedUrl = String(url); + return new Response(DDG_HTML, { status: 200, headers: { "content-type": "text/html" } }); + }) as typeof fetch; + + try { + const result = await handleSearch({ + query: "claude api", + provider: "duckduckgo-free", + maxResults: 5, + searchType: "web", + credentials: {}, + log: null, + }); + + assert.equal(result.success, true); + assert.ok(capturedUrl.includes("lite.duckduckgo.com"), "must hit the DDG lite endpoint"); + assert.equal(result.data.provider, "duckduckgo-free"); + assert.equal(result.data.usage.search_cost_usd, 0, "free provider has zero cost"); + assert.equal(result.data.results.length, 2); + assert.equal(result.data.results[0].url, "https://platform.claude.com/docs"); + assert.equal(result.data.results[0].title, "Claude API Docs"); + assert.ok(result.data.results[0].snippet.includes("Claude API reference")); + assert.equal(result.data.results[0].citation.provider, "duckduckgo-free"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleSearch fails over to duckduckgo-free when the primary provider errors", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url: string | URL) => { + if (String(url).includes("lite.duckduckgo.com")) { + return new Response(DDG_HTML, { status: 200, headers: { "content-type": "text/html" } }); + } + // Primary (e.g. searxng on localhost) is unreachable. + return new Response("bad gateway", { status: 502 }); + }) as typeof fetch; + + try { + const result = await handleSearch({ + query: "claude api", + provider: "searxng-search", + maxResults: 5, + searchType: "web", + credentials: {}, + alternateProvider: "duckduckgo-free", + alternateCredentials: {}, + log: null, + }); + + assert.equal(result.success, true); + assert.equal( + result.data.provider, + "duckduckgo-free", + "failover must land on the free last-resort provider" + ); + assert.equal(result.data.results.length, 2); + } finally { + globalThis.fetch = originalFetch; + } +}); From 5ffc2f72b6c83bff33850ba354c011c4641d8cb3 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Wed, 17 Jun 2026 23:43:44 -0300 Subject: [PATCH 4/6] chore(quality): strict log type in DDG handler + rebaseline search.ts --- config/quality/file-size-baseline.json | 3 ++- open-sse/handlers/search.ts | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 8627eb3dfd29..8043eada0cdf 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -32,6 +32,7 @@ "_rebaseline_2026_06_17_4034_usage_command": "PR #4034 own growth: apiKeys.ts 1633->1661 (+28 = the allow_usage_command additive column — fallback definition, parseAllowUsageCommand, prepared-statement SELECT column, and the metadata/create/update wiring, mirroring the existing disable_non_public_models accessor pattern) and chat.ts 1425->1432 (+7 = the handleInternalUsageCommand intercept hook at the existing post-auth chokepoint in handleChat). The reusable command logic lives in the new src/lib/usage/internalUsageCommand.ts (well under cap). Cohesive opt-in feature at the established column/dispatch boundaries; not extractable without splitting the api-keys domain module.", "_rebaseline_2026_06_17_ctx_editing_relays": "Context Editing relay coverage + 400-fallback (F4.2/F4.3) own growth: base.ts 1244->1292 (+48 at the existing dispatch chokepoint: extend the injection gate to anthropic-compatible-cc-* relays with an expanded comment documenting the claude-web / generic-anthropic-compatible exclusions; the contextEditingDisabled loop flag; and the 400-fallback block that, when an upstream rejects context_management with a 400, strips the param, re-serializes + re-signs (CCH), and retries the same URL once). Both edits sit on the per-URL request-build/fetch path and cannot be extracted without splitting the executor's fetch loop. No new file warranted — the logic is a few lines tightly coupled to the in-loop transformedBody/bodyString/response locals.", "_rebaseline_2026_06_17_ctx_editing_telemetry": "Context Editing telemetry (F4.1) own growth: chatCore.ts 5875->5898 (+23 = a best-effort, Claude-only, non-streaming block at the existing non-streaming response handler that extracts the provider's server-side context-editing receipt from responseBody and records it under engine \"context-editing\"). The pure extractor (extractContextEditingTelemetry) lives in the existing small open-sse/config/contextEditing.ts and the DB writer (recordContextEditingTelemetry) in src/lib/db/compressionAnalytics.ts — both under cap. Cohesive at the response-handling chokepoint next to the existing attachCompressionUsageReceipt call; not extractable without hiding the receipt-recording boundary.", + "_rebaseline_2026_06_17_duckduckgo_free": "Free web search (free-claude-code port, Fase 6) own growth: handlers/search.ts 1442->1546 (+104 = tryDuckDuckGoFreeProvider, a dedicated provider path mirroring the existing tryZaiMCPProvider special-case: the lite endpoint returns HTML, not the JSON the generic tryProvider()/normalizeResponse() flow expects, so it cannot reuse that path). The reusable parser + guarded fetch (parseDuckDuckGoLite, freeWebSearch) live in the new small open-sse/services/freeWebSearch.ts (well under cap). Cohesive at the per-provider dispatch chokepoint; not extractable without hiding the provider boundary.", "cap": 800, "frozen": { "open-sse/config/providerRegistry.ts": 4731, @@ -50,7 +51,7 @@ "open-sse/handlers/chatCore.ts": 5898, "open-sse/handlers/imageGeneration.ts": 3777, "open-sse/handlers/responseSanitizer.ts": 1103, - "open-sse/handlers/search.ts": 1442, + "open-sse/handlers/search.ts": 1546, "open-sse/handlers/sseParser.ts": 812, "open-sse/handlers/videoGeneration.ts": 1078, "open-sse/mcp-server/schemas/tools.ts": 1437, diff --git a/open-sse/handlers/search.ts b/open-sse/handlers/search.ts index 3d87f8bab64c..262e10d220c0 100644 --- a/open-sse/handlers/search.ts +++ b/open-sse/handlers/search.ts @@ -1290,14 +1290,17 @@ async function tryDuckDuckGoFreeProvider( params: Omit, startTime: number, globalStartTime: number, - log?: any + log?: { + info?: (tag: string, message: string) => void; + error?: (tag: string, message: string) => void; + } | null ): Promise { const { query, searchType, maxResults } = params; const remainingGlobal = GLOBAL_TIMEOUT_MS - (Date.now() - globalStartTime); const timeout = Math.min(config.timeoutMs, Math.max(remainingGlobal, 1000)); if (log) { - log.info("SEARCH", `${config.id} | query: "${query.slice(0, 80)}" | type: ${searchType}`); + log.info?.("SEARCH", `${config.id} | query: "${query.slice(0, 80)}" | type: ${searchType}`); } const requestBody = { @@ -1347,11 +1350,11 @@ async function tryDuckDuckGoFreeProvider( errors: [], }, }; - } catch (err: any) { + } catch (err) { const duration = Date.now() - startTime; const message = sanitizeErrorMessage(err); if (log) { - log.error("SEARCH", `${config.id} error: ${message}`); + log.error?.("SEARCH", `${config.id} error: ${message}`); } saveCallLog({ From 56af51ee8fc7675678f5bb7d0a624531accdfcc9 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Wed, 17 Jun 2026 23:48:16 -0300 Subject: [PATCH 5/6] =?UTF-8?q?fix(search):=20harden=20DDG=20parser=20?= =?UTF-8?q?=E2=80=94=20bound=20regexes=20(ReDoS),=20https-only=20URLs,=20n?= =?UTF-8?q?o=20redirect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- open-sse/services/freeWebSearch.ts | 37 ++++++++++++++++++++++-------- tests/unit/free-web-search.test.ts | 21 +++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/open-sse/services/freeWebSearch.ts b/open-sse/services/freeWebSearch.ts index f5c2c96f2345..3f0e890f8e70 100644 --- a/open-sse/services/freeWebSearch.ts +++ b/open-sse/services/freeWebSearch.ts @@ -24,10 +24,16 @@ const DDG_USER_AGENT = // Real lite shape: `Title` (href usually // before class; quotes may be single or double) and `…`. -const ANCHOR_RE = /]*?class=['"][^'"]*result-link[^'"]*['"][^>]*)>([\s\S]*?)<\/a>/gi; +// The inner-content captures are HARD-BOUNDED ({0,N}?) and the whole body is truncated +// (MAX_HTML_BYTES) so adversarial/unclosed HTML can't cause catastrophic backtracking +// (ReDoS) — real titles/snippets are short. See CLAUDE.md PII learnings §1. +const ANCHOR_RE = + /]*?class=['"][^'"]*result-link[^'"]*['"][^>]*)>([\s\S]{0,512}?)<\/a>/gi; const HREF_RE = /href=['"]([^'"]+)['"]/i; const SNIPPET_RE = - /]*?class=['"][^'"]*result-snippet[^'"]*['"][^>]*>([\s\S]*?)<\/td>/gi; + /]*?class=['"][^'"]*result-snippet[^'"]*['"][^>]*>([\s\S]{0,2048}?)<\/td>/gi; +// Generous cap for the lite endpoint (~50 KB real) — bounds the regex input size. +const MAX_HTML_BYTES = 256 * 1024; function decodeEntities(text: string): string { return text @@ -43,19 +49,26 @@ function stripTags(html: string): string { return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim(); } -/** Resolve a lite result href to a real absolute URL. */ +/** + * Resolve a lite result href to a real absolute URL. Returns "" for anything that + * is not a plain http(s) URL — the decoded `uddg` value is external data, so we must + * never surface a `javascript:`/`data:`/`file:`/internal-IP URL to the caller (LLM + * or client); the parser skips empty URLs. + */ function resolveResultUrl(href: string): string { + let candidate = href; // Older/HTML endpoints wrap the target in a redirect: //duckduckgo.com/l/?uddg=&… const redirect = href.match(/[?&]uddg=([^&]+)/); if (redirect) { try { - return decodeURIComponent(redirect[1]); + candidate = decodeURIComponent(redirect[1]); } catch { - // malformed encoding — fall through to the raw href + candidate = href; // malformed encoding — fall back to the raw href } + } else if (href.startsWith("//")) { + candidate = `https:${href}`; } - if (href.startsWith("//")) return `https:${href}`; - return href; + return /^https?:\/\//i.test(candidate) ? candidate : ""; } /** @@ -63,8 +76,10 @@ function resolveResultUrl(href: string): string { * fully unit-testable. Result link N aligns with snippet N (the lite layout emits * them 1:1); a missing snippet yields an empty string rather than a crash. */ -export function parseDuckDuckGoLite(html: string): FreeSearchResult[] { - if (!html) return []; +export function parseDuckDuckGoLite(rawHtml: string): FreeSearchResult[] { + if (!rawHtml) return []; + // Bound the regex input size as the first ReDoS guard. + const html = rawHtml.length > MAX_HTML_BYTES ? rawHtml.slice(0, MAX_HTML_BYTES) : rawHtml; const snippets = [...html.matchAll(SNIPPET_RE)].map((m) => stripTags(m[1])); const results: FreeSearchResult[] = []; @@ -105,7 +120,9 @@ export async function freeWebSearch( }, body: new URLSearchParams({ q: query }).toString(), guard: "public-only", - allowRedirect: true, + // Keep redirects manual: the public-only guard only validates the initial URL, + // so following a 3xx could reach an internal host. DDG lite answers POST with 200. + allowRedirect: false, timeoutMs, }); diff --git a/tests/unit/free-web-search.test.ts b/tests/unit/free-web-search.test.ts index 178b1c877c8c..6b5dc44cf496 100644 --- a/tests/unit/free-web-search.test.ts +++ b/tests/unit/free-web-search.test.ts @@ -68,6 +68,27 @@ test("parseDuckDuckGoLite returns [] when there are no result links", () => { assert.deepEqual(parseDuckDuckGoLite(""), []); }); +test("parseDuckDuckGoLite drops non-http(s) result URLs (javascript:/data:/file:)", () => { + const html = `XSS + bad + Safe + good`; + const results = parseDuckDuckGoLite(html); + assert.equal(results.length, 1, "the javascript: link must be discarded"); + assert.equal(results[0].url, "https://ok.example.com"); + assert.doesNotMatch(JSON.stringify(results), /javascript:/); +}); + +test("parseDuckDuckGoLite is bounded on adversarial HTML (no catastrophic backtracking)", () => { + // Many unclosed result-link anchors + a huge filler tail must return promptly. + const pathological = + ``.repeat(2000) + "x".repeat(2_000_000); + const start = Date.now(); + const results = parseDuckDuckGoLite(pathological); + assert.ok(Date.now() - start < 1000, "must not hang on adversarial HTML"); + assert.ok(Array.isArray(results)); +}); + test("parseDuckDuckGoLite tolerates a missing snippet (empty string, not crash)", () => { const html = `Only a link`; const results = parseDuckDuckGoLite(html); From 744539445ef18574aab46da3a12a5f263956a5f2 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Thu, 18 Jun 2026 00:45:34 -0300 Subject: [PATCH 6/6] test(search): expect 13 providers after duckduckgo-free addition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GET /v1/search registry-listing test still asserted 12 providers and the pre-duckduckgo id list. duckduckgo-free is registered last in SEARCH_PROVIDERS, so getAllSearchProviders() returns it at the tail — update the count and append the id to the deepEqual. --- tests/unit/search-route.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/search-route.test.ts b/tests/unit/search-route.test.ts index 08ba7c9598ab..adada3fa8fa4 100644 --- a/tests/unit/search-route.test.ts +++ b/tests/unit/search-route.test.ts @@ -45,14 +45,14 @@ test.after(() => { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); -test("v1 search GET lists all 12 search providers", async () => { +test("v1 search GET lists all 13 search providers", async () => { const response = await searchRoute.GET(); const body = (await response.json()) as any; const ids = body.data.map((item: { id: string }) => item.id); assert.equal(response.status, 200); assert.equal(body.object, "list"); - assert.equal(body.data.length, 12); + assert.equal(body.data.length, 13); assert.deepEqual(ids, [ "serper-search", "brave-search", @@ -66,6 +66,7 @@ test("v1 search GET lists all 12 search providers", async () => { "searxng-search", "ollama-search", "zai-search", + "duckduckgo-free", ]); });