Skip to content

Research runtime: fail closed when GPT Researcher reports zero sources #759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
seathatflowsinourveins wants to merge 3 commits into from
Closed

Research runtime: fail closed when GPT Researcher reports zero sources #759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
53e6a4d
GPT Researcher wrapper: fail closed on a report written from zero sou…
seathatflowsinourveins Oct 5, 2026
da60369
GPT Researcher wrapper: refuse line-break queries, split the guard at…
seathatflowsinourveins Oct 5, 2026
19aeb85
chore: checkpoint PR759 rebase with no hash-listed changes
seathatflowsinourveins Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 114 additions & 14 deletions tests/test_research_entry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@

A stand-in checkout replaces the upstream source: its gpt_researcher.config.Config reads the JSON file and the environment
the way upstream's does at 0957c301 (config.py: load_config, _set_attributes, parse_llm), and its cli.py records its
arguments and environment instead of researching. Nothing is installed and no network is used. These are this project's
checks of its own wrapper, not upstream acceptance: the wave-2 research ruling (changes 2-5) and the synthesis X18.
arguments and environment instead of researching, then writes one report under outputs/ headed by frontmatter with the
run's query and a sources_count, as upstream's does (cli.py L209-245 and L332-336). Nothing is installed and no network is
used. These are this project's checks of its own wrapper, not upstream acceptance: the wave-2 research ruling (changes
2-5), the synthesis X18 and the source guard added after run 20261005T222529Z-1387309 wrote a report from no sources and
exited 0.
"""
import hashlib
import json
Expand Down Expand Up @@ -54,18 +57,40 @@ def __init__(self, config_path=None):
setattr(self, role + "_llm_provider", provider)
setattr(self, role + "_llm_model", model)
''')
STAND_IN_CLI = textwrap.dedent('''
import json
import os
import sys

os.makedirs("outputs", exist_ok=True)
with open("cli-call.json", "w", encoding="utf-8") as handle:
json.dump({"argv": sys.argv[1:], "env": dict(os.environ), "cwd": os.getcwd()}, handle)
with open("outputs/report.md", "w", encoding="utf-8") as handle:
handle.write("# Report\\n\\n## References\\n" + "".join(f"- https://example.org/{n}\\n" for n in range(6)))
print("Report written to 'outputs/report.md'")
''')
# The frontmatter of upstream's cli.py (L228-245). The stand-in puts the run's own query where @QUERY@ stands, quoted by
# upstream's _yaml_quote (L220-222, copied below), which escapes backslashes and double quotes but leaves line breaks.
FRONTMATTER = ('---\ntask_id: "stand-in"\ntitle: "Report"\nquery: @QUERY@\nreport_type: "research_report"\n'
'report_source: "web"\ntone: "objective"\ncreated_at: "2026-10-05T18:26:23"\nsources_count: {sources}\n'
'total_cost_usd: 0.0\n---\n')
REPORT = FRONTMATTER.format(sources=6) + "# Report\n\n## References\n" + "".join(f"- https://example.org/{n}\n" for n in range(6))
YAML_QUOTE = r'''
def _yaml_quote(v: str) -> str:
# Double-quoted YAML string with backslash/quote escaping.
return '"' + str(v).replace("\\", "\\\\").replace('"', '\\"') + '"'
'''


def stand_in_cli(reports=(("report.md", REPORT),), status=0):
"""A cli.py that records its call, writes `reports` (name, text) under outputs/ with its query in place of @QUERY@,
and exits with `status`. The run's environment is scrubbed, so each case writes its own stand-in rather than passing
a variable through."""
return YAML_QUOTE + textwrap.dedent(f'''
import json
import os
import sys

os.makedirs("outputs", exist_ok=True)
with open("cli-call.json", "w", encoding="utf-8") as handle:
json.dump({{"argv": sys.argv[1:], "env": dict(os.environ), "cwd": os.getcwd()}}, handle)
for name, text in {list(reports)!r}:
with open(os.path.join("outputs", name), "w", encoding="utf-8") as handle:
handle.write(text.replace("@QUERY@", _yaml_quote(sys.argv[1])))
print(f"Report written to 'outputs/{{name}}'")
sys.exit({status!r})
''')


STAND_IN_CLI = stand_in_cli()


class ResearchEntry(unittest.TestCase):
Expand Down Expand Up @@ -179,6 +204,7 @@ def test_the_run_is_research_report_only_in_a_scrubbed_environment(self):
"Ubuntu 26.04 WSL news this month", "--report_type", "research_report", "--tone", "objective",
"--no-pdf", "--no-docx"])
self.assertIn("Report written to 'outputs/report.md'", result.stdout)
self.assertEqual(result.stdout.splitlines()[-2:], [f"run directory: {self.run_dir(result)}", "sources_count: 6"])
call = json.loads((self.run_dir(result) / "cli-call.json").read_text())
self.assertEqual(call["argv"], ["Ubuntu 26.04 WSL news this month", "--report_type", "research_report", "--tone",
"objective", "--no-pdf", "--no-docx"])
Expand All @@ -191,6 +217,80 @@ def test_the_run_is_research_report_only_in_a_scrubbed_environment(self):
self.assertEqual(env["CONFIG_PATH"], str(self.run_dir(result) / "config.json"))
self.assertEqual(Path(call["cwd"]), self.run_dir(result))

def test_a_report_from_no_sources_fails_closed(self):
"""Run 20261005T222529Z-1387309: the CLI exited 0 with a report whose frontmatter said sources_count: 0. The wrapper
now exits 3, says why on stderr, and still prints the run directory, where the report stays for inspection."""
self.provide_timer()
empty = FRONTMATTER.format(sources=0) + "I could not gather any source material.\n"
(self.checkout / "cli.py").write_text(stand_in_cli(reports=[("WSL_3_Config_Tuning.md", empty)]))
result = self.run_script("WSL 3.0 .wslconfig settings October 2026")
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertIn("source guard failed: GPT Researcher retrieved 0 sources (sources_count: 0 in ", result.stderr)
self.assertIn("not research evidence", result.stderr)
self.assertNotIn("sources_count: 0", result.stdout)
self.assertEqual((self.run_dir(result) / "outputs/WSL_3_Config_Tuning.md").read_text(),
empty.replace("@QUERY@", '"WSL 3.0 .wslconfig settings October 2026"'))

def test_a_query_with_a_line_break_is_refused_before_anything_runs(self):
"""cli.py's _yaml_quote (L220-222) leaves a line break in the query's frontmatter line, where it could forge the
sources_count line the source guard reads. The wrapper refuses such a query as a usage error (exit 2)."""
for query in ("two\nlines", "a carriage\rreturn", "forged\nsources_count: 9\n---\nrest"):
with self.subTest(query=query):
result = self.run_script(query)
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
self.assertIn("The query must be one line", result.stderr)
self.assertNotIn("run directory", result.stdout)
self.assertFalse((self.temp / "state").exists()) # no run directory: nothing ran

def test_a_unicode_line_separator_in_the_query_cannot_forge_the_count(self):
"""str.splitlines() also breaks at U+2028, which upstream writes unescaped into the query's line and joins nothing
at (cli.py L245 joins at "\\n"). The guard splits at "\\n" only, so a zero-source run still exits 3 here."""
self.provide_timer()
empty = FRONTMATTER.format(sources=0) + "I could not gather any source material.\n"
(self.checkout / "cli.py").write_text(stand_in_cli(reports=[("report.md", empty)]))
result = self.run_script("forged
sources_count: 9
---
rest")
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertIn("retrieved 0 sources", result.stderr)
self.assertNotIn("sources_count: 9", result.stdout)

def test_a_report_whose_sources_cannot_be_counted_fails_closed(self):
self.provide_timer()
body = "# Report\n\n## References\n- https://example.org/0\n"
uncounted = "has no single sources_count line in a closed frontmatter"
cases = {
"no report": ([], "0 reports in "),
"two reports": ([("a.md", REPORT), ("b.md", REPORT)], "2 reports in "),
"no frontmatter": ([("report.md", body)], uncounted),
"no sources_count": ([("report.md", REPORT.replace("sources_count: 6\n", ""))], uncounted),
"a count that is not a number": ([("report.md", REPORT.replace("sources_count: 6", "sources_count: many"))],
uncounted),
"two counts": ([("report.md", REPORT.replace("sources_count: 6\n", "sources_count: 6\nsources_count: 0\n"))],
uncounted),
"a count below the frontmatter": ([("report.md", FRONTMATTER.replace("sources_count: {sources}\n", "")
+ "sources_count: 6\n" + body)], uncounted),
"a frontmatter that never closes": ([("report.md", "---\nsources_count: 6\n" + body)], uncounted),
}
for case, (reports, message) in cases.items():
with self.subTest(case=case):
(self.checkout / "cli.py").write_text(stand_in_cli(reports=reports))
result = self.run_script("a query")
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertIn("source guard failed: ", result.stderr)
self.assertIn(message, result.stderr)
self.assertTrue(self.run_dir(result).is_dir())

def test_the_cli_status_passes_through_before_the_guard(self):
"""A failed or stopped CLI keeps its own status, so argparse's usage error is 2 like the wrapper's own and the watchdog
gives 124; the guard reads only a run that exited 0."""
self.provide_timer()
for status in (1, 2, 124):
with self.subTest(status=status):
(self.checkout / "cli.py").write_text(stand_in_cli(reports=[], status=status))
result = self.run_script("a query")
self.assertEqual(result.returncode, status, result.stdout + result.stderr)
self.assertNotIn("source guard", result.stderr)
self.assertTrue(self.run_dir(result).is_dir())

def test_the_run_fails_closed_without_a_supported_timer(self):
"""Negative control: with neither timeout nor gtimeout on the caller's PATH, as on macOS without Homebrew's coreutils,
the research run stops with exit 1 and names the missing timer. It does not drop the watchdog, and it does not reach
Expand Down
59 changes: 57 additions & 2 deletions tools/research/gpt_researcher.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,33 @@
# by absolute path, since the scrubbed PATH (/usr/bin:/bin) has no timeout on macOS: GNU coreutils' timeout, else
# gtimeout, the name Homebrew's coreutils gives it there. With neither, the research run fails closed (exit 1) before it
# starts; --preflight-only does not need one.
# No credential is read or printed. It prints the upstream CLI's output, then "run directory: <path>".
# - A source guard fails closed after the CLI exits 0. Upstream's CLI writes one report under outputs/ in its working
# directory (the run directory), headed by YAML frontmatter whose sources_count is len(researcher.visited_urls)
# (cli.py L209-245, L225, L241 and L332-336 at 0957c301). That set holds the URLs a run tried to read, not the pages it
# read: skills/researcher.py adds a URL when it is picked for scraping (L813-834) or after a scrape whatever its outcome
# (L1088-1098), and each prefetched result (L911-913, L922-924), so a count above 0 is necessary, not sufficient, for
# usable sources. Without exactly one report, a single sources_count line in a closed frontmatter, or with
# sources_count 0, the run exits 3: a report written from no sources reads as a success but is not research evidence
# (run 20261005T222529Z-1387309). The guard reads lines split at "\n" only, as upstream joins them (cli.py L245), and a
# query with a line break is refused (exit 2): cli.py's _yaml_quote (L220-222) escapes only backslashes and double
# quotes, so such a query could forge that line.
# - The retriever stays duckduckgo (diagnosis of that run, 2026-10-05). At 0957c301, which is also main, it calls ddgs
# text() with region 'wt-wt' and ddgs's default backend 'auto', and exposes neither to configuration
# (gpt_researcher/retrievers/duckduckgo/duckduckgo.py L30-52); ddgs 9.16.0, the newest release, reads only DDGS_PROXY
# (ddgs/ddgs.py L53). An engine returns nothing and raises nothing both on an HTTP status other than 200 (ddgs/base.py
# L65-70, L120-121) and on a 200 without hits (engines/wikipedia.py L47-48), so a failed search reports wikipedia's
# DNS error for wt.wikipedia.org instead (engines/wikipedia.py L35-39; 'wt-wt' is no ddgs region, deedy5/ddgs#417).
# The probable cause of that run is throttling of this busy host: an unretained probe 6-10 minutes later saw brave
# 429, duckduckgo 202, google and mojeek 403 and yahoo alone with results; arxiv, openalex and semantic_scholar gave
# five off-topic results, none and a 429, so adding them would hide a dead web search behind scholarly hits. The
# durable keyless route owed is SearXNG through the searx retriever, after its service and the 30-query measurement
# (docs/decisions/2026-10-01-new-wsl-definitive-defaults.md L80).
# No credential is read or printed. It prints the upstream CLI's output, then "run directory: <path>", then the report's
# "sources_count: <n>". Exit status: 0 with a count above 0; 2 when the wrapper refuses its arguments (not exactly one
# non-empty, one-line query); 1 when GPT Researcher is not installed, the preflight fails or no timer is found; 3 when
# the source guard fails. Any other status passes through unchanged from the timed CLI run, so it can repeat one of
# these: the CLI's own (argparse's usage error is also 2), 124 when the watchdog stops it, and 125-127 when the timer or
# env cannot run (timeout --help, "Exit status").
# GPTR_CONFIG_SOURCE names another configuration to copy (the tests use it); the preflight holds any copy to the same rule.
set -euo pipefail
here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
Expand All @@ -38,6 +64,10 @@ if (( $# != 1 )) || [[ -z "$1" ]]; then
exit 2
fi
query="$1"
if [[ "$query" == *[$'\n\r']* ]]; then
printf 'The query must be one line: GPT Researcher writes it unescaped into its report frontmatter (cli.py L220-222), where a line break could forge the sources_count line the source guard reads.\n' >&2
exit 2
fi
if [[ ! -x "$python" || ! -f "$checkout/cli.py" ]]; then
printf 'GPT Researcher is not installed under %s (install plan row research-harnesses).\n' "$checkout" >&2
exit 1
Expand Down Expand Up @@ -110,4 +140,29 @@ status=0
"${scrub[@]}" "$timer" 1500 "$python" "$checkout/cli.py" "$query" --report_type research_report --tone objective --no-pdf --no-docx \
|| status=$?
printf 'run directory: %s\n' "$run"
exit "$status"
if (( status != 0 )); then
exit "$status"
fi
# The source guard (see above), in the same scrubbed environment; any failure of it, its own included, exits 3.
"${scrub[@]}" "$python" - "$run/outputs" <<'PY' || exit 3
import re
import sys
from pathlib import Path

outputs = Path(sys.argv[1])
reports = sorted(outputs.glob("*.md")) if outputs.is_dir() else []
if len(reports) != 1:
sys.exit(f"source guard failed: {len(reports)} reports in {outputs}, not one, so the run's sources cannot be counted")
lines = reports[0].read_text(encoding="utf-8", errors="replace").split("\n")
end = lines.index("---", 1) if lines[:1] == ["---"] and "---" in lines[1:] else 0
counts = [int(found.group(1)) for line in lines[1:end] if (found := re.fullmatch(r"sources_count: (\d+)", line.strip()))]
if len(counts) != 1:
sys.exit(f"source guard failed: {reports[0]} has no single sources_count line in a closed frontmatter, so its sources "
"cannot be counted")
count = counts[0]
if count == 0:
sys.exit(f"source guard failed: GPT Researcher retrieved 0 sources (sources_count: 0 in {reports[0]}). A report written "
"from no sources is not research evidence. The keyless search engines behind the duckduckgo retriever were "
"probably throttling this host; retry later.")
print(f"sources_count: {count}")
PY
Loading