From 46a3c68eb05c6156f981151e6e15a09ea4349c59 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Tue, 11 Aug 2026 11:08:04 +0200 Subject: [PATCH 1/4] fix: disable Firefox JSON viewer so evaluate works on JSON APIs Firefox renders application/json documents in a built-in viewer whose own CSP () blocks Playwright's eval-based page.evaluate, crashing /v1 with a 500 on JSON APIs (closes #394). Setting devtools.jsonview.enabled=false renders JSON as plain text, which also returns the raw JSON body instead of the viewer's syntax-highlighted HTML. --- src/utils.py | 6 ++++++ tests/main_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/utils.py b/src/utils.py index e34bca9..83f1470 100644 --- a/src/utils.py +++ b/src/utils.py @@ -96,6 +96,12 @@ async def get_browser( proxy=proxy_config, humanize=True, locale=BROWSER_LOCALE or "auto", + # Firefox renders application/json documents in a built-in viewer whose + # own CSP (`script-src resource:`) blocks Playwright's eval-based + # evaluate(), crashing /v1 on JSON APIs (issue #394). Disabling the + # viewer renders JSON as plain text: evaluate works and consumers get + # the raw JSON instead of the viewer's HTML markup. + extra_prefs={"devtools.jsonview.enabled": False}, ) as browser_raw: # InvisiblePlaywright yields a Browser instance browser = cast("Browser", browser_raw) diff --git a/tests/main_test.py b/tests/main_test.py index ced8a60..cc8a5dd 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -58,6 +58,34 @@ def test_bypass(website: str): assert response.status_code == HTTPStatus.OK +def test_json_api(): + """JSON APIs must return 200, not crash on the UA evaluate. + + Firefox renders application/json in a built-in viewer whose CSP blocks + Playwright's eval-based evaluate() (issue #394). The browser must be + launched with the viewer disabled so /v1 works and returns the raw JSON. + """ + url = "https://api.ipify.org?format=json" + test_request = httpx2.get(url) + if test_request.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: + pytest.skip( + f"Skipping JSON API test - upstream error ({test_request.status_code})" + ) + + response = client.post( + "/v1", + json=LinkRequest.model_construct(url=url, cmd="request.get").model_dump(), + ) + + if response.status_code == HTTPStatus.REQUEST_TIMEOUT: + pytest.skip("Skipping JSON API test - timed out (upstream issue)") + + assert response.status_code == HTTPStatus.OK + solution = response.json()["solution"] + assert solution["user_agent"] + assert '"ip"' in solution["response"] + + def test_health_check(): """ Tests the health check endpoint. From d3a828e8146b8594606dbe0e4989f20fc32a76e2 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Tue, 11 Aug 2026 11:14:10 +0200 Subject: [PATCH 2/4] fix(v1): source User-Agent from request headers; evaluate only as fallback page.evaluate runs eval() in the page's main world, which fails with 'call to eval() blocked by CSP' under any CSP that disallows unsafe-eval - HTTP headers (already stripped), meta tags (not strippable), or internal viewer documents (#394). The navigation request already carries the UA the site actually saw, so take user_agent from page_request.request.headers and keep evaluate only as a best-effort fallback whose failure can no longer 500 the request. --- src/endpoints.py | 17 ++++++++++++++++- tests/main_test.py | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 346b1b4..a44f6a5 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -153,10 +153,25 @@ async def strip_csp_route(route) -> None: else await dep.page.content() ) + # The User-Agent the target site actually saw, taken from the navigation + # request headers. page.evaluate falls through to eval() in the page's + # main world and fails whenever the page CSP disallows eval (e.g. the + # Firefox JSON viewer in #394, or uncatchable meta-tag CSP), so evaluate + # is only a fallback and its failure must not turn the request into a 500. + user_agent = ( + page_request.request.headers.get("user-agent") if page_request else None + ) + if user_agent is None: + try: + user_agent = await dep.page.evaluate("navigator.userAgent") + except Exception: + logger.warning("Could not determine User-Agent via page.evaluate") + user_agent = "" + return LinkResponse( message="Success", solution=Solution( - user_agent=await dep.page.evaluate("navigator.userAgent"), + user_agent=user_agent, url=final_url if final_url is not None else dep.page.url, status=status, cookies=cookies, diff --git a/tests/main_test.py b/tests/main_test.py index cc8a5dd..06bc993 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -139,7 +139,9 @@ def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass: page = AsyncMock() page.url = "https://example.test/login" page.goto.return_value = MagicMock( - status=HTTPStatus.OK, headers={"content-type": "text/html"} + status=HTTPStatus.OK, + headers={"content-type": "text/html"}, + request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}), ) page.title.return_value = "Login" page.evaluate.return_value = "UnitTestBrowser/1.0" @@ -181,3 +183,22 @@ async def test_domcontentloaded_timeout_returns_408(): ) assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT + + +@pytest.mark.asyncio +async def test_user_agent_survives_csp_blocked_evaluate(): + """UA comes from request headers when page CSP blocks evaluate (#394). + + No CSP configuration (header, meta tag, or internal viewer document) may + turn /v1 into a 500. + """ + dep = fake_dep() + dep.page.evaluate.side_effect = Exception("call to eval() blocked by CSP") + + response = await read_item( + LinkRequest(url="https://example.test/login"), + dep, + ) + + assert response.status == "ok" + assert response.solution.user_agent == "UnitTestBrowser/1.0" From 9b933ea70cb03de03c5726a569cb81476f884aee Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Tue, 11 Aug 2026 11:23:31 +0200 Subject: [PATCH 3/4] chore: drop explanatory comments --- src/endpoints.py | 5 ----- src/utils.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index a44f6a5..f32f5cf 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -153,11 +153,6 @@ async def strip_csp_route(route) -> None: else await dep.page.content() ) - # The User-Agent the target site actually saw, taken from the navigation - # request headers. page.evaluate falls through to eval() in the page's - # main world and fails whenever the page CSP disallows eval (e.g. the - # Firefox JSON viewer in #394, or uncatchable meta-tag CSP), so evaluate - # is only a fallback and its failure must not turn the request into a 500. user_agent = ( page_request.request.headers.get("user-agent") if page_request else None ) diff --git a/src/utils.py b/src/utils.py index 83f1470..abc4ff2 100644 --- a/src/utils.py +++ b/src/utils.py @@ -96,11 +96,6 @@ async def get_browser( proxy=proxy_config, humanize=True, locale=BROWSER_LOCALE or "auto", - # Firefox renders application/json documents in a built-in viewer whose - # own CSP (`script-src resource:`) blocks Playwright's eval-based - # evaluate(), crashing /v1 on JSON APIs (issue #394). Disabling the - # viewer renders JSON as plain text: evaluate works and consumers get - # the raw JSON instead of the viewer's HTML markup. extra_prefs={"devtools.jsonview.enabled": False}, ) as browser_raw: # InvisiblePlaywright yields a Browser instance From d7792b8fcd17ff198ff0c68aae680ed3d6954dc5 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Tue, 11 Aug 2026 11:40:49 +0200 Subject: [PATCH 4/4] fix(test): assert camelCase userAgent key in JSON response --- tests/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index 6bd4854..b3136ac 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -83,7 +83,7 @@ def test_json_api(): assert response.status_code == HTTPStatus.OK solution = response.json()["solution"] - assert solution["user_agent"] + assert solution["userAgent"] assert '"ip"' in solution["response"]