From 89a31c9fc8447edbc80760f9aa3e50a75143e3bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:24:23 +0000 Subject: [PATCH 1/4] chore(deps): bump nltk from 3.9.4 to 3.10.3 in /envs/textarena_env Bumps [nltk](https://github.com/nltk/nltk) from 3.9.4 to 3.10.3. - [Release notes](https://github.com/nltk/nltk/releases) - [Changelog](https://github.com/nltk/nltk/blob/develop/ChangeLog) - [Commits](https://github.com/nltk/nltk/compare/3.9.4...v3.10.3) --- updated-dependencies: - dependency-name: nltk dependency-version: 3.10.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- envs/textarena_env/uv.lock | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/envs/textarena_env/uv.lock b/envs/textarena_env/uv.lock index 496a7c2151..4c7096b200 100644 --- a/envs/textarena_env/uv.lock +++ b/envs/textarena_env/uv.lock @@ -668,6 +668,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/bd/05055d8360cef0757d79367157f3b15c0a0715e81e08f86a04018ec045f0/cyclopts-4.10.2-py3-none-any.whl", hash = "sha256:a1f2d6f8f7afac9456b48f75a40b36658778ddc9c6d406b520d017ae32c990fe", size = 204314, upload-time = "2026-04-08T23:57:46.969Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1403,17 +1412,18 @@ wheels = [ [[package]] name = "nltk" -version = "3.9.4" +version = "3.10.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, + { name = "defusedxml" }, { name = "joblib" }, { name = "regex" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/e6/fe51d2bb1a3b446f59c5c8165999a9fee208bc346af90a7cbf7657bc0d75/nltk-3.10.3.tar.gz", hash = "sha256:bb9327a461c3811c2fa4900e03840401f2126adfb30c0072827c433bd2444ea4", size = 5137152, upload-time = "2026-08-12T23:46:37.258Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6d/ebd2af4640b12168fdf0cb74b6118df2f32a2f62ec7e0c06fbfd80706639/nltk-3.10.3-py3-none-any.whl", hash = "sha256:ff9598a8e20518ee0d557745890cc4435b9578489e2dcbc69c4f81fa060caf7c", size = 1798643, upload-time = "2026-08-12T23:44:13.478Z" }, ] [[package]] @@ -1775,10 +1785,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -1847,9 +1857,9 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/da/99/b342345300f13440fe9fe385c3c481e2d9a595ee3bab4d3219247ac94e9a/pandas-3.0.2.tar.gz", hash = "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", size = 4645855, upload-time = "2026-03-31T06:48:30.816Z" } wheels = [ @@ -2736,8 +2746,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "jeepney", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ From fd769a31cb48639e37df553919018082cef5dd62 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 10 Sep 2026 10:49:34 +0200 Subject: [PATCH 2/4] fix: handle nltk no-proxy downloads --- envs/textarena_env/pyproject.toml | 2 +- envs/textarena_env/server/environment.py | 40 +++++++- envs/textarena_env/uv.lock | 2 +- tests/envs/test_textarena_nltk.py | 122 +++++++++++++++++++++++ 4 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 tests/envs/test_textarena_nltk.py diff --git a/envs/textarena_env/pyproject.toml b/envs/textarena_env/pyproject.toml index 206fe49c5c..9c8af63c8a 100644 --- a/envs/textarena_env/pyproject.toml +++ b/envs/textarena_env/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ # "openspiel>=1.0.0", # "smolagents>=1.22.0,<2", "textarena>=0.6.1", - "nltk>=3.9.3", + "nltk>=3.10.3", # For custom Gradio tab (server/gradio_ui.py) when ENABLE_WEB_INTERFACE=true "gradio>=6.15.1", ] diff --git a/envs/textarena_env/server/environment.py b/envs/textarena_env/server/environment.py index 83b7d6c912..9944e01119 100644 --- a/envs/textarena_env/server/environment.py +++ b/envs/textarena_env/server/environment.py @@ -8,7 +8,10 @@ from __future__ import annotations +import os +import subprocess import sys +import urllib.request from typing import Any, Dict, Iterable, List, Optional from uuid import uuid4 @@ -51,8 +54,41 @@ def _ensure_nltk_data() -> None: "NLTK is required for TextArena environments. " "Install textarena_env dependencies (including nltk)." ) from exc - nltk.download("words", quiet=True) - nltk.download("averaged_perceptron_tagger_eng", quiet=True) + packages = ("words", "averaged_perceptron_tagger_eng") + opener = urllib.request._opener + explicit_proxy = opener is not None and any( + isinstance(handler, urllib.request.ProxyHandler) and handler.proxies + for handler in opener.handlers + ) + if ( + nltk.__version__ == "3.10.3" + and set(urllib.request.getproxies()) == {"no"} + and not explicit_proxy + ): + # NLTK #3748 mistakes NO_PROXY alone for a carrying proxy. Isolate the + # workaround so other server threads retain their proxy configuration. + # Real proxies still use NLTK's normal, security-enforcing download path. + download_env = os.environ.copy() + download_env.pop("NO_PROXY", None) + download_env.pop("no_proxy", None) + subprocess.run( + [ + sys.executable, + "-m", + "nltk.downloader", + "--quiet", + "--exit-on-error", + "--dir", + nltk.downloader.Downloader().default_download_dir(), + *packages, + ], + env=download_env, + check=True, + timeout=120, + ) + else: + for package in packages: + nltk.download(package, quiet=True, raise_on_error=True) _NLTK_DOWNLOADED = True diff --git a/envs/textarena_env/uv.lock b/envs/textarena_env/uv.lock index 4c7096b200..5519cfa3b5 100644 --- a/envs/textarena_env/uv.lock +++ b/envs/textarena_env/uv.lock @@ -1663,7 +1663,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.115.0" }, { name = "gradio", specifier = ">=6.15.1" }, - { name = "nltk", specifier = ">=3.9.3" }, + { name = "nltk", specifier = ">=3.10.3" }, { name = "openenv", specifier = ">=0.2.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, diff --git a/tests/envs/test_textarena_nltk.py b/tests/envs/test_textarena_nltk.py new file mode 100644 index 0000000000..ee01133ad3 --- /dev/null +++ b/tests/envs/test_textarena_nltk.py @@ -0,0 +1,122 @@ +"""NLTK startup compatibility without changing process-wide proxy settings.""" + +import os +import subprocess +import sys +import urllib.request +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from textarena_env.server import environment + + +@pytest.fixture +def nltk_download(monkeypatch, tmp_path): + nltk = SimpleNamespace( + __version__="3.10.3", + download=Mock(), + downloader=SimpleNamespace( + Downloader=lambda: SimpleNamespace( + default_download_dir=lambda: str(tmp_path) + ) + ), + ) + monkeypatch.setitem(sys.modules, "nltk", nltk) + monkeypatch.setattr(environment, "_NLTK_DOWNLOADED", False) + monkeypatch.setattr(urllib.request, "_opener", None) + monkeypatch.setattr( + urllib.request, "getproxies", urllib.request.getproxies_environment + ) + for key in list(os.environ): + if key.lower().endswith("_proxy"): + monkeypatch.delenv(key) + return nltk + + +@pytest.mark.parametrize("proxy_key", ["NO_PROXY", "no_proxy"]) +def test_exclusion_only_download_is_isolated(monkeypatch, nltk_download, proxy_key): + monkeypatch.setenv(proxy_key, "localhost,127.0.0.1") + original_env = dict(os.environ) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + environment._ensure_nltk_data() + environment._ensure_nltk_data() + + run.assert_called_once() + command = run.call_args.args[0] + assert command[:3] == [sys.executable, "-m", "nltk.downloader"] + assert "--exit-on-error" in command + assert command[-2:] == ["words", "averaged_perceptron_tagger_eng"] + assert run.call_args.kwargs["check"] is True + assert run.call_args.kwargs["env"] == { + key: value for key, value in original_env.items() if key != proxy_key + } + assert dict(os.environ) == original_env + nltk_download.download.assert_not_called() + + +@pytest.mark.parametrize( + "proxy_key", ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] +) +def test_carrying_proxy_keeps_nltk_security_checks( + monkeypatch, nltk_download, proxy_key +): + monkeypatch.setenv("NO_PROXY", "localhost") + monkeypatch.setenv(proxy_key, "http://proxy.example:3128") + nltk_download.download.side_effect = PermissionError("proxied fetch") + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + with pytest.raises(PermissionError, match="proxied fetch"): + environment._ensure_nltk_data() + + run.assert_not_called() + nltk_download.download.assert_called_once_with( + "words", quiet=True, raise_on_error=True + ) + assert not environment._NLTK_DOWNLOADED + + +def test_explicit_proxy_is_not_bypassed(monkeypatch, nltk_download): + monkeypatch.setenv("NO_PROXY", "localhost") + monkeypatch.setattr( + urllib.request, + "_opener", + urllib.request.build_opener( + urllib.request.ProxyHandler({"https": "http://proxy.example:3128"}) + ), + ) + nltk_download.download.side_effect = PermissionError("proxied fetch") + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + with pytest.raises(PermissionError, match="proxied fetch"): + environment._ensure_nltk_data() + run.assert_not_called() + + +def test_failed_subprocess_does_not_cache_success(monkeypatch, nltk_download): + monkeypatch.setenv("NO_PROXY", "localhost") + run = Mock(side_effect=subprocess.CalledProcessError(1, "nltk.downloader")) + monkeypatch.setattr(subprocess, "run", run) + for _ in range(2): + with pytest.raises(subprocess.CalledProcessError): + environment._ensure_nltk_data() + assert not environment._NLTK_DOWNLOADED + assert run.call_count == 2 + + +@pytest.mark.parametrize("version,proxies", [("3.10.3", False), ("3.10.4", True)]) +def test_unaffected_download_uses_normal_path( + monkeypatch, nltk_download, version, proxies +): + nltk_download.__version__ = version + if proxies: + monkeypatch.setenv("NO_PROXY", "localhost") + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + environment._ensure_nltk_data() + run.assert_not_called() + assert nltk_download.download.call_count == 2 + assert environment._NLTK_DOWNLOADED From 57c14c033b7f07e7a8b861a2c5923adb775737c7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 10 Sep 2026 10:55:18 +0200 Subject: [PATCH 3/4] fix: recognize exclusion-only proxy handlers --- envs/textarena_env/server/environment.py | 3 ++- tests/envs/test_textarena_nltk.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/envs/textarena_env/server/environment.py b/envs/textarena_env/server/environment.py index 9944e01119..26c01145a2 100644 --- a/envs/textarena_env/server/environment.py +++ b/envs/textarena_env/server/environment.py @@ -57,7 +57,8 @@ def _ensure_nltk_data() -> None: packages = ("words", "averaged_perceptron_tagger_eng") opener = urllib.request._opener explicit_proxy = opener is not None and any( - isinstance(handler, urllib.request.ProxyHandler) and handler.proxies + isinstance(handler, urllib.request.ProxyHandler) + and any(scheme != "no" for scheme in handler.proxies) for handler in opener.handlers ) if ( diff --git a/tests/envs/test_textarena_nltk.py b/tests/envs/test_textarena_nltk.py index ee01133ad3..463dd929b4 100644 --- a/tests/envs/test_textarena_nltk.py +++ b/tests/envs/test_textarena_nltk.py @@ -120,3 +120,21 @@ def test_unaffected_download_uses_normal_path( run.assert_not_called() assert nltk_download.download.call_count == 2 assert environment._NLTK_DOWNLOADED + + +def test_exclusion_only_global_opener_is_not_a_carrying_proxy( + monkeypatch, nltk_download +): + monkeypatch.setenv("NO_PROXY", "localhost") + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({"no": "localhost"}) + ) + monkeypatch.setattr(urllib.request, "_opener", opener) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + environment._ensure_nltk_data() + + run.assert_called_once() + nltk_download.download.assert_not_called() + assert urllib.request._opener is opener From 73f16b3133102b1dbae206eb3648938c6c7b2f55 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 10 Sep 2026 11:01:18 +0200 Subject: [PATCH 4/4] fix: verify downloaded nltk resources --- envs/textarena_env/server/environment.py | 4 ++++ tests/envs/test_textarena_nltk.py | 25 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/envs/textarena_env/server/environment.py b/envs/textarena_env/server/environment.py index 26c01145a2..5591f41ad1 100644 --- a/envs/textarena_env/server/environment.py +++ b/envs/textarena_env/server/environment.py @@ -90,6 +90,10 @@ def _ensure_nltk_data() -> None: else: for package in packages: nltk.download(package, quiet=True, raise_on_error=True) + # NLTK 3.10.3's CLI can exit zero after a failed download. Verify the + # resources in this process before caching successful initialization. + nltk.data.find("corpora/words") + nltk.data.find("taggers/averaged_perceptron_tagger_eng") _NLTK_DOWNLOADED = True diff --git a/tests/envs/test_textarena_nltk.py b/tests/envs/test_textarena_nltk.py index 463dd929b4..2eeabdcdc4 100644 --- a/tests/envs/test_textarena_nltk.py +++ b/tests/envs/test_textarena_nltk.py @@ -16,6 +16,7 @@ def nltk_download(monkeypatch, tmp_path): nltk = SimpleNamespace( __version__="3.10.3", download=Mock(), + data=SimpleNamespace(find=Mock()), downloader=SimpleNamespace( Downloader=lambda: SimpleNamespace( default_download_dir=lambda: str(tmp_path) @@ -138,3 +139,27 @@ def test_exclusion_only_global_opener_is_not_a_carrying_proxy( run.assert_called_once() nltk_download.download.assert_not_called() assert urllib.request._opener is opener + + +@pytest.mark.parametrize( + "missing_resource", ["corpora/words", "taggers/averaged_perceptron_tagger_eng"] +) +def test_cli_zero_exit_without_corpora_does_not_cache_success( + monkeypatch, nltk_download, missing_resource +): + monkeypatch.setenv("NO_PROXY", "localhost") + # NLTK's CLI can exit zero after reporting a download/security error. + run = Mock(return_value=subprocess.CompletedProcess([], 0)) + monkeypatch.setattr(subprocess, "run", run) + + def find(resource): + if resource == missing_resource: + raise LookupError("corpus missing") + return resource + + nltk_download.data.find.side_effect = find + for _ in range(2): + with pytest.raises(LookupError, match="corpus missing"): + environment._ensure_nltk_data() + assert not environment._NLTK_DOWNLOADED + assert run.call_count == 2