Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion scripts/run_tests_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
HERMES_TEST_WORKERS Override worker count (default: os.cpu_count())
HERMES_TEST_PATHS Override discovery roots (colon-sep, default: 'tests')

The runner intentionally neutralizes runtime voice/TTS environment flags in
pytest subprocesses. Local interactive sessions can toggle those flags on;
tests must never inherit them and play real audio.

Exit code: 0 if every file's pytest exited 0; 1 otherwise.
"""

Expand Down Expand Up @@ -91,6 +95,22 @@
_DURATIONS_FILE = "test_durations.json"


# Runtime-only interactive voice flags must not leak from a developer's live
# Hermes process into test subprocesses. If ``HERMES_VOICE_TTS=1`` is inherited,
# gateway tests that complete fake assistant turns can call the real TTS stack
# and play audio (for example the fixture string "partial answer complete").
_PYTEST_CHILD_ENV_OVERRIDES = {
"HERMES_VOICE": "0",
"HERMES_VOICE_TTS": "0",
"HERMES_VOICE_DEBUG": "0",
}


def _pytest_child_env() -> dict[str, str]:
"""Return a copy of the environment safe for pytest subprocesses."""
return {**os.environ, **_PYTEST_CHILD_ENV_OVERRIDES}


def _approximately_count_tests(
files: List[Path], repo_root: Path
) -> dict[Path, int]:
Expand Down Expand Up @@ -260,7 +280,7 @@ def _run_one_file(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
env=os.environ,
env=_pytest_child_env(),
# POSIX: place the child at the head of its own process group so
# _kill_tree can SIGKILL the group atomically.
# Windows: this maps to CREATE_NEW_PROCESS_GROUP in CPython 3.12+;
Expand Down
32 changes: 29 additions & 3 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@
CI. Code using ``Path.home() / ".hermes"`` instead of the canonical
``get_hermes_home()`` is a bug to fix at the callsite.)
3. **Deterministic runtime.** TZ=UTC, LANG=C.UTF-8, PYTHONHASHSEED=0.
4. **No HERMES_SESSION_* inheritance** — the agent's current gateway
session must not leak into tests.
4. **No runtime Hermes state inheritance** — the agent's current gateway
session, voice/TTS toggles, and other live-session flags must not leak into
tests.

These invariants make the local test run match CI closely. Gaps that
remain (CPU count, xdist worker count) are addressed by the canonical
Expand All @@ -26,6 +27,17 @@

import pytest

# These process-wide runtime flags must be neutralized while this root
# conftest is imported, before pytest imports any test module for collection.
# Function-scoped fixtures are too late for collection-time module code.
_HERMES_RUNTIME_ZERO_VARS = frozenset({
"HERMES_VOICE",
"HERMES_VOICE_TTS",
"HERMES_VOICE_DEBUG",
})
for _runtime_var in _HERMES_RUNTIME_ZERO_VARS:
os.environ[_runtime_var] = "0"

# Ensure project root is importable
PROJECT_ROOT = Path(__file__).parent.parent
if str(PROJECT_ROOT) not in sys.path:
Expand Down Expand Up @@ -170,6 +182,12 @@ def _looks_like_credential(name: str) -> bool:
# unconditionally — individual tests that need them set do so explicitly.
_HERMES_BEHAVIORAL_VARS = frozenset({
"HERMES_YOLO_MODE",
# Runtime voice toggles are process-wide env flags in the TUI gateway.
# A live developer session can leave them enabled; tests must opt in
# explicitly or fake assistant completions may invoke real audio playback.
"HERMES_VOICE",
"HERMES_VOICE_TTS",
"HERMES_VOICE_DEBUG",
"HERMES_INTERACTIVE",
"HERMES_QUIET",
"HERMES_TOOL_PROGRESS",
Expand Down Expand Up @@ -324,6 +342,11 @@ def _looks_like_credential(name: str) -> bool:
"MATRIX_REQUIRE_MENTION",
})

# Voice flags are special: they are checked by background gateway threads after
# a test emits ``message.complete``. They are neutralized at root-conftest import
# for collection safety and kept at zero across fixture teardown so background
# work cannot observe an inherited live interactive value between tests.


@pytest.fixture(autouse=True)
def _hermetic_environment(tmp_path, monkeypatch):
Expand All @@ -340,7 +363,10 @@ def _hermetic_environment(tmp_path, monkeypatch):

# 2. Blank behavioral HERMES_* vars that could change test semantics.
for name in _HERMES_BEHAVIORAL_VARS:
monkeypatch.delenv(name, raising=False)
if name in _HERMES_RUNTIME_ZERO_VARS:
os.environ[name] = "0"
else:
monkeypatch.delenv(name, raising=False)

# 3. Redirect HERMES_HOME to a per-test tempdir. Code that reads
# ``~/.hermes/*`` via ``get_hermes_home()`` now gets the tempdir.
Expand Down
180 changes: 180 additions & 0 deletions tests/test_run_tests_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,3 +277,183 @@ def test_positional_path_not_treated_as_flag(tmp_path: Path) -> None:
# Discovery found the probe file (2 tests), proving the positional path
# was consumed as a root, not forwarded to pytest as a bad flag.
assert "test_flagprobe.py" in proc.stdout, proc.stdout


def test_runner_scrubs_inherited_voice_tts_env(tmp_path: Path) -> None:
"""Runtime voice flags from an interactive parent must not reach pytest.

The probe is intentionally outside ``tests/`` so the repo-level conftest
does not mask the runner behavior.
"""
repo_root = Path(__file__).resolve().parent.parent
runner = repo_root / "scripts" / "run_tests_parallel.py"

probe_dir = tmp_path / "probe"
probe_dir.mkdir()
(probe_dir / "test_voice_env_probe.py").write_text(
textwrap.dedent(
"""
import os

def test_voice_env_flags_are_neutralized():
assert os.environ.get("HERMES_VOICE") == "0"
assert os.environ.get("HERMES_VOICE_TTS") == "0"
assert os.environ.get("HERMES_VOICE_DEBUG") == "0"
"""
).strip()
+ "\n"
)

env = os.environ.copy()
env.update(
{
"HERMES_VOICE": "1",
"HERMES_VOICE_TTS": "1",
"HERMES_VOICE_DEBUG": "1",
}
)
proc = subprocess.run(
[
sys.executable,
str(runner),
"--paths",
str(probe_dir),
"-j",
"1",
"--file-timeout",
"30",
],
cwd=repo_root,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=60,
)

assert proc.returncode == 0, proc.stdout


def test_pytest_conftest_scrubs_voice_env_before_test_module_collection(
tmp_path: Path,
) -> None:
"""Direct pytest runs must neutralize voice flags before module import."""
repo_root = Path(__file__).resolve().parent.parent
nonce = f"{os.getpid()}-{int(time.time() * 1000)}"
probe = repo_root / "tests" / f"_tmp_voice_collection_probe_{nonce}.py"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This probe is created inside the source checkout. Even with the finally cleanup below, an interrupted pytest process can leave an untracked Python file under tests/; please keep the probe in a temporary test root while explicitly loading the root conftest.

try:
probe.write_text(
textwrap.dedent(
"""
import os

assert os.environ.get("HERMES_VOICE") == "0"
assert os.environ.get("HERMES_VOICE_TTS") == "0"
assert os.environ.get("HERMES_VOICE_DEBUG") == "0"

def test_collection_observed_neutral_voice_env():
pass
"""
).strip()
+ "\n"
)

env = os.environ.copy()
env.update(
{
"HERMES_VOICE": "1",
"HERMES_VOICE_TTS": "1",
"HERMES_VOICE_DEBUG": "1",
}
)
proc = subprocess.run(
[sys.executable, "-m", "pytest", "-q", str(probe)],
cwd=repo_root,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=60,
)

assert proc.returncode == 0, proc.stdout
finally:
try:
probe.unlink()
except FileNotFoundError:
pass


def test_pytest_conftest_keeps_voice_tts_neutralized_after_test_teardown(
tmp_path: Path,
) -> None:
"""Background work after fixture teardown must still see voice TTS off."""
repo_root = Path(__file__).resolve().parent.parent
nonce = f"{os.getpid()}-{int(time.time() * 1000)}"
probe = repo_root / "tests" / f"_tmp_voice_teardown_probe_{nonce}.py"
handoff = tmp_path / "voice-env-after-teardown.txt"
try:
probe.write_text(
textwrap.dedent(
f"""
import json, os, threading, time
from pathlib import Path

HANDOFF = Path({str(handoff)!r})
VOICE_ENV_VARS = (
"HERMES_VOICE",
"HERMES_VOICE_TTS",
"HERMES_VOICE_DEBUG",
)

def test_background_thread_observes_voice_env_after_return(monkeypatch):
for name in VOICE_ENV_VARS:
monkeypatch.setenv(name, "1")
assert os.environ[name] == "1"

def worker():
deadline = time.monotonic() + 5
while time.monotonic() < deadline and any(
os.environ.get(name) == "1" for name in VOICE_ENV_VARS
):
time.sleep(0.01)
HANDOFF.write_text(json.dumps({{
name: os.environ.get(name, "<missing>")
for name in VOICE_ENV_VARS
}}))

threading.Thread(target=worker, daemon=False).start()
"""
).strip()
+ "\n"
)

env = os.environ.copy()
env.update(
{
"HERMES_VOICE": "1",
"HERMES_VOICE_TTS": "1",
"HERMES_VOICE_DEBUG": "1",
}
)
proc = subprocess.run(
[sys.executable, "-m", "pytest", "-q", str(probe)],
cwd=repo_root,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=60,
)

assert proc.returncode == 0, proc.stdout
assert json.loads(handoff.read_text()) == {
"HERMES_VOICE": "0",
"HERMES_VOICE_TTS": "0",
"HERMES_VOICE_DEBUG": "0",
}
finally:
try:
probe.unlink()
except FileNotFoundError:
pass
Loading