From f77cc936cdb790aeb0079d3bdc6053bb1b88430a Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 13 Jul 2026 14:19:54 -0700 Subject: [PATCH 01/12] feat(tools): add Tenki cloud sandbox terminal backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Tenki (tenki.cloud) as a seventh terminal execution backend alongside local, docker, ssh, singularity, modal, and daytona. Hermes creates Tenki sandboxes on demand for terminal, file tools, and execute_code, and terminates them on cleanup by default (opt-in pause/resume persistence via container_persistent: true). Core: - tools/environments/tenki.py: TenkiEnvironment — sandbox lifecycle, exec, pause/resume persistence, remote file sync-back - tools/tenki_config.py: profile-scope-aware auth/workspace/project/endpoint resolution from the Tenki CLI config or environment - Shared _container_config_from_env_config() helper replaces the three duplicated container-config dicts (terminal, file tools, execute_code) - Setup wizard, doctor, status, gateway, and CLI wiring; website docs, env-var reference, and cli-config.yaml.example - Optional tenki extra (tenki-sandbox==0.1.1), lazy-installed like modal/daytona Security & correctness hardening: - Do not inject the supervisor's control-plane Tenki token into the model-controlled guest env; host-side SDK auth is unchanged. Nested-sandbox creation is an explicit opt-in via terminal.tenki_forward_env (which also forwards the resolved token so `tenki login` credentials work), and logs a warning when the control-plane token is forwarded. - Resolve Tenki credentials and forwarded env through agent.secret_scope so an active profile scope wins over process-global os.environ and the shared machine CLI login is skipped when a profile scope is authoritative. - Strip TENKI_AUTH_TOKEN / TENKI_API_KEY from spawned subprocess environments (provider blocklist + always-strip tier), matching modal/daytona. - Namespace persistent sandbox identity by a per-profile token (name + metadata + reuse match) and resolve the snapshot-store path per profile, bound at construction so background-thread cleanup writes to the right home. - Durability gate: a non-durable snapshot is not recorded (cleanup pauses and preserves prior state); a failed pause leaves the sandbox live rather than terminating it. Restore falls back to a base image only for an unrecoverable snapshot (gone / non-durable / snapshot-specific invalid state), preserving the pointer on transient errors. - Config: blank tenki_api_endpoint default across both config loaders so the documented env/CLI fallback is reachable; allow the guest-home subtree (/home/tenki/*) as a valid cwd at all container-cwd guards. Known follow-up (pre-existing, backend-agnostic): the process-global terminal environment cache (_active_environments, keyed "default") is not profile-scoped, so under the multiplexing gateway forwarded credentials are not isolated across profiles. Tracked separately; documented in the credential-forwarding notes. Tests: tests/tools/test_tenki_environment.py plus terminal/file/config/scrub coverage, including profile-scope, durability, restore-classification, and cwd-subtree regression pins. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- agent/prompt_builder.py | 18 +- cli-config.yaml.example | 30 +- cli.py | 39 +- docs/security/network-egress-isolation.md | 2 +- gateway/run.py | 15 +- hermes_cli/config.py | 75 +- hermes_cli/doctor.py | 53 + hermes_cli/setup.py | 85 +- hermes_cli/status.py | 27 + hermes_cli/tips.py | 2 +- hermes_cli/web_server.py | 2 +- nix/packages.nix | 1 + pyproject.toml | 3 +- tests/agent/test_prompt_builder.py | 3 +- tests/gateway/test_config_cwd_bridge.py | 9 + tests/hermes_cli/test_config_env_expansion.py | 18 + tests/hermes_cli/test_setup.py | 32 +- tests/test_project_metadata.py | 2 +- tests/tools/test_browser_ssrf_local.py | 2 +- tests/tools/test_command_guards.py | 4 + tests/tools/test_container_cwd_sanitize.py | 107 +- tests/tools/test_docker_network_config.py | 45 +- .../tools/test_file_tools_container_config.py | 83 ++ tests/tools/test_hardline_blocklist.py | 4 +- tests/tools/test_local_env_blocklist.py | 4 + tests/tools/test_modal_sandbox_fixes.py | 7 + tests/tools/test_parse_env_var.py | 55 + tests/tools/test_skills_tool.py | 2 +- tests/tools/test_tenki_environment.py | 1304 +++++++++++++++++ tests/tools/test_terminal_config_env_sync.py | 76 + tests/tools/test_terminal_requirements.py | 41 + .../tools/test_terminal_tool_requirements.py | 29 + tools/approval.py | 4 +- tools/browser_tool.py | 2 +- tools/code_execution_tool.py | 18 +- tools/env_probe.py | 4 +- tools/environments/__init__.py | 2 +- tools/environments/base.py | 8 +- tools/environments/local.py | 4 + tools/environments/tenki.py | 1087 ++++++++++++++ tools/file_operations.py | 6 +- tools/file_tools.py | 24 +- tools/lazy_deps.py | 1 + tools/skills_tool.py | 2 +- tools/tenki_config.py | 201 +++ tools/terminal_tool.py | 195 ++- tools/tool_result_storage.py | 2 +- uv.lock | 29 +- website/docs/developer-guide/architecture.md | 6 +- website/docs/getting-started/nix-setup.md | 1 + website/docs/guides/tips.md | 4 +- website/docs/index.mdx | 2 +- .../docs/reference/environment-variables.md | 23 +- website/docs/user-guide/configuration.md | 52 +- website/docs/user-guide/features/tools.md | 4 +- website/docs/user-guide/security.md | 8 +- .../autonomous-ai-agents-hermes-agent.md | 2 +- .../software-development-plan.md | 2 +- website/scripts/generate-llms-txt.py | 2 +- 62 files changed, 3744 insertions(+), 136 deletions(-) create mode 100644 tests/tools/test_tenki_environment.py create mode 100644 tools/environments/tenki.py create mode 100644 tools/tenki_config.py diff --git a/AGENTS.md b/AGENTS.md index a3d5e5be8413f..6737b92a98d3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,7 +240,7 @@ hermes-agent/ ├── agent/ # Agent internals (provider adapters, memory, caching, compression, etc.) ├── hermes_cli/ # CLI subcommands, setup wizard, plugins loader, skin engine ├── tools/ # Tool implementations — auto-discovered via tools/registry.py -│ └── environments/ # Terminal backends (local, docker, ssh, modal, daytona, singularity) +│ └── environments/ # Terminal backends (local, docker, ssh, modal, daytona, singularity, tenki) ├── gateway/ # Messaging gateway — run.py + session.py + platforms/ │ ├── platforms/ # Adapter per platform (telegram, discord, slack, whatsapp, │ │ # homeassistant, signal, matrix, mattermost, email, sms, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46581d820037d..d2a6c43979cb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -257,7 +257,7 @@ hermes-agent/ │ ├── skill_tools.py # Skill search, load, manage │ └── environments/ # Terminal execution backends │ ├── base.py # BaseEnvironment ABC -│ ├── local.py, docker.py, ssh.py, singularity.py, modal.py, daytona.py +│ ├── local.py, docker.py, ssh.py, singularity.py, modal.py, daytona.py, tenki.py │ ├── gateway/ # Messaging gateway │ ├── run.py # GatewayRunner — platform lifecycle, message routing, cron diff --git a/README.md b/README.md index ba1322a389207..3c6dddd8f6a5d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Use any model you want — [Nous Portal](https://portal.nousresearch.com), OpenR A closed learning loopAgent-curated memory with periodic nudges. Autonomous skill creation after complex tasks. Skills self-improve during use. FTS5 session search with LLM summarization for cross-session recall. Honcho dialectic user modeling. Compatible with the agentskills.io open standard. Scheduled automationsBuilt-in cron scheduler with delivery to any platform. Daily reports, nightly backups, weekly audits — all in natural language, running unattended. Delegates and parallelizesSpawn isolated subagents for parallel workstreams. Write Python scripts that call tools via RPC, collapsing multi-step pipelines into zero-context-cost turns. -Runs anywhere, not just your laptopSix terminal backends — local, Docker, SSH, Singularity, Modal, and Daytona. Daytona and Modal offer serverless persistence — your agent's environment hibernates when idle and wakes on demand, costing nearly nothing between sessions. Run it on a $5 VPS or a GPU cluster. +Runs anywhere, not just your laptopSeven terminal backends — local, Docker, SSH, Singularity, Modal, Daytona, and Tenki. Cloud backends let your agent run isolated compute away from your host. Run it on a $5 VPS, a GPU cluster, or on-demand cloud sandboxes. Research-readyBatch trajectory generation, trajectory compression for training the next generation of tool-calling models. diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index b5b2b58c3621e..ceaa2a6a958bc 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -889,7 +889,7 @@ def format_steer_marker(steer_text: str) -> str: # runs. For these backends, host info (Windows/Linux/macOS, $HOME, cwd) is # misleading — the agent should only see the machine it can actually touch. _REMOTE_TERMINAL_BACKENDS = frozenset({ - "docker", "singularity", "modal", "daytona", "ssh", + "docker", "singularity", "modal", "daytona", "tenki", "ssh", "managed_modal", }) @@ -904,6 +904,7 @@ def format_steer_marker(steer_text: str) -> str: "modal": "a Modal sandbox (Linux)", "managed_modal": "a managed Modal sandbox (Linux)", "daytona": "a Daytona workspace (Linux)", + "tenki": "a Tenki sandbox (Linux)", "ssh": "a remote host reached over SSH (likely Linux)", } @@ -964,6 +965,8 @@ def _probe_remote_backend(env_type: str) -> str | None: image = config.get("modal_image", "") elif env_type == "daytona": image = config.get("daytona_image", "") + elif env_type == "tenki": + image = config.get("tenki_image", "") else: image = "" @@ -978,7 +981,7 @@ def _probe_remote_backend(env_type: str) -> str | None: } container_config = None - if env_type in {"docker", "singularity", "modal", "daytona"}: + if env_type in {"docker", "singularity", "modal", "daytona", "tenki"}: container_config = { "container_cpu": config.get("container_cpu", 1), "container_memory": config.get("container_memory", 5120), @@ -993,6 +996,15 @@ def _probe_remote_backend(env_type: str) -> str | None: "docker_extra_args": config.get("docker_extra_args", []), "docker_persist_across_processes": config.get("docker_persist_across_processes", True), "docker_orphan_reaper": config.get("docker_orphan_reaper", True), + "tenki_api_endpoint": config.get("tenki_api_endpoint", ""), + "tenki_workspace_id": config.get("tenki_workspace_id", ""), + "tenki_project_id": config.get("tenki_project_id", ""), + "tenki_name_prefix": config.get("tenki_name_prefix", "hermes"), + "tenki_allow_inbound": config.get("tenki_allow_inbound", False), + "tenki_allow_outbound": config.get("tenki_allow_outbound", True), + "tenki_max_duration": config.get("tenki_max_duration", 3600), + "tenki_idle_timeout": config.get("tenki_idle_timeout", 0), + "tenki_pause_retention": config.get("tenki_pause_retention", 0), } env = _create_environment( @@ -1068,7 +1080,7 @@ def build_environment_hints() -> str: and a Windows-only note that `terminal` shells out to bash, not PowerShell). - For **remote / sandbox** terminal backends (docker, singularity, - modal, daytona, ssh): host info is **suppressed** + modal, daytona, tenki, ssh): host info is **suppressed** because the agent's tools can't touch the host — only the backend matters. A live probe inside the backend reports its OS, user, $HOME, and cwd. Falls back to a static summary if the probe fails. diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 69acc68680578..fd426419a69b0 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -306,8 +306,34 @@ terminal: # daytona_image: "nikolaik/python-nodejs:python3.11-nodejs20" # container_disk: 10240 # Daytona max is 10GB per sandbox -# -# --- Container resource limits (docker, singularity, modal, daytona -- ignored for local/ssh) --- +# ----------------------------------------------------------------------------- +# OPTION 7: Tenki cloud execution +# Commands run in Tenki cloud sandboxes, created on demand +# Great for: On-demand cloud compute, isolated ephemeral sandboxes +# Requires: pip install tenki-sandbox, plus `tenki login` or the +# TENKI_AUTH_TOKEN / TENKI_API_KEY env var +# ----------------------------------------------------------------------------- +# terminal: +# backend: "tenki" +# cwd: "/home/tenki" # Path INSIDE the sandbox +# timeout: 180 +# lifetime_seconds: 300 +# container_persistent: false # Tenki default: terminate sandboxes on cleanup +# tenki_image: "" # Optional image/template; blank uses Tenki default +# tenki_api_endpoint: "https://api.tenki.cloud" +# tenki_workspace_id: "" # Blank falls back to Tenki CLI config +# tenki_project_id: "" # Blank falls back to Tenki CLI config +# tenki_name_prefix: "hermes" +# tenki_allow_inbound: false +# tenki_allow_outbound: true +# tenki_max_duration: 3600 # Max sandbox lifetime in seconds +# tenki_idle_timeout: 0 # Auto-pause after idle seconds (0 = disabled) +# tenki_pause_retention: 0 # Retention for paused sandboxes (0 = disabled) +# tenki_sync_hermes_home: false # Opt-in sync of selected ~/.hermes files +# tenki_forward_env: [] # Env vars to forward (e.g. GITHUB_TOKEN) + +# +# --- Container resource limits (docker, singularity, modal, daytona, tenki -- ignored for local/ssh) --- # These settings apply to all container backends. They control the resources # allocated to the sandbox and whether its filesystem persists across sessions. container_cpu: 1 # CPU cores diff --git a/cli.py b/cli.py index 9887bb0297807..fe5c0910a7414 100644 --- a/cli.py +++ b/cli.py @@ -405,6 +405,21 @@ def load_cli_config() -> Dict[str, Any]: "singularity_image": "docker://nikolaik/python-nodejs:python3.11-nodejs20", "modal_image": "nikolaik/python-nodejs:python3.11-nodejs20", "daytona_image": "nikolaik/python-nodejs:python3.11-nodejs20", + "tenki_image": "", + # Blank so the TENKI_API_ENDPOINT / TENKI_API_URL and Tenki CLI + # fallbacks in resolve_tenki_api_endpoint() stay reachable; a + # non-blank default here is bridged as explicit and would mask them. + "tenki_api_endpoint": "", + "tenki_workspace_id": "", + "tenki_project_id": "", + "tenki_name_prefix": "hermes", + "tenki_allow_inbound": False, + "tenki_allow_outbound": True, + "tenki_max_duration": 3600, + "tenki_idle_timeout": 0, + "tenki_pause_retention": 0, + "tenki_sync_hermes_home": False, + "tenki_forward_env": [], "docker_volumes": [], # host:container volume mounts for Docker backend "docker_mount_cwd_to_workspace": False, # explicit opt-in only; default off for sandbox isolation }, @@ -509,6 +524,7 @@ def load_cli_config() -> Dict[str, Any]: # overwrite env vars that were already set by .env -- only a user's config # file should be authoritative. _file_has_terminal_config = False + file_config: dict[str, Any] = {} # Load from file if exists if config_path.exists(): @@ -568,7 +584,7 @@ def load_cli_config() -> Dict[str, Any]: logger.warning("Failed to load cli-config.yaml: %s", e) # Expand ${ENV_VAR} references in config values before bridging to env vars. - from hermes_cli.config import _expand_env_vars + from hermes_cli.config import _deep_merge, _expand_env_vars, _normalize_terminal_backend_defaults defaults = _expand_env_vars(defaults) # Managed scope: overlay administrator-pinned values LAST so they win over @@ -581,7 +597,11 @@ def load_cli_config() -> Dict[str, Any]: # normalization, leaf-merge) and is fail-open. from hermes_cli import managed_scope + managed_config = managed_scope.load_managed_config() defaults = managed_scope.apply_managed_overlay(defaults) + raw_terminal_defaults_source = file_config + if managed_config: + raw_terminal_defaults_source = _deep_merge(raw_terminal_defaults_source, managed_config) # Apply terminal config to environment variables (so terminal_tool picks them up) terminal_config = defaults.get("terminal", {}) @@ -591,6 +611,9 @@ def load_cli_config() -> Dict[str, Any]: # Accept both, with "backend" taking precedence (it's the documented key). if "backend" in terminal_config: terminal_config["env_type"] = terminal_config["backend"] + + defaults = _normalize_terminal_backend_defaults(defaults, raw_terminal_defaults_source) + terminal_config = defaults.get("terminal", {}) # CWD resolution for CLI/TUI. The gateway has its own config bridge in # gateway/run.py but may lazily import cli.py (triggering this code). @@ -617,12 +640,24 @@ def load_cli_config() -> Dict[str, Any]: "singularity_image": "TERMINAL_SINGULARITY_IMAGE", "modal_image": "TERMINAL_MODAL_IMAGE", "daytona_image": "TERMINAL_DAYTONA_IMAGE", + "tenki_image": "TERMINAL_TENKI_IMAGE", + "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", + "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", + "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", + "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", + "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", + "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", + "tenki_max_duration": "TERMINAL_TENKI_MAX_DURATION", + "tenki_idle_timeout": "TERMINAL_TENKI_IDLE_TIMEOUT", + "tenki_pause_retention": "TERMINAL_TENKI_PAUSE_RETENTION", + "tenki_sync_hermes_home": "TERMINAL_TENKI_SYNC_HERMES_HOME", + "tenki_forward_env": "TERMINAL_TENKI_FORWARD_ENV", # SSH config "ssh_host": "TERMINAL_SSH_HOST", "ssh_user": "TERMINAL_SSH_USER", "ssh_port": "TERMINAL_SSH_PORT", "ssh_key": "TERMINAL_SSH_KEY", - # Container resource config (docker, singularity, modal, daytona -- ignored for local/ssh) + # Container resource config (docker, singularity, modal, daytona, tenki -- ignored for local/ssh) "container_cpu": "TERMINAL_CONTAINER_CPU", "container_memory": "TERMINAL_CONTAINER_MEMORY", "container_disk": "TERMINAL_CONTAINER_DISK", diff --git a/docs/security/network-egress-isolation.md b/docs/security/network-egress-isolation.md index 46cde2fd7471a..0c356d32de2ec 100644 --- a/docs/security/network-egress-isolation.md +++ b/docs/security/network-egress-isolation.md @@ -182,7 +182,7 @@ docker compose exec gateway \ *container's* network. If you use the default local terminal backend, tool commands execute inside the same container. For stronger isolation, combine network segmentation with a sandboxed terminal backend (Docker, Modal, - Daytona). + Daytona, Tenki). - **Platform adapters need egress:** The gateway service needs outbound access to reach messaging platform APIs. If you add new platform adapters, add their diff --git a/gateway/run.py b/gateway/run.py index d5b3fbff4743e..c0779e0740795 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1484,7 +1484,7 @@ def _profile_runtime_scope(profile_home: "Path"): with open(_config_path, encoding="utf-8") as _f: _cfg = _yaml.safe_load(_f) or {} # Expand ${ENV_VAR} references before bridging to env vars. - from hermes_cli.config import _expand_env_vars + from hermes_cli.config import _expand_env_vars, _normalize_terminal_backend_defaults _cfg = _expand_env_vars(_cfg) # Managed scope: overlay administrator-pinned values BEFORE bridging to # env vars, so a managed timezone / redact_secrets / max_turns / terminal @@ -1497,6 +1497,7 @@ def _profile_runtime_scope(profile_home: "Path"): _cfg = managed_scope.apply_managed_overlay(_cfg) except Exception: pass + _cfg = _normalize_terminal_backend_defaults(_cfg, _cfg) # Top-level simple values (fallback only — don't override .env) for _key, _val in _cfg.items(): if isinstance(_val, (str, int, float, bool)) and _key not in os.environ: @@ -1519,6 +1520,18 @@ def _profile_runtime_scope(profile_home: "Path"): "singularity_image": "TERMINAL_SINGULARITY_IMAGE", "modal_image": "TERMINAL_MODAL_IMAGE", "daytona_image": "TERMINAL_DAYTONA_IMAGE", + "tenki_image": "TERMINAL_TENKI_IMAGE", + "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", + "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", + "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", + "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", + "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", + "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", + "tenki_max_duration": "TERMINAL_TENKI_MAX_DURATION", + "tenki_idle_timeout": "TERMINAL_TENKI_IDLE_TIMEOUT", + "tenki_pause_retention": "TERMINAL_TENKI_PAUSE_RETENTION", + "tenki_sync_hermes_home": "TERMINAL_TENKI_SYNC_HERMES_HOME", + "tenki_forward_env": "TERMINAL_TENKI_FORWARD_ENV", "ssh_host": "TERMINAL_SSH_HOST", "ssh_user": "TERMINAL_SSH_USER", "ssh_port": "TERMINAL_SSH_PORT", diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 30a8ae4643c37..b747196bffb1d 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1219,7 +1219,23 @@ def _ensure_hermes_home_managed(home: Path): "singularity_image": "docker://nikolaik/python-nodejs:python3.11-nodejs20", "modal_image": "nikolaik/python-nodejs:python3.11-nodejs20", "daytona_image": "nikolaik/python-nodejs:python3.11-nodejs20", - # Container resource limits (docker, singularity, modal, daytona — ignored for local/ssh) + "tenki_image": "", + # Blank so the documented TENKI_API_ENDPOINT / TENKI_API_URL and Tenki + # CLI-config fallbacks in resolve_tenki_api_endpoint() are reachable. A + # non-blank default here is bridged as an explicit value and would mask + # them. Blank resolves to https://api.tenki.cloud downstream. + "tenki_api_endpoint": "", + "tenki_workspace_id": "", + "tenki_project_id": "", + "tenki_name_prefix": "hermes", + "tenki_allow_inbound": False, + "tenki_allow_outbound": True, + "tenki_max_duration": 3600, + "tenki_idle_timeout": 0, + "tenki_pause_retention": 0, + "tenki_sync_hermes_home": False, + "tenki_forward_env": [], + # Container resource limits (docker, singularity, modal, daytona, tenki — ignored for local/ssh) "container_cpu": 1, "container_memory": 5120, # MB (default 5GB) "container_disk": 51200, # MB (default 50GB) @@ -6608,6 +6624,31 @@ def _normalize_max_turns_config(config: Dict[str, Any]) -> Dict[str, Any]: return config +def _normalize_terminal_backend_defaults( + config: Dict[str, Any], + raw_config: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Apply backend-specific terminal defaults without writing them to config.yaml.""" + terminal = config.get("terminal") + if not isinstance(terminal, dict): + return config + + backend = str(terminal.get("backend") or terminal.get("env_type") or "").lower() + if backend != "tenki": + return config + + raw_terminal = raw_config.get("terminal") if isinstance(raw_config, dict) else {} + explicit_persistence = isinstance(raw_terminal, dict) and "container_persistent" in raw_terminal + if explicit_persistence: + return config + + config = dict(config) + terminal = dict(terminal) + terminal["container_persistent"] = False + config["terminal"] = terminal + return config + + def cfg_get(cfg: Optional[Dict[str, Any]], *keys: str, default: Any = None) -> Any: """Traverse nested dict keys safely, returning ``default`` on any miss. @@ -6822,6 +6863,18 @@ def write_platform_config_field( "singularity_image": "TERMINAL_SINGULARITY_IMAGE", "modal_image": "TERMINAL_MODAL_IMAGE", "daytona_image": "TERMINAL_DAYTONA_IMAGE", + "tenki_image": "TERMINAL_TENKI_IMAGE", + "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", + "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", + "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", + "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", + "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", + "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", + "tenki_max_duration": "TERMINAL_TENKI_MAX_DURATION", + "tenki_idle_timeout": "TERMINAL_TENKI_IDLE_TIMEOUT", + "tenki_pause_retention": "TERMINAL_TENKI_PAUSE_RETENTION", + "tenki_sync_hermes_home": "TERMINAL_TENKI_SYNC_HERMES_HOME", + "tenki_forward_env": "TERMINAL_TENKI_FORWARD_ENV", "ssh_host": "TERMINAL_SSH_HOST", "ssh_user": "TERMINAL_SSH_USER", "ssh_port": "TERMINAL_SSH_PORT", @@ -6877,10 +6930,20 @@ def apply_terminal_config_to_env( target = os.environ if env is None else env raw_config = read_raw_config() + raw_terminal_defaults_source: Dict[str, Any] = raw_config + try: + from hermes_cli import managed_scope + + managed_config = managed_scope.load_managed_config() + if managed_config: + raw_terminal_defaults_source = _deep_merge(raw_terminal_defaults_source, managed_config) + except Exception: + pass file_has_terminal_config = isinstance(raw_config.get("terminal"), dict) should_override = file_has_terminal_config if override is None else override cfg = config if config is not None else load_config_readonly() + cfg = _normalize_terminal_backend_defaults(cfg, raw_terminal_defaults_source) terminal_cfg = cfg.get("terminal", {}) if isinstance(cfg, dict) else {} if not isinstance(terminal_cfg, dict): return target @@ -6951,6 +7014,7 @@ def _load_config_impl(*, want_deepcopy: bool) -> Dict[str, Any]: return copy.deepcopy(cached[4]) if want_deepcopy else cached[4] config = copy.deepcopy(DEFAULT_CONFIG) + user_config: Dict[str, Any] = {} if user_sig is not None: try: @@ -7014,9 +7078,12 @@ def _load_config_impl(*, want_deepcopy: bool) -> Dict[str, Any]: # This deliberately inverts the usual env-over-config precedence for the # keys the managed layer pins — see docs/design/managed-scope.md §4.1. managed_config = managed_scope.load_managed_config() + raw_terminal_defaults_source: Dict[str, Any] = user_config if managed_config: managed_expanded = _expand_env_vars(managed_config) expanded = _deep_merge(expanded, managed_expanded) + raw_terminal_defaults_source = _deep_merge(raw_terminal_defaults_source, managed_config) + expanded = _normalize_terminal_backend_defaults(expanded, raw_terminal_defaults_source) _LAST_EXPANDED_CONFIG_BY_PATH[path_key] = copy.deepcopy(expanded) if cache_sig is not None: # Cache stores a separate deepcopy so subsequent ``load_config()`` @@ -7967,6 +8034,12 @@ def show_config(): print(f" Daytona image: {terminal.get('daytona_image', 'nikolaik/python-nodejs:python3.11-nodejs20')}") daytona_key = get_env_value('DAYTONA_API_KEY') print(f" API key: {'configured' if daytona_key else '(not set)'}") + elif terminal.get('backend') == 'tenki': + print(f" Tenki image: {terminal.get('tenki_image') or '(Tenki default)'}") + print(f" Endpoint: {terminal.get('tenki_api_endpoint') or 'https://api.tenki.cloud'}") + print(f" Workspace: {terminal.get('tenki_workspace_id') or '(from Tenki CLI)'}") + print(f" Project: {terminal.get('tenki_project_id') or '(from Tenki CLI)'}") + print(f" Sync .hermes: {'enabled' if terminal.get('tenki_sync_hermes_home') else 'disabled'}") elif terminal.get('backend') == 'ssh': ssh_host = get_env_value('TERMINAL_SSH_HOST') ssh_user = get_env_value('TERMINAL_SSH_USER') diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 4122e7af295d6..2f7cacc2cb709 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -8,6 +8,7 @@ import sys import subprocess import shutil +import importlib.util from pathlib import Path from hermes_cli.config import get_project_root, get_hermes_home, get_env_path @@ -1549,6 +1550,58 @@ def run_doctor(args): issues, ) + # Tenki (if using tenki backend) + if terminal_env == "tenki": + try: + from hermes_cli.config import load_config_readonly + from tools.tenki_config import ( + has_tenki_auth, + resolve_tenki_project_id, + resolve_tenki_workspace_id, + ) + except Exception: + load_config_readonly = lambda: {} # noqa: E731 + has_tenki_auth = lambda: False # noqa: E731 + resolve_tenki_project_id = lambda _explicit="": "" # noqa: E731 + resolve_tenki_workspace_id = lambda _explicit="": "" # noqa: E731 + terminal_cfg = load_config_readonly().get("terminal", {}) + if not isinstance(terminal_cfg, dict): + terminal_cfg = {} + + if has_tenki_auth(): + check_ok("Tenki auth", "(configured)") + else: + _fail_and_issue( + "Tenki auth not found", + "(required for TERMINAL_ENV=tenki)", + "Run tenki login or set TENKI_AUTH_TOKEN/TENKI_API_KEY", + issues, + ) + + workspace_id = resolve_tenki_workspace_id( + os.getenv("TERMINAL_TENKI_WORKSPACE_ID") or terminal_cfg.get("tenki_workspace_id", "") + ) + project_id = resolve_tenki_project_id( + os.getenv("TERMINAL_TENKI_PROJECT_ID") or terminal_cfg.get("tenki_project_id", "") + ) + if workspace_id and project_id: + check_ok("Tenki workspace/project", "(configured)") + else: + check_warn( + "Tenki workspace/project not configured", + "(optional for sessions; required for volume-backed workflows)", + ) + + if importlib.util.find_spec("tenki_sandbox") is not None: + check_ok("tenki-sandbox SDK", "(installed)") + else: + _fail_and_issue( + "tenki-sandbox SDK not installed", + "(pip install tenki-sandbox==0.1.1)", + "Install Tenki SDK: pip install tenki-sandbox==0.1.1", + issues, + ) + # Node.js + agent-browser (for browser automation tools) if _safe_which("node"): check_ok("Node.js") diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 54f4e5f676d5d..fd2177af9b8d6 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -673,7 +673,7 @@ def _print_setup_summary(config: dict, hermes_home): def _prompt_container_resources(config: dict): - """Prompt for container resource settings (Docker, Singularity, Modal, Daytona).""" + """Prompt for container resource settings (Docker, Singularity, Modal, Daytona, Tenki).""" terminal = config.setdefault("terminal", {}) print() @@ -1196,11 +1196,12 @@ def setup_terminal_backend(config: dict): "Modal - serverless cloud sandbox", "SSH - run on a remote machine", "Daytona - persistent cloud development environment", + "Tenki Agent - Tenki cloud sandbox", ] - idx_to_backend = {0: "local", 1: "docker", 2: "modal", 3: "ssh", 4: "daytona"} - backend_to_idx = {"local": 0, "docker": 1, "modal": 2, "ssh": 3, "daytona": 4} + idx_to_backend = {0: "local", 1: "docker", 2: "modal", 3: "ssh", 4: "daytona", 5: "tenki"} + backend_to_idx = {"local": 0, "docker": 1, "modal": 2, "ssh": 3, "daytona": 4, "tenki": 5} - next_idx = 5 + next_idx = 6 if is_linux: terminal_choices.append("Singularity/Apptainer - HPC-friendly container") idx_to_backend[next_idx] = "singularity" @@ -1386,6 +1387,82 @@ def setup_terminal_backend(config: dict): "daytona_image", "nikolaik/python-nodejs:python3.11-nodejs20" ) + elif selected_backend == "tenki": + print_success("Terminal backend: Tenki Agent") + print_info("Cloud sandboxes are created on demand and terminated by default.") + print_info("Requires Tenki CLI login or TENKI_AUTH_TOKEN/TENKI_API_KEY.") + + try: + __import__("tenki_sandbox") + except ImportError: + print_info("Installing Tenki SDK...") + import subprocess + + uv_bin = shutil.which("uv") + package = "tenki-sandbox==0.1.1" + if uv_bin: + result = subprocess.run( + [uv_bin, "pip", "install", "--python", sys.executable, package], + capture_output=True, + text=True, + ) + else: + result = subprocess.run( + [sys.executable, "-m", "pip", "install", package], + capture_output=True, + text=True, + ) + if result.returncode == 0: + print_success("Tenki SDK installed") + else: + print_warning("Install failed — run manually: pip install tenki-sandbox==0.1.1") + if result.stderr: + print_info(f" Error: {result.stderr.strip().splitlines()[-1]}") + + from tools.tenki_config import ( + has_tenki_auth, + resolve_tenki_api_endpoint, + resolve_tenki_project_id, + resolve_tenki_workspace_id, + ) + + terminal = config.setdefault("terminal", {}) + endpoint = resolve_tenki_api_endpoint(terminal.get("tenki_api_endpoint", "")) + workspace_id = resolve_tenki_workspace_id(terminal.get("tenki_workspace_id", "")) + project_id = resolve_tenki_project_id(terminal.get("tenki_project_id", "")) + + terminal["tenki_api_endpoint"] = endpoint + if workspace_id: + terminal["tenki_workspace_id"] = workspace_id + if project_id: + terminal["tenki_project_id"] = project_id + terminal.setdefault("tenki_image", "") + terminal.setdefault("tenki_name_prefix", "hermes") + terminal.setdefault("tenki_allow_inbound", False) + terminal.setdefault("tenki_allow_outbound", True) + terminal.setdefault("tenki_max_duration", 3600) + terminal.setdefault("tenki_idle_timeout", 0) + terminal.setdefault("tenki_pause_retention", 0) + terminal.setdefault("tenki_sync_hermes_home", False) + if current_backend == "tenki": + terminal.setdefault("container_persistent", False) + terminal.setdefault("cwd", "/home/tenki") + else: + terminal["container_persistent"] = False + terminal["cwd"] = "/home/tenki" + + print_info(f" Endpoint: {endpoint}") + print_info(f" Workspace: {workspace_id or '(not found; run tenki login)'}") + print_info(f" Project: {project_id or '(not found; run tenki login)'}") + if has_tenki_auth(): + print_info(" Tenki auth: already configured") + else: + print_warning(" Tenki auth not found") + token = prompt(" Tenki token/API key (optional; leave blank to run tenki login)", password=True) + if token: + save_env_value("TENKI_API_KEY", token) + print_success(" Configured") + elif selected_backend == "ssh": print_success("Terminal backend: SSH") print_info("Run commands on a remote machine via SSH.") diff --git a/hermes_cli/status.py b/hermes_cli/status.py index 088460fb63fe6..43c7a18fcaf7a 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -425,6 +425,33 @@ def _resolve_env(env_ref) -> str: elif terminal_env == "daytona": daytona_image = os.getenv("TERMINAL_DAYTONA_IMAGE", "nikolaik/python-nodejs:python3.11-nodejs20") print(f" Daytona Image: {daytona_image}") + elif terminal_env == "tenki": + from tools.tenki_config import ( + resolve_tenki_api_endpoint, + resolve_tenki_project_id, + resolve_tenki_workspace_id, + ) + + tenki_image = os.getenv("TERMINAL_TENKI_IMAGE") or terminal_cfg.get("tenki_image", "") + tenki_endpoint = resolve_tenki_api_endpoint( + os.getenv("TERMINAL_TENKI_API_ENDPOINT") or terminal_cfg.get("tenki_api_endpoint", "") + ) + tenki_workspace = resolve_tenki_workspace_id( + os.getenv("TERMINAL_TENKI_WORKSPACE_ID") or terminal_cfg.get("tenki_workspace_id", "") + ) + tenki_project = resolve_tenki_project_id( + os.getenv("TERMINAL_TENKI_PROJECT_ID") or terminal_cfg.get("tenki_project_id", "") + ) + tenki_sync_value = os.getenv("TERMINAL_TENKI_SYNC_HERMES_HOME") + if tenki_sync_value is None: + tenki_sync = bool(terminal_cfg.get("tenki_sync_hermes_home", False)) + else: + tenki_sync = tenki_sync_value.lower() in {"true", "1", "yes"} + print(f" Tenki Image: {tenki_image or '(Tenki default)'}") + print(f" Endpoint: {tenki_endpoint}") + print(f" Workspace: {tenki_workspace or '(not found)'}") + print(f" Project: {tenki_project or '(not found)'}") + print(f" Sync .hermes: {check_mark(tenki_sync)} {'enabled' if tenki_sync else 'disabled'}") sudo_password = os.getenv("SUDO_PASSWORD", "") print(f" Sudo: {check_mark(bool(sudo_password))} {'enabled' if sudo_password else 'disabled'}") diff --git a/hermes_cli/tips.py b/hermes_cli/tips.py index 6f365771a1b39..9b8a8466dd4a6 100644 --- a/hermes_cli/tips.py +++ b/hermes_cli/tips.py @@ -147,7 +147,7 @@ "/moa routes one hard prompt through your configured Mixture of Agents model set.", "Terminal commands support background mode with notify_on_complete for long-running tasks.", "Terminal background processes support watch_patterns to alert on specific output lines.", - "The terminal tool supports 6 backends: local, Docker, SSH, Modal, Daytona, and Singularity.", + "The terminal tool supports 7 backends: local, Docker, SSH, Modal, Daytona, Tenki, and Singularity.", # --- Profiles --- "Each profile gets its own config, API keys, memory, sessions, skills, and cron jobs.", diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 7365d241c09ff..55692d36b2c9f 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -624,7 +624,7 @@ async def _token_auth_seam(request: Request, call_next): "terminal.backend": { "type": "select", "description": "Terminal execution backend", - "options": ["local", "docker", "ssh", "modal", "daytona", "singularity"], + "options": ["local", "docker", "ssh", "modal", "daytona", "tenki", "singularity"], }, "terminal.modal_mode": { "type": "select", diff --git a/nix/packages.nix b/nix/packages.nix index d11a21d2cb712..95ab693f89578 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -35,6 +35,7 @@ "messaging" "modal" "parallel-web" + "tenki" "tts-premium" "voice" ] diff --git a/pyproject.toml b/pyproject.toml index 851473b13b3fd..58fb521425d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,7 @@ fal = ["fal-client==0.13.1"] edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] +tenki = ["tenki-sandbox==0.1.1"] hindsight = ["hindsight-client==0.6.1"] dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82) messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.27.0", "slack-sdk==3.40.1", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265 @@ -282,7 +283,7 @@ all = [ # # Removed from [all] on 2026-05-12 (covered by lazy-install): # anthropic, exa, firecrawl, parallel-web, fal, edge-tts, - # modal, daytona, messaging (telegram/discord/slack), + # modal, daytona, tenki, messaging (telegram/discord/slack), # matrix, slack, honcho, voice (faster-whisper), # dingtalk, feishu, bedrock, tts-premium (elevenlabs) # diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index 858c880ec8fb6..d57165dee067f 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -1299,7 +1299,7 @@ def _fake_create_environment(*, env_type, **kwargs): def test_remote_backend_list_covers_known_sandboxes(self): """Regression guard: if someone adds a remote backend, they must list it here.""" import agent.prompt_builder as _pb - for backend in ("docker", "singularity", "modal", "daytona", "ssh"): + for backend in ("docker", "singularity", "modal", "daytona", "tenki", "ssh"): assert backend in _pb._REMOTE_TERMINAL_BACKENDS, ( f"{backend!r} must be in _REMOTE_TERMINAL_BACKENDS so its host " f"info is suppressed in the system prompt" @@ -1645,4 +1645,3 @@ def test_not_duplicated_in_google_guidance(self): # Budget warning history stripping # ========================================================================= - diff --git a/tests/gateway/test_config_cwd_bridge.py b/tests/gateway/test_config_cwd_bridge.py index ca449b94b62b9..e865dced77fbf 100644 --- a/tests/gateway/test_config_cwd_bridge.py +++ b/tests/gateway/test_config_cwd_bridge.py @@ -22,6 +22,9 @@ def _simulate_config_bridge(cfg: dict, initial_env: dict | None = None): Returns the resulting env dict (only TERMINAL_* and MESSAGING_CWD keys). """ env = dict(initial_env or {}) + from hermes_cli.config import _normalize_terminal_backend_defaults + + cfg = _normalize_terminal_backend_defaults(cfg, cfg) # --- Replicate lines 54-56: generic top-level bridge (for context) --- for key, val in cfg.items(): @@ -111,6 +114,12 @@ def test_top_level_backend_sets_terminal_env(self): result = _simulate_config_bridge(cfg) assert result["TERMINAL_ENV"] == "docker" + def test_tenki_backend_defaults_to_terminate_only_over_stale_env(self): + cfg = {"terminal": {"backend": "tenki"}} + result = _simulate_config_bridge(cfg, {"TERMINAL_CONTAINER_PERSISTENT": "true"}) + assert result["TERMINAL_ENV"] == "tenki" + assert result["TERMINAL_CONTAINER_PERSISTENT"] == "False" + def test_top_level_cwd_and_backend(self): cfg = {"backend": "local", "cwd": "/home/hermes/projects"} result = _simulate_config_bridge(cfg) diff --git a/tests/hermes_cli/test_config_env_expansion.py b/tests/hermes_cli/test_config_env_expansion.py index 75ef62592d1a1..4738c180dc272 100644 --- a/tests/hermes_cli/test_config_env_expansion.py +++ b/tests/hermes_cli/test_config_env_expansion.py @@ -1,5 +1,7 @@ """Tests for ${ENV_VAR} substitution in config.yaml values.""" +import os + import pytest from hermes_cli.config import _expand_env_vars, load_config @@ -196,3 +198,19 @@ def test_cli_config_unresolved_kept_verbatim(self, tmp_path, monkeypatch): config = load_cli_config() assert config["auxiliary"]["vision"]["api_key"] == "${UNSET_CLI_VAR_ABC}" + + def test_cli_tenki_backend_overrides_stale_persistent_env(self, tmp_path, monkeypatch): + config_yaml = "terminal:\n backend: tenki\n" + config_file = tmp_path / "config.yaml" + config_file.write_text(config_yaml) + + monkeypatch.setenv("TERMINAL_CONTAINER_PERSISTENT", "true") + monkeypatch.setattr("cli._hermes_home", tmp_path) + + from cli import load_cli_config + config = load_cli_config() + + assert config["terminal"]["container_persistent"] is False + assert config["terminal"]["env_type"] == "tenki" + assert config["terminal"]["backend"] == "tenki" + assert os.environ["TERMINAL_CONTAINER_PERSISTENT"] == "False" diff --git a/tests/hermes_cli/test_setup.py b/tests/hermes_cli/test_setup.py index 9cf2f737eb22b..a84f74d6069f6 100644 --- a/tests/hermes_cli/test_setup.py +++ b/tests/hermes_cli/test_setup.py @@ -491,6 +491,37 @@ def fake_prompt_choice(question, choices, default=0): assert config["terminal"]["modal_mode"] == "direct" +def test_tenki_setup_switch_resets_inherited_persistent_container(monkeypatch): + config = { + "terminal": { + "backend": "docker", + "container_persistent": True, + "cwd": "/workspace", + } + } + + def fake_prompt_choice(question, choices, default=0): + if question == "Select terminal backend:": + assert "Tenki Agent - Tenki cloud sandbox" in choices + return choices.index("Tenki Agent - Tenki cloud sandbox") + raise AssertionError(f"Unexpected prompt_choice call: {question}") + + monkeypatch.setitem(sys.modules, "tenki_sandbox", types.ModuleType("tenki_sandbox")) + monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") + monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) + monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") + monkeypatch.setattr("hermes_cli.setup.save_config", lambda _config: None) + monkeypatch.setattr("hermes_cli.setup.save_env_value", lambda *_args, **_kwargs: None) + + from hermes_cli.setup import setup_terminal_backend + + setup_terminal_backend(config) + + assert config["terminal"]["backend"] == "tenki" + assert config["terminal"]["container_persistent"] is False + assert config["terminal"]["cwd"] == "/home/tenki" + + # test_setup_slack_* moved to tests/gateway/test_slack_plugin_setup.py — the # _setup_slack wizard migrated to the slack plugin's interactive_setup (#41112). @@ -539,4 +570,3 @@ def _interrupt(*_a, **_k): with pytest.raises(SystemExit): setup_mod.prompt_yes_no("Install it now?", True) - diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index f2f4887d60911..7bc84858be2de 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -70,7 +70,7 @@ def test_lazy_installable_extras_excluded_from_all(): "fal", "edge-tts", "tts-premium", "voice", # faster-whisper / sounddevice / numpy - "modal", "daytona", + "modal", "daytona", "tenki", "messaging", "slack", "matrix", "dingtalk", "feishu", "honcho", "hindsight", "supermemory", "mem0", diff --git a/tests/tools/test_browser_ssrf_local.py b/tests/tools/test_browser_ssrf_local.py index 9536e09891de3..a9c24a5dec9b8 100644 --- a/tests/tools/test_browser_ssrf_local.py +++ b/tests/tools/test_browser_ssrf_local.py @@ -190,7 +190,7 @@ def test_cloud_provider_is_not_local(self, monkeypatch): assert browser_tool._is_local_backend() is False - @pytest.mark.parametrize("backend", ["docker", "modal", "daytona", "ssh", "singularity"]) + @pytest.mark.parametrize("backend", ["docker", "modal", "daytona", "tenki", "ssh", "singularity"]) def test_container_terminal_backend_is_not_local(self, monkeypatch, backend): """Terminal running in a container → NOT local (browser on host can access internal networks).""" monkeypatch.setattr(browser_tool, "_is_camofox_mode", lambda: False) diff --git a/tests/tools/test_command_guards.py b/tests/tools/test_command_guards.py index 9b8a93c30bf83..85bb2febce4ea 100644 --- a/tests/tools/test_command_guards.py +++ b/tests/tools/test_command_guards.py @@ -74,6 +74,10 @@ def test_daytona_skips_both(self): result = check_all_command_guards("rm -rf /", "daytona") assert result["approved"] is True + def test_tenki_skips_both(self): + result = check_all_command_guards("rm -rf /", "tenki") + assert result["approved"] is True + # --------------------------------------------------------------------------- # tirith allow + safe command diff --git a/tests/tools/test_container_cwd_sanitize.py b/tests/tools/test_container_cwd_sanitize.py index 00a155e8bb826..2f06b60ae2908 100644 --- a/tests/tools/test_container_cwd_sanitize.py +++ b/tests/tools/test_container_cwd_sanitize.py @@ -62,10 +62,35 @@ def test_host_prefixes_include_windows_and_posix(self): def test_container_backends_set(self): assert tt._CONTAINER_BACKENDS == frozenset( - {"docker", "singularity", "modal", "daytona"} + {"docker", "singularity", "modal", "daytona", "tenki"} ) +class TestBackendGuestSubpath: + """The Tenki guest home /home/tenki (and its subtree) is a real sandbox + path, so it must be exempt from the host-path guard even though it shares + the /home/ prefix that other backends reject.""" + + def test_tenki_guest_home_root_is_subpath(self): + assert tt._is_backend_guest_subpath("tenki", "/home/tenki") is True + + def test_tenki_guest_home_child_is_subpath(self): + assert tt._is_backend_guest_subpath("tenki", "/home/tenki/project") is True + + def test_tenki_unrelated_home_is_not_subpath(self): + # A different user's home is still a host path even on tenki. + assert tt._is_backend_guest_subpath("tenki", "/home/someoneelse") is False + + def test_docker_has_no_guest_home_exemption(self): + # Docker's default cwd is /root, so /home/... stays a rejected host path. + assert tt._is_backend_guest_subpath("docker", "/home/tenki/project") is False + + def test_tenki_guest_subpath_survives_full_guard(self): + # The combined check the call sites use: unusable-by-prefix but exempt. + assert tt._is_unusable_container_cwd("/home/tenki/project") is True + assert tt._is_backend_guest_subpath("tenki", "/home/tenki/project") is True + + class TestOverrideCwdSanitizedAtCallSite: """E2E pin: a per-task cwd OVERRIDE that is a host path must NOT reach the container builder. This is the actual reported bug — the gateway/TUI @@ -145,6 +170,62 @@ def test_valid_container_override_is_preserved(self, monkeypatch): cwd = self._run_and_capture_cwd(monkeypatch, "/workspace/task42") assert cwd == "/workspace/task42" + def _run_tenki_and_capture_cwd(self, monkeypatch, override_cwd): + """Drive terminal_tool() on the tenki backend with a cwd override and + return the cwd that reached _create_environment.""" + captured = {} + config = { + "env_type": "tenki", + "tenki_image": "", + "cwd": "/home/tenki", + "host_cwd": None, + "timeout": 180, + "lifetime_seconds": 300, + "container_cpu": 1, + "container_memory": 5120, + "container_disk": 51200, + "container_persistent": False, + "modal_mode": "auto", + } + + class _DummyEnv: + cwd = "/home/tenki" + + def execute(self, *a, **k): + return {"output": "", "exit_code": 0} + + def fake_create_environment(env_type, image, cwd, timeout, **kwargs): + captured["cwd"] = cwd + return _DummyEnv() + + monkeypatch.setattr(tt, "_get_env_config", lambda: config) + monkeypatch.setattr(tt, "_start_cleanup_thread", lambda: None) + monkeypatch.setattr(tt, "_check_all_guards", lambda *a, **k: {"approved": True}) + monkeypatch.setattr(tt, "_create_environment", fake_create_environment) + monkeypatch.setattr(tt, "_active_environments", {}) + monkeypatch.setattr(tt, "_last_activity", {}) + + task_id = "sess-tenki-cwd" + tt.register_task_env_overrides(task_id, {"cwd": override_cwd}) + try: + tt.terminal_tool(command="pwd", task_id=task_id) + finally: + tt.clear_task_env_overrides(task_id) + tt._active_environments.pop(task_id, None) + tt._active_environments.pop("default", None) + return captured.get("cwd") + + def test_tenki_guest_subpath_override_is_preserved(self, monkeypatch): + # A real Tenki guest path registered as a per-task override must NOT be + # collapsed to /home/tenki (the #9b override-path fix). + cwd = self._run_tenki_and_capture_cwd(monkeypatch, "/home/tenki/project") + assert cwd == "/home/tenki/project" + + def test_tenki_foreign_host_override_still_sanitized(self, monkeypatch): + # A genuine host path is still discarded on tenki. + cwd = self._run_tenki_and_capture_cwd(monkeypatch, "/home/someoneelse/x") + assert cwd == "/home/tenki" + class TestFileOpsCwdSanitizedAtCallSite: """E2E pin: file tools (_get_file_ops) must sanitize a host/relative cwd @@ -171,6 +252,18 @@ def _run_and_capture_cwd(self, monkeypatch, override_cwd, env_type="docker", "singularity_image": "docker://pytorch/pytorch:latest", "modal_image": "pytorch/pytorch:latest", "daytona_image": "pytorch/pytorch:latest", + "tenki_image": "", + "tenki_api_endpoint": "", + "tenki_workspace_id": "", + "tenki_project_id": "", + "tenki_name_prefix": "hermes", + "tenki_allow_inbound": False, + "tenki_allow_outbound": True, + "tenki_max_duration": 3600, + "tenki_idle_timeout": 0, + "tenki_pause_retention": 0, + "tenki_sync_hermes_home": False, + "tenki_forward_env": [], "cwd": config_cwd, "host_cwd": None, "timeout": 180, @@ -255,3 +348,15 @@ def test_host_override_sanitized_on_modal(self, monkeypatch): cwd = self._run_and_capture_cwd( monkeypatch, "/Users/me/workspace", env_type="modal") assert cwd == "/workspace" + + def test_tenki_guest_subpath_override_is_preserved(self, monkeypatch): + # File tools must honor the same Tenki guest-home exemption as the + # terminal tool: /home/tenki/project is a real sandbox path. + cwd = self._run_and_capture_cwd( + monkeypatch, "/home/tenki/project", env_type="tenki", config_cwd="/home/tenki") + assert cwd == "/home/tenki/project" + + def test_tenki_foreign_host_override_still_sanitized(self, monkeypatch): + cwd = self._run_and_capture_cwd( + monkeypatch, "/home/someoneelse/x", env_type="tenki", config_cwd="/home/tenki") + assert cwd == "/home/tenki" diff --git a/tests/tools/test_docker_network_config.py b/tests/tools/test_docker_network_config.py index 776f5ef6d7c71..ced0c8c00a809 100644 --- a/tests/tools/test_docker_network_config.py +++ b/tests/tools/test_docker_network_config.py @@ -85,10 +85,15 @@ def test_docker_network_config_is_bridged_everywhere(): def test_sibling_container_config_sites_carry_docker_network(): - """Every container_config dict that carries docker_run_as_host_user must - also carry docker_network — otherwise that code path silently falls back - to networked containers while the terminal path honors the lockdown - (the probe/exec asymmetry reported on issue #46358). + """Every container_config construction site must carry docker_network — + otherwise that code path silently falls back to networked containers + while the terminal path honors the lockdown (the probe/exec asymmetry + reported on issue #46358). + + The sibling tool modules build their container_config through the shared + _container_config_from_env_config() helper, so a site is either an inline + dict (which must carry docker_network alongside docker_run_as_host_user) + or a call to that helper (whose output is asserted below). """ import ast import inspect @@ -96,20 +101,32 @@ def test_sibling_container_config_sites_carry_docker_network(): import tools.code_execution_tool as code_execution_tool import tools.file_tools as file_tools + assert terminal_tool._container_config_from_env_config({})["docker_network"] is True + assert ( + terminal_tool._container_config_from_env_config({"docker_network": False})[ + "docker_network" + ] + is False + ) + for module in (terminal_tool, file_tools, code_execution_tool): tree = ast.parse(inspect.getsource(module)) sites = 0 for node in ast.walk(tree): - if not isinstance(node, ast.Dict): - continue - keys = {k.value for k in node.keys if isinstance(k, ast.Constant)} - if "docker_run_as_host_user" in keys: - sites += 1 - assert "docker_network" in keys, ( - f"{module.__name__} builds a container_config with " - f"docker_run_as_host_user but without docker_network " - f"(line {node.lineno})" - ) + if isinstance(node, ast.Dict): + keys = {k.value for k in node.keys if isinstance(k, ast.Constant)} + if "docker_run_as_host_user" in keys: + sites += 1 + assert "docker_network" in keys, ( + f"{module.__name__} builds a container_config with " + f"docker_run_as_host_user but without docker_network " + f"(line {node.lineno})" + ) + elif isinstance(node, ast.Call): + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "") + if name == "_container_config_from_env_config": + sites += 1 assert sites >= 1, f"expected at least one container_config site in {module.__name__}" diff --git a/tests/tools/test_file_tools_container_config.py b/tests/tools/test_file_tools_container_config.py index f8a79a37e4edb..3a010504a51b4 100644 --- a/tests/tools/test_file_tools_container_config.py +++ b/tests/tools/test_file_tools_container_config.py @@ -1,6 +1,8 @@ """Tests for docker container_config key propagation in file_tools.""" +import threading from unittest.mock import patch, MagicMock +import tools.code_execution_tool as code_execution_tool import tools.file_tools as file_tools @@ -21,6 +23,21 @@ def _make_env_config(**overrides): "docker_volumes": [], "docker_mount_cwd_to_workspace": True, "docker_forward_env": ["MY_SECRET", "API_KEY"], + "docker_env": {"A": "B"}, + "docker_extra_args": ["--shm-size=1g"], + "docker_persist_across_processes": False, + "docker_orphan_reaper": False, + "tenki_api_endpoint": "https://api.tenki.test", + "tenki_workspace_id": "ws-123", + "tenki_project_id": "prj-456", + "tenki_name_prefix": "agent", + "tenki_allow_inbound": True, + "tenki_allow_outbound": False, + "tenki_max_duration": 7200, + "tenki_idle_timeout": 600, + "tenki_pause_retention": 3600, + "tenki_sync_hermes_home": True, + "tenki_forward_env": ["GITHUB_TOKEN"], } base.update(overrides) return base @@ -73,6 +90,26 @@ def test_docker_forward_env_defaults_to_empty_list(self): cc = self._run(cfg, "t4").get("container_config", {}) assert cc.get("docker_forward_env") == [] + def test_shared_container_config_fields_are_forwarded(self): + """File tools use the same container-config builder as terminal execution.""" + cc = self._run(_make_env_config(), "t5").get("container_config", {}) + + assert cc.get("docker_env") == {"A": "B"} + assert cc.get("docker_extra_args") == ["--shm-size=1g"] + assert cc.get("docker_persist_across_processes") is False + assert cc.get("docker_orphan_reaper") is False + assert cc.get("tenki_name_prefix") == "agent" + assert cc.get("tenki_api_endpoint") == "https://api.tenki.test" + assert cc.get("tenki_workspace_id") == "ws-123" + assert cc.get("tenki_project_id") == "prj-456" + assert cc.get("tenki_allow_inbound") is True + assert cc.get("tenki_allow_outbound") is False + assert cc.get("tenki_max_duration") == 7200 + assert cc.get("tenki_idle_timeout") == 600 + assert cc.get("tenki_pause_retention") == 3600 + assert cc.get("tenki_sync_hermes_home") is True + assert cc.get("tenki_forward_env") == ["GITHUB_TOKEN"] + def test_cwd_only_raw_task_override_reaches_file_environment(self): """CWD-only task overrides collapse to default but must keep their cwd.""" captured = self._run( @@ -83,3 +120,49 @@ def test_cwd_only_raw_task_override_reaches_file_environment(self): assert captured["task_id"] == "default" assert captured["cwd"] == "/workspace/session" + + +class TestExecuteCodeContainerConfig: + def test_execute_code_uses_shared_container_config_for_tenki(self): + captured = {} + mock_env = MagicMock() + env_config = _make_env_config( + env_type="tenki", + tenki_image="tenki-image", + cwd="/home/tenki", + ) + + def fake_create_env(**kwargs): + captured.update(kwargs) + return mock_env + + with patch("tools.terminal_tool._get_env_config", return_value=env_config), \ + patch("tools.terminal_tool._task_env_overrides", {}), \ + patch("tools.terminal_tool._active_environments", {}), \ + patch("tools.terminal_tool._last_activity", {}), \ + patch("tools.terminal_tool._env_lock", threading.Lock()), \ + patch("tools.terminal_tool._creation_locks", {}), \ + patch("tools.terminal_tool._creation_locks_lock", threading.Lock()), \ + patch("tools.terminal_tool._create_environment", side_effect=fake_create_env), \ + patch("tools.terminal_tool._start_cleanup_thread"): + env, env_type = code_execution_tool._get_or_create_env("exec-tenki") + + assert env is mock_env + assert env_type == "tenki" + assert captured["env_type"] == "tenki" + assert captured["image"] == "tenki-image" + assert captured["cwd"] == "/home/tenki" + assert captured["task_id"] == "default" + cc = captured["container_config"] + assert cc["container_persistent"] is False + assert cc["tenki_api_endpoint"] == "https://api.tenki.test" + assert cc["tenki_workspace_id"] == "ws-123" + assert cc["tenki_project_id"] == "prj-456" + assert cc["tenki_name_prefix"] == "agent" + assert cc["tenki_allow_inbound"] is True + assert cc["tenki_allow_outbound"] is False + assert cc["tenki_max_duration"] == 7200 + assert cc["tenki_idle_timeout"] == 600 + assert cc["tenki_pause_retention"] == 3600 + assert cc["tenki_sync_hermes_home"] is True + assert cc["tenki_forward_env"] == ["GITHUB_TOKEN"] diff --git a/tests/tools/test_hardline_blocklist.py b/tests/tools/test_hardline_blocklist.py index 38f9d4d7a8429..2c8cee7b4ab76 100644 --- a/tests/tools/test_hardline_blocklist.py +++ b/tests/tools/test_hardline_blocklist.py @@ -513,7 +513,7 @@ def test_container_backends_still_bypass(clean_session): Hardline only protects environments with real host impact (local, ssh). """ - for env in ("docker", "singularity", "modal", "daytona"): + for env in ("docker", "singularity", "modal", "daytona", "tenki"): r1 = check_dangerous_command("rm -rf /", env) assert r1["approved"] is True, f"container {env} should still bypass" r2 = check_all_command_guards("rm -rf /", env) @@ -644,7 +644,7 @@ def test_sudo_stdin_guard_not_blocked_by_yolo(clean_session, monkeypatch): def test_sudo_stdin_guard_container_bypass(clean_session): """Containerized backends still bypass — they can't touch the host.""" - for env in ("docker", "singularity", "modal", "daytona"): + for env in ("docker", "singularity", "modal", "daytona", "tenki"): for cmd in _SUDO_STDIN_BLOCK: result = check_all_command_guards(cmd, env) assert result["approved"] is True, f"container {env} should bypass sudo guard on {cmd!r}" diff --git a/tests/tools/test_local_env_blocklist.py b/tests/tools/test_local_env_blocklist.py index 2e8332470ae86..890bb04a2845c 100644 --- a/tests/tools/test_local_env_blocklist.py +++ b/tests/tools/test_local_env_blocklist.py @@ -210,6 +210,8 @@ def test_tool_and_gateway_vars_are_stripped(self): "MODAL_TOKEN_ID": "modal-id", "MODAL_TOKEN_SECRET": "modal-secret", "DAYTONA_API_KEY": "daytona-key", + "TENKI_AUTH_TOKEN": "tenki-token", + "TENKI_API_KEY": "tenki-key", } result_env = _run_with_env(extra_os_env=leaked_vars) @@ -459,6 +461,8 @@ def test_gateway_runtime_vars_are_in_blocklist(self): "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "DAYTONA_API_KEY", + "TENKI_AUTH_TOKEN", + "TENKI_API_KEY", } assert extras.issubset(_HERMES_PROVIDER_ENV_BLOCKLIST) diff --git a/tests/tools/test_modal_sandbox_fixes.py b/tests/tools/test_modal_sandbox_fixes.py index dddfe134edb65..d8ac891b2b2a3 100644 --- a/tests/tools/test_modal_sandbox_fixes.py +++ b/tests/tools/test_modal_sandbox_fixes.py @@ -112,6 +112,12 @@ def test_default_cwd_is_root_for_container_backends(self, backend, monkeypatch): f"Backend {backend}: expected /root default, got {config['cwd']}" ) + def test_default_cwd_is_tenki_home_for_tenki(self, monkeypatch): + monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.delenv("TERMINAL_CWD", raising=False) + config = _tt_mod._get_env_config() + assert config["cwd"] == "/home/tenki" + def test_docker_default_cwd_maps_current_directory_when_enabled(self, monkeypatch): """Docker should use /workspace when cwd mounting is explicitly enabled.""" monkeypatch.setattr("tools.terminal_tool.os.getcwd", lambda: "/home/user/project") @@ -345,6 +351,7 @@ def test_should_skip_container_guards(self): assert A._should_skip_container_guards("modal", has_host_access=True) is True assert A._should_skip_container_guards("singularity") is True assert A._should_skip_container_guards("daytona") is True + assert A._should_skip_container_guards("tenki") is True assert A._should_skip_container_guards("local") is False def test_isolated_docker_keeps_fast_path(self, monkeypatch): diff --git a/tests/tools/test_parse_env_var.py b/tests/tools/test_parse_env_var.py index 8cbbce6985827..e495217a3efbb 100644 --- a/tests/tools/test_parse_env_var.py +++ b/tests/tools/test_parse_env_var.py @@ -39,6 +39,45 @@ def test_get_env_config_parses_docker_forward_env_json(self): config = _tt_mod._get_env_config() assert config["docker_forward_env"] == ["GITHUB_TOKEN", "NPM_TOKEN"] + def test_get_env_config_parses_tenki_forward_env_json(self): + with patch.dict("os.environ", { + "TERMINAL_ENV": "tenki", + "TERMINAL_TENKI_FORWARD_ENV": '["GITHUB_TOKEN", "GH_TOKEN"]', + }, clear=False): + config = _tt_mod._get_env_config() + assert config["tenki_forward_env"] == ["GITHUB_TOKEN", "GH_TOKEN"] + + def test_get_env_config_parses_tenki_numeric_settings(self): + with patch.dict("os.environ", { + "TERMINAL_ENV": "tenki", + "TERMINAL_TENKI_MAX_DURATION": "7200", + "TERMINAL_TENKI_IDLE_TIMEOUT": "600", + "TERMINAL_TENKI_PAUSE_RETENTION": "300", + }, clear=False): + config = _tt_mod._get_env_config() + assert config["tenki_max_duration"] == 7200 + assert config["tenki_idle_timeout"] == 600 + assert config["tenki_pause_retention"] == 300 + + def test_stale_invalid_tenki_value_does_not_break_local_backend(self): + # A stale/invalid tenki value bridged from config.yaml must not abort + # _get_env_config() for a local session that never uses tenki. + with patch.dict("os.environ", { + "TERMINAL_ENV": "local", + "TERMINAL_TENKI_MAX_DURATION": "not-a-number", + }, clear=False): + config = _tt_mod._get_env_config() + assert config["env_type"] == "local" + assert config["tenki_max_duration"] == 3600 + + def test_invalid_tenki_value_raises_when_tenki_backend_active(self): + with patch.dict("os.environ", { + "TERMINAL_ENV": "tenki", + "TERMINAL_TENKI_MAX_DURATION": "not-a-number", + }, clear=False): + with pytest.raises(ValueError, match="TERMINAL_TENKI_MAX_DURATION"): + _tt_mod._get_env_config() + def test_create_environment_passes_docker_forward_env(self): fake_env = object() with patch.object(_tt_mod, "_DockerEnvironment", return_value=fake_env) as mock_docker: @@ -53,6 +92,22 @@ def test_create_environment_passes_docker_forward_env(self): assert result is fake_env assert mock_docker.call_args.kwargs["forward_env"] == ["GITHUB_TOKEN"] + def test_create_environment_passes_tenki_forward_env(self): + import tools.environments.tenki as tenki_module + + fake_env = object() + with patch.object(tenki_module, "TenkiEnvironment", return_value=fake_env) as mock_tenki: + result = _tt_mod._create_environment( + "tenki", + image="", + cwd="/home/tenki", + timeout=180, + container_config={"tenki_forward_env": ["GITHUB_TOKEN", "GH_TOKEN"]}, + ) + + assert result is fake_env + assert mock_tenki.call_args.kwargs["forward_env"] == ["GITHUB_TOKEN", "GH_TOKEN"] + def test_falls_back_to_default(self): with patch.dict("os.environ", {}, clear=False): # Remove the var if it exists, rely on default diff --git a/tests/tools/test_skills_tool.py b/tests/tools/test_skills_tool.py index a7445207c7118..731f89e280100 100644 --- a/tests/tools/test_skills_tool.py +++ b/tests/tools/test_skills_tool.py @@ -978,7 +978,7 @@ def test_local_env_missing_keeps_setup_needed(self, tmp_path, monkeypatch): @pytest.mark.parametrize( "backend", - ["ssh", "daytona", "docker", "singularity", "modal"], + ["ssh", "daytona", "tenki", "docker", "singularity", "modal"], ) def test_remote_backend_becomes_available_after_local_secret_capture( self, tmp_path, monkeypatch, backend diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py new file mode 100644 index 0000000000000..23a06e24b5db8 --- /dev/null +++ b/tests/tools/test_tenki_environment.py @@ -0,0 +1,1304 @@ +from __future__ import annotations + +import sys +import threading +import time +import types +from types import SimpleNamespace + +import pytest + + +class _FakeFS: + def __init__(self): + self.mkdir_calls: list[tuple[tuple, dict]] = [] + self.upload_calls: list[tuple[str, str]] = [] + self.download_calls: list[tuple[str, str]] = [] + + @staticmethod + def _assert_remote_path(path: str) -> None: + if path.startswith("/") and path != "/home/tenki" and not path.startswith("/home/tenki/"): + raise AssertionError(f"Tenki fs path must be under /home/tenki, got {path!r}") + + def mkdir(self, path, **kwargs): + self._assert_remote_path(str(path)) + self.mkdir_calls.append(((path,), kwargs)) + + def upload(self, local_path, remote_path, **_kwargs): + self._assert_remote_path(str(remote_path)) + self.upload_calls.append((str(local_path), str(remote_path))) + + def download(self, remote_path, local_path, **_kwargs): + self._assert_remote_path(str(remote_path)) + self.download_calls.append((str(remote_path), str(local_path))) + + +class _FakeResult: + def __init__(self, stdout: str = "", stderr: str = "", exit_code: int = 0): + self.stdout_text = stdout + self.stderr_text = stderr + self.exit_code = exit_code + + +class _FakeProcess: + def __init__( + self, + result: _FakeResult, + *, + stdin_data: str | None = None, + block_until_killed: bool = False, + ): + self._result = result + self.stdin_data = stdin_data + self.closed_stdin = False + self.killed = False + self._block_until_killed = block_until_killed + self._done = threading.Event() + + def close_stdin(self): + self.closed_stdin = True + + def kill(self): + self.killed = True + self._done.set() + + def wait(self, *_args, **_kwargs): + if self._block_until_killed: + self._done.wait(timeout=5) + return _FakeResult(stdout="", exit_code=143) + return self._result + + +class _FakeSandbox: + def __init__( + self, + *, + name: str = "sb-test", + state: str = "RUNNING", + metadata: dict | None = None, + ): + self.exec_calls: list[tuple[tuple, dict]] = [] + self.start_calls: list[tuple[tuple, dict]] = [] + self.last_process: _FakeProcess | None = None + self.snapshots: list[tuple[str | None, bool]] = [] + self.terminated = False + self.paused = False + self.resumed = False + self.waited = False + self.refreshed = False + self.id = "sb-test" + self.name = name + self.state = state + self.info = SimpleNamespace(name=name, metadata=metadata or {}) + self.fs = _FakeFS() + + @staticmethod + def _result_for_command(args): + command = args[-1] if args else "" + if "echo \"$HOME\"" in command: + return _FakeResult(stdout="/home/tenki\n") + return _FakeResult(stdout="ran\n", exit_code=0) + + def exec(self, *args, **kwargs): + self.exec_calls.append((args, kwargs)) + return self._result_for_command(args) + + def start(self, *args, **kwargs): + self.start_calls.append((args, kwargs)) + command = args[-1] if args else "" + self.last_process = _FakeProcess( + self._result_for_command(args), + stdin_data=kwargs.get("stdin"), + block_until_killed="sleep infinity" in command, + ) + return self.last_process + + def refresh(self): + self.refreshed = True + return self.info + + def terminate(self): + self.terminated = True + self.state = "TERMINATED" + + def pause(self): + self.paused = True + self.state = "PAUSED" + + def resume(self): + self.resumed = True + self.state = "RUNNING" + + def wait_ready(self, *_args, **_kwargs): + self.waited = True + + def snapshot(self, *, name=None, wait=True): + self.snapshots.append((name, wait)) + return SimpleNamespace(id=f"snap-{self.name}") + + +def _last_started_command(sandbox: _FakeSandbox) -> str: + return sandbox.start_calls[-1][0][-1] + + +class _FakeSnapshotNotFoundError(Exception): + """Mirrors tenki_sandbox.SnapshotNotFoundError for the fake SDK.""" + + +class _FakeRegistryArtifactNotFoundError(Exception): + """Mirrors tenki_sandbox.RegistryArtifactNotFoundError for the fake SDK.""" + + +class _FakeSnapshotNotDurableError(Exception): + """Mirrors tenki_sandbox.SnapshotNotDurableError for the fake SDK.""" + + +class _FakeInvalidStateError(Exception): + """Mirrors tenki_sandbox.InvalidStateError for the fake SDK.""" + + +class _FakeSandboxFactory: + created_kwargs: list[dict] = [] + failed_kwargs: list[dict] = [] + sandboxes: list[_FakeSandbox] = [] + fail_snapshot_ids: set[str] = set() + # When a snapshot id is in fail_snapshot_ids, raise this exception type with + # this message. Defaults to the confirmed-not-found error; tests set them to + # a transient error / generic message to prove the pointer is preserved, or + # to a snapshot-specific InvalidStateError to prove base-image fallback. + snapshot_error: type[Exception] = _FakeSnapshotNotFoundError + snapshot_error_msg: str = "restore failed" + + @classmethod + def create(cls, **kwargs): + if kwargs.get("snapshot_id") in cls.fail_snapshot_ids: + cls.failed_kwargs.append(kwargs) + raise cls.snapshot_error(cls.snapshot_error_msg) + sandbox = _FakeSandbox( + name=kwargs.get("name", "sb-test"), + metadata=kwargs.get("metadata", {}), + ) + cls.created_kwargs.append(kwargs) + cls.sandboxes.append(sandbox) + return sandbox + + +class _FakeClient: + listed_sandboxes: list[_FakeSandbox] = [] + closed_count = 0 + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.snapshots = SimpleNamespace(wait_durable=lambda *_args, **_kwargs: None) + + def create(self, **kwargs): + return _FakeSandboxFactory.create(**kwargs) + + def list(self, **_kwargs): + return list(self.listed_sandboxes) + + def list_project(self, *_args, **_kwargs): + return list(self.listed_sandboxes) + + def list_workspace(self, *_args, **_kwargs): + return list(self.listed_sandboxes) + + def close(self): + type(self).closed_count += 1 + + +def _install_fake_tenki(monkeypatch): + module = types.ModuleType("tenki_sandbox") + _FakeSandboxFactory.created_kwargs = [] + _FakeSandboxFactory.failed_kwargs = [] + _FakeSandboxFactory.sandboxes = [] + _FakeSandboxFactory.fail_snapshot_ids = set() + _FakeSandboxFactory.snapshot_error = _FakeSnapshotNotFoundError + _FakeSandboxFactory.snapshot_error_msg = "restore failed" + _FakeClient.listed_sandboxes = [] + _FakeClient.closed_count = 0 + module.Client = _FakeClient + module.Sandbox = _FakeSandboxFactory + module.SnapshotNotFoundError = _FakeSnapshotNotFoundError + module.RegistryArtifactNotFoundError = _FakeRegistryArtifactNotFoundError + module.SnapshotNotDurableError = _FakeSnapshotNotDurableError + module.InvalidStateError = _FakeInvalidStateError + monkeypatch.setitem(sys.modules, "tenki_sandbox", module) + + +def _clear_tenki_auth_env(monkeypatch): + monkeypatch.delenv("TENKI_AUTH_TOKEN", raising=False) + monkeypatch.delenv("TENKI_API_KEY", raising=False) + + +def _clear_env_passthrough_cache(): + try: + import tools.env_passthrough as env_passthrough + + env_passthrough.clear_env_passthrough() + env_passthrough._config_passthrough = None + except Exception: + pass + + +def test_tenki_cli_auth_token_is_normalized_for_sdk_cookie_auth(monkeypatch, tmp_path): + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: cli-cookie\n", encoding="utf-8") + + from tools.tenki_config import resolve_tenki_auth_token + + assert resolve_tenki_auth_token() == "cookie:cli-cookie" + + +def test_tenki_cli_auth_token_preserves_sdk_prefixes(monkeypatch, tmp_path): + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + + from tools.tenki_config import resolve_tenki_auth_token + + for token in ("cookie:cli-cookie", "ory_st_session", "sk-api-key"): + (tmp_path / "config.yaml").write_text(f"auth_token: {token}\n", encoding="utf-8") + assert resolve_tenki_auth_token() == token + + +def test_tenki_cli_api_key_is_not_treated_as_cookie(monkeypatch, tmp_path): + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("api_key: provider-key\n", encoding="utf-8") + + from tools.tenki_config import resolve_tenki_auth_token + + assert resolve_tenki_auth_token() == "provider-key" + + +def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "\n".join( + [ + "api_endpoint: https://api.tenki.test", + "current_workspace_id: ws-123", + "current_project_id: prj-456", + "auth_token: tok-secret", + ] + ), + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment( + image="", + task_id="session 1", + persistent_filesystem=False, + allow_inbound=False, + allow_outbound=True, + ) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + # The control-plane credential is used host-side to create the sandbox... + assert kwargs["base_url"] == "https://api.tenki.test" + assert kwargs["workspace_id"] == "ws-123" + assert kwargs["project_id"] == "prj-456" + assert kwargs["auth_token"] == "cookie:tok-secret" + # ...but is NEVER injected into the model-controlled guest environment + # (an empty env is omitted from the create kwargs entirely). + guest_env = kwargs.get("env", {}) + assert "TENKI_AUTH_TOKEN" not in guest_env + assert "TENKI_API_KEY" not in guest_env + assert "TENKI_API_ENDPOINT" not in guest_env + assert "TENKI_WORKSPACE_ID" not in guest_env + assert "TENKI_PROJECT_ID" not in guest_env + assert kwargs["allow_inbound"] is False + assert kwargs["allow_outbound"] is True + assert kwargs["cpu_cores"] == 1 + assert "idle_timeout" not in kwargs + assert "idle_timeout_minutes" not in kwargs + assert "pause_retention" not in kwargs + assert kwargs["metadata"]["hermes_backend"] == "tenki" + assert kwargs["metadata"]["hermes_profile"] + assert kwargs["name"].startswith("hermes-") + assert kwargs["name"].endswith("session-1") + + output, exit_code = env._exec_raw("echo ok", timeout=5) + assert output == "ran\n" + assert exit_code == 0 + + sandbox = _FakeSandboxFactory.sandboxes[0] + assert "TENKI_AUTH_TOKEN" not in sandbox.exec_calls[-1][1]["env"] + env.cleanup() + assert sandbox.terminated is True + assert sandbox.paused is False + + +def test_tenki_environment_does_not_inject_control_plane_token_by_default(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_API_KEY", "sk-test-key") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="api-key") + + kwargs = _FakeSandboxFactory.created_kwargs[0] + # Host-side create still authenticates with the credential... + assert kwargs["auth_token"] == "sk-test-key" + # ...but the guest never receives it unless explicitly forwarded. + guest_env = kwargs.get("env", {}) + assert "TENKI_AUTH_TOKEN" not in guest_env + assert "TENKI_API_KEY" not in guest_env + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_environment_forwards_control_plane_token_only_when_opted_in(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_API_KEY", "sk-test-key") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + # Nested-sandbox support: the operator explicitly forwards the credential. + env = TenkiEnvironment(task_id="api-key", forward_env=["TENKI_API_KEY"]) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["env"]["TENKI_API_KEY"] == "sk-test-key" + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_environment_honors_tenki_forward_env_from_process_env(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + monkeypatch.setenv("GH_TOKEN", "gho-process") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="gh-token", forward_env=["GH_TOKEN"]) + + assert _FakeSandboxFactory.created_kwargs[0]["env"]["GH_TOKEN"] == "gho-process" + env.execute("echo ok", timeout=5) + assert env._sandbox.start_calls[-1][1]["env"]["GH_TOKEN"] == "gho-process" + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_environment_honors_tenki_forward_env_from_hermes_dotenv(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + (tmp_path / ".env").write_text("GITHUB_TOKEN=ghp-dotenv\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="github-token", forward_env=["GITHUB_TOKEN"]) + + assert _FakeSandboxFactory.created_kwargs[0]["env"]["GITHUB_TOKEN"] == "ghp-dotenv" + env.execute("echo ok", timeout=5) + assert env._sandbox.start_calls[-1][1]["env"]["GITHUB_TOKEN"] == "ghp-dotenv" + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_forwarded_env_prefers_profile_scope_over_process_env(monkeypatch, tmp_path): + """Under a multiplexed profile scope, a forwarded credential must resolve + to the active profile's value, never another profile's raw os.environ.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + # Another profile's value leaking through the process environment... + monkeypatch.setenv("GH_TOKEN", "gho-other-profile") + + from agent import secret_scope + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(secret_scope, "_MULTIPLEX_ACTIVE", True) + token = secret_scope.set_secret_scope({"GH_TOKEN": "gho-this-profile"}) + try: + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="scoped", forward_env=["GH_TOKEN"]) + # ...must be overridden by the active profile scope. + assert _FakeSandboxFactory.created_kwargs[0]["env"]["GH_TOKEN"] == "gho-this-profile" + env.cleanup() + finally: + secret_scope.reset_secret_scope(token) + _clear_env_passthrough_cache() + + +def test_tenki_auth_token_prefers_profile_scope_over_process_env(monkeypatch, tmp_path): + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok-other-profile") + + from agent import secret_scope + from tools.tenki_config import resolve_tenki_auth_token + + monkeypatch.setattr(secret_scope, "_MULTIPLEX_ACTIVE", True) + tok = secret_scope.set_secret_scope({"TENKI_AUTH_TOKEN": "tok-this-profile"}) + try: + assert resolve_tenki_auth_token() == "tok-this-profile" + finally: + secret_scope.reset_secret_scope(tok) + + +def test_tenki_auth_token_fails_closed_when_multiplex_active_and_unscoped(monkeypatch, tmp_path): + """Multiplex on + no scope installed: an os.environ token must NOT leak + through (fail closed), rather than serving another profile's value.""" + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok-leaked-from-process-env") + + from agent import secret_scope + from tools.tenki_config import resolve_tenki_auth_token + + monkeypatch.setattr(secret_scope, "_MULTIPLEX_ACTIVE", True) + # No set_secret_scope() — this is the fail-closed branch. + assert secret_scope.current_secret_scope() is None + assert resolve_tenki_auth_token() == "" + + +def test_tenki_forwarded_env_fails_closed_when_multiplex_active_and_unscoped(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + monkeypatch.setenv("GH_TOKEN", "gho-leaked-from-process-env") + + from agent import secret_scope + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(secret_scope, "_MULTIPLEX_ACTIVE", True) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="scoped", forward_env=["GH_TOKEN"]) + + # No scope installed while multiplexing → the process-env value is not leaked. + assert "GH_TOKEN" not in _FakeSandboxFactory.created_kwargs[0].get("env", {}) + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_control_plane_token_forwarded_from_cli_config_when_opted_in(monkeypatch, tmp_path): + """The opt-in must work even when auth came from `tenki login` (CLI config), + whose secret never lands in os.environ.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + # Credential lives ONLY in the Tenki CLI config, not the environment. + (tmp_path / "config.yaml").write_text("auth_token: tok-cli-login\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="nested", forward_env=["TENKI_AUTH_TOKEN"]) + + guest_env = _FakeSandboxFactory.created_kwargs[0].get("env", {}) + assert guest_env["TENKI_AUTH_TOKEN"] == "cookie:tok-cli-login" + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_environment_honors_safe_env_passthrough(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + _clear_env_passthrough_cache() + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + monkeypatch.setenv("CUSTOM_TASK_ENV", "task-value") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n" + "terminal:\n" + " env_passthrough:\n" + " - CUSTOM_TASK_ENV\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="safe-passthrough") + + assert _FakeSandboxFactory.created_kwargs[0]["env"]["CUSTOM_TASK_ENV"] == "task-value" + env.execute("echo ok", timeout=5) + assert env._sandbox.start_calls[-1][1]["env"]["CUSTOM_TASK_ENV"] == "task-value" + env.cleanup() + _clear_env_passthrough_cache() + + +def test_tenki_environment_snapshots_when_persistent(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + sandbox = _FakeSandboxFactory.sandboxes[0] + env.cleanup() + assert len(sandbox.snapshots) == 1 + snap_name, snap_wait = sandbox.snapshots[0] + assert snap_name.endswith("persist") and snap_wait is True + assert sandbox.paused is False + assert sandbox.terminated is True + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + assert _FakeSandboxFactory.created_kwargs[-1]["snapshot_id"].endswith("persist") + assert "image" not in _FakeSandboxFactory.created_kwargs[-1] + env.cleanup() + + +def test_tenki_environment_falls_back_when_persistent_snapshot_is_stale(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-stale") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-stale"} + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.failed_kwargs[0]["snapshot_id"] == "snap-stale" + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + env.cleanup() + + +def test_tenki_environment_preserves_snapshot_on_transient_restore_error(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-transient") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-transient"} + # A transient failure (not a confirmed not-found) must NOT boot a blank + # base image or drop the recovery pointer. + _FakeSandboxFactory.snapshot_error = RuntimeError + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + with pytest.raises(RuntimeError): + TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + # No base-image fallback happened, and the snapshot pointer is retained. + assert _FakeSandboxFactory.created_kwargs == [] + assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-transient", False) + + +def test_tenki_environment_skips_snapshot_when_not_durable(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + def _fail_durable(*_args, **_kwargs): + raise RuntimeError("not durable yet") + + def _init(self, **kw): + self.kwargs = kw + self.snapshots = SimpleNamespace(wait_durable=_fail_durable) + + monkeypatch.setattr(_FakeClient, "__init__", _init) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = _FakeSandboxFactory.sandboxes[0] + + env.cleanup() + + # Durability failed → do NOT record the snapshot and do NOT terminate the + # live sandbox; pause it so state is preserved for recovery. + assert sandbox.paused is True + assert sandbox.terminated is False + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + + +def test_tenki_environment_resumes_existing_persistent_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + existing = _FakeSandbox( + name=f"hermes-{token}-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": token}, + ) + _FakeClient.listed_sandboxes = [existing] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert env._sandbox is existing + assert existing.resumed is True + assert existing.waited is True + assert _FakeSandboxFactory.created_kwargs == [] + # The resumed guest never receives the control-plane credential either. + assert "TENKI_AUTH_TOKEN" not in existing.exec_calls[-1][1]["env"] + env.cleanup() + + +def test_tenki_environment_does_not_reuse_other_profiles_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + # A live sandbox on the same Tenki account belonging to a DIFFERENT profile + # (foreign token) with the same task id must never be resumed. + foreign = _FakeSandbox( + name="hermes-deadbeef00-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": "deadbeef00"}, + ) + _FakeClient.listed_sandboxes = [foreign] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert env._sandbox is not foreign + assert foreign.resumed is False + assert _FakeSandboxFactory.created_kwargs, "should create its own sandbox" + env.cleanup() + + +def test_tenki_reuse_rejects_name_match_with_foreign_profile_metadata(monkeypatch, tmp_path): + """Defense-in-depth: even if a candidate's NAME matches, a differing + hermes_profile in metadata must block reuse.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + # Same name (as if a token collision), but metadata says a different profile. + collider = _FakeSandbox( + name=f"hermes-{token}-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": "foreign-token"}, + ) + _FakeClient.listed_sandboxes = [collider] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert env._sandbox is not collider + assert collider.resumed is False + env.cleanup() + + +def test_tenki_restore_falls_back_on_nondurable_snapshot(monkeypatch, tmp_path): + """A snapshot that EXISTS but is permanently unusable (non-durable) must + drop the pointer and boot the base image, not wedge the task forever.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-nondurable") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-nondurable"} + _FakeSandboxFactory.snapshot_error = _FakeSnapshotNotDurableError + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.failed_kwargs[0]["snapshot_id"] == "snap-nondurable" + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + env.cleanup() + + +def test_tenki_restore_preserves_pointer_on_invalid_state_error(monkeypatch, tmp_path): + """InvalidStateError is a generic precondition failure, NOT snapshot-gone, + so it must be treated as transient: preserve the pointer, do not base-boot.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-invalidstate") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-invalidstate"} + _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + with pytest.raises(_FakeInvalidStateError): + TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs == [] + assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-invalidstate", False) + + +def test_tenki_restore_falls_back_on_snapshot_specific_invalid_state(monkeypatch, tmp_path): + """A generic InvalidStateError whose message identifies the snapshot (the + SDK's collapsed representation of a bad/non-durable snapshot on restore) + IS unrecoverable → drop the pointer and boot the base image.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-badstate") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-badstate"} + _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError + _FakeSandboxFactory.snapshot_error_msg = "snapshot is not durable" + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + env.cleanup() + + +def test_tenki_snapshot_store_bound_to_construction_profile(monkeypatch, tmp_path): + """Cleanup (which may run in a background thread without the per-turn + HERMES_HOME contextvar) must write the snapshot pointer to the profile that + was active at construction, not whatever home is ambient at cleanup time.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + home_a = tmp_path / "profiles" / "a" + home_b = tmp_path / "profiles" / "b" + home_a.mkdir(parents=True) + home_b.mkdir(parents=True) + + monkeypatch.setenv("HERMES_HOME", str(home_a)) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + # Simulate a background cleanup running under the WRONG ambient home. + monkeypatch.setenv("HERMES_HOME", str(home_b)) + env.cleanup() + + # Pointer landed in profile A's store (construction-time), not B's. + assert (home_a / "tenki_snapshots.json").exists() + assert not (home_b / "tenki_snapshots.json").exists() + + +def test_tenki_persistent_not_terminated_when_snapshot_and_pause_both_fail(monkeypatch, tmp_path): + """Durability failed AND pause failed: the sandbox must be left live (not + terminated), so the only copy of un-snapshotted state is preserved.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + def _fail_durable(*_args, **_kwargs): + raise RuntimeError("not durable") + + def _init(self, **kw): + self.kwargs = kw + self.snapshots = SimpleNamespace(wait_durable=_fail_durable) + + monkeypatch.setattr(_FakeClient, "__init__", _init) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = _FakeSandboxFactory.sandboxes[0] + + def _fail_pause(): + raise RuntimeError("pause unavailable") + + sandbox.pause = _fail_pause + + env.cleanup() + + # Neither snapshot durable nor pause succeeded → sandbox left live. + assert sandbox.terminated is False + + +def test_tenki_environment_resumes_paused_cached_sandbox_before_execute(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="paused-cache") + sandbox = env._sandbox + sandbox.state = "PAUSED" + + env.execute("echo ok", timeout=5) + + assert sandbox.refreshed is True + assert sandbox.resumed is True + assert sandbox.waited is True + assert env._sandbox is sandbox + env.cleanup() + + +def test_tenki_environment_recreates_terminated_cached_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="terminated-cache") + first = env._sandbox + first.state = "TERMINATED" + + env.execute("echo ok", timeout=5) + + assert len(_FakeSandboxFactory.sandboxes) == 2 + assert env._sandbox is _FakeSandboxFactory.sandboxes[1] + assert env._sandbox is not first + env.cleanup() + + +def test_tenki_environment_ignores_mismatched_persistent_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + _FakeClient.listed_sandboxes = [ + _FakeSandbox( + name="hermes-other", + state="PAUSED", + metadata={"hermes_task_id": "other"}, + ) + ] + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs + assert _FakeSandboxFactory.created_kwargs[0]["name"].endswith("persist") + env.cleanup() + + +def test_tenki_environment_converts_idle_timeout_to_sdk_minutes(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="idle", cpu=1.2, idle_timeout=61) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["cpu_cores"] == 2 + assert kwargs["idle_timeout_minutes"] == 2 + env.cleanup() + + +def test_tenki_environment_omits_non_positive_pause_retention(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="pause-default", pause_retention=0) + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert "pause_retention" not in kwargs + env.cleanup() + + env = TenkiEnvironment(task_id="pause-negative", pause_retention=-1) + kwargs = _FakeSandboxFactory.created_kwargs[1] + assert "pause_retention" not in kwargs + env.cleanup() + + +def test_tenki_environment_passes_positive_pause_retention(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="pause-positive", pause_retention=3600) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["pause_retention"] == 3600 + env.cleanup() + + +def test_tenki_sync_hermes_home_is_opt_in(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + calls = [] + + class FakeSyncManager: + def __init__(self, **kwargs): + calls.append(("init", kwargs)) + + def sync(self, *, force=False): + calls.append(("sync", force)) + + def sync_back(self): + calls.append(("sync_back", None)) + + monkeypatch.setattr(tenki_module, "FileSyncManager", FakeSyncManager) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="no-sync", sync_hermes_home=False) + assert calls == [] + env.cleanup() + + env = TenkiEnvironment(task_id="sync", sync_hermes_home=True) + assert calls[0][0] == "init" + assert calls[1] == ("sync", True) + env.cleanup() + assert ("sync_back", None) in calls + + +def test_tenki_bulk_sync_stages_tar_under_home_not_tmp(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="bulk-sync") + host_file = tmp_path / "skill.md" + host_file.write_text("content", encoding="utf-8") + + env._tenki_bulk_upload([(str(host_file), "/home/tenki/.hermes/skills/skill.md")]) + + remote_tar = env._sandbox.fs.upload_calls[-1][1] + assert remote_tar.startswith("/home/tenki/.hermes_tenki_sync.") + assert not remote_tar.startswith("/tmp/") + env.cleanup() + + +def test_tenki_bulk_sync_uses_documented_fs_root_when_home_differs(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="root-home") + env._remote_home = "/root" + + assert env._remote_transfer_path(".hermes_tenki_sync").startswith("/home/tenki/") + env.cleanup() + + +def test_tenki_cleanup_sync_back_uses_original_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="cleanup-sync") + original = env._sandbox + created_before = len(_FakeSandboxFactory.created_kwargs) + + class FakeSyncManager: + def sync_back(self): + env._tenki_bulk_download(tmp_path / "sync-back.tar") + + env._sync_manager = FakeSyncManager() + env.cleanup() + + assert len(_FakeSandboxFactory.created_kwargs) == created_before + assert original.fs.download_calls + remote_tar = original.fs.download_calls[-1][0] + assert remote_tar.startswith("/home/tenki/.hermes_tenki_sync_back.") + assert original.terminated is True + + +def test_tenki_cleanup_blocks_public_execution_while_syncing(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="cleanup-guard") + created_before = len(_FakeSandboxFactory.created_kwargs) + + with env._lock: + env._cleanup_in_progress = True + env._cleanup_sandbox = env._sandbox + try: + try: + env.execute("echo should-not-run", timeout=5) + except RuntimeError as exc: + assert "cleanup" in str(exc) + else: + raise AssertionError("execute should fail while cleanup is in progress") + assert len(_FakeSandboxFactory.created_kwargs) == created_before + finally: + with env._lock: + env._cleanup_in_progress = False + env._cleanup_sandbox = None + env.cleanup() + + +def test_tenki_execute_passes_stdin_natively_not_as_heredoc(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="stdin") + large_stdin = "x" * 200_000 + + env.execute("cat > /home/tenki/out.txt", stdin_data=large_stdin, timeout=5) + + sandbox = env._sandbox + command = _last_started_command(sandbox) + assert large_stdin not in command + assert "HERMES_STDIN_" not in command + assert sandbox.start_calls[-1][1]["stdin"] == large_stdin + assert "TENKI_AUTH_TOKEN" not in sandbox.start_calls[-1][1]["env"] + env.cleanup() + + +def test_tenki_cancel_kills_process_without_tearing_down_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="cancel-process") + sandbox = env._sandbox + handle = env._run_bash("sleep infinity", timeout=30) + for _ in range(100): + if sandbox.last_process is not None: + break + time.sleep(0.01) + + handle.kill() + handle.wait(timeout=1) + + assert sandbox.last_process is not None + assert sandbox.last_process.killed is True + assert sandbox.terminated is False + assert sandbox.paused is False + env.cleanup() + + +def test_tenki_non_sudo_command_does_not_probe_sudo(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + def fail_probe(self): + raise AssertionError("sudo should not be probed for commands without sudo") + + monkeypatch.setattr(TenkiEnvironment, "_sudo_nopasswd_works", fail_probe) + + env = TenkiEnvironment(task_id="no-sudo") + env.execute("echo ok", timeout=5) + env.cleanup() + + +def test_tenki_passwordless_sudo_does_not_prompt_or_rewrite(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.delenv("SUDO_PASSWORD", raising=False) + monkeypatch.setenv("HERMES_INTERACTIVE", "1") + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + monkeypatch.setattr(TenkiEnvironment, "_sudo_nopasswd_works", lambda self: True) + + def fail_prompt(*_args, **_kwargs): + raise AssertionError("Tenki sudo should not prompt for a host password") + + monkeypatch.setattr("tools.terminal_tool._prompt_for_sudo_password", fail_prompt) + + env = TenkiEnvironment(task_id="sudo-nopasswd") + env.execute("sudo whoami", timeout=5) + + command = _last_started_command(_FakeSandboxFactory.sandboxes[0]) + assert "sudo whoami" in command + assert "sudo -S" not in command + assert "sudo -n whoami" not in command + env.cleanup() + + +def test_tenki_sudo_without_nopasswd_fails_fast_without_host_password(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setenv("SUDO_PASSWORD", "host-secret") + monkeypatch.setenv("HERMES_INTERACTIVE", "1") + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + monkeypatch.setattr(TenkiEnvironment, "_sudo_nopasswd_works", lambda self: False) + + def fail_prompt(*_args, **_kwargs): + raise AssertionError("Tenki sudo should not prompt for a host password") + + monkeypatch.setattr("tools.terminal_tool._prompt_for_sudo_password", fail_prompt) + + env = TenkiEnvironment(task_id="sudo-no-nopasswd") + env.execute("sudo whoami", timeout=5) + + command = _last_started_command(_FakeSandboxFactory.sandboxes[0]) + assert "sudo -n whoami" in command + assert "sudo -S" not in command + assert "host-secret" not in command + env.cleanup() diff --git a/tests/tools/test_terminal_config_env_sync.py b/tests/tools/test_terminal_config_env_sync.py index 5f6668fd62a01..94bb361709fb9 100644 --- a/tests/tools/test_terminal_config_env_sync.py +++ b/tests/tools/test_terminal_config_env_sync.py @@ -254,6 +254,59 @@ def test_docker_extra_args_is_bridged_everywhere(): assert "TERMINAL_DOCKER_EXTRA_ARGS" in _terminal_tool_env_var_names() +def test_tenki_config_backend_defaults_to_terminate_only(monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + from hermes_cli import config as hc_config + + hc_config._LOAD_CONFIG_CACHE.clear() + (tmp_path / "config.yaml").write_text("terminal:\n backend: tenki\n", encoding="utf-8") + + cfg = hc_config.load_config() + assert cfg["terminal"]["container_persistent"] is False + + env = {} + hc_config.apply_terminal_config_to_env(env=env, config=cfg) + assert env["TERMINAL_CONTAINER_PERSISTENT"] == "False" + + hc_config._LOAD_CONFIG_CACHE.clear() + (tmp_path / "config.yaml").write_text( + "terminal:\n backend: tenki\n container_persistent: true\n", + encoding="utf-8", + ) + + cfg = hc_config.load_config() + assert cfg["terminal"]["container_persistent"] is True + + +def test_tenki_managed_persistence_survives_env_bridge(monkeypatch, tmp_path): + home = tmp_path / "home" + managed = tmp_path / "managed" + home.mkdir() + managed.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed)) + + from hermes_cli import config as hc_config + from hermes_cli import managed_scope + + hc_config._LOAD_CONFIG_CACHE.clear() + hc_config._RAW_CONFIG_CACHE.clear() + managed_scope.invalidate_managed_cache() + (home / "config.yaml").write_text("terminal:\n backend: tenki\n", encoding="utf-8") + (managed / "config.yaml").write_text( + "terminal:\n container_persistent: true\n", + encoding="utf-8", + ) + + cfg = hc_config.load_config() + assert cfg["terminal"]["container_persistent"] is True + + env = {"TERMINAL_CONTAINER_PERSISTENT": "false"} + hc_config.apply_terminal_config_to_env(env=env, config=cfg, override=True) + assert env["TERMINAL_CONTAINER_PERSISTENT"] == "True" + + def test_docker_persist_across_processes_is_bridged_everywhere(): """Regression pin for the cross-process container reuse toggle. @@ -322,3 +375,26 @@ def test_docker_forward_env_is_bridged_everywhere(): assert "docker_forward_env" in _gateway_env_map_keys() assert "docker_forward_env" in _save_config_env_sync_keys() assert "TERMINAL_DOCKER_FORWARD_ENV" in _terminal_tool_env_var_names() + + +def test_tenki_config_is_bridged_everywhere(): + """Tenki backend config must reach every Hermes entry point.""" + tenki_keys = { + "tenki_image": "TERMINAL_TENKI_IMAGE", + "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", + "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", + "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", + "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", + "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", + "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", + "tenki_max_duration": "TERMINAL_TENKI_MAX_DURATION", + "tenki_idle_timeout": "TERMINAL_TENKI_IDLE_TIMEOUT", + "tenki_pause_retention": "TERMINAL_TENKI_PAUSE_RETENTION", + "tenki_sync_hermes_home": "TERMINAL_TENKI_SYNC_HERMES_HOME", + "tenki_forward_env": "TERMINAL_TENKI_FORWARD_ENV", + } + for key, env_var in tenki_keys.items(): + assert key in _cli_env_map_keys() + assert key in _gateway_env_map_keys() + assert key in _save_config_env_sync_keys() + assert env_var in _terminal_tool_env_var_names() diff --git a/tests/tools/test_terminal_requirements.py b/tests/tools/test_terminal_requirements.py index a2c1f00e12f26..d37a0b42a3f76 100644 --- a/tests/tools/test_terminal_requirements.py +++ b/tests/tools/test_terminal_requirements.py @@ -16,6 +16,12 @@ def _clear_terminal_env(monkeypatch): "TERMINAL_DOCKER_VOLUMES", "TERMINAL_LIFETIME_SECONDS", "TERMINAL_MODAL_MODE", + "TERMINAL_TENKI_API_ENDPOINT", + "TERMINAL_TENKI_PROJECT_ID", + "TERMINAL_TENKI_WORKSPACE_ID", + "TENKI_API_KEY", + "TENKI_AUTH_TOKEN", + "TENKI_CONFIG_PATH", "TERMINAL_SSH_HOST", "TERMINAL_SSH_PORT", "TERMINAL_SSH_USER", @@ -185,3 +191,38 @@ def test_modal_backend_managed_mode_without_feature_flag_logs_clear_error(monkey "Nous Tool Gateway access is not currently available" in record.getMessage() for record in caplog.records ) + + +def test_tenki_backend_with_sdk_and_cli_auth_returns_true(monkeypatch, tmp_path): + _clear_terminal_env(monkeypatch) + monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "tenki.yaml")) + (tmp_path / "tenki.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + monkeypatch.setattr( + terminal_tool_module.importlib.util, + "find_spec", + lambda name: object() if name == "tenki_sandbox" else None, + ) + + assert terminal_tool_module.check_terminal_requirements() is True + + +def test_tenki_backend_without_auth_logs_specific_error(monkeypatch, caplog, tmp_path): + _clear_terminal_env(monkeypatch) + monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setattr( + terminal_tool_module.importlib.util, + "find_spec", + lambda name: object() if name == "tenki_sandbox" else None, + ) + + with caplog.at_level(logging.ERROR): + ok = terminal_tool_module.check_terminal_requirements() + + assert ok is False + assert any( + "no Tenki auth was found" in record.getMessage() + for record in caplog.records + ) diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index 4608fe868aece..c1a56765069e4 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -65,6 +65,35 @@ def test_terminal_and_execute_code_tools_resolve_for_managed_modal(self, monkeyp assert "terminal" in names assert "execute_code" in names + def test_tenki_requires_auth_only_for_plain_sessions(self, monkeypatch, tmp_path): + original_find_spec = terminal_tool_module.importlib.util.find_spec + + def fake_find_spec(name): + if name == "tenki_sandbox": + return object() + return original_find_spec(name) + + monkeypatch.setattr(terminal_tool_module.importlib.util, "find_spec", fake_find_spec) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.delenv("TENKI_WORKSPACE_ID", raising=False) + monkeypatch.delenv("TENKI_WORKSPACE", raising=False) + monkeypatch.delenv("TENKI_PROJECT_ID", raising=False) + monkeypatch.delenv("TENKI_PROJECT", raising=False) + monkeypatch.setattr( + terminal_tool_module, + "_get_env_config", + lambda: { + "env_type": "tenki", + "tenki_workspace_id": "", + "tenki_project_id": "", + }, + ) + + assert terminal_tool_module.check_terminal_requirements() is False + + monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") + assert terminal_tool_module.check_terminal_requirements() is True + class TestCheckFnTransientFailureSuppression: """The check_fn TTL cache should absorb transient probe failures. diff --git a/tools/approval.py b/tools/approval.py index 7c58062ed58bc..7545353fd01ff 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -342,7 +342,7 @@ def _is_gateway_approval_context() -> bool: # # Hardline only applies to environments that can actually damage the host # (local, ssh, container-host cron). Containerized backends (docker, -# singularity, modal, daytona) already bypass the dangerous-command layer +# singularity, modal, daytona, tenki) already bypass the dangerous-command layer # because nothing they do can touch the host, so we leave that behavior # alone. # @@ -2326,7 +2326,7 @@ def _should_skip_container_guards(env_type: str, has_host_access: bool = False) """ if env_type == "docker": return not has_host_access - return env_type in ("singularity", "modal", "daytona") + return env_type in ("singularity", "modal", "daytona", "tenki") def check_dangerous_command(command: str, env_type: str, diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 82c248a3f715a..0ec35075db190 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -795,7 +795,7 @@ def _is_local_backend() -> bool: and network access on the same machine, so the check adds no security value. - However, when the terminal runs in a container (docker, modal, daytona, + However, when the terminal runs in a container (docker, modal, daytona, tenki, ssh, singularity), the browser on the host can access internal networks that the terminal cannot. In this case, SSRF protection should be enabled even though the browser is technically "local". diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index 54772d7d1a0a2..2a1a97e65516d 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -16,7 +16,7 @@ **Remote backends (file-based RPC):** 1. Parent generates `hermes_tools.py` with file-based RPC stubs 2. Parent ships both files to the remote environment - 3. Script runs inside the terminal backend (Docker/SSH/Modal/Daytona/etc.) + 3. Script runs inside the terminal backend (Docker/SSH/Modal/Daytona/Tenki/etc.) 4. Tool calls are written as request files; a polling thread on the parent reads them via env.execute(), dispatches, and writes response files 5. The script polls for response files and continues @@ -636,6 +636,8 @@ def _get_or_create_env(task_id: str): _get_env_config, _last_activity, _start_cleanup_thread, _creation_locks, _creation_locks_lock, _task_env_overrides, _resolve_container_task_id, + _CONTAINER_BACKENDS, + _container_config_from_env_config, ) effective_task_id = _resolve_container_task_id(task_id) @@ -670,22 +672,16 @@ def _get_or_create_env(task_id: str): image = overrides.get("modal_image") or config["modal_image"] elif env_type == "daytona": image = overrides.get("daytona_image") or config["daytona_image"] + elif env_type == "tenki": + image = overrides.get("tenki_image") or config["tenki_image"] else: image = "" cwd = overrides.get("cwd") or config["cwd"] container_config = None - if env_type in {"docker", "singularity", "modal", "daytona"}: - container_config = { - "container_cpu": config.get("container_cpu", 1), - "container_memory": config.get("container_memory", 5120), - "container_disk": config.get("container_disk", 51200), - "container_persistent": config.get("container_persistent", True), - "docker_volumes": config.get("docker_volumes", []), - "docker_run_as_host_user": config.get("docker_run_as_host_user", False), - "docker_network": config.get("docker_network", True), - } + if env_type in _CONTAINER_BACKENDS: + container_config = _container_config_from_env_config(config) ssh_config = None if env_type == "ssh": diff --git a/tools/env_probe.py b/tools/env_probe.py index 71a1c8116cf4e..25b0fb851c2ca 100644 --- a/tools/env_probe.py +++ b/tools/env_probe.py @@ -19,7 +19,7 @@ environment looks normal (python3+pip both present and matched, no PEP 668), it emits nothing — no token cost. -Remote terminal backends (docker, modal, ssh, …) are skipped: the +Remote terminal backends (docker, modal, tenki, ssh, …) are skipped: the host's Python state is irrelevant when tools run inside a sandbox. The sandbox has its own existing probe (``_probe_remote_backend``) in ``agent/prompt_builder.py``. @@ -49,7 +49,7 @@ # Duplicated rather than imported to avoid a circular import (prompt_builder # imports nothing from tools). _REMOTE_BACKENDS = frozenset({ - "docker", "singularity", "modal", "daytona", "ssh", "managed_modal", + "docker", "singularity", "modal", "daytona", "tenki", "ssh", "managed_modal", }) diff --git a/tools/environments/__init__.py b/tools/environments/__init__.py index 1eebcab42a086..664dea6134f01 100644 --- a/tools/environments/__init__.py +++ b/tools/environments/__init__.py @@ -2,7 +2,7 @@ Each backend provides the same interface (BaseEnvironment ABC) for running shell commands in a specific execution context: local, Docker, SSH, -Singularity, Modal, or Daytona. (Modal additionally has direct and +Singularity, Modal, Daytona, or Tenki. (Modal additionally has direct and Nous-managed modes, selected via terminal.modal_mode.) The terminal_tool.py factory (_create_environment) selects the backend diff --git a/tools/environments/base.py b/tools/environments/base.py index 9762902cb2240..5d69683b73408 100644 --- a/tools/environments/base.py +++ b/tools/environments/base.py @@ -189,7 +189,7 @@ def _file_mtime_key(host_path: str) -> tuple[float, int] | None: class ProcessHandle(Protocol): """Duck type that every backend's _run_bash() must return. - subprocess.Popen satisfies this natively. SDK backends (Modal, Daytona) + subprocess.Popen satisfies this natively. SDK backends (Modal, Daytona, Tenki) return _ThreadedProcessHandle which adapts their blocking calls. """ @@ -205,7 +205,7 @@ def returncode(self) -> int | None: ... class _ThreadedProcessHandle: - """Adapter for SDK backends (Modal, Daytona) that have no real subprocess. + """Adapter for SDK backends (Modal, Daytona, Tenki) that have no real subprocess. Wraps a blocking ``exec_fn() -> (output_str, exit_code)`` in a background thread and exposes a ProcessHandle-compatible interface. An optional @@ -848,7 +848,7 @@ def _extract_cwd_from_output(self, result: dict): """Parse the __HERMES_CWD_{session}__ marker from stdout output. Updates self.cwd and strips the marker from result["output"]. - Used by remote backends (Docker, SSH, Modal, Daytona, Singularity). + Used by remote backends (Docker, SSH, Modal, Daytona, Tenki, Singularity). """ output = result.get("output", "") marker = self._cwd_marker @@ -885,7 +885,7 @@ def _extract_cwd_from_output(self, result: dict): def _before_execute(self) -> None: """Hook called before each command execution. - Remote backends (SSH, Modal, Daytona) override this to trigger + Remote backends (SSH, Modal, Daytona, Tenki) override this to trigger their FileSyncManager. Bind-mount backends (Docker, Singularity) and Local don't need file sync — the host filesystem is directly visible inside the container/process. diff --git a/tools/environments/local.py b/tools/environments/local.py index 1052837051923..56dd1dde29dd0 100644 --- a/tools/environments/local.py +++ b/tools/environments/local.py @@ -244,6 +244,8 @@ def _build_provider_env_blocklist() -> frozenset: "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "DAYTONA_API_KEY", + "TENKI_AUTH_TOKEN", + "TENKI_API_KEY", "GATEWAY_RELAY_ID", "GATEWAY_RELAY_SECRET", "GATEWAY_RELAY_DELIVERY_KEY", @@ -465,6 +467,8 @@ def _sanitize_subprocess_env(base_env: dict | None, extra_env: dict | None = Non "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "DAYTONA_API_KEY", + "TENKI_AUTH_TOKEN", + "TENKI_API_KEY", }) diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py new file mode 100644 index 0000000000000..958aa34311f7d --- /dev/null +++ b/tools/environments/tenki.py @@ -0,0 +1,1087 @@ +"""Tenki cloud sandbox execution environment.""" + +from __future__ import annotations + +import hashlib +import inspect +import logging +import math +import os +import re +import shlex +import tarfile +import tempfile +import threading +from pathlib import Path +from typing import Any + +from hermes_constants import get_hermes_home +from tools.environments.base import ( + BaseEnvironment, + _ThreadedProcessHandle, + _load_json_store, + _save_json_store, +) +from tools.environments.file_sync import ( + FileSyncManager, + iter_sync_files, + quoted_mkdir_command, + quoted_rm_command, + unique_parent_dirs, +) +from tools.tenki_config import ( + resolve_tenki_api_endpoint, + resolve_tenki_auth_token, + resolve_tenki_project_id, + resolve_tenki_workspace_id, +) + +logger = logging.getLogger(__name__) +_SNAPSHOT_NAMESPACE = "direct" +_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def _snapshot_store_path() -> Path: + """Resolve the snapshot registry path for the *active* profile. + + Resolved per call (not frozen at import) so the multiplexing gateway, + which overrides ``HERMES_HOME`` per turn, writes each profile's snapshot + pointers into that profile's own home instead of whichever profile + happened to import this module first. + """ + return get_hermes_home() / "tenki_snapshots.json" + + +def _profile_token() -> str: + """Short, stable identifier for the active Hermes profile. + + Two profiles sharing one Tenki account must get distinct sandbox + names/metadata so they can never attach to or restore each other's + sandbox. Prefer the canonical ``HERMES_PROFILE`` id, which is stable across + machines and survives a home-directory move; only fall back to a + *normalized* ``HERMES_HOME`` path when no profile id is set (the default + profile). Resolving per call handles the multiplexing gateway's per-turn + ``HERMES_HOME`` override (same reason as :func:`_snapshot_store_path`). + """ + profile = os.getenv("HERMES_PROFILE", "").strip() + if profile: + basis = f"profile:{profile}" + else: + try: + basis = str(get_hermes_home().resolve()) + except Exception: + basis = str(get_hermes_home()) + return hashlib.sha1(basis.encode("utf-8")).hexdigest()[:10] + + +def _load_snapshots(store_path: Path | None = None) -> dict: + return _load_json_store(store_path or _snapshot_store_path()) + + +def _save_snapshots(data: dict, store_path: Path | None = None) -> None: + _save_json_store(store_path or _snapshot_store_path(), data) + + +def _snapshot_key(task_id: str) -> str: + return f"{_SNAPSHOT_NAMESPACE}:{task_id}" + + +def _get_snapshot_restore_candidate( + task_id: str, store_path: Path | None = None +) -> tuple[str | None, bool]: + snapshots = _load_snapshots(store_path) + namespaced_key = _snapshot_key(task_id) + snapshot_id = snapshots.get(namespaced_key) + if isinstance(snapshot_id, str) and snapshot_id: + return snapshot_id, False + legacy_snapshot_id = snapshots.get(task_id) + if isinstance(legacy_snapshot_id, str) and legacy_snapshot_id: + return legacy_snapshot_id, True + return None, False + + +def _store_snapshot(task_id: str, snapshot_id: str, store_path: Path | None = None) -> None: + snapshots = _load_snapshots(store_path) + snapshots[_snapshot_key(task_id)] = snapshot_id + snapshots.pop(task_id, None) + _save_snapshots(snapshots, store_path) + + +def _delete_snapshot( + task_id: str, snapshot_id: str | None = None, store_path: Path | None = None +) -> None: + snapshots = _load_snapshots(store_path) + updated = False + for key in (_snapshot_key(task_id), task_id): + value = snapshots.get(key) + if value is None: + continue + if snapshot_id is None or value == snapshot_id: + snapshots.pop(key, None) + updated = True + if updated: + _save_snapshots(snapshots, store_path) + + +def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for item in forward_env or []: + if not isinstance(item, str): + logger.warning("Ignoring non-string tenki_forward_env entry: %r", item) + continue + name = item.strip() + if not name: + continue + if not _ENV_NAME_RE.match(name): + logger.warning("Ignoring invalid tenki_forward_env entry: %r", item) + continue + if name not in seen: + normalized.append(name) + seen.add(name) + return normalized + + +def _safe_name(value: str, *, fallback: str = "default", max_len: int = 48) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", value or "").strip("-._") + return (safe or fallback)[:max_len] + + +def _supports_any_kwargs(sig: inspect.Signature | None) -> bool: + if sig is None: + return True + return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in sig.parameters.values()) + + +def _add_supported( + kwargs: dict[str, Any], + sig: inspect.Signature | None, + names: tuple[str, ...], + value: Any, +) -> None: + if value in (None, "", [], {}): + return + if sig is not None: + for name in names: + if name in sig.parameters: + kwargs[name] = value + return + if _supports_any_kwargs(sig): + kwargs[names[0]] = value + + +def _result_attr(result: Any, names: tuple[str, ...]) -> Any: + for name in names: + if not hasattr(result, name): + continue + value = getattr(result, name) + if callable(value): + try: + value = value() + except TypeError: + pass + if value is not None: + return value + return None + + +def _text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + +def _positive_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +def _rewrite_sudo_noninteractive(command: str) -> tuple[str, int]: + """Add ``-n`` to real sudo invocations so Tenki never prompts.""" + from tools.terminal_tool import _looks_like_env_assignment, _read_shell_token + + out: list[str] = [] + i = 0 + n = len(command) + command_start = True + sudo_count = 0 + + while i < n: + ch = command[i] + + if ch.isspace(): + out.append(ch) + if ch == "\n": + command_start = True + i += 1 + continue + + if ch == "#" and command_start: + comment_end = command.find("\n", i) + if comment_end == -1: + out.append(command[i:]) + break + out.append(command[i:comment_end]) + i = comment_end + continue + + if command.startswith("&&", i) or command.startswith("||", i) or command.startswith(";;", i): + out.append(command[i:i + 2]) + i += 2 + command_start = True + continue + + if ch in ";|&(": + out.append(ch) + i += 1 + command_start = True + continue + + if ch == ")": + out.append(ch) + i += 1 + command_start = False + continue + + token, next_i = _read_shell_token(command, i) + if command_start and token == "sudo": + out.append("sudo -n") + sudo_count += 1 + else: + out.append(token) + + if command_start and _looks_like_env_assignment(token): + command_start = True + else: + command_start = False + i = next_i + + return "".join(out), sudo_count + + +class TenkiEnvironment(BaseEnvironment): + """Tenki sandbox backend. + + Tenki's SDK exposes process handles inside a remote sandbox, so this adapts + them to the normal Hermes ``ProcessHandle`` contract with + ``_ThreadedProcessHandle``. + """ + + _stdin_mode = "pipe" + _snapshot_timeout = 60 + _terminal_states = frozenset({"TERMINATING", "TERMINATED", "DELETED", "FAILED", "ERROR"}) + + def __init__( + self, + image: str = "", + cwd: str = "/home/tenki", + timeout: int = 60, + cpu: float = 1, + memory: int = 5120, + disk: int = 51200, + persistent_filesystem: bool = False, + task_id: str = "default", + api_endpoint: str = "", + workspace_id: str = "", + project_id: str = "", + name_prefix: str = "hermes", + allow_inbound: bool = False, + allow_outbound: bool = True, + max_duration: int = 3600, + idle_timeout: int = 0, + pause_retention: int = 0, + sync_hermes_home: bool = False, + forward_env: list[str] | None = None, + ): + super().__init__(cwd=cwd, timeout=timeout) + + try: + from tools.lazy_deps import ensure as _lazy_ensure + + _lazy_ensure("terminal.tenki", prompt=False) + except ImportError: + pass + except Exception as exc: + raise ImportError(str(exc)) + + from tenki_sandbox import Client, Sandbox + + self._Client = Client + self._Sandbox = Sandbox + self._client = None + self._sandbox = None + self._lock = threading.Lock() + self._persistent = persistent_filesystem + self._sync_hermes_home = sync_hermes_home + self._sync_manager: FileSyncManager | None = None + self._cleanup_in_progress = False + self._cleanup_sandbox = None + self._task_id = task_id + self._profile_token = _profile_token() + # Bind the profile's snapshot-store path at construction, while the + # correct HERMES_HOME context is active. Cleanup (and the idle-reaper + # snapshot save) can run in a background thread that does NOT inherit + # the per-turn HERMES_HOME contextvar, so re-resolving there would write + # the pointer into the wrong profile's home. + self._snapshot_store = _snapshot_store_path() + self._snapshot_restore_id: str | None = None + self._snapshot_restore_from_legacy_key = False + self._image = image + self._cpu = cpu + self._memory = memory + self._disk = disk + self._api_endpoint = resolve_tenki_api_endpoint(api_endpoint) + self._workspace_id = resolve_tenki_workspace_id(workspace_id) + self._project_id = resolve_tenki_project_id(project_id) + self._auth_token = resolve_tenki_auth_token() + self._name_prefix = _safe_name(name_prefix, fallback="hermes", max_len=28) + self._allow_inbound = allow_inbound + self._allow_outbound = allow_outbound + self._max_duration = max_duration + self._idle_timeout = idle_timeout + self._pause_retention = pause_retention + self._forward_env = _normalize_forward_env_names(forward_env) + self._remote_home = "/home/tenki" + if self._persistent: + self._snapshot_restore_id, self._snapshot_restore_from_legacy_key = ( + _get_snapshot_restore_candidate(self._task_id, self._snapshot_store) + ) + + self._ensure_sandbox() + self._resolve_remote_home() + if self._sync_hermes_home: + self._sync_manager = FileSyncManager( + get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"), + upload_fn=self._tenki_upload, + delete_fn=self._tenki_delete, + bulk_upload_fn=self._tenki_bulk_upload, + bulk_download_fn=self._tenki_bulk_download, + ) + self._sync_manager.sync(force=True) + self.init_session() + + def _sandbox_create_signature(self) -> inspect.Signature | None: + try: + return inspect.signature(self._Sandbox.create) + except (TypeError, ValueError): + return None + + def _create_kwargs(self) -> dict[str, Any]: + sig = self._sandbox_create_signature() + kwargs: dict[str, Any] = {} + sandbox_name = self._sandbox_name() + + _add_supported(kwargs, sig, ("name",), sandbox_name) + if self._snapshot_restore_id: + _add_supported(kwargs, sig, ("snapshot_id",), self._snapshot_restore_id) + else: + _add_supported(kwargs, sig, ("image", "template"), self._image) + cpu_cores = max(1, math.ceil(float(self._cpu))) if self._cpu else None + _add_supported(kwargs, sig, ("cpu_cores", "cpu"), cpu_cores) + _add_supported(kwargs, sig, ("memory_mb", "memory"), self._memory) + + if self._disk: + disk_gb = max(1, math.ceil(float(self._disk) / 1024)) + _add_supported(kwargs, sig, ("disk_size_gb", "disk_gb", "disk"), disk_gb) + + _add_supported(kwargs, sig, ("allow_inbound",), self._allow_inbound) + _add_supported(kwargs, sig, ("allow_outbound",), self._allow_outbound) + _add_supported(kwargs, sig, ("max_duration",), self._max_duration) + idle_timeout = _positive_float(self._idle_timeout) + if idle_timeout is not None: + idle_timeout_minutes = max(1, math.ceil(idle_timeout / 60)) + _add_supported(kwargs, sig, ("idle_timeout_minutes",), idle_timeout_minutes) + pause_retention = _positive_float(self._pause_retention) + if pause_retention is not None: + _add_supported(kwargs, sig, ("pause_retention",), pause_retention) + _add_supported(kwargs, sig, ("workspace_id",), self._workspace_id) + _add_supported(kwargs, sig, ("project_id",), self._project_id) + _add_supported(kwargs, sig, ("base_url", "api_endpoint"), self._api_endpoint) + _add_supported(kwargs, sig, ("auth_token", "api_key"), self._auth_token) + _add_supported(kwargs, sig, ("env",), self._sandbox_env()) + _add_supported( + kwargs, + sig, + ("metadata",), + { + "hermes_task_id": self._task_id, + "hermes_backend": "tenki", + "hermes_profile": self._profile_token, + }, + ) + _add_supported(kwargs, sig, ("tags",), ["hermes-agent"]) + _add_supported(kwargs, sig, ("wait",), True) + # Do NOT emit a create-time ``timeout`` here: the SDK's Sandbox.create + # pops ``timeout`` into the *Client* (HTTP) timeout, while Client.create + # treats ``timeout`` as the *wait-for-ready* budget — so the same value + # would mean two different things across the two create paths. The HTTP + # timeout is set explicitly in _create_client(); readiness uses the + # SDK's default wait budget. + return kwargs + + def _sandbox_env(self) -> dict[str, str]: + """Environment variables injected into Tenki sandbox processes. + + The supervisor's Tenki control-plane credential is used host-side to + create and manage the sandbox (see ``_create_kwargs`` / + ``_create_client``); it is deliberately NOT injected into the guest. + Guest code is model-controlled and can print, exfiltrate, or reuse + whatever is in its environment, and the sandbox is billed against the + supervisor's account — so a leaked ``TENKI_AUTH_TOKEN`` would let guest + code create, terminate, and bill account resources outside the + parent's configured limits. Nested-sandbox support is still available + as an explicit opt-in: list ``TENKI_AUTH_TOKEN`` (or ``TENKI_API_KEY``) + in ``terminal.tenki_forward_env``. + + ``terminal.tenki_forward_env`` is the explicit allowlist for + task-specific credentials such as GitHub tokens; the generic + ``terminal.env_passthrough`` allowlist is also honored for skill + variables that are not protected by Hermes' provider-secret blocklist. + """ + env: dict[str, str] = {} + env.update(self._resolve_forwarded_env(self._forward_env)) + env.update(self._passthrough_env()) + # If the operator explicitly opted into forwarding the control-plane + # credential (for nested-sandbox creation), supply the already-resolved + # token. Re-reading the env var here would miss a `tenki login` + # credential, which lives in the Tenki CLI config, not the environment — + # so the documented opt-in would silently forward nothing. + if self._auth_token: + for key in ("TENKI_AUTH_TOKEN", "TENKI_API_KEY"): + if key in self._forward_env and not env.get(key): + env[key] = self._auth_token + logger.warning( + "Tenki: forwarding the control-plane credential %s into the " + "sandbox as requested by terminal.tenki_forward_env. Guest code " + "can read it and create/terminate/bill account resources. Note " + "that forwarded credentials are NOT profile-isolated under the " + "multiplexing gateway's shared terminal cache.", + key, + ) + return env + + @staticmethod + def _resolve_forwarded_env(keys: list[str] | set[str] | tuple[str, ...]) -> dict[str, str]: + if not keys: + return {} + from tools.tenki_config import _global_credential_fallback_allowed, _scoped_env + + get_env_value = None + if _global_credential_fallback_allowed(): + try: + from hermes_cli.config import get_env_value + except Exception: + get_env_value = None + + env: dict[str, str] = {} + for key in keys: + # Scope-aware read first: under a multiplexed profile turn this + # resolves the active profile's value, never another profile's raw + # os.environ. The ~/.hermes/.env fallback is consulted only when no + # profile scope is authoritative. + value = _scoped_env(key) + if not value and get_env_value is not None: + try: + value = get_env_value(key) or "" + except Exception: + value = "" + if value: + env[key] = value + return env + + @staticmethod + def _passthrough_env() -> dict[str, str]: + try: + from tools.env_passthrough import get_all_passthrough + + keys = sorted(get_all_passthrough()) + except Exception: + keys = [] + return TenkiEnvironment._resolve_forwarded_env(keys) + + def _create_client(self): + if self._client is None: + self._client = self._Client( + auth_token=self._auth_token, + base_url=self._api_endpoint, + timeout=max(60, self.timeout), + ) + return self._client + + def _sandbox_name(self) -> str: + # The profile token namespaces the name so two profiles sharing one + # Tenki account never collide on a name or reuse each other's sandbox. + return f"{self._name_prefix}-{self._profile_token}-{_safe_name(self._task_id)}" + + @staticmethod + def _sandbox_state(sandbox: Any) -> str: + state = getattr(sandbox, "state", "") + if callable(state): + try: + state = state() + except TypeError: + state = "" + return str(state or "").upper() + + def _sandbox_matches_task(self, sandbox: Any) -> bool: + name = getattr(sandbox, "name", "") + info = getattr(sandbox, "info", None) + if not name and info is not None: + name = getattr(info, "name", "") + if name != self._sandbox_name(): + return False + metadata = getattr(info, "metadata", {}) if info is not None else {} + # Never reuse another profile's sandbox: if the candidate carries a + # profile token it must match ours (the name already encodes it, but + # metadata is the authoritative, defense-in-depth check). + if isinstance(metadata, dict) and metadata.get("hermes_profile"): + if metadata.get("hermes_profile") != self._profile_token: + return False + if isinstance(metadata, dict) and metadata.get("hermes_task_id"): + return metadata.get("hermes_task_id") == self._task_id + return True + + def _find_persistent_sandbox(self): + if not self._persistent: + return None + client = self._create_client() + try: + if self._project_id and hasattr(client, "list_project"): + candidates = client.list_project(self._project_id, tags=["hermes-agent"]) + elif self._workspace_id and hasattr(client, "list_workspace"): + candidates = client.list_workspace(self._workspace_id, tags=["hermes-agent"]) + else: + candidates = client.list(tags=["hermes-agent"]) + except Exception as exc: + logger.debug("Tenki: could not list persistent sandboxes: %s", exc) + return None + + usable = [] + for sandbox in candidates: + if not self._sandbox_matches_task(sandbox): + continue + state = self._sandbox_state(sandbox) + if state in self._terminal_states: + continue + usable.append((state, sandbox)) + if not usable: + return None + usable.sort(key=lambda item: 0 if item[0] == "RUNNING" else 1) + return usable[0][1] + + def _resume_persistent_sandbox(self): + sandbox = self._find_persistent_sandbox() + if sandbox is None: + return None + if not self._ensure_sandbox_ready(sandbox): + logger.info( + "Tenki: existing sandbox for task %s is no longer reusable; creating a fresh sandbox", + self._task_id, + ) + return None + sandbox_id = getattr(sandbox, "id", None) or getattr(sandbox, "sandbox_id", None) + logger.info("Tenki: resumed sandbox %s for task %s", sandbox_id or "", self._task_id) + return sandbox + + def _ensure_sandbox_ready(self, sandbox: Any) -> bool: + refresh = getattr(sandbox, "refresh", None) + if callable(refresh): + try: + refresh() + except Exception as exc: + logger.info("Tenki: sandbox refresh failed for task %s: %s", self._task_id, exc) + return False + + state = self._sandbox_state(sandbox) + if state in self._terminal_states: + return False + + try: + if state and state != "RUNNING": + resume = getattr(sandbox, "resume", None) + if callable(resume): + resume() + wait_ready = getattr(sandbox, "wait_ready", None) + if callable(wait_ready): + wait_ready(max(60, self.timeout)) + except Exception as exc: + logger.info("Tenki: could not make sandbox ready for task %s: %s", self._task_id, exc) + return False + + return self._sandbox_state(sandbox) not in self._terminal_states + + def _ensure_sandbox(self) -> None: + with self._lock: + if self._cleanup_in_progress: + raise RuntimeError("Tenki cleanup is in progress") + if self._sandbox is not None: + if self._ensure_sandbox_ready(self._sandbox): + return + self._sandbox = None + self._sandbox = self._resume_persistent_sandbox() + if self._sandbox is not None: + return + self._sandbox = self._create_sandbox_with_snapshot_fallback() + sandbox_id = getattr(self._sandbox, "id", None) or getattr(self._sandbox, "sandbox_id", None) + logger.info("Tenki: created sandbox %s for task %s", sandbox_id or "", self._task_id) + + def _create_sandbox_from_kwargs(self, kwargs: dict[str, Any]): + if self._persistent: + client = self._create_client() + create_kwargs = dict(kwargs) + for key in ("auth_token", "api_key", "base_url", "api_endpoint"): + create_kwargs.pop(key, None) + return client.create(**create_kwargs) + return self._Sandbox.create(**kwargs) + + # Snapshot errors that mean the recorded snapshot can never restore, so + # dropping the pointer and booting the base image is the right recovery. + # A *transient* failure (network, rate-limit, ambiguous) is NOT in this set: + # it must propagate with the pointer intact so a later attempt can still + # recover the persistent state instead of silently booting an empty sandbox. + # Snapshot-specific error type names that mean the snapshot can never + # restore. Note InvalidStateError is intentionally NOT here: the restore RPC + # maps a generic FAILED_PRECONDITION (workspace/policy/etc.) to it, so it is + # only unrecoverable when its message points at the snapshot itself + # (handled by message inspection below); a bare InvalidStateError stays + # transient so an unrelated precondition can't destroy a valid pointer. + _UNRECOVERABLE_SNAPSHOT_ERRORS = frozenset({ + "SnapshotNotFoundError", # snapshot is gone + "RegistryArtifactNotFoundError", # backing artifact is gone + "SnapshotNotDurableError", # explicitly never reached durability + }) + + @classmethod + def _snapshot_unrecoverable(cls, exc: BaseException) -> bool: + """True when the error confirms the snapshot can never restore. + + Covers "gone" (not-found), "explicitly non-durable", and a generic + ``InvalidStateError`` whose message identifies the snapshot as the + failing precondition (the SDK's restore RPC collapses a bad/non-durable + snapshot into a generic FAILED_PRECONDITION → InvalidStateError). Only + these justify discarding the recovery pointer and booting a base image; + every other error (including a bare InvalidStateError, rate-limit, + quota, auth blip, or network failure) is transient and re-raised with + the pointer preserved. + """ + def _is_snapshot_specific_invalid_state(e: BaseException, invalid_state_cls) -> bool: + if invalid_state_cls is not None and not isinstance(e, invalid_state_cls): + return False + if invalid_state_cls is None and type(e).__name__ != "InvalidStateError": + return False + msg = str(e).lower() + return "snapshot" in msg or "durable" in msg + + try: + from tenki_sandbox import ( + InvalidStateError, + RegistryArtifactNotFoundError, + SnapshotNotDurableError, + SnapshotNotFoundError, + ) + + if isinstance( + exc, + (SnapshotNotFoundError, RegistryArtifactNotFoundError, SnapshotNotDurableError), + ): + return True + if _is_snapshot_specific_invalid_state(exc, InvalidStateError): + return True + return False + except Exception: + pass + # Name-based fallback for SDK builds that don't export every class. + for typ in type(exc).__mro__: + if typ.__name__ in cls._UNRECOVERABLE_SNAPSHOT_ERRORS: + return True + return _is_snapshot_specific_invalid_state(exc, None) + + def _create_sandbox_with_snapshot_fallback(self): + kwargs = self._create_kwargs() + try: + sandbox = self._create_sandbox_from_kwargs(kwargs) + except Exception as exc: + if not self._snapshot_restore_id: + raise + if not self._snapshot_unrecoverable(exc): + # Ambiguous/transient failure — keep the snapshot pointer so a + # later attempt can still recover, rather than deleting it and + # booting a blank base image (silent loss of persistent state). + logger.warning( + "Tenki: snapshot restore %s for task %s failed transiently (%s); " + "preserving it for retry", + self._snapshot_restore_id, + self._task_id, + exc, + ) + raise + logger.warning( + "Tenki: snapshot %s for task %s is unrecoverable; creating from base image: %s", + self._snapshot_restore_id, + self._task_id, + exc, + ) + _delete_snapshot(self._task_id, self._snapshot_restore_id, self._snapshot_store) + self._snapshot_restore_id = None + self._snapshot_restore_from_legacy_key = False + sandbox = self._create_sandbox_from_kwargs(self._create_kwargs()) + else: + if self._snapshot_restore_id and self._snapshot_restore_from_legacy_key: + _store_snapshot(self._task_id, self._snapshot_restore_id, self._snapshot_store) + return sandbox + + def _remote_transfer_path(self, prefix: str) -> str: + base = (self._remote_home or "/home/tenki").rstrip("/") or "/home/tenki" + if base != "/home/tenki" and not base.startswith("/home/tenki/"): + base = "/home/tenki" + return f"{base}/{prefix}.{os.getpid()}.{self._session_id}.tar" + + def _resolve_remote_home(self) -> None: + try: + result = self._exec_raw("echo \"$HOME\"", timeout=15) + home = result[0].strip() if result[1] == 0 else "" + if home: + self._remote_home = home + if self.cwd in {"~", "/home/tenki"}: + self.cwd = home + except Exception: + pass + + def _tenki_upload(self, host_path: str, remote_path: str) -> None: + self._ensure_sandbox() + parent = str(Path(remote_path).parent) + self._sandbox.fs.mkdir(parent, recursive=True) + self._sandbox.fs.upload(host_path, remote_path) + + def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: + if not files: + return + + self._ensure_sandbox() + parents = unique_parent_dirs(files) + if parents: + self._exec_raw(quoted_mkdir_command(parents), timeout=30) + + remote_tar = self._remote_transfer_path(".hermes_tenki_sync") + with tempfile.NamedTemporaryFile(suffix=".tar") as tmp: + with tarfile.open(fileobj=tmp, mode="w") as tar: + for host_path, remote_path in files: + tar.add(host_path, arcname=remote_path.lstrip("/")) + tmp.flush() + self._sandbox.fs.upload(tmp.name, remote_tar) + + try: + output, exit_code = self._exec_raw( + f"tar xf {shlex.quote(remote_tar)} -C /", + timeout=120, + ) + if exit_code != 0: + raise RuntimeError(f"Tenki bulk upload failed (exit {exit_code}): {output}") + finally: + try: + self._exec_raw(f"rm -f {shlex.quote(remote_tar)}", timeout=10) + except Exception: + pass + + def _tenki_bulk_download(self, dest: Path) -> None: + sandbox = self._transfer_sandbox() + remote_tar = self._remote_transfer_path(".hermes_tenki_sync_back") + rel_base = f"{self._remote_home}/.hermes".lstrip("/") + try: + output, exit_code = self._exec_raw_on_sandbox( + sandbox, + f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(rel_base)}", + timeout=120, + ) + if exit_code != 0: + raise RuntimeError(f"Tenki bulk download failed (exit {exit_code}): {output}") + sandbox.fs.download(remote_tar, str(dest)) + finally: + try: + self._exec_raw_on_sandbox(sandbox, f"rm -f {shlex.quote(remote_tar)}", timeout=10) + except Exception: + pass + + def _transfer_sandbox(self): + if self._cleanup_in_progress and self._cleanup_sandbox is not None: + return self._cleanup_sandbox + self._ensure_sandbox() + return self._sandbox + + def _tenki_delete(self, remote_paths: list[str]) -> None: + if not remote_paths: + return + self._exec_raw(quoted_rm_command(remote_paths), timeout=30) + + def _exec_raw(self, command: str, *, login: bool = False, timeout: int = 120) -> tuple[str, int]: + self._ensure_sandbox() + return self._exec_raw_on_sandbox(self._sandbox, command, login=login, timeout=timeout) + + def _exec_raw_on_sandbox( + self, + sandbox: Any, + command: str, + *, + login: bool = False, + timeout: int = 120, + ) -> tuple[str, int]: + flag = "-lc" if login else "-c" + result = sandbox.exec("bash", flag, command, timeout=timeout, env=self._sandbox_env()) + return self._result_to_output(result) + + @staticmethod + def _result_to_output(result: Any) -> tuple[str, int]: + stdout = _text(_result_attr(result, ("stdout_text", "stdout", "output", "result", "text"))) + stderr = _text(_result_attr(result, ("stderr_text", "stderr"))) + exit_code = _result_attr(result, ("exit_code", "returncode", "status_code")) + if exit_code is None: + ok = _result_attr(result, ("ok", "success")) + exit_code = 0 if ok is True else 1 + if stdout and stderr and not stdout.endswith("\n"): + output = stdout + "\n" + stderr + else: + output = stdout + stderr + return output, int(exit_code) + + def _start_process( + self, + cmd_string: str, + *, + login: bool, + timeout: int, + stdin_data: str | None, + process_ref: dict[str, Any] | None = None, + ) -> tuple[str, int]: + self._ensure_sandbox() + flag = "-lc" if login else "-c" + start = getattr(self._sandbox, "start", None) + if not callable(start): + kwargs: dict[str, Any] = {"timeout": timeout, "env": self._sandbox_env()} + if stdin_data is not None: + kwargs["input"] = stdin_data + result = self._sandbox.exec("bash", flag, cmd_string, **kwargs) + return self._result_to_output(result) + + process = start( + "bash", + flag, + cmd_string, + timeout=timeout, + stdin=stdin_data, + env=self._sandbox_env(), + ) + if process_ref is not None: + process_ref["process"] = process + if stdin_data is None: + close_stdin = getattr(process, "close_stdin", None) + if callable(close_stdin): + close_stdin() + result = process.wait(timeout=timeout + 5 if timeout is not None else None) + return self._result_to_output(result) + + def _sudo_nopasswd_works(self) -> bool: + try: + _output, exit_code = self._exec_raw("sudo -n true", timeout=10) + except Exception: + return False + return exit_code == 0 + + def _prepare_command(self, command: str | None) -> tuple[str | None, str | None]: + if command is None: + return None, None + + # Tenki sandboxes should rely on their own sudoers policy. Do not ask + # the user for a host sudo password, and do not send SUDO_PASSWORD to a + # remote cloud sandbox. The default Tenki image supports NOPASSWD sudo. + transformed, sudo_count = _rewrite_sudo_noninteractive(command) + if sudo_count == 0: + return command, None + if self._sudo_nopasswd_works(): + return command, None + return transformed, None + + def _before_execute(self) -> None: + self._ensure_sandbox() + if self._sync_manager: + self._sync_manager.sync() + + def _run_bash( + self, + cmd_string: str, + *, + login: bool = False, + timeout: int = 120, + stdin_data: str | None = None, + ): + process_ref: dict[str, Any] = {} + + def cancel() -> None: + process = process_ref.get("process") + kill = getattr(process, "kill", None) + if callable(kill): + try: + kill() + return + except Exception: + pass + with self._lock: + sandbox = self._sandbox + # Drop our reference so the next command resumes (persistent) or + # recreates (ephemeral) a sandbox instead of reusing a torn-down + # one. + self._sandbox = None + if sandbox is None: + return + # For a persistent sandbox, pause (preserve the filesystem) instead of + # terminating: an interrupted or timed-out command must not destroy + # state the user asked to keep. The paused sandbox is re-discovered and + # resumed on the next command via _resume_persistent_sandbox(). + if self._persistent: + pause = getattr(sandbox, "pause", None) + if callable(pause): + try: + pause() + return + except Exception: + pass # fall through to terminate if pause is unavailable + for method_name in ("terminate", "close"): + method = getattr(sandbox, method_name, None) + if callable(method): + try: + method() + except Exception: + pass + return + + def exec_fn() -> tuple[str, int]: + return self._start_process( + cmd_string, + login=login, + timeout=timeout, + stdin_data=stdin_data, + process_ref=process_ref, + ) + + return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel) + + def cleanup(self): + with self._lock: + sandbox = self._sandbox + sync_manager = self._sync_manager + self._sync_manager = None + client = self._client + self._cleanup_in_progress = True + self._cleanup_sandbox = sandbox + if sandbox is None: + self._close_client(client) + with self._lock: + if self._client is client: + self._client = None + self._cleanup_in_progress = False + self._cleanup_sandbox = None + return + + try: + if sync_manager: + logger.info("Tenki: syncing files from sandbox...") + try: + sync_manager.sync_back() + except Exception as exc: + logger.warning("Tenki: sync_back failed: %s", exc) + + snapshot_saved = False + if self._persistent: + snapshot_saved = self._save_persistent_snapshot(sandbox) + + if self._persistent and not snapshot_saved: + # Persistent state was NOT durably snapshotted. Terminating now + # would destroy the only copy, so prefer pause; and if pause + # fails, still do NOT terminate — leave the sandbox live for a + # later recovery attempt (the max-duration / idle reaper bounds + # the cost). Terminating here would break the preservation + # guarantee that the durability gate exists to uphold. + pause = getattr(sandbox, "pause", None) + if callable(pause): + try: + pause() + logger.info("Tenki: paused sandbox for task %s", self._task_id) + except Exception as exc: + logger.warning( + "Tenki: pause failed for task %s; leaving sandbox live to " + "preserve un-snapshotted state (not terminating): %s", + self._task_id, exc, + ) + else: + logger.warning( + "Tenki: no durable snapshot and no pause support for task %s; " + "leaving sandbox live to preserve state (not terminating)", + self._task_id, + ) + return + + for method_name in ("terminate", "close"): + method = getattr(sandbox, method_name, None) + if not callable(method): + continue + try: + method() + logger.info("Tenki: terminated sandbox for task %s", self._task_id) + except Exception as exc: + logger.warning("Tenki: cleanup failed: %s", exc) + return + finally: + self._close_client(client) + with self._lock: + if self._sandbox is sandbox: + self._sandbox = None + if self._client is client: + self._client = None + self._cleanup_in_progress = False + self._cleanup_sandbox = None + + def _save_persistent_snapshot(self, sandbox: Any) -> bool: + snapshot_id: str | None = None + try: + snapshot = sandbox.snapshot(name=self._sandbox_name(), wait=True) + snapshot_id = getattr(snapshot, "id", None) or getattr(snapshot, "snapshot_id", None) + except Exception as exc: + logger.warning("Tenki: filesystem snapshot failed: %s", exc) + return False + if not snapshot_id: + logger.warning("Tenki: snapshot completed without an id; preserving paused sandbox instead") + return False + # snapshot(wait=True) only waits for READY; durability is a separate, + # required gate. If durability is not confirmed the snapshot may not be + # a safe recovery copy, so we must NOT record it as the persistent + # pointer or let the caller terminate the live sandbox. Return False so + # cleanup pauses the sandbox and the prior (known-durable) snapshot + # pointer is left intact for recovery. + if self._client is not None: + snapshots = getattr(self._client, "snapshots", None) + wait_durable = getattr(snapshots, "wait_durable", None) + if callable(wait_durable): + try: + wait_durable(snapshot_id, timeout=300) + except Exception as exc: + logger.warning( + "Tenki: snapshot %s for task %s did not reach durability (%s); " + "preserving paused sandbox and prior snapshot instead", + snapshot_id, self._task_id, exc, + ) + return False + _store_snapshot(self._task_id, snapshot_id, self._snapshot_store) + logger.info("Tenki: saved filesystem snapshot %s for task %s", snapshot_id, self._task_id) + return True + + @staticmethod + def _close_client(client: Any) -> None: + if client is None: + return + close = getattr(client, "close", None) + if callable(close): + close() diff --git a/tools/file_operations.py b/tools/file_operations.py index 76446befaa5e5..9741579dce1ac 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -3,7 +3,7 @@ File Operations Module Provides file manipulation capabilities (read, write, patch, search) that work -across all terminal backends (local, docker, ssh, singularity, modal, daytona). +across all terminal backends (local, docker, ssh, singularity, modal, daytona, tenki). The key insight is that all file operations can be expressed as shell commands, so we wrap the terminal backend's execute() interface to provide a unified file API. @@ -794,7 +794,7 @@ class ShellFileOperations(FileOperations): File operations implemented via shell commands. Works with ANY terminal backend that has execute(command, cwd) method. - This includes local, docker, singularity, ssh, modal, and daytona environments. + This includes local, docker, singularity, ssh, modal, daytona, and tenki environments. """ def __init__(self, terminal_env, cwd: str = None): @@ -1879,7 +1879,7 @@ def _lsp_local_only(self) -> bool: LSP servers run on the host process — they need access to the files they're linting. Remote/sandboxed backends (Docker, - Modal, SSH, Daytona) keep files inside the sandbox where the + Modal, SSH, Daytona, Tenki) keep files inside the sandbox where the host-side LSP server can't reach them, so we skip the LSP path for those entirely. """ diff --git a/tools/file_tools.py b/tools/file_tools.py index e602e8e0a6756..0312c145ce0f8 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -1052,7 +1052,9 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: _creation_locks_lock, _resolve_container_task_id, _is_unusable_container_cwd, + _is_backend_guest_subpath, _CONTAINER_BACKENDS, + _container_config_from_env_config, ) import time @@ -1110,6 +1112,8 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: image = overrides.get("modal_image") or config["modal_image"] elif env_type == "daytona": image = overrides.get("daytona_image") or config["daytona_image"] + elif env_type == "tenki": + image = overrides.get("tenki_image") or config["tenki_image"] else: image = "" @@ -1126,7 +1130,11 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: # bypass the guard. Valid in-container override paths (RL/benchmark # sandboxes that set cwd to /workspace, /root, etc.) are absolute # non-host paths and pass through untouched. - if env_type in _CONTAINER_BACKENDS and _is_unusable_container_cwd(cwd): + if ( + env_type in _CONTAINER_BACKENDS + and _is_unusable_container_cwd(cwd) + and not _is_backend_guest_subpath(env_type, cwd) + ): if cwd != config["cwd"]: logger.info( "Ignoring host/relative cwd override %r for %s backend " @@ -1137,18 +1145,8 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: logger.info("Creating new %s environment for task %s...", env_type, task_id[:8]) container_config = None - if env_type in {"docker", "singularity", "modal", "daytona"}: - container_config = { - "container_cpu": config.get("container_cpu", 1), - "container_memory": config.get("container_memory", 5120), - "container_disk": config.get("container_disk", 51200), - "container_persistent": config.get("container_persistent", True), - "docker_volumes": config.get("docker_volumes", []), - "docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False), - "docker_forward_env": config.get("docker_forward_env", []), - "docker_run_as_host_user": config.get("docker_run_as_host_user", False), - "docker_network": config.get("docker_network", True), - } + if env_type in _CONTAINER_BACKENDS: + container_config = _container_config_from_env_config(config) ssh_config = None if env_type == "ssh": diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index d35feb861cf5e..232ac714c21a7 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -204,6 +204,7 @@ # ─── Terminal backends ───────────────────────────────────────────────── "terminal.modal": ("modal==1.3.4",), "terminal.daytona": ("daytona==0.155.0",), + "terminal.tenki": ("tenki-sandbox==0.1.1",), # ─── Skills ──────────────────────────────────────────────────────────── "skill.google_workspace": ( diff --git a/tools/skills_tool.py b/tools/skills_tool.py index a5613f62c4c87..823a08cd48f7d 100644 --- a/tools/skills_tool.py +++ b/tools/skills_tool.py @@ -171,7 +171,7 @@ def _skills_dir() -> Path: } _ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _REMOTE_ENV_BACKENDS = frozenset( - {"docker", "singularity", "modal", "ssh", "daytona"} + {"docker", "singularity", "modal", "ssh", "daytona", "tenki"} ) _secret_capture_callback = None diff --git a/tools/tenki_config.py b/tools/tenki_config.py new file mode 100644 index 0000000000000..78f07771d9d8c --- /dev/null +++ b/tools/tenki_config.py @@ -0,0 +1,201 @@ +"""Helpers for reading Tenki CLI configuration without exposing secrets.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from utils import fast_safe_load + +TENKI_DEFAULT_API_ENDPOINT = "https://api.tenki.cloud" + +_SECRET_KEYS = frozenset({ + "auth_token", + "api_key", + "access_token", + "session_token", + "token", +}) + +_SDK_AUTH_PREFIXES = ("cookie:", "ory_st_", "sk-") + + +def tenki_cli_config_path() -> Path: + """Return the Tenki CLI config path. + + ``TENKI_CONFIG_PATH`` is honored for tests and uncommon CLI installs. + """ + override = os.getenv("TENKI_CONFIG_PATH") + if override: + return Path(override).expanduser() + return Path.home() / ".config" / "tenki" / "config.yaml" + + +def load_tenki_cli_config() -> dict[str, Any]: + """Load Tenki CLI config, returning ``{}`` on missing or invalid files.""" + path = tenki_cli_config_path() + try: + data = fast_safe_load(path.read_text(encoding="utf-8")) or {} + except OSError: + return {} + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def _string(value: Any) -> str: + return value.strip() if isinstance(value, str) else "" + + +def _scoped_env(name: str) -> str: + """Read a credential env var honoring the active profile secret scope. + + Under a multiplexed gateway turn a profile scope is installed, and the + token must come from *that* profile's secrets — never from a raw + ``os.environ`` read that could hold another profile's value. When no + multiplexing is active this behaves exactly like ``os.getenv``. + """ + try: + from agent.secret_scope import get_secret + + return _string(get_secret(name, "")) + except Exception: + # Fail closed: an unscoped read under active multiplexing (or any + # resolution error) must NOT silently leak a process-global value. + return "" + + +def _global_credential_fallback_allowed() -> bool: + """Whether machine-global credential sources (the shared Tenki CLI login) + may be consulted. + + Skipped whenever a profile secret scope is authoritative — a multiplexed + profile without its own Tenki token must not borrow the machine-global + ``tenki login`` credential that another profile may be relying on. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + return current_secret_scope() is None and not is_multiplex_active() + except Exception: + return True + + +def _first_string(data: dict[str, Any], keys: tuple[str, ...]) -> str: + for key in keys: + value = _string(data.get(key)) + if value: + return value + return "" + + +def _normalize_cli_auth_token(secret: str, key: str = "") -> str: + """Return a Tenki SDK-compatible auth token from Tenki CLI config. + + Tenki CLI v0.6 stores its browser session cookie as a bare ``auth_token``. + The Python SDK expects cookie credentials to be prefixed with ``cookie:``; + otherwise it sends the value as a bearer token and the API returns + ``sandbox: unauthorized``. + """ + secret = _string(secret) + if not secret or secret.startswith(_SDK_AUTH_PREFIXES): + return secret + if key.lower() == "auth_token": + return f"cookie:{secret}" + return secret + + +def _find_secret_value(data: Any) -> str: + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(key, str) and key.lower() in _SECRET_KEYS: + secret = _string(value) + if secret: + return _normalize_cli_auth_token(secret, key) + found = _find_secret_value(value) + if found: + return found + elif isinstance(data, list): + for item in data: + found = _find_secret_value(item) + if found: + return found + return "" + + +def resolve_tenki_api_endpoint(explicit: str = "") -> str: + """Resolve the Tenki API endpoint from config/env/CLI defaults. + + Scope-aware (see :func:`_scoped_env`): under a multiplexed profile turn the + active profile's setting wins, and the shared machine Tenki CLI config is + consulted only when no profile scope is authoritative. + """ + explicit = _string(explicit) + if explicit: + return explicit + for env_name in ("TENKI_API_ENDPOINT", "TENKI_API_URL"): + value = _scoped_env(env_name) + if value: + return value + if _global_credential_fallback_allowed(): + cfg = load_tenki_cli_config() + endpoint = _first_string(cfg, ("api_endpoint", "api_url", "endpoint")) + if endpoint: + return endpoint + return TENKI_DEFAULT_API_ENDPOINT + + +def resolve_tenki_workspace_id(explicit: str = "") -> str: + """Resolve the Tenki workspace id. Scope-aware; workspace/project decide + where sandboxes are created, so a multiplexed profile must not silently + borrow the machine-global workspace of another tenant.""" + explicit = _string(explicit) + if explicit: + return explicit + for env_name in ("TENKI_WORKSPACE_ID", "TENKI_WORKSPACE"): + value = _scoped_env(env_name) + if value: + return value + if not _global_credential_fallback_allowed(): + return "" + return _first_string(load_tenki_cli_config(), ("current_workspace_id", "workspace_id", "workspace")) + + +def resolve_tenki_project_id(explicit: str = "") -> str: + """Resolve the Tenki project id. Scope-aware for the same reason as + :func:`resolve_tenki_workspace_id`.""" + explicit = _string(explicit) + if explicit: + return explicit + for env_name in ("TENKI_PROJECT_ID", "TENKI_PROJECT"): + value = _scoped_env(env_name) + if value: + return value + if not _global_credential_fallback_allowed(): + return "" + return _first_string(load_tenki_cli_config(), ("current_project_id", "project_id", "project")) + + +def resolve_tenki_auth_token(explicit: str = "") -> str: + """Resolve a Tenki auth token/API key without logging or persisting it. + + Reads are profile-scope-aware (see :func:`_scoped_env`): under a + multiplexed gateway turn the active profile's secrets win, and the shared + machine ``tenki login`` credential is consulted only when no profile scope + is authoritative. + """ + explicit = _string(explicit) + if explicit: + return explicit + for env_name in ("TENKI_AUTH_TOKEN", "TENKI_API_KEY"): + value = _scoped_env(env_name) + if value: + return value + if not _global_credential_fallback_allowed(): + return "" + return _find_secret_value(load_tenki_cli_config()) + + +def has_tenki_auth() -> bool: + return bool(resolve_tenki_auth_token()) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index fc13367eb116c..e788538e459e0 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -3,16 +3,17 @@ Terminal Tool Module A terminal tool that executes commands in local, Docker, Modal, SSH, -Singularity, and Daytona environments. Supports local execution, +Singularity, Daytona, and Tenki environments. Supports local execution, containerized backends, and cloud sandboxes, including managed Modal mode. Supported environments: - "local": Execute directly on the host machine (default, fastest) - "docker": Execute in Docker containers (isolated, requires Docker) - "modal": Execute in Modal cloud sandboxes (direct Modal or managed gateway) +- "tenki": Execute in Tenki cloud sandboxes Features: -- Multiple execution backends (local, docker, modal) +- Multiple execution backends (local, docker, modal, tenki, etc.) - Background task support - VM/container lifecycle management - Automatic cleanup after inactivity @@ -1060,7 +1061,7 @@ def _maybe_reap_docker_orphans(container_config: Dict[str, Any]) -> None: # Per-task environment overrides registry. -# Allows environments (e.g., TerminalBench2Env) to specify a custom Docker/Modal +# Allows environments (e.g., TerminalBench2Env) to specify a custom Docker/Modal/Tenki # image for a specific task_id BEFORE the agent loop starts. When the terminal or # file tools create a new sandbox for that task_id, they check this registry first # and fall back to the TERMINAL_MODAL_IMAGE (etc.) env var if no override is set. @@ -1079,6 +1080,7 @@ def register_task_env_overrides(task_id: str, overrides: Dict[str, Any]): Supported override keys: - modal_image: str -- Path to Dockerfile or Docker Hub image name + - tenki_image: str -- Tenki sandbox image/template identifier - docker_image: str -- Docker image name - cwd: str -- Working directory inside the sandbox @@ -1146,7 +1148,7 @@ def _resolve_container_task_id(task_id: Optional[str]) -> str: """ _ISOLATION_KEYS = frozenset({ "docker_image", "modal_image", "singularity_image", - "daytona_image", "env_type", + "daytona_image", "tenki_image", "env_type", }) if task_id and task_id in _task_env_overrides: overrides = _task_env_overrides[task_id] @@ -1213,7 +1215,20 @@ def _safe_getcwd() -> str: # cwd looks when it leaks toward a Linux container's ``-w`` flag. _HOST_CWD_PREFIXES = ("/Users/", "/home/", "C:\\", "C:/") -_CONTAINER_BACKENDS = frozenset({"docker", "singularity", "modal", "daytona"}) +_CONTAINER_BACKENDS = frozenset({"docker", "singularity", "modal", "daytona", "tenki"}) + +# Guest-home roots whose subtree is a valid container cwd even though the root +# shares a host-looking prefix. Tenki's guest home is /home/tenki, so +# /home/tenki/project is a real sandbox path, not a host path to discard. +_CONTAINER_GUEST_HOME_ROOTS = {"tenki": "/home/tenki"} + + +def _is_backend_guest_subpath(env_type: str, cwd: str) -> bool: + """True when *cwd* is the backend's guest-home root or a path beneath it.""" + root = _CONTAINER_GUEST_HOME_ROOTS.get(env_type) + if not root or not cwd: + return False + return cwd == root or cwd.startswith(root.rstrip("/") + "/") def _is_ssh_remote_tilde_cwd(backend: str, cwd: str) -> bool: @@ -1260,7 +1275,7 @@ def _get_env_config() -> Dict[str, Any]: env_type = os.getenv("TERMINAL_ENV", "local") mount_docker_cwd = os.getenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower() in {"true", "1", "yes"} - container_backend = env_type in {"docker", "singularity", "modal", "daytona"} + container_backend = env_type in _CONTAINER_BACKENDS docker_backend = env_type == "docker" # Docker/container-only env vars may be bridged from config.yaml even when @@ -1287,6 +1302,22 @@ def _get_env_config() -> Dict[str, Any]: docker_env = {} docker_extra_args = [] + # Tenki settings may be bridged from config.yaml even when the active + # backend is local/ssh. Do not parse their numeric/JSON payloads until the + # tenki backend is selected; a stale or invalid value must not make the + # local terminal unusable (mirrors the container_/docker_ guards above). + tenki_backend = env_type == "tenki" + if tenki_backend: + tenki_forward_env = _parse_env_var("TERMINAL_TENKI_FORWARD_ENV", "[]", json.loads, "valid JSON") + tenki_max_duration = _parse_env_var("TERMINAL_TENKI_MAX_DURATION", "3600") + tenki_idle_timeout = _parse_env_var("TERMINAL_TENKI_IDLE_TIMEOUT", "0") + tenki_pause_retention = _parse_env_var("TERMINAL_TENKI_PAUSE_RETENTION", "0") + else: + tenki_forward_env = [] + tenki_max_duration = 3600 + tenki_idle_timeout = 0 + tenki_pause_retention = 0 + # Default cwd: local uses the host's current directory, ssh uses the # remote home, and everything else starts in the backend's default # root-like cwd. @@ -1294,6 +1325,8 @@ def _get_env_config() -> Dict[str, Any]: default_cwd = _safe_getcwd() elif env_type == "ssh": default_cwd = "~" + elif env_type == "tenki": + default_cwd = "/home/tenki" else: default_cwd = "/root" @@ -1315,8 +1348,10 @@ def _get_env_config() -> Dict[str, Any]: host_cwd = candidate cwd = "/workspace" elif env_type in _CONTAINER_BACKENDS and cwd: - # Host paths and relative paths that won't work inside containers - if _is_unusable_container_cwd(cwd) and cwd != default_cwd: + # Host paths and relative paths that won't work inside containers. A + # path inside the backend's own guest-home subtree is valid even though + # it may share a host-looking prefix (see _is_backend_guest_subpath). + if _is_unusable_container_cwd(cwd) and not _is_backend_guest_subpath(env_type, cwd): logger.info("Ignoring TERMINAL_CWD=%r for %s backend " "(host/relative path won't work in sandbox). Using %r instead.", cwd, env_type, default_cwd) @@ -1330,6 +1365,18 @@ def _get_env_config() -> Dict[str, Any]: "singularity_image": os.getenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"), "modal_image": os.getenv("TERMINAL_MODAL_IMAGE", default_image), "daytona_image": os.getenv("TERMINAL_DAYTONA_IMAGE", default_image), + "tenki_image": os.getenv("TERMINAL_TENKI_IMAGE", ""), + "tenki_api_endpoint": os.getenv("TERMINAL_TENKI_API_ENDPOINT", ""), + "tenki_workspace_id": os.getenv("TERMINAL_TENKI_WORKSPACE_ID", ""), + "tenki_project_id": os.getenv("TERMINAL_TENKI_PROJECT_ID", ""), + "tenki_name_prefix": os.getenv("TERMINAL_TENKI_NAME_PREFIX", "hermes"), + "tenki_allow_inbound": os.getenv("TERMINAL_TENKI_ALLOW_INBOUND", "false").lower() in {"true", "1", "yes"}, + "tenki_allow_outbound": os.getenv("TERMINAL_TENKI_ALLOW_OUTBOUND", "true").lower() in {"true", "1", "yes"}, + "tenki_max_duration": tenki_max_duration, + "tenki_idle_timeout": tenki_idle_timeout, + "tenki_pause_retention": tenki_pause_retention, + "tenki_sync_hermes_home": os.getenv("TERMINAL_TENKI_SYNC_HERMES_HOME", "false").lower() in {"true", "1", "yes"}, + "tenki_forward_env": tenki_forward_env, "cwd": cwd, "host_cwd": host_cwd, "docker_mount_cwd_to_workspace": mount_docker_cwd, @@ -1349,11 +1396,14 @@ def _get_env_config() -> Dict[str, Any]: ).lower() in {"true", "1", "yes"}, "local_persistent": os.getenv("TERMINAL_LOCAL_PERSISTENT", "false").lower() in {"true", "1", "yes"}, # Container resource config (applies to docker, singularity, modal, - # daytona -- ignored for local/ssh) + # daytona, tenki -- ignored for local/ssh) "container_cpu": container_cpu, "container_memory": container_memory, # MB (default 5GB) "container_disk": container_disk, # MB (default 50GB) - "container_persistent": os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() in {"true", "1", "yes"}, + "container_persistent": os.getenv( + "TERMINAL_CONTAINER_PERSISTENT", + "false" if env_type == "tenki" else "true", + ).lower() in {"true", "1", "yes"}, "docker_volumes": docker_volumes, "docker_env": docker_env, "docker_run_as_host_user": os.getenv("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false").lower() in {"true", "1", "yes"}, @@ -1387,6 +1437,37 @@ def _get_modal_backend_state(modal_mode: object | None) -> Dict[str, Any]: ) +def _container_config_from_env_config(config: Dict[str, Any]) -> Dict[str, Any]: + """Build the shared container-backend config passed to _create_environment.""" + return { + "container_cpu": config.get("container_cpu", 1), + "container_memory": config.get("container_memory", 5120), + "container_disk": config.get("container_disk", 51200), + "container_persistent": config.get("container_persistent", True), + "modal_mode": config.get("modal_mode", "auto"), + "docker_volumes": config.get("docker_volumes", []), + "docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False), + "docker_forward_env": config.get("docker_forward_env", []), + "docker_env": config.get("docker_env", {}), + "docker_run_as_host_user": config.get("docker_run_as_host_user", False), + "docker_extra_args": config.get("docker_extra_args", []), + "docker_network": config.get("docker_network", True), + "docker_persist_across_processes": config.get("docker_persist_across_processes", True), + "docker_orphan_reaper": config.get("docker_orphan_reaper", True), + "tenki_api_endpoint": config.get("tenki_api_endpoint", ""), + "tenki_workspace_id": config.get("tenki_workspace_id", ""), + "tenki_project_id": config.get("tenki_project_id", ""), + "tenki_name_prefix": config.get("tenki_name_prefix", "hermes"), + "tenki_allow_inbound": config.get("tenki_allow_inbound", False), + "tenki_allow_outbound": config.get("tenki_allow_outbound", True), + "tenki_max_duration": config.get("tenki_max_duration", 3600), + "tenki_idle_timeout": config.get("tenki_idle_timeout", 0), + "tenki_pause_retention": config.get("tenki_pause_retention", 0), + "tenki_sync_hermes_home": config.get("tenki_sync_hermes_home", False), + "tenki_forward_env": config.get("tenki_forward_env", []), + } + + def _create_environment(env_type: str, image: str, cwd: str, timeout: int, ssh_config: dict = None, container_config: dict = None, local_config: dict = None, @@ -1397,7 +1478,7 @@ def _create_environment(env_type: str, image: str, cwd: str, timeout: int, Args: env_type: One of "local", "docker", "singularity", "modal", - "daytona", "ssh" + "daytona", "tenki", "ssh" image: Docker/Singularity/Modal image name (ignored for local/ssh) cwd: Working directory timeout: Default command timeout @@ -1520,6 +1601,31 @@ def _create_environment(env_type: str, image: str, cwd: str, timeout: int, persistent_filesystem=persistent, task_id=task_id, ) + elif env_type == "tenki": + from tools.environments.tenki import TenkiEnvironment as _TenkiEnvironment + + return _TenkiEnvironment( + image=image, + cwd=cwd, + timeout=timeout, + cpu=cpu, + memory=memory, + disk=disk, + persistent_filesystem=persistent, + task_id=task_id, + api_endpoint=cc.get("tenki_api_endpoint", ""), + workspace_id=cc.get("tenki_workspace_id", ""), + project_id=cc.get("tenki_project_id", ""), + name_prefix=cc.get("tenki_name_prefix", "hermes"), + allow_inbound=cc.get("tenki_allow_inbound", False), + allow_outbound=cc.get("tenki_allow_outbound", True), + max_duration=cc.get("tenki_max_duration", 3600), + idle_timeout=cc.get("tenki_idle_timeout", 0), + pause_retention=cc.get("tenki_pause_retention", 0), + sync_hermes_home=cc.get("tenki_sync_hermes_home", False), + forward_env=cc.get("tenki_forward_env", []), + ) + elif env_type == "ssh": if not ssh_config or not ssh_config.get("host") or not ssh_config.get("user"): raise ValueError("SSH environment requires ssh_host and ssh_user to be configured") @@ -1535,7 +1641,7 @@ def _create_environment(env_type: str, image: str, cwd: str, timeout: int, else: raise ValueError( f"Unknown environment type: {env_type}. Use 'local', 'docker', " - f"'singularity', 'modal', 'daytona', or 'ssh'" + f"'singularity', 'modal', 'daytona', 'tenki', or 'ssh'" ) @@ -2090,6 +2196,8 @@ def terminal_tool( image = overrides.get("modal_image") or config["modal_image"] elif env_type == "daytona": image = overrides.get("daytona_image") or config["daytona_image"] + elif env_type == "tenki": + image = overrides.get("tenki_image") or config["tenki_image"] else: image = "" @@ -2105,7 +2213,11 @@ def terminal_tool( # Valid in-container override paths (RL/benchmark sandboxes that set # cwd to /workspace, /root, etc.) are absolute non-host paths and pass # through untouched. - if env_type in _CONTAINER_BACKENDS and _is_unusable_container_cwd(cwd): + if ( + env_type in _CONTAINER_BACKENDS + and _is_unusable_container_cwd(cwd) + and not _is_backend_guest_subpath(env_type, cwd) + ): if cwd != config["cwd"]: logger.info( "Ignoring host/relative cwd override %r for %s backend " @@ -2198,23 +2310,8 @@ def terminal_tool( } container_config = None - if env_type in {"docker", "singularity", "modal", "daytona"}: - container_config = { - "container_cpu": config.get("container_cpu", 1), - "container_memory": config.get("container_memory", 5120), - "container_disk": config.get("container_disk", 51200), - "container_persistent": config.get("container_persistent", True), - "modal_mode": config.get("modal_mode", "auto"), - "docker_volumes": config.get("docker_volumes", []), - "docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False), - "docker_forward_env": config.get("docker_forward_env", []), - "docker_env": config.get("docker_env", {}), - "docker_run_as_host_user": config.get("docker_run_as_host_user", False), - "docker_extra_args": config.get("docker_extra_args", []), - "docker_network": config.get("docker_network", True), - "docker_persist_across_processes": config.get("docker_persist_across_processes", True), - "docker_orphan_reaper": config.get("docker_orphan_reaper", True), - } + if env_type in _CONTAINER_BACKENDS: + container_config = _container_config_from_env_config(config) local_config = None if env_type == "local": @@ -2897,10 +2994,42 @@ def check_terminal_requirements() -> bool: from daytona import Daytona # noqa: F401 — SDK presence check return os.getenv("DAYTONA_API_KEY") is not None + elif env_type == "tenki": + if importlib.util.find_spec("tenki_sandbox") is None: + try: + from tools.lazy_deps import ensure as _lazy_ensure + + _lazy_ensure("terminal.tenki", prompt=False) + importlib.invalidate_caches() + except Exception as exc: + logger.error( + "tenki-sandbox is required for Tenki terminal backend: " + "pip install tenki-sandbox==0.1.1 (%s)", + exc, + ) + return False + if importlib.util.find_spec("tenki_sandbox") is None: + logger.error( + "tenki-sandbox is required for Tenki terminal backend: " + "pip install tenki-sandbox==0.1.1" + ) + return False + try: + from tools.tenki_config import has_tenki_auth + except Exception: + has_tenki_auth = lambda: False # noqa: E731 + if not has_tenki_auth(): + logger.error( + "Tenki backend selected but no Tenki auth was found. Run `tenki login` " + "or set TENKI_AUTH_TOKEN/TENKI_API_KEY." + ) + return False + return True + else: logger.error( "Unknown TERMINAL_ENV '%s'. Use one of: local, docker, singularity, " - "modal, daytona, ssh.", + "modal, daytona, tenki, ssh.", env_type, ) return False @@ -2919,6 +3048,7 @@ def check_terminal_requirements() -> bool: print(f" Environment type: {config['env_type']}") print(f" Docker image: {config['docker_image']}") print(f" Modal image: {config['modal_image']}") + print(f" Tenki image: {config['tenki_image'] or '(Tenki default)'}") print(f" Working directory: {config['cwd']}") print(f" Default timeout: {config['timeout']}s") print(f" Lifetime: {config['lifetime_seconds']}s") @@ -2943,12 +3073,13 @@ def check_terminal_requirements() -> bool: print( " TERMINAL_ENV: " f"{os.getenv('TERMINAL_ENV', 'local')} " - "(local/docker/singularity/modal/daytona/ssh)" + "(local/docker/singularity/modal/daytona/tenki/ssh)" ) print(f" TERMINAL_DOCKER_IMAGE: {os.getenv('TERMINAL_DOCKER_IMAGE', default_img)}") print(f" TERMINAL_SINGULARITY_IMAGE: {os.getenv('TERMINAL_SINGULARITY_IMAGE', f'docker://{default_img}')}") print(f" TERMINAL_MODAL_IMAGE: {os.getenv('TERMINAL_MODAL_IMAGE', default_img)}") print(f" TERMINAL_DAYTONA_IMAGE: {os.getenv('TERMINAL_DAYTONA_IMAGE', default_img)}") + print(f" TERMINAL_TENKI_IMAGE: {os.getenv('TERMINAL_TENKI_IMAGE', '(Tenki default)')}") print(f" TERMINAL_CWD: {os.getenv('TERMINAL_CWD', _safe_getcwd())}") from hermes_constants import display_hermes_home as _dhh print(f" TERMINAL_SANDBOX_DIR: {os.getenv('TERMINAL_SANDBOX_DIR', f'{_dhh()}/sandboxes')}") diff --git a/tools/tool_result_storage.py b/tools/tool_result_storage.py index b9ceccf75b433..fc1bd1afee880 100644 --- a/tools/tool_result_storage.py +++ b/tools/tool_result_storage.py @@ -152,7 +152,7 @@ def maybe_persist_tool_result( """Layer 2: persist oversized result into the sandbox, return preview + path. Writes via env.execute() so the file is accessible from any backend - (local, Docker, SSH, Modal, Daytona). Falls back to inline truncation + (local, Docker, SSH, Modal, Daytona, Tenki). Falls back to inline truncation if write fails or no env is available. Args: diff --git a/uv.lock b/uv.lock index 21bd0827c77ec..07dde5e690152 100644 --- a/uv.lock +++ b/uv.lock @@ -1683,6 +1683,9 @@ teams = [ { name = "aiohttp" }, { name = "microsoft-teams-apps" }, ] +tenki = [ + { name = "tenki-sandbox" }, +] termux = [ { name = "agent-client-protocol" }, { name = "honcho-ai" }, @@ -1845,6 +1848,7 @@ requires-dist = [ { name = "starlette", marker = "extra == 'web'", specifier = "==1.0.1" }, { name = "supermemory", marker = "extra == 'supermemory'", specifier = "==3.50.0" }, { name = "tenacity", specifier = "==9.1.4" }, + { name = "tenki-sandbox", marker = "extra == 'tenki'", specifier = "==0.1.1" }, { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.21" }, { name = "tzdata", marker = "sys_platform == 'win32'", specifier = "==2025.3" }, { name = "urllib3", specifier = ">=2.7.0,<3" }, @@ -1853,7 +1857,7 @@ requires-dist = [ { name = "websockets", specifier = "==15.0.1" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "tenki", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [[package]] name = "hf-xet" @@ -4061,6 +4065,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "tenki-sandbox" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/7f/6e59a084acece58d2349c80d95996cfc4971dcbc61038d6cc9a822edf3dc/tenki_sandbox-0.1.1.tar.gz", hash = "sha256:2748d3cb381c1f955163b368189899517aa04f70e59d64fb06a999f400dfb0b6", size = 107483, upload-time = "2026-06-10T09:23:28.337Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/6b/9e3a0eaf5064d47e3ec1115ba9585d117feb5507312e195d15ee9a9bba28/tenki_sandbox-0.1.1-py3-none-any.whl", hash = "sha256:826ab21db80b3ebe74c9a67f3eeb09f8d880535677f8d5dc3d03ad402210f97c", size = 94883, upload-time = "2026-06-10T09:23:26.958Z" }, +] + [[package]] name = "termcolor" version = "3.3.0" @@ -4395,6 +4413,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "15.0.1" diff --git a/website/docs/developer-guide/architecture.md b/website/docs/developer-guide/architecture.md index 6d6ec8695ae47..6ec69cb6f14b3 100644 --- a/website/docs/developer-guide/architecture.md +++ b/website/docs/developer-guide/architecture.md @@ -40,7 +40,7 @@ This page is the top-level map of Hermes Agent internals. Use it to orient yours ▼ ▼ ┌───────────────────┐ ┌──────────────────────┐ │ Session Storage │ │ Tool Backends │ -│ (SQLite + FTS5) │ │ Terminal (6 backends) │ +│ (SQLite + FTS5) │ │ Terminal (7 backends) │ │ hermes_state.py │ │ Browser (5 backends) │ │ gateway/session.py│ │ Web (4 backends) │ └───────────────────┘ │ MCP (dynamic) │ @@ -106,7 +106,7 @@ hermes-agent/ │ ├── credential_files.py # File-based credential passthrough │ ├── env_passthrough.py # Env var passthrough for sandboxes │ ├── ansi_strip.py # ANSI escape stripping -│ └── environments/ # Terminal backends (local, docker, ssh, modal, daytona, singularity) +│ └── environments/ # Terminal backends (local, docker, ssh, modal, daytona, tenki, singularity) │ ├── gateway/ # Messaging platform gateway │ ├── run.py # GatewayRunner — message dispatch (large file) @@ -211,7 +211,7 @@ A shared runtime resolver used by CLI, gateway, cron, ACP, and auxiliary calls. ### Tool System -Central tool registry (`tools/registry.py`) with 70+ registered tools across ~28 toolsets. Each tool file self-registers at import time. The registry handles schema collection, dispatch, availability checking, and error wrapping. Terminal tools support 6 backends (local, Docker, SSH, Daytona, Modal, Singularity). +Central tool registry (`tools/registry.py`) with 70+ registered tools across ~28 toolsets. Each tool file self-registers at import time. The registry handles schema collection, dispatch, availability checking, and error wrapping. Terminal tools support 7 backends (local, Docker, SSH, Daytona, Tenki, Modal, Singularity). → [Tools Runtime](./tools-runtime.md) diff --git a/website/docs/getting-started/nix-setup.md b/website/docs/getting-started/nix-setup.md index 17d34883c9e66..8d3d364924e49 100644 --- a/website/docs/getting-started/nix-setup.md +++ b/website/docs/getting-started/nix-setup.md @@ -700,6 +700,7 @@ This is resolved by uv alongside core dependencies — no PYTHONPATH patching, n | `hindsight` | Hindsight memory provider | | `modal` | Modal terminal backend | | `daytona` | Daytona terminal backend | +| `tenki` | Tenki terminal backend | | `exa` | Exa web search | | `firecrawl` | Firecrawl web search | | `fal` | FAL image generation | diff --git a/website/docs/guides/tips.md b/website/docs/guides/tips.md index fd4501080a0d5..4c792c9c93366 100644 --- a/website/docs/guides/tips.md +++ b/website/docs/guides/tips.md @@ -181,7 +181,7 @@ By default, messaging sessions never auto-reset — context lives until you `/re ### Use Docker for Untrusted Code -When working with untrusted repositories or running unfamiliar code, use Docker or Daytona as your terminal backend. Set `TERMINAL_BACKEND=docker` in your `.env`. Destructive commands inside a container can't harm your host system. +When working with untrusted repositories or running unfamiliar code, use Docker, Daytona, or Tenki as your terminal backend. Set `TERMINAL_BACKEND=docker` in your `.env`. Destructive commands inside a container can't harm your host system. ```bash # In your .env: @@ -217,7 +217,7 @@ When the agent triggers a dangerous command approval (`rm -rf`, `DROP TABLE`, et Hermes checks every command against a curated list of dangerous patterns before execution. This includes recursive deletes, SQL drops, piping curl to shell, and more. Don't disable this in production — it exists for good reasons. :::warning -When running in a container backend (Docker, Singularity, Modal, Daytona), dangerous command checks are **skipped** because the container is the security boundary. Make sure your container images are properly locked down. +When running in a container backend (Docker, Singularity, Modal, Daytona, Tenki), dangerous command checks are **skipped** because the container is the security boundary. Make sure your container images are properly locked down. ::: ### Use Allowlists for Messaging Bots diff --git a/website/docs/index.mdx b/website/docs/index.mdx index f7a943cba9f73..f8d07e24c2727 100644 --- a/website/docs/index.mdx +++ b/website/docs/index.mdx @@ -121,7 +121,7 @@ It's not a coding copilot tethered to an IDE or a chatbot wrapper around a singl ## Key Features - **A closed learning loop** — Agent-curated memory with periodic nudges, autonomous skill creation, skill self-improvement during use, FTS5 cross-session recall with LLM summarization, and [Honcho](https://github.com/plastic-labs/honcho) dialectic user modeling -- **Runs anywhere, not just your laptop** — 6 terminal backends: local, Docker, SSH, Daytona, Singularity, Modal. Daytona and Modal offer serverless persistence — your environment hibernates when idle, costing nearly nothing +- **Runs anywhere, not just your laptop** — 7 terminal backends: local, Docker, SSH, Daytona, Tenki, Singularity, Modal. Cloud backends let your agent run isolated compute away from your host - **Lives where you do** — CLI, Telegram, Discord, Slack, WhatsApp, Signal, Matrix, Mattermost, Email, SMS, DingTalk, Feishu, WeCom, Weixin, QQ Bot, Yuanbao, BlueBubbles, Home Assistant, Microsoft Teams, Google Chat, and more — 20+ platforms from one gateway - **Built by model trainers** — Created by [Nous Research](https://nousresearch.com), the lab behind Hermes, Nomos, and Psyche. Works with [Nous Portal](https://portal.nousresearch.com), [OpenRouter](https://openrouter.ai), OpenAI, or any endpoint - **Scheduled automations** — Built-in cron with delivery to any platform diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 53f7a18a0bedc..97d7470ff8cdf 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -162,6 +162,11 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | `HINDSIGHT_TIMEOUT` | Timeout in seconds for Hindsight memory-provider API calls (default: `60`). Bump this if your Hindsight instance is slow to respond during `/sync` or `on_session_switch` and you're seeing timeouts in `errors.log`. | | `SUPERMEMORY_API_KEY` | Semantic long-term memory with profile recall and session ingest ([supermemory.ai](https://supermemory.ai)) | | `DAYTONA_API_KEY` | Daytona cloud sandboxes ([daytona.io](https://daytona.io/)) | +| `TENKI_AUTH_TOKEN` / `TENKI_API_KEY` | Tenki cloud sandboxes ([tenki.cloud](https://tenki.cloud)); alternatively run `tenki login` | +| `TENKI_CONFIG_PATH` | Override the Tenki CLI config path Hermes reads for auth, workspace, and project defaults | +| `TENKI_API_ENDPOINT` / `TENKI_API_URL` | Direct Tenki API endpoint override used when terminal config is blank | +| `TENKI_WORKSPACE_ID` / `TENKI_WORKSPACE` | Direct Tenki workspace ID override used when terminal config is blank | +| `TENKI_PROJECT_ID` / `TENKI_PROJECT` | Direct Tenki project ID override used when terminal config is blank | ### Skill API Keys @@ -205,7 +210,7 @@ These variables configure the [Tool Gateway](/user-guide/features/tool-gateway) | Variable | Description | |----------|-------------| -| `TERMINAL_ENV` | Backend: `local`, `docker`, `ssh`, `singularity`, `modal`, `daytona` | +| `TERMINAL_ENV` | Backend: `local`, `docker`, `ssh`, `singularity`, `modal`, `daytona`, `tenki` | | `HERMES_DOCKER_BINARY` | Override the container binary Hermes shells out to (e.g. `podman`, `/usr/local/bin/docker`). When unset, Hermes auto-discovers `docker` or `podman` on `PATH`. Needed when both are installed and you want the non-default, or when the binary lives outside `PATH`. | | `TERMINAL_DOCKER_IMAGE` | Docker image (default: `nikolaik/python-nodejs:python3.11-nodejs20`) | | `TERMINAL_DOCKER_FORWARD_ENV` | JSON array of env var names to explicitly forward into Docker terminal sessions. Note: skill-declared `required_environment_variables` are forwarded automatically — you only need this for vars not declared by any skill. | @@ -214,6 +219,18 @@ These variables configure the [Tool Gateway](/user-guide/features/tool-gateway) | `TERMINAL_SINGULARITY_IMAGE` | Singularity image or `.sif` path | | `TERMINAL_MODAL_IMAGE` | Modal container image | | `TERMINAL_DAYTONA_IMAGE` | Daytona sandbox image | +| `TERMINAL_TENKI_IMAGE` | Optional Tenki sandbox image/template; blank uses Tenki default | +| `TERMINAL_TENKI_API_ENDPOINT` | Tenki API endpoint (default: `https://api.tenki.cloud`) | +| `TERMINAL_TENKI_WORKSPACE_ID` | Tenki workspace ID; blank falls back to Tenki CLI config | +| `TERMINAL_TENKI_PROJECT_ID` | Tenki project ID; blank falls back to Tenki CLI config | +| `TERMINAL_TENKI_NAME_PREFIX` | Prefix for Hermes-created Tenki sandbox names (default: `hermes`) | +| `TERMINAL_TENKI_ALLOW_INBOUND` | Allow inbound network access in Tenki sandboxes (`true`/`false`, default: `false`) | +| `TERMINAL_TENKI_ALLOW_OUTBOUND` | Allow outbound network access in Tenki sandboxes (`true`/`false`, default: `true`) | +| `TERMINAL_TENKI_MAX_DURATION` | Tenki sandbox maximum duration in seconds (default: `3600`) | +| `TERMINAL_TENKI_IDLE_TIMEOUT` | Tenki sandbox idle timeout in seconds; converted to minutes for the SDK (`0` disables explicit idle timeout) | +| `TERMINAL_TENKI_PAUSE_RETENTION` | Tenki pause retention duration in seconds (`0` uses Tenki default) | +| `TERMINAL_TENKI_SYNC_HERMES_HOME` | Opt-in sync of selected `~/.hermes` credentials, skills, and cache files into Tenki sandboxes (`true`/`false`, default: `false`) | +| `TERMINAL_TENKI_FORWARD_ENV` | JSON array of env var names to explicitly forward into Tenki sandboxes. Use this for task credentials such as `GITHUB_TOKEN`; values are resolved from the shell first, then `~/.hermes/.env`. | | `TERMINAL_TIMEOUT` | Command timeout in seconds | | `TERMINAL_LIFETIME_SECONDS` | Max lifetime for terminal sessions in seconds | | `TERMINAL_CWD` | Deprecated direct override for gateway/cron terminal sessions. Prefer `terminal.cwd` in `config.yaml`; CLI still uses the launch directory. | @@ -231,14 +248,14 @@ For cloud sandbox backends, persistence is filesystem-oriented. `TERMINAL_LIFETI | `TERMINAL_SSH_KEY` | Path to private key | | `TERMINAL_SSH_PERSISTENT` | Override persistent shell for SSH (default: follows `TERMINAL_PERSISTENT_SHELL`) | -## Container Resources (Docker, Singularity, Modal, Daytona) +## Container Resources (Docker, Singularity, Modal, Daytona, Tenki) | Variable | Description | |----------|-------------| | `TERMINAL_CONTAINER_CPU` | CPU cores (default: 1) | | `TERMINAL_CONTAINER_MEMORY` | Memory in MB (default: 5120) | | `TERMINAL_CONTAINER_DISK` | Disk in MB (default: 51200) | -| `TERMINAL_CONTAINER_PERSISTENT` | Persist container filesystem across sessions (default: `true`) | +| `TERMINAL_CONTAINER_PERSISTENT` | Persist container filesystem across sessions (default: `true`; Tenki defaults to `false` unless explicitly set) | | `TERMINAL_SANDBOX_DIR` | Host directory for workspaces and overlays (default: `~/.hermes/sandboxes/`) | ## Persistent Shell diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index e65399a67d853..e2e00b7a03200 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -108,11 +108,11 @@ Before that stash step, Hermes also restores tracked `package-lock.json` diffs l ## Terminal Backend Configuration -Hermes supports six terminal backends. Each determines where the agent's shell commands actually execute — your local machine, a Docker container, a remote server via SSH, a Modal cloud sandbox (direct or via the Nous-managed gateway), a Daytona workspace, or a Singularity/Apptainer container. +Hermes supports seven terminal backends. Each determines where the agent's shell commands actually execute — your local machine, a Docker container, a remote server via SSH, a Modal cloud sandbox (direct or via the Nous-managed gateway), a Daytona workspace, a Tenki sandbox, or a Singularity/Apptainer container. ```yaml terminal: - backend: local # local | docker | ssh | modal | daytona | singularity + backend: local # local | docker | ssh | modal | daytona | tenki | singularity cwd: "." # Gateway/cron working directory (CLI always uses launch dir) timeout: 180 # Per-command timeout in seconds home_mode: auto # auto | real | profile — subprocess HOME policy @@ -120,9 +120,15 @@ terminal: singularity_image: "docker://nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Singularity backend modal_image: "nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Modal backend daytona_image: "nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Daytona backend + tenki_image: "" # Optional Tenki image/template; blank uses Tenki default + tenki_api_endpoint: "https://api.tenki.cloud" + tenki_workspace_id: "" # Blank falls back to Tenki CLI config + tenki_project_id: "" # Blank falls back to Tenki CLI config + tenki_sync_hermes_home: false # Opt-in sync of selected ~/.hermes files + tenki_forward_env: [] # Explicit host env vars to forward into Tenki ``` -For cloud sandboxes such as Modal and Daytona, `container_persistent: true` means Hermes will try to preserve filesystem state across sandbox recreation. It does not promise that the same live sandbox, PID space, or background processes will still be running later. +For cloud sandboxes such as Modal, Daytona, and Tenki, `container_persistent: true` means Hermes will try to preserve filesystem state across sandbox recreation. It does not promise that the same live sandbox, PID space, or background processes will still be running later. Tenki defaults to `container_persistent: false` so sandboxes are terminated when Hermes cleans them up; when persistence is enabled, Hermes pauses and later resumes the matching Tenki sandbox. ### Backend Overview @@ -133,6 +139,7 @@ For cloud sandboxes such as Modal and Daytona, `container_persistent: true` mean | **ssh** | Remote server via SSH | Network boundary | Remote dev, powerful hardware | | **modal** | Modal cloud sandbox | Full (cloud VM) | Ephemeral cloud compute, evals | | **daytona** | Daytona workspace | Full (cloud container) | Managed cloud dev environments | +| **tenki** | Tenki sandbox | Full (cloud sandbox) | On-demand cloud compute | | **singularity** | Singularity/Apptainer container | Namespaces (--containall) | HPC clusters, shared machines | ### Local Backend @@ -370,6 +377,42 @@ terminal: **Disk limit:** Daytona enforces a 10 GiB maximum. Requests above this are capped with a warning. +### Tenki Backend + +Runs commands in a [Tenki](https://tenki.cloud) sandbox. Hermes creates sandboxes on demand and terminates them by default. + +```yaml +terminal: + backend: tenki + cwd: "/home/tenki" + container_persistent: false # Default for Tenki + tenki_api_endpoint: "https://api.tenki.cloud" + tenki_workspace_id: "" # Falls back to Tenki CLI config + tenki_project_id: "" # Falls back to Tenki CLI config + tenki_name_prefix: "hermes" + tenki_allow_inbound: false + tenki_allow_outbound: true + tenki_max_duration: 3600 + tenki_idle_timeout: 0 + tenki_pause_retention: 0 + tenki_sync_hermes_home: false # Opt in only if child sandboxes need selected ~/.hermes files + tenki_forward_env: [] # Explicit credentials like GITHUB_TOKEN / GH_TOKEN +``` + +**Required:** Tenki CLI login, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`. Hermes also reads the Tenki CLI config for the current workspace and project IDs. + +**Persistence:** Tenki is terminate-only by default. Set `container_persistent: true` only if you intentionally want Hermes to pause and resume a task-named sandbox. + +**Sudo:** Tenki sandboxes use the sandbox's own sudoers policy. Hermes never prompts for or forwards the host `SUDO_PASSWORD` to Tenki. The default Tenki image supports passwordless sudo. + +**Optional `.hermes` sync:** Set `tenki_sync_hermes_home: true` if a Tenki sandbox needs the same selected credential, skill, and cache files that Modal and Daytona receive. Leave it off when Tenki is only an execution sandbox and secrets should remain with the supervisor process. + +**Credential forwarding:** `terminal.env_passthrough` intentionally blocks common credential names such as `GITHUB_TOKEN` and `GH_TOKEN`. For Git or package-manager tokens that must be visible inside Tenki sandboxes, list the variable names in `terminal.tenki_forward_env`; Hermes resolves them profile-scope-aware (from your current shell / the active profile scope first, then `~/.hermes/.env`). The supervisor's own Tenki control-plane token (`TENKI_AUTH_TOKEN` / `TENKI_API_KEY`) is **not** forwarded by default; add it to `tenki_forward_env` only when child sandboxes must create nested Tenki sandboxes, and note that anything forwarded is readable by (model-controlled) guest code. + +> **Multiplexing caveat:** forwarded credentials are not profile-isolated under the multiplexing gateway's shared terminal cache — two profiles served by one gateway process can reuse the same live sandbox. Avoid forwarding sensitive credentials (especially the control-plane token) into sandboxes when running multiple profiles from a single multiplexed gateway. + +**Fully remote supervisor pattern:** To make Hermes itself live remotely, run the Hermes process inside a long-lived Tenki supervisor sandbox and configure that process with `terminal.backend: tenki`. The supervisor owns `~/.hermes`, model credentials, sessions, memory, and gateway/dashboard processes. Terminal, file, `execute_code`, and delegated subagent execution then create child Tenki sandboxes on demand. Give the supervisor Tenki credentials with `tenki login`, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`; child sandboxes do not need host sudo passwords. + ### Singularity/Apptainer Backend Runs commands in a [Singularity/Apptainer](https://apptainer.org) container. Designed for HPC clusters and shared machines where Docker isn't available. @@ -400,13 +443,14 @@ If terminal commands fail immediately or the terminal tool is reported as disabl - **SSH** — Both `TERMINAL_SSH_HOST` and `TERMINAL_SSH_USER` must be set. Hermes logs a clear error if either is missing. - **Modal** — Needs `MODAL_TOKEN_ID` env var or `~/.modal.toml`. Run `hermes doctor` to check. - **Daytona** — Needs `DAYTONA_API_KEY`. The Daytona SDK handles server URL configuration. +- **Tenki** — Needs Tenki CLI login or `TENKI_AUTH_TOKEN`/`TENKI_API_KEY`. Workspace/project can come from the Tenki CLI config or `terminal.tenki_workspace_id` / `terminal.tenki_project_id`. - **Singularity** — Needs `apptainer` or `singularity` in `$PATH`. Common on HPC clusters. When in doubt, set `terminal.backend` back to `local` and verify that commands run there first. ### Remote-to-Host File Sync on Teardown -For the **SSH**, **Modal**, and **Daytona** backends (anywhere the agent's working tree lives on a different machine than the host running Hermes), Hermes tracks files the agent touched inside the remote sandbox and, on session teardown / sandbox cleanup, **syncs the modified files back to the host** under `~/.hermes/cache/remote-syncs//`. +For the **SSH**, **Modal**, **Daytona**, and **Tenki** backends (anywhere the agent's working tree lives on a different machine than the host running Hermes), Hermes tracks files the agent touched inside the remote sandbox and, on session teardown / sandbox cleanup, **syncs the modified files back to the host** under `~/.hermes/cache/remote-syncs//`. Tenki also supports opt-in selected `.hermes` credential/skill/cache sync with `terminal.tenki_sync_hermes_home: true`; it is disabled by default. - Triggers on: session close, `/new`, `/reset`, gateway message timeout, `delegate_task` subagent completion when the child used a remote backend. - Covers the whole tree the agent modified, not just files it explicitly opened. Additions, edits, and deletions are all captured. diff --git a/website/docs/user-guide/features/tools.md b/website/docs/user-guide/features/tools.md index 92a5bc0690470..22a57e5f4dceb 100644 --- a/website/docs/user-guide/features/tools.md +++ b/website/docs/user-guide/features/tools.md @@ -71,7 +71,7 @@ The terminal tool can execute commands in different environments: ```yaml # In ~/.hermes/config.yaml terminal: - backend: local # or: docker, ssh, singularity, modal, daytona + backend: local # or: docker, ssh, singularity, modal, daytona, tenki cwd: "." # Working directory timeout: 180 # Command timeout in seconds ``` @@ -128,7 +128,7 @@ Configure CPU, memory, disk, and persistence for all container backends: ```yaml terminal: - backend: docker # or singularity, modal, daytona + backend: docker # or singularity, modal, daytona, tenki container_cpu: 1 # CPU cores (default: 1) container_memory: 5120 # Memory in MB (default: 5GB) container_disk: 51200 # Disk in MB (default: 50GB) diff --git a/website/docs/user-guide/security.md b/website/docs/user-guide/security.md index 71d1b131d2ede..6848ac159912c 100644 --- a/website/docs/user-guide/security.md +++ b/website/docs/user-guide/security.md @@ -183,7 +183,7 @@ The following patterns trigger approval prompts (defined in `tools/approval.py`) | `gateway run` with `&`/`disown`/`nohup`/`setsid` | Prevents starting gateway outside service manager | :::info -**Container bypass**: When running in `docker`, `singularity`, `modal`, or `daytona` backends, dangerous command checks are **skipped** because the container itself is the security boundary. Destructive commands inside a container can't harm the host. +**Container bypass**: When running in `docker`, `singularity`, `modal`, `daytona`, or `tenki` backends, dangerous command checks are **skipped** because the container itself is the security boundary. Destructive commands inside a container can't harm the host. ::: ### Approval Flow (CLI) @@ -399,11 +399,11 @@ terminal: - **Ephemeral mode** (`container_persistent: false`): Uses tmpfs for workspace — everything is lost on cleanup :::tip -For production gateway deployments, use `docker`, `modal`, or `daytona` backend to isolate agent commands from your host system. This eliminates the need for dangerous command approval entirely. +For production gateway deployments, use `docker`, `modal`, `daytona`, or `tenki` backend to isolate agent commands from your host system. This eliminates the need for dangerous command approval entirely. ::: :::warning -If you add names to `terminal.docker_forward_env`, those variables are intentionally injected into the container for terminal commands. This is useful for task-specific credentials like `GITHUB_TOKEN`, but it also means code running in the container can read and exfiltrate them. +If you add names to `terminal.docker_forward_env` or `terminal.tenki_forward_env`, those variables are intentionally injected into the sandbox for terminal commands. This is useful for task-specific credentials like `GITHUB_TOKEN`, but it also means code running in the sandbox can read and exfiltrate them. ::: ## Terminal Backend Security Comparison @@ -416,6 +416,7 @@ If you add names to `terminal.docker_forward_env`, those variables are intention | **singularity** | Container | ❌ Skipped | HPC environments | | **modal** | Cloud sandbox | ❌ Skipped | Scalable cloud isolation | | **daytona** | Cloud sandbox | ❌ Skipped | Persistent cloud workspaces | +| **tenki** | Cloud sandbox | ❌ Skipped | On-demand cloud compute | ## Environment Variable Passthrough {#environment-variable-passthrough} @@ -491,6 +492,7 @@ Paths are relative to `~/.hermes/`. Files are mounted to `/root/.hermes/` inside | **terminal** (local) | Blocks explicit Hermes infrastructure vars (provider keys, gateway tokens, tool API keys) | ✅ Passthrough vars bypass the blocklist | | **terminal** (Docker) | No host env vars by default | ✅ Passthrough vars + `docker_forward_env` forwarded via `-e` | | **terminal** (Modal) | No host env/files by default | ✅ Credential files mounted; env passthrough via sync | +| **terminal** (Tenki) | No arbitrary host env vars by default | ✅ Passthrough vars + `tenki_forward_env` forwarded via sandbox env | | **MCP** | Blocks everything except safe system vars + explicitly configured `env` | ❌ Not affected by passthrough (use MCP `env` config instead) | ### Security Considerations diff --git a/website/docs/user-guide/skills/bundled/autonomous-ai-agents/autonomous-ai-agents-hermes-agent.md b/website/docs/user-guide/skills/bundled/autonomous-ai-agents/autonomous-ai-agents-hermes-agent.md index 2dde2ad9d1244..8978978d6d76b 100644 --- a/website/docs/user-guide/skills/bundled/autonomous-ai-agents/autonomous-ai-agents-hermes-agent.md +++ b/website/docs/user-guide/skills/bundled/autonomous-ai-agents/autonomous-ai-agents-hermes-agent.md @@ -1034,7 +1034,7 @@ See `tests/agent/test_prompt_builder.py::TestEnvironmentHints` for a worked exam Factual guidance about the host OS, user home, cwd, terminal backend, and shell (bash vs. PowerShell on Windows) is emitted from `agent/prompt_builder.py::build_environment_hints()`. This is also where the WSL hint and per-backend probe logic live. The convention: - **Local terminal backend** → emit host info (OS, `$HOME`, cwd) + Windows-specific notes (hostname ≠ username, `terminal` uses bash not PowerShell). -- **Remote terminal backend** (anything in `_REMOTE_TERMINAL_BACKENDS`: `docker, singularity, modal, daytona, ssh, managed_modal`) → **suppress** host info entirely and describe only the backend. A live `uname`/`whoami`/`pwd` probe runs inside the backend via `tools.environments.get_environment(...).execute(...)`, cached per process in `_BACKEND_PROBE_CACHE`, with a static fallback if the probe times out. +- **Remote terminal backend** (anything in `_REMOTE_TERMINAL_BACKENDS`: `docker, singularity, modal, daytona, tenki, ssh, managed_modal`) → **suppress** host info entirely and describe only the backend. A live `uname`/`whoami`/`pwd` probe runs inside the backend via `tools.environments.get_environment(...).execute(...)`, cached per process in `_BACKEND_PROBE_CACHE`, with a static fallback if the probe times out. - **Key fact for prompt authoring:** when `TERMINAL_ENV != "local"`, *every* file tool (`read_file`, `write_file`, `patch`, `search_files`) runs inside the backend container, not on the host. The system prompt must never describe the host in that case — the agent can't touch it. Full design notes, the exact emitted strings, and testing pitfalls: diff --git a/website/docs/user-guide/skills/bundled/software-development/software-development-plan.md b/website/docs/user-guide/skills/bundled/software-development/software-development-plan.md index 36d390bd2f340..f1d43b2b60980 100644 --- a/website/docs/user-guide/skills/bundled/software-development/software-development-plan.md +++ b/website/docs/user-guide/skills/bundled/software-development/software-development-plan.md @@ -63,7 +63,7 @@ If the task is code-related, include exact file paths, likely test targets, and Save the plan with `write_file` under: - `.hermes/plans/YYYY-MM-DD_HHMMSS-.md` -Treat that as relative to the active working directory / backend workspace. Hermes file tools are backend-aware, so using this relative path keeps the plan with the workspace on local, docker, ssh, modal, and daytona backends. +Treat that as relative to the active working directory / backend workspace. Hermes file tools are backend-aware, so using this relative path keeps the plan with the workspace on local, docker, ssh, modal, daytona, and tenki backends. If the runtime provides a specific target path, use that exact path. If not, create a sensible timestamped filename yourself under `.hermes/plans/`. diff --git a/website/scripts/generate-llms-txt.py b/website/scripts/generate-llms-txt.py index a34c57792a3df..fbeb79fc5e65b 100644 --- a/website/scripts/generate-llms-txt.py +++ b/website/scripts/generate-llms-txt.py @@ -204,7 +204,7 @@ def emit_llms_index() -> str: "autonomous coding and task agent with persistent memory, agent-created skills, " "and a messaging gateway that lives on 21+ messaging platforms — 19 native to " "the gateway plus IRC and Microsoft Teams via plugins (Telegram, Discord, Slack, " - "SMS, Matrix, ...). Runs on local, Docker, SSH, Daytona, Modal, or Singularity " + "SMS, Matrix, ...). Runs on local, Docker, SSH, Daytona, Tenki, Modal, or Singularity " "backends. Works with Nous Portal, OpenRouter, OpenAI, Anthropic, Google, or any " "OpenAI-compatible endpoint." ) From 1dd2ceb97e83714dfa78c20818f721ec7bd07eb3 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 13 Jul 2026 22:55:14 -0700 Subject: [PATCH 02/12] chore(release): map @hashbender in AUTHOR_MAP Required by contributor-check for nick@luxor.tech commits in #64190. Co-Authored-By: Claude Fable 5 --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 84a39b54c377a..675159235eaa2 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -75,6 +75,7 @@ "AndreasHiltner@users.noreply.github.com": "AndreasHiltner", # PR #56854 salvage (gateway: route multiplex profile responses through the profile's own adapter — 53-site _adapter_for_source sweep) "AlexFucuson9@users.noreply.github.com": "AlexFucuson9", # PR #61347 salvage (agent: reapply provider headers after model switch; #61099) "allenliang2022@users.noreply.github.com": "allenliang2022", # PR #56932 test coverage folded into #56909 salvage (408 → retryable timeout) + "nick@luxor.tech": "hashbender", # PR #64190 (tenki cloud sandbox terminal backend) "m888.braun@hotmail.com": "ManniBr", # PR #57417 partial salvage (gateway: fail-closed adapter resolution for unregistered secondary profiles) "poowis2011@hotmail.com": "Umi4Life", # PR #47377 salvage (agent: emit one-shot fallback switch notice on successful fallback so gateway users see model/provider change; #35419) "austin@openvm067.space": "austinlaw076", # PR #57563 partial salvage (auth: lazy per-profile Anthropic OAuth file; gateway: whatsapp_cloud/line added to port-binding platform set) From 4773552e84ec7da81d4fd1297d68315c28d41d49 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 13 Jul 2026 23:26:30 -0700 Subject: [PATCH 03/12] fix(tools): capture sandbox ref under lock before exec to avoid cancel race cancel() can null self._sandbox between _ensure_sandbox() and the dereference in _start_process/_exec_raw/_transfer_sandbox, turning a user interrupt into an AttributeError. _require_sandbox() captures the reference under the lock and raises a typed RuntimeError if the sandbox was torn down. Found by Tenki Code Reviewer on the mirrored PR. Co-Authored-By: Claude Fable 5 --- tests/tools/test_tenki_environment.py | 34 +++++++++++++++++++++++++++ tools/environments/tenki.py | 23 ++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py index 23a06e24b5db8..6f998a2af879c 100644 --- a/tests/tools/test_tenki_environment.py +++ b/tests/tools/test_tenki_environment.py @@ -1302,3 +1302,37 @@ def fail_prompt(*_args, **_kwargs): assert "sudo -S" not in command assert "host-secret" not in command env.cleanup() + +def test_exec_survives_cancel_clearing_sandbox_mid_operation(monkeypatch, tmp_path): + """cancel() nulling self._sandbox between _ensure_sandbox() and the exec + call must not crash the in-flight command: operations run against the + reference captured by _require_sandbox(), and a genuinely torn-down + sandbox surfaces a clean RuntimeError instead of an AttributeError.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_API_KEY", "sk-test-key") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="cancel-race") + + sandbox = _FakeSandboxFactory.sandboxes[0] + orig_exec = sandbox.exec + + def exec_and_teardown(*args, **kwargs): + env._sandbox = None # what cancel() does concurrently + return orig_exec(*args, **kwargs) + + monkeypatch.setattr(sandbox, "exec", exec_and_teardown) + output, exit_code = env._exec_raw("echo ok", timeout=5) + assert exit_code == 0 + + # After the teardown a fruitless ensure must fail loud and typed. + monkeypatch.setattr(env, "_ensure_sandbox", lambda: None) + env._sandbox = None + with pytest.raises(RuntimeError, match="torn down"): + env._exec_raw("echo ok", timeout=5) diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 958aa34311f7d..00d9886521a9b 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -630,6 +630,17 @@ def _ensure_sandbox(self) -> None: sandbox_id = getattr(self._sandbox, "id", None) or getattr(self._sandbox, "sandbox_id", None) logger.info("Tenki: created sandbox %s for task %s", sandbox_id or "", self._task_id) + def _require_sandbox(self) -> Any: + # Capture the reference under the lock: cancel() may null out + # self._sandbox between _ensure_sandbox() and the caller's use of it, + # and the operation must run against the sandbox that ensure produced. + self._ensure_sandbox() + with self._lock: + sandbox = self._sandbox + if sandbox is None: + raise RuntimeError("Tenki sandbox was torn down mid-operation") + return sandbox + def _create_sandbox_from_kwargs(self, kwargs: dict[str, Any]): if self._persistent: client = self._create_client() @@ -810,8 +821,7 @@ def _tenki_bulk_download(self, dest: Path) -> None: def _transfer_sandbox(self): if self._cleanup_in_progress and self._cleanup_sandbox is not None: return self._cleanup_sandbox - self._ensure_sandbox() - return self._sandbox + return self._require_sandbox() def _tenki_delete(self, remote_paths: list[str]) -> None: if not remote_paths: @@ -819,8 +829,7 @@ def _tenki_delete(self, remote_paths: list[str]) -> None: self._exec_raw(quoted_rm_command(remote_paths), timeout=30) def _exec_raw(self, command: str, *, login: bool = False, timeout: int = 120) -> tuple[str, int]: - self._ensure_sandbox() - return self._exec_raw_on_sandbox(self._sandbox, command, login=login, timeout=timeout) + return self._exec_raw_on_sandbox(self._require_sandbox(), command, login=login, timeout=timeout) def _exec_raw_on_sandbox( self, @@ -857,14 +866,14 @@ def _start_process( stdin_data: str | None, process_ref: dict[str, Any] | None = None, ) -> tuple[str, int]: - self._ensure_sandbox() + sandbox = self._require_sandbox() flag = "-lc" if login else "-c" - start = getattr(self._sandbox, "start", None) + start = getattr(sandbox, "start", None) if not callable(start): kwargs: dict[str, Any] = {"timeout": timeout, "env": self._sandbox_env()} if stdin_data is not None: kwargs["input"] = stdin_data - result = self._sandbox.exec("bash", flag, cmd_string, **kwargs) + result = sandbox.exec("bash", flag, cmd_string, **kwargs) return self._result_to_output(result) process = start( From e7e95e2fb75c4d2d6aae7e1fca8b0d368997989f Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 13 Jul 2026 23:42:48 -0700 Subject: [PATCH 04/12] fix(tools): harden tenki cleanup close + unify probe container config Two findings from Tenki Code Reviewer on the true-base verification PR: - _close_client: swallow close() exceptions. cleanup() resets _cleanup_in_progress only after closing the client, so an escaping network error during teardown left the flag stuck and every later _ensure_sandbox() failed with 'Tenki cleanup is in progress'. - prompt_builder probe: replace the fourth inline container-config copy with the shared _container_config_from_env_config() builder; the inline dict omitted tenki_sync_hermes_home, tenki_forward_env, and docker_network, so probe environments diverged from real ones. Co-Authored-By: Claude Fable 5 --- agent/prompt_builder.py | 34 +++++++--------------------- tests/agent/test_prompt_builder.py | 36 ++++++++++++++++++++++++++++++ tools/environments/tenki.py | 8 ++++++- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index ceaa2a6a958bc..3d3638f5b02ca 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -945,7 +945,12 @@ def _probe_remote_backend(env_type: str) -> str | None: try: # Import locally: tools/ imports are heavy and only relevant when a # non-local backend is actually configured. - from tools.terminal_tool import _create_environment, _get_env_config # type: ignore + from tools.terminal_tool import ( # type: ignore + _CONTAINER_BACKENDS, + _container_config_from_env_config, + _create_environment, + _get_env_config, + ) except Exception as e: logger.debug("Backend probe unavailable (import failed): %s", e) _BACKEND_PROBE_CACHE[cache_key] = "" @@ -981,31 +986,8 @@ def _probe_remote_backend(env_type: str) -> str | None: } container_config = None - if env_type in {"docker", "singularity", "modal", "daytona", "tenki"}: - container_config = { - "container_cpu": config.get("container_cpu", 1), - "container_memory": config.get("container_memory", 5120), - "container_disk": config.get("container_disk", 51200), - "container_persistent": config.get("container_persistent", True), - "modal_mode": config.get("modal_mode", "auto"), - "docker_volumes": config.get("docker_volumes", []), - "docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False), - "docker_forward_env": config.get("docker_forward_env", []), - "docker_env": config.get("docker_env", {}), - "docker_run_as_host_user": config.get("docker_run_as_host_user", False), - "docker_extra_args": config.get("docker_extra_args", []), - "docker_persist_across_processes": config.get("docker_persist_across_processes", True), - "docker_orphan_reaper": config.get("docker_orphan_reaper", True), - "tenki_api_endpoint": config.get("tenki_api_endpoint", ""), - "tenki_workspace_id": config.get("tenki_workspace_id", ""), - "tenki_project_id": config.get("tenki_project_id", ""), - "tenki_name_prefix": config.get("tenki_name_prefix", "hermes"), - "tenki_allow_inbound": config.get("tenki_allow_inbound", False), - "tenki_allow_outbound": config.get("tenki_allow_outbound", True), - "tenki_max_duration": config.get("tenki_max_duration", 3600), - "tenki_idle_timeout": config.get("tenki_idle_timeout", 0), - "tenki_pause_retention": config.get("tenki_pause_retention", 0), - } + if env_type in _CONTAINER_BACKENDS: + container_config = _container_config_from_env_config(config) env = _create_environment( env_type=env_type, diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index d57165dee067f..6ecddf57f343f 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -1296,6 +1296,42 @@ def _fake_create_environment(*, env_type, **kwargs): assert "Linux 6.8.0" in line assert "root" in line + def test_probe_container_config_uses_shared_builder(self, monkeypatch): + """The probe must pass the canonical container config from + ``_container_config_from_env_config`` — a stale inline copy omitted + ``tenki_sync_hermes_home`` / ``tenki_forward_env`` (and + ``docker_network``), so probe environments were built with different + settings than real ones.""" + import agent.prompt_builder as _pb + import tools.terminal_tool as _tt + + monkeypatch.setenv("TERMINAL_ENV", "tenki") + _pb._clear_backend_probe_cache() + + class _FakeEnv: + def execute(self, cmd, timeout=None): + return { + "returncode": 0, + "output": ( + "os=Linux\nkernel=6.8.0\nhome=/home/tenki\n" + "cwd=/home/tenki\nuser=tenki\n" + ), + } + + created = {} + + def _fake_create_environment(*, env_type, **kwargs): + created["container_config"] = kwargs.get("container_config") + return _FakeEnv() + + monkeypatch.setattr(_tt, "_create_environment", _fake_create_environment) + + assert _pb._probe_remote_backend("tenki") is not None + container_config = created["container_config"] + assert container_config is not None + for key in ("tenki_sync_hermes_home", "tenki_forward_env", "docker_network"): + assert key in container_config + def test_remote_backend_list_covers_known_sandboxes(self): """Regression guard: if someone adds a remote backend, they must list it here.""" import agent.prompt_builder as _pb diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 00d9886521a9b..0b0000da1a251 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -1089,8 +1089,14 @@ def _save_persistent_snapshot(self, sandbox: Any) -> bool: @staticmethod def _close_client(client: Any) -> None: + # Best-effort: a failed close must never propagate — cleanup() resets + # _cleanup_in_progress after this call, and an escaping exception would + # leave the flag stuck and brick the environment. if client is None: return close = getattr(client, "close", None) if callable(close): - close() + try: + close() + except Exception as exc: + logger.warning("Tenki: client close failed: %s", exc) From f2fb6836729dc987086d6ab72fe4151e1210a3a8 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 27 Jul 2026 13:12:45 -0700 Subject: [PATCH 05/12] fix(tools): capture one sandbox for tenki upload flows _tenki_upload and _tenki_bulk_upload re-read self._sandbox for each step, but cancel() nulls that field out concurrently. A cancel landing mid-flow made the mkdir and the upload target different sandboxes, or dereferenced None outright (AttributeError: 'NoneType' has no attribute 'fs'). The bulk flow was worse: its mkdir/upload/untar/rm could span two sandboxes, extracting the tar somewhere other than where it landed. Capture the sandbox once via _require_sandbox() and thread that one reference through every filesystem and exec call, matching what _tenki_bulk_download already does with _transfer_sandbox(). Co-Authored-By: Claude Opus 5 (1M context) --- tests/tools/test_tenki_environment.py | 72 +++++++++++++++++++++++++++ tools/environments/tenki.py | 24 ++++++--- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py index 6f998a2af879c..dd872131ad671 100644 --- a/tests/tools/test_tenki_environment.py +++ b/tests/tools/test_tenki_environment.py @@ -1,5 +1,6 @@ from __future__ import annotations +import shlex import sys import threading import time @@ -1336,3 +1337,74 @@ def exec_and_teardown(*args, **kwargs): env._sandbox = None with pytest.raises(RuntimeError, match="torn down"): env._exec_raw("echo ok", timeout=5) + + +def _tenki_env_for_upload_race(monkeypatch, tmp_path, task_id): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_API_KEY", "sk-test-key") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + return TenkiEnvironment(task_id=task_id) + + +def test_tenki_upload_targets_one_sandbox_when_cancel_races(monkeypatch, tmp_path): + """cancel() nulls self._sandbox, so re-reading it per call could send the + mkdir and the upload of a single file to two different sandboxes (or to + None). Both must run against one captured reference.""" + env = _tenki_env_for_upload_race(monkeypatch, tmp_path, "upload-race") + original = env._sandbox + created_before = len(_FakeSandboxFactory.created_kwargs) + + orig_mkdir = original.fs.mkdir + + def mkdir_and_teardown(*args, **kwargs): + env._sandbox = None # what cancel() does concurrently + return orig_mkdir(*args, **kwargs) + + monkeypatch.setattr(original.fs, "mkdir", mkdir_and_teardown) + + host_file = tmp_path / "skill.md" + host_file.write_text("content", encoding="utf-8") + env._tenki_upload(str(host_file), "/home/tenki/.hermes/skills/skill.md") + + assert len(_FakeSandboxFactory.created_kwargs) == created_before + assert original.fs.mkdir_calls, "mkdir ran on the captured sandbox" + assert original.fs.upload_calls[-1] == ( + str(host_file), + "/home/tenki/.hermes/skills/skill.md", + ) + + +def test_tenki_bulk_upload_targets_one_sandbox_when_cancel_races(monkeypatch, tmp_path): + """Same single-capture rule for the bulk flow: mkdir, tar upload, untar and + the cleanup rm must all land on one sandbox, or the tar gets extracted + somewhere other than where it was uploaded.""" + env = _tenki_env_for_upload_race(monkeypatch, tmp_path, "bulk-upload-race") + original = env._sandbox + created_before = len(_FakeSandboxFactory.created_kwargs) + + orig_exec = original.exec + + def exec_and_teardown(*args, **kwargs): + env._sandbox = None # cancel() lands during the mkdir + return orig_exec(*args, **kwargs) + + monkeypatch.setattr(original, "exec", exec_and_teardown) + + host_file = tmp_path / "skill.md" + host_file.write_text("content", encoding="utf-8") + env._tenki_bulk_upload([(str(host_file), "/home/tenki/.hermes/skills/skill.md")]) + + assert len(_FakeSandboxFactory.created_kwargs) == created_before + remote_tar = original.fs.upload_calls[-1][1] + commands = [call[0][-1] for call in original.exec_calls] + assert any(cmd.startswith("mkdir ") for cmd in commands) + # The untar and the cleanup both reference the tar this sandbox received. + assert any(f"tar xf {shlex.quote(remote_tar)}" in cmd for cmd in commands) + assert any(f"rm -f {shlex.quote(remote_tar)}" in cmd for cmd in commands) diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 0b0000da1a251..62847cf6e0c77 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -764,19 +764,26 @@ def _resolve_remote_home(self) -> None: pass def _tenki_upload(self, host_path: str, remote_path: str) -> None: - self._ensure_sandbox() + # One capture for the whole flow: cancel() nulls out self._sandbox, so + # re-reading it per call could send the mkdir and the upload to two + # different sandboxes (or to None). + sandbox = self._require_sandbox() parent = str(Path(remote_path).parent) - self._sandbox.fs.mkdir(parent, recursive=True) - self._sandbox.fs.upload(host_path, remote_path) + sandbox.fs.mkdir(parent, recursive=True) + sandbox.fs.upload(host_path, remote_path) def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: if not files: return - self._ensure_sandbox() + # Same single-capture rule as _tenki_upload: the mkdir, the tar upload, + # the untar, and the cleanup rm must all target one sandbox, or a + # concurrent cancel() can leave the tar extracted into a different + # sandbox than the one it was uploaded to. + sandbox = self._require_sandbox() parents = unique_parent_dirs(files) if parents: - self._exec_raw(quoted_mkdir_command(parents), timeout=30) + self._exec_raw_on_sandbox(sandbox, quoted_mkdir_command(parents), timeout=30) remote_tar = self._remote_transfer_path(".hermes_tenki_sync") with tempfile.NamedTemporaryFile(suffix=".tar") as tmp: @@ -784,10 +791,11 @@ def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: for host_path, remote_path in files: tar.add(host_path, arcname=remote_path.lstrip("/")) tmp.flush() - self._sandbox.fs.upload(tmp.name, remote_tar) + sandbox.fs.upload(tmp.name, remote_tar) try: - output, exit_code = self._exec_raw( + output, exit_code = self._exec_raw_on_sandbox( + sandbox, f"tar xf {shlex.quote(remote_tar)} -C /", timeout=120, ) @@ -795,7 +803,7 @@ def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: raise RuntimeError(f"Tenki bulk upload failed (exit {exit_code}): {output}") finally: try: - self._exec_raw(f"rm -f {shlex.quote(remote_tar)}", timeout=10) + self._exec_raw_on_sandbox(sandbox, f"rm -f {shlex.quote(remote_tar)}", timeout=10) except Exception: pass From fc92659fb77ac9857bb29583f64ce8b98bbac5c3 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 27 Jul 2026 13:13:34 -0700 Subject: [PATCH 06/12] refactor(tools): drop the dead tenki project surface Tenki removed projects from its API: the SDK has no project_id on sandbox creation, no list_project, and no IdentityProject. The terminal.tenki_project_id key and its TERMINAL_TENKI_PROJECT_ID / TENKI_PROJECT_ID env overrides therefore configure nothing. Remove the config key from both loaders, the gateway env bridge, and the container-config plumbing, along with the documentation rows that advertised it. The workspace remains the unit that decides where sandboxes are created. Co-Authored-By: Claude Opus 5 (1M context) --- cli-config.yaml.example | 1 - cli.py | 2 -- gateway/run.py | 1 - hermes_cli/config.py | 3 --- tests/tools/test_container_cwd_sanitize.py | 1 - tests/tools/test_file_tools_container_config.py | 3 --- tests/tools/test_terminal_config_env_sync.py | 1 - tests/tools/test_terminal_tool_requirements.py | 3 --- tools/terminal_tool.py | 3 --- website/docs/reference/environment-variables.md | 4 +--- website/docs/user-guide/configuration.md | 6 ++---- 11 files changed, 3 insertions(+), 25 deletions(-) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index e9f10df4adae9..e8c82892ca934 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -323,7 +323,6 @@ terminal: # tenki_image: "" # Optional image/template; blank uses Tenki default # tenki_api_endpoint: "https://api.tenki.cloud" # tenki_workspace_id: "" # Blank falls back to Tenki CLI config -# tenki_project_id: "" # Blank falls back to Tenki CLI config # tenki_name_prefix: "hermes" # tenki_allow_inbound: false # tenki_allow_outbound: true diff --git a/cli.py b/cli.py index 59d91dc65cf1b..72d3c4b33b74e 100644 --- a/cli.py +++ b/cli.py @@ -411,7 +411,6 @@ def load_cli_config() -> Dict[str, Any]: # non-blank default here is bridged as explicit and would mask them. "tenki_api_endpoint": "", "tenki_workspace_id": "", - "tenki_project_id": "", "tenki_name_prefix": "hermes", "tenki_allow_inbound": False, "tenki_allow_outbound": True, @@ -643,7 +642,6 @@ def load_cli_config() -> Dict[str, Any]: "tenki_image": "TERMINAL_TENKI_IMAGE", "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", - "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", diff --git a/gateway/run.py b/gateway/run.py index c8ba34d1c1e98..6e1e041240cd2 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1523,7 +1523,6 @@ def _profile_runtime_scope(profile_home: "Path"): "tenki_image": "TERMINAL_TENKI_IMAGE", "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", - "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", diff --git a/hermes_cli/config.py b/hermes_cli/config.py index c0f8f7b6a3d93..8a213d3e767bd 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1233,7 +1233,6 @@ def _ensure_hermes_home_managed(home: Path): # them. Blank resolves to https://api.tenki.cloud downstream. "tenki_api_endpoint": "", "tenki_workspace_id": "", - "tenki_project_id": "", "tenki_name_prefix": "hermes", "tenki_allow_inbound": False, "tenki_allow_outbound": True, @@ -6957,7 +6956,6 @@ def write_platform_config_field( "tenki_image": "TERMINAL_TENKI_IMAGE", "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", - "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", @@ -8143,7 +8141,6 @@ def show_config(): print(f" Tenki image: {terminal.get('tenki_image') or '(Tenki default)'}") print(f" Endpoint: {terminal.get('tenki_api_endpoint') or 'https://api.tenki.cloud'}") print(f" Workspace: {terminal.get('tenki_workspace_id') or '(from Tenki CLI)'}") - print(f" Project: {terminal.get('tenki_project_id') or '(from Tenki CLI)'}") print(f" Sync .hermes: {'enabled' if terminal.get('tenki_sync_hermes_home') else 'disabled'}") elif terminal.get('backend') == 'ssh': ssh_host = get_env_value('TERMINAL_SSH_HOST') diff --git a/tests/tools/test_container_cwd_sanitize.py b/tests/tools/test_container_cwd_sanitize.py index 2f06b60ae2908..2b283cb05b283 100644 --- a/tests/tools/test_container_cwd_sanitize.py +++ b/tests/tools/test_container_cwd_sanitize.py @@ -255,7 +255,6 @@ def _run_and_capture_cwd(self, monkeypatch, override_cwd, env_type="docker", "tenki_image": "", "tenki_api_endpoint": "", "tenki_workspace_id": "", - "tenki_project_id": "", "tenki_name_prefix": "hermes", "tenki_allow_inbound": False, "tenki_allow_outbound": True, diff --git a/tests/tools/test_file_tools_container_config.py b/tests/tools/test_file_tools_container_config.py index 3a010504a51b4..cf549ae14876c 100644 --- a/tests/tools/test_file_tools_container_config.py +++ b/tests/tools/test_file_tools_container_config.py @@ -29,7 +29,6 @@ def _make_env_config(**overrides): "docker_orphan_reaper": False, "tenki_api_endpoint": "https://api.tenki.test", "tenki_workspace_id": "ws-123", - "tenki_project_id": "prj-456", "tenki_name_prefix": "agent", "tenki_allow_inbound": True, "tenki_allow_outbound": False, @@ -101,7 +100,6 @@ def test_shared_container_config_fields_are_forwarded(self): assert cc.get("tenki_name_prefix") == "agent" assert cc.get("tenki_api_endpoint") == "https://api.tenki.test" assert cc.get("tenki_workspace_id") == "ws-123" - assert cc.get("tenki_project_id") == "prj-456" assert cc.get("tenki_allow_inbound") is True assert cc.get("tenki_allow_outbound") is False assert cc.get("tenki_max_duration") == 7200 @@ -157,7 +155,6 @@ def fake_create_env(**kwargs): assert cc["container_persistent"] is False assert cc["tenki_api_endpoint"] == "https://api.tenki.test" assert cc["tenki_workspace_id"] == "ws-123" - assert cc["tenki_project_id"] == "prj-456" assert cc["tenki_name_prefix"] == "agent" assert cc["tenki_allow_inbound"] is True assert cc["tenki_allow_outbound"] is False diff --git a/tests/tools/test_terminal_config_env_sync.py b/tests/tools/test_terminal_config_env_sync.py index 94bb361709fb9..f2f851a79077a 100644 --- a/tests/tools/test_terminal_config_env_sync.py +++ b/tests/tools/test_terminal_config_env_sync.py @@ -383,7 +383,6 @@ def test_tenki_config_is_bridged_everywhere(): "tenki_image": "TERMINAL_TENKI_IMAGE", "tenki_api_endpoint": "TERMINAL_TENKI_API_ENDPOINT", "tenki_workspace_id": "TERMINAL_TENKI_WORKSPACE_ID", - "tenki_project_id": "TERMINAL_TENKI_PROJECT_ID", "tenki_name_prefix": "TERMINAL_TENKI_NAME_PREFIX", "tenki_allow_inbound": "TERMINAL_TENKI_ALLOW_INBOUND", "tenki_allow_outbound": "TERMINAL_TENKI_ALLOW_OUTBOUND", diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index c1a56765069e4..3c0fe8ad6cc7b 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -77,15 +77,12 @@ def fake_find_spec(name): monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) monkeypatch.delenv("TENKI_WORKSPACE_ID", raising=False) monkeypatch.delenv("TENKI_WORKSPACE", raising=False) - monkeypatch.delenv("TENKI_PROJECT_ID", raising=False) - monkeypatch.delenv("TENKI_PROJECT", raising=False) monkeypatch.setattr( terminal_tool_module, "_get_env_config", lambda: { "env_type": "tenki", "tenki_workspace_id": "", - "tenki_project_id": "", }, ) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index c79d08502e4e3..03ce0f2a62879 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -1368,7 +1368,6 @@ def _get_env_config() -> Dict[str, Any]: "tenki_image": os.getenv("TERMINAL_TENKI_IMAGE", ""), "tenki_api_endpoint": os.getenv("TERMINAL_TENKI_API_ENDPOINT", ""), "tenki_workspace_id": os.getenv("TERMINAL_TENKI_WORKSPACE_ID", ""), - "tenki_project_id": os.getenv("TERMINAL_TENKI_PROJECT_ID", ""), "tenki_name_prefix": os.getenv("TERMINAL_TENKI_NAME_PREFIX", "hermes"), "tenki_allow_inbound": os.getenv("TERMINAL_TENKI_ALLOW_INBOUND", "false").lower() in {"true", "1", "yes"}, "tenki_allow_outbound": os.getenv("TERMINAL_TENKI_ALLOW_OUTBOUND", "true").lower() in {"true", "1", "yes"}, @@ -1456,7 +1455,6 @@ def _container_config_from_env_config(config: Dict[str, Any]) -> Dict[str, Any]: "docker_orphan_reaper": config.get("docker_orphan_reaper", True), "tenki_api_endpoint": config.get("tenki_api_endpoint", ""), "tenki_workspace_id": config.get("tenki_workspace_id", ""), - "tenki_project_id": config.get("tenki_project_id", ""), "tenki_name_prefix": config.get("tenki_name_prefix", "hermes"), "tenki_allow_inbound": config.get("tenki_allow_inbound", False), "tenki_allow_outbound": config.get("tenki_allow_outbound", True), @@ -1615,7 +1613,6 @@ def _create_environment(env_type: str, image: str, cwd: str, timeout: int, task_id=task_id, api_endpoint=cc.get("tenki_api_endpoint", ""), workspace_id=cc.get("tenki_workspace_id", ""), - project_id=cc.get("tenki_project_id", ""), name_prefix=cc.get("tenki_name_prefix", "hermes"), allow_inbound=cc.get("tenki_allow_inbound", False), allow_outbound=cc.get("tenki_allow_outbound", True), diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 7be664c285171..553491f14858b 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -165,10 +165,9 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | `SUPERMEMORY_API_KEY` | Semantic long-term memory with profile recall and session ingest ([supermemory.ai](https://supermemory.ai)) | | `DAYTONA_API_KEY` | Daytona cloud sandboxes ([daytona.io](https://daytona.io/)) | | `TENKI_AUTH_TOKEN` / `TENKI_API_KEY` | Tenki cloud sandboxes ([tenki.cloud](https://tenki.cloud)); alternatively run `tenki login` | -| `TENKI_CONFIG_PATH` | Override the Tenki CLI config path Hermes reads for auth, workspace, and project defaults | +| `TENKI_CONFIG_PATH` | Override the Tenki CLI config path Hermes reads for auth and workspace defaults | | `TENKI_API_ENDPOINT` / `TENKI_API_URL` | Direct Tenki API endpoint override used when terminal config is blank | | `TENKI_WORKSPACE_ID` / `TENKI_WORKSPACE` | Direct Tenki workspace ID override used when terminal config is blank | -| `TENKI_PROJECT_ID` / `TENKI_PROJECT` | Direct Tenki project ID override used when terminal config is blank | ### Skill API Keys @@ -224,7 +223,6 @@ These variables configure the [Tool Gateway](/user-guide/features/tool-gateway) | `TERMINAL_TENKI_IMAGE` | Optional Tenki sandbox image/template; blank uses Tenki default | | `TERMINAL_TENKI_API_ENDPOINT` | Tenki API endpoint (default: `https://api.tenki.cloud`) | | `TERMINAL_TENKI_WORKSPACE_ID` | Tenki workspace ID; blank falls back to Tenki CLI config | -| `TERMINAL_TENKI_PROJECT_ID` | Tenki project ID; blank falls back to Tenki CLI config | | `TERMINAL_TENKI_NAME_PREFIX` | Prefix for Hermes-created Tenki sandbox names (default: `hermes`) | | `TERMINAL_TENKI_ALLOW_INBOUND` | Allow inbound network access in Tenki sandboxes (`true`/`false`, default: `false`) | | `TERMINAL_TENKI_ALLOW_OUTBOUND` | Allow outbound network access in Tenki sandboxes (`true`/`false`, default: `true`) | diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 5a76d1a2925b5..f81473f3664cc 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -123,7 +123,6 @@ terminal: tenki_image: "" # Optional Tenki image/template; blank uses Tenki default tenki_api_endpoint: "https://api.tenki.cloud" tenki_workspace_id: "" # Blank falls back to Tenki CLI config - tenki_project_id: "" # Blank falls back to Tenki CLI config tenki_sync_hermes_home: false # Opt-in sync of selected ~/.hermes files tenki_forward_env: [] # Explicit host env vars to forward into Tenki ``` @@ -388,7 +387,6 @@ terminal: container_persistent: false # Default for Tenki tenki_api_endpoint: "https://api.tenki.cloud" tenki_workspace_id: "" # Falls back to Tenki CLI config - tenki_project_id: "" # Falls back to Tenki CLI config tenki_name_prefix: "hermes" tenki_allow_inbound: false tenki_allow_outbound: true @@ -399,7 +397,7 @@ terminal: tenki_forward_env: [] # Explicit credentials like GITHUB_TOKEN / GH_TOKEN ``` -**Required:** Tenki CLI login, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`. Hermes also reads the Tenki CLI config for the current workspace and project IDs. +**Required:** Tenki CLI login, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`. Hermes also reads the Tenki CLI config for the current workspace ID. **Persistence:** Tenki is terminate-only by default. Set `container_persistent: true` only if you intentionally want Hermes to pause and resume a task-named sandbox. @@ -443,7 +441,7 @@ If terminal commands fail immediately or the terminal tool is reported as disabl - **SSH** — Both `TERMINAL_SSH_HOST` and `TERMINAL_SSH_USER` must be set. Hermes logs a clear error if either is missing. - **Modal** — Needs `MODAL_TOKEN_ID` env var or `~/.modal.toml`. Run `hermes doctor` to check. - **Daytona** — Needs `DAYTONA_API_KEY`. The Daytona SDK handles server URL configuration. -- **Tenki** — Needs Tenki CLI login or `TENKI_AUTH_TOKEN`/`TENKI_API_KEY`. Workspace/project can come from the Tenki CLI config or `terminal.tenki_workspace_id` / `terminal.tenki_project_id`. +- **Tenki** — Needs Tenki CLI login or `TENKI_AUTH_TOKEN`/`TENKI_API_KEY`. The workspace can come from the Tenki CLI config or `terminal.tenki_workspace_id`. - **Singularity** — Needs `apptainer` or `singularity` in `$PATH`. Common on HPC clusters. When in doubt, set `terminal.backend` back to `local` and verify that commands run there first. From a40ba721ee78e2a99002d93b489d0fe56ee812be Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 27 Jul 2026 13:14:35 -0700 Subject: [PATCH 07/12] feat(tools): bump tenki SDK to 0.5.1 with bounded range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per AGENTS.md's dependency policy, use a bounded range rather than an exact runtime pin: pre-1.0 packages get a two-minor ceiling, so tenki>=0.5.1,<0.7. uv.lock refreshed. The package was renamed on PyPI: 0.5.1 ships as `tenki` (tenki-sandbox stops at 0.4.0). The compat `tenki_sandbox` module still ships inside `tenki`, but new code should import `tenki`, so the imports, the find_spec probes, and every install hint move over. 0.5 is a real API migration, not just a version string: - Client.create no longer accepts project_id, and list_project / list_workspace folded into list(workspace_id=...). _create_kwargs filtered against Sandbox.create, which is a bare **kwargs passthrough that names nothing it accepts, so filtering was a no-op and any dropped kwarg reached the client as an unexpected keyword. Introspect Client.create — the real validator — and pass the client-construction kwargs (base_url, auth_token) explicitly, since Sandbox.create pops those before forwarding. - RegistryArtifactNotFoundError was renamed RegistryImageNotFoundError. The single combined `from tenki_sandbox import (...)` failed outright on the rename, silently dropping the isinstance check for every class in it and leaving only the name-based fallback. Resolve each class independently and accept both names. Tests: the fake SDK now mirrors 0.5 — Client.create declares its real parameter list with no **kwargs catch-all, so an unsupported name raises TypeError exactly as the real client would. Adds regressions for the kwarg filtering and for snapshot-error classification across the rename. Co-Authored-By: Claude Opus 5 (1M context) --- cli-config.yaml.example | 2 +- hermes_cli/doctor.py | 21 +-- hermes_cli/setup.py | 11 +- hermes_cli/status.py | 5 - pyproject.toml | 2 +- tests/hermes_cli/test_setup.py | 2 +- tests/tools/test_tenki_environment.py | 169 +++++++++++++++--- tests/tools/test_terminal_requirements.py | 5 +- .../tools/test_terminal_tool_requirements.py | 2 +- tools/environments/tenki.py | 133 ++++++++++---- tools/lazy_deps.py | 2 +- tools/tenki_config.py | 21 +-- tools/terminal_tool.py | 12 +- uv.lock | 12 +- 14 files changed, 281 insertions(+), 118 deletions(-) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index e8c82892ca934..29d09237e9003 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -311,7 +311,7 @@ terminal: # OPTION 7: Tenki cloud execution # Commands run in Tenki cloud sandboxes, created on demand # Great for: On-demand cloud compute, isolated ephemeral sandboxes -# Requires: pip install tenki-sandbox, plus `tenki login` or the +# Requires: pip install tenki, plus `tenki login` or the # TENKI_AUTH_TOKEN / TENKI_API_KEY env var # ----------------------------------------------------------------------------- # terminal: diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 5ece76cc9e1ea..b0a63082a8aa3 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -1561,13 +1561,11 @@ def run_doctor(args): from hermes_cli.config import load_config_readonly from tools.tenki_config import ( has_tenki_auth, - resolve_tenki_project_id, resolve_tenki_workspace_id, ) except Exception: load_config_readonly = lambda: {} # noqa: E731 has_tenki_auth = lambda: False # noqa: E731 - resolve_tenki_project_id = lambda _explicit="": "" # noqa: E731 resolve_tenki_workspace_id = lambda _explicit="": "" # noqa: E731 terminal_cfg = load_config_readonly().get("terminal", {}) if not isinstance(terminal_cfg, dict): @@ -1586,24 +1584,21 @@ def run_doctor(args): workspace_id = resolve_tenki_workspace_id( os.getenv("TERMINAL_TENKI_WORKSPACE_ID") or terminal_cfg.get("tenki_workspace_id", "") ) - project_id = resolve_tenki_project_id( - os.getenv("TERMINAL_TENKI_PROJECT_ID") or terminal_cfg.get("tenki_project_id", "") - ) - if workspace_id and project_id: - check_ok("Tenki workspace/project", "(configured)") + if workspace_id: + check_ok("Tenki workspace", "(configured)") else: check_warn( - "Tenki workspace/project not configured", + "Tenki workspace not configured", "(optional for sessions; required for volume-backed workflows)", ) - if importlib.util.find_spec("tenki_sandbox") is not None: - check_ok("tenki-sandbox SDK", "(installed)") + if importlib.util.find_spec("tenki") is not None: + check_ok("tenki SDK", "(installed)") else: _fail_and_issue( - "tenki-sandbox SDK not installed", - "(pip install tenki-sandbox==0.1.1)", - "Install Tenki SDK: pip install tenki-sandbox==0.1.1", + "tenki SDK not installed", + "(pip install 'tenki>=0.5.1,<0.7')", + "Install Tenki SDK: pip install 'tenki>=0.5.1,<0.7'", issues, ) diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index fd2177af9b8d6..0b801e365f3ee 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -1393,13 +1393,13 @@ def setup_terminal_backend(config: dict): print_info("Requires Tenki CLI login or TENKI_AUTH_TOKEN/TENKI_API_KEY.") try: - __import__("tenki_sandbox") + __import__("tenki") except ImportError: print_info("Installing Tenki SDK...") import subprocess uv_bin = shutil.which("uv") - package = "tenki-sandbox==0.1.1" + package = "tenki>=0.5.1,<0.7" if uv_bin: result = subprocess.run( [uv_bin, "pip", "install", "--python", sys.executable, package], @@ -1415,27 +1415,23 @@ def setup_terminal_backend(config: dict): if result.returncode == 0: print_success("Tenki SDK installed") else: - print_warning("Install failed — run manually: pip install tenki-sandbox==0.1.1") + print_warning("Install failed — run manually: pip install 'tenki>=0.5.1,<0.7'") if result.stderr: print_info(f" Error: {result.stderr.strip().splitlines()[-1]}") from tools.tenki_config import ( has_tenki_auth, resolve_tenki_api_endpoint, - resolve_tenki_project_id, resolve_tenki_workspace_id, ) terminal = config.setdefault("terminal", {}) endpoint = resolve_tenki_api_endpoint(terminal.get("tenki_api_endpoint", "")) workspace_id = resolve_tenki_workspace_id(terminal.get("tenki_workspace_id", "")) - project_id = resolve_tenki_project_id(terminal.get("tenki_project_id", "")) terminal["tenki_api_endpoint"] = endpoint if workspace_id: terminal["tenki_workspace_id"] = workspace_id - if project_id: - terminal["tenki_project_id"] = project_id terminal.setdefault("tenki_image", "") terminal.setdefault("tenki_name_prefix", "hermes") terminal.setdefault("tenki_allow_inbound", False) @@ -1453,7 +1449,6 @@ def setup_terminal_backend(config: dict): print_info(f" Endpoint: {endpoint}") print_info(f" Workspace: {workspace_id or '(not found; run tenki login)'}") - print_info(f" Project: {project_id or '(not found; run tenki login)'}") if has_tenki_auth(): print_info(" Tenki auth: already configured") else: diff --git a/hermes_cli/status.py b/hermes_cli/status.py index 9bf6ec96ae3b3..b2d8b3d751b53 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -430,7 +430,6 @@ def _resolve_env(env_ref) -> str: elif terminal_env == "tenki": from tools.tenki_config import ( resolve_tenki_api_endpoint, - resolve_tenki_project_id, resolve_tenki_workspace_id, ) @@ -441,9 +440,6 @@ def _resolve_env(env_ref) -> str: tenki_workspace = resolve_tenki_workspace_id( os.getenv("TERMINAL_TENKI_WORKSPACE_ID") or terminal_cfg.get("tenki_workspace_id", "") ) - tenki_project = resolve_tenki_project_id( - os.getenv("TERMINAL_TENKI_PROJECT_ID") or terminal_cfg.get("tenki_project_id", "") - ) tenki_sync_value = os.getenv("TERMINAL_TENKI_SYNC_HERMES_HOME") if tenki_sync_value is None: tenki_sync = bool(terminal_cfg.get("tenki_sync_hermes_home", False)) @@ -452,7 +448,6 @@ def _resolve_env(env_ref) -> str: print(f" Tenki Image: {tenki_image or '(Tenki default)'}") print(f" Endpoint: {tenki_endpoint}") print(f" Workspace: {tenki_workspace or '(not found)'}") - print(f" Project: {tenki_project or '(not found)'}") print(f" Sync .hermes: {check_mark(tenki_sync)} {'enabled' if tenki_sync else 'disabled'}") sudo_password = os.getenv("SUDO_PASSWORD", "") diff --git a/pyproject.toml b/pyproject.toml index 409ed852b962c..a9723ddaccaae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,7 +156,7 @@ fal = ["fal-client==0.13.1"] edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] -tenki = ["tenki-sandbox==0.1.1"] +tenki = ["tenki>=0.5.1,<0.7"] # published as `tenki` (was `tenki-sandbox` pre-0.5) hindsight = ["hindsight-client==0.6.1"] dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82) messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.29.0", "slack-sdk==3.43.0", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265 diff --git a/tests/hermes_cli/test_setup.py b/tests/hermes_cli/test_setup.py index a84f74d6069f6..2f44614c3dbf8 100644 --- a/tests/hermes_cli/test_setup.py +++ b/tests/hermes_cli/test_setup.py @@ -506,7 +506,7 @@ def fake_prompt_choice(question, choices, default=0): return choices.index("Tenki Agent - Tenki cloud sandbox") raise AssertionError(f"Unexpected prompt_choice call: {question}") - monkeypatch.setitem(sys.modules, "tenki_sandbox", types.ModuleType("tenki_sandbox")) + monkeypatch.setitem(sys.modules, "tenki", types.ModuleType("tenki")) monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py index dd872131ad671..32a9313cedbdd 100644 --- a/tests/tools/test_tenki_environment.py +++ b/tests/tools/test_tenki_environment.py @@ -143,22 +143,41 @@ def _last_started_command(sandbox: _FakeSandbox) -> str: class _FakeSnapshotNotFoundError(Exception): - """Mirrors tenki_sandbox.SnapshotNotFoundError for the fake SDK.""" + """Mirrors tenki.SnapshotNotFoundError for the fake SDK.""" -class _FakeRegistryArtifactNotFoundError(Exception): - """Mirrors tenki_sandbox.RegistryArtifactNotFoundError for the fake SDK.""" +class _FakeRegistryImageNotFoundError(Exception): + """Mirrors tenki.RegistryImageNotFoundError for the fake SDK. + + Named ``RegistryArtifactNotFoundError`` before tenki 0.5. + """ class _FakeSnapshotNotDurableError(Exception): - """Mirrors tenki_sandbox.SnapshotNotDurableError for the fake SDK.""" + """Mirrors tenki.SnapshotNotDurableError for the fake SDK.""" class _FakeInvalidStateError(Exception): - """Mirrors tenki_sandbox.InvalidStateError for the fake SDK.""" + """Mirrors tenki.InvalidStateError for the fake SDK.""" + + +# Sentinel so the fake records exactly which kwargs the environment passed, +# rather than the defaults it didn't. +_UNSET = object() + +# Kwargs that tenki's Sandbox.create pops into the Client it builds instead of +# forwarding to Client.create. +_CLIENT_ONLY_KWARGS = ("auth_token", "base_url", "gateway_url", "cookie_name", "timeout") class _FakeSandboxFactory: + """Mirrors ``tenki.Sandbox``. + + ``create`` is a bare ``**kwargs`` passthrough — exactly like the real SDK's + — that pops the client-construction kwargs and forwards the rest to + ``Client.create``, which is the thing that actually validates names. + """ + created_kwargs: list[dict] = [] failed_kwargs: list[dict] = [] sandboxes: list[_FakeSandbox] = [] @@ -172,6 +191,11 @@ class _FakeSandboxFactory: @classmethod def create(cls, **kwargs): + client_kwargs = {key: kwargs.pop(key) for key in _CLIENT_ONLY_KWARGS if key in kwargs} + return _FakeClient(**client_kwargs).create(**kwargs) + + @classmethod + def _record_and_build(cls, kwargs: dict): if kwargs.get("snapshot_id") in cls.fail_snapshot_ids: cls.failed_kwargs.append(kwargs) raise cls.snapshot_error(cls.snapshot_error_msg) @@ -192,16 +216,44 @@ def __init__(self, **kwargs): self.kwargs = kwargs self.snapshots = SimpleNamespace(wait_durable=lambda *_args, **_kwargs: None) - def create(self, **kwargs): - return _FakeSandboxFactory.create(**kwargs) - - def list(self, **_kwargs): - return list(self.listed_sandboxes) - - def list_project(self, *_args, **_kwargs): - return list(self.listed_sandboxes) - - def list_workspace(self, *_args, **_kwargs): + # The parameter list mirrors tenki 0.5's Client.create and deliberately has + # NO **kwargs catch-all: a name the SDK dropped (project_id, removed in 0.5) + # raises TypeError here exactly as it would against the real client, so the + # environment's create-kwarg filtering is genuinely under test. + def create( + self, + *, + workspace_id=_UNSET, + name=_UNSET, + wait=_UNSET, + timeout=_UNSET, + allow_inbound=_UNSET, + allow_outbound=_UNSET, + max_duration=_UNSET, + idle_timeout_minutes=_UNSET, + pause_retention=_UNSET, + cpu_cores=_UNSET, + memory_mb=_UNSET, + disk_size_gb=_UNSET, + metadata=_UNSET, + tags=_UNSET, + env=_UNSET, + ssh_authorized_keys=_UNSET, + snapshot_id=_UNSET, + image=_UNSET, + sticky=_UNSET, + ): + passed = { + key: value + for key, value in locals().items() + if key != "self" and value is not _UNSET + } + # In the real SDK the client kwargs live on the client; merge them so + # tests can assert against the whole create call in one dict. + return _FakeSandboxFactory._record_and_build({**self.kwargs, **passed}) + + # tenki 0.5 folded list_project/list_workspace into list(workspace_id=...). + def list(self, *, workspace_id=None, tags=None, sticky=None): return list(self.listed_sandboxes) def close(self): @@ -209,7 +261,7 @@ def close(self): def _install_fake_tenki(monkeypatch): - module = types.ModuleType("tenki_sandbox") + module = types.ModuleType("tenki") _FakeSandboxFactory.created_kwargs = [] _FakeSandboxFactory.failed_kwargs = [] _FakeSandboxFactory.sandboxes = [] @@ -221,10 +273,10 @@ def _install_fake_tenki(monkeypatch): module.Client = _FakeClient module.Sandbox = _FakeSandboxFactory module.SnapshotNotFoundError = _FakeSnapshotNotFoundError - module.RegistryArtifactNotFoundError = _FakeRegistryArtifactNotFoundError + module.RegistryImageNotFoundError = _FakeRegistryImageNotFoundError module.SnapshotNotDurableError = _FakeSnapshotNotDurableError module.InvalidStateError = _FakeInvalidStateError - monkeypatch.setitem(sys.modules, "tenki_sandbox", module) + monkeypatch.setitem(sys.modules, "tenki", module) def _clear_tenki_auth_env(monkeypatch): @@ -284,7 +336,6 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch [ "api_endpoint: https://api.tenki.test", "current_workspace_id: ws-123", - "current_project_id: prj-456", "auth_token: tok-secret", ] ), @@ -307,7 +358,6 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch # The control-plane credential is used host-side to create the sandbox... assert kwargs["base_url"] == "https://api.tenki.test" assert kwargs["workspace_id"] == "ws-123" - assert kwargs["project_id"] == "prj-456" assert kwargs["auth_token"] == "cookie:tok-secret" # ...but is NEVER injected into the model-controlled guest environment # (an empty env is omitted from the create kwargs entirely). @@ -316,7 +366,6 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch assert "TENKI_API_KEY" not in guest_env assert "TENKI_API_ENDPOINT" not in guest_env assert "TENKI_WORKSPACE_ID" not in guest_env - assert "TENKI_PROJECT_ID" not in guest_env assert kwargs["allow_inbound"] is False assert kwargs["allow_outbound"] is True assert kwargs["cpu_cores"] == 1 @@ -1408,3 +1457,81 @@ def exec_and_teardown(*args, **kwargs): # The untar and the cleanup both reference the tar this sandbox received. assert any(f"tar xf {shlex.quote(remote_tar)}" in cmd for cmd in commands) assert any(f"rm -f {shlex.quote(remote_tar)}" in cmd for cmd in commands) + + +def test_tenki_create_kwargs_filters_names_the_installed_sdk_dropped(monkeypatch, tmp_path): + """``Sandbox.create`` is a bare ``**kwargs`` passthrough, so the accepted + set has to be read off ``Client.create`` — the real validator. Filtering + against the passthrough accepts every name, and one the SDK has dropped + (``project_id``, gone in 0.5) reaches the client as an unexpected keyword + and kills sandbox creation with a TypeError.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "missing.yaml")) + monkeypatch.setenv("TENKI_API_KEY", "sk-test-key") + + # A deliberately narrow client: no cpu_cores/memory_mb/disk_size_gb/ + # allow_*/max_duration/idle_timeout_minutes/pause_retention, and no + # **kwargs to swallow them. + def narrow_create(self, *, name=None, image=None, env=None, metadata=None, tags=None, wait=True): + return _FakeSandboxFactory._record_and_build( + {"name": name, "image": image, "env": env, "metadata": metadata, "tags": tags, "wait": wait} + ) + + monkeypatch.setattr(_FakeClient, "create", narrow_create) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="narrow-sdk", + image="base-image", + cpu=2, + memory=2048, + idle_timeout=120, + pause_retention=60, + ) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["image"] == "base-image" + for dropped in ( + "cpu_cores", + "memory_mb", + "disk_size_gb", + "allow_inbound", + "allow_outbound", + "max_duration", + "idle_timeout_minutes", + "pause_retention", + ): + assert dropped not in kwargs + env.cleanup() + + +def test_snapshot_unrecoverable_resolves_renamed_registry_error(monkeypatch): + """tenki 0.5 renamed RegistryArtifactNotFoundError to + RegistryImageNotFoundError. Importing the whole set in one statement made a + single rename drop the isinstance check for every class in it.""" + _install_fake_tenki(monkeypatch) + + from tools.environments.tenki import TenkiEnvironment + + # 0.5 name, exported by the installed SDK. + assert TenkiEnvironment._snapshot_unrecoverable(_FakeRegistryImageNotFoundError("gone")) is True + assert TenkiEnvironment._snapshot_unrecoverable(_FakeSnapshotNotFoundError("gone")) is True + assert TenkiEnvironment._snapshot_unrecoverable(_FakeSnapshotNotDurableError("nope")) is True + + # Pre-0.5 name: absent from a 0.5 SDK's exports, so only the MRO name + # fallback can classify it. Getting this wrong silently boots a base image. + class RegistryArtifactNotFoundError(Exception): + pass + + assert TenkiEnvironment._snapshot_unrecoverable(RegistryArtifactNotFoundError("gone")) is True + + # Transient failures must keep the snapshot pointer for a later retry. + assert TenkiEnvironment._snapshot_unrecoverable(RuntimeError("network blip")) is False + assert TenkiEnvironment._snapshot_unrecoverable(_FakeInvalidStateError("workspace suspended")) is False + # ...but an InvalidStateError that names the snapshot is unrecoverable. + assert TenkiEnvironment._snapshot_unrecoverable(_FakeInvalidStateError("snapshot is not durable")) is True diff --git a/tests/tools/test_terminal_requirements.py b/tests/tools/test_terminal_requirements.py index d37a0b42a3f76..51bb123f5d0c4 100644 --- a/tests/tools/test_terminal_requirements.py +++ b/tests/tools/test_terminal_requirements.py @@ -17,7 +17,6 @@ def _clear_terminal_env(monkeypatch): "TERMINAL_LIFETIME_SECONDS", "TERMINAL_MODAL_MODE", "TERMINAL_TENKI_API_ENDPOINT", - "TERMINAL_TENKI_PROJECT_ID", "TERMINAL_TENKI_WORKSPACE_ID", "TENKI_API_KEY", "TENKI_AUTH_TOKEN", @@ -202,7 +201,7 @@ def test_tenki_backend_with_sdk_and_cli_auth_returns_true(monkeypatch, tmp_path) monkeypatch.setattr( terminal_tool_module.importlib.util, "find_spec", - lambda name: object() if name == "tenki_sandbox" else None, + lambda name: object() if name == "tenki" else None, ) assert terminal_tool_module.check_terminal_requirements() is True @@ -215,7 +214,7 @@ def test_tenki_backend_without_auth_logs_specific_error(monkeypatch, caplog, tmp monkeypatch.setattr( terminal_tool_module.importlib.util, "find_spec", - lambda name: object() if name == "tenki_sandbox" else None, + lambda name: object() if name == "tenki" else None, ) with caplog.at_level(logging.ERROR): diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index 3c0fe8ad6cc7b..6b99770afbb1b 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -69,7 +69,7 @@ def test_tenki_requires_auth_only_for_plain_sessions(self, monkeypatch, tmp_path original_find_spec = terminal_tool_module.importlib.util.find_spec def fake_find_spec(name): - if name == "tenki_sandbox": + if name == "tenki": return object() return original_find_spec(name) diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 62847cf6e0c77..2aebc3b0cb9cb 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -32,7 +32,6 @@ from tools.tenki_config import ( resolve_tenki_api_endpoint, resolve_tenki_auth_token, - resolve_tenki_project_id, resolve_tenki_workspace_id, ) @@ -153,6 +152,16 @@ def _supports_any_kwargs(sig: inspect.Signature | None) -> bool: return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in sig.parameters.values()) +def _named_parameters(sig: inspect.Signature) -> set[str]: + """Explicitly named parameters, ignoring ``self``/``*args``/``**kwargs``.""" + return { + name + for name, param in sig.parameters.items() + if name != "self" + and param.kind not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL) + } + + def _add_supported( kwargs: dict[str, Any], sig: inspect.Signature | None, @@ -288,7 +297,6 @@ def __init__( task_id: str = "default", api_endpoint: str = "", workspace_id: str = "", - project_id: str = "", name_prefix: str = "hermes", allow_inbound: bool = False, allow_outbound: bool = True, @@ -309,7 +317,7 @@ def __init__( except Exception as exc: raise ImportError(str(exc)) - from tenki_sandbox import Client, Sandbox + from tenki import Client, Sandbox self._Client = Client self._Sandbox = Sandbox @@ -337,7 +345,6 @@ def __init__( self._disk = disk self._api_endpoint = resolve_tenki_api_endpoint(api_endpoint) self._workspace_id = resolve_tenki_workspace_id(workspace_id) - self._project_id = resolve_tenki_project_id(project_id) self._auth_token = resolve_tenki_auth_token() self._name_prefix = _safe_name(name_prefix, fallback="hermes", max_len=28) self._allow_inbound = allow_inbound @@ -366,10 +373,35 @@ def __init__( self.init_session() def _sandbox_create_signature(self) -> inspect.Signature | None: - try: - return inspect.signature(self._Sandbox.create) - except (TypeError, ValueError): - return None + """Signature that actually validates the sandbox create kwargs. + + ``Sandbox.create`` is a bare ``**kwargs`` passthrough: it pops the + client-construction kwargs and forwards everything else to + ``Client.create``. Introspecting it therefore accepts *every* name and + filters nothing, so a kwarg the SDK has dropped (``project_id``, gone + in tenki 0.5) sails through and only fails as a ``TypeError`` at call + time. ``Client.create`` is the real validator in both paths, so prefer + it and fall back to ``Sandbox.create`` only if it can't be introspected. + """ + candidates = ( + getattr(self._Client, "create", None), + getattr(self._Sandbox, "create", None), + ) + fallback: inspect.Signature | None = None + for target in candidates: + if target is None: + continue + try: + sig = inspect.signature(target) + except (TypeError, ValueError): + continue + # A pure **kwargs passthrough names nothing it accepts; keep it only + # as a last resort so _add_supported still degrades to "send it". + if _supports_any_kwargs(sig) and not _named_parameters(sig): + fallback = fallback or sig + continue + return sig + return fallback def _create_kwargs(self) -> dict[str, Any]: sig = self._sandbox_create_signature() @@ -400,9 +432,14 @@ def _create_kwargs(self) -> dict[str, Any]: if pause_retention is not None: _add_supported(kwargs, sig, ("pause_retention",), pause_retention) _add_supported(kwargs, sig, ("workspace_id",), self._workspace_id) - _add_supported(kwargs, sig, ("project_id",), self._project_id) - _add_supported(kwargs, sig, ("base_url", "api_endpoint"), self._api_endpoint) - _add_supported(kwargs, sig, ("auth_token", "api_key"), self._auth_token) + # Client-construction kwargs, NOT sandbox kwargs: Sandbox.create pops + # these into the Client it builds, so they never appear in the + # create-signature we filter against. The persistent path builds its own + # client and strips them again in _create_sandbox_from_kwargs. + if self._api_endpoint: + kwargs["base_url"] = self._api_endpoint + if self._auth_token: + kwargs["auth_token"] = self._auth_token _add_supported(kwargs, sig, ("env",), self._sandbox_env()) _add_supported( kwargs, @@ -546,17 +583,31 @@ def _sandbox_matches_task(self, sandbox: Any) -> bool: return metadata.get("hermes_task_id") == self._task_id return True + def _list_kwargs(self, client: Any) -> dict[str, Any]: + """Kwargs for the sandbox listing used to re-attach a persistent sandbox. + + tenki 0.5 folded the old ``list_project`` / ``list_workspace`` helpers + into ``Client.list``, which takes the workspace as a keyword. Older SDK + builds don't accept it, so scope the listing only when the installed + signature says it will be honored. + """ + kwargs: dict[str, Any] = {"tags": ["hermes-agent"]} + if not self._workspace_id: + return kwargs + try: + sig = inspect.signature(client.list) + except (TypeError, ValueError): + return kwargs + if "workspace_id" in sig.parameters or _supports_any_kwargs(sig): + kwargs["workspace_id"] = self._workspace_id + return kwargs + def _find_persistent_sandbox(self): if not self._persistent: return None client = self._create_client() try: - if self._project_id and hasattr(client, "list_project"): - candidates = client.list_project(self._project_id, tags=["hermes-agent"]) - elif self._workspace_id and hasattr(client, "list_workspace"): - candidates = client.list_workspace(self._workspace_id, tags=["hermes-agent"]) - else: - candidates = client.list(tags=["hermes-agent"]) + candidates = client.list(**self._list_kwargs(client)) except Exception as exc: logger.debug("Tenki: could not list persistent sandboxes: %s", exc) return None @@ -661,9 +712,12 @@ def _create_sandbox_from_kwargs(self, kwargs: dict[str, Any]): # only unrecoverable when its message points at the snapshot itself # (handled by message inspection below); a bare InvalidStateError stays # transient so an unrelated precondition can't destroy a valid pointer. + # ``RegistryArtifactNotFoundError`` was renamed ``RegistryImageNotFoundError`` + # in tenki 0.5; both names are listed so either SDK generation resolves. _UNRECOVERABLE_SNAPSHOT_ERRORS = frozenset({ "SnapshotNotFoundError", # snapshot is gone - "RegistryArtifactNotFoundError", # backing artifact is gone + "RegistryImageNotFoundError", # backing image is gone (0.5+) + "RegistryArtifactNotFoundError", # same, pre-0.5 name "SnapshotNotDurableError", # explicitly never reached durability }) @@ -688,25 +742,38 @@ def _is_snapshot_specific_invalid_state(e: BaseException, invalid_state_cls) -> msg = str(e).lower() return "snapshot" in msg or "durable" in msg + # Resolve each class independently: a single `from tenki import (...)` + # of all four fails outright when one has been renamed (as + # RegistryArtifactNotFoundError was in 0.5), silently dropping the + # isinstance check for the classes that *are* present. + sdk: Any = None try: - from tenki_sandbox import ( - InvalidStateError, - RegistryArtifactNotFoundError, - SnapshotNotDurableError, - SnapshotNotFoundError, - ) + import tenki - if isinstance( - exc, - (SnapshotNotFoundError, RegistryArtifactNotFoundError, SnapshotNotDurableError), - ): - return True - if _is_snapshot_specific_invalid_state(exc, InvalidStateError): - return True - return False + sdk = tenki except Exception: pass - # Name-based fallback for SDK builds that don't export every class. + + if sdk is not None: + unrecoverable = tuple( + cls_obj + for cls_obj in ( + getattr(sdk, name, None) for name in cls._UNRECOVERABLE_SNAPSHOT_ERRORS + ) + if isinstance(cls_obj, type) and issubclass(cls_obj, BaseException) + ) + if unrecoverable and isinstance(exc, unrecoverable): + return True + invalid_state_cls = getattr(sdk, "InvalidStateError", None) + if ( + isinstance(invalid_state_cls, type) + and issubclass(invalid_state_cls, BaseException) + and _is_snapshot_specific_invalid_state(exc, invalid_state_cls) + ): + return True + + # Name-based fallback for SDK builds that don't export every class, and + # for a subclass the installed SDK no longer exports under its own name. for typ in type(exc).__mro__: if typ.__name__ in cls._UNRECOVERABLE_SNAPSHOT_ERRORS: return True diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 1a198efa13404..78d6e02ee21dd 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -204,7 +204,7 @@ # ─── Terminal backends ───────────────────────────────────────────────── "terminal.modal": ("modal==1.3.4",), "terminal.daytona": ("daytona==0.155.0",), - "terminal.tenki": ("tenki-sandbox==0.1.1",), + "terminal.tenki": ("tenki>=0.5.1,<0.7",), # renamed from `tenki-sandbox` in 0.5 # ─── Skills ──────────────────────────────────────────────────────────── "skill.google_workspace": ( diff --git a/tools/tenki_config.py b/tools/tenki_config.py index 78f07771d9d8c..d624cb0508390 100644 --- a/tools/tenki_config.py +++ b/tools/tenki_config.py @@ -147,9 +147,9 @@ def resolve_tenki_api_endpoint(explicit: str = "") -> str: def resolve_tenki_workspace_id(explicit: str = "") -> str: - """Resolve the Tenki workspace id. Scope-aware; workspace/project decide - where sandboxes are created, so a multiplexed profile must not silently - borrow the machine-global workspace of another tenant.""" + """Resolve the Tenki workspace id. Scope-aware; the workspace decides where + sandboxes are created, so a multiplexed profile must not silently borrow + the machine-global workspace of another tenant.""" explicit = _string(explicit) if explicit: return explicit @@ -162,21 +162,6 @@ def resolve_tenki_workspace_id(explicit: str = "") -> str: return _first_string(load_tenki_cli_config(), ("current_workspace_id", "workspace_id", "workspace")) -def resolve_tenki_project_id(explicit: str = "") -> str: - """Resolve the Tenki project id. Scope-aware for the same reason as - :func:`resolve_tenki_workspace_id`.""" - explicit = _string(explicit) - if explicit: - return explicit - for env_name in ("TENKI_PROJECT_ID", "TENKI_PROJECT"): - value = _scoped_env(env_name) - if value: - return value - if not _global_credential_fallback_allowed(): - return "" - return _first_string(load_tenki_cli_config(), ("current_project_id", "project_id", "project")) - - def resolve_tenki_auth_token(explicit: str = "") -> str: """Resolve a Tenki auth token/API key without logging or persisting it. diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 03ce0f2a62879..966a88b41e118 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -3023,7 +3023,7 @@ def check_terminal_requirements() -> bool: return os.getenv("DAYTONA_API_KEY") is not None elif env_type == "tenki": - if importlib.util.find_spec("tenki_sandbox") is None: + if importlib.util.find_spec("tenki") is None: try: from tools.lazy_deps import ensure as _lazy_ensure @@ -3031,15 +3031,15 @@ def check_terminal_requirements() -> bool: importlib.invalidate_caches() except Exception as exc: logger.error( - "tenki-sandbox is required for Tenki terminal backend: " - "pip install tenki-sandbox==0.1.1 (%s)", + "tenki is required for Tenki terminal backend: " + "pip install 'tenki>=0.5.1,<0.7' (%s)", exc, ) return False - if importlib.util.find_spec("tenki_sandbox") is None: + if importlib.util.find_spec("tenki") is None: logger.error( - "tenki-sandbox is required for Tenki terminal backend: " - "pip install tenki-sandbox==0.1.1" + "tenki is required for Tenki terminal backend: " + "pip install 'tenki>=0.5.1,<0.7'" ) return False try: diff --git a/uv.lock b/uv.lock index 7f37fad77ba8f..608582ec71dbb 100644 --- a/uv.lock +++ b/uv.lock @@ -1684,7 +1684,7 @@ teams = [ { name = "microsoft-teams-apps" }, ] tenki = [ - { name = "tenki-sandbox" }, + { name = "tenki" }, ] termux = [ { name = "agent-client-protocol" }, @@ -1848,7 +1848,7 @@ requires-dist = [ { name = "starlette", marker = "extra == 'web'", specifier = "==1.0.1" }, { name = "supermemory", marker = "extra == 'supermemory'", specifier = "==3.50.0" }, { name = "tenacity", specifier = "==9.1.4" }, - { name = "tenki-sandbox", marker = "extra == 'tenki'", specifier = "==0.1.1" }, + { name = "tenki", marker = "extra == 'tenki'", specifier = ">=0.5.1,<0.7" }, { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.21" }, { name = "tzdata", marker = "sys_platform == 'win32'", specifier = "==2025.3" }, { name = "urllib3", specifier = ">=2.7.0,<3" }, @@ -4066,17 +4066,17 @@ wheels = [ ] [[package]] -name = "tenki-sandbox" -version = "0.1.1" +name = "tenki" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "websocket-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/35/7f/6e59a084acece58d2349c80d95996cfc4971dcbc61038d6cc9a822edf3dc/tenki_sandbox-0.1.1.tar.gz", hash = "sha256:2748d3cb381c1f955163b368189899517aa04f70e59d64fb06a999f400dfb0b6", size = 107483, upload-time = "2026-06-10T09:23:28.337Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/24/40a827e1795d7cafaffbf615c3b13ad5aa8462359db01bab969f3f00913b/tenki-0.5.1.tar.gz", hash = "sha256:a1ed4e753db3d4a38a71cfb8d9cbf0cd05b28463f4c718fc9ec37f3155f28b4c", size = 184908, upload-time = "2026-07-25T02:22:23.961Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/6b/9e3a0eaf5064d47e3ec1115ba9585d117feb5507312e195d15ee9a9bba28/tenki_sandbox-0.1.1-py3-none-any.whl", hash = "sha256:826ab21db80b3ebe74c9a67f3eeb09f8d880535677f8d5dc3d03ad402210f97c", size = 94883, upload-time = "2026-06-10T09:23:26.958Z" }, + { url = "https://files.pythonhosted.org/packages/58/cb/cba98c901307d5e266a072015d7963834f28ba14cf74598149a98093c655/tenki-0.5.1-py3-none-any.whl", hash = "sha256:2e296544c85b8709f85e84728b5de0c4cb42009fac5475899a1ac238b99ab459", size = 158595, upload-time = "2026-07-25T02:22:22.46Z" }, ] [[package]] From e1f032e6aa586ddd79a31cd33d6622a4b194444e Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 27 Jul 2026 19:39:54 -0700 Subject: [PATCH 08/12] fix(tools): harden Tenki sandbox lifecycle --- agent/prompt_builder.py | 47 +- apps/desktop/src/app/settings/constants.ts | 4 +- apps/desktop/src/app/settings/helpers.test.ts | 3 +- cli-config.yaml.example | 4 +- hermes_cli/config.py | 40 + hermes_cli/setup.py | 26 +- hermes_cli/web_server.py | 109 +- model_tools.py | 16 +- tests/agent/test_prompt_builder.py | 88 +- tests/hermes_cli/test_setup.py | 69 +- tests/hermes_cli/test_web_server.py | 107 +- tests/tools/test_credential_files.py | 38 +- tests/tools/test_file_sync.py | 54 + .../tools/test_file_tools_container_config.py | 264 +- tests/tools/test_shared_container_task_id.py | 153 + tests/tools/test_tenki_environment.py | 3690 ++++++++++++++++- .../tools/test_terminal_tool_requirements.py | 123 + tools/code_execution_tool.py | 52 +- tools/credential_files.py | 26 +- tools/environments/file_sync.py | 23 +- tools/environments/tenki.py | 2982 +++++++++++-- tools/file_tools.py | 170 +- tools/registry.py | 29 +- tools/terminal_tool.py | 639 ++- .../docs/reference/environment-variables.md | 8 +- website/docs/user-guide/configuration.md | 30 +- 26 files changed, 7976 insertions(+), 818 deletions(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index e919cfce0b318..0d63b0ee91422 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -983,7 +983,7 @@ def format_steer_marker(steer_text: str) -> str: # a mid-process backend switch rebuilds the string. Kept in-module (not on # disk) because the probe captures live backend state that may change # across Hermes restarts. -_BACKEND_PROBE_CACHE: dict[tuple[str, str], str] = {} +_BACKEND_PROBE_CACHE: dict[tuple[str, str, str], str] = {} _WINDOWS_BASH_SHELL_HINT = ( @@ -1006,11 +1006,27 @@ def _probe_remote_backend(env_type: str) -> str | None: operate on a different machine than the host Hermes runs on. """ cwd_hint = os.getenv("TERMINAL_CWD", "") - cache_key = (env_type, cwd_hint) + try: + from hermes_constants import get_hermes_home + + profile_key = str(get_hermes_home()) + except Exception: + profile_key = "" + cache_key = (env_type, cwd_hint, profile_key) cached = _BACKEND_PROBE_CACHE.get(cache_key) if cached is not None: return cached or None + # Tenki environments are billable cloud resources and the prompt builder + # has no registry ownership or side-effect-free teardown seam for them. + # Creating a one-off sandbox here could forward configured credentials, + # sync profile files, or persist resources before the first tool call. + # Use the existing static fallback; the agent can probe its real sandbox + # with a terminal call once it actually needs one. + if env_type == "tenki": + _BACKEND_PROBE_CACHE[cache_key] = "" + return None + try: # Import locally: tools/ imports are heavy and only relevant when a # non-local backend is actually configured. @@ -1039,8 +1055,6 @@ def _probe_remote_backend(env_type: str) -> str | None: image = config.get("modal_image", "") elif env_type == "daytona": image = config.get("daytona_image", "") - elif env_type == "tenki": - image = config.get("tenki_image", "") else: image = "" @@ -1133,8 +1147,9 @@ def build_environment_hints() -> str: - For **remote / sandbox** terminal backends (docker, singularity, modal, daytona, tenki, ssh): host info is **suppressed** because the agent's tools can't touch the host — only the backend - matters. A live probe inside the backend reports its OS, user, $HOME, - and cwd. Falls back to a static summary if the probe fails. + matters. A live probe inside most backends reports its OS, user, $HOME, + and cwd, with a static fallback if the probe fails. Tenki always uses + the static summary so prompt construction never creates a cloud sandbox. The WSL environment hint is appended unchanged when running under WSL. """ @@ -1144,6 +1159,15 @@ def build_environment_hints() -> str: hints: list[str] = [] backend = (os.getenv("TERMINAL_ENV") or "local").strip().lower() + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + if is_multiplex_active() and current_secret_scope() is not None: + from tools.terminal_tool import _get_env_config + + backend = str(_get_env_config().get("env_type") or backend).strip().lower() + except Exception: + logger.debug("Could not resolve profile-scoped terminal backend", exc_info=True) is_remote_backend = backend in _REMOTE_TERMINAL_BACKENDS if not is_remote_backend: @@ -1190,6 +1214,17 @@ def build_environment_hints() -> str: f"of the Hermes process are irrelevant; only the following " f"backend state matters:\n{probe}" ) + elif backend == "tenki": + hints.append( + "Terminal backend: tenki. Your `terminal`, `read_file`, " + "`write_file`, `patch`, and `search_files` tools all operate " + "inside a Tenki sandbox (Linux) — NOT on the machine where " + "Hermes itself runs. Prompt construction intentionally " + "defers sandbox creation until the first tool call that needs " + "the backend, " + "so the sandbox's current user, $HOME, and working directory " + "are not known yet." + ) else: description = _BACKEND_FALLBACK_DESCRIPTIONS.get( backend, f"a {backend} environment (likely Linux)" diff --git a/apps/desktop/src/app/settings/constants.ts b/apps/desktop/src/app/settings/constants.ts index f041b2992470e..2473be05a0cf2 100644 --- a/apps/desktop/src/app/settings/constants.ts +++ b/apps/desktop/src/app/settings/constants.ts @@ -256,8 +256,8 @@ export const ENUM_OPTIONS: Record = { // shadow that and hide user-installed/pip providers (#49513). // Terminal execution backends — kept in sync with the dispatch ladder in // tools/terminal_tool.py::_create_environment (local/docker/singularity/ - // modal/daytona/ssh). Remote backends need extra env (image, tokens, host). - 'terminal.backend': ['local', 'docker', 'singularity', 'modal', 'daytona', 'ssh'], + // modal/daytona/tenki/ssh). Remote backends need extra env (image, tokens, host). + 'terminal.backend': ['local', 'docker', 'singularity', 'modal', 'daytona', 'tenki', 'ssh'], 'stt.elevenlabs.model_id': ['scribe_v2', 'scribe_v1'], 'stt.local.model': ['tiny', 'base', 'small', 'medium', 'large-v3'], // Speech-to-text backends — kept in sync with the stt block in diff --git a/apps/desktop/src/app/settings/helpers.test.ts b/apps/desktop/src/app/settings/helpers.test.ts index 860c89cb31f09..482e17f118a8d 100644 --- a/apps/desktop/src/app/settings/helpers.test.ts +++ b/apps/desktop/src/app/settings/helpers.test.ts @@ -206,7 +206,8 @@ describe('settings helpers', () => { it('renders a dropdown for the terminal execution backend', () => { const opts = enumOptionsFor('terminal.backend', 'local', config) - expect(opts).toEqual(['local', 'docker', 'singularity', 'modal', 'daytona', 'ssh']) + expect(opts).toContain('tenki') + expect(new Set(opts).size).toBe(opts?.length) }) it('appends a hand-typed value not in the known list so it stays selected', () => { diff --git a/cli-config.yaml.example b/cli-config.yaml.example index b69b2a9f94c93..7fb7a224576a8 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -320,7 +320,7 @@ terminal: # timeout: 180 # lifetime_seconds: 300 # container_persistent: false # Tenki default: terminate sandboxes on cleanup -# tenki_image: "" # Optional image/template; blank uses Tenki default +# tenki_image: "" # Optional registry image reference; blank uses Tenki default # tenki_api_endpoint: "https://api.tenki.cloud" # tenki_workspace_id: "" # Blank falls back to Tenki CLI config # tenki_name_prefix: "hermes" @@ -328,7 +328,7 @@ terminal: # tenki_allow_outbound: true # tenki_max_duration: 3600 # Max sandbox lifetime in seconds # tenki_idle_timeout: 0 # Auto-pause after idle seconds (0 = disabled) -# tenki_pause_retention: 0 # Retention for paused sandboxes (0 = disabled) +# tenki_pause_retention: 0 # Retention for paused sandboxes (0 = Tenki default) # tenki_sync_hermes_home: false # Opt-in sync of selected ~/.hermes files # tenki_forward_env: [] # Env vars to forward (e.g. GITHUB_TOKEN) diff --git a/hermes_cli/config.py b/hermes_cli/config.py index e2433fe2c2915..788daad2c0cd8 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -7357,6 +7357,46 @@ def _normalize_terminal_backend_defaults( return config +_PRE_TENKI_SETTINGS_KEY = "_pre_tenki_backend_settings" + + +def apply_terminal_backend_transition( + terminal: Dict[str, Any], + backend: str, +) -> None: + """Switch terminal backend while round-tripping Tenki-safe overrides.""" + backend = str(backend or "").strip().lower() + current = str( + terminal.get("backend") or terminal.get("env_type") or "local" + ).strip().lower() + if backend == "tenki" and current != "tenki": + if _PRE_TENKI_SETTINGS_KEY not in terminal: + keys = ("container_persistent", "cwd") + terminal[_PRE_TENKI_SETTINGS_KEY] = { + "present": [key for key in keys if key in terminal], + "values": { + key: terminal[key] + for key in keys + if key in terminal + }, + } + terminal["container_persistent"] = False + terminal["cwd"] = "/home/tenki" + elif current == "tenki" and backend != "tenki": + saved = terminal.pop(_PRE_TENKI_SETTINGS_KEY, None) + if isinstance(saved, dict): + present = set(saved.get("present") or []) + values = saved.get("values") + if not isinstance(values, dict): + values = {} + for key in ("container_persistent", "cwd"): + if key in present and key in values: + terminal[key] = values[key] + else: + terminal.pop(key, None) + terminal["backend"] = backend + + def is_provider_enabled(provider_cfg: Optional[Dict[str, Any]]) -> bool: """Return whether a ``providers.`` config block is enabled. diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index b6724cbbf47af..6cfba1cdac869 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -136,6 +136,7 @@ def _set_reasoning_effort(config: Dict[str, Any], effort: str) -> None: # Import config helpers from hermes_cli.config import ( + apply_terminal_backend_transition, cfg_get, DEFAULT_CONFIG, get_hermes_home, @@ -1224,7 +1225,8 @@ def setup_terminal_backend(config: dict): print_info(f"Keeping current backend: {current_backend}") return - config.setdefault("terminal", {})["backend"] = selected_backend + terminal_config = config.setdefault("terminal", {}) + apply_terminal_backend_transition(terminal_config, selected_backend) if selected_backend == "local": print_success("Terminal backend: Local") @@ -1449,12 +1451,16 @@ def setup_terminal_backend(config: dict): ) terminal = config.setdefault("terminal", {}) - endpoint = resolve_tenki_api_endpoint(terminal.get("tenki_api_endpoint", "")) - workspace_id = resolve_tenki_workspace_id(terminal.get("tenki_workspace_id", "")) + # Keep fallback-derived values dynamic. Persisting the currently + # resolved endpoint/workspace would turn a Tenki CLI or environment + # fallback into an explicit config override, so later CLI/profile + # changes could never take effect. + for key in ("tenki_api_endpoint", "tenki_workspace_id"): + value = terminal.get(key) + terminal[key] = value.strip() if isinstance(value, str) else "" + endpoint = resolve_tenki_api_endpoint(terminal["tenki_api_endpoint"]) + workspace_id = resolve_tenki_workspace_id(terminal["tenki_workspace_id"]) - terminal["tenki_api_endpoint"] = endpoint - if workspace_id: - terminal["tenki_workspace_id"] = workspace_id terminal.setdefault("tenki_image", "") terminal.setdefault("tenki_name_prefix", "hermes") terminal.setdefault("tenki_allow_inbound", False) @@ -1463,12 +1469,8 @@ def setup_terminal_backend(config: dict): terminal.setdefault("tenki_idle_timeout", 0) terminal.setdefault("tenki_pause_retention", 0) terminal.setdefault("tenki_sync_hermes_home", False) - if current_backend == "tenki": - terminal.setdefault("container_persistent", False) - terminal.setdefault("cwd", "/home/tenki") - else: - terminal["container_persistent"] = False - terminal["cwd"] = "/home/tenki" + terminal.setdefault("container_persistent", False) + terminal.setdefault("cwd", "/home/tenki") print_info(f" Endpoint: {endpoint}") print_info(f" Workspace: {workspace_id or '(not found; run tenki login)'}") diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 7497b7f39fa28..e54ca1b156b12 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -16443,6 +16443,11 @@ async def run_toolset_post_setup( "label": "Daytona", "description": "Run commands in a Daytona cloud sandbox.", }, + { + "name": "tenki", + "label": "Tenki", + "description": "Run commands in a Tenki cloud sandbox.", + }, { "name": "ssh", "label": "SSH", @@ -16550,7 +16555,44 @@ def _probe_daytona_backend() -> tuple: return ("needs_setup", "Set DAYTONA_API_KEY to use the Daytona backend.") -def _probe_terminal_backend(name: str, terminal_cfg: dict) -> tuple: +def _probe_tenki_backend(profile_secrets: Optional[Dict[str, str]] = None) -> tuple: + try: + __import__("tenki") + except ImportError: + return ( + "needs_setup", + "Tenki SDK not found — install `tenki>=0.5.1,<0.7` or run `hermes setup terminal`.", + ) + + if profile_secrets is not None: + # A named profile is an isolation boundary. Its .env/external-secret + # mapping is authoritative; never fall through to the dashboard + # process environment or the machine-wide `tenki login` credential. + has_auth = any( + str(profile_secrets.get(key) or "").strip() + for key in ("TENKI_AUTH_TOKEN", "TENKI_API_KEY") + ) + else: + try: + from tools.tenki_config import has_tenki_auth + + has_auth = has_tenki_auth() + except Exception: + has_auth = False + if has_auth: + return ("ready", "") + return ( + "needs_setup", + "Tenki credentials not found — run `tenki login` or configure TENKI_AUTH_TOKEN/TENKI_API_KEY.", + ) + + +def _probe_terminal_backend( + name: str, + terminal_cfg: dict, + *, + profile_secrets: Optional[Dict[str, str]] = None, +) -> tuple: """Return ``(status, detail)`` for one backend. Never raises.""" try: if name == "local": @@ -16565,6 +16607,8 @@ def _probe_terminal_backend(name: str, terminal_cfg: dict) -> tuple: return _probe_modal_backend() if name == "daytona": return _probe_daytona_backend() + if name == "tenki": + return _probe_tenki_backend(profile_secrets) return ("unavailable", f"Unknown backend: {name}") except Exception as exc: # pragma: no cover — belt-and-braces guard return ("unavailable", f"Probe failed: {exc}") @@ -16580,26 +16624,45 @@ async def get_terminal_backends(profile: Optional[str] = None): Probes are fast (<~2s each) and defensive — a probe failure surfaces as a status, never an error response. """ - with _profile_scope(profile): - config = load_config() - terminal_cfg = config.get("terminal") - if not isinstance(terminal_cfg, dict): - terminal_cfg = {} - active = str(terminal_cfg.get("backend") or "local").strip().lower() - if active not in _TERMINAL_BACKEND_NAMES: - active = "local" - - backends = [] - for row in _TERMINAL_BACKENDS: - status, detail = _probe_terminal_backend(row["name"], terminal_cfg) - backends.append({ - "name": row["name"], - "label": row["label"], - "description": row["description"], - "active": row["name"] == active, - "status": status, - "detail": detail, - }) + with _profile_scope(profile) as profile_dir: + secret_token = None + profile_secrets = None + if profile_dir is not None: + from agent.secret_scope import ( + build_profile_secret_scope, + reset_secret_scope, + set_secret_scope, + ) + + profile_secrets = build_profile_secret_scope(Path(profile_dir)) + secret_token = set_secret_scope(profile_secrets) + try: + config = load_config() + terminal_cfg = config.get("terminal") + if not isinstance(terminal_cfg, dict): + terminal_cfg = {} + active = str(terminal_cfg.get("backend") or "local").strip().lower() + if active not in _TERMINAL_BACKEND_NAMES: + active = "local" + + backends = [] + for row in _TERMINAL_BACKENDS: + status, detail = _probe_terminal_backend( + row["name"], + terminal_cfg, + profile_secrets=profile_secrets, + ) + backends.append({ + "name": row["name"], + "label": row["label"], + "description": row["description"], + "active": row["name"] == active, + "status": status, + "detail": detail, + }) + finally: + if secret_token is not None: + reset_secret_scope(secret_token) return {"active": active, "backends": backends} @@ -16632,7 +16695,9 @@ async def select_terminal_backend( if not isinstance(terminal_cfg, dict): terminal_cfg = {} config["terminal"] = terminal_cfg - terminal_cfg["backend"] = backend + from hermes_cli.config import apply_terminal_backend_transition + + apply_terminal_backend_transition(terminal_cfg, backend) save_config(config) return {"ok": True, "backend": backend} diff --git a/model_tools.py b/model_tools.py index 7b5811e8fbfe4..8ba89cc8b7c6d 100644 --- a/model_tools.py +++ b/model_tools.py @@ -29,7 +29,11 @@ import time from typing import Dict, Any, List, Optional, Tuple -from tools.registry import discover_builtin_tools, registry +from tools.registry import ( + _CHECK_FN_TTL_SECONDS, + discover_builtin_tools, + registry, +) from toolsets import resolve_toolset, validate_toolset logger = logging.getLogger(__name__) @@ -321,15 +325,25 @@ def get_tool_definitions( try: from hermes_cli.config import get_config_path cfg_path = get_config_path() + cfg_path_key = str(cfg_path.resolve()) cfg_stat = cfg_path.stat() cfg_fp = (cfg_stat.st_mtime_ns, cfg_stat.st_size) except (FileNotFoundError, OSError, ImportError): + try: + cfg_path_key = str(cfg_path) + except UnboundLocalError: + cfg_path_key = "" cfg_fp = None cache_key = ( frozenset(enabled_toolsets) if enabled_toolsets is not None else None, frozenset(disabled_toolsets) if disabled_toolsets else None, registry._generation, + cfg_path_key, cfg_fp, + # The outer schema cache must eventually re-enter registry + # check_fn probes after credentials/services change. Otherwise it + # can outlive the inner availability TTL indefinitely. + int(time.monotonic() // _CHECK_FN_TTL_SECONDS), bool(os.environ.get("HERMES_KANBAN_TASK")), bool(skip_tool_search_assembly), _is_delegated_child_context(), diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index 6f4a309865ee7..5194205654480 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -1375,16 +1375,86 @@ def _fake_create_environment(*, env_type, **kwargs): assert "Linux 6.8.0" in line assert "root" in line + def test_tenki_prompt_probe_uses_static_fallback_without_creating_sandbox(self, monkeypatch): + import agent.prompt_builder as _pb + import tools.terminal_tool as _tt + + monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.setenv("TERMINAL_CONTAINER_PERSISTENT", "true") + monkeypatch.setenv("TERMINAL_TENKI_SYNC_HERMES_HOME", "true") + monkeypatch.setenv("TERMINAL_TENKI_FORWARD_ENV", '["NOTION_TOKEN"]') + monkeypatch.setenv("NOTION_TOKEN", "must-not-enter-a-probe") + _pb._clear_backend_probe_cache() + + factory_calls = [] + + def record_create(**kwargs): + factory_calls.append(kwargs) + return object() + + monkeypatch.setattr(_tt, "_create_environment", record_create) + + assert _pb._probe_remote_backend("tenki") is None + hint = _pb.build_environment_hints() + assert factory_calls == [] + assert "Terminal backend: tenki" in hint + assert "defers sandbox creation until the first tool call" in hint + assert "probe directly" not in hint + + def test_multiplexed_profile_hint_uses_scoped_tenki_backend( + self, + monkeypatch, + tmp_path, + ): + import agent.prompt_builder as _pb + import tools.terminal_tool as _tt + from agent.secret_scope import is_multiplex_active, set_multiplex_active + from gateway.run import _profile_runtime_scope + from hermes_cli import config as hermes_config + + profile_home = tmp_path / "profile" + profile_home.mkdir() + (profile_home / "config.yaml").write_text( + "terminal:\n" + " backend: tenki\n" + " cwd: /home/tenki\n", + encoding="utf-8", + ) + (profile_home / ".env").write_text( + "TENKI_AUTH_TOKEN=profile-token\n", + encoding="utf-8", + ) + monkeypatch.setenv("TERMINAL_ENV", "local") + previous = is_multiplex_active() + set_multiplex_active(True) + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() + _pb._clear_backend_probe_cache() + factory_calls = [] + monkeypatch.setattr( + _tt, + "_create_environment", + lambda **kwargs: factory_calls.append(kwargs), + ) + try: + with _profile_runtime_scope(profile_home): + assert _tt._get_env_config()["env_type"] == "tenki" + hint = _pb.build_environment_hints() + finally: + set_multiplex_active(previous) + + assert factory_calls == [] + assert "Terminal backend: tenki" in hint + assert "Host:" not in hint + def test_probe_container_config_uses_shared_builder(self, monkeypatch): """The probe must pass the canonical container config from ``_container_config_from_env_config`` — a stale inline copy omitted - ``tenki_sync_hermes_home`` / ``tenki_forward_env`` (and - ``docker_network``), so probe environments were built with different - settings than real ones.""" + backend settings such as ``docker_network``.""" import agent.prompt_builder as _pb import tools.terminal_tool as _tt - monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.setenv("TERMINAL_ENV", "docker") _pb._clear_backend_probe_cache() class _FakeEnv: @@ -1392,8 +1462,8 @@ def execute(self, cmd, timeout=None): return { "returncode": 0, "output": ( - "os=Linux\nkernel=6.8.0\nhome=/home/tenki\n" - "cwd=/home/tenki\nuser=tenki\n" + "os=Linux\nkernel=6.8.0\nhome=/root\n" + "cwd=/workspace\nuser=root\n" ), } @@ -1405,11 +1475,10 @@ def _fake_create_environment(*, env_type, **kwargs): monkeypatch.setattr(_tt, "_create_environment", _fake_create_environment) - assert _pb._probe_remote_backend("tenki") is not None + assert _pb._probe_remote_backend("docker") is not None container_config = created["container_config"] assert container_config is not None - for key in ("tenki_sync_hermes_home", "tenki_forward_env", "docker_network"): - assert key in container_config + assert "docker_network" in container_config def test_remote_backend_list_covers_known_sandboxes(self): """Regression guard: if someone adds a remote backend, they must list it here.""" @@ -1759,4 +1828,3 @@ def test_not_duplicated_in_google_guidance(self): # ========================================================================= # Budget warning history stripping # ========================================================================= - diff --git a/tests/hermes_cli/test_setup.py b/tests/hermes_cli/test_setup.py index 2f44614c3dbf8..4ae12d6e15b30 100644 --- a/tests/hermes_cli/test_setup.py +++ b/tests/hermes_cli/test_setup.py @@ -491,12 +491,16 @@ def fake_prompt_choice(question, choices, default=0): assert config["terminal"]["modal_mode"] == "direct" -def test_tenki_setup_switch_resets_inherited_persistent_container(monkeypatch): +def test_tenki_setup_switch_resets_inherited_persistent_container(tmp_path, monkeypatch): config = { "terminal": { "backend": "docker", "container_persistent": True, "cwd": "/workspace", + # YAML blank scalars load as None and must not become explicit + # "None" overrides when setup saves the config. + "tenki_api_endpoint": None, + "tenki_workspace_id": None, } } @@ -507,19 +511,80 @@ def fake_prompt_choice(question, choices, default=0): raise AssertionError(f"Unexpected prompt_choice call: {question}") monkeypatch.setitem(sys.modules, "tenki", types.ModuleType("tenki")) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") + monkeypatch.setenv("TENKI_API_ENDPOINT", "https://profile.tenki.test") + monkeypatch.setenv("TENKI_WORKSPACE_ID", "workspace-from-profile") monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") - monkeypatch.setattr("hermes_cli.setup.save_config", lambda _config: None) monkeypatch.setattr("hermes_cli.setup.save_env_value", lambda *_args, **_kwargs: None) from hermes_cli.setup import setup_terminal_backend + from hermes_cli.config import load_config_readonly + from tools.tenki_config import ( + resolve_tenki_api_endpoint, + resolve_tenki_workspace_id, + ) setup_terminal_backend(config) assert config["terminal"]["backend"] == "tenki" assert config["terminal"]["container_persistent"] is False assert config["terminal"]["cwd"] == "/home/tenki" + # Resolved fallbacks are displayed, not frozen into config. A later + # profile/CLI change must still be able to supply different values. + assert config["terminal"]["tenki_api_endpoint"] == "" + assert config["terminal"]["tenki_workspace_id"] == "" + + reloaded = load_config_readonly()["terminal"] + assert reloaded["tenki_api_endpoint"] == "" + assert reloaded["tenki_workspace_id"] == "" + + monkeypatch.setenv("TENKI_API_ENDPOINT", "https://later-profile.tenki.test") + monkeypatch.setenv("TENKI_WORKSPACE_ID", "workspace-from-later-profile") + assert ( + resolve_tenki_api_endpoint(reloaded["tenki_api_endpoint"]) + == "https://later-profile.tenki.test" + ) + assert ( + resolve_tenki_workspace_id(reloaded["tenki_workspace_id"]) + == "workspace-from-later-profile" + ) + + +def test_tenki_setup_preserves_explicit_endpoint_and_workspace(tmp_path, monkeypatch): + config = { + "terminal": { + "backend": "tenki", + "tenki_api_endpoint": " https://explicit.tenki.test ", + "tenki_workspace_id": " workspace-explicit ", + } + } + + def fake_prompt_choice(question, choices, default=0): + if question == "Select terminal backend:": + return choices.index("Tenki Agent - Tenki cloud sandbox") + raise AssertionError(f"Unexpected prompt_choice call: {question}") + + monkeypatch.setitem(sys.modules, "tenki", types.ModuleType("tenki")) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") + monkeypatch.setenv("TENKI_API_ENDPOINT", "https://fallback.tenki.test") + monkeypatch.setenv("TENKI_WORKSPACE_ID", "workspace-fallback") + monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) + monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") + monkeypatch.setattr("hermes_cli.setup.save_env_value", lambda *_args, **_kwargs: None) + + from hermes_cli.setup import setup_terminal_backend + from hermes_cli.config import load_config_readonly + + setup_terminal_backend(config) + + assert config["terminal"]["tenki_api_endpoint"] == "https://explicit.tenki.test" + assert config["terminal"]["tenki_workspace_id"] == "workspace-explicit" + reloaded = load_config_readonly()["terminal"] + assert reloaded["tenki_api_endpoint"] == "https://explicit.tenki.test" + assert reloaded["tenki_workspace_id"] == "workspace-explicit" # test_setup_slack_* moved to tests/gateway/test_slack_plugin_setup.py — the diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 24ef3d0f8c403..1aec046e3f160 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -6831,7 +6831,8 @@ def test_get_terminal_backends_shape_and_local_ready(self, monkeypatch): assert resp.status_code == 200 body = resp.json() names = [row["name"] for row in body["backends"]] - assert names == ["local", "docker", "singularity", "modal", "daytona", "ssh"] + assert "tenki" in names + assert len(names) == len(set(names)) assert body["active"] in set(names) for row in body["backends"]: assert row["status"] in {"ready", "needs_setup", "unavailable"} @@ -6948,6 +6949,110 @@ def test_terminal_ssh_probe_ready_when_configured(self, monkeypatch): assert ssh["status"] == "ready" assert "hermes@devbox.example.com" in ssh["detail"] + def test_terminal_tenki_probe_reports_auth_readiness(self, monkeypatch): + """Tenki readiness is a local SDK/auth check and never creates a sandbox.""" + import hermes_cli.web_server as web_server + import tools.tenki_config as tenki_config + + monkeypatch.setitem(sys.modules, "tenki", SimpleNamespace()) + monkeypatch.setattr(tenki_config, "has_tenki_auth", lambda: False) + assert web_server._probe_tenki_backend()[0] == "needs_setup" + + monkeypatch.setattr(tenki_config, "has_tenki_auth", lambda: True) + assert web_server._probe_tenki_backend() == ("ready", "") + + def test_terminal_tenki_probe_isolates_named_profile_secrets(self, monkeypatch): + """A target profile must see its own token and never borrow the + dashboard process token or another profile's machine-wide login.""" + import hermes_cli.web_server as web_server + from hermes_cli.profiles import get_profile_dir + + monkeypatch.setitem(sys.modules, "tenki", SimpleNamespace()) + monkeypatch.setenv("TENKI_API_KEY", "process-token-must-not-leak") + + with_token = get_profile_dir("tenki-ready") + with_token.mkdir(parents=True) + (with_token / ".env").write_text( + "TENKI_AUTH_TOKEN=profile-token\n", + encoding="utf-8", + ) + + without_token = get_profile_dir("tenki-missing") + without_token.mkdir(parents=True) + (without_token / ".env").write_text("OTHER_SECRET=x\n", encoding="utf-8") + + ready = self.client.get( + "/api/tools/terminal/backends?profile=tenki-ready" + ).json() + missing = self.client.get( + "/api/tools/terminal/backends?profile=tenki-missing" + ).json() + + ready_tenki = next(row for row in ready["backends"] if row["name"] == "tenki") + missing_tenki = next( + row for row in missing["backends"] if row["name"] == "tenki" + ) + assert ready_tenki["status"] == "ready" + assert missing_tenki["status"] == "needs_setup" + + def test_select_terminal_backend_accepts_and_preserves_tenki(self, monkeypatch): + """Dashboard GET/PUT must not rewrite a configured Tenki backend to local.""" + import hermes_cli.web_server as web_server + from hermes_cli.config import load_config, save_config + + monkeypatch.setattr( + web_server, + "_probe_tenki_backend", + lambda _profile_secrets=None: ("ready", ""), + ) + config = load_config() + config.setdefault("terminal", {}).update({ + "backend": "docker", + "container_persistent": True, + "cwd": "/workspace", + }) + save_config(config) + + resp = self.client.put( + "/api/tools/terminal/backend", json={"backend": "tenki"} + ) + assert resp.status_code == 200 + assert resp.json() == {"ok": True, "backend": "tenki"} + + body = self.client.get("/api/tools/terminal/backends").json() + assert body["active"] == "tenki" + tenki = next(r for r in body["backends"] if r["name"] == "tenki") + assert tenki["active"] is True + assert tenki["status"] == "ready" + terminal = load_config()["terminal"] + assert terminal["container_persistent"] is False + assert terminal["cwd"] == "/home/tenki" + + resp = self.client.put( + "/api/tools/terminal/backend", + json={"backend": "docker"}, + ) + assert resp.status_code == 200 + terminal = load_config()["terminal"] + assert terminal["backend"] == "docker" + assert terminal["container_persistent"] is True + assert terminal["cwd"] == "/workspace" + assert "_pre_tenki_backend_settings" not in terminal + + def test_terminal_backend_unknown_config_falls_back_to_known_active(self): + from hermes_cli.config import load_config, save_config + + config = load_config() + config.setdefault("terminal", {})["backend"] = "not-a-backend" + save_config(config) + + body = self.client.get("/api/tools/terminal/backends").json() + names = {row["name"] for row in body["backends"]} + assert body["active"] in names + assert [row["name"] for row in body["backends"] if row["active"]] == [ + body["active"] + ] + def test_select_terminal_backend_persists_config(self, monkeypatch): """PUT .../backend writes terminal.backend and the list reflects it.""" import hermes_cli.web_server as web_server diff --git a/tests/tools/test_credential_files.py b/tests/tools/test_credential_files.py index 0862b6722c848..906c4686c86a3 100644 --- a/tests/tools/test_credential_files.py +++ b/tests/tools/test_credential_files.py @@ -24,10 +24,10 @@ def _clean_state(): """Reset module state between tests.""" import tools.credential_files as _cred_mod clear_credential_files() - _cred_mod._config_files = None + _cred_mod._config_files.clear() yield clear_credential_files() - _cred_mod._config_files = None + _cred_mod._config_files.clear() class TestRegisterCredentialFiles: @@ -364,6 +364,40 @@ def test_config_legitimate_file_works(self, tmp_path, monkeypatch): assert len(mounts) == 1 assert "oauth.json" in mounts[0]["container_path"] + def test_config_credential_cache_is_profile_scoped(self, tmp_path): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + from hermes_cli import config as hermes_config + + home_a = tmp_path / "a" + home_b = tmp_path / "b" + home_a.mkdir() + home_b.mkdir() + (home_a / "service.json").write_text("profile-a", encoding="utf-8") + self._write_config(home_a, ["service.json"]) + self._write_config(home_b, []) + + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() + token = set_hermes_home_override(home_a) + try: + mounts_a = get_credential_file_mounts() + finally: + reset_hermes_home_override(token) + + token = set_hermes_home_override(home_b) + try: + mounts_b = get_credential_file_mounts() + finally: + reset_hermes_home_override(token) + + assert [Path(row["host_path"]) for row in mounts_a] == [ + home_a / "service.json" + ] + assert mounts_b == [] + # --------------------------------------------------------------------------- # Cache directory mounts diff --git a/tests/tools/test_file_sync.py b/tests/tools/test_file_sync.py index ce49b43647940..fbf59d4779a74 100644 --- a/tests/tools/test_file_sync.py +++ b/tests/tools/test_file_sync.py @@ -3,6 +3,7 @@ import io import os import tarfile +import threading import time from pathlib import Path from unittest.mock import MagicMock, patch @@ -401,3 +402,56 @@ def test_bulk_upload_rollback_on_failure(self, tmp_files): mgr.sync(force=True) bulk_upload.assert_called_once() assert len(bulk_upload.call_args[0][0]) == 3 + + def test_parallel_sync_transactions_are_serialized(self, tmp_path): + source = tmp_path / "changing.txt" + source.write_text("initial", encoding="utf-8") + get_calls = 0 + get_lock = threading.Lock() + active_uploads = 0 + max_active_uploads = 0 + upload_calls = 0 + upload_lock = threading.Lock() + + def get_files(): + nonlocal get_calls + with get_lock: + get_calls += 1 + source.write_text(f"version-{get_calls}", encoding="utf-8") + return [(str(source), "/root/.hermes/cache/changing.txt")] + + def bulk_upload(_files): + nonlocal active_uploads, max_active_uploads, upload_calls + with upload_lock: + active_uploads += 1 + upload_calls += 1 + max_active_uploads = max(max_active_uploads, active_uploads) + time.sleep(0.03) + with upload_lock: + active_uploads -= 1 + + manager = FileSyncManager( + get_files_fn=get_files, + upload_fn=MagicMock(), + delete_fn=MagicMock(), + bulk_upload_fn=bulk_upload, + ) + start = threading.Barrier(3) + + def sync(): + start.wait() + manager.sync(force=True) + + first = threading.Thread(target=sync) + second = threading.Thread(target=sync) + first.start() + second.start() + start.wait() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() + assert not second.is_alive() + assert upload_calls == 2 + assert max_active_uploads == 1 + assert manager._pushed_hashes["/root/.hermes/cache/changing.txt"] diff --git a/tests/tools/test_file_tools_container_config.py b/tests/tools/test_file_tools_container_config.py index cf549ae14876c..de8a961252fb7 100644 --- a/tests/tools/test_file_tools_container_config.py +++ b/tests/tools/test_file_tools_container_config.py @@ -2,8 +2,10 @@ import threading from unittest.mock import patch, MagicMock +import pytest import tools.code_execution_tool as code_execution_tool import tools.file_tools as file_tools +import tools.terminal_tool as terminal_tool def _make_env_config(**overrides): @@ -138,6 +140,7 @@ def fake_create_env(**kwargs): patch("tools.terminal_tool._task_env_overrides", {}), \ patch("tools.terminal_tool._active_environments", {}), \ patch("tools.terminal_tool._last_activity", {}), \ + patch("tools.terminal_tool._retiring_environments", {}), \ patch("tools.terminal_tool._env_lock", threading.Lock()), \ patch("tools.terminal_tool._creation_locks", {}), \ patch("tools.terminal_tool._creation_locks_lock", threading.Lock()), \ @@ -150,7 +153,8 @@ def fake_create_env(**kwargs): assert captured["env_type"] == "tenki" assert captured["image"] == "tenki-image" assert captured["cwd"] == "/home/tenki" - assert captured["task_id"] == "default" + assert captured["task_id"].startswith("tenki:") + assert captured["task_id"].endswith(":default") cc = captured["container_config"] assert cc["container_persistent"] is False assert cc["tenki_api_endpoint"] == "https://api.tenki.test" @@ -163,3 +167,261 @@ def fake_create_env(**kwargs): assert cc["tenki_pause_retention"] == 3600 assert cc["tenki_sync_hermes_home"] is True assert cc["tenki_forward_env"] == ["GITHUB_TOKEN"] + + def test_execute_code_sanitizes_host_cwd_when_it_creates_tenki_first(self): + captured = {} + mock_env = MagicMock() + env_config = _make_env_config( + env_type="tenki", + tenki_image="tenki-image", + cwd="/home/tenki", + ) + overrides = { + "desktop-session": { + "cwd": "/Users/alice/workspace", + }, + } + + def fake_create_env(**kwargs): + captured.update(kwargs) + return mock_env + + with patch("tools.terminal_tool._get_env_config", return_value=env_config), \ + patch("tools.terminal_tool._task_env_overrides", overrides), \ + patch("tools.terminal_tool._active_environments", {}), \ + patch("tools.terminal_tool._last_activity", {}), \ + patch("tools.terminal_tool._retiring_environments", {}), \ + patch("tools.terminal_tool._env_lock", threading.Lock()), \ + patch("tools.terminal_tool._creation_locks", {}), \ + patch("tools.terminal_tool._creation_locks_lock", threading.Lock()), \ + patch("tools.terminal_tool._create_environment", side_effect=fake_create_env), \ + patch("tools.terminal_tool._start_cleanup_thread"): + code_execution_tool._get_or_create_env("desktop-session") + + assert captured["cwd"] == "/home/tenki" + + +def test_tenki_live_cwd_registration_is_sanitized_for_file_operations(): + env_config = _make_env_config( + env_type="tenki", + tenki_image="tenki-image", + cwd="/home/tenki", + ) + raw_task_id = "desktop-live-cwd" + effective_task_id = terminal_tool._resolve_environment_cache_key( + raw_task_id, + "tenki", + ) + live_env = MagicMock() + live_env.cwd = "/home/tenki" + live_env.execute.return_value = { + "output": "ok", + "returncode": 0, + } + active = {effective_task_id: live_env} + session_cwds = {} + + with patch("tools.terminal_tool._get_env_config", return_value=env_config), \ + patch("tools.terminal_tool._task_env_overrides", {}), \ + patch("tools.terminal_tool._session_cwd", session_cwds), \ + patch("tools.terminal_tool._active_environments", active), \ + patch("tools.terminal_tool._last_activity", {}), \ + patch("tools.terminal_tool._retiring_environments", {}), \ + patch("tools.terminal_tool._env_lock", threading.Lock()), \ + patch("tools.terminal_tool._creation_locks", {}), \ + patch("tools.terminal_tool._creation_locks_lock", threading.Lock()), \ + patch("tools.file_tools._file_ops_cache", {}), \ + patch("tools.file_tools._file_ops_lock", threading.Lock()): + terminal_tool.register_task_env_overrides( + raw_task_id, + {"cwd": "/Users/alice/project"}, + ) + file_ops = file_tools._get_file_ops(raw_task_id) + file_ops._exec("pwd") + + # Keep the host workspace in session state for Desktop/ACP, but never send + # that host-only path to the Tenki guest. + assert session_cwds[raw_task_id] == "/Users/alice/project" + assert live_env.cwd == "/home/tenki" + live_env.execute.assert_called_once_with("pwd", cwd="/home/tenki") + + +def test_environment_creation_slot_survives_retirement_until_last_waiter(): + task_id = "tenki:profile:creation-slot" + old_env = MagicMock() + completed = threading.Event() + active = {task_id: old_env} + retirements = {task_id: completed} + creation_slots = {} + env_lock = threading.Lock() + creation_slots_lock = threading.Lock() + first_selected = threading.Event() + release_first = threading.Event() + second_acquired = threading.Event() + + def first_creator(): + with terminal_tool._environment_creation_lock(task_id): + _key, env = terminal_tool._select_active_environment(task_id) + assert env is None + first_selected.set() + assert release_first.wait(timeout=2) + + def second_creator(): + with terminal_tool._environment_creation_lock(task_id): + second_acquired.set() + + with patch("tools.terminal_tool._active_environments", active), \ + patch("tools.terminal_tool._last_activity", {task_id: 0}), \ + patch("tools.terminal_tool._retiring_environments", retirements), \ + patch("tools.terminal_tool._env_lock", env_lock), \ + patch("tools.terminal_tool._creation_locks", creation_slots), \ + patch("tools.terminal_tool._creation_locks_lock", creation_slots_lock), \ + patch("tools.file_tools._file_ops_cache", {}), \ + patch("tools.file_tools._file_ops_lock", threading.Lock()): + first = threading.Thread(target=first_creator) + first.start() + + # The first creator owns the slot but waits behind the retirement + # tombstone. Finishing retirement must release the tombstone without + # deleting that slot generation. + for _ in range(100): + with creation_slots_lock: + if task_id in creation_slots: + break + threading.Event().wait(0.01) + terminal_tool._finish_environment_retirement( + task_id, + old_env, + completed, + ) + assert first_selected.wait(timeout=2) + + with creation_slots_lock: + original_slot = creation_slots[task_id] + second = threading.Thread(target=second_creator) + second.start() + threading.Event().wait(0.05) + + with creation_slots_lock: + assert creation_slots[task_id] is original_slot + assert original_slot.users == 2 + assert second_acquired.is_set() is False + + release_first.set() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() + assert not second.is_alive() + assert second_acquired.is_set() is True + assert creation_slots == {} + + +def test_environment_registration_discards_losing_candidate(): + task_id = "tenki:profile:registration-loser" + existing = MagicMock() + candidate = MagicMock() + candidate.shares_remote_resource_with.return_value = False + + with patch( + "tools.terminal_tool._active_environments", + {task_id: existing}, + ), patch("tools.terminal_tool._env_lock", threading.Lock()): + selected = terminal_tool._register_active_environment( + task_id, + candidate, + ) + + assert selected is existing + candidate.discard.assert_called_once_with() + + +@pytest.mark.parametrize("entrypoint", ["terminal", "file", "code"]) +def test_environment_entrypoints_wait_before_selecting_retiring_tenki( + entrypoint, +): + env_config = _make_env_config( + env_type="tenki", + tenki_image="tenki-image", + cwd="/home/tenki", + ) + raw_task_id = "retirement-barrier" + effective_task_id = terminal_tool._resolve_environment_cache_key( + raw_task_id, + "tenki", + ) + old_env = MagicMock() + old_env.cwd = "/home/tenki" + replacement = MagicMock() + replacement.cwd = "/home/tenki" + replacement.execute.return_value = { + "output": "ok", + "returncode": 0, + } + active = {effective_task_id: old_env} + last_activity = {effective_task_id: 0} + retirements = {} + env_lock = threading.Lock() + wait_entered = threading.Event() + create_called = threading.Event() + + class TrackingEvent(threading.Event): + def wait(self, timeout=None): + wait_entered.set() + return super().wait(timeout) + + retirement = TrackingEvent() + retirements[effective_task_id] = retirement + + def fake_create_env(**_kwargs): + create_called.set() + return replacement + + result = [] + + def invoke(): + if entrypoint == "terminal": + result.append( + terminal_tool.terminal_tool( + "echo ok", + task_id=raw_task_id, + force=True, + ) + ) + elif entrypoint == "file": + result.append(file_tools._get_file_ops(raw_task_id)) + else: + result.append( + code_execution_tool._get_or_create_env(raw_task_id) + ) + + with patch("tools.terminal_tool._get_env_config", return_value=env_config), \ + patch("tools.terminal_tool._active_environments", active), \ + patch("tools.terminal_tool._last_activity", last_activity), \ + patch("tools.terminal_tool._retiring_environments", retirements), \ + patch("tools.terminal_tool._env_lock", env_lock), \ + patch("tools.terminal_tool._creation_locks", {}), \ + patch("tools.terminal_tool._creation_locks_lock", threading.Lock()), \ + patch("tools.terminal_tool._create_environment", side_effect=fake_create_env), \ + patch("tools.terminal_tool._start_cleanup_thread"), \ + patch("tools.file_tools._file_ops_cache", {}), \ + patch("tools.file_tools._file_ops_lock", threading.Lock()): + worker = threading.Thread(target=invoke) + worker.start() + assert wait_entered.wait(timeout=2) + assert create_called.is_set() is False + assert old_env.execute.called is False + + with env_lock: + active.pop(effective_task_id) + retirements.pop(effective_task_id) + retirement.set() + worker.join(timeout=2) + + assert not worker.is_alive() + assert create_called.is_set() is True + assert old_env.execute.called is False + if entrypoint == "file": + assert result[0].env is replacement + elif entrypoint == "code": + assert result[0] == (replacement, "tenki") diff --git a/tests/tools/test_shared_container_task_id.py b/tests/tools/test_shared_container_task_id.py index 3a66cde441eaf..7286aef04db84 100644 --- a/tests/tools/test_shared_container_task_id.py +++ b/tests/tools/test_shared_container_task_id.py @@ -54,6 +54,16 @@ def test_arbitrary_session_id_collapses_to_default(): assert terminal_tool._resolve_container_task_id("sess-123e4567-e89b-12d3") == "default" +def test_tenki_shaped_task_id_does_not_bypass_non_tenki_collapse(): + assert ( + terminal_tool._resolve_environment_cache_key( + "tenki:foreign-profile:default", + "docker", + ) + == "default" + ) + + def test_rl_task_with_override_keeps_its_own_id(): # RL / benchmark pattern: register a per-task image, then the task_id # must survive ``_resolve_container_task_id`` so the rollout lands in @@ -151,3 +161,146 @@ def test_env_type_override_keeps_own_id(): ) finally: terminal_tool.clear_task_env_overrides("bench-env") + + +def test_multiplexed_tenki_profiles_isolate_config_auth_and_tool_caches( + monkeypatch, + tmp_path, +): + """Two profiles in one gateway process must never share a Tenki sandbox, + workspace, credential, file-ops wrapper, or execute-code environment.""" + from agent.secret_scope import is_multiplex_active, set_multiplex_active + from gateway.run import _profile_runtime_scope + from hermes_cli import config as hermes_config + from hermes_constants import get_hermes_home + from tools import code_execution_tool, file_tools + from tools.tenki_config import resolve_tenki_auth_token + + # This test exercises process-wide cleanup, so give it an isolated + # lifecycle registry instead of consuming any environment a prior test + # intentionally left alive. + monkeypatch.setattr(terminal_tool, "_active_environments", {}) + monkeypatch.setattr(terminal_tool, "_last_activity", {}) + monkeypatch.setattr(terminal_tool, "_retiring_environments", {}) + monkeypatch.setattr(terminal_tool, "_creation_locks", {}) + + home_a = tmp_path / "profiles" / "a" + home_b = tmp_path / "profiles" / "b" + for home, workspace, token in ( + (home_a, "workspace-a", "token-a"), + (home_b, "workspace-b", "token-b"), + ): + home.mkdir(parents=True) + (home / "config.yaml").write_text( + "terminal:\n" + " backend: tenki\n" + " cwd: /home/tenki\n" + f" tenki_workspace_id: {workspace}\n", + encoding="utf-8", + ) + (home / ".env").write_text( + f"TENKI_AUTH_TOKEN={token}\n", + encoding="utf-8", + ) + + # These process-wide values belong to neither scoped profile. Config and + # auth resolution must override/fail closed around them. + monkeypatch.setenv("HERMES_PROFILE", "primary-process-profile") + monkeypatch.setenv("TERMINAL_ENV", "local") + monkeypatch.setenv("TERMINAL_TENKI_WORKSPACE_ID", "process-workspace-leak") + monkeypatch.setenv("TENKI_AUTH_TOKEN", "process-token-leak") + monkeypatch.setattr(terminal_tool, "_start_cleanup_thread", lambda: None) + + created = [] + + class FakeTenkiEnv: + def __init__(self, *, cwd, profile_home, workspace, token, task_id): + self.cwd = cwd + self.profile_home = profile_home + self.workspace = workspace + self.token = token + self.task_id = task_id + self.cleanup_calls = 0 + + def execute(self, _command, **_kwargs): + return { + "output": f"{self.profile_home}:{self.workspace}:{self.token}", + "returncode": 0, + } + + def cleanup(self): + self.cleanup_calls += 1 + + def fake_create_environment(**kwargs): + env = FakeTenkiEnv( + cwd=kwargs["cwd"], + profile_home=str(get_hermes_home()), + workspace=kwargs["container_config"]["tenki_workspace_id"], + token=resolve_tenki_auth_token(), + task_id=kwargs["task_id"], + ) + created.append(env) + return env + + monkeypatch.setattr( + terminal_tool, + "_create_environment", + fake_create_environment, + ) + + previous_multiplex = is_multiplex_active() + set_multiplex_active(True) + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() + try: + with _profile_runtime_scope(home_a): + terminal_tool.terminal_tool("printf a", task_id="shared", force=True) + env_a = terminal_tool.get_active_env("shared") + file_ops_a = file_tools._get_file_ops("shared") + code_env_a, code_backend_a = code_execution_tool._get_or_create_env( + "shared" + ) + + with _profile_runtime_scope(home_b): + terminal_tool.terminal_tool("printf b", task_id="shared", force=True) + env_b = terminal_tool.get_active_env("shared") + file_ops_b = file_tools._get_file_ops("shared") + code_env_b, code_backend_b = code_execution_tool._get_or_create_env( + "shared" + ) + + assert len(created) == 2 + assert env_a is created[0] + assert env_b is created[1] + assert env_a is not env_b + assert (env_a.workspace, env_a.token) == ("workspace-a", "token-a") + assert (env_b.workspace, env_b.token) == ("workspace-b", "token-b") + assert env_a.profile_home == str(home_a) + assert env_b.profile_home == str(home_b) + assert env_a.task_id != env_b.task_id + assert env_a.task_id.startswith("tenki:") + assert env_b.task_id.startswith("tenki:") + + assert file_ops_a.env is env_a + assert file_ops_b.env is env_b + assert file_ops_a is not file_ops_b + assert (code_env_a, code_backend_a) == (env_a, "tenki") + assert (code_env_b, code_backend_b) == (env_b, "tenki") + + # Process shutdown operates on canonical registry keys outside either + # profile scope. It must still retire both profile-owned wrappers. + assert terminal_tool.cleanup_all_environments() == 2 + assert env_a.cleanup_calls == 1 + assert env_b.cleanup_calls == 1 + assert not terminal_tool._active_environments + finally: + set_multiplex_active(previous_multiplex) + for env in created: + for key, value in list(terminal_tool._active_environments.items()): + if value is env: + terminal_tool._active_environments.pop(key, None) + terminal_tool._last_activity.pop(key, None) + terminal_tool._creation_locks.pop(key, None) + file_tools.clear_file_ops_cache() + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py index 32a9313cedbdd..c645baa1fecda 100644 --- a/tests/tools/test_tenki_environment.py +++ b/tests/tools/test_tenki_environment.py @@ -77,6 +77,7 @@ def __init__( name: str = "sb-test", state: str = "RUNNING", metadata: dict | None = None, + sandbox_id: str | None = "sb-test", ): self.exec_calls: list[tuple[tuple, dict]] = [] self.start_calls: list[tuple[tuple, dict]] = [] @@ -87,7 +88,7 @@ def __init__( self.resumed = False self.waited = False self.refreshed = False - self.id = "sb-test" + self.id = sandbox_id self.name = name self.state = state self.info = SimpleNamespace(name=name, metadata=metadata or {}) @@ -161,6 +162,10 @@ class _FakeInvalidStateError(Exception): """Mirrors tenki.InvalidStateError for the fake SDK.""" +class _FakeSessionNotFoundError(Exception): + """Mirrors tenki.SessionNotFoundError for the fake SDK.""" + + # Sentinel so the fake records exactly which kwargs the environment passed, # rather than the defaults it didn't. _UNSET = object() @@ -181,6 +186,7 @@ class _FakeSandboxFactory: created_kwargs: list[dict] = [] failed_kwargs: list[dict] = [] sandboxes: list[_FakeSandbox] = [] + next_sandbox_id = 0 fail_snapshot_ids: set[str] = set() # When a snapshot id is in fail_snapshot_ids, raise this exception type with # this message. Defaults to the confirmed-not-found error; tests set them to @@ -199,9 +205,11 @@ def _record_and_build(cls, kwargs: dict): if kwargs.get("snapshot_id") in cls.fail_snapshot_ids: cls.failed_kwargs.append(kwargs) raise cls.snapshot_error(cls.snapshot_error_msg) + cls.next_sandbox_id += 1 sandbox = _FakeSandbox( name=kwargs.get("name", "sb-test"), metadata=kwargs.get("metadata", {}), + sandbox_id=f"sb-created-{cls.next_sandbox_id}", ) cls.created_kwargs.append(kwargs) cls.sandboxes.append(sandbox) @@ -210,11 +218,36 @@ def _record_and_build(cls, kwargs: dict): class _FakeClient: listed_sandboxes: list[_FakeSandbox] = [] + remote_sandboxes: list[_FakeSandbox] = [] + snapshot_get_results: dict[str, object] = {} closed_count = 0 + active_generation = 0 + deleted_snapshot_ids: list[str] = [] def __init__(self, **kwargs): self.kwargs = kwargs - self.snapshots = SimpleNamespace(wait_durable=lambda *_args, **_kwargs: None) + self._generation = type(self).active_generation + + def get_snapshot(snapshot_id): + if snapshot_id in _FakeSandboxFactory.fail_snapshot_ids: + raise _FakeSandboxFactory.snapshot_error( + _FakeSandboxFactory.snapshot_error_msg + ) + if snapshot_id in type(self).snapshot_get_results: + return type(self).snapshot_get_results[snapshot_id] + return SimpleNamespace( + id=snapshot_id, + state="READY", + durability_state="DURABLE", + ) + + self.snapshots = SimpleNamespace( + get=get_snapshot, + wait_durable=lambda *_args, **_kwargs: None, + delete=lambda snapshot_id: type(self).deleted_snapshot_ids.append( + snapshot_id + ), + ) # The parameter list mirrors tenki 0.5's Client.create and deliberately has # NO **kwargs catch-all: a name the SDK dropped (project_id, removed in 0.5) @@ -254,10 +287,36 @@ def create( # tenki 0.5 folded list_project/list_workspace into list(workspace_id=...). def list(self, *, workspace_id=None, tags=None, sticky=None): - return list(self.listed_sandboxes) + candidates = [*self.listed_sandboxes, *_FakeSandboxFactory.sandboxes] + result = [] + seen: set[str] = set() + for sandbox in candidates: + if sandbox.id in seen: + continue + seen.add(sandbox.id) + result.append(sandbox) + if sandbox not in type(self).remote_sandboxes: + type(self).remote_sandboxes.append(sandbox) + return result + + def get(self, sandbox_id): + candidates = [ + *_FakeSandboxFactory.sandboxes, + *type(self).remote_sandboxes, + *type(self).listed_sandboxes, + ] + for sandbox in candidates: + if sandbox.id == sandbox_id: + return sandbox + raise _FakeSessionNotFoundError(sandbox_id) def close(self): - type(self).closed_count += 1 + # A delayed close from a prior test must not mutate the current test's + # generation after _install_fake_tenki resets shared fake state. + if getattr(self, "_generation", type(self).active_generation) == ( + type(self).active_generation + ): + type(self).closed_count += 1 def _install_fake_tenki(monkeypatch): @@ -269,16 +328,29 @@ def _install_fake_tenki(monkeypatch): _FakeSandboxFactory.snapshot_error = _FakeSnapshotNotFoundError _FakeSandboxFactory.snapshot_error_msg = "restore failed" _FakeClient.listed_sandboxes = [] + _FakeClient.remote_sandboxes = [] + _FakeClient.snapshot_get_results = {} + _FakeClient.active_generation += 1 _FakeClient.closed_count = 0 + _FakeClient.deleted_snapshot_ids = [] module.Client = _FakeClient module.Sandbox = _FakeSandboxFactory module.SnapshotNotFoundError = _FakeSnapshotNotFoundError module.RegistryImageNotFoundError = _FakeRegistryImageNotFoundError module.SnapshotNotDurableError = _FakeSnapshotNotDurableError module.InvalidStateError = _FakeInvalidStateError + module.SessionNotFoundError = _FakeSessionNotFoundError monkeypatch.setitem(sys.modules, "tenki", module) +def _release_quarantined_locks_since(tenki_module, start: int) -> None: + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + lock_files = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[start:] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[start:] + for lock_file in lock_files: + tenki_module._release_task_ownership_lock(lock_file) + + def _clear_tenki_auth_env(monkeypatch): monkeypatch.delenv("TENKI_AUTH_TOKEN", raising=False) monkeypatch.delenv("TENKI_API_KEY", raising=False) @@ -342,6 +414,7 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch encoding="utf-8", ) + from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) @@ -374,6 +447,8 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch assert "pause_retention" not in kwargs assert kwargs["metadata"]["hermes_backend"] == "tenki" assert kwargs["metadata"]["hermes_profile"] + assert kwargs["metadata"]["hermes_create_attempt"] + assert kwargs["wait"] is False assert kwargs["name"].startswith("hermes-") assert kwargs["name"].endswith("session-1") @@ -386,6 +461,7 @@ def test_tenki_environment_uses_cli_config_and_terminates_by_default(monkeypatch env.cleanup() assert sandbox.terminated is True assert sandbox.paused is False + assert tenki_module._get_create_attempt("session 1") is None def test_tenki_environment_does_not_inject_control_plane_token_by_default(monkeypatch, tmp_path): @@ -647,232 +723,405 @@ def test_tenki_environment_snapshots_when_persistent(monkeypatch, tmp_path): env.cleanup() -def test_tenki_environment_falls_back_when_persistent_snapshot_is_stale(monkeypatch, tmp_path): +def test_tenki_persistent_snapshot_retires_superseded_remote_copy( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - tenki_module._store_snapshot("persist", "snap-stale") - _FakeSandboxFactory.fail_snapshot_ids = {"snap-stale"} + tenki_module._store_snapshot("persist", "snap-prior") monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = env._sandbox - env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) - - assert _FakeSandboxFactory.failed_kwargs[0]["snapshot_id"] == "snap-stale" - assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" - assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) env.cleanup() + replacement_id = f"snap-{sandbox.name}" + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + replacement_id, + False, + ) + assert _FakeClient.deleted_snapshot_ids == ["snap-prior"] + assert sandbox.terminated is True + -def test_tenki_environment_preserves_snapshot_on_transient_restore_error(monkeypatch, tmp_path): +def test_tenki_snapshot_delete_failure_keeps_new_durable_pointer( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - tenki_module._store_snapshot("persist", "snap-transient") - _FakeSandboxFactory.fail_snapshot_ids = {"snap-transient"} - # A transient failure (not a confirmed not-found) must NOT boot a blank - # base image or drop the recovery pointer. - _FakeSandboxFactory.snapshot_error = RuntimeError + tenki_module._store_snapshot("persist", "snap-prior") + + delete_attempts = [] + + def flaky_delete(snapshot_id): + delete_attempts.append(snapshot_id) + if len(delete_attempts) == 1: + raise RuntimeError("control plane unavailable") + _FakeClient.deleted_snapshot_ids.append(snapshot_id) + + def _init(self, **kw): + self.kwargs = kw + self.snapshots = SimpleNamespace( + wait_durable=lambda *_args, **_kwargs: None, + delete=flaky_delete, + ) + + monkeypatch.setattr(_FakeClient, "__init__", _init) monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = env._sandbox - with pytest.raises(RuntimeError): - TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + env.cleanup() - # No base-image fallback happened, and the snapshot pointer is retained. - assert _FakeSandboxFactory.created_kwargs == [] - assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-transient", False) + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + f"snap-{sandbox.name}", + False, + ) + assert sandbox.terminated is True + assert tenki_module._pending_snapshot_retirements() == ("snap-prior",) + # The next lifecycle retries the durable profile-scoped journal without + # needing the failed predecessor to remain the active restore pointer. + retry_env = TenkiEnvironment( + task_id="persist", + persistent_filesystem=True, + ) -def test_tenki_environment_skips_snapshot_when_not_durable(monkeypatch, tmp_path): + assert delete_attempts[:2] == ["snap-prior", "snap-prior"] + assert _FakeClient.deleted_snapshot_ids == ["snap-prior"] + assert tenki_module._pending_snapshot_retirements() == () + with pytest.raises( + tenki_module._SnapshotPointerConflict, + match="already retired", + ): + tenki_module._store_snapshot("stale-writer", "snap-prior") + retry_env.cleanup() + + +def test_tenki_retirement_claim_is_durable_before_remote_delete( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - - def _fail_durable(*_args, **_kwargs): - raise RuntimeError("not durable yet") + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="retirement-claim") + tenki_module._queue_snapshot_retirement( + "snap-old", + env._snapshot_store, + ) + real_atomic_save = tenki_module._atomic_save_snapshots + + def fail_before_claim_replace(path, snapshots): + if snapshots.get( + tenki_module._snapshot_retired_key("snap-old") + ) == "snap-old": + raise OSError("ENOSPC before tombstone replace") + return real_atomic_save(path, snapshots) + + monkeypatch.setattr( + tenki_module, + "_atomic_save_snapshots", + fail_before_claim_replace, + ) - def _init(self, **kw): - self.kwargs = kw - self.snapshots = SimpleNamespace(wait_durable=_fail_durable) + assert env._retire_pending_snapshot_if_unreferenced( + "snap-old", + reason="claim failure probe", + ) is False + assert _FakeClient.deleted_snapshot_ids == [] - monkeypatch.setattr(_FakeClient, "__init__", _init) - monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) - sandbox = _FakeSandboxFactory.sandboxes[0] + def visible_but_uncertain_claim(path, snapshots): + real_atomic_save(path, snapshots) + raise tenki_module._SnapshotPointerCommitUncertain( + "directory fsync failed", + ) + monkeypatch.setattr( + tenki_module, + "_atomic_save_snapshots", + visible_but_uncertain_claim, + ) + assert env._retire_pending_snapshot_if_unreferenced( + "snap-old", + reason="uncertain claim probe", + ) is False + assert _FakeClient.deleted_snapshot_ids == [] + + # The visible uncertain claim is not acted on until its store is explicitly + # made durable. Once durable, deletion and non-resurrection are atomic in + # the required direction. + monkeypatch.setattr( + tenki_module, + "_atomic_save_snapshots", + real_atomic_save, + ) + tenki_module._confirm_snapshot_store_durable(env._snapshot_store) + assert env._retire_pending_snapshot_if_unreferenced( + "snap-old", + reason="claim retry", + ) is True + assert _FakeClient.deleted_snapshot_ids == ["snap-old"] + with pytest.raises( + tenki_module._SnapshotPointerConflict, + match="already retired", + ): + tenki_module._store_snapshot("stale-writer", "snap-old") env.cleanup() - # Durability failed → do NOT record the snapshot and do NOT terminate the - # live sandbox; pause it so state is preserved for recovery. - assert sandbox.paused is True - assert sandbox.terminated is False - assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) - -def test_tenki_environment_resumes_existing_persistent_sandbox(monkeypatch, tmp_path): +def test_tenki_windows_write_through_failure_keeps_remote_snapshot( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - token = tenki_module._profile_token() - existing = _FakeSandbox( - name=f"hermes-{token}-persist", - state="PAUSED", - metadata={"hermes_task_id": "persist", "hermes_profile": token}, - ) - _FakeClient.listed_sandboxes = [existing] - monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + env = TenkiEnvironment(task_id="windows-retirement") + tenki_module._queue_snapshot_retirement( + "snap-windows", + env._snapshot_store, + ) + monkeypatch.setattr(tenki_module, "_snapshot_platform", lambda: "nt") + monkeypatch.setattr( + tenki_module, + "_windows_replace_file_write_through", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("write-through replacement failed") + ), + ) - assert env._sandbox is existing - assert existing.resumed is True - assert existing.waited is True - assert _FakeSandboxFactory.created_kwargs == [] - # The resumed guest never receives the control-plane credential either. - assert "TENKI_AUTH_TOKEN" not in existing.exec_calls[-1][1]["env"] + assert env._retire_pending_snapshot_if_unreferenced( + "snap-windows", + reason="Windows durability probe", + ) is False + assert _FakeClient.deleted_snapshot_ids == [] + monkeypatch.setattr(tenki_module, "_snapshot_platform", lambda: "posix") env.cleanup() -def test_tenki_environment_does_not_reuse_other_profiles_sandbox(monkeypatch, tmp_path): +def test_tenki_environment_falls_back_when_persistent_snapshot_is_stale(monkeypatch, tmp_path): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - # A live sandbox on the same Tenki account belonging to a DIFFERENT profile - # (foreign token) with the same task id must never be resumed. - foreign = _FakeSandbox( - name="hermes-deadbeef00-persist", - state="PAUSED", - metadata={"hermes_task_id": "persist", "hermes_profile": "deadbeef00"}, - ) - _FakeClient.listed_sandboxes = [foreign] - + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-stale") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-stale"} monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) - assert env._sandbox is not foreign - assert foreign.resumed is False - assert _FakeSandboxFactory.created_kwargs, "should create its own sandbox" + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + # The snapshot is rejected by an authoritative preflight before the + # non-idempotent create RPC is issued. + assert _FakeSandboxFactory.failed_kwargs == [] + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + assert _FakeClient.deleted_snapshot_ids == ["snap-stale"] env.cleanup() -def test_tenki_reuse_rejects_name_match_with_foreign_profile_metadata(monkeypatch, tmp_path): - """Defense-in-depth: even if a candidate's NAME matches, a differing - hermes_profile in metadata must block reuse.""" +def test_tenki_snapshot_error_after_commit_never_creates_second_remote( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - token = tenki_module._profile_token() - # Same name (as if a token collision), but metadata says a different profile. - collider = _FakeSandbox( - name=f"hermes-{token}-persist", - state="PAUSED", - metadata={"hermes_task_id": "persist", "hermes_profile": "foreign-token"}, - ) - _FakeClient.listed_sandboxes = [collider] + task_id = "snapshot-error-after-commit" + tenki_module._store_snapshot(task_id, "snap-existing") + original_create = _FakeClient.create + + def commit_then_snapshot_error(self, **kwargs): + original_create(self, **kwargs) + raise _FakeSnapshotNotFoundError("decode failed after commit") + monkeypatch.setattr(_FakeClient, "create", commit_then_snapshot_error) monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) - assert env._sandbox is not collider - assert collider.resumed is False + env = TenkiEnvironment( + task_id=task_id, + image="base-image", + persistent_filesystem=True, + ) + + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert env._sandbox is _FakeSandboxFactory.sandboxes[0] + assert tenki_module._get_snapshot_restore_candidate(task_id) == ( + "snap-existing", + False, + ) + assert _FakeClient.deleted_snapshot_ids == [] env.cleanup() -def test_tenki_restore_falls_back_on_nondurable_snapshot(monkeypatch, tmp_path): - """A snapshot that EXISTS but is permanently unusable (non-durable) must - drop the pointer and boot the base image, not wedge the task forever.""" +@pytest.mark.parametrize( + ("state", "durability_state"), + [ + ("FAILED", "PROPAGATION_FAILED"), + ("READY", "UNSPECIFIED"), + ("DELETING", "DURABLE"), + ], +) +def test_tenki_snapshot_preflight_rejects_returned_unusable_state( + monkeypatch, + tmp_path, + state, + durability_state, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - tenki_module._store_snapshot("persist", "snap-nondurable") - _FakeSandboxFactory.fail_snapshot_ids = {"snap-nondurable"} - _FakeSandboxFactory.snapshot_error = _FakeSnapshotNotDurableError + task_id = f"bad-snapshot-{state.lower()}" + snapshot_id = f"snap-{state.lower()}" + tenki_module._store_snapshot(task_id, snapshot_id) + _FakeClient.snapshot_get_results[snapshot_id] = SimpleNamespace( + id=snapshot_id, + state=state, + durability_state=durability_state, + failure_reason="injected failure", + ) monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + env = TenkiEnvironment( + task_id=task_id, + image="base-image", + persistent_filesystem=True, + ) - assert _FakeSandboxFactory.failed_kwargs[0]["snapshot_id"] == "snap-nondurable" + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert "snapshot_id" not in _FakeSandboxFactory.created_kwargs[0] assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" - assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + assert tenki_module._get_snapshot_restore_candidate(task_id) == ( + None, + False, + ) env.cleanup() -def test_tenki_restore_preserves_pointer_on_invalid_state_error(monkeypatch, tmp_path): - """InvalidStateError is a generic precondition failure, NOT snapshot-gone, - so it must be treated as transient: preserve the pointer, do not base-boot.""" +@pytest.mark.parametrize( + ("state", "durability_state"), + [ + ("CREATING", "UNSPECIFIED"), + ("READY", "PROPAGATING"), + ], +) +def test_tenki_snapshot_preflight_preserves_transition_state( + monkeypatch, + tmp_path, + state, + durability_state, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - tenki_module._store_snapshot("persist", "snap-invalidstate") - _FakeSandboxFactory.fail_snapshot_ids = {"snap-invalidstate"} - _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError + task_id = f"transition-snapshot-{state.lower()}" + snapshot_id = f"snap-{state.lower()}" + tenki_module._store_snapshot(task_id, snapshot_id) + _FakeClient.snapshot_get_results[snapshot_id] = SimpleNamespace( + id=snapshot_id, + state=state, + durability_state=durability_state, + ) monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - with pytest.raises(_FakeInvalidStateError): - TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + with pytest.raises(RuntimeError, match="not durably restorable"): + TenkiEnvironment( + task_id=task_id, + image="base-image", + persistent_filesystem=True, + ) - assert _FakeSandboxFactory.created_kwargs == [] - assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-invalidstate", False) + assert not _FakeSandboxFactory.created_kwargs + assert tenki_module._get_snapshot_restore_candidate(task_id) == ( + snapshot_id, + False, + ) + assert _FakeClient.deleted_snapshot_ids == [] -def test_tenki_restore_falls_back_on_snapshot_specific_invalid_state(monkeypatch, tmp_path): - """A generic InvalidStateError whose message identifies the snapshot (the - SDK's collapsed representation of a bad/non-durable snapshot on restore) - IS unrecoverable → drop the pointer and boot the base image.""" +def test_tenki_environment_preserves_snapshot_on_transient_restore_error(monkeypatch, tmp_path): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) @@ -883,65 +1132,2417 @@ def test_tenki_restore_falls_back_on_snapshot_specific_invalid_state(monkeypatch from tools.environments.tenki import TenkiEnvironment monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - tenki_module._store_snapshot("persist", "snap-badstate") - _FakeSandboxFactory.fail_snapshot_ids = {"snap-badstate"} - _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError - _FakeSandboxFactory.snapshot_error_msg = "snapshot is not durable" + tenki_module._store_snapshot("persist", "snap-transient") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-transient"} + # A transient failure (not a confirmed not-found) must NOT boot a blank + # base image or drop the recovery pointer. + _FakeSandboxFactory.snapshot_error = RuntimeError monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + with pytest.raises(RuntimeError): + TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) - assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" - assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) - env.cleanup() + # No base-image fallback happened, and the snapshot pointer is retained. + assert _FakeSandboxFactory.created_kwargs == [] + assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-transient", False) -def test_tenki_snapshot_store_bound_to_construction_profile(monkeypatch, tmp_path): - """Cleanup (which may run in a background thread without the per-turn - HERMES_HOME contextvar) must write the snapshot pointer to the profile that - was active at construction, not whatever home is ambient at cleanup time.""" +def test_tenki_readiness_error_keeps_exact_snapshot_remote_and_pointer( + monkeypatch, + tmp_path, +): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) - (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment - home_a = tmp_path / "profiles" / "a" - home_b = tmp_path / "profiles" / "b" - home_a.mkdir(parents=True) - home_b.mkdir(parents=True) + tenki_module._store_snapshot("persist", "snap-ready-error") - monkeypatch.setenv("HERMES_HOME", str(home_a)) + def fail_readiness(self, *_args, **_kwargs): + raise _FakeSnapshotNotFoundError("readiness failed after create") + + monkeypatch.setattr(_FakeSandbox, "wait_ready", fail_readiness) monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) - # Simulate a background cleanup running under the WRONG ambient home. - monkeypatch.setenv("HERMES_HOME", str(home_b)) - env.cleanup() + with pytest.raises( + _FakeSnapshotNotFoundError, + match="readiness failed after create", + ): + TenkiEnvironment( + task_id="persist", + persistent_filesystem=True, + ) - # Pointer landed in profile A's store (construction-time), not B's. - assert (home_a / "tenki_snapshots.json").exists() - assert not (home_b / "tenki_snapshots.json").exists() + sandbox = _FakeSandboxFactory.sandboxes[0] + assert _FakeSandboxFactory.created_kwargs[0]["wait"] is False + assert sandbox.paused is True + assert sandbox.terminated is False + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + "snap-ready-error", + False, + ) -def test_tenki_persistent_not_terminated_when_snapshot_and_pause_both_fail(monkeypatch, tmp_path): - """Durability failed AND pause failed: the sandbox must be left live (not - terminated), so the only copy of un-snapshotted state is preserved.""" +def test_tenki_environment_skips_snapshot_when_not_durable(monkeypatch, tmp_path): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + from tools.environments import tenki as tenki_module from tools.environments.tenki import TenkiEnvironment monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-prior") def _fail_durable(*_args, **_kwargs): - raise RuntimeError("not durable") + raise RuntimeError("not durable yet") + + def _init(self, **kw): + self.kwargs = kw + self.snapshots = SimpleNamespace( + wait_durable=_fail_durable, + delete=lambda snapshot_id: type(self).deleted_snapshot_ids.append( + snapshot_id + ), + ) + + monkeypatch.setattr(_FakeClient, "__init__", _init) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = _FakeSandboxFactory.sandboxes[0] + + env.cleanup() + + # Durability failed → do NOT record the snapshot and do NOT terminate the + # live sandbox; pause it so state is preserved for recovery. + assert sandbox.paused is True + assert sandbox.terminated is False + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + "snap-prior", + False, + ) + assert _FakeClient.deleted_snapshot_ids == [f"snap-{sandbox.name}"] + + +def test_tenki_snapshot_pointer_failure_preserves_prior_and_retires_new_copy( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + tenki_module._store_snapshot("persist", "snap-prior") + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = env._sandbox + + def fail_pointer_store(*_args, **_kwargs): + raise OSError("disk full") + + monkeypatch.setattr(tenki_module, "_store_snapshot", fail_pointer_store) + env.cleanup() + + assert sandbox.paused is True + assert sandbox.terminated is False + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + "snap-prior", + False, + ) + assert _FakeClient.deleted_snapshot_ids == [f"snap-{sandbox.name}"] + + +def test_tenki_directory_fsync_uncertainty_keeps_both_snapshots_and_live_sandbox( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + tenki_module._store_snapshot("persist", "snap-prior") + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = env._sandbox + original_fsync = tenki_module.os.fsync + fsync_calls = 0 + + def fail_directory_fsync(fd): + nonlocal fsync_calls + fsync_calls += 1 + if fsync_calls == 2: + raise OSError("injected directory fsync failure") + return original_fsync(fd) + + monkeypatch.setattr(tenki_module.os, "fsync", fail_directory_fsync) + env.cleanup() + + replacement_id = f"snap-{sandbox.name}" + # os.replace made the new pointer visible, but failed directory fsync + # means a crash may reveal either pointer. Keep both remote copies and the + # live sandbox; do not act as though the handoff committed durably. + assert tenki_module._get_snapshot_restore_candidate("persist") == ( + replacement_id, + False, + ) + assert _FakeClient.deleted_snapshot_ids == [] + assert sandbox.paused is True + assert sandbox.terminated is False + + +def test_tenki_environment_refuses_unmanaged_persistent_sandbox( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + existing = _FakeSandbox( + name=f"hermes-{token}-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": token}, + ) + _FakeClient.listed_sandboxes = [existing] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert existing.resumed is False + assert existing.waited is False + assert _FakeSandboxFactory.created_kwargs == [] + assert existing.terminated is False + binding = tenki_module._load_snapshots()[ + tenki_module._remote_binding_key("persist") + ] + assert binding["conflicted"] is True + assert binding["unmanaged"] is True + assert binding["conflict_ids"] == [existing.id] + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_unidentified_collision_is_durably_unresolvable( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + unidentified = _FakeSandbox( + sandbox_id=None, + name=f"hermes-{token}-persist", + metadata={"hermes_task_id": "persist", "hermes_profile": token}, + ) + _FakeClient.listed_sandboxes = [unidentified] + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment(task_id="persist", persistent_filesystem=True) + assert tenki_module._remote_binding_state("persist") == ( + None, + None, + False, + True, + (), + True, + ) + assert not _FakeSandboxFactory.created_kwargs + _release_quarantined_locks_since(tenki_module, quarantine_start) + + # A later omission cannot erase an observed branch whose exact id was + # unavailable, so automatic recovery remains deliberately disabled. + _FakeClient.listed_sandboxes = [] + with pytest.raises( + RuntimeError, + match="unresolvable durable persistent-lineage conflict", + ): + TenkiEnvironment(task_id="persist", persistent_filesystem=True) + assert not _FakeSandboxFactory.created_kwargs + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_mixed_unmanaged_collision_preserves_known_ids( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + task_id = "mixed-unmanaged" + token = tenki_module._profile_token() + metadata = { + "hermes_task_id": task_id, + "hermes_profile": token, + } + known = _FakeSandbox( + sandbox_id="known-unmanaged", + name=f"hermes-{token}-{task_id}", + metadata=metadata, + ) + unidentified = _FakeSandbox( + sandbox_id=None, + name=f"hermes-{token}-{task_id}", + metadata=metadata, + ) + _FakeClient.listed_sandboxes = [known, unidentified] + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment(task_id=task_id, persistent_filesystem=True) + + assert tenki_module._remote_binding_state(task_id) == ( + known.id, + None, + False, + True, + (known.id,), + True, + ) + assert not _FakeSandboxFactory.created_kwargs + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_registration_loser_sharing_remote_releases_only_wrapper( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools import terminal_tool + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + winner = TenkiEnvironment( + task_id="shared-remote", + persistent_filesystem=True, + ) + remote = winner._sandbox + # Production cannot construct a second wrapper while this lifetime lock is + # held. Release it deliberately to exercise only the defensive + # same-remote registration path. + winner._release_task_ownership() + _FakeClient.listed_sandboxes = [remote] + loser = TenkiEnvironment( + task_id="shared-remote", + persistent_filesystem=True, + ) + + assert loser._sandbox is remote + assert loser.shares_remote_resource_with(winner) is True + + registry_key = "tenki:profile:shared-remote" + with monkeypatch.context() as context: + context.setattr( + terminal_tool, + "_active_environments", + {registry_key: winner}, + ) + context.setattr(terminal_tool, "_env_lock", threading.Lock()) + selected = terminal_tool._register_active_environment( + registry_key, + loser, + ) + + assert selected is winner + assert winner._sandbox is remote + assert remote.terminated is False + assert loser._sandbox is None + assert loser._client is None + assert loser._cleanup_complete is True + assert _FakeClient.closed_count == 1 + winner.cleanup() + + +def test_tenki_persistent_list_failure_never_authorizes_creation( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + def fail_list(self, **_kwargs): + raise RuntimeError("control plane unavailable") + + monkeypatch.setattr(_FakeClient, "list", fail_list) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + with pytest.raises(RuntimeError, match="could not check"): + TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs == [] + + +def test_tenki_exact_binding_survives_list_omission_on_create_and_restart( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + monkeypatch.setattr(_FakeClient, "list", lambda self, **_kwargs: []) + task_id = "list-omits-owned-binding" + + first = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + remote = first._sandbox + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert tenki_module._remote_binding_state(task_id) == ( + remote.id, + remote.info.metadata["hermes_create_attempt"], + True, + False, + (), + False, + ) + + # Simulate a process exit: the durable exact-id binding survives, while + # the list API still omits the live remote. + first._release_task_ownership() + first._sandbox = None + first._client = None + first._cleanup_complete = True + + successor = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert successor._sandbox is remote + assert len(_FakeSandboxFactory.created_kwargs) == 1 + successor.cleanup() + + +def test_tenki_persistent_readiness_errors_preserve_exact_sandbox( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + sandbox = env._sandbox + created_before = len(_FakeSandboxFactory.created_kwargs) + + def fail_refresh(): + raise RuntimeError("transient refresh failure") + + monkeypatch.setattr(sandbox, "refresh", fail_refresh) + + with pytest.raises(RuntimeError, match="could not refresh"): + env._ensure_sandbox() + + assert env._sandbox is sandbox + assert len(_FakeSandboxFactory.created_kwargs) == created_before + + +def test_tenki_unmanaged_persistent_remote_never_resumes_or_forks( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + existing = _FakeSandbox( + name=f"hermes-{token}-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": token}, + ) + + def fail_resume(): + raise RuntimeError("transient resume failure") + + existing.resume = fail_resume + _FakeClient.listed_sandboxes = [existing] + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs == [] + assert existing.resumed is False + assert existing.terminated is False + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_profile_identity_is_stable_across_runtime_modes( + monkeypatch, + tmp_path, +): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + from tools.environments import tenki as tenki_module + + profile_home = tmp_path / "profiles" / "coder" + profile_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(profile_home)) + monkeypatch.setenv("HERMES_PROFILE", "coder") + standalone_token = tenki_module._profile_token() + + token = set_hermes_home_override(profile_home) + try: + multiplexed_token = tenki_module._profile_token() + finally: + reset_hermes_home_override(token) + + assert multiplexed_token == standalone_token + old_name_token = tenki_module._profile_token_for_basis("profile:coder") + assert old_name_token in tenki_module._legacy_profile_tokens() + + +def test_tenki_migrates_legacy_profile_snapshot_key(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + profile_home = tmp_path / "profiles" / "coder" + profile_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(profile_home)) + monkeypatch.setenv("HERMES_PROFILE", "coder") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + canonical_token = tenki_module._profile_token() + legacy_token = tenki_module._profile_token_for_basis("profile:coder") + canonical_task = f"tenki:{canonical_token}:default" + legacy_task = f"tenki:{legacy_token}:default" + tenki_module._store_snapshot(legacy_task, "snap-legacy-profile") + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment( + task_id=canonical_task, + persistent_filesystem=True, + ) + + assert _FakeSandboxFactory.created_kwargs[0]["snapshot_id"] == ( + "snap-legacy-profile" + ) + assert tenki_module._get_snapshot_restore_candidate( + canonical_task, + ) == ("snap-legacy-profile", False) + assert tenki_module._get_snapshot_restore_candidate( + legacy_task, + ) == (None, False) + env.cleanup() + + +def test_tenki_legacy_migration_cas_never_overwrites_newer_pointer( + monkeypatch, + tmp_path, +): + from tools.environments import tenki as tenki_module + + store = tmp_path / "tenki_snapshots.json" + legacy_task = "tenki:legacy:default" + canonical_task = "tenki:canonical:default" + tenki_module._store_snapshot( + legacy_task, + "snap-legacy", + store, + ) + tenki_module._store_snapshot( + canonical_task, + "snap-newer", + store, + ) + + with pytest.raises( + tenki_module._SnapshotPointerConflict, + match="advanced", + ): + tenki_module._migrate_snapshot_pointer( + canonical_task, + legacy_task, + "snap-legacy", + store, + ) + + assert tenki_module._get_snapshot_restore_candidate( + canonical_task, + store, + ) == ("snap-newer", False) + assert tenki_module._get_snapshot_restore_candidate( + legacy_task, + store, + ) == ("snap-legacy", False) + + +def test_tenki_legacy_pointer_migration_failure_keeps_restored_sandbox( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + profile_home = tmp_path / "profiles" / "coder" + profile_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(profile_home)) + monkeypatch.setenv("HERMES_PROFILE", "coder") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + canonical_token = tenki_module._profile_token() + legacy_token = tenki_module._profile_token_for_basis("profile:coder") + canonical_task = f"tenki:{canonical_token}:default" + legacy_task = f"tenki:{legacy_token}:default" + tenki_module._store_snapshot(legacy_task, "snap-legacy-profile") + original_migrate = tenki_module._migrate_snapshot_pointer + + def fail_migration(*_args, **_kwargs): + raise OSError("local pointer store unavailable") + + monkeypatch.setattr( + tenki_module, + "_migrate_snapshot_pointer", + fail_migration, + ) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment( + task_id=canonical_task, + persistent_filesystem=True, + ) + restored = env._sandbox + + assert restored is _FakeSandboxFactory.sandboxes[0] + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert _FakeSandboxFactory.created_kwargs[0]["snapshot_id"] == ( + "snap-legacy-profile" + ) + assert tenki_module._get_snapshot_restore_candidate(legacy_task) == ( + "snap-legacy-profile", + False, + ) + assert env._snapshot_restore_task_id == legacy_task + assert env._save_persistent_snapshot(restored) is False + assert restored.snapshots == [] + + monkeypatch.setattr( + tenki_module, + "_migrate_snapshot_pointer", + original_migrate, + ) + env._exec_raw("echo migration retry") + + assert env._sandbox is restored + assert tenki_module._get_snapshot_restore_candidate(canonical_task) == ( + "snap-legacy-profile", + False, + ) + assert tenki_module._get_snapshot_restore_candidate(legacy_task) == ( + None, + False, + ) + env.cleanup() + + +def test_tenki_refuses_legacy_profile_named_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + profile_home = tmp_path / "profiles" / "coder" + profile_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(profile_home)) + monkeypatch.setenv("HERMES_PROFILE", "coder") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + canonical_token = tenki_module._profile_token() + legacy_token = tenki_module._profile_token_for_basis("profile:coder") + canonical_task = f"tenki:{canonical_token}:default" + legacy_task = f"tenki:{legacy_token}:default" + existing = _FakeSandbox( + name=( + f"hermes-{legacy_token}-" + f"{tenki_module._safe_name(legacy_task)}" + ), + state="PAUSED", + metadata={ + "hermes_task_id": legacy_task, + "hermes_profile": legacy_token, + }, + ) + _FakeClient.listed_sandboxes = [existing] + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment( + task_id=canonical_task, + persistent_filesystem=True, + ) + + assert existing.resumed is False + assert existing.terminated is False + assert _FakeSandboxFactory.created_kwargs == [] + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_environment_does_not_reuse_other_profiles_sandbox(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + # A live sandbox on the same Tenki account belonging to a DIFFERENT profile + # (foreign token) with the same task id must never be resumed. + foreign = _FakeSandbox( + name="hermes-deadbeef00-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": "deadbeef00"}, + ) + _FakeClient.listed_sandboxes = [foreign] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert env._sandbox is not foreign + assert foreign.resumed is False + assert _FakeSandboxFactory.created_kwargs, "should create its own sandbox" + env.cleanup() + + +def test_tenki_reuse_rejects_name_match_with_foreign_profile_metadata(monkeypatch, tmp_path): + """Defense-in-depth: even if a candidate's NAME matches, a differing + hermes_profile in metadata must block reuse.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + # Same name (as if a token collision), but metadata says a different profile. + collider = _FakeSandbox( + name=f"hermes-{token}-persist", + state="PAUSED", + metadata={"hermes_task_id": "persist", "hermes_profile": "foreign-token"}, + ) + _FakeClient.listed_sandboxes = [collider] + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert env._sandbox is not collider + assert collider.resumed is False + env.cleanup() + + +def test_tenki_restore_falls_back_on_nondurable_snapshot(monkeypatch, tmp_path): + """A snapshot that EXISTS but is permanently unusable (non-durable) must + drop the pointer and boot the base image, not wedge the task forever.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-nondurable") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-nondurable"} + _FakeSandboxFactory.snapshot_error = _FakeSnapshotNotDurableError + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.failed_kwargs == [] + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + env.cleanup() + + +def test_tenki_restore_preserves_pointer_on_invalid_state_error(monkeypatch, tmp_path): + """InvalidStateError is a generic precondition failure, NOT snapshot-gone, + so it must be treated as transient: preserve the pointer, do not base-boot.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-invalidstate") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-invalidstate"} + _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + with pytest.raises(_FakeInvalidStateError): + TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs == [] + assert tenki_module._get_snapshot_restore_candidate("persist") == ("snap-invalidstate", False) + + +def test_tenki_restore_falls_back_on_snapshot_specific_invalid_state(monkeypatch, tmp_path): + """A generic InvalidStateError whose message identifies the snapshot (the + SDK's collapsed representation of a bad/non-durable snapshot on restore) + IS unrecoverable → drop the pointer and boot the base image.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + tenki_module._store_snapshot("persist", "snap-badstate") + _FakeSandboxFactory.fail_snapshot_ids = {"snap-badstate"} + _FakeSandboxFactory.snapshot_error = _FakeInvalidStateError + _FakeSandboxFactory.snapshot_error_msg = "snapshot is not durable" + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + env = TenkiEnvironment(task_id="persist", image="base-image", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs[0]["image"] == "base-image" + assert tenki_module._get_snapshot_restore_candidate("persist") == (None, False) + env.cleanup() + + +def test_tenki_snapshot_store_bound_to_construction_profile(monkeypatch, tmp_path): + """Cleanup (which may run in a background thread without the per-turn + HERMES_HOME contextvar) must write the snapshot pointer to the profile that + was active at construction, not whatever home is ambient at cleanup time.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + home_a = tmp_path / "profiles" / "a" + home_b = tmp_path / "profiles" / "b" + home_a.mkdir(parents=True) + home_b.mkdir(parents=True) + + monkeypatch.setenv("HERMES_HOME", str(home_a)) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + # Simulate a background cleanup running under the WRONG ambient home. + monkeypatch.setenv("HERMES_HOME", str(home_b)) + env.cleanup() + + # Pointer landed in profile A's store (construction-time), not B's. + assert (home_a / "tenki_snapshots.json").exists() + assert not (home_b / "tenki_snapshots.json").exists() + + +def test_tenki_sync_enumeration_and_cleanup_stay_bound_to_owner_profile( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + process_home = tmp_path / "process-home" + process_home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(process_home)) + tenki_config = tmp_path / "tenki-cli.yaml" + tenki_config.write_text("auth_token: tok-secret\n", encoding="utf-8") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tenki_config)) + + from hermes_cli import config as hermes_config + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + from tools import credential_files + from tools.environments.tenki import TenkiEnvironment + + home_a = tmp_path / "profiles" / "a" + home_b = tmp_path / "profiles" / "b" + home_a.mkdir(parents=True) + home_b.mkdir(parents=True) + (home_a / "service.json").write_text("profile-a", encoding="utf-8") + (home_a / "config.yaml").write_text( + "terminal:\n credential_files:\n - service.json\n", + encoding="utf-8", + ) + (home_b / "config.yaml").write_text( + "terminal:\n credential_files: []\n", + encoding="utf-8", + ) + credential_files._config_files.clear() + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + token = set_hermes_home_override(home_a) + try: + env_a = TenkiEnvironment(task_id="sync-a", sync_hermes_home=True) + finally: + reset_hermes_home_override(token) + token = set_hermes_home_override(home_b) + try: + env_b = TenkiEnvironment(task_id="sync-b", sync_hermes_home=True) + finally: + reset_hermes_home_override(token) + + files_a = env_a._sync_manager._get_files_fn() + files_b = env_b._sync_manager._get_files_fn() + assert any(host == str(home_a / "service.json") for host, _remote in files_a) + assert all(host != str(home_a / "service.json") for host, _remote in files_b) + + sync_back_homes = [] + monkeypatch.setattr( + env_a._sync_manager, + "sync_back", + lambda home=None: sync_back_homes.append(home), + ) + monkeypatch.setattr( + env_b._sync_manager, + "sync_back", + lambda home=None: sync_back_homes.append(home), + ) + first = threading.Thread(target=env_a.cleanup) + second = threading.Thread(target=env_b.cleanup) + first.start() + second.start() + first.join(timeout=2) + second.join(timeout=2) + + assert sync_back_homes == [home_a, home_b] or sync_back_homes == [home_b, home_a] + assert process_home not in sync_back_homes + + +def test_tenki_snapshot_pointer_updates_are_serialized_and_atomic( + monkeypatch, + tmp_path, +): + from tools.environments import tenki as tenki_module + + store = tmp_path / "tenki_snapshots.json" + active_loads = 0 + max_active_loads = 0 + state_lock = threading.Lock() + original_load = tenki_module._load_json_store + + def tracked_load(path): + nonlocal active_loads, max_active_loads + with state_lock: + active_loads += 1 + max_active_loads = max(max_active_loads, active_loads) + time.sleep(0.02) + try: + return original_load(path) + finally: + with state_lock: + active_loads -= 1 + + monkeypatch.setattr(tenki_module, "_load_json_store", tracked_load) + start = threading.Barrier(3) + + def save(task_id, snapshot_id): + start.wait() + tenki_module._store_snapshot(task_id, snapshot_id, store) + + first = threading.Thread(target=save, args=("task-a", "snap-a")) + second = threading.Thread(target=save, args=("task-b", "snap-b")) + first.start() + second.start() + start.wait() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() + assert not second.is_alive() + assert max_active_loads == 1 + assert tenki_module._load_snapshots(store) == { + "direct:task-a": "snap-a", + "direct:task-b": "snap-b", + } + assert list(tmp_path.glob(".tenki_snapshots.json.*.tmp")) == [] + + +def test_tenki_malformed_recovery_registry_fails_closed( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + store = tmp_path / "tenki_snapshots.json" + malformed = '{"direct:task":"sole-snapshot"' + store.write_text(malformed, encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + + with pytest.raises(RuntimeError, match="recovery registry is unreadable"): + TenkiEnvironment( + task_id="task", + persistent_filesystem=True, + ) + + assert store.read_text(encoding="utf-8") == malformed + assert not _FakeSandboxFactory.created_kwargs + + +def test_tenki_snapshot_store_uses_windows_cross_process_lock( + monkeypatch, + tmp_path, +): + from tools.environments import tenki as tenki_module + + lock_calls = [] + fake_msvcrt = SimpleNamespace( + LK_LOCK=1, + LK_UNLCK=2, + locking=lambda fd, mode, size: lock_calls.append((fd, mode, size)), + ) + store = tmp_path / "tenki_snapshots.json" + monkeypatch.setattr(tenki_module, "_fcntl", None) + monkeypatch.setattr(tenki_module, "_msvcrt", fake_msvcrt) + + tenki_module._store_snapshot("task", "snap", store) + + assert [mode for _fd, mode, _size in lock_calls] == [1, 2] + assert all(size == 1 for _fd, _mode, size in lock_calls) + assert tenki_module._get_snapshot_restore_candidate("task", store) == ( + "snap", + False, + ) + + +def test_tenki_snapshot_store_uses_windows_write_through_replace_and_confirm( + monkeypatch, + tmp_path, +): + from tools.environments import tenki as tenki_module + + store = tmp_path / "tenki_snapshots.json" + replacements = [] + + def write_through_replace(source, destination): + replacements.append((source, destination)) + tenki_module.os.replace(source, destination) + + monkeypatch.setattr(tenki_module, "_snapshot_platform", lambda: "nt") + monkeypatch.setattr( + tenki_module, + "_windows_replace_file_write_through", + write_through_replace, + ) + + tenki_module._store_snapshot("task", "snap", store) + assert len(replacements) == 1 + assert tenki_module._get_snapshot_restore_candidate("task", store) == ( + "snap", + False, + ) + + tenki_module._confirm_snapshot_store_durable(store) + assert len(replacements) == 2 + assert tenki_module._get_snapshot_restore_candidate("task", store) == ( + "snap", + False, + ) + + +def test_tenki_windows_replace_requests_movefile_write_through( + monkeypatch, + tmp_path, +): + import ctypes + + from tools.environments import tenki as tenki_module + + calls = [] + + class FakeMoveFileEx: + argtypes = None + restype = None + + def __call__(self, source, destination, flags): + calls.append((source, destination, flags)) + return True + + fake_move = FakeMoveFileEx() + fake_kernel = SimpleNamespace(MoveFileExW=fake_move) + monkeypatch.setattr( + ctypes, + "WinDLL", + lambda *_args, **_kwargs: fake_kernel, + raising=False, + ) + + source = str(tmp_path / "source.json") + destination = tmp_path / "destination.json" + tenki_module._windows_replace_file_write_through( + source, + destination, + ) + + assert calls == [ + ( + tenki_module.os.path.abspath(source), + tenki_module.os.path.abspath(destination), + 0x1 | 0x8, + ) + ] + + +def test_tenki_snapshot_store_fails_closed_without_os_file_lock( + monkeypatch, + tmp_path, +): + from tools.environments import tenki as tenki_module + + store = tmp_path / "tenki_snapshots.json" + monkeypatch.setattr(tenki_module, "_fcntl", None) + monkeypatch.setattr(tenki_module, "_msvcrt", None) + + with pytest.raises(RuntimeError, match="requires fcntl or msvcrt"): + tenki_module._store_snapshot("task", "snap", store) + + assert store.exists() is False + + +def test_tenki_task_ownership_lock_spans_wrapper_lifetime( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + owner = TenkiEnvironment( + task_id="owned-task", + persistent_filesystem=True, + ) + + with pytest.raises(RuntimeError, match="already active"): + TenkiEnvironment( + task_id="owned-task", + persistent_filesystem=True, + ) + + assert len(_FakeSandboxFactory.created_kwargs) == 1 + owner.cleanup() + + successor = TenkiEnvironment( + task_id="owned-task", + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 2 + successor.cleanup() + + +def test_tenki_failed_initialization_quiesces_before_releasing_ownership( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + partial_wrappers = [] + + def fail_init_session(self): + partial_wrappers.append(self) + raise RuntimeError("session initialization failed") + + monkeypatch.setattr(TenkiEnvironment, "init_session", fail_init_session) + with pytest.raises(RuntimeError, match="session initialization failed"): + TenkiEnvironment( + task_id="failed-init", + persistent_filesystem=True, + ) + + partial = partial_wrappers[0] + sandbox = _FakeSandboxFactory.sandboxes[0] + assert sandbox.paused is True + assert sandbox.terminated is False + assert partial._sandbox is None + assert partial._cleanup_complete is True + + # A successor can safely resume the exact quiesced remote. Delayed cleanup + # of the failed wrapper is inert and cannot terminate it underneath the + # successor. + _FakeClient.listed_sandboxes = [sandbox] + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + successor = TenkiEnvironment( + task_id="failed-init", + persistent_filesystem=True, + ) + assert successor._sandbox is sandbox + partial.cleanup() + assert sandbox.terminated is False + successor.cleanup() + + +def test_tenki_failed_initialization_quarantines_unsafe_task_lock( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + partial_wrappers = [] + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + def fail_pause(self): + raise RuntimeError("pause unavailable") + + def fail_init_session(self): + partial_wrappers.append(self) + raise RuntimeError("session initialization failed") + + monkeypatch.setattr(_FakeSandbox, "pause", fail_pause) + monkeypatch.setattr(tenki_module, "_TERMINATE_RETRY_DELAYS", ()) + monkeypatch.setattr(TenkiEnvironment, "init_session", fail_init_session) + with pytest.raises(RuntimeError, match="session initialization failed"): + TenkiEnvironment( + task_id="unsafe-failed-init", + persistent_filesystem=True, + ) + + partial = partial_wrappers[0] + sandbox = _FakeSandboxFactory.sandboxes[0] + assert sandbox.terminated is False + assert partial._cleanup_complete is True + with pytest.raises(RuntimeError, match="already active"): + TenkiEnvironment( + task_id="unsafe-failed-init", + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + partial.cleanup() + assert sandbox.terminated is False + + # Release this test's fail-closed quarantine explicitly; production keeps + # it until process exit. + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + quarantined = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in quarantined: + tenki_module._release_task_ownership_lock(lock_file) + + +def test_tenki_ambiguous_create_is_reconciled_before_ownership_release( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + original_create = _FakeClient.create + + def commit_then_timeout(self, **kwargs): + sandbox = original_create(self, **kwargs) + _FakeClient.listed_sandboxes = [sandbox] + raise TimeoutError("response lost after commit") + + monkeypatch.setattr(_FakeClient, "create", commit_then_timeout) + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(TimeoutError, match="response lost"): + TenkiEnvironment(task_id="ambiguous-create") + + first_remote = _FakeSandboxFactory.sandboxes[0] + assert first_remote.terminated is True + assert len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) == quarantine_start + + monkeypatch.setattr(_FakeClient, "create", original_create) + successor = TenkiEnvironment(task_id="ambiguous-create") + assert successor._sandbox is not first_remote + assert len(_FakeSandboxFactory.created_kwargs) == 2 + successor.cleanup() + + +def test_tenki_unreconciled_ambiguous_create_quarantines_task_lock( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + def timeout_without_visible_remote(self, **_kwargs): + raise TimeoutError("ambiguous create") + + monkeypatch.setattr( + _FakeClient, + "create", + timeout_without_visible_remote, + ) + with pytest.raises(TimeoutError, match="ambiguous create"): + TenkiEnvironment(task_id="unreconciled-create") + + with pytest.raises(RuntimeError, match="already active"): + TenkiEnvironment(task_id="unreconciled-create") + assert _FakeSandboxFactory.created_kwargs == [] + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + quarantined = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in quarantined: + tenki_module._release_task_ownership_lock(lock_file) + + +def test_tenki_ambiguous_persistent_lineages_are_never_terminated( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + token = tenki_module._profile_token() + task_id = "ambiguous-persistent" + old_remote = _FakeSandbox( + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + original_create = _FakeClient.create + list_calls = 0 + + def eventually_consistent_list( + self, + *, + workspace_id=None, + tags=None, + sticky=None, + ): + nonlocal list_calls + list_calls += 1 + if list_calls == 1: + return [] + return [old_remote, *_FakeSandboxFactory.sandboxes] + + def commit_then_timeout(self, **kwargs): + original_create(self, **kwargs) + raise TimeoutError("response lost after duplicate commit") + + monkeypatch.setattr(_FakeClient, "list", eventually_consistent_list) + monkeypatch.setattr(_FakeClient, "create", commit_then_timeout) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(TimeoutError, match="duplicate commit"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + new_remote = _FakeSandboxFactory.sandboxes[0] + _FakeClient.remote_sandboxes = [old_remote] + assert old_remote.terminated is False + assert new_remote.terminated is False + attempt_id = new_remote.info.metadata["hermes_create_attempt"] + assert tenki_module._get_create_attempt(task_id) is None + assert tenki_module._remote_binding_state(task_id) == ( + min(old_remote.id, new_remote.id), + attempt_id, + False, + True, + tuple(sorted((old_remote.id, new_remote.id))), + False, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + + # Simulate process exit releasing only the kernel lock. The durable marker + # makes the next owner reconcile the same exact attempt; both lineages are + # still left untouched and no third branch is created. + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + first_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in first_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + monkeypatch.setattr(_FakeClient, "list", lambda self, **_kwargs: []) + with pytest.raises(RuntimeError, match="durable persistent-lineage conflict"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert old_remote.terminated is False + assert new_remote.terminated is False + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + second_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in second_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + +def test_tenki_unmanaged_collision_remains_durable_when_list_omits_remote( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "pre-binding-persistent" + token = tenki_module._profile_token() + existing = _FakeSandbox( + sandbox_id="sb-existing", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + _FakeClient.listed_sandboxes = [existing] + + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert tenki_module._get_remote_binding(task_id) == existing.id + assert not _FakeSandboxFactory.created_kwargs + _release_quarantined_locks_since(tenki_module, quarantine_start) + + # A later eventual-consistency omission cannot turn the collision into an + # ownership claim or authorize a create. + _FakeClient.listed_sandboxes = [] + + with pytest.raises(RuntimeError, match="durable persistent-lineage conflict"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert not _FakeSandboxFactory.created_kwargs + assert existing.terminated is False + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_known_conflict_clears_only_after_exact_remote_is_terminal( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "resolved-unmanaged-conflict" + token = tenki_module._profile_token() + existing = _FakeSandbox( + sandbox_id="sb-existing", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + _FakeClient.listed_sandboxes = [existing] + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): + TenkiEnvironment(task_id=task_id, persistent_filesystem=True) + _release_quarantined_locks_since(tenki_module, quarantine_start) + + existing.terminate() + _FakeClient.listed_sandboxes = [] + successor = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + assert successor._sandbox is not existing + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert tenki_module._remote_binding_state(task_id)[2:4] == ( + True, + False, + ) + successor.cleanup() + + +def test_tenki_terminal_binding_is_authoritatively_cleared_after_crash( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "terminated-binding-crash" + first = TenkiEnvironment(task_id=task_id) + terminated = first._sandbox + assert tenki_module._get_remote_binding(task_id) == terminated.id + + # The server-side termination commits, then the process dies before the + # local binding removal. A successor proves terminal state with get(id), + # clears that exact binding, and creates one replacement. + terminated.terminate() + first._release_task_ownership() + first._sandbox = None + first._client = None + first._cleanup_complete = True + + successor = TenkiEnvironment(task_id=task_id) + assert successor._sandbox is not terminated + assert len(_FakeSandboxFactory.created_kwargs) == 2 + assert tenki_module._get_remote_binding(task_id) == successor._sandbox.id + successor.cleanup() + + +def test_tenki_local_create_rejection_clears_attempt_before_retry( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "local-create-rejection" + original_create_kwargs = TenkiEnvironment._create_kwargs + + def invalid_create_kwargs(self): + kwargs = original_create_kwargs(self) + kwargs["removed_sdk_kwarg"] = True + return kwargs + + monkeypatch.setattr( + TenkiEnvironment, + "_create_kwargs", + invalid_create_kwargs, + ) + with pytest.raises(TypeError, match="do not match the installed SDK"): + TenkiEnvironment(task_id=task_id) + + assert tenki_module._get_create_attempt(task_id) is None + assert tenki_module._get_remote_binding(task_id) is None + assert not _FakeSandboxFactory.sandboxes + + monkeypatch.setattr( + TenkiEnvironment, + "_create_kwargs", + original_create_kwargs, + ) + successor = TenkiEnvironment(task_id=task_id) + assert len(_FakeSandboxFactory.sandboxes) == 1 + successor.cleanup() + + +def test_tenki_post_commit_value_error_retains_attempt_and_blocks_duplicate( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "post-commit-value-error" + original_create = _FakeClient.create + + def commit_then_value_error(self, **kwargs): + original_create(self, **kwargs) + raise ValueError("response decoding failed after commit") + + monkeypatch.setattr(_FakeClient, "create", commit_then_value_error) + monkeypatch.setattr(_FakeClient, "list", lambda self, **_kwargs: []) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(ValueError, match="response decoding failed"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + committed = _FakeSandboxFactory.sandboxes[0] + attempt_id = committed.info.metadata["hermes_create_attempt"] + assert tenki_module._get_create_attempt(task_id) == attempt_id + assert tenki_module._get_remote_binding(task_id) is None + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + first_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in first_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + with pytest.raises(RuntimeError, match="prior create is unresolved"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert committed.terminated is False + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + second_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in second_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + +def test_tenki_expired_empty_attempt_cannot_lock_task_forever( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "expired-empty-attempt" + tenki_module._begin_create_attempt( + task_id, + "pre-rpc-crash", + time.time() - 1, + tmp_path / "tenki_snapshots.json", + ) + + env = TenkiEnvironment(task_id=task_id, max_duration=0) + + assert tenki_module._get_create_attempt(task_id) is None + assert len(_FakeSandboxFactory.created_kwargs) == 1 + # A positive server-side lifetime is what makes eventual expiry + # authoritative even when a crash happened immediately before the RPC. + assert _FakeSandboxFactory.created_kwargs[0]["max_duration"] == 3600 + env.cleanup() + + +def test_tenki_terminal_list_row_requires_authoritative_get_before_clear( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "stale-terminal-list-row" + attempt_id = "expected-attempt" + token = tenki_module._profile_token() + metadata = { + "hermes_backend": "tenki", + "hermes_task_id": task_id, + "hermes_profile": token, + "hermes_create_attempt": attempt_id, + } + listed = _FakeSandbox( + sandbox_id="sb-known", + name=f"hermes-{token}-{task_id}", + state="TERMINATED", + metadata=metadata, + ) + authoritative = _FakeSandbox( + sandbox_id="sb-known", + name=f"hermes-{token}-{task_id}", + state="RUNNING", + metadata=metadata, + ) + _FakeClient.listed_sandboxes = [listed] + _FakeClient.remote_sandboxes = [authoritative] + tenki_module._begin_create_attempt( + task_id, + attempt_id, + time.time() + 7200, + ) + + env = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + assert env._sandbox is authoritative + assert len(_FakeSandboxFactory.created_kwargs) == 0 + assert tenki_module._remote_binding_state(task_id)[2:4] == ( + True, + False, + ) + env.cleanup() + + +def test_tenki_expired_attempt_never_adopts_unmanaged_match( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "expired-attempt-unmanaged" + token = tenki_module._profile_token() + unmanaged = _FakeSandbox( + sandbox_id="external-id", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + _FakeClient.listed_sandboxes = [unmanaged] + tenki_module._begin_create_attempt( + task_id, + "expired-attempt", + time.time() - 1, + ) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="prior create is unresolved"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + assert tenki_module._get_create_attempt(task_id) is None + assert tenki_module._remote_binding_state(task_id) == ( + unmanaged.id, + "expired-attempt", + False, + True, + (unmanaged.id,), + False, + ) + assert not _FakeSandboxFactory.created_kwargs + assert unmanaged.resumed is False + _release_quarantined_locks_since(tenki_module, quarantine_start) + + _FakeClient.listed_sandboxes = [] + with pytest.raises(RuntimeError, match="durable persistent-lineage conflict"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert not _FakeSandboxFactory.created_kwargs + assert unmanaged.resumed is False + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_hidden_exact_conflict_preserves_visible_known_id( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "hidden-exact-visible-other" + token = tenki_module._profile_token() + visible = _FakeSandbox( + sandbox_id="visible-known", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + _FakeClient.listed_sandboxes = [visible] + tenki_module._begin_create_attempt( + task_id, + "hidden-exact-attempt", + time.time() + 7200, + ) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="prior create is unresolved"): + TenkiEnvironment(task_id=task_id, persistent_filesystem=True) + + assert tenki_module._remote_binding_state(task_id) == ( + visible.id, + "hidden-exact-attempt", + False, + True, + (visible.id,), + True, + ) + assert not _FakeSandboxFactory.created_kwargs + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_unvalidated_binding_requires_durable_expected_attempt( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "mismatched-bound-attempt" + token = tenki_module._profile_token() + remote = _FakeSandbox( + sandbox_id="sb-bound", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_backend": "tenki", + "hermes_task_id": task_id, + "hermes_profile": token, + "hermes_create_attempt": "attacker-attempt", + }, + ) + _FakeClient.remote_sandboxes = [remote] + tenki_module._begin_create_attempt( + task_id, + "expected-attempt", + time.time() + 7200, + ) + tenki_module._store_remote_binding( + task_id, + remote.id, + "expected-attempt", + validated=False, + ) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="cannot validate the new lineage"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + assert tenki_module._remote_binding_state(task_id) == ( + remote.id, + "expected-attempt", + False, + False, + (), + False, + ) + assert not _FakeSandboxFactory.created_kwargs + assert remote.resumed is False + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_transient_bound_id_lookup_never_falls_back_to_create( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "transient-bound-get" + first = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + remote = first._sandbox + first._release_task_ownership() + first._sandbox = None + first._client = None + first._cleanup_complete = True + original_get = _FakeClient.get + + def fail_get(self, sandbox_id): + raise TimeoutError(f"lookup timed out for {sandbox_id}") + + monkeypatch.setattr(_FakeClient, "get", fail_get) + with pytest.raises(RuntimeError, match="could not resolve bound remote"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert tenki_module._get_remote_binding(task_id) == remote.id + + monkeypatch.setattr(_FakeClient, "get", original_get) + successor = TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert successor._sandbox is remote + assert len(_FakeSandboxFactory.created_kwargs) == 1 + successor.cleanup() + + +def test_tenki_unvalidated_binding_stays_fail_closed_when_list_omits_remote( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "unvalidated-binding" + token = tenki_module._profile_token() + older = _FakeSandbox( + sandbox_id="sb-older", + name=f"hermes-{token}-{task_id}", + metadata={ + "hermes_task_id": task_id, + "hermes_profile": token, + }, + ) + _FakeClient.remote_sandboxes = [older] + list_calls = 0 + + def reveal_fork_after_create( + self, + *, + workspace_id=None, + tags=None, + sticky=None, + ): + nonlocal list_calls + list_calls += 1 + if list_calls == 1: + return [] + return [older, *_FakeSandboxFactory.sandboxes] + + monkeypatch.setattr(_FakeClient, "list", reveal_fork_after_create) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + with pytest.raises(RuntimeError, match="multiple active sandbox lineages"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + + created = _FakeSandboxFactory.sandboxes[0] + assert tenki_module._get_create_attempt(task_id) is None + assert tenki_module._remote_binding_state(task_id) == ( + created.id, + created.info.metadata["hermes_create_attempt"], + False, + True, + tuple(sorted((created.id, older.id))), + False, + ) + assert older.terminated is False + assert created.terminated is False + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + first_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in first_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + # Even an empty eventual-consistency result cannot convert the exact, + # pending binding into absence or authorize a third create. + monkeypatch.setattr(_FakeClient, "list", lambda self, **_kwargs: []) + with pytest.raises(RuntimeError, match="durable persistent-lineage conflict"): + TenkiEnvironment( + task_id=task_id, + persistent_filesystem=True, + ) + assert len(_FakeSandboxFactory.created_kwargs) == 1 + assert older.terminated is False + assert created.terminated is False + + with tenki_module._QUARANTINED_TASK_OWNERSHIP_GUARD: + second_quarantine = tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + del tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES[ + quarantine_start: + ] + for lock_file in second_quarantine: + tenki_module._release_task_ownership_lock(lock_file) + + +def test_tenki_post_create_mixed_conflict_preserves_known_ids( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + task_id = "post-create-mixed-conflict" + token = tenki_module._profile_token() + metadata = { + "hermes_task_id": task_id, + "hermes_profile": token, + } + known = _FakeSandbox( + sandbox_id="known-other", + name=f"hermes-{token}-{task_id}", + metadata=metadata, + ) + unidentified = _FakeSandbox( + sandbox_id=None, + name=f"hermes-{token}-{task_id}", + metadata=metadata, + ) + list_calls = 0 + + def reveal_mixed_conflict(self, **_kwargs): + nonlocal list_calls + list_calls += 1 + if list_calls == 1: + return [] + return [*_FakeSandboxFactory.sandboxes, known, unidentified] + + monkeypatch.setattr(_FakeClient, "list", reveal_mixed_conflict) + quarantine_start = len( + tenki_module._QUARANTINED_TASK_OWNERSHIP_FILES + ) + + with pytest.raises(RuntimeError, match="multiple active sandbox lineages"): + TenkiEnvironment(task_id=task_id, persistent_filesystem=True) + + created = _FakeSandboxFactory.sandboxes[0] + assert tenki_module._remote_binding_state(task_id) == ( + created.id, + created.info.metadata["hermes_create_attempt"], + False, + True, + tuple(sorted((created.id, known.id))), + True, + ) + assert not created.terminated + assert not known.terminated + _release_quarantined_locks_since(tenki_module, quarantine_start) + + +def test_tenki_runtime_recreate_reconciles_ambiguous_commit_before_retry( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="runtime-ambiguous-create") + original_remote = env._sandbox + original_remote.terminate() + env._sandbox = None + assert env._clear_create_attempt_marker() is True + original_create = _FakeClient.create + + def commit_then_timeout(self, **kwargs): + sandbox = original_create(self, **kwargs) + _FakeClient.listed_sandboxes = [sandbox] + raise TimeoutError("runtime create response lost") + + monkeypatch.setattr(_FakeClient, "create", commit_then_timeout) + with pytest.raises(TimeoutError, match="response lost"): + env._ensure_sandbox() + + ambiguous_remote = _FakeSandboxFactory.sandboxes[-1] + assert ambiguous_remote is not original_remote + assert ambiguous_remote.terminated is True + assert env._create_outcome_uncertain is False + + monkeypatch.setattr(_FakeClient, "create", original_create) + env._ensure_sandbox() + assert env._sandbox not in (original_remote, ambiguous_remote) + env.cleanup() + + +def test_tenki_persistent_not_terminated_when_snapshot_and_pause_both_fail(monkeypatch, tmp_path): + """Durability failed AND pause failed: the sandbox must be left live (not + terminated), so the only copy of un-snapshotted state is preserved.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + def _fail_durable(*_args, **_kwargs): + raise RuntimeError("not durable") def _init(self, **kw): self.kwargs = kw @@ -1025,14 +3626,82 @@ def test_tenki_environment_ignores_mismatched_persistent_sandbox(monkeypatch, tm from tools.environments.tenki import TenkiEnvironment monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + env = TenkiEnvironment(task_id="persist", persistent_filesystem=True) + + assert _FakeSandboxFactory.created_kwargs + assert _FakeSandboxFactory.created_kwargs[0]["name"].endswith("persist") + env.cleanup() + + +def test_tenki_environment_converts_idle_timeout_to_sdk_minutes(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="idle", cpu=1.2, idle_timeout=61) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["cpu_cores"] == 2 + assert kwargs["idle_timeout_minutes"] == 2 + env.cleanup() + + +def test_tenki_environment_passes_sdk_resource_boundaries(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="resource-boundaries", + cpu=16, + memory=65_536, + disk=100 * 1024, + ) + + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert kwargs["cpu_cores"] == 16 + assert kwargs["memory_mb"] == 65_536 + assert kwargs["disk_size_gb"] == 100 + env.cleanup() + + +def test_tenki_environment_omits_resources_outside_sdk_bounds(monkeypatch, tmp_path): + """Shared container settings can exceed Tenki's API ranges. Omit invalid + values so the SDK/workspace defaults apply instead of making create fail.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="resource-out-of-range", + cpu=17, + memory=64, + disk=4 * 1024, + ) - assert _FakeSandboxFactory.created_kwargs - assert _FakeSandboxFactory.created_kwargs[0]["name"].endswith("persist") + kwargs = _FakeSandboxFactory.created_kwargs[0] + assert "cpu_cores" not in kwargs + assert "memory_mb" not in kwargs + assert "disk_size_gb" not in kwargs env.cleanup() -def test_tenki_environment_converts_idle_timeout_to_sdk_minutes(monkeypatch, tmp_path): +def test_tenki_environment_omits_unaligned_sdk_memory(monkeypatch, tmp_path): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) @@ -1042,11 +3711,13 @@ def test_tenki_environment_converts_idle_timeout_to_sdk_minutes(monkeypatch, tmp from tools.environments.tenki import TenkiEnvironment monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) - env = TenkiEnvironment(task_id="idle", cpu=1.2, idle_timeout=61) + env = TenkiEnvironment( + task_id="resource-unaligned-memory", + memory=129, + ) kwargs = _FakeSandboxFactory.created_kwargs[0] - assert kwargs["cpu_cores"] == 2 - assert kwargs["idle_timeout_minutes"] == 2 + assert "memory_mb" not in kwargs env.cleanup() @@ -1108,7 +3779,7 @@ def __init__(self, **kwargs): def sync(self, *, force=False): calls.append(("sync", force)) - def sync_back(self): + def sync_back(self, _hermes_home=None): calls.append(("sync_back", None)) monkeypatch.setattr(tenki_module, "FileSyncManager", FakeSyncManager) @@ -1161,6 +3832,10 @@ def test_tenki_bulk_sync_uses_documented_fs_root_when_home_differs(monkeypatch, env._remote_home = "/root" assert env._remote_transfer_path(".hermes_tenki_sync").startswith("/home/tenki/") + assert ( + env._remote_transfer_path(".hermes_tenki_sync") + != env._remote_transfer_path(".hermes_tenki_sync") + ) env.cleanup() @@ -1179,7 +3854,7 @@ def test_tenki_cleanup_sync_back_uses_original_sandbox(monkeypatch, tmp_path): created_before = len(_FakeSandboxFactory.created_kwargs) class FakeSyncManager: - def sync_back(self): + def sync_back(self, _hermes_home=None): env._tenki_bulk_download(tmp_path / "sync-back.tar") env._sync_manager = FakeSyncManager() @@ -1223,6 +3898,652 @@ def test_tenki_cleanup_blocks_public_execution_while_syncing(monkeypatch, tmp_pa env.cleanup() +def test_tenki_require_sandbox_rejects_cleanup_claimed_in_capture_gap(monkeypatch, tmp_path): + """cleanup may claim the sandbox after ensure releases the lock; the later + capture must recheck the guard instead of returning cleanup's reference.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="cleanup-capture-race") + original_ensure = env._ensure_sandbox + ensured = threading.Event() + cleanup_claimed = threading.Event() + + def ensure_then_wait_for_cleanup(): + original_ensure() + ensured.set() + assert cleanup_claimed.wait(timeout=2) + + monkeypatch.setattr(env, "_ensure_sandbox", ensure_then_wait_for_cleanup) + errors: list[BaseException] = [] + + def require_sandbox(): + try: + env._require_sandbox() + except BaseException as exc: + errors.append(exc) + + worker = threading.Thread(target=require_sandbox) + worker.start() + assert ensured.wait(timeout=2) + with env._lock: + env._cleanup_in_progress = True + env._cleanup_sandbox = env._sandbox + cleanup_claimed.set() + worker.join(timeout=2) + + assert not worker.is_alive() + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + assert "cleanup" in str(errors[0]) + + monkeypatch.setattr(env, "_ensure_sandbox", original_ensure) + with env._lock: + env._cleanup_in_progress = False + env._cleanup_sandbox = None + env.cleanup() + + +def test_tenki_ephemeral_cleanup_closes_environment_owned_client(monkeypatch, tmp_path): + """The default ephemeral path must not leak Sandbox.create's hidden client.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="ephemeral-client-close") + + assert env._persistent is False + assert isinstance(env._client, _FakeClient) + assert _FakeClient.closed_count == 0 + + env.cleanup() + + assert _FakeClient.closed_count == 1 + assert env._client is None + + +def test_tenki_failed_termination_remains_retryable_and_tracked( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TERMINAL_ENV", "tenki") + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools import terminal_tool + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + monkeypatch.setattr(tenki_module, "_TERMINATE_RETRY_DELAYS", (0, 0)) + env = TenkiEnvironment(task_id="retry-termination") + sandbox = env._sandbox + original_terminate = sandbox.terminate + + def fail_terminate(): + raise RuntimeError("control plane unavailable") + + monkeypatch.setattr(sandbox, "terminate", fail_terminate) + cache_key = terminal_tool._resolve_environment_cache_key( + "retry-termination", + "tenki", + ) + terminal_tool._active_environments[cache_key] = env + terminal_tool._last_activity[cache_key] = 0 + + terminal_tool._cleanup_inactive_envs(0) + + assert terminal_tool._active_environments[cache_key] is env + assert cache_key not in terminal_tool._retiring_environments + assert env._sandbox is sandbox + assert env._client is not None + assert env._cleanup_complete is False + assert _FakeClient.closed_count == 0 + + monkeypatch.setattr(sandbox, "terminate", original_terminate) + terminal_tool.cleanup_vm("retry-termination") + + assert cache_key not in terminal_tool._active_environments + assert sandbox.terminated is True + assert _FakeClient.closed_count == 1 + + +def test_tenki_concurrent_cleanup_terminates_and_closes_once(monkeypatch, tmp_path): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment(task_id="concurrent-cleanup") + sandbox = env._sandbox + terminate_entered = threading.Event() + release_terminate = threading.Event() + terminate_calls = [] + original_terminate = sandbox.terminate + + def blocking_terminate(): + terminate_calls.append(True) + terminate_entered.set() + assert release_terminate.wait(timeout=2) + original_terminate() + + monkeypatch.setattr(sandbox, "terminate", blocking_terminate) + cleanup_waiting = threading.Event() + original_wait = env._lifecycle_condition.wait + + def tracked_wait(timeout=None): + cleanup_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(env._lifecycle_condition, "wait", tracked_wait) + first = threading.Thread(target=env.cleanup) + second = threading.Thread(target=env.cleanup) + first.start() + assert terminate_entered.wait(timeout=2) + second.start() + + assert cleanup_waiting.wait(timeout=2) + assert _FakeClient.closed_count == 0 + release_terminate.set() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() + assert not second.is_alive() + assert len(terminate_calls) == 1 + assert _FakeClient.closed_count == 1 + + +@pytest.mark.parametrize("persistent", [False, True]) +def test_tenki_cleanup_waits_for_cancel_before_closing_client( + monkeypatch, + tmp_path, + persistent, +): + """A detached sandbox still needs its control-plane client until cancel's + terminate RPC completes; cleanup must not close that shared channel first.""" + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text("auth_token: tok-secret\n", encoding="utf-8") + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id=f"cancel-cleanup-{persistent}", + persistent_filesystem=persistent, + ) + sandbox = env._sandbox + # Force cancel's sandbox-level fallback instead of the process.kill fast path. + monkeypatch.setattr(sandbox, "start", None) + terminate_entered = threading.Event() + release_terminate = threading.Event() + action_name = "pause" if persistent else "terminate" + original_action = getattr(sandbox, action_name) + + def blocking_action(): + terminate_entered.set() + assert release_terminate.wait(timeout=2) + original_action() + + monkeypatch.setattr(sandbox, action_name, blocking_action) + handle = env._run_bash("echo running", timeout=5) + cancel_thread = threading.Thread(target=handle.kill) + cancel_thread.start() + assert terminate_entered.wait(timeout=2) + + cleanup_thread = threading.Thread(target=env.cleanup) + cleanup_waiting = threading.Event() + original_wait = env._lifecycle_condition.wait + + def tracked_wait(timeout=None): + cleanup_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(env._lifecycle_condition, "wait", tracked_wait) + cleanup_thread.start() + assert cleanup_waiting.wait(timeout=2) + assert _FakeClient.closed_count == 0 + + release_terminate.set() + cancel_thread.join(timeout=2) + cleanup_thread.join(timeout=2) + handle.wait(timeout=2) + + assert not cancel_thread.is_alive() + assert not cleanup_thread.is_alive() + assert getattr(sandbox, "paused" if persistent else "terminated") is True + if persistent: + assert sandbox.terminated is True + assert _FakeClient.closed_count == 1 + + +def test_tenki_persistent_cancel_never_terminates_when_pause_fails( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="cancel-pause-failure", + persistent_filesystem=True, + ) + sandbox = env._sandbox + # Force sandbox-level cancellation before a process handle can supply a + # working kill method, then make the preservation pause fail. + monkeypatch.setattr(sandbox, "start", None) + + def fail_pause(): + raise RuntimeError("pause control plane unavailable") + + monkeypatch.setattr(sandbox, "pause", fail_pause) + handle = env._run_bash("echo running", timeout=5) + handle.kill() + handle.wait(timeout=2) + + assert env._sandbox is sandbox + assert sandbox.paused is False + assert sandbox.terminated is False + assert sandbox.state == "RUNNING" + + +def test_tenki_persistent_cancel_blocks_ensure_and_reuses_exact_sandbox( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="cancel-exact-reference", + persistent_filesystem=True, + ) + sandbox = env._sandbox + monkeypatch.setattr(sandbox, "start", None) + pause_entered = threading.Event() + release_pause = threading.Event() + original_pause = sandbox.pause + + def blocking_pause(): + pause_entered.set() + assert release_pause.wait(timeout=2) + original_pause() + + monkeypatch.setattr(sandbox, "pause", blocking_pause) + handle = env._run_bash("echo running", timeout=5) + cancel_thread = threading.Thread(target=handle.kill) + cancel_thread.start() + assert pause_entered.wait(timeout=2) + + # If ensure incorrectly falls back to discovery after cancel, this turns + # that mistake into an error instead of silently finding the same sandbox. + def fail_list(self, **_kwargs): + raise RuntimeError("transient list miss") + + monkeypatch.setattr(_FakeClient, "list", fail_list) + ensured = [] + ensure_errors = [] + + def execute_again(): + try: + result = env._exec_raw("echo after cancel") + ensured.append((env._sandbox, result)) + except Exception as exc: + ensure_errors.append(exc) + + ensure_thread = threading.Thread(target=execute_again) + ensure_thread.start() + time.sleep(0.05) + + assert ensure_thread.is_alive() + assert env._sandbox is sandbox + + release_pause.set() + cancel_thread.join(timeout=2) + ensure_thread.join(timeout=2) + handle.wait(timeout=2) + + assert not cancel_thread.is_alive() + assert not ensure_thread.is_alive() + assert ensure_errors == [] + assert ensured == [(sandbox, ("ran\n", 0))] + assert env._sandbox is sandbox + assert sandbox.resumed is True + assert len(_FakeSandboxFactory.created_kwargs) == 1 + + +def test_tenki_completed_cancel_generation_rechecks_before_lease( + monkeypatch, + tmp_path, +): + _install_fake_tenki(monkeypatch) + _clear_tenki_auth_env(monkeypatch) + monkeypatch.setattr("tools.lazy_deps.ensure", lambda *_args, **_kwargs: None) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("TENKI_CONFIG_PATH", str(tmp_path / "config.yaml")) + (tmp_path / "config.yaml").write_text( + "auth_token: tok-secret\n", + encoding="utf-8", + ) + + from tools.environments import tenki as tenki_module + from tools.environments.tenki import TenkiEnvironment + + monkeypatch.setattr(TenkiEnvironment, "init_session", lambda self: None) + env = TenkiEnvironment( + task_id="cancel-generation", + persistent_filesystem=True, + ) + sandbox = env._sandbox + monkeypatch.setattr(sandbox, "start", None) + + class CapturedHandle: + def __init__(self, _exec_fn, cancel_fn=None): + self._cancel_fn = cancel_fn + + def kill(self): + self._cancel_fn() + + def wait(self, timeout=None): + return 0 + + monkeypatch.setattr(tenki_module, "_ThreadedProcessHandle", CapturedHandle) + cancel_handle = env._run_bash("echo cancelled", timeout=5) + + original_require = env._require_sandbox + resolved_once = threading.Event() + release_resolved = threading.Event() + require_calls = 0 + + def require_with_gap(): + nonlocal require_calls + resolved = original_require() + require_calls += 1 + if require_calls == 1: + resolved_once.set() + assert release_resolved.wait(timeout=2) + return resolved + + monkeypatch.setattr(env, "_require_sandbox", require_with_gap) + leased = [] + lease_errors = [] + + def take_lease(): + try: + with env._sandbox_operation() as leased_sandbox: + leased.append( + (leased_sandbox, leased_sandbox.state) + ) + except Exception as exc: + lease_errors.append(exc) + + lease_thread = threading.Thread(target=take_lease) + lease_thread.start() + assert resolved_once.wait(timeout=2) + + # Complete the entire cancel inside the resolve→publish gap. Object + # identity is unchanged, so only the generation check can detect that the + # resolved sandbox was paused after readiness validation. + cancel_handle.kill() + assert sandbox.state == "PAUSED" + assert env._cancel_in_progress == 0 + release_resolved.set() + lease_thread.join(timeout=2) + + assert not lease_thread.is_alive() + assert lease_errors == [] + assert require_calls >= 2 + assert leased == [(sandbox, "RUNNING")] + assert sandbox.resumed is True + assert env._active_operations == 0 + + +def test_tenki_cleanup_waits_for_running_execution(monkeypatch, tmp_path): + """Cleanup cannot terminate the sandbox while a leased command is still + using it, even when that command is not in the background-process registry.""" + env = _tenki_env_for_upload_race(monkeypatch, tmp_path, "cleanup-exec-lease") + sandbox = env._sandbox + client = env._client + client_close_calls = [] + original_close = client.close + + def tracked_close(): + client_close_calls.append(True) + original_close() + + monkeypatch.setattr(client, "close", tracked_close) + handle = env._run_bash("sleep infinity", timeout=30) + for _ in range(100): + if sandbox.last_process is not None: + break + time.sleep(0.01) + assert sandbox.last_process is not None + + cleanup_waiting = threading.Event() + original_wait = env._lifecycle_condition.wait + + def tracked_wait(timeout=None): + cleanup_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(env._lifecycle_condition, "wait", tracked_wait) + cleanup_thread = threading.Thread(target=env.cleanup) + cleanup_thread.start() + + assert cleanup_waiting.wait(timeout=2) + assert sandbox.terminated is False + assert client_close_calls == [] + + sandbox.last_process._done.set() + handle.wait(timeout=2) + cleanup_thread.join(timeout=2) + + assert not cleanup_thread.is_alive() + assert sandbox.terminated is True + assert client_close_calls == [True] + + +def test_tenki_cleanup_waits_for_running_upload(monkeypatch, tmp_path): + """The mkdir/upload pair holds one operation lease until both calls finish.""" + env = _tenki_env_for_upload_race(monkeypatch, tmp_path, "cleanup-upload-lease") + sandbox = env._sandbox + host_file = tmp_path / "skill.md" + host_file.write_text("content", encoding="utf-8") + upload_entered = threading.Event() + release_upload = threading.Event() + original_upload = sandbox.fs.upload + + def blocking_upload(*args, **kwargs): + upload_entered.set() + assert release_upload.wait(timeout=2) + return original_upload(*args, **kwargs) + + monkeypatch.setattr(sandbox.fs, "upload", blocking_upload) + upload_thread = threading.Thread( + target=env._tenki_upload, + args=(str(host_file), "/home/tenki/.hermes/skills/skill.md"), + ) + upload_thread.start() + assert upload_entered.wait(timeout=2) + + cleanup_waiting = threading.Event() + original_wait = env._lifecycle_condition.wait + + def tracked_wait(timeout=None): + cleanup_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(env._lifecycle_condition, "wait", tracked_wait) + cleanup_thread = threading.Thread(target=env.cleanup) + cleanup_thread.start() + + assert cleanup_waiting.wait(timeout=2) + assert sandbox.terminated is False + assert _FakeClient.closed_count == 0 + + release_upload.set() + upload_thread.join(timeout=2) + cleanup_thread.join(timeout=2) + + assert not upload_thread.is_alive() + assert not cleanup_thread.is_alive() + assert sandbox.fs.upload_calls + assert sandbox.terminated is True + assert _FakeClient.closed_count == 1 + + +def test_tenki_idle_reaper_waits_for_running_execution(monkeypatch, tmp_path): + """A retirement tombstone keeps all creators from replacing the wrapper + until its active operation and cleanup have both completed.""" + env = _tenki_env_for_upload_race(monkeypatch, tmp_path, "reaper-exec-lease") + sandbox = env._sandbox + handle = env._run_bash("sleep infinity", timeout=30) + for _ in range(100): + if sandbox.last_process is not None: + break + time.sleep(0.01) + assert sandbox.last_process is not None + + from tools import terminal_tool + from tools import code_execution_tool + from tools import file_tools + + monkeypatch.setenv("TERMINAL_ENV", "tenki") + cache_key = terminal_tool._resolve_environment_cache_key( + "reaper-exec-lease", + "tenki", + ) + terminal_tool._active_environments[cache_key] = env + terminal_tool._last_activity[cache_key] = 0 + cleanup_waiting = threading.Event() + original_wait = env._lifecycle_condition.wait + + def tracked_wait(timeout=None): + cleanup_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(env._lifecycle_condition, "wait", tracked_wait) + reaper_thread = threading.Thread( + target=terminal_tool._cleanup_inactive_envs, + args=(0,), + ) + reaper_thread.start() + + assert cleanup_waiting.wait(timeout=2) + assert terminal_tool._active_environments[cache_key] is env + assert cache_key in terminal_tool._retiring_environments + assert sandbox.terminated is False + + replacement_created = threading.Event() + + class ReplacementEnv: + cwd = "/home/tenki" + + def cleanup(self): + return None + + replacement = ReplacementEnv() + + def create_replacement(**_kwargs): + assert sandbox.terminated is True + replacement_created.set() + return replacement + + monkeypatch.setattr( + terminal_tool, + "_create_environment", + create_replacement, + ) + cache_clear_entered = threading.Event() + release_cache_clear = threading.Event() + original_clear_file_ops_cache = file_tools.clear_file_ops_cache + + def blocked_clear_file_ops_cache(task_id=None): + cache_clear_entered.set() + assert release_cache_clear.wait(timeout=2) + return original_clear_file_ops_cache(task_id) + + monkeypatch.setattr( + file_tools, + "clear_file_ops_cache", + blocked_clear_file_ops_cache, + ) + creator_result = [] + + def create_for_code_execution(): + creator_result.append( + code_execution_tool._get_or_create_env("reaper-exec-lease") + ) + + creator_thread = threading.Thread(target=create_for_code_execution) + creator_thread.start() + time.sleep(0.05) + assert creator_thread.is_alive() + assert replacement_created.is_set() is False + + sandbox.last_process._done.set() + handle.wait(timeout=2) + assert cache_clear_entered.wait(timeout=2) + assert creator_thread.is_alive() + assert replacement_created.is_set() is False + release_cache_clear.set() + reaper_thread.join(timeout=2) + creator_thread.join(timeout=2) + + assert not reaper_thread.is_alive() + assert not creator_thread.is_alive() + assert sandbox.terminated is True + assert _FakeClient.closed_count == 1 + assert replacement_created.is_set() is True + assert creator_result == [(replacement, "tenki")] + terminal_tool._active_environments.pop(cache_key, None) + terminal_tool._last_activity.pop(cache_key, None) + terminal_tool._creation_locks.pop(cache_key, None) + + def test_tenki_execute_passes_stdin_natively_not_as_heredoc(monkeypatch, tmp_path): _install_fake_tenki(monkeypatch) _clear_tenki_auth_env(monkeypatch) @@ -1510,6 +4831,49 @@ def narrow_create(self, *, name=None, image=None, env=None, metadata=None, tags= env.cleanup() +def test_installed_tenki_sdk_exposes_adapter_contract_without_network(): + """Exercise the real optional SDK surface that the adapter introspects.""" + import inspect + + tenki = pytest.importorskip("tenki") + client_create = inspect.signature(tenki.Client.create).parameters + assert { + "workspace_id", + "snapshot_id", + "image", + "cpu_cores", + "memory_mb", + "disk_size_gb", + } <= set(client_create) + assert "project_id" not in client_create + assert "workspace_id" in inspect.signature(tenki.Client.list).parameters + assert any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in inspect.signature( + tenki.Sandbox.create + ).parameters.values() + ) + + # Client construction is local; no request is made until a resource + # method is called. Verify cleanup/durability APIs without contacting + # Tenki. + client = tenki.Client( + auth_token="contract-test", + base_url="https://example.invalid", + ) + try: + assert callable(client.snapshots.wait_durable) + assert callable(client.snapshots.delete) + assert "snapshot_id" in inspect.signature( + client.snapshots.wait_durable + ).parameters + assert "snapshot_id" in inspect.signature( + client.snapshots.delete + ).parameters + finally: + client.close() + + def test_snapshot_unrecoverable_resolves_renamed_registry_error(monkeypatch): """tenki 0.5 renamed RegistryArtifactNotFoundError to RegistryImageNotFoundError. Importing the whole set in one statement made a @@ -1535,3 +4899,13 @@ class RegistryArtifactNotFoundError(Exception): assert TenkiEnvironment._snapshot_unrecoverable(_FakeInvalidStateError("workspace suspended")) is False # ...but an InvalidStateError that names the snapshot is unrecoverable. assert TenkiEnvironment._snapshot_unrecoverable(_FakeInvalidStateError("snapshot is not durable")) is True + # A generic FAILED_PRECONDITION may mention a snapshot while describing a + # transient workspace/policy block; that must preserve the restore pointer. + assert ( + TenkiEnvironment._snapshot_unrecoverable( + _FakeInvalidStateError( + "snapshot restore temporarily blocked by workspace policy" + ) + ) + is False + ) diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index 6b99770afbb1b..8d851699fbe52 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -1,6 +1,7 @@ """Tests for terminal/file tool availability in local dev environments.""" import importlib +import os import pytest @@ -91,6 +92,92 @@ def fake_find_spec(name): monkeypatch.setenv("TENKI_AUTH_TOKEN", "tok") assert terminal_tool_module.check_terminal_requirements() is True + def test_tool_definition_cache_isolates_profiles_and_rechecks_credentials( + self, + monkeypatch, + tmp_path, + ): + import model_tools + import tools.registry as registry_module + from agent.secret_scope import is_multiplex_active, set_multiplex_active + from gateway.run import _profile_runtime_scope + from hermes_cli import config as hermes_config + + home_a = tmp_path / "a" + home_b = tmp_path / "b" + config_text = "terminal:\n backend: tenki\n cwd: /home/tenki\n" + for home in (home_a, home_b): + home.mkdir() + (home / "config.yaml").write_text(config_text, encoding="utf-8") + (home_a / ".env").write_text( + "TENKI_AUTH_TOKEN=token-a\n", + encoding="utf-8", + ) + (home_b / ".env").write_text("", encoding="utf-8") + # Same config contents and fingerprint make profile identity, not mtime + # luck, the distinguishing cache-key invariant. + stat_a = (home_a / "config.yaml").stat() + os.utime( + home_b / "config.yaml", + ns=(stat_a.st_atime_ns, stat_a.st_mtime_ns), + ) + + monkeypatch.setenv("TERMINAL_ENV", "local") + monkeypatch.setenv("TENKI_AUTH_TOKEN", "process-token-must-not-leak") + clock = {"now": 1000.0} + monkeypatch.setattr(model_tools.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr( + registry_module.time, + "monotonic", + lambda: clock["now"], + ) + previous = is_multiplex_active() + set_multiplex_active(True) + model_tools._clear_tool_defs_cache() + registry_module.invalidate_check_fn_cache() + hermes_config._LOAD_CONFIG_CACHE.clear() + hermes_config._RAW_CONFIG_CACHE.clear() + try: + with _profile_runtime_scope(home_a): + names_a = { + tool["function"]["name"] + for tool in get_tool_definitions( + enabled_toolsets=["terminal"], + quiet_mode=True, + ) + } + with _profile_runtime_scope(home_b): + names_b = { + tool["function"]["name"] + for tool in get_tool_definitions( + enabled_toolsets=["terminal"], + quiet_mode=True, + ) + } + + assert "terminal" in names_a + assert "terminal" not in names_b + assert len(model_tools._tool_defs_cache) == 2 + + (home_b / ".env").write_text( + "TENKI_AUTH_TOKEN=token-b\n", + encoding="utf-8", + ) + clock["now"] += registry_module._CHECK_FN_TTL_SECONDS + 1 + with _profile_runtime_scope(home_b): + names_b_after_rotation = { + tool["function"]["name"] + for tool in get_tool_definitions( + enabled_toolsets=["terminal"], + quiet_mode=True, + ) + } + assert "terminal" in names_b_after_rotation + finally: + set_multiplex_active(previous) + model_tools._clear_tool_defs_cache() + registry_module.invalidate_check_fn_cache() + class TestCheckFnTransientFailureSuppression: """The check_fn TTL cache should absorb transient probe failures. @@ -159,6 +246,42 @@ def never(): monkeypatch.setattr(reg.time, "monotonic", lambda: t["now"]) assert reg._check_fn_cached(never) is False + def test_cache_is_scoped_to_profile_secret_context(self, tmp_path): + """A success under profile A cannot make profile B's unavailable + backend appear ready through the shared check_fn TTL cache.""" + import tools.registry as reg + from agent.secret_scope import reset_secret_scope, set_secret_scope + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + home_a = tmp_path / "a" + home_b = tmp_path / "b" + home_a.mkdir() + home_b.mkdir() + + def profile_probe(): + from hermes_constants import get_hermes_home + + return get_hermes_home() == home_a + + home_token = set_hermes_home_override(home_a) + secret_token = set_secret_scope({"TENKI_AUTH_TOKEN": "token-a"}) + try: + assert reg._check_fn_cached(profile_probe) is True + finally: + reset_secret_scope(secret_token) + reset_hermes_home_override(home_token) + + home_token = set_hermes_home_override(home_b) + secret_token = set_secret_scope({}) + try: + assert reg._check_fn_cached(profile_probe) is False + finally: + reset_secret_scope(secret_token) + reset_hermes_home_override(home_token) + def test_grace_expiry_lets_real_outage_through(self, monkeypatch): import tools.registry as reg diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index 194333c549c18..b3ea3e5e371bf 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -712,37 +712,34 @@ def _get_or_create_env(task_id: str): Returns ``(env, env_type)`` tuple. """ from tools.terminal_tool import ( - _active_environments, _env_lock, _create_environment, - _get_env_config, _last_activity, _start_cleanup_thread, - _creation_locks, _creation_locks_lock, _task_env_overrides, - _resolve_container_task_id, + _create_environment, + _get_env_config, _start_cleanup_thread, + _environment_creation_lock, + _register_active_environment, + _resolve_environment_cache_key, + _resolve_environment_cwd, + _select_active_environment, + resolve_task_overrides, _CONTAINER_BACKENDS, _container_config_from_env_config, ) - effective_task_id = _resolve_container_task_id(task_id) + config = _get_env_config() + env_type = config["env_type"] + effective_task_id = _resolve_environment_cache_key(task_id, env_type) # Fast path: environment already exists - with _env_lock: - if effective_task_id in _active_environments: - _last_activity[effective_task_id] = time.time() - return _active_environments[effective_task_id], _get_env_config()["env_type"] + _selected_key, env = _select_active_environment(effective_task_id) + if env is not None: + return env, env_type # Slow path: create environment (same pattern as file_tools._get_file_ops) - with _creation_locks_lock: - if effective_task_id not in _creation_locks: - _creation_locks[effective_task_id] = threading.Lock() - task_lock = _creation_locks[effective_task_id] - - with task_lock: - with _env_lock: - if effective_task_id in _active_environments: - _last_activity[effective_task_id] = time.time() - return _active_environments[effective_task_id], _get_env_config()["env_type"] + with _environment_creation_lock(effective_task_id): + _selected_key, env = _select_active_environment(effective_task_id) + if env is not None: + return env, env_type - config = _get_env_config() - env_type = config["env_type"] - overrides = _task_env_overrides.get(effective_task_id, {}) + overrides = resolve_task_overrides(task_id) if env_type == "docker": image = overrides.get("docker_image") or config["docker_image"] @@ -757,7 +754,12 @@ def _get_or_create_env(task_id: str): else: image = "" - cwd = overrides.get("cwd") or config["cwd"] + cwd = _resolve_environment_cwd( + task_id, + env_type, + config, + overrides, + ) container_config = None if env_type in _CONTAINER_BACKENDS: @@ -793,9 +795,7 @@ def _get_or_create_env(task_id: str): host_cwd=config.get("host_cwd"), ) - with _env_lock: - _active_environments[effective_task_id] = env - _last_activity[effective_task_id] = time.time() + env = _register_active_environment(effective_task_id, env) _start_cleanup_thread() logger.info("%s environment ready for execute_code task %s", diff --git a/tools/credential_files.py b/tools/credential_files.py index 583d9973b5472..88e437d125088 100644 --- a/tools/credential_files.py +++ b/tools/credential_files.py @@ -50,8 +50,8 @@ def _get_registered() -> Dict[str, str]: return val -# Cache for config-based file list (loaded once per process). -_config_files: List[Dict[str, str]] | None = None +# Cache for config-based file lists, isolated by profile/Hermes home. +_config_files: Dict[str, List[Dict[str, str]]] = {} def _resolve_hermes_home() -> Path: @@ -174,15 +174,24 @@ def register_credential_files( def _load_config_files() -> List[Dict[str, str]]: - """Load ``terminal.credential_files`` from config.yaml (cached).""" + """Load ``terminal.credential_files`` from config.yaml (profile-cached).""" global _config_files - if _config_files is not None: - return _config_files + if not isinstance(_config_files, dict): + # Backward-compatible with tests/extensions that cleared the former + # singleton cache by assigning None. + _config_files = {} + hermes_home = _resolve_hermes_home() + try: + cache_key = str(hermes_home.resolve()) + except OSError: + cache_key = str(hermes_home) + cached = _config_files.get(cache_key) + if cached is not None: + return cached result: List[Dict[str, str]] = [] try: from hermes_cli.config import read_raw_config - hermes_home = _resolve_hermes_home() cfg = read_raw_config() cred_files = cfg_get(cfg, "terminal", "credential_files") if isinstance(cred_files, list): @@ -214,8 +223,8 @@ def _load_config_files() -> List[Dict[str, str]]: except Exception as e: logger.warning("Could not read terminal.credential_files from config: %s", e) - _config_files = result - return _config_files + _config_files[cache_key] = result + return result def get_credential_file_mounts() -> List[Dict[str, str]]: @@ -522,4 +531,3 @@ def clear_credential_files() -> None: """Reset the skill-scoped registry (e.g. on session reset).""" _get_registered().clear() - diff --git a/tools/environments/file_sync.py b/tools/environments/file_sync.py index 2357228ef9c53..2ae9cc5fd1621 100644 --- a/tools/environments/file_sync.py +++ b/tools/environments/file_sync.py @@ -48,6 +48,7 @@ BulkDownloadFn = Callable[[Path], None] # (dest_tar_path) -> writes tar archive, raises on failure DeleteFn = Callable[[list[str]], None] # (remote_paths) -> raises on failure GetFilesFn = Callable[[], list[tuple[str, str]]] # () -> [(host_path, remote_path), ...] +GetUploadOnlyHostPathsFn = Callable[[], set[str]] def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]: @@ -150,19 +151,29 @@ def __init__( sync_interval: float = _SYNC_INTERVAL_SECONDS, bulk_upload_fn: BulkUploadFn | None = None, bulk_download_fn: BulkDownloadFn | None = None, + get_upload_only_host_paths_fn: GetUploadOnlyHostPathsFn | None = None, ): self._get_files_fn = get_files_fn self._upload_fn = upload_fn self._bulk_upload_fn = bulk_upload_fn self._bulk_download_fn = bulk_download_fn self._delete_fn = delete_fn + self._get_upload_only_host_paths_fn = ( + get_upload_only_host_paths_fn or _credential_host_paths + ) self._synced_files: dict[str, tuple[float, int]] = {} # remote_path -> (mtime, size) self._pushed_hashes: dict[str, str] = {} # remote_path -> sha256 hex digest self._upload_only_host_paths: set[str] = set() self._last_sync_time: float = 0.0 # monotonic; 0 ensures first sync runs self._sync_interval = sync_interval + self._sync_lock = threading.RLock() def sync(self, *, force: bool = False) -> None: + """Serialize one upload transaction against sync-back and peer syncs.""" + with self._sync_lock: + self._sync_once(force=force) + + def _sync_once(self, *, force: bool = False) -> None: """Run a sync cycle: upload changed files, delete removed files. Rate-limited to once per ``sync_interval`` unless *force* is True @@ -177,7 +188,9 @@ def sync(self, *, force: bool = False) -> None: return current_files = self._get_files_fn() - self._upload_only_host_paths.update(_credential_host_paths()) + self._upload_only_host_paths.update( + self._get_upload_only_host_paths_fn() + ) current_remote_paths = {remote for _, remote in current_files} # --- Uploads: new or changed files --- @@ -248,6 +261,11 @@ def sync(self, *, force: bool = False) -> None: # ------------------------------------------------------------------ def sync_back(self, hermes_home: Path | None = None) -> None: + """Serialize teardown sync-back against every upload transaction.""" + with self._sync_lock: + self._sync_back_serialized(hermes_home) + + def _sync_back_serialized(self, hermes_home: Path | None = None) -> None: """Pull remote changes back to the host filesystem. Downloads the remote ``.hermes/`` directory as a tar archive, @@ -372,7 +390,8 @@ def _sync_back_impl(self) -> None: applied = 0 upload_only_host_paths = ( - self._upload_only_host_paths | _credential_host_paths() + self._upload_only_host_paths + | self._get_upload_only_host_paths_fn() ) for dirpath, _dirnames, filenames in os.walk(staging): for fname in filenames: diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 2aebc3b0cb9cb..5a9aff9f6b341 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -4,6 +4,7 @@ import hashlib import inspect +import json import logging import math import os @@ -12,6 +13,10 @@ import tarfile import tempfile import threading +import time +import uuid +from contextlib import contextmanager +from contextvars import copy_context from pathlib import Path from typing import Any @@ -19,11 +24,10 @@ from tools.environments.base import ( BaseEnvironment, _ThreadedProcessHandle, - _load_json_store, - _save_json_store, ) from tools.environments.file_sync import ( FileSyncManager, + _credential_host_paths, iter_sync_files, quoted_mkdir_command, quoted_rm_command, @@ -37,7 +41,65 @@ logger = logging.getLogger(__name__) _SNAPSHOT_NAMESPACE = "direct" +_SNAPSHOT_RETIREMENT_NAMESPACE = "retire" +_SNAPSHOT_RETIRED_NAMESPACE = "retired" +_CREATE_ATTEMPT_NAMESPACE = "create-attempt" +_REMOTE_BINDING_NAMESPACE = "remote-binding" +_CREATE_ATTEMPT_EXPIRY_GRACE = 3600 _ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_TENKI_CPU_RANGE = (1, 16) +_TENKI_MEMORY_MB_RANGE = (128, 65_536) +_TENKI_DISK_GB_RANGE = (5, 100) +_SNAPSHOT_LOCKS: dict[str, threading.RLock] = {} +_SNAPSHOT_LOCKS_GUARD = threading.Lock() +_QUARANTINED_TASK_OWNERSHIP_FILES: list[Any] = [] +_QUARANTINED_TASK_OWNERSHIP_GUARD = threading.Lock() +_TERMINATE_RETRY_DELAYS = (0.1, 0.5) + +try: + import fcntl as _fcntl +except ImportError: # pragma: no cover - Windows + _fcntl = None + +try: + import msvcrt as _msvcrt +except ImportError: # pragma: no cover - POSIX + _msvcrt = None + + +class _SnapshotPointerCommitUncertain(OSError): + """The new pointer is visible, but its directory entry may not be durable.""" + + def __init__(self, message: str): + super().__init__(message) + self.previous_snapshot_id: str | None = None + self.new_snapshot_id: str | None = None + + +class _SnapshotPointerConflict(RuntimeError): + """A stale writer attempted to replace newer or retired recovery state.""" + + +def _load_json_store(path: Path) -> dict: + """Load Tenki recovery state, failing closed when it is unreadable. + + Treating a malformed existing registry as empty would let the next + read-modify-write erase the only snapshot pointer, create attempt, or + exact remote binding. A missing file is the sole valid empty state. + """ + if not path.exists(): + return {} + try: + value = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + raise RuntimeError( + f"Tenki recovery registry is unreadable: {path}" + ) from exc + if not isinstance(value, dict): + raise RuntimeError( + f"Tenki recovery registry must contain a JSON object: {path}" + ) + return value def _snapshot_store_path() -> Path: @@ -51,40 +113,740 @@ def _snapshot_store_path() -> Path: return get_hermes_home() / "tenki_snapshots.json" +def _task_ownership_lock_path(profile_home: Path, task_id: str) -> Path: + task_hash = hashlib.sha256(task_id.encode("utf-8")).hexdigest() + return profile_home / "locks" / "tenki" / f"{task_hash}.lock" + + +def _acquire_task_ownership_lock(profile_home: Path, task_id: str): + """Acquire one profile/task lifetime lock or fail without remote mutation.""" + lock_path = _task_ownership_lock_path(profile_home, task_id) + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock_file = lock_path.open("a+", encoding="utf-8") + try: + if _fcntl is not None: + _fcntl.flock( + lock_file.fileno(), + _fcntl.LOCK_EX | _fcntl.LOCK_NB, + ) + elif _msvcrt is not None: + lock_file.seek(0, os.SEEK_END) + if lock_file.tell() == 0: + lock_file.write(" ") + lock_file.flush() + lock_file.seek(0) + _msvcrt.locking( + lock_file.fileno(), + _msvcrt.LK_NBLCK, + 1, + ) + else: + raise RuntimeError( + "Tenki task ownership requires fcntl or msvcrt " + "cross-process file locking" + ) + except (BlockingIOError, OSError) as exc: + lock_file.close() + raise RuntimeError( + "Tenki sandbox task is already active in another Hermes process: " + f"{task_id}" + ) from exc + except Exception: + lock_file.close() + raise + return lock_file + + +def _release_task_ownership_lock(lock_file: Any) -> None: + if lock_file is None: + return + try: + if _fcntl is not None: + _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_UN) + elif _msvcrt is not None: + lock_file.seek(0) + _msvcrt.locking( + lock_file.fileno(), + _msvcrt.LK_UNLCK, + 1, + ) + except OSError: + pass + finally: + lock_file.close() + + +def _quarantine_task_ownership_lock(lock_file: Any) -> None: + """Keep a failed task exclusively owned until this process exits.""" + if lock_file is None: + return + with _QUARANTINED_TASK_OWNERSHIP_GUARD: + _QUARANTINED_TASK_OWNERSHIP_FILES.append(lock_file) + + def _profile_token() -> str: - """Short, stable identifier for the active Hermes profile. - - Two profiles sharing one Tenki account must get distinct sandbox - names/metadata so they can never attach to or restore each other's - sandbox. Prefer the canonical ``HERMES_PROFILE`` id, which is stable across - machines and survives a home-directory move; only fall back to a - *normalized* ``HERMES_HOME`` path when no profile id is set (the default - profile). Resolving per call handles the multiplexing gateway's per-turn - ``HERMES_HOME`` override (same reason as :func:`_snapshot_store_path`). - """ - profile = os.getenv("HERMES_PROFILE", "").strip() - if profile: - basis = f"profile:{profile}" - else: - try: - basis = str(get_hermes_home().resolve()) - except Exception: - basis = str(get_hermes_home()) + """Canonical profile identity, stable across standalone/multiplex modes.""" + try: + home = str(get_hermes_home().resolve()) + except Exception: + home = str(get_hermes_home()) + return _profile_token_for_basis(f"home:{home}") + + +def _profile_token_for_basis(basis: str) -> str: return hashlib.sha1(basis.encode("utf-8")).hexdigest()[:10] +def _legacy_profile_tokens() -> tuple[str, ...]: + """Tokens emitted before profile identity was canonicalized to home.""" + try: + home_path = get_hermes_home().resolve() + except Exception: + home_path = get_hermes_home() + + bases = [str(home_path)] # old default-profile path basis + if home_path.parent.name == "profiles": + profile_name = home_path.name + else: + profile_name = os.getenv("HERMES_PROFILE", "").strip() + if profile_name: + bases.append(f"profile:{profile_name}") + + canonical = _profile_token() + return tuple( + token + for token in dict.fromkeys( + _profile_token_for_basis(basis) for basis in bases + ) + if token != canonical + ) + + def _load_snapshots(store_path: Path | None = None) -> dict: - return _load_json_store(store_path or _snapshot_store_path()) + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + return _load_json_store(path) def _save_snapshots(data: dict, store_path: Path | None = None) -> None: - _save_json_store(store_path or _snapshot_store_path(), data) + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + _atomic_save_snapshots(path, data) + + +def _snapshot_platform() -> str: + return os.name + + +def _windows_replace_file_write_through(source: str, destination: Path) -> None: + """Atomically replace *destination* with Windows write-through semantics.""" + import ctypes + from ctypes import wintypes + + move_file_ex = ctypes.WinDLL( + "kernel32", + use_last_error=True, + ).MoveFileExW + move_file_ex.argtypes = ( + wintypes.LPCWSTR, + wintypes.LPCWSTR, + wintypes.DWORD, + ) + move_file_ex.restype = wintypes.BOOL + flags = 0x1 | 0x8 # MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH + if move_file_ex( + os.path.abspath(source), + os.path.abspath(destination), + flags, + ): + return + error_code = ctypes.get_last_error() + raise OSError( + error_code, + ctypes.FormatError(error_code), + str(destination), + ) + + +@contextmanager +def _snapshot_store_lock(path: Path): + """Serialize snapshot-pointer RMW in-process and across processes.""" + key = str(path) + with _SNAPSHOT_LOCKS_GUARD: + lock = _SNAPSHOT_LOCKS.setdefault(key, threading.RLock()) + with lock: + lock_path = path.with_suffix(path.suffix + ".lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock_file = lock_path.open("a+", encoding="utf-8") + os_locked = False + try: + if _fcntl is not None: + _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_EX) + os_locked = True + elif _msvcrt is not None: + # Windows byte-range locks require a real byte at the current + # file position. Concurrent initializers may append more than + # one byte, but every process locks byte zero. + lock_file.seek(0, os.SEEK_END) + if lock_file.tell() == 0: + lock_file.write(" ") + lock_file.flush() + lock_file.seek(0) + _msvcrt.locking( + lock_file.fileno(), + _msvcrt.LK_LOCK, + 1, + ) + os_locked = True + else: + # Pointer RMW without a kernel lock can report success in two + # processes while silently losing one task's sole recovery + # pointer. Persistent state must fail closed on unsupported + # platforms. + raise RuntimeError( + "Tenki snapshot registry requires fcntl or msvcrt " + "cross-process file locking" + ) + yield + finally: + if os_locked and _fcntl is not None: + try: + _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_UN) + except OSError: + pass + elif os_locked and _msvcrt is not None: + try: + lock_file.seek(0) + _msvcrt.locking( + lock_file.fileno(), + _msvcrt.LK_UNLCK, + 1, + ) + except OSError: + pass + lock_file.close() + + +def _atomic_save_snapshots(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temp_path: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as tmp: + import json + + json.dump(data, tmp, indent=2) + tmp.flush() + os.fsync(tmp.fileno()) + temp_path = tmp.name + platform = _snapshot_platform() + if platform == "nt": + try: + _windows_replace_file_write_through(temp_path, path) + except OSError as exc: + raise _SnapshotPointerCommitUncertain( + f"could not durably replace snapshot-pointer file {path}" + ) from exc + return + if platform != "posix": + raise RuntimeError( + "Tenki snapshot registry cannot prove rename durability on " + f"platform {platform!r}" + ) + + os.replace(temp_path, path) + # The file contents were fsynced before replace; fsync the containing + # directory too so the pointer rename itself survives a host crash. + # A POSIX failure is an *uncertain commit*: the visible file is new, + # but a crash may roll the directory entry back. The caller must keep + # both remote snapshots and the live sandbox in that state. + dir_fd: int | None = None + try: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + dir_fd = os.open(path.parent, flags) + os.fsync(dir_fd) + except OSError as exc: + raise _SnapshotPointerCommitUncertain( + f"could not fsync snapshot-pointer directory {path.parent}" + ) from exc + finally: + if dir_fd is not None: + try: + os.close(dir_fd) + except OSError: + pass + finally: + if temp_path and os.path.exists(temp_path): + try: + os.unlink(temp_path) + except OSError: + pass def _snapshot_key(task_id: str) -> str: return f"{_SNAPSHOT_NAMESPACE}:{task_id}" +def _snapshot_retirement_key(snapshot_id: str) -> str: + return f"{_SNAPSHOT_RETIREMENT_NAMESPACE}:{snapshot_id}" + + +def _snapshot_retired_key(snapshot_id: str) -> str: + return f"{_SNAPSHOT_RETIRED_NAMESPACE}:{snapshot_id}" + + +def _create_attempt_key(task_id: str) -> str: + return f"{_CREATE_ATTEMPT_NAMESPACE}:{task_id}" + + +def _remote_binding_key(task_id: str) -> str: + return f"{_REMOTE_BINDING_NAMESPACE}:{task_id}" + + +def _create_attempt_state( + task_id: str, + store_path: Path | None = None, +) -> tuple[str | None, float | None]: + snapshots = _load_snapshots(store_path) + value = snapshots.get(_create_attempt_key(task_id)) + if isinstance(value, str) and value: + return value, None + if not isinstance(value, dict): + return None, None + attempt_id = value.get("attempt_id") + expires_at = value.get("expires_at") + return ( + attempt_id if isinstance(attempt_id, str) and attempt_id else None, + float(expires_at) if isinstance(expires_at, (int, float)) else None, + ) + + +def _get_create_attempt( + task_id: str, + store_path: Path | None = None, +) -> str | None: + return _create_attempt_state(task_id, store_path)[0] + + +def _remote_binding_state( + task_id: str, + store_path: Path | None = None, +) -> tuple[ + str | None, + str | None, + bool, + bool, + tuple[str, ...], + bool, +]: + snapshots = _load_snapshots(store_path) + value = snapshots.get(_remote_binding_key(task_id)) + if isinstance(value, str) and value: + # This unversioned shape was used only by unpublished development + # candidates. It cannot prove how ownership was established, so never + # auto-adopt it as a supported Hermes lineage. + return value, None, False, True, (value,), False + if not isinstance(value, dict): + return None, None, False, False, (), False + remote_id = value.get("remote_id") + attempt_id = value.get("attempt_id") + conflict_ids = value.get("conflict_ids") + parsed_conflict_ids = tuple( + sorted({ + item + for item in ( + conflict_ids if isinstance(conflict_ids, list) else [] + ) + if isinstance(item, str) and item + }) + ) + parsed_remote_id = ( + remote_id if isinstance(remote_id, str) and remote_id else None + ) + parsed_attempt_id = ( + attempt_id if isinstance(attempt_id, str) and attempt_id else None + ) + conflicted = bool(value.get("conflicted", False)) + return ( + parsed_remote_id, + parsed_attempt_id, + bool(value.get("validated", False)) and parsed_attempt_id is not None, + conflicted, + parsed_conflict_ids, + bool(value.get("unresolvable", conflicted and not parsed_conflict_ids)), + ) + + +def _get_remote_binding( + task_id: str, + store_path: Path | None = None, +) -> str | None: + return _remote_binding_state(task_id, store_path)[0] + + +def _begin_create_attempt( + task_id: str, + attempt_id: str, + expires_at: float, + store_path: Path | None = None, +) -> None: + """Durably journal one unique remote create before issuing its RPC.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _create_attempt_key(task_id) + existing = snapshots.get(key) + if existing not in (None, attempt_id): + existing_id = ( + existing.get("attempt_id") + if isinstance(existing, dict) + else existing + ) + if existing_id != attempt_id: + raise _SnapshotPointerConflict( + f"task {task_id} already has unresolved create attempt " + f"{existing_id}" + ) + snapshots[key] = { + "attempt_id": attempt_id, + "expires_at": expires_at, + } + _atomic_save_snapshots(path, snapshots) + + +def _clear_create_attempt( + task_id: str, + attempt_id: str, + store_path: Path | None = None, +) -> None: + """Durably clear exactly the create attempt whose remote is gone.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _create_attempt_key(task_id) + existing = snapshots.get(key) + if existing is None: + # A prior removal may be visible after an uncertain directory/ + # write-through commit. Re-publish that marker-free state so this + # retry establishes durability before ownership can be released. + _atomic_save_snapshots(path, snapshots) + return + existing_id = ( + existing.get("attempt_id") + if isinstance(existing, dict) + else existing + ) + if existing_id != attempt_id: + raise _SnapshotPointerConflict( + f"create attempt advanced from {attempt_id} to {existing_id}" + ) + snapshots.pop(key, None) + _atomic_save_snapshots(path, snapshots) + + +def _store_remote_binding( + task_id: str, + remote_id: str, + attempt_id: str | None, + *, + validated: bool, + store_path: Path | None = None, +) -> None: + """Bind a task to one authoritative Tenki id before the remote is used.""" + if validated and attempt_id is None: + raise ValueError( + "a validated Tenki binding requires its durable create attempt id" + ) + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + attempt_key = _create_attempt_key(task_id) + existing_attempt = snapshots.get(attempt_key) + if isinstance(existing_attempt, dict): + existing_attempt = existing_attempt.get("attempt_id") + if attempt_id is not None and existing_attempt != attempt_id: + raise _SnapshotPointerConflict( + f"create attempt advanced from {attempt_id} to " + f"{existing_attempt}" + ) + binding_key = _remote_binding_key(task_id) + existing_binding = snapshots.get(binding_key) + if isinstance(existing_binding, dict): + existing_binding = existing_binding.get("remote_id") + if existing_binding not in (None, remote_id): + raise _SnapshotPointerConflict( + f"remote binding advanced from {remote_id} to " + f"{existing_binding}" + ) + snapshots[binding_key] = { + "remote_id": remote_id, + "attempt_id": attempt_id, + "validated": validated, + "conflicted": False, + "unresolvable": False, + } + if attempt_id is not None: + snapshots.pop(attempt_key, None) + _atomic_save_snapshots(path, snapshots) + + +def _replace_create_attempt_with_lineage_conflict( + task_id: str, + attempt_id: str, + remote_ids: list[str], + *, + unresolvable: bool, + store_path: Path | None = None, +) -> None: + """Atomically replace an uncertain create with a durable conflict.""" + ids = sorted(set(remote_ids)) + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + attempt_key = _create_attempt_key(task_id) + existing_attempt = snapshots.get(attempt_key) + existing_attempt_id = ( + existing_attempt.get("attempt_id") + if isinstance(existing_attempt, dict) + else existing_attempt + ) + if existing_attempt_id != attempt_id: + raise _SnapshotPointerConflict( + f"create attempt advanced from {attempt_id} to " + f"{existing_attempt_id}" + ) + binding_key = _remote_binding_key(task_id) + if snapshots.get(binding_key) is not None: + raise _SnapshotPointerConflict( + f"task {task_id} acquired a remote binding during reconciliation" + ) + snapshots[binding_key] = { + "remote_id": ids[0] if ids else None, + "attempt_id": attempt_id, + "validated": False, + "conflicted": True, + "conflict_ids": ids, + "unresolvable": unresolvable, + } + snapshots.pop(attempt_key, None) + _atomic_save_snapshots(path, snapshots) + + +def _mark_remote_binding_validated( + task_id: str, + remote_id: str, + store_path: Path | None = None, +) -> None: + """Durably publish positive sole-lineage validation for a bound remote.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _remote_binding_key(task_id) + existing = snapshots.get(key) + existing_id = ( + existing.get("remote_id") + if isinstance(existing, dict) + else existing + ) + if existing_id != remote_id: + raise _SnapshotPointerConflict( + f"remote binding advanced from {remote_id} to {existing_id}" + ) + if isinstance(existing, dict) and existing.get("conflicted"): + raise _SnapshotPointerConflict( + f"remote binding {remote_id} has a durable lineage conflict" + ) + if isinstance(existing, dict): + existing = dict(existing) + else: + existing = {"remote_id": remote_id, "attempt_id": None} + existing["validated"] = True + snapshots[key] = existing + _atomic_save_snapshots(path, snapshots) + + +def _mark_remote_binding_conflicted( + task_id: str, + remote_id: str, + conflict_ids: list[str], + *, + unresolvable: bool, + store_path: Path | None = None, +) -> None: + """Permanently remember an observed fork despite later list omissions.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _remote_binding_key(task_id) + existing = snapshots.get(key) + existing_id = ( + existing.get("remote_id") + if isinstance(existing, dict) + else existing + ) + if existing_id != remote_id: + raise _SnapshotPointerConflict( + f"remote binding advanced from {remote_id} to {existing_id}" + ) + if isinstance(existing, dict): + conflict_record: dict[str, Any] = dict(existing) + else: + conflict_record = { + "remote_id": remote_id, + "attempt_id": None, + } + conflict_record["validated"] = False + conflict_record["conflicted"] = True + conflict_record["conflict_ids"] = sorted(set(conflict_ids)) + conflict_record["unresolvable"] = unresolvable + snapshots[key] = conflict_record + _atomic_save_snapshots(path, snapshots) + + +def _store_unmanaged_lineage_conflict( + task_id: str, + remote_ids: list[str], + *, + unresolvable: bool, + store_path: Path | None = None, +) -> None: + """Record visible task-name collisions without claiming their ownership.""" + ids = sorted(set(remote_ids)) + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + if snapshots.get(_create_attempt_key(task_id)) is not None: + raise _SnapshotPointerConflict( + f"task {task_id} acquired a create attempt during collision check" + ) + binding_key = _remote_binding_key(task_id) + if snapshots.get(binding_key) is not None: + raise _SnapshotPointerConflict( + f"task {task_id} acquired a remote binding during collision check" + ) + snapshots[binding_key] = { + # None means at least one visible collision had no authoritative + # id. That conflict cannot be auto-cleared, because no exact + # lookup can ever prove the unidentified branch terminated. + "remote_id": ids[0] if ids else None, + "attempt_id": None, + "validated": False, + "conflicted": True, + "unmanaged": True, + "conflict_ids": ids, + "unresolvable": unresolvable, + } + _atomic_save_snapshots(path, snapshots) + + +def _clear_remote_binding( + task_id: str, + remote_id: str, + store_path: Path | None = None, +) -> None: + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _remote_binding_key(task_id) + existing = snapshots.get(key) + if isinstance(existing, dict): + existing = existing.get("remote_id") + if existing is None: + _atomic_save_snapshots(path, snapshots) + return + if existing != remote_id: + raise _SnapshotPointerConflict( + f"remote binding advanced from {remote_id} to {existing}" + ) + snapshots.pop(key, None) + _atomic_save_snapshots(path, snapshots) + + +def _pending_snapshot_retirements( + store_path: Path | None = None, +) -> tuple[str, ...]: + snapshots = _load_snapshots(store_path) + prefix = f"{_SNAPSHOT_RETIREMENT_NAMESPACE}:" + return tuple( + value + for key, value in snapshots.items() + if key.startswith(prefix) and isinstance(value, str) and value + ) + + +def _queue_snapshot_retirement( + snapshot_id: str, + store_path: Path | None = None, +) -> None: + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + if _snapshot_retired_key(snapshot_id) in snapshots: + return + snapshots[_snapshot_retirement_key(snapshot_id)] = snapshot_id + _atomic_save_snapshots(path, snapshots) + + +def _queue_snapshot_pointer_retirement( + task_id: str, + snapshot_id: str, + store_path: Path | None = None, +) -> None: + """Atomically detach one unusable pointer and journal its retirement.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + for key in (_snapshot_key(task_id), task_id): + if snapshots.get(key) == snapshot_id: + snapshots.pop(key, None) + if _snapshot_retired_key(snapshot_id) not in snapshots: + snapshots[_snapshot_retirement_key(snapshot_id)] = snapshot_id + _atomic_save_snapshots(path, snapshots) + + +def _confirm_snapshot_store_durable( + store_path: Path | None = None, +) -> None: + """Re-establish local durability before acting on retirement records. + + A prior pointer replace may be visible even though its directory fsync + failed. Retrying that fsync before any remote deletion makes the visible + pointer+journal commit durable and preserves the uncertain-commit rule: + neither predecessor is retired until local recovery metadata is safe. + """ + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + if not path.exists(): + return + with path.open("rb") as store_file: + os.fsync(store_file.fileno()) + platform = _snapshot_platform() + if platform == "nt": + # Re-publish the visible state through MoveFileExW with + # MOVEFILE_WRITE_THROUGH. This also upgrades state left by an + # uncertain prior replacement before a retirement retry can delete + # a remote recovery copy. + _atomic_save_snapshots(path, _load_json_store(path)) + return + if platform != "posix": + raise RuntimeError( + "Tenki snapshot registry cannot confirm rename durability on " + f"platform {platform!r}" + ) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + dir_fd = os.open(path.parent, flags) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + + def _get_snapshot_restore_candidate( task_id: str, store_path: Path | None = None ) -> tuple[str | None, bool]: @@ -99,27 +861,87 @@ def _get_snapshot_restore_candidate( return None, False -def _store_snapshot(task_id: str, snapshot_id: str, store_path: Path | None = None) -> None: - snapshots = _load_snapshots(store_path) - snapshots[_snapshot_key(task_id)] = snapshot_id - snapshots.pop(task_id, None) - _save_snapshots(snapshots, store_path) +def _store_snapshot( + task_id: str, + snapshot_id: str, + store_path: Path | None = None, +) -> str | None: + """Atomically install *snapshot_id* and return its predecessor, if any.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + key = _snapshot_key(task_id) + if _snapshot_retired_key(snapshot_id) in snapshots: + raise _SnapshotPointerConflict( + f"snapshot {snapshot_id} was already retired" + ) + previous = snapshots.get(key) + if previous is None: + previous = snapshots.get(task_id) + snapshots[key] = snapshot_id + snapshots.pop(task_id, None) + if ( + isinstance(previous, str) + and previous + and previous != snapshot_id + ): + # Journal the predecessor in the same atomic commit as the new + # pointer. A crash or transient delete failure can then retry + # retirement without ever forgetting the old remote snapshot. + snapshots[_snapshot_retirement_key(previous)] = previous + try: + _atomic_save_snapshots(path, snapshots) + except _SnapshotPointerCommitUncertain as exc: + exc.previous_snapshot_id = ( + previous if isinstance(previous, str) and previous else None + ) + exc.new_snapshot_id = snapshot_id + raise + return previous if isinstance(previous, str) and previous else None -def _delete_snapshot( - task_id: str, snapshot_id: str | None = None, store_path: Path | None = None +def _migrate_snapshot_pointer( + task_id: str, + source_task_id: str, + snapshot_id: str, + store_path: Path | None = None, ) -> None: - snapshots = _load_snapshots(store_path) - updated = False - for key in (_snapshot_key(task_id), task_id): - value = snapshots.get(key) - if value is None: - continue - if snapshot_id is None or value == snapshot_id: - snapshots.pop(key, None) - updated = True - if updated: - _save_snapshots(snapshots, store_path) + """CAS-migrate one legacy pointer without overwriting newer state.""" + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + if _snapshot_retired_key(snapshot_id) in snapshots: + raise _SnapshotPointerConflict( + f"legacy snapshot {snapshot_id} was already retired" + ) + + target_key = _snapshot_key(task_id) + target_value = snapshots.get(target_key) + source_keys = ( + _snapshot_key(source_task_id), + source_task_id, + ) + source_matches = any( + snapshots.get(key) == snapshot_id for key in source_keys + ) + + if target_value not in (None, snapshot_id): + raise _SnapshotPointerConflict( + f"canonical pointer advanced to {target_value}" + ) + if target_value is None and not source_matches: + raise _SnapshotPointerConflict( + "legacy source pointer changed before migration" + ) + + snapshots[target_key] = snapshot_id + for source_key in source_keys: + if source_key != target_key and snapshots.get(source_key) == snapshot_id: + snapshots.pop(source_key, None) + # Same-task legacy format is the plain task key. + if snapshots.get(task_id) == snapshot_id: + snapshots.pop(task_id, None) + _atomic_save_snapshots(path, snapshots) def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]: @@ -207,7 +1029,41 @@ def _positive_float(value: Any) -> float | None: number = float(value) except (TypeError, ValueError): return None - return number if number > 0 else None + return number if number > 0 and math.isfinite(number) else None + + +def _tenki_resource_value( + value: Any, + *, + minimum: int, + maximum: int, + label: str, + alignment: int = 1, +) -> int | None: + """Return a supported Tenki resource value, otherwise use SDK defaults.""" + try: + number = math.ceil(float(value)) + except (TypeError, ValueError, OverflowError): + number = None + if ( + number is not None + and minimum <= number <= maximum + and number % alignment == 0 + ): + return number + alignment_hint = ( + f" and aligned to {alignment}" if alignment > 1 else "" + ) + logger.warning( + "Tenki: ignoring unsupported %s %r (supported range: %s-%s%s); " + "using the Tenki workspace default", + label, + value, + minimum, + maximum, + alignment_hint, + ) + return None def _rewrite_sudo_noninteractive(command: str) -> tuple[str, int]: @@ -323,22 +1179,58 @@ def __init__( self._Sandbox = Sandbox self._client = None self._sandbox = None + self._create_outcome_uncertain = False + self._create_attempt_id: str | None = None + self._create_attempt_expires_at: float | None = None + self._remote_binding_id: str | None = None + self._remote_binding_attempt_id: str | None = None + self._remote_binding_validated = False + self._remote_binding_conflicted = False + self._remote_binding_conflict_ids: tuple[str, ...] = () + self._remote_binding_unresolvable = False + self._create_lineage_ambiguous = False self._lock = threading.Lock() + self._lifecycle_condition = threading.Condition(self._lock) self._persistent = persistent_filesystem self._sync_hermes_home = sync_hermes_home self._sync_manager: FileSyncManager | None = None self._cleanup_in_progress = False + self._cleanup_complete = False self._cleanup_sandbox = None + self._cancel_in_progress = 0 + self._cancel_generation = 0 + self._active_operations = 0 self._task_id = task_id self._profile_token = _profile_token() + canonical_prefix = f"tenki:{self._profile_token}:" + task_suffix = ( + self._task_id[len(canonical_prefix):] + if self._task_id.startswith(canonical_prefix) + else None + ) + self._profile_task_candidates = [ + (self._profile_token, self._task_id), + *[ + ( + token, + f"tenki:{token}:{task_suffix}" + if task_suffix is not None + else self._task_id, + ) + for token in _legacy_profile_tokens() + ], + ] # Bind the profile's snapshot-store path at construction, while the # correct HERMES_HOME context is active. Cleanup (and the idle-reaper # snapshot save) can run in a background thread that does NOT inherit # the per-turn HERMES_HOME contextvar, so re-resolving there would write # the pointer into the wrong profile's home. self._snapshot_store = _snapshot_store_path() + self._profile_home = get_hermes_home() + self._profile_sync_context = copy_context() self._snapshot_restore_id: str | None = None self._snapshot_restore_from_legacy_key = False + self._snapshot_restore_task_id = self._task_id self._image = image self._cpu = cpu self._memory = memory @@ -350,27 +1242,83 @@ def __init__( self._allow_inbound = allow_inbound self._allow_outbound = allow_outbound self._max_duration = max_duration + self._effective_max_duration = _positive_float(max_duration) or 3600 self._idle_timeout = idle_timeout self._pause_retention = pause_retention self._forward_env = _normalize_forward_env_names(forward_env) self._remote_home = "/home/tenki" - if self._persistent: - self._snapshot_restore_id, self._snapshot_restore_from_legacy_key = ( - _get_snapshot_restore_candidate(self._task_id, self._snapshot_store) + # Hold this profile/task lock for the wrapper's full lifetime. It is + # acquired before pointer discovery and remote listing/creation, so two + # Hermes processes cannot fork a task or terminate one another's live + # sandbox. OS crash/exit releases the kernel lock automatically. + self._task_ownership_file = _acquire_task_ownership_lock( + self._profile_home, + self._task_id, + ) + try: + ( + self._create_attempt_id, + self._create_attempt_expires_at, + ) = _create_attempt_state( + self._task_id, + self._snapshot_store, ) - - self._ensure_sandbox() - self._resolve_remote_home() - if self._sync_hermes_home: - self._sync_manager = FileSyncManager( - get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"), - upload_fn=self._tenki_upload, - delete_fn=self._tenki_delete, - bulk_upload_fn=self._tenki_bulk_upload, - bulk_download_fn=self._tenki_bulk_download, - ) - self._sync_manager.sync(force=True) - self.init_session() + ( + self._remote_binding_id, + self._remote_binding_attempt_id, + self._remote_binding_validated, + self._remote_binding_conflicted, + self._remote_binding_conflict_ids, + self._remote_binding_unresolvable, + ) = _remote_binding_state( + self._task_id, + self._snapshot_store, + ) + if self._create_attempt_id is not None and ( + self._remote_binding_id is not None + or self._remote_binding_conflicted + ): + raise RuntimeError( + "Tenki task has both an unresolved create and a remote " + f"binding: {self._task_id}" + ) + self._create_outcome_uncertain = bool(self._create_attempt_id) + if self._persistent: + seen_task_ids: set[str] = set() + for _token, candidate_task_id in self._profile_task_candidates: + if candidate_task_id in seen_task_ids: + continue + seen_task_ids.add(candidate_task_id) + snapshot_id, from_legacy_key = ( + _get_snapshot_restore_candidate( + candidate_task_id, + self._snapshot_store, + ) + ) + if snapshot_id: + self._snapshot_restore_id = snapshot_id + self._snapshot_restore_from_legacy_key = from_legacy_key + self._snapshot_restore_task_id = candidate_task_id + break + + self._ensure_sandbox() + self._resolve_remote_home() + if self._sync_hermes_home: + self._sync_manager = FileSyncManager( + get_files_fn=self._profile_sync_files, + upload_fn=self._tenki_upload, + delete_fn=self._tenki_delete, + bulk_upload_fn=self._tenki_bulk_upload, + bulk_download_fn=self._tenki_bulk_download, + get_upload_only_host_paths_fn=( + self._profile_credential_host_paths + ), + ) + self._sync_manager.sync(force=True) + self.init_session() + except BaseException: + self._abort_failed_initialization() + raise def _sandbox_create_signature(self) -> inspect.Signature | None: """Signature that actually validates the sandbox create kwargs. @@ -413,17 +1361,42 @@ def _create_kwargs(self) -> dict[str, Any]: _add_supported(kwargs, sig, ("snapshot_id",), self._snapshot_restore_id) else: _add_supported(kwargs, sig, ("image", "template"), self._image) - cpu_cores = max(1, math.ceil(float(self._cpu))) if self._cpu else None + cpu_cores = _tenki_resource_value( + self._cpu, + minimum=_TENKI_CPU_RANGE[0], + maximum=_TENKI_CPU_RANGE[1], + label="CPU cores", + ) _add_supported(kwargs, sig, ("cpu_cores", "cpu"), cpu_cores) - _add_supported(kwargs, sig, ("memory_mb", "memory"), self._memory) + memory_mb = _tenki_resource_value( + self._memory, + minimum=_TENKI_MEMORY_MB_RANGE[0], + maximum=_TENKI_MEMORY_MB_RANGE[1], + label="memory (MB)", + alignment=2, + ) + _add_supported(kwargs, sig, ("memory_mb", "memory"), memory_mb) - if self._disk: - disk_gb = max(1, math.ceil(float(self._disk) / 1024)) - _add_supported(kwargs, sig, ("disk_size_gb", "disk_gb", "disk"), disk_gb) + try: + disk_gb_value = float(self._disk) / 1024 + except (TypeError, ValueError, OverflowError): + disk_gb_value = self._disk + disk_gb = _tenki_resource_value( + disk_gb_value, + minimum=_TENKI_DISK_GB_RANGE[0], + maximum=_TENKI_DISK_GB_RANGE[1], + label="root disk (GB)", + ) + _add_supported(kwargs, sig, ("disk_size_gb", "disk_gb", "disk"), disk_gb) _add_supported(kwargs, sig, ("allow_inbound",), self._allow_inbound) _add_supported(kwargs, sig, ("allow_outbound",), self._allow_outbound) - _add_supported(kwargs, sig, ("max_duration",), self._max_duration) + _add_supported( + kwargs, + sig, + ("max_duration",), + self._effective_max_duration, + ) idle_timeout = _positive_float(self._idle_timeout) if idle_timeout is not None: idle_timeout_minutes = max(1, math.ceil(idle_timeout / 60)) @@ -452,7 +1425,11 @@ def _create_kwargs(self) -> dict[str, Any]: }, ) _add_supported(kwargs, sig, ("tags",), ["hermes-agent"]) - _add_supported(kwargs, sig, ("wait",), True) + # Keep readiness outside Client.create so every post-RPC readiness + # failure occurs after Hermes owns the exact Sandbox handle. Otherwise + # the SDK can construct a Sandbox, fail in its internal wait, and raise + # without returning the only identity that is safe to clean up. + _add_supported(kwargs, sig, ("wait",), False) # Do NOT emit a create-time ``timeout`` here: the SDK's Sandbox.create # pops ``timeout`` into the *Client* (HTTP) timeout, while Client.create # treats ``timeout`` as the *wait-for-ready* budget — so the same value @@ -495,9 +1472,9 @@ def _sandbox_env(self) -> dict[str, str]: logger.warning( "Tenki: forwarding the control-plane credential %s into the " "sandbox as requested by terminal.tenki_forward_env. Guest code " - "can read it and create/terminate/bill account resources. Note " - "that forwarded credentials are NOT profile-isolated under the " - "multiplexing gateway's shared terminal cache.", + "can read it and create/terminate/bill account resources. " + "Tenki sandboxes are profile-isolated, but every process inside " + "this profile's sandbox can read the forwarded value.", key, ) return env @@ -553,7 +1530,10 @@ def _create_client(self): def _sandbox_name(self) -> str: # The profile token namespaces the name so two profiles sharing one # Tenki account never collide on a name or reuse each other's sandbox. - return f"{self._name_prefix}-{self._profile_token}-{_safe_name(self._task_id)}" + return self._sandbox_name_for(self._profile_token, self._task_id) + + def _sandbox_name_for(self, profile_token: str, task_id: str) -> str: + return f"{self._name_prefix}-{profile_token}-{_safe_name(task_id)}" @staticmethod def _sandbox_state(sandbox: Any) -> str: @@ -570,19 +1550,48 @@ def _sandbox_matches_task(self, sandbox: Any) -> bool: info = getattr(sandbox, "info", None) if not name and info is not None: name = getattr(info, "name", "") - if name != self._sandbox_name(): + matched_identity = next( + ( + (profile_token, task_id) + for profile_token, task_id in self._profile_task_candidates + if name == self._sandbox_name_for(profile_token, task_id) + ), + None, + ) + if matched_identity is None: return False + matched_token, matched_task_id = matched_identity metadata = getattr(info, "metadata", {}) if info is not None else {} # Never reuse another profile's sandbox: if the candidate carries a # profile token it must match ours (the name already encodes it, but # metadata is the authoritative, defense-in-depth check). if isinstance(metadata, dict) and metadata.get("hermes_profile"): - if metadata.get("hermes_profile") != self._profile_token: + if metadata.get("hermes_profile") != matched_token: return False if isinstance(metadata, dict) and metadata.get("hermes_task_id"): - return metadata.get("hermes_task_id") == self._task_id + return metadata.get("hermes_task_id") == matched_task_id return True + def _sandbox_has_owned_identity( + self, + sandbox: Any, + expected_attempt_id: str, + ) -> bool: + """Require the canonical metadata Hermes writes before claiming use.""" + name = getattr(sandbox, "name", "") + info = getattr(sandbox, "info", None) + if not name and info is not None: + name = getattr(info, "name", "") + metadata = getattr(info, "metadata", None) if info is not None else None + return ( + name == self._sandbox_name() + and isinstance(metadata, dict) + and metadata.get("hermes_backend") == "tenki" + and metadata.get("hermes_task_id") == self._task_id + and metadata.get("hermes_profile") == self._profile_token + and metadata.get("hermes_create_attempt") == expected_attempt_id + ) + def _list_kwargs(self, client: Any) -> dict[str, Any]: """Kwargs for the sandbox listing used to re-attach a persistent sandbox. @@ -602,118 +1611,436 @@ def _list_kwargs(self, client: Any) -> dict[str, Any]: kwargs["workspace_id"] = self._workspace_id return kwargs - def _find_persistent_sandbox(self): + def _assert_no_unmanaged_persistent_sandbox(self) -> None: + """Refuse visible name collisions that have no local ownership state. + + Tenki support is new in this unmerged change, so supported Hermes + sandboxes always have a create-attempt or exact-id binding in this + profile store. A matching remote with neither is external/unpublished + state: listing may detect it, but listing can never confer ownership. + """ if not self._persistent: - return None + return client = self._create_client() try: candidates = client.list(**self._list_kwargs(client)) except Exception as exc: - logger.debug("Tenki: could not list persistent sandboxes: %s", exc) - return None - - usable = [] + # A failed listing is not proof that no sandbox exists. Creating + # from the last snapshot in this state can fork a live sandbox + # containing newer, unsnapshotted state. + raise RuntimeError( + "Tenki could not check for unmanaged persistent sandbox " + f"collisions for task {self._task_id}" + ) from exc + + remote_ids: list[str] = [] + unidentified_collision = False for sandbox in candidates: if not self._sandbox_matches_task(sandbox): continue state = self._sandbox_state(sandbox) if state in self._terminal_states: continue - usable.append((state, sandbox)) - if not usable: - return None - usable.sort(key=lambda item: 0 if item[0] == "RUNNING" else 1) - return usable[0][1] - - def _resume_persistent_sandbox(self): - sandbox = self._find_persistent_sandbox() - if sandbox is None: - return None - if not self._ensure_sandbox_ready(sandbox): - logger.info( - "Tenki: existing sandbox for task %s is no longer reusable; creating a fresh sandbox", + remote_id = self._sandbox_identity(sandbox) + if remote_id is None: + unidentified_collision = True + continue + remote_ids.append(remote_id) + if not remote_ids and not unidentified_collision: + return + try: + _store_unmanaged_lineage_conflict( self._task_id, + remote_ids, + unresolvable=unidentified_collision, + store_path=self._snapshot_store, ) - return None - sandbox_id = getattr(sandbox, "id", None) or getattr(sandbox, "sandbox_id", None) - logger.info("Tenki: resumed sandbox %s for task %s", sandbox_id or "", self._task_id) - return sandbox + except BaseException: + self._create_lineage_ambiguous = True + raise + self._remote_binding_id = ( + None + if unidentified_collision + else sorted(set(remote_ids))[0] + ) + self._remote_binding_attempt_id = None + self._remote_binding_validated = False + self._remote_binding_conflicted = True + self._remote_binding_conflict_ids = ( + tuple(sorted(set(remote_ids))) + ) + self._remote_binding_unresolvable = unidentified_collision + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki found an unmanaged persistent sandbox collision for task " + f"{self._task_id}; terminate the reported remote ids before retrying" + ) def _ensure_sandbox_ready(self, sandbox: Any) -> bool: + """Return False only when *sandbox* is definitively terminal. + + All ambiguous control-plane failures propagate. Callers may create a + replacement only after a successful refresh proves the old sandbox is + terminal; treating a transient refresh/resume/readiness error as + absence would fork persistent state. + """ refresh = getattr(sandbox, "refresh", None) if callable(refresh): try: refresh() except Exception as exc: - logger.info("Tenki: sandbox refresh failed for task %s: %s", self._task_id, exc) - return False + raise RuntimeError( + "Tenki could not refresh the persistent sandbox for task " + f"{self._task_id}" + ) from exc state = self._sandbox_state(sandbox) if state in self._terminal_states: return False try: - if state and state != "RUNNING": + if state != "RUNNING": resume = getattr(sandbox, "resume", None) - if callable(resume): - resume() + if not callable(resume): + raise RuntimeError( + "persistent sandbox is not running and cannot be resumed" + ) + resume() wait_ready = getattr(sandbox, "wait_ready", None) if callable(wait_ready): wait_ready(max(60, self.timeout)) except Exception as exc: - logger.info("Tenki: could not make sandbox ready for task %s: %s", self._task_id, exc) - return False + raise RuntimeError( + "Tenki could not make the persistent sandbox ready for task " + f"{self._task_id}" + ) from exc - return self._sandbox_state(sandbox) not in self._terminal_states + final_state = self._sandbox_state(sandbox) + if final_state in self._terminal_states: + return False + if final_state != "RUNNING": + raise RuntimeError( + "Tenki could not confirm that the persistent sandbox is " + f"running for task {self._task_id} (state={final_state!r})" + ) + return True def _ensure_sandbox(self) -> None: - with self._lock: - if self._cleanup_in_progress: + with self._lifecycle_condition: + # Sandbox-level cancellation pauses/terminates outside the lock. + # Do not inspect or replace its exact reference until cancellation + # has restored the ownership invariant and notified waiters. + while self._cancel_in_progress: + self._lifecycle_condition.wait() + if self._cleanup_in_progress or self._cleanup_complete: raise RuntimeError("Tenki cleanup is in progress") + if ( + self._remote_binding_id is not None + or self._remote_binding_conflicted + ): + self._resolve_remote_binding() + if self._create_outcome_uncertain: + if not self._reconcile_uncertain_create(): + raise RuntimeError( + "Tenki cannot create a replacement while a prior " + f"create is unresolved for task {self._task_id}" + ) if self._sandbox is not None: - if self._ensure_sandbox_ready(self._sandbox): + sandbox = self._sandbox + if not self._remote_binding_validated: + self._validate_created_lineage() + if self._ensure_sandbox_ready(sandbox): + self._after_sandbox_ownership_confirmed() return self._sandbox = None - self._sandbox = self._resume_persistent_sandbox() - if self._sandbox is not None: - return - self._sandbox = self._create_sandbox_with_snapshot_fallback() + if not self._clear_remote_binding_marker(sandbox): + raise RuntimeError( + "Tenki cannot replace a terminal sandbox until its " + f"remote binding is durably cleared for {self._task_id}" + ) + self._assert_no_unmanaged_persistent_sandbox() + try: + self._sandbox = self._create_sandbox_with_snapshot_fallback() + except BaseException: + if self._create_outcome_uncertain: + self._reconcile_uncertain_create() + raise + self._validate_created_lineage() + self._wait_created_sandbox_ready(self._sandbox) + self._after_sandbox_ownership_confirmed() sandbox_id = getattr(self._sandbox, "id", None) or getattr(self._sandbox, "sandbox_id", None) logger.info("Tenki: created sandbox %s for task %s", sandbox_id or "", self._task_id) + def _after_sandbox_ownership_confirmed(self) -> None: + """Retry local housekeeping only after exact live ownership exists.""" + if not self._persistent: + return + self._migrate_loaded_snapshot_pointer() + self._retry_pending_snapshot_retirements() + def _require_sandbox(self) -> Any: # Capture the reference under the lock: cancel() may null out # self._sandbox between _ensure_sandbox() and the caller's use of it, # and the operation must run against the sandbox that ensure produced. self._ensure_sandbox() with self._lock: + # cleanup() can claim the sandbox after _ensure_sandbox() releases + # the lock but before this capture. Never hand a caller the same + # reference cleanup is syncing, snapshotting, or terminating. + if self._cleanup_in_progress or self._cleanup_complete: + raise RuntimeError("Tenki cleanup is in progress") sandbox = self._sandbox if sandbox is None: raise RuntimeError("Tenki sandbox was torn down mid-operation") return sandbox + @contextmanager + def _sandbox_operation(self): + """Lease one sandbox reference for the full duration of an operation.""" + while True: + with self._lifecycle_condition: + while self._cancel_in_progress: + self._lifecycle_condition.wait() + cancel_generation = self._cancel_generation + sandbox = self._require_sandbox() + with self._lifecycle_condition: + if self._cancel_in_progress: + # Cancellation may pause or detach the reference returned + # above. Wait for exact ownership to be restored, then run + # ensure again so a paused persistent sandbox is resumed. + while self._cancel_in_progress: + self._lifecycle_condition.wait() + continue + if self._cancel_generation != cancel_generation: + # A cancel completed entirely between readiness resolution + # and lease publication. Even when persistent ownership + # restored the same object, it may now be PAUSED; run the + # readiness path again before publishing a lease. + continue + if self._cleanup_in_progress or self._cleanup_complete: + raise RuntimeError("Tenki cleanup is in progress") + if self._sandbox is not sandbox: + # An ephemeral cancel detached this candidate between + # require() and lease publication. Resolve a fresh one. + continue + self._active_operations += 1 + break + try: + yield sandbox + finally: + with self._lifecycle_condition: + self._active_operations -= 1 + self._lifecycle_condition.notify_all() + + @contextmanager + def _transfer_operation(self): + """Use cleanup's owned sandbox for sync-back, otherwise take a lease.""" + with self._lifecycle_condition: + cleanup_sandbox = ( + self._cleanup_sandbox if self._cleanup_in_progress else None + ) + if cleanup_sandbox is not None: + yield cleanup_sandbox + return + with self._sandbox_operation() as sandbox: + yield sandbox + def _create_sandbox_from_kwargs(self, kwargs: dict[str, Any]): - if self._persistent: - client = self._create_client() - create_kwargs = dict(kwargs) - for key in ("auth_token", "api_key", "base_url", "api_endpoint"): - create_kwargs.pop(key, None) - return client.create(**create_kwargs) - return self._Sandbox.create(**kwargs) + # Always create through the client owned by this environment. In Tenki + # 0.5.1, Sandbox.create() constructs a hidden Client and marks the + # sandbox as owning it, but Sandbox.close()/terminate() does not close + # that client; only the SDK context-manager exit does. Keeping one + # explicit client lets cleanup close the control-plane channel for + # ephemeral sandboxes too, and reuse it safely after cancellation. + client = self._create_client() + create_kwargs = dict(kwargs) + for key in ("auth_token", "api_key", "base_url", "api_endpoint"): + create_kwargs.pop(key, None) + + # Prove Python call-shape failures before journaling or issuing the + # non-idempotent RPC. Anything that escapes Client.create() after this + # point is outcome-uncertain: Tenki decodes the response into + # SandboxInfo only after create_session commits, and that post-RPC + # conversion can itself raise TypeError or ValueError. + try: + inspect.signature(client.create).bind(**create_kwargs) + except (TypeError, ValueError) as exc: + raise TypeError( + "Tenki create arguments do not match the installed SDK" + ) from exc + + if self._create_attempt_id is not None: + raise RuntimeError( + "Tenki cannot start a new create while attempt " + f"{self._create_attempt_id} is unresolved" + ) + attempt_id = uuid.uuid4().hex + expires_at = ( + time.time() + + self._effective_max_duration + + _CREATE_ATTEMPT_EXPIRY_GRACE + ) + _begin_create_attempt( + self._task_id, + attempt_id, + expires_at, + self._snapshot_store, + ) + self._create_attempt_id = attempt_id + self._create_attempt_expires_at = expires_at + metadata = dict(create_kwargs.get("metadata") or {}) + metadata["hermes_create_attempt"] = attempt_id + create_kwargs["metadata"] = metadata + + # The RPC may commit remotely and then time out before returning a + # handle. The durable attempt id remains both local and remotely + # queryable until that exact remote is gone. + self._create_outcome_uncertain = True + sandbox = client.create(**create_kwargs) + if not self._bind_remote_sandbox(sandbox, attempt_id=attempt_id): + raise RuntimeError( + "Tenki created a sandbox but could not durably bind its " + f"remote identity for task {self._task_id}" + ) + self._create_outcome_uncertain = False + return sandbox - # Snapshot errors that mean the recorded snapshot can never restore, so - # dropping the pointer and booting the base image is the right recovery. - # A *transient* failure (network, rate-limit, ambiguous) is NOT in this set: - # it must propagate with the pointer intact so a later attempt can still - # recover the persistent state instead of silently booting an empty sandbox. - # Snapshot-specific error type names that mean the snapshot can never - # restore. Note InvalidStateError is intentionally NOT here: the restore RPC - # maps a generic FAILED_PRECONDITION (workspace/policy/etc.) to it, so it is - # only unrecoverable when its message points at the snapshot itself - # (handled by message inspection below); a bare InvalidStateError stays - # transient so an unrelated precondition can't destroy a valid pointer. - # ``RegistryArtifactNotFoundError`` was renamed ``RegistryImageNotFoundError`` - # in tenki 0.5; both names are listed so either SDK generation resolves. + def _validate_created_lineage(self) -> None: + """Validate exact ownership; use list only to detect visible conflicts.""" + sandbox = self._sandbox + expected_attempt_id = self._remote_binding_attempt_id + remote_id = self._sandbox_identity(sandbox) + client = self._client + get_sandbox = getattr(client, "get", None) + list_sandboxes = getattr(client, "list", None) + if ( + expected_attempt_id is None + or remote_id is None + or remote_id != self._remote_binding_id + or not self._sandbox_has_owned_identity( + sandbox, + expected_attempt_id, + ) + or not callable(get_sandbox) + or not callable(list_sandboxes) + ): + self._create_lineage_ambiguous = True + raise RuntimeError( + f"Tenki cannot validate the new lineage for task {self._task_id}" + ) + try: + exact_sandbox = get_sandbox(remote_id) + if ( + self._sandbox_identity(exact_sandbox) != remote_id + or not self._sandbox_has_owned_identity( + exact_sandbox, + expected_attempt_id, + ) + ): + raise RuntimeError( + "Tenki exact-id lookup did not preserve the bound " + "task/profile/create-attempt identity" + ) + if self._sandbox_state(exact_sandbox) in self._terminal_states: + raise RuntimeError( + "Tenki exact-id lookup returned a terminal sandbox" + ) + candidates = list_sandboxes(**self._list_kwargs(client)) + active = [ + candidate + for candidate in candidates + if self._sandbox_matches_task(candidate) + and self._sandbox_state(candidate) not in self._terminal_states + ] + except BaseException as exc: + raise RuntimeError( + "Tenki could not validate that a new sandbox is the sole " + f"lineage for task {self._task_id}" + ) from exc + + other = [ + candidate + for candidate in active + if self._sandbox_identity(candidate) != remote_id + ] + if other: + other_ids = [ + candidate_id + for candidate in other + if (candidate_id := self._sandbox_identity(candidate)) + ] + # An active result without an id is an unresolvable conflict, but + # every known id remains durable and actionable. + unresolvable = len(other_ids) != len(other) + conflict_ids = [remote_id, *other_ids] + # Once a fork has been positively observed, omission-prone future + # listings may never erase that fact or promote one branch. + try: + _mark_remote_binding_conflicted( + self._task_id, + remote_id, + conflict_ids, + unresolvable=unresolvable, + store_path=self._snapshot_store, + ) + except BaseException as exc: + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki detected multiple lineages but could not durably " + f"record the conflict for task {self._task_id}" + ) from exc + self._remote_binding_validated = False + self._remote_binding_conflicted = True + self._remote_binding_conflict_ids = tuple( + sorted(set(conflict_ids)) + ) + self._remote_binding_unresolvable = unresolvable + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki detected multiple active sandbox lineages for task " + f"{self._task_id}; leaving every lineage untouched" + ) + try: + _mark_remote_binding_validated( + self._task_id, + remote_id, + self._snapshot_store, + ) + except BaseException as exc: + raise RuntimeError( + "Tenki could not durably record sole-lineage validation for " + f"task {self._task_id}" + ) from exc + self._remote_binding_validated = True + + def _wait_created_sandbox_ready(self, sandbox: Any) -> None: + """Wait only after the exact newly created Sandbox handle is owned.""" + wait_ready = getattr(sandbox, "wait_ready", None) + if callable(wait_ready): + wait_ready(max(60, self.timeout)) + state = self._sandbox_state(sandbox) + if state in self._terminal_states: + raise RuntimeError( + f"Tenki create ended in terminal state {state!r}" + ) + if state != "RUNNING": + raise RuntimeError( + f"Tenki create did not reach RUNNING state (state={state!r})" + ) + + # Snapshot errors that mean the recorded snapshot can never restore, so + # dropping the pointer and booting the base image is the right recovery. + # A *transient* failure (network, rate-limit, ambiguous) is NOT in this set: + # it must propagate with the pointer intact so a later attempt can still + # recover the persistent state instead of silently booting an empty sandbox. + # Snapshot-specific error type names that mean the snapshot can never + # restore. Note InvalidStateError is intentionally NOT here: the restore RPC + # maps a generic FAILED_PRECONDITION (workspace/policy/etc.) to it, so it is + # only unrecoverable when its message points at the snapshot itself + # (handled by message inspection below); a bare InvalidStateError stays + # transient so an unrelated precondition can't destroy a valid pointer. + # ``RegistryArtifactNotFoundError`` was renamed ``RegistryImageNotFoundError`` + # in tenki 0.5; both names are listed so either SDK generation resolves. _UNRECOVERABLE_SNAPSHOT_ERRORS = frozenset({ "SnapshotNotFoundError", # snapshot is gone "RegistryImageNotFoundError", # backing image is gone (0.5+) @@ -740,7 +2067,20 @@ def _is_snapshot_specific_invalid_state(e: BaseException, invalid_state_cls) -> if invalid_state_cls is None and type(e).__name__ != "InvalidStateError": return False msg = str(e).lower() - return "snapshot" in msg or "durable" in msg + # FAILED_PRECONDITION is a broad service status: workspace policy, + # quota, or a temporarily blocked restore can all map to the same + # InvalidStateError. Discard the pointer only for an explicit, + # permanent non-durability statement; ambiguous mentions of a + # snapshot remain retryable. + return any( + phrase in msg + for phrase in ( + "snapshot is not durable", + "snapshot not durable", + "non-durable snapshot", + "snapshot is non-durable", + ) + ) # Resolve each class independently: a single `from tenki import (...)` # of all four fails outright when one has been renamed (as @@ -779,7 +2119,85 @@ def _is_snapshot_specific_invalid_state(e: BaseException, invalid_state_cls) -> return True return _is_snapshot_specific_invalid_state(exc, None) + def _retire_unrecoverable_restore_pointer( + self, + exc: BaseException, + ) -> None: + snapshot_id = self._snapshot_restore_id + if snapshot_id is None: + return + logger.warning( + "Tenki: snapshot %s for task %s is unrecoverable; creating from " + "base image: %s", + snapshot_id, + self._task_id, + exc, + ) + _queue_snapshot_pointer_retirement( + self._snapshot_restore_task_id, + snapshot_id, + self._snapshot_store, + ) + self._delete_remote_snapshot( + snapshot_id, + reason="unrecoverable restore pointer", + ) + self._snapshot_restore_id = None + self._snapshot_restore_from_legacy_key = False + self._snapshot_restore_task_id = self._task_id + + def _preflight_snapshot_restore(self) -> None: + """Reject a known-bad snapshot before issuing a create RPC.""" + snapshot_id = self._snapshot_restore_id + if snapshot_id is None: + return + client = self._create_client() + snapshots = getattr(client, "snapshots", None) + get_snapshot = getattr(snapshots, "get", None) + if not callable(get_snapshot): + return + try: + snapshot = get_snapshot(snapshot_id) + except BaseException as exc: + if not self._snapshot_unrecoverable(exc): + raise + self._retire_unrecoverable_restore_pointer(exc) + return + + state = str(getattr(snapshot, "state", "") or "").upper() + durability = str( + getattr(snapshot, "durability_state", "") or "" + ).upper() + if state in {"FAILED", "DELETING", "DELETED"}: + reason = str(getattr(snapshot, "failure_reason", "") or "") + detail = f": {reason}" if reason else "" + self._retire_unrecoverable_restore_pointer( + RuntimeError( + f"snapshot {snapshot_id} is {state}{detail}" + ) + ) + return + if durability == "PROPAGATION_FAILED" or ( + state == "READY" and durability == "UNSPECIFIED" + ): + self._retire_unrecoverable_restore_pointer( + RuntimeError( + f"snapshot {snapshot_id} has unusable durability state " + f"{durability or ''}" + ) + ) + return + if durability != "DURABLE": + # A pending/unknown durability transition is not permanent, but + # only Tenki's positive DURABLE state can authorize a restore + # create. Preserve the pointer and retry on a later lifecycle. + raise RuntimeError( + f"snapshot {snapshot_id} is not durably restorable " + f"(state={state!r}, durability={durability!r})" + ) + def _create_sandbox_with_snapshot_fallback(self): + self._preflight_snapshot_restore() kwargs = self._create_kwargs() try: sandbox = self._create_sandbox_from_kwargs(kwargs) @@ -798,26 +2216,184 @@ def _create_sandbox_with_snapshot_fallback(self): exc, ) raise - logger.warning( - "Tenki: snapshot %s for task %s is unrecoverable; creating from base image: %s", + # Client.create can commit and then raise while decoding even with + # wait=False. Reconcile the exact durable attempt before changing + # the snapshot pointer or issuing another non-idempotent create. + if not self._reconcile_uncertain_create(): + raise RuntimeError( + "Tenki could not reconcile the rejected snapshot " + f"create attempt for task {self._task_id}" + ) from exc + if self._sandbox is not None: + # The supposedly failed RPC actually committed. Preserve and + # validate that exact restored sandbox instead of retrying. + return self._sandbox + self._retire_unrecoverable_restore_pointer(exc) + sandbox = self._create_sandbox_from_kwargs(self._create_kwargs()) + return sandbox + + def _migrate_loaded_snapshot_pointer(self) -> None: + """Best-effort move of a loaded legacy pointer to the canonical key. + + Migration is metadata housekeeping performed *after* the remote + sandbox has been restored. A local store/fsync/delete failure must not + discard ownership of that known-good live sandbox; leave the source + pointer and migration flags intact so a later call can retry. + """ + if not self._snapshot_restore_id or not ( + self._snapshot_restore_from_legacy_key + or self._snapshot_restore_task_id != self._task_id + ): + return + source_task_id = self._snapshot_restore_task_id + try: + _migrate_snapshot_pointer( + self._task_id, + source_task_id, self._snapshot_restore_id, + self._snapshot_store, + ) + except _SnapshotPointerConflict: + # This is not a retryable local-I/O failure: another writer has + # advanced or retired recovery state. Never overwrite/delete that + # newer pointer with this wrapper's stale legacy snapshot. + raise + except Exception as exc: + logger.warning( + "Tenki: could not migrate snapshot pointer for task %s from " + "%s; keeping the restored sandbox and source pointer for a " + "later retry: %s", self._task_id, + source_task_id, exc, ) - _delete_snapshot(self._task_id, self._snapshot_restore_id, self._snapshot_store) - self._snapshot_restore_id = None - self._snapshot_restore_from_legacy_key = False - sandbox = self._create_sandbox_from_kwargs(self._create_kwargs()) - else: - if self._snapshot_restore_id and self._snapshot_restore_from_legacy_key: - _store_snapshot(self._task_id, self._snapshot_restore_id, self._snapshot_store) - return sandbox + return + self._snapshot_restore_task_id = self._task_id + self._snapshot_restore_from_legacy_key = False + + def _retry_pending_snapshot_retirements(self) -> None: + """Best-effort retry of durable, profile-scoped remote deletions.""" + try: + _confirm_snapshot_store_durable(self._snapshot_store) + pending = _pending_snapshot_retirements(self._snapshot_store) + except Exception as exc: + logger.warning( + "Tenki: could not load pending snapshot retirements for " + "profile %s: %s", + self._profile_token, + exc, + ) + return + for snapshot_id in pending: + self._retire_pending_snapshot_if_unreferenced( + snapshot_id, + reason="retrying pending retirement", + ) + + def _retire_pending_snapshot_if_unreferenced( + self, + snapshot_id: str, + *, + reason: str, + ) -> bool: + """Reference-check, remote-delete, and tombstone under one store lock.""" + path = self._snapshot_store + with _snapshot_store_lock(path): + snapshots = _load_json_store(path) + pending_key = _snapshot_retirement_key(snapshot_id) + if snapshots.get(pending_key) != snapshot_id: + return True + metadata_prefixes = ( + f"{_SNAPSHOT_RETIREMENT_NAMESPACE}:", + f"{_SNAPSHOT_RETIRED_NAMESPACE}:", + f"{_CREATE_ATTEMPT_NAMESPACE}:", + f"{_REMOTE_BINDING_NAMESPACE}:", + ) + referenced = any( + isinstance(value, str) + and value == snapshot_id + and not key.startswith(metadata_prefixes) + for key, value in snapshots.items() + ) + if referenced: + return False + + # Publish the permanent non-resurrection claim before deleting the + # remote, while retaining the pending retry record. If this commit + # fails or is uncertain, leave the remote intact. A later lifecycle + # first fsyncs visible uncertain state and then safely retries. + retired_key = _snapshot_retired_key(snapshot_id) + if snapshots.get(retired_key) != snapshot_id: + snapshots[retired_key] = snapshot_id + try: + _atomic_save_snapshots(path, snapshots) + except Exception as exc: + logger.warning( + "Tenki: could not durably claim retirement of snapshot " + "%s (%s); leaving the remote intact: %s", + snapshot_id, + reason, + exc, + ) + return False + + remote_snapshots = getattr(self._client, "snapshots", None) + delete = getattr(remote_snapshots, "delete", None) + if not callable(delete): + logger.warning( + "Tenki: cannot retire snapshot %s (%s): SDK delete API " + "unavailable", + snapshot_id, + reason, + ) + return False + try: + delete(snapshot_id) + except Exception as exc: + if not self._snapshot_unrecoverable(exc): + logger.warning( + "Tenki: could not retire snapshot %s (%s): %s", + snapshot_id, + reason, + exc, + ) + return False + + # The tombstone was durable before the delete. Completion only + # clears retry metadata; the tombstone remains indefinitely. + snapshots.pop(pending_key, None) + try: + _atomic_save_snapshots(path, snapshots) + except Exception as exc: + # The remote is gone, but the durable tombstone and pending + # record remain safe. A later not-found retry clears metadata. + logger.warning( + "Tenki: retired remote snapshot %s but could not durably " + "clear its pending-retirement record: %s", + snapshot_id, + exc, + ) + return True + logger.info("Tenki: retired snapshot %s (%s)", snapshot_id, reason) + return True def _remote_transfer_path(self, prefix: str) -> str: base = (self._remote_home or "/home/tenki").rstrip("/") or "/home/tenki" if base != "/home/tenki" and not base.startswith("/home/tenki/"): base = "/home/tenki" - return f"{base}/{prefix}.{os.getpid()}.{self._session_id}.tar" + nonce = uuid.uuid4().hex + return f"{base}/{prefix}.{os.getpid()}.{self._session_id}.{nonce}.tar" + + def _profile_sync_files(self) -> list[tuple[str, str]]: + context = self._profile_sync_context.copy() + return context.run( + iter_sync_files, + f"{self._remote_home}/.hermes", + ) + + def _profile_credential_host_paths(self) -> set[str]: + context = self._profile_sync_context.copy() + return context.run(_credential_host_paths) def _resolve_remote_home(self) -> None: try: @@ -834,10 +2410,10 @@ def _tenki_upload(self, host_path: str, remote_path: str) -> None: # One capture for the whole flow: cancel() nulls out self._sandbox, so # re-reading it per call could send the mkdir and the upload to two # different sandboxes (or to None). - sandbox = self._require_sandbox() - parent = str(Path(remote_path).parent) - sandbox.fs.mkdir(parent, recursive=True) - sandbox.fs.upload(host_path, remote_path) + with self._sandbox_operation() as sandbox: + parent = str(Path(remote_path).parent) + sandbox.fs.mkdir(parent, recursive=True) + sandbox.fs.upload(host_path, remote_path) def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: if not files: @@ -847,56 +2423,67 @@ def _tenki_bulk_upload(self, files: list[tuple[str, str]]) -> None: # the untar, and the cleanup rm must all target one sandbox, or a # concurrent cancel() can leave the tar extracted into a different # sandbox than the one it was uploaded to. - sandbox = self._require_sandbox() - parents = unique_parent_dirs(files) - if parents: - self._exec_raw_on_sandbox(sandbox, quoted_mkdir_command(parents), timeout=30) - - remote_tar = self._remote_transfer_path(".hermes_tenki_sync") - with tempfile.NamedTemporaryFile(suffix=".tar") as tmp: - with tarfile.open(fileobj=tmp, mode="w") as tar: - for host_path, remote_path in files: - tar.add(host_path, arcname=remote_path.lstrip("/")) - tmp.flush() - sandbox.fs.upload(tmp.name, remote_tar) + with self._sandbox_operation() as sandbox: + parents = unique_parent_dirs(files) + if parents: + self._exec_raw_on_sandbox( + sandbox, + quoted_mkdir_command(parents), + timeout=30, + ) + + remote_tar = self._remote_transfer_path(".hermes_tenki_sync") + with tempfile.NamedTemporaryFile(suffix=".tar") as tmp: + with tarfile.open(fileobj=tmp, mode="w") as tar: + for host_path, remote_path in files: + tar.add(host_path, arcname=remote_path.lstrip("/")) + tmp.flush() + sandbox.fs.upload(tmp.name, remote_tar) - try: - output, exit_code = self._exec_raw_on_sandbox( - sandbox, - f"tar xf {shlex.quote(remote_tar)} -C /", - timeout=120, - ) - if exit_code != 0: - raise RuntimeError(f"Tenki bulk upload failed (exit {exit_code}): {output}") - finally: try: - self._exec_raw_on_sandbox(sandbox, f"rm -f {shlex.quote(remote_tar)}", timeout=10) - except Exception: - pass + output, exit_code = self._exec_raw_on_sandbox( + sandbox, + f"tar xf {shlex.quote(remote_tar)} -C /", + timeout=120, + ) + if exit_code != 0: + raise RuntimeError( + f"Tenki bulk upload failed (exit {exit_code}): {output}" + ) + finally: + try: + self._exec_raw_on_sandbox( + sandbox, + f"rm -f {shlex.quote(remote_tar)}", + timeout=10, + ) + except Exception: + pass def _tenki_bulk_download(self, dest: Path) -> None: - sandbox = self._transfer_sandbox() - remote_tar = self._remote_transfer_path(".hermes_tenki_sync_back") - rel_base = f"{self._remote_home}/.hermes".lstrip("/") - try: - output, exit_code = self._exec_raw_on_sandbox( - sandbox, - f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(rel_base)}", - timeout=120, - ) - if exit_code != 0: - raise RuntimeError(f"Tenki bulk download failed (exit {exit_code}): {output}") - sandbox.fs.download(remote_tar, str(dest)) - finally: + with self._transfer_operation() as sandbox: + remote_tar = self._remote_transfer_path(".hermes_tenki_sync_back") + rel_base = f"{self._remote_home}/.hermes".lstrip("/") try: - self._exec_raw_on_sandbox(sandbox, f"rm -f {shlex.quote(remote_tar)}", timeout=10) - except Exception: - pass - - def _transfer_sandbox(self): - if self._cleanup_in_progress and self._cleanup_sandbox is not None: - return self._cleanup_sandbox - return self._require_sandbox() + output, exit_code = self._exec_raw_on_sandbox( + sandbox, + f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(rel_base)}", + timeout=120, + ) + if exit_code != 0: + raise RuntimeError( + f"Tenki bulk download failed (exit {exit_code}): {output}" + ) + sandbox.fs.download(remote_tar, str(dest)) + finally: + try: + self._exec_raw_on_sandbox( + sandbox, + f"rm -f {shlex.quote(remote_tar)}", + timeout=10, + ) + except Exception: + pass def _tenki_delete(self, remote_paths: list[str]) -> None: if not remote_paths: @@ -904,7 +2491,13 @@ def _tenki_delete(self, remote_paths: list[str]) -> None: self._exec_raw(quoted_rm_command(remote_paths), timeout=30) def _exec_raw(self, command: str, *, login: bool = False, timeout: int = 120) -> tuple[str, int]: - return self._exec_raw_on_sandbox(self._require_sandbox(), command, login=login, timeout=timeout) + with self._sandbox_operation() as sandbox: + return self._exec_raw_on_sandbox( + sandbox, + command, + login=login, + timeout=timeout, + ) def _exec_raw_on_sandbox( self, @@ -941,33 +2534,36 @@ def _start_process( stdin_data: str | None, process_ref: dict[str, Any] | None = None, ) -> tuple[str, int]: - sandbox = self._require_sandbox() - flag = "-lc" if login else "-c" - start = getattr(sandbox, "start", None) - if not callable(start): - kwargs: dict[str, Any] = {"timeout": timeout, "env": self._sandbox_env()} - if stdin_data is not None: - kwargs["input"] = stdin_data - result = sandbox.exec("bash", flag, cmd_string, **kwargs) + with self._sandbox_operation() as sandbox: + flag = "-lc" if login else "-c" + start = getattr(sandbox, "start", None) + if not callable(start): + kwargs: dict[str, Any] = { + "timeout": timeout, + "env": self._sandbox_env(), + } + if stdin_data is not None: + kwargs["input"] = stdin_data + result = sandbox.exec("bash", flag, cmd_string, **kwargs) + return self._result_to_output(result) + + process = start( + "bash", + flag, + cmd_string, + timeout=timeout, + stdin=stdin_data, + env=self._sandbox_env(), + ) + if process_ref is not None: + process_ref["process"] = process + if stdin_data is None: + close_stdin = getattr(process, "close_stdin", None) + if callable(close_stdin): + close_stdin() + result = process.wait(timeout=timeout + 5 if timeout is not None else None) return self._result_to_output(result) - process = start( - "bash", - flag, - cmd_string, - timeout=timeout, - stdin=stdin_data, - env=self._sandbox_env(), - ) - if process_ref is not None: - process_ref["process"] = process - if stdin_data is None: - close_stdin = getattr(process, "close_stdin", None) - if callable(close_stdin): - close_stdin() - result = process.wait(timeout=timeout + 5 if timeout is not None else None) - return self._result_to_output(result) - def _sudo_nopasswd_works(self) -> bool: try: _output, exit_code = self._exec_raw("sudo -n true", timeout=10) @@ -1013,34 +2609,81 @@ def cancel() -> None: return except Exception: pass - with self._lock: + with self._lifecycle_condition: + # cleanup owns the sandbox once it starts. Let that one owner + # finish instead of racing it with a second pause/terminate. + if self._cleanup_in_progress or self._cleanup_complete: + return sandbox = self._sandbox - # Drop our reference so the next command resumes (persistent) or - # recreates (ephemeral) a sandbox instead of reusing a torn-down - # one. - self._sandbox = None + # Persistent cancellation must retain this exact reference: + # discovery can fail transiently, and recreating from an older + # snapshot would fork/strand unsnapshotted state. Ephemeral + # sandboxes are deliberately detached before termination. + if not self._persistent: + self._sandbox = None + if sandbox is not None and self._create_attempt_id is not None: + self._create_outcome_uncertain = True + if sandbox is not None: + self._cancel_in_progress += 1 if sandbox is None: return - # For a persistent sandbox, pause (preserve the filesystem) instead of - # terminating: an interrupted or timed-out command must not destroy - # state the user asked to keep. The paused sandbox is re-discovered and - # resumed on the next command via _resume_persistent_sandbox(). - if self._persistent: - pause = getattr(sandbox, "pause", None) - if callable(pause): - try: - pause() - return - except Exception: - pass # fall through to terminate if pause is unavailable - for method_name in ("terminate", "close"): - method = getattr(sandbox, method_name, None) - if callable(method): - try: - method() - except Exception: - pass + try: + # For a persistent sandbox, pause (preserve the filesystem) + # instead of terminating: an interrupted or timed-out command + # must not destroy state the user asked to keep. The paused + # sandbox is re-discovered and resumed on the next command. + if self._persistent: + pause = getattr(sandbox, "pause", None) + if callable(pause): + try: + pause() + return + except Exception as exc: + logger.warning( + "Tenki: cancel could not pause persistent sandbox " + "for task %s; leaving it live to preserve " + "unsnapshotted state: %s", + self._task_id, + exc, + ) + else: + logger.warning( + "Tenki: cancel found no pause support for persistent " + "task %s; leaving sandbox live to preserve " + "unsnapshotted state", + self._task_id, + ) return + for method_name in ("terminate", "close"): + method = getattr(sandbox, method_name, None) + if callable(method): + try: + method() + except Exception as exc: + logger.warning( + "Tenki: cancel could not dispose ephemeral " + "sandbox for task %s: %s", + self._task_id, + exc, + ) + continue + if not self._clear_remote_binding_marker(sandbox): + logger.error( + "Tenki: cancel terminated task %s but could not " + "clear its durable remote binding", + self._task_id, + ) + return + finally: + with self._lifecycle_condition: + if self._persistent and not self._cleanup_complete: + # Reassert exact ownership before releasing ensure() or + # cleanup() waiters, including pause failure/missing + # paths. + self._sandbox = sandbox + self._cancel_generation += 1 + self._cancel_in_progress -= 1 + self._lifecycle_condition.notify_all() def exec_fn() -> tuple[str, int]: return self._start_process( @@ -1053,36 +2696,692 @@ def exec_fn() -> tuple[str, int]: return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel) - def cleanup(self): - with self._lock: + @staticmethod + def _sandbox_identity(sandbox: Any) -> str | None: + if sandbox is None: + return None + identity = getattr(sandbox, "id", None) or getattr( + sandbox, + "sandbox_id", + None, + ) + return str(identity) if identity else None + + @staticmethod + def _remote_identity(env: Any) -> str | None: + return TenkiEnvironment._sandbox_identity( + getattr(env, "_sandbox", None) + ) + + def _bind_remote_sandbox( + self, + sandbox: Any, + *, + attempt_id: str | None, + validated: bool | None = None, + ) -> bool: + remote_id = self._sandbox_identity(sandbox) + if remote_id is None: + return False + if validated is None: + validated = False + try: + _store_remote_binding( + self._task_id, + remote_id, + attempt_id, + validated=validated, + store_path=self._snapshot_store, + ) + except BaseException as exc: + logger.error( + "Tenki: could not durably bind remote %s for task %s: %s", + remote_id, + self._task_id, + exc, + ) + return False + self._remote_binding_id = remote_id + self._remote_binding_attempt_id = attempt_id + self._remote_binding_validated = validated + self._remote_binding_conflicted = False + self._remote_binding_conflict_ids = () + self._remote_binding_unresolvable = False + if attempt_id is not None: + self._create_attempt_id = None + self._create_attempt_expires_at = None + return True + + def _clear_remote_binding_marker(self, sandbox: Any) -> bool: + remote_id = self._remote_binding_id or self._sandbox_identity(sandbox) + if remote_id is None: + return True + try: + _clear_remote_binding( + self._task_id, + remote_id, + self._snapshot_store, + ) + except BaseException as exc: + logger.error( + "Tenki: could not durably clear remote binding %s for task " + "%s: %s", + remote_id, + self._task_id, + exc, + ) + return False + self._remote_binding_id = None + self._remote_binding_attempt_id = None + self._remote_binding_validated = False + self._remote_binding_conflicted = False + self._remote_binding_conflict_ids = () + self._remote_binding_unresolvable = False + return True + + @staticmethod + def _remote_definitively_absent(exc: BaseException) -> bool: + try: + from tenki import SessionNotFoundError + except (ImportError, AttributeError): + pass + else: + if isinstance(exc, SessionNotFoundError): + return True + return any( + typ.__name__ == "SessionNotFoundError" + for typ in type(exc).__mro__ + ) + + def _resolve_remote_binding(self) -> None: + """Resolve a durable task binding through authoritative Client.get.""" + remote_id = self._remote_binding_id + if ( + (remote_id is None and not self._remote_binding_conflicted) + or self._sandbox is not None + ): + return + if self._remote_binding_conflicted: + if ( + not self._remote_binding_conflict_ids + and not self._remote_binding_unresolvable + ): + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki task has a malformed durable persistent-lineage " + f"conflict for {self._task_id}" + ) + client = self._create_client() + active_ids: list[str] = [] + for conflict_id in self._remote_binding_conflict_ids: + try: + candidate = client.get(conflict_id) + except BaseException as exc: + if self._remote_definitively_absent(exc): + continue + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki could not resolve every remote in the durable " + f"lineage conflict for task {self._task_id}" + ) from exc + if self._sandbox_identity(candidate) != conflict_id: + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki conflict lookup returned a different sandbox id" + ) + if self._sandbox_state(candidate) not in self._terminal_states: + active_ids.append(conflict_id) + if active_ids: + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki task has a durable persistent-lineage conflict; " + f"active remote ids for {self._task_id}: " + + ", ".join(active_ids) + ) + if self._remote_binding_unresolvable: + self._create_lineage_ambiguous = True + known_ids = ", ".join( + self._remote_binding_conflict_ids + ) or "" + raise RuntimeError( + "Tenki task has an unresolvable durable " + f"persistent-lineage conflict for {self._task_id}; " + f"known remote ids: {known_ids}; manual remote cleanup " + "and local conflict removal are required" + ) + if not self._clear_remote_binding_marker(None): + self._create_lineage_ambiguous = True + raise RuntimeError( + "Tenki could not clear a fully terminated lineage conflict " + f"for task {self._task_id}" + ) + return + client = self._create_client() + get_sandbox = getattr(client, "get", None) + if not callable(get_sandbox): + raise RuntimeError( + "Tenki SDK cannot resolve a durable sandbox binding by id" + ) + try: + sandbox = get_sandbox(remote_id) + except BaseException as exc: + if self._remote_definitively_absent(exc): + if not self._clear_remote_binding_marker(None): + raise RuntimeError( + "Tenki remote is absent but its durable binding could " + f"not be cleared for task {self._task_id}" + ) from exc + return + raise RuntimeError( + f"Tenki could not resolve bound remote {remote_id}" + ) from exc + + if self._sandbox_identity(sandbox) != remote_id: + raise RuntimeError( + "Tenki by-id lookup returned a different sandbox identity" + ) + if self._sandbox_state(sandbox) in self._terminal_states: + if not self._clear_remote_binding_marker(sandbox): + raise RuntimeError( + "Tenki terminal remote binding could not be cleared for " + f"task {self._task_id}" + ) + return + if self._persistent: + self._sandbox = sandbox + if not self._remote_binding_validated: + self._validate_created_lineage() + return + + last_exc: BaseException | None = None + for method_name in ("terminate", "close"): + method = getattr(sandbox, method_name, None) + if not callable(method): + continue + try: + method() + except BaseException as exc: + last_exc = exc + continue + if self._clear_remote_binding_marker(sandbox): + return + last_exc = RuntimeError( + "remote terminated but its binding could not be cleared" + ) + raise RuntimeError( + f"Tenki could not dispose bound ephemeral remote {remote_id}: " + f"{last_exc or 'no supported cleanup method'}" + ) + + def shares_remote_resource_with(self, other: Any) -> bool: + """Whether two wrappers point at the same Tenki sandbox.""" + mine = self._remote_identity(self) + theirs = self._remote_identity(other) + return bool(mine and theirs and mine == theirs) + + @staticmethod + def _sandbox_create_attempt(sandbox: Any) -> str | None: + info = getattr(sandbox, "info", None) + metadata = getattr(info, "metadata", {}) if info is not None else {} + if not isinstance(metadata, dict): + return None + attempt_id = metadata.get("hermes_create_attempt") + return ( + str(attempt_id) + if isinstance(attempt_id, str) and attempt_id + else None + ) + + def _clear_create_attempt_marker(self) -> bool: + attempt_id = self._create_attempt_id + if attempt_id is None: + self._create_outcome_uncertain = False + return True + try: + _clear_create_attempt( + self._task_id, + attempt_id, + self._snapshot_store, + ) + except BaseException as exc: + logger.error( + "Tenki: could not durably clear create attempt %s for task " + "%s: %s", + attempt_id, + self._task_id, + exc, + ) + return False + self._create_attempt_id = None + self._create_attempt_expires_at = None + self._create_outcome_uncertain = False + return True + + def _persist_create_attempt_conflict( + self, + remote_ids: list[str], + *, + unresolvable: bool, + ) -> bool: + """Durably retain an observed fork before releasing attempt state.""" + attempt_id = self._create_attempt_id + if attempt_id is None: + return False + try: + _replace_create_attempt_with_lineage_conflict( + self._task_id, + attempt_id, + remote_ids, + unresolvable=unresolvable, + store_path=self._snapshot_store, + ) + except BaseException as exc: + self._create_lineage_ambiguous = True + logger.error( + "Tenki: could not durably record create-attempt conflict %s " + "for task %s: %s", + attempt_id, + self._task_id, + exc, + ) + return False + ids = tuple(sorted(set(remote_ids))) + self._create_attempt_id = None + self._create_attempt_expires_at = None + self._create_outcome_uncertain = False + self._remote_binding_id = ids[0] if ids else None + self._remote_binding_attempt_id = attempt_id + self._remote_binding_validated = False + self._remote_binding_conflicted = True + self._remote_binding_conflict_ids = ids + self._remote_binding_unresolvable = unresolvable + self._create_lineage_ambiguous = True + return True + + def _reconcile_uncertain_create(self) -> bool: + """Resolve exactly one durable create attempt without touching siblings.""" + attempt_id = self._create_attempt_id + if attempt_id is None: + self._create_outcome_uncertain = False + return True + try: + client = self._create_client() + except BaseException as exc: + logger.error( + "Tenki: could not create a client to reconcile attempt %s for " + "task %s: %s", + attempt_id, + self._task_id, + exc, + ) + return False + list_sandboxes = getattr(client, "list", None) + if not callable(list_sandboxes): + return False + try: + candidates = list_sandboxes(**self._list_kwargs(client)) + except BaseException as exc: + logger.error( + "Tenki: could not reconcile uncertain create for task %s: %s", + self._task_id, + exc, + ) + return False + + exact_candidates: dict[str, Any] = {} + exact_without_id = False + other_active: dict[str, Any] = {} + other_without_id = False + try: + for candidate in candidates: + if not self._sandbox_matches_task(candidate): + continue + state = self._sandbox_state(candidate) + exact = self._sandbox_create_attempt(candidate) == attempt_id + candidate_id = self._sandbox_identity(candidate) + if exact: + if candidate_id is None: + exact_without_id = True + else: + exact_candidates[candidate_id] = candidate + continue + if state in self._terminal_states: + continue + if candidate_id is None: + other_without_id = True + else: + other_active[candidate_id] = candidate + except BaseException as exc: + logger.error( + "Tenki: could not inspect uncertain create candidates for task " + "%s: %s", + self._task_id, + exc, + ) + return False + + expired = ( + self._create_attempt_expires_at is not None + and time.time() >= self._create_attempt_expires_at + ) + + if ( + exact_without_id + or other_without_id + or other_active + or len(exact_candidates) > 1 + ): + # Persist the conflict before returning. If the exact create is not + # visible and has not expired, its unknown identity must also be + # represented, which makes the conflict permanently unresolvable + # rather than allowing a later list omission to promote a branch. + unresolvable = ( + exact_without_id + or other_without_id + or (not exact_candidates and not expired) + ) + conflict_ids = [ + *exact_candidates, + *other_active, + ] + self._persist_create_attempt_conflict( + conflict_ids, + unresolvable=unresolvable, + ) + logger.error( + "Tenki: create attempt %s for task %s conflicts with %s other " + "active lineage(s) and %s duplicate exact attempt(s)", + attempt_id, + self._task_id, + len(other_active), + max(0, len(exact_candidates) - 1), + ) + return False + + if not exact_candidates: + if expired and not other_active: + # Even if the RPC committed immediately before a crash, the + # configured server-side max duration plus a one-hour grace has + # elapsed. No remote from this attempt can remain live. + return self._clear_create_attempt_marker() + # A timeout can race an eventually consistent list. An empty result + # is therefore not proof of absence; fail closed by quarantining the + # task lock instead of allowing a duplicate create. + logger.error( + "Tenki: create attempt %s for task %s returned no exact " + "reconcilable sandbox; absence is unproven", + attempt_id, + self._task_id, + ) + return False + + remote_id = next(iter(exact_candidates)) + get_sandbox = getattr(client, "get", None) + if not callable(get_sandbox): + return False + try: + sandbox = get_sandbox(remote_id) + except BaseException as exc: + if self._remote_definitively_absent(exc): + return self._clear_create_attempt_marker() + logger.error( + "Tenki: could not authoritatively resolve uncertain create " + "%s for task %s: %s", + attempt_id, + self._task_id, + exc, + ) + return False + if ( + self._sandbox_identity(sandbox) != remote_id + or not self._sandbox_has_owned_identity(sandbox, attempt_id) + ): + self._persist_create_attempt_conflict( + [], + unresolvable=True, + ) + logger.error( + "Tenki: exact-id recovery did not preserve canonical ownership " + "for attempt %s and task %s", + attempt_id, + self._task_id, + ) + return False + if self._sandbox_state(sandbox) in self._terminal_states: + return self._clear_create_attempt_marker() + + # A terminal list row cannot authorize clearing: Client.get(id) above + # is authoritative and may reveal that the same remote is still live. + if self._persistent: + # Bind the exact recovered attempt, but keep it unvalidated until + # the normal exact-identity + conflict scan completes. + if not self._bind_remote_sandbox( + sandbox, + attempt_id=attempt_id, + validated=False, + ): + return False + self._sandbox = sandbox + self._create_outcome_uncertain = False + return True + + disposed = False + last_exc: BaseException | None = None + for method_name in ("terminate", "close"): + method = getattr(sandbox, method_name, None) + if not callable(method): + continue + for attempt in range(len(_TERMINATE_RETRY_DELAYS) + 1): + try: + method() + disposed = True + break + except BaseException as exc: + last_exc = exc + if attempt < len(_TERMINATE_RETRY_DELAYS): + time.sleep(_TERMINATE_RETRY_DELAYS[attempt]) + if disposed: + break + if not disposed: + logger.error( + "Tenki: could not dispose exact uncertain create %s for task " + "%s: %s", + attempt_id, + self._task_id, + last_exc or "no supported remote cleanup method", + ) + return False + return self._clear_create_attempt_marker() + + def _abort_failed_initialization(self) -> None: + """Synchronously make a partially initialized wrapper inert. + + The lifetime task lock covers the whole rollback. Persistent sandboxes + are paused so their state remains discoverable; ephemeral sandboxes are + terminated. If the remote cannot be made safe, the kernel lock is + quarantined for the rest of the process so no successor can attach. + """ + sandbox = getattr(self, "_sandbox", None) + client = getattr(self, "_client", None) + create_uncertain = getattr(self, "_create_outcome_uncertain", False) + remote_safe = sandbox is None and not create_uncertain + last_exc: BaseException | None = None + + try: + if self._create_lineage_ambiguous: + last_exc = RuntimeError( + "multiple remote lineages require fail-closed ownership" + ) + elif sandbox is None and create_uncertain: + remote_safe = self._reconcile_uncertain_create() + if not remote_safe: + last_exc = RuntimeError( + "uncertain remote creation could not be reconciled" + ) + elif sandbox is not None: + method_names = ( + ("pause",) + if self._persistent + else ("terminate", "close") + ) + for method_name in method_names: + method = getattr(sandbox, method_name, None) + if not callable(method): + continue + for attempt in range(len(_TERMINATE_RETRY_DELAYS) + 1): + try: + method() + remote_safe = True + break + except BaseException as exc: + last_exc = exc + if attempt < len(_TERMINATE_RETRY_DELAYS): + time.sleep(_TERMINATE_RETRY_DELAYS[attempt]) + if remote_safe: + break + except BaseException as exc: + # Rollback itself must never obscure the constructor failure or + # release ownership in an unknown remote state. + last_exc = exc + + if remote_safe and self._create_lineage_ambiguous: + remote_safe = False + last_exc = RuntimeError( + "multiple remote lineages require fail-closed ownership" + ) + if ( + remote_safe + and sandbox is not None + and not self._persistent + and not self._clear_remote_binding_marker(sandbox) + ): + remote_safe = False + last_exc = RuntimeError( + "terminated remote binding could not be cleared" + ) + + with self._lifecycle_condition: + self._sandbox = None + self._client = None + self._sync_manager = None + self._cleanup_sandbox = None + self._cleanup_in_progress = False + self._cleanup_complete = True + self._lifecycle_condition.notify_all() + try: + self._close_client(client) + except BaseException as exc: + logger.warning( + "Tenki: client close raised during failed initialization: %s", + exc, + ) + + if remote_safe: + self._release_task_ownership() + return + + lock_file = getattr(self, "_task_ownership_file", None) + self._task_ownership_file = None + _quarantine_task_ownership_lock(lock_file) + logger.error( + "Tenki: failed initialization could not safely quiesce task %s; " + "retaining exclusive task ownership until process exit: %s", + self._task_id, + last_exc or "no supported remote cleanup method", + ) + + def _release_task_ownership(self) -> None: + lock_file = getattr(self, "_task_ownership_file", None) + self._task_ownership_file = None + _release_task_ownership_lock(lock_file) + + def release_duplicate_wrapper(self, winner: Any) -> None: + """Close only this wrapper/client when *winner* owns the same remote.""" + with self._lifecycle_condition: + while ( + self._cleanup_in_progress + or self._cancel_in_progress + or self._active_operations + ): + self._lifecycle_condition.wait() + if self._cleanup_complete: + return + client = self._client + if client is getattr(winner, "_client", None): + client = None + self._sandbox = None + self._client = None + self._sync_manager = None + self._cleanup_complete = True + self._cleanup_in_progress = False + self._cleanup_sandbox = None + self._lifecycle_condition.notify_all() + self._close_client(client) + self._release_task_ownership() + + def discard(self) -> None: + """Dispose an unregistered duplicate without snapshot side effects.""" + self.cleanup(discard=True) + + def cleanup(self, *, discard: bool = False): + with self._lifecycle_condition: + while self._cleanup_in_progress: + self._lifecycle_condition.wait() + if self._cleanup_complete: + return + + self._cleanup_in_progress = True + # cancel() may already own a detached sandbox while it performs a + # pause/terminate RPC outside the lock. Do not close the shared + # control-plane client until that RPC has completed. + while self._cancel_in_progress: + self._lifecycle_condition.wait() + while self._active_operations: + self._lifecycle_condition.wait() + sandbox = self._sandbox sync_manager = self._sync_manager self._sync_manager = None client = self._client - self._cleanup_in_progress = True self._cleanup_sandbox = sandbox - if sandbox is None: - self._close_client(client) - with self._lock: - if self._client is client: - self._client = None - self._cleanup_in_progress = False - self._cleanup_sandbox = None - return + cleanup_succeeded = False try: - if sync_manager: + if self._create_lineage_ambiguous: + raise RuntimeError( + "Tenki cleanup is blocked because multiple remote lineages " + f"exist for task {self._task_id}" + ) + if sandbox is None: + if self._remote_binding_id is not None: + self._resolve_remote_binding() + sandbox = self._sandbox + client = self._client + with self._lifecycle_condition: + self._cleanup_sandbox = sandbox + if self._create_attempt_id is not None: + self._create_outcome_uncertain = True + if not self._reconcile_uncertain_create(): + raise RuntimeError( + "Tenki cleanup cannot release task ownership while " + f"a create is unresolved for task {self._task_id}" + ) + if sandbox is None: + cleanup_succeeded = True + return + + if sync_manager and not discard: logger.info("Tenki: syncing files from sandbox...") try: - sync_manager.sync_back() + sync_manager.sync_back(self._profile_home) except Exception as exc: logger.warning("Tenki: sync_back failed: %s", exc) snapshot_saved = False - if self._persistent: + if self._persistent and not discard: snapshot_saved = self._save_persistent_snapshot(sandbox) - if self._persistent and not snapshot_saved: + if self._persistent and not discard and not snapshot_saved: # Persistent state was NOT durably snapshotted. Terminating now # would destroy the only copy, so prefer pause; and if pause # fails, still do NOT terminate — leave the sandbox live for a @@ -1106,29 +3405,75 @@ def cleanup(self): "leaving sandbox live to preserve state (not terminating)", self._task_id, ) + cleanup_succeeded = True return for method_name in ("terminate", "close"): method = getattr(sandbox, method_name, None) if not callable(method): continue - try: - method() - logger.info("Tenki: terminated sandbox for task %s", self._task_id) - except Exception as exc: - logger.warning("Tenki: cleanup failed: %s", exc) - return + last_exc: Exception | None = None + attempts = len(_TERMINATE_RETRY_DELAYS) + 1 + for attempt in range(attempts): + try: + method() + if not self._clear_remote_binding_marker(sandbox): + raise RuntimeError( + "remote terminated but its durable create " + "binding could not be cleared" + ) + logger.info( + "Tenki: terminated sandbox for task %s", + self._task_id, + ) + cleanup_succeeded = True + return + except Exception as exc: + last_exc = exc + if attempt < len(_TERMINATE_RETRY_DELAYS): + time.sleep(_TERMINATE_RETRY_DELAYS[attempt]) + raise RuntimeError( + f"Tenki cleanup failed after {attempts} attempts " + f"for task {self._task_id}: {last_exc}" + ) from last_exc + raise RuntimeError( + f"Tenki sandbox for task {self._task_id} has no cleanup method" + ) finally: - self._close_client(client) - with self._lock: - if self._sandbox is sandbox: - self._sandbox = None - if self._client is client: - self._client = None + if cleanup_succeeded: + self._close_client(client) + with self._lifecycle_condition: + if cleanup_succeeded: + if self._sandbox is sandbox: + self._sandbox = None + if self._client is client: + self._client = None + self._cleanup_complete = True self._cleanup_in_progress = False self._cleanup_sandbox = None + self._lifecycle_condition.notify_all() + if cleanup_succeeded: + self._release_task_ownership() def _save_persistent_snapshot(self, sandbox: Any) -> bool: + # A legacy source pointer is the predecessor of any new canonical + # snapshot. If its migration still cannot commit, writing a new + # canonical pointer would forget that predecessor and leak its remote + # snapshot. Retry here (cleanup may be the first post-construction + # lifecycle), then fail closed and pause if it remains unresolved. + self._migrate_loaded_snapshot_pointer() + if self._snapshot_restore_id and ( + self._snapshot_restore_from_legacy_key + or self._snapshot_restore_task_id != self._task_id + ): + logger.warning( + "Tenki: snapshot pointer migration is still pending for task " + "%s; preserving the live sandbox instead of replacing an " + "untracked legacy predecessor", + self._task_id, + ) + return False + snapshot_id: str | None = None try: snapshot = sandbox.snapshot(name=self._sandbox_name(), wait=True) @@ -1145,23 +3490,98 @@ def _save_persistent_snapshot(self, sandbox: Any) -> bool: # pointer or let the caller terminate the live sandbox. Return False so # cleanup pauses the sandbox and the prior (known-durable) snapshot # pointer is left intact for recovery. - if self._client is not None: - snapshots = getattr(self._client, "snapshots", None) - wait_durable = getattr(snapshots, "wait_durable", None) - if callable(wait_durable): - try: - wait_durable(snapshot_id, timeout=300) - except Exception as exc: - logger.warning( - "Tenki: snapshot %s for task %s did not reach durability (%s); " - "preserving paused sandbox and prior snapshot instead", - snapshot_id, self._task_id, exc, - ) - return False - _store_snapshot(self._task_id, snapshot_id, self._snapshot_store) + snapshots = getattr(self._client, "snapshots", None) + wait_durable = getattr(snapshots, "wait_durable", None) + if not callable(wait_durable): + logger.warning( + "Tenki: cannot confirm durability for snapshot %s for task %s; " + "preserving paused sandbox and prior snapshot instead", + snapshot_id, + self._task_id, + ) + self._delete_remote_snapshot( + snapshot_id, + reason="unverifiable replacement", + ) + return False + try: + wait_durable(snapshot_id, timeout=300) + except Exception as exc: + logger.warning( + "Tenki: snapshot %s for task %s did not reach durability (%s); " + "preserving paused sandbox and prior snapshot instead", + snapshot_id, self._task_id, exc, + ) + self._delete_remote_snapshot( + snapshot_id, + reason="non-durable replacement", + ) + return False + try: + previous_snapshot_id = _store_snapshot( + self._task_id, + snapshot_id, + self._snapshot_store, + ) + except _SnapshotPointerCommitUncertain as exc: + # The rename is visible, but a host crash may reveal either the old + # or new pointer. Keep both remote snapshots and the live sandbox; + # deleting either recovery copy would make one crash outcome lossy. + logger.warning( + "Tenki: durable snapshot pointer commit for %s is uncertain " + "(%s; previous=%s, new=%s); retaining both snapshots and " + "preserving the live sandbox", + self._task_id, + exc, + exc.previous_snapshot_id or "", + exc.new_snapshot_id or snapshot_id, + ) + return False + except Exception as exc: + logger.warning( + "Tenki: could not persist durable snapshot pointer %s for task %s " + "(%s); preserving paused sandbox and prior snapshot instead", + snapshot_id, + self._task_id, + exc, + ) + self._delete_remote_snapshot( + snapshot_id, + reason="unrecorded replacement", + ) + return False + logger.info("Tenki: saved filesystem snapshot %s for task %s", snapshot_id, self._task_id) + if previous_snapshot_id and previous_snapshot_id != snapshot_id: + self._retire_pending_snapshot_if_unreferenced( + previous_snapshot_id, + reason=f"superseded by {snapshot_id}", + ) return True + def _delete_remote_snapshot( + self, + snapshot_id: str, + *, + reason: str, + ) -> bool: + """Safely retire an unreferenced remote snapshot through the journal.""" + try: + _queue_snapshot_retirement(snapshot_id, self._snapshot_store) + except Exception as exc: + logger.warning( + "Tenki: could not durably journal snapshot %s for retirement " + "(%s); leaving the remote intact: %s", + snapshot_id, + reason, + exc, + ) + return False + return self._retire_pending_snapshot_if_unreferenced( + snapshot_id, + reason=reason, + ) + @staticmethod def _close_client(client: Any) -> None: # Best-effort: a failed close must never propagate — cleanup() resets diff --git a/tools/file_tools.py b/tools/file_tools.py index 2ca9a0c287f8f..d855c6348c748 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -164,25 +164,36 @@ def _resolve_path(filepath: str, task_id: str = "default") -> Path | PurePosixPa # (gateway/run.py); the file/terminal-tool layer must do likewise so CLI # sessions get the same protection. See references/worktree-cwd-discipline.md. _TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"}) -_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona"}) +_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset( + {"docker", "singularity", "modal", "daytona", "tenki"} +) def _terminal_env_type_for_task(task_id: str = "default") -> str: """Best-effort terminal backend type for path-resolution decisions.""" try: from tools.terminal_tool import ( - _active_environments, - _env_lock, _get_env_config, - _resolve_container_task_id, + _resolve_environment_cache_key, + _select_active_environment, ) + cfg = _get_env_config() try: - container_key = _resolve_container_task_id(task_id) + container_key = _resolve_environment_cache_key( + task_id, + str(cfg.get("env_type") or "local"), + ) except Exception: container_key = task_id - with _env_lock: - env = _active_environments.get(container_key) or _active_environments.get(task_id) + fallback = ( + task_id if not container_key.startswith("tenki:") else None + ) + _selected_key, env = _select_active_environment( + container_key, + fallback, + touch=False, + ) if env is not None: name = env.__class__.__name__.lower() if "local" in name: @@ -197,7 +208,8 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str: return "modal" if "daytona" in name: return "daytona" - cfg = _get_env_config() + if "tenki" in name: + return "tenki" return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower() except Exception: return str(os.getenv("TERMINAL_ENV") or "local").lower() @@ -627,19 +639,24 @@ def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | Non """Return the container-side Hermes mirror prefix for Docker file tools.""" try: from tools.terminal_tool import ( - _active_environments, - _env_lock, _get_env_config, - _resolve_container_task_id, + _resolve_environment_cache_key, + _select_active_environment, ) - container_key = _resolve_container_task_id(task_id) + config = _get_env_config() + env_type = str(config.get("env_type") or "local") + container_key = _resolve_environment_cache_key(task_id, env_type) except Exception: return None try: - with _env_lock: - env = _active_environments.get(container_key) or _active_environments.get(task_id) + fallback = task_id if env_type != "tenki" else None + _selected_key, env = _select_active_environment( + container_key, + fallback, + touch=False, + ) if env is not None: if env.__class__.__name__ == "DockerEnvironment" and bool( @@ -648,7 +665,6 @@ def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | Non return "/root/.hermes" return None - config = _get_env_config() except Exception: return None @@ -941,67 +957,52 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: a registered env override keep their isolation. """ from tools.terminal_tool import ( - _active_environments, _env_lock, _create_environment, - _get_env_config, _last_activity, _start_cleanup_thread, - _creation_locks, - _creation_locks_lock, - _resolve_container_task_id, - _is_unusable_container_cwd, - _is_backend_guest_subpath, + _create_environment, + _get_env_config, _start_cleanup_thread, + _commit_active_environment_use, + _environment_creation_lock, + _register_active_environment, + _resolve_environment_cwd, + _resolve_environment_cache_key, + _select_active_environment, _CONTAINER_BACKENDS, _container_config_from_env_config, ) - import time raw_task_id = task_id or "default" - task_id = _resolve_container_task_id(raw_task_id) + config = _get_env_config() + env_type = config["env_type"] + task_id = _resolve_environment_cache_key(raw_task_id, env_type) + _selected_key, terminal_env = _select_active_environment(task_id) # Fast path: check cache -- but also verify the underlying environment # is still alive (it may have been killed by the cleanup thread). with _file_ops_lock: cached = _file_ops_cache.get(task_id) if cached is not None: - with _env_lock: - if task_id in _active_environments: - _last_activity[task_id] = time.time() - return cached - else: - # Environment was cleaned up -- preserve the old cwd in the - # session record before invalidating the stale cache entry - # (fixes #26211: silent file-creation failures in long-running - # conversations). Usually a no-op: every completed command - # already recorded its cwd. - old_cwd = getattr(cached, "cwd", None) - if old_cwd: - try: - from tools.terminal_tool import record_session_cwd - record_session_cwd(raw_task_id, old_cwd) - except Exception: - pass - with _file_ops_lock: - _file_ops_cache.pop(task_id, None) + if terminal_env is not None and getattr(cached, "env", None) is terminal_env: + return cached + # Environment was cleaned up or replaced -- preserve the old cwd in + # the session record before invalidating the stale cache entry. + old_cwd = getattr(cached, "cwd", None) + if old_cwd: + try: + from tools.terminal_tool import record_session_cwd + record_session_cwd(raw_task_id, old_cwd) + except Exception: + pass + with _file_ops_lock: + _file_ops_cache.pop(task_id, None) # Need to ensure the environment exists before building file_ops. # Acquire per-task lock so only one thread creates the sandbox. - with _creation_locks_lock: - if task_id not in _creation_locks: - _creation_locks[task_id] = threading.Lock() - task_lock = _creation_locks[task_id] - - with task_lock: + with _environment_creation_lock(task_id): # Double-check: another thread may have created it while we waited - with _env_lock: - if task_id in _active_environments: - _last_activity[task_id] = time.time() - terminal_env = _active_environments[task_id] - else: - terminal_env = None + _selected_key, terminal_env = _select_active_environment(task_id) if terminal_env is None: from tools.terminal_tool import resolve_task_overrides - config = _get_env_config() - env_type = config["env_type"] overrides = resolve_task_overrides(raw_task_id) if env_type == "docker": @@ -1017,36 +1018,12 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: else: image = "" - try: - from tools.terminal_tool import get_session_cwd - recorded_cwd = get_session_cwd(raw_task_id) - except Exception: - recorded_cwd = None - cwd = overrides.get("cwd") or recorded_cwd or config["cwd"] - # Re-apply the container cwd guard that _get_env_config() already - # ran on config["cwd"] (see #50636). A per-task cwd override - # registered by the gateway/TUI/ACP for workspace tracking is a - # raw host path (e.g. a Desktop session's /Users//workspace or - # C:\\Users\\). On a container backend that reaches - # ``docker run -w `` and the container starts in a - # directory that doesn't exist inside the sandbox, so search_files - # and friends silently return empty results (#54447). Sanitize it - # back to the already-validated config["cwd"] so the override can't - # bypass the guard. Valid in-container override paths (RL/benchmark - # sandboxes that set cwd to /workspace, /root, etc.) are absolute - # non-host paths and pass through untouched. - if ( - env_type in _CONTAINER_BACKENDS - and _is_unusable_container_cwd(cwd) - and not _is_backend_guest_subpath(env_type, cwd) - ): - if cwd != config["cwd"]: - logger.info( - "Ignoring host/relative cwd override %r for %s backend " - "(won't exist in sandbox). Using %r instead.", - cwd, env_type, config["cwd"], - ) - cwd = config["cwd"] + cwd = _resolve_environment_cwd( + raw_task_id, + env_type, + config, + overrides, + ) logger.info("Creating new %s environment for task %s...", env_type, task_id[:8]) container_config = None @@ -1081,17 +1058,26 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: host_cwd=config.get("host_cwd"), ) - with _env_lock: - _active_environments[task_id] = terminal_env - _last_activity[task_id] = time.time() + terminal_env = _register_active_environment( + task_id, + terminal_env, + ) _start_cleanup_thread() logger.info("%s environment ready for task %s", env_type, task_id[:8]) # Build file_ops from the (guaranteed live) environment and cache it file_ops = ShellFileOperations(terminal_env) - with _file_ops_lock: - _file_ops_cache[task_id] = file_ops + def cache_file_ops() -> None: + with _file_ops_lock: + _file_ops_cache[task_id] = file_ops + + if not _commit_active_environment_use( + task_id, + terminal_env, + cache_file_ops, + ): + return _get_file_ops(raw_task_id) return file_ops diff --git a/tools/registry.py b/tools/registry.py index 354da7123fd7d..d7b3791b741fe 100644 --- a/tools/registry.py +++ b/tools/registry.py @@ -145,12 +145,26 @@ def __init__(self, name, toolset, schema, handler, check_fn, # as a flake (last-good True is served) rather than a real outage. Kept short # so a genuinely-down backend is reflected within a couple of turns. _CHECK_FN_FAILURE_GRACE_SECONDS = 60.0 -_check_fn_cache: Dict[Callable, tuple[float, bool]] = {} +_check_fn_cache: Dict[tuple[Callable, str], tuple[float, bool]] = {} # Monotonic timestamp of the most recent True result per check_fn. -_check_fn_last_good: Dict[Callable, float] = {} +_check_fn_last_good: Dict[tuple[Callable, str], float] = {} _check_fn_cache_lock = threading.Lock() +def _check_fn_context_key() -> str: + """Scope availability verdicts to the active profile when one is bound.""" + try: + from agent.secret_scope import current_secret_scope + + if current_secret_scope() is not None: + from hermes_constants import get_hermes_home + + return str(get_hermes_home()) + except Exception: + pass + return "" + + def _check_fn_cached(fn: Callable) -> bool: """Return bool(fn()), TTL-cached across calls. @@ -161,8 +175,9 @@ def _check_fn_cached(fn: Callable) -> bool: contention, probe timeout) from silently stripping tools mid-session. """ now = time.monotonic() + cache_key = (fn, _check_fn_context_key()) with _check_fn_cache_lock: - cached = _check_fn_cache.get(fn) + cached = _check_fn_cache.get(cache_key) if cached is not None: ts, value = cached if now - ts < _CHECK_FN_TTL_SECONDS: @@ -177,11 +192,11 @@ def _check_fn_cached(fn: Callable) -> bool: with _check_fn_cache_lock: if value: - _check_fn_last_good[fn] = now - _check_fn_cache[fn] = (now, True) + _check_fn_last_good[cache_key] = now + _check_fn_cache[cache_key] = (now, True) return True - last_good = _check_fn_last_good.get(fn) + last_good = _check_fn_last_good.get(cache_key) if last_good is not None and now - last_good < _CHECK_FN_FAILURE_GRACE_SECONDS: # Recent success → treat this failure as a flake. Serve last-good # True and do NOT cache the failure, so the next call re-probes @@ -202,7 +217,7 @@ def _check_fn_cached(fn: Callable) -> bool: getattr(fn, "__qualname__", fn), "raised" if raised else "returned False", ) - _check_fn_cache[fn] = (now, False) + _check_fn_cache[cache_key] = (now, False) return False diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 3d2eb7e39a098..79698a303ac37 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -43,8 +43,9 @@ import atexit import shutil import subprocess +from contextlib import contextmanager from pathlib import Path -from typing import Optional, Dict, Any, List +from typing import Optional, Dict, Any, List, Callable from utils import env_var_enabled @@ -983,7 +984,18 @@ def _transform_sudo_command(command: str | None) -> tuple[str | None, str | None _active_environments: Dict[str, Any] = {} _last_activity: Dict[str, float] = {} _env_lock = threading.Lock() -_creation_locks: Dict[str, threading.Lock] = {} # Per-task locks for sandbox creation +_retiring_environments: Dict[str, threading.Event] = {} +class _EnvironmentCreationSlot: + """One generation-safe per-task creation lock with waiter accounting.""" + + __slots__ = ("lock", "users") + + def __init__(self) -> None: + self.lock = threading.Lock() + self.users = 0 + + +_creation_locks: Dict[str, _EnvironmentCreationSlot] = {} _creation_locks_lock = threading.Lock() # Protects _creation_locks dict itself _cleanup_thread = None _cleanup_running = False @@ -995,6 +1007,213 @@ def _transform_sudo_command(command: str | None) -> tuple[str | None, str | None _docker_orphan_reaper_lock = threading.Lock() +def _select_active_environment( + task_id: str, + fallback_task_id: Optional[str] = None, + *, + touch: bool = True, +) -> tuple[Optional[str], Any]: + """Atomically wait past retirement and select a live registry entry. + + The tombstone check and environment read happen under the same lock. A + caller therefore cannot finish waiting, race with retirement installation, + and then select an entry that was already marked for teardown. + """ + keys = tuple( + dict.fromkeys( + key for key in (task_id, fallback_task_id) if key + ) + ) + while True: + with _env_lock: + retired = next( + ( + _retiring_environments[key] + for key in keys + if key in _retiring_environments + ), + None, + ) + if retired is None: + for key in keys: + env = _active_environments.get(key) + if env is not None: + if touch: + _last_activity[key] = time.time() + return key, env + return None, None + retired.wait() + + +@contextmanager +def _environment_creation_lock(task_id: str): + """Serialize creators without allowing keyed-lock ABA. + + A slot remains in the registry while any holder or waiter retains its + generation. It is removed only after the final user releases it, so + retirement cannot delete a lock out from under an already-waiting creator + and let a newcomer enter through a second lock. + """ + with _creation_locks_lock: + slot = _creation_locks.get(task_id) + if slot is None: + slot = _EnvironmentCreationSlot() + _creation_locks[task_id] = slot + slot.users += 1 + slot.lock.acquire() + try: + yield + finally: + slot.lock.release() + with _creation_locks_lock: + slot.users -= 1 + if slot.users == 0 and _creation_locks.get(task_id) is slot: + _creation_locks.pop(task_id, None) + + +def _discard_unregistered_environment( + env: Any, + existing: Any, + task_id: str, +) -> None: + """Best-effort disposal for a candidate that lost registry publication.""" + shares_remote = getattr(env, "shares_remote_resource_with", None) + release_wrapper = getattr(env, "release_duplicate_wrapper", None) + try: + if ( + callable(shares_remote) + and shares_remote(existing) + and callable(release_wrapper) + ): + # Persistent wrappers can independently reattach to the same + # remote Tenki sandbox. Terminating the losing wrapper's remote + # would also terminate the registered winner; close only the + # losing client/wrapper in that case. + release_wrapper(existing) + return + except Exception: + logger.exception( + "Failed to compare/release duplicate environment wrapper for " + "task %s", + task_id, + ) + return + + discard = getattr(env, "discard", None) + cleanup = getattr(env, "cleanup", None) + action = discard if callable(discard) else cleanup + if not callable(action): + logger.error( + "Environment candidate for task %s lost registration and has no " + "discard/cleanup method", + task_id, + ) + return + try: + action() + except Exception: + logger.exception( + "Failed to dispose unregistered environment candidate for task %s", + task_id, + ) + + +def _register_active_environment(task_id: str, env: Any) -> Any: + """Publish a newly created environment under the registry protocol.""" + with _env_lock: + if task_id in _retiring_environments: + raise RuntimeError( + f"Cannot register environment while {task_id!r} is retiring" + ) + existing = _active_environments.get(task_id) + if existing is None: + _active_environments[task_id] = env + _last_activity[task_id] = time.time() + return env + if existing is env: + return env + # This is unreachable for callers using _environment_creation_lock, but + # do not leak a billable remote resource if a future/third-party path + # violates that protocol. + _discard_unregistered_environment(env, existing, task_id) + return existing + + +def _commit_active_environment_use( + task_id: str, + env: Any, + callback: Callable[[], None], +) -> bool: + """Run a small registry-adjacent commit only if *env* is still selectable.""" + with _env_lock: + if ( + task_id in _retiring_environments + or _active_environments.get(task_id) is not env + ): + return False + _last_activity[task_id] = time.time() + callback() + return True + + +def _begin_environment_retirement( + task_id: str, + env: Any, +) -> tuple[threading.Event, bool]: + """Install a tombstone without removing the live registry entry.""" + with _env_lock: + existing = _retiring_environments.get(task_id) + if existing is not None: + return existing, False + if _active_environments.get(task_id) is not env: + completed = threading.Event() + completed.set() + return completed, False + completed = threading.Event() + _retiring_environments[task_id] = completed + _last_activity.pop(task_id, None) + return completed, True + + +def _finish_environment_retirement( + task_id: str, + env: Any, + completed: threading.Event, +) -> None: + """Remove a successfully cleaned environment, then release waiters.""" + with _env_lock: + if _active_environments.get(task_id) is env: + _active_environments.pop(task_id, None) + _last_activity.pop(task_id, None) + # Keep the tombstone installed until the creation/file caches have + # been invalidated. Otherwise a waiter can observe "not retiring" and + # reuse a ShellFileOperations object that still points at the dead env. + try: + from tools.file_tools import clear_file_ops_cache + + clear_file_ops_cache(task_id) + except ImportError: + pass + with _env_lock: + if _retiring_environments.get(task_id) is completed: + _retiring_environments.pop(task_id, None) + completed.set() + + +def _abort_environment_retirement( + task_id: str, + env: Any, + completed: threading.Event, +) -> None: + """Keep a failed cleanup registered and retryable on a later reap.""" + with _env_lock: + if _active_environments.get(task_id) is env: + _last_activity[task_id] = time.time() + if _retiring_environments.get(task_id) is completed: + _retiring_environments.pop(task_id, None) + completed.set() + + def _maybe_reap_docker_orphans(container_config: Dict[str, Any]) -> None: """Run the docker orphan reaper once per process, if enabled. @@ -1132,7 +1351,7 @@ def register_task_env_overrides(task_id: str, overrides: Dict[str, Any]): Supported override keys: - modal_image: str -- Path to Dockerfile or Docker Hub image name - - tenki_image: str -- Tenki sandbox image/template identifier + - tenki_image: str -- Tenki registry image reference - docker_image: str -- Docker image name - cwd: str -- Working directory inside the sandbox @@ -1157,11 +1376,23 @@ def register_task_env_overrides(task_id: str, overrides: Dict[str, Any]): # isolation-keyed rollouts. Try the raw id first, then the container id, # so a CWD-only override (which collapses to "default") still finds and # updates the originating session's env. - container_id = _resolve_container_task_id(task_id) - with _env_lock: - env = _active_environments.get(task_id) or _active_environments.get(container_id) + config = _get_env_config() + env_type = config["env_type"] + container_id = _resolve_environment_cache_key(task_id, env_type) + fallback_task_id = ( + task_id if not container_id.startswith("tenki:") else None + ) + _selected_key, env = _select_active_environment( + container_id, + fallback_task_id, + ) if env is not None and getattr(env, "cwd", None) is not None: - env.cwd = new_cwd + env.cwd = _resolve_environment_cwd( + task_id, + env_type, + config, + overrides, + ) def clear_task_env_overrides(task_id: str): @@ -1209,6 +1440,27 @@ def _resolve_container_task_id(task_id: Optional[str]) -> str: return "default" +def _resolve_environment_cache_key(task_id: Optional[str], env_type: str) -> str: + """Return the registry key for one backend environment. + + Most backends intentionally share ``default`` across sessions/subagents. + Tenki is the exception: credentials, workspace, synced files, and billing + are profile-owned, so multiplexed profiles must never share one cached + sandbox. Subagents within the same profile still collapse to the same base + key and therefore keep the shared-sandbox contract. + """ + raw = str(task_id or "default") + base = _resolve_container_task_id(task_id) + if str(env_type or "").strip().lower() != "tenki": + return base + from tools.environments.tenki import _profile_token + + prefix = f"tenki:{_profile_token()}:" + if raw.startswith(prefix): + return raw + return f"{prefix}{base}" + + def resolve_task_overrides(task_id: Optional[str]) -> Dict[str, Any]: """Return the env overrides for *task_id*, raw key first then collapsed. @@ -1231,13 +1483,20 @@ def resolve_task_overrides(task_id: Optional[str]) -> Dict[str, Any]: # Configuration from environment variables -def _parse_env_var(name: str, default: str, converter: Any = int, type_label: str = "integer"): +def _parse_env_var( + name: str, + default: str, + converter: Any = int, + type_label: str = "integer", + *, + env: Optional[Dict[str, str]] = None, +): """Parse an environment variable with *converter*, raising a clear error on bad values. Without this wrapper, a single malformed env var (e.g. TERMINAL_TIMEOUT=5m) causes an unhandled ValueError that kills every terminal command. """ - raw = os.getenv(name, default) + raw = (os.environ if env is None else env).get(name, default) try: return converter(raw) except (ValueError, json.JSONDecodeError): @@ -1320,6 +1579,32 @@ def _is_unusable_container_cwd(cwd: str) -> bool: return False +def _resolve_environment_cwd( + task_id: Optional[str], + env_type: str, + config: Dict[str, Any], + overrides: Optional[Dict[str, Any]] = None, +) -> str: + """Resolve one session cwd and sanitize host paths for guest backends.""" + overrides = overrides if overrides is not None else resolve_task_overrides(task_id) + cwd = overrides.get("cwd") or get_session_cwd(task_id) or config["cwd"] + if ( + env_type in _CONTAINER_BACKENDS + and _is_unusable_container_cwd(cwd) + and not _is_backend_guest_subpath(env_type, cwd) + ): + if cwd != config["cwd"]: + logger.info( + "Ignoring host/relative cwd override %r for %s backend " + "(won't exist in sandbox). Using %r instead.", + cwd, + env_type, + config["cwd"], + ) + cwd = config["cwd"] + return cwd + + # One-shot guard for the config-fallback bridge below. Purely an # optimization: after the first attempt either TERMINAL_ENV is set (bridge # succeeded — merged config always carries terminal.backend) or the import @@ -1361,14 +1646,44 @@ def _ensure_terminal_env_bridged() -> None: logger.debug("terminal config → env fallback bridge failed", exc_info=True) +def _runtime_terminal_env() -> Dict[str, str]: + """Return terminal env values for the active runtime/profile context. + + Single-profile processes keep the historical process environment. A + multiplexed turn instead overlays the active profile's config into a + private mapping, avoiding cross-profile mutation of ``os.environ`` while + preserving exported values for settings that profile did not configure. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + if is_multiplex_active() and current_secret_scope() is not None: + from hermes_cli.config import ( + apply_terminal_config_to_env, + load_config_readonly, + ) + + target = dict(os.environ) + return apply_terminal_config_to_env( + env=target, + config=load_config_readonly(), + override=None, + ) + except Exception: + logger.debug("profile terminal config resolution failed", exc_info=True) + _ensure_terminal_env_bridged() + return dict(os.environ) + + def _get_env_config() -> Dict[str, Any]: """Get terminal environment configuration from environment variables.""" # Default image with Python and Node.js for maximum compatibility default_image = "nikolaik/python-nodejs:python3.11-nodejs20" - _ensure_terminal_env_bridged() - env_type = os.getenv("TERMINAL_ENV", "local") + runtime_env = _runtime_terminal_env() + getenv = runtime_env.get + env_type = getenv("TERMINAL_ENV", "local") - mount_docker_cwd = os.getenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower() in {"true", "1", "yes"} + mount_docker_cwd = getenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower() in {"true", "1", "yes"} container_backend = env_type in _CONTAINER_BACKENDS docker_backend = env_type == "docker" @@ -1377,19 +1692,19 @@ def _get_env_config() -> Dict[str, Any]: # until a backend that can consume them is selected; a stale or invalid # Docker value should not make local terminal/execute_code unusable. if container_backend: - container_cpu = _parse_env_var("TERMINAL_CONTAINER_CPU", "1", float, "number") - container_memory = _parse_env_var("TERMINAL_CONTAINER_MEMORY", "5120") - container_disk = _parse_env_var("TERMINAL_CONTAINER_DISK", "51200") + container_cpu = _parse_env_var("TERMINAL_CONTAINER_CPU", "1", float, "number", env=runtime_env) + container_memory = _parse_env_var("TERMINAL_CONTAINER_MEMORY", "5120", env=runtime_env) + container_disk = _parse_env_var("TERMINAL_CONTAINER_DISK", "51200", env=runtime_env) else: container_cpu = 1.0 container_memory = 5120 container_disk = 51200 if docker_backend: - docker_forward_env = _parse_env_var("TERMINAL_DOCKER_FORWARD_ENV", "[]", json.loads, "valid JSON") - docker_volumes = _parse_env_var("TERMINAL_DOCKER_VOLUMES", "[]", json.loads, "valid JSON") - docker_env = _parse_env_var("TERMINAL_DOCKER_ENV", "{}", json.loads, "valid JSON") - docker_extra_args = _parse_env_var("TERMINAL_DOCKER_EXTRA_ARGS", "[]", json.loads, "valid JSON") + docker_forward_env = _parse_env_var("TERMINAL_DOCKER_FORWARD_ENV", "[]", json.loads, "valid JSON", env=runtime_env) + docker_volumes = _parse_env_var("TERMINAL_DOCKER_VOLUMES", "[]", json.loads, "valid JSON", env=runtime_env) + docker_env = _parse_env_var("TERMINAL_DOCKER_ENV", "{}", json.loads, "valid JSON", env=runtime_env) + docker_extra_args = _parse_env_var("TERMINAL_DOCKER_EXTRA_ARGS", "[]", json.loads, "valid JSON", env=runtime_env) else: docker_forward_env = [] docker_volumes = [] @@ -1402,10 +1717,10 @@ def _get_env_config() -> Dict[str, Any]: # local terminal unusable (mirrors the container_/docker_ guards above). tenki_backend = env_type == "tenki" if tenki_backend: - tenki_forward_env = _parse_env_var("TERMINAL_TENKI_FORWARD_ENV", "[]", json.loads, "valid JSON") - tenki_max_duration = _parse_env_var("TERMINAL_TENKI_MAX_DURATION", "3600") - tenki_idle_timeout = _parse_env_var("TERMINAL_TENKI_IDLE_TIMEOUT", "0") - tenki_pause_retention = _parse_env_var("TERMINAL_TENKI_PAUSE_RETENTION", "0") + tenki_forward_env = _parse_env_var("TERMINAL_TENKI_FORWARD_ENV", "[]", json.loads, "valid JSON", env=runtime_env) + tenki_max_duration = _parse_env_var("TERMINAL_TENKI_MAX_DURATION", "3600", env=runtime_env) + tenki_idle_timeout = _parse_env_var("TERMINAL_TENKI_IDLE_TIMEOUT", "0", env=runtime_env) + tenki_pause_retention = _parse_env_var("TERMINAL_TENKI_PAUSE_RETENTION", "0", env=runtime_env) else: tenki_forward_env = [] tenki_max_duration = 3600 @@ -1428,12 +1743,12 @@ def _get_env_config() -> Dict[str, Any]: # If Docker cwd passthrough is explicitly enabled, remap the host path to # /workspace and track the original host path separately. Otherwise keep the # normal sandbox behavior and discard host paths. - cwd = os.getenv("TERMINAL_CWD", default_cwd) + cwd = getenv("TERMINAL_CWD", default_cwd) if cwd and not _is_ssh_remote_tilde_cwd(env_type, cwd): cwd = os.path.expanduser(cwd) host_cwd = None if env_type == "docker" and mount_docker_cwd: - docker_cwd_source = os.getenv("TERMINAL_CWD") or _safe_getcwd() + docker_cwd_source = getenv("TERMINAL_CWD") or _safe_getcwd() candidate = os.path.abspath(os.path.expanduser(docker_cwd_source)) if ( any(candidate.startswith(p) for p in _HOST_CWD_PREFIXES) @@ -1453,54 +1768,54 @@ def _get_env_config() -> Dict[str, Any]: return { "env_type": env_type, - "modal_mode": coerce_modal_mode(os.getenv("TERMINAL_MODAL_MODE", "auto")), - "docker_image": os.getenv("TERMINAL_DOCKER_IMAGE", default_image), + "modal_mode": coerce_modal_mode(getenv("TERMINAL_MODAL_MODE", "auto")), + "docker_image": getenv("TERMINAL_DOCKER_IMAGE", default_image), "docker_forward_env": docker_forward_env, - "singularity_image": os.getenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"), - "modal_image": os.getenv("TERMINAL_MODAL_IMAGE", default_image), - "daytona_image": os.getenv("TERMINAL_DAYTONA_IMAGE", default_image), - "tenki_image": os.getenv("TERMINAL_TENKI_IMAGE", ""), - "tenki_api_endpoint": os.getenv("TERMINAL_TENKI_API_ENDPOINT", ""), - "tenki_workspace_id": os.getenv("TERMINAL_TENKI_WORKSPACE_ID", ""), - "tenki_name_prefix": os.getenv("TERMINAL_TENKI_NAME_PREFIX", "hermes"), - "tenki_allow_inbound": os.getenv("TERMINAL_TENKI_ALLOW_INBOUND", "false").lower() in {"true", "1", "yes"}, - "tenki_allow_outbound": os.getenv("TERMINAL_TENKI_ALLOW_OUTBOUND", "true").lower() in {"true", "1", "yes"}, + "singularity_image": getenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"), + "modal_image": getenv("TERMINAL_MODAL_IMAGE", default_image), + "daytona_image": getenv("TERMINAL_DAYTONA_IMAGE", default_image), + "tenki_image": getenv("TERMINAL_TENKI_IMAGE", ""), + "tenki_api_endpoint": getenv("TERMINAL_TENKI_API_ENDPOINT", ""), + "tenki_workspace_id": getenv("TERMINAL_TENKI_WORKSPACE_ID", ""), + "tenki_name_prefix": getenv("TERMINAL_TENKI_NAME_PREFIX", "hermes"), + "tenki_allow_inbound": getenv("TERMINAL_TENKI_ALLOW_INBOUND", "false").lower() in {"true", "1", "yes"}, + "tenki_allow_outbound": getenv("TERMINAL_TENKI_ALLOW_OUTBOUND", "true").lower() in {"true", "1", "yes"}, "tenki_max_duration": tenki_max_duration, "tenki_idle_timeout": tenki_idle_timeout, "tenki_pause_retention": tenki_pause_retention, - "tenki_sync_hermes_home": os.getenv("TERMINAL_TENKI_SYNC_HERMES_HOME", "false").lower() in {"true", "1", "yes"}, + "tenki_sync_hermes_home": getenv("TERMINAL_TENKI_SYNC_HERMES_HOME", "false").lower() in {"true", "1", "yes"}, "tenki_forward_env": tenki_forward_env, "cwd": cwd, "host_cwd": host_cwd, "docker_mount_cwd_to_workspace": mount_docker_cwd, - "timeout": _parse_env_var("TERMINAL_TIMEOUT", "180"), - "lifetime_seconds": _parse_env_var("TERMINAL_LIFETIME_SECONDS", "300"), + "timeout": _parse_env_var("TERMINAL_TIMEOUT", "180", env=runtime_env), + "lifetime_seconds": _parse_env_var("TERMINAL_LIFETIME_SECONDS", "300", env=runtime_env), # SSH-specific config - "ssh_host": os.getenv("TERMINAL_SSH_HOST", ""), - "ssh_user": os.getenv("TERMINAL_SSH_USER", ""), - "ssh_port": _parse_env_var("TERMINAL_SSH_PORT", "22"), - "ssh_key": os.getenv("TERMINAL_SSH_KEY", ""), + "ssh_host": getenv("TERMINAL_SSH_HOST", ""), + "ssh_user": getenv("TERMINAL_SSH_USER", ""), + "ssh_port": _parse_env_var("TERMINAL_SSH_PORT", "22", env=runtime_env), + "ssh_key": getenv("TERMINAL_SSH_KEY", ""), # Persistent shell: SSH defaults to the config-level persistent_shell # setting (true by default for non-local backends); local is always opt-in. # Per-backend env vars override if explicitly set. - "ssh_persistent": os.getenv( + "ssh_persistent": getenv( "TERMINAL_SSH_PERSISTENT", - os.getenv("TERMINAL_PERSISTENT_SHELL", "true"), + getenv("TERMINAL_PERSISTENT_SHELL", "true"), ).lower() in {"true", "1", "yes"}, - "local_persistent": os.getenv("TERMINAL_LOCAL_PERSISTENT", "false").lower() in {"true", "1", "yes"}, + "local_persistent": getenv("TERMINAL_LOCAL_PERSISTENT", "false").lower() in {"true", "1", "yes"}, # Container resource config (applies to docker, singularity, modal, # daytona, tenki -- ignored for local/ssh) "container_cpu": container_cpu, "container_memory": container_memory, # MB (default 5GB) "container_disk": container_disk, # MB (default 50GB) - "container_persistent": os.getenv( + "container_persistent": getenv( "TERMINAL_CONTAINER_PERSISTENT", "false" if env_type == "tenki" else "true", ).lower() in {"true", "1", "yes"}, "docker_volumes": docker_volumes, "docker_env": docker_env, - "docker_run_as_host_user": os.getenv("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false").lower() in {"true", "1", "yes"}, - "docker_network": os.getenv("TERMINAL_DOCKER_NETWORK", "true").lower() in {"true", "1", "yes"}, + "docker_run_as_host_user": getenv("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false").lower() in {"true", "1", "yes"}, + "docker_network": getenv("TERMINAL_DOCKER_NETWORK", "true").lower() in {"true", "1", "yes"}, "docker_extra_args": docker_extra_args, # Cross-process container reuse (issue #20561). The docs claim # "ONE long-lived container shared across sessions" — this toggle @@ -1508,14 +1823,14 @@ def _get_env_config() -> Dict[str, Any]: # attaching to it instead of always starting a fresh one. Set to # ``false`` for hard per-process isolation (no reuse, container is # removed on exit). - "docker_persist_across_processes": os.getenv( + "docker_persist_across_processes": getenv( "TERMINAL_DOCKER_PERSIST_ACROSS_PROCESSES", "true" ).lower() in {"true", "1", "yes"}, # Startup orphan reaper for hermes-tagged containers left behind by # crashed / SIGKILL'd previous processes that bypassed atexit. # Conservative: only sweeps Exited containers older than 2× the # idle-reap window AND scoped to the current profile. Issue #20561. - "docker_orphan_reaper": os.getenv( + "docker_orphan_reaper": getenv( "TERMINAL_DOCKER_ORPHAN_REAPER", "true" ).lower() in {"true", "1", "yes"}, } @@ -1750,36 +2065,25 @@ def _cleanup_inactive_envs(lifetime_seconds: int = 300): except ImportError: pass - # Phase 1: collect stale entries and remove them from tracking dicts while - # holding the lock. Do NOT call env.cleanup() inside the lock -- Modal and - # Docker teardown can block for 10-15s, which would stall every concurrent - # terminal/file tool call waiting on _env_lock. - envs_to_stop = [] # list of (task_id, env) pairs + # Phase 1: tombstone stale entries while retaining their registry slots. + # Creators wait on the tombstone, so they cannot build a second wrapper + # while cleanup is waiting for an in-flight operation to finish. + envs_to_stop = [] # list of (task_id, env, completion-event) tuples with _env_lock: for task_id, last_time in list(_last_activity.items()): if current_time - last_time > lifetime_seconds: - env = _active_environments.pop(task_id, None) + env = _active_environments.get(task_id) + if env is None or task_id in _retiring_environments: + continue + completed = threading.Event() + _retiring_environments[task_id] = completed _last_activity.pop(task_id, None) - if env is not None: - envs_to_stop.append((task_id, env)) - - # Also purge per-task creation locks for cleaned-up tasks - with _creation_locks_lock: - for task_id, _ in envs_to_stop: - _creation_locks.pop(task_id, None) + envs_to_stop.append((task_id, env, completed)) # Phase 2: stop the actual sandboxes OUTSIDE the lock so other tool calls # are not blocked while Modal/Docker sandboxes shut down. - for task_id, env in envs_to_stop: - # Invalidate stale file_ops cache entry (Bug fix: prevents - # ShellFileOperations from referencing a dead sandbox) - try: - from tools.file_tools import clear_file_ops_cache - clear_file_ops_cache(task_id) - except ImportError: - pass - + for task_id, env, completed in envs_to_stop: try: if hasattr(env, 'cleanup'): env.cleanup() @@ -1789,12 +2093,15 @@ def _cleanup_inactive_envs(lifetime_seconds: int = 300): env.terminate() logger.info("Cleaned up inactive environment for task: %s", task_id) + _finish_environment_retirement(task_id, env, completed) except Exception as e: error_str = str(e) if "404" in error_str or "not found" in error_str.lower(): + _finish_environment_retirement(task_id, env, completed) logger.info("Environment for task %s already cleaned up", task_id) else: + _abort_environment_retirement(task_id, env, completed) logger.warning("Error cleaning up environment for task %s: %s", task_id, e) @@ -1837,9 +2144,15 @@ def _stop_cleanup_thread(): def get_active_env(task_id: str): """Return the active BaseEnvironment for *task_id*, or None.""" - lookup = _resolve_container_task_id(task_id) - with _env_lock: - return _active_environments.get(lookup) or _active_environments.get(task_id) + env_type = _get_env_config()["env_type"] + lookup = _resolve_environment_cache_key(task_id, env_type) + fallback = task_id if env_type != "tenki" else None + _selected_key, env = _select_active_environment( + lookup, + fallback, + touch=False, + ) + return env def is_persistent_env(task_id: str) -> bool: @@ -1868,7 +2181,15 @@ def cleanup_all_environments(): for task_id in task_ids: try: - cleanup_vm(task_id) + # These are already canonical registry keys. Re-resolving a + # profile-prefixed Tenki key under the process's current profile + # would deliberately reject it as foreign and leak that sandbox + # during gateway/atexit cleanup. + _cleanup_environment_by_key( + task_id, + display_task_id=task_id, + force_remove=False, + ) cleaned += 1 except Exception as e: logger.error("Error cleaning %s: %s", task_id, e, exc_info=True) @@ -1909,27 +2230,39 @@ def cleanup_vm(task_id: str, *, force_remove: bool = False): via this function), so persist-mode idle envs are similarly no-op'd — only the orphan reaper at next startup reclaims them. """ - # Remove from tracking dicts while holding the lock, but defer the - # actual (potentially slow) env.cleanup() call to outside the lock - # so other tool calls aren't blocked. - env = None + env_type = _get_env_config()["env_type"] + lookup = _resolve_environment_cache_key(task_id, env_type) + raw_lookup = task_id if env_type != "tenki" else None with _env_lock: - env = _active_environments.pop(task_id, None) - _last_activity.pop(task_id, None) - - # Clean up per-task creation lock - with _creation_locks_lock: - _creation_locks.pop(task_id, None) + actual_lookup = lookup + env = _active_environments.get(actual_lookup) + if env is None and raw_lookup is not None: + actual_lookup = raw_lookup + _cleanup_environment_by_key( + actual_lookup, + display_task_id=task_id, + force_remove=force_remove, + ) - # Invalidate stale file_ops cache entry - try: - from tools.file_tools import clear_file_ops_cache - clear_file_ops_cache(task_id) - except ImportError: - pass +def _cleanup_environment_by_key( + actual_lookup: str, + *, + display_task_id: str, + force_remove: bool, +) -> None: + """Clean one exact registry key without re-resolving profile identity.""" + with _env_lock: + env = _active_environments.get(actual_lookup) if env is None: return + completed, owns_retirement = _begin_environment_retirement( + actual_lookup, + env, + ) + if not owns_retirement: + completed.wait() + return try: if hasattr(env, 'cleanup'): @@ -1946,14 +2279,39 @@ def cleanup_vm(task_id: str, *, force_remove: bool = False): elif hasattr(env, 'terminate'): env.terminate() - logger.info("Manually cleaned up environment for task: %s", task_id) + logger.info( + "Manually cleaned up environment for task: %s", + display_task_id, + ) + _finish_environment_retirement( + actual_lookup, + env, + completed, + ) except Exception as e: error_str = str(e) if "404" in error_str or "not found" in error_str.lower(): - logger.info("Environment for task %s already cleaned up", task_id) + _finish_environment_retirement( + actual_lookup, + env, + completed, + ) + logger.info( + "Environment for task %s already cleaned up", + display_task_id, + ) else: - logger.warning("Error cleaning up environment for task %s: %s", task_id, e) + _abort_environment_retirement( + actual_lookup, + env, + completed, + ) + logger.warning( + "Error cleaning up environment for task %s: %s", + display_task_id, + e, + ) def _atexit_cleanup(): @@ -2264,7 +2622,7 @@ def terminal_tool( # task_ids collapse back to "default" so the top-level agent and # every delegate_task child share one container; only task_ids with # a registered env override (RL benchmarks) get isolated sandboxes. - effective_task_id = _resolve_container_task_id(task_id) + effective_task_id = _resolve_environment_cache_key(task_id, env_type) # Check per-task overrides (set by environments like TerminalBench2Env) # before falling back to global env var config. ``resolve_task_overrides`` @@ -2288,30 +2646,12 @@ def terminal_tool( else: image = "" - cwd = overrides.get("cwd") or get_session_cwd(task_id) or config["cwd"] - # A per-task cwd override (registered by the gateway/TUI for workspace - # tracking, or by RL/benchmark envs) wins over config["cwd"] — but - # config["cwd"] was already sanitized for container backends in - # _get_env_config() while the override is raw. On a container backend a - # raw host path (e.g. a Windows desktop session's C:\Users\, or a - # POSIX /home/) reaches `docker run -w ` and the - # container fails to start (exit 125). Re-apply the same host/relative - # path guard to the *resolved* cwd so the override can't bypass it. - # Valid in-container override paths (RL/benchmark sandboxes that set - # cwd to /workspace, /root, etc.) are absolute non-host paths and pass - # through untouched. - if ( - env_type in _CONTAINER_BACKENDS - and _is_unusable_container_cwd(cwd) - and not _is_backend_guest_subpath(env_type, cwd) - ): - if cwd != config["cwd"]: - logger.info( - "Ignoring host/relative cwd override %r for %s backend " - "(won't exist in sandbox). Using %r instead.", - cwd, env_type, config["cwd"], - ) - cwd = config["cwd"] + cwd = _resolve_environment_cwd( + task_id, + env_type, + config, + overrides, + ) default_timeout = config["timeout"] effective_timeout = timeout or default_timeout @@ -2345,42 +2685,25 @@ def terminal_tool( # Use a per-task creation lock so concurrent tool calls for the same # task_id wait for the first one to finish creating the sandbox, # instead of each creating their own (wasting Modal resources). - env = None - with _env_lock: - # Prefer the collapsed container id, but fall back to an env cached - # under the raw task_id. Per-session surfaces (ACP/gateway/dashboard) - # with a CWD-only override collapse to "default" for container - # sharing, yet an env may already be cached under the originating - # task_id; honor it instead of spawning a duplicate. - _existing_key = ( - effective_task_id if effective_task_id in _active_environments - else (task_id if task_id and task_id in _active_environments else None) - ) - if _existing_key is not None: - _last_activity[_existing_key] = time.time() - env = _active_environments[_existing_key] - needs_creation = False - else: - needs_creation = True + fallback_task_id = ( + task_id if env_type != "tenki" and task_id else None + ) + _existing_key, env = _select_active_environment( + effective_task_id, + fallback_task_id, + ) + needs_creation = env is None if needs_creation: - # Per-task lock: only one thread creates the sandbox, others wait - with _creation_locks_lock: - if effective_task_id not in _creation_locks: - _creation_locks[effective_task_id] = threading.Lock() - task_lock = _creation_locks[effective_task_id] - - with task_lock: + # Per-task slot: only one thread creates the sandbox, and the slot + # generation stays stable across retirement for all waiters. + with _environment_creation_lock(effective_task_id): # Double-check after acquiring the per-task lock - with _env_lock: - _existing_key = ( - effective_task_id if effective_task_id in _active_environments - else (task_id if task_id and task_id in _active_environments else None) - ) - if _existing_key is not None: - _last_activity[_existing_key] = time.time() - env = _active_environments[_existing_key] - needs_creation = False + _existing_key, env = _select_active_environment( + effective_task_id, + fallback_task_id, + ) + needs_creation = env is None if needs_creation: if env_type == "singularity": @@ -2426,10 +2749,10 @@ def terminal_tool( "status": "disabled" }, ensure_ascii=False) - with _env_lock: - _active_environments[effective_task_id] = new_env - _last_activity[effective_task_id] = time.time() - env = new_env + env = _register_active_environment( + effective_task_id, + new_env, + ) logger.info("%s environment ready for task %s", env_type, effective_task_id[:8]) if env is None: diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 67fcc76bd8286..215bab7418802 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -220,7 +220,7 @@ These variables configure the [Tool Gateway](/user-guide/features/tool-gateway) | `TERMINAL_SINGULARITY_IMAGE` | Singularity image or `.sif` path | | `TERMINAL_MODAL_IMAGE` | Modal container image | | `TERMINAL_DAYTONA_IMAGE` | Daytona sandbox image | -| `TERMINAL_TENKI_IMAGE` | Optional Tenki sandbox image/template; blank uses Tenki default | +| `TERMINAL_TENKI_IMAGE` | Optional Tenki registry image reference; blank uses Tenki default | | `TERMINAL_TENKI_API_ENDPOINT` | Tenki API endpoint (default: `https://api.tenki.cloud`) | | `TERMINAL_TENKI_WORKSPACE_ID` | Tenki workspace ID; blank falls back to Tenki CLI config | | `TERMINAL_TENKI_NAME_PREFIX` | Prefix for Hermes-created Tenki sandbox names (default: `hermes`) | @@ -252,9 +252,9 @@ For cloud sandbox backends, persistence is filesystem-oriented. `TERMINAL_LIFETI | Variable | Description | |----------|-------------| -| `TERMINAL_CONTAINER_CPU` | CPU cores (default: 1) | -| `TERMINAL_CONTAINER_MEMORY` | Memory in MB (default: 5120) | -| `TERMINAL_CONTAINER_DISK` | Disk in MB (default: 51200) | +| `TERMINAL_CONTAINER_CPU` | CPU cores (default: 1; Tenki accepts 1-16) | +| `TERMINAL_CONTAINER_MEMORY` | Memory in MB (default: 5120; Tenki accepts even values from 128-65536) | +| `TERMINAL_CONTAINER_DISK` | Disk in MB (default: 51200; Tenki accepts 5-100 GB) | | `TERMINAL_CONTAINER_PERSISTENT` | Persist container filesystem across sessions (default: `true`; Tenki defaults to `false` unless explicitly set) | | `TERMINAL_SANDBOX_DIR` | Host directory for workspaces and overlays (default: `~/.hermes/sandboxes/`) | diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index df37df586fbe7..6c3bdf69df0fc 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -126,14 +126,14 @@ terminal: singularity_image: "docker://nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Singularity backend modal_image: "nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Modal backend daytona_image: "nikolaik/python-nodejs:python3.11-nodejs20" # Container image for Daytona backend - tenki_image: "" # Optional Tenki image/template; blank uses Tenki default + tenki_image: "" # Optional Tenki registry image reference; blank uses Tenki default tenki_api_endpoint: "https://api.tenki.cloud" tenki_workspace_id: "" # Blank falls back to Tenki CLI config tenki_sync_hermes_home: false # Opt-in sync of selected ~/.hermes files tenki_forward_env: [] # Explicit host env vars to forward into Tenki ``` -For cloud sandboxes such as Modal, Daytona, and Tenki, `container_persistent: true` means Hermes will try to preserve filesystem state across sandbox recreation. It does not promise that the same live sandbox, PID space, or background processes will still be running later. Tenki defaults to `container_persistent: false` so sandboxes are terminated when Hermes cleans them up; when persistence is enabled, Hermes pauses and later resumes the matching Tenki sandbox. +For cloud sandboxes such as Modal, Daytona, and Tenki, `container_persistent: true` means Hermes will try to preserve filesystem state across sandbox recreation. It does not promise that the same live sandbox, PID space, or background processes will still be running later. Tenki defaults to `container_persistent: false`; when persistence is enabled, cleanup normally snapshots and terminates the sandbox, then the next environment restores that snapshot. Hermes only keeps the live sandbox paused when it cannot confirm a durable snapshot. ### Backend Overview @@ -391,6 +391,9 @@ terminal: backend: tenki cwd: "/home/tenki" container_persistent: false # Default for Tenki + container_cpu: 1 # Tenki supports 1-16 cores + container_memory: 5120 # Tenki supports even values, 128-65536 MB + container_disk: 51200 # MB; Tenki supports 5-100 GB tenki_api_endpoint: "https://api.tenki.cloud" tenki_workspace_id: "" # Falls back to Tenki CLI config tenki_name_prefix: "hermes" @@ -405,15 +408,15 @@ terminal: **Required:** Tenki CLI login, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`. Hermes also reads the Tenki CLI config for the current workspace ID. -**Persistence:** Tenki is terminate-only by default. Set `container_persistent: true` only if you intentionally want Hermes to pause and resume a task-named sandbox. +**Persistence:** Tenki is ephemeral by default: Hermes terminates the sandbox during cleanup. With `container_persistent: true`, Hermes first creates a durable snapshot, records its ID in the active profile's `tenki_snapshots.json`, attempts to delete the superseded remote snapshot, and then terminates the sandbox. The next environment restores the recorded snapshot. If snapshot durability or local pointer persistence cannot be confirmed, Hermes preserves the prior pointer and pauses the live sandbox instead of terminating the only copy; a later run can rediscover and resume it. **Sudo:** Tenki sandboxes use the sandbox's own sudoers policy. Hermes never prompts for or forwards the host `SUDO_PASSWORD` to Tenki. The default Tenki image supports passwordless sudo. -**Optional `.hermes` sync:** Set `tenki_sync_hermes_home: true` if a Tenki sandbox needs the same selected credential, skill, and cache files that Modal and Daytona receive. Leave it off when Tenki is only an execution sandbox and secrets should remain with the supervisor process. +**Optional `.hermes` sync:** Set `tenki_sync_hermes_home: true` if a Tenki sandbox needs selected credential, skill, and cache files from `~/.hermes`. This setting only synchronizes that selected `.hermes` data; it does **not** copy the sandbox working directory or arbitrary command outputs back to the host. Leave it off when Tenki is only an execution sandbox and secrets should remain with the supervisor process. **Credential forwarding:** `terminal.env_passthrough` intentionally blocks common credential names such as `GITHUB_TOKEN` and `GH_TOKEN`. For Git or package-manager tokens that must be visible inside Tenki sandboxes, list the variable names in `terminal.tenki_forward_env`; Hermes resolves them profile-scope-aware (from your current shell / the active profile scope first, then `~/.hermes/.env`). The supervisor's own Tenki control-plane token (`TENKI_AUTH_TOKEN` / `TENKI_API_KEY`) is **not** forwarded by default; add it to `tenki_forward_env` only when child sandboxes must create nested Tenki sandboxes, and note that anything forwarded is readable by (model-controlled) guest code. -> **Multiplexing caveat:** forwarded credentials are not profile-isolated under the multiplexing gateway's shared terminal cache — two profiles served by one gateway process can reuse the same live sandbox. Avoid forwarding sensitive credentials (especially the control-plane token) into sandboxes when running multiple profiles from a single multiplexed gateway. +When the multiplexing gateway serves several profiles, Hermes keys Tenki environments and availability checks by profile. Each profile therefore resolves its own backend settings, workspace, auth scope, sandbox cache, file operations, and snapshots. **Fully remote supervisor pattern:** To make Hermes itself live remotely, run the Hermes process inside a long-lived Tenki supervisor sandbox and configure that process with `terminal.backend: tenki`. The supervisor owns `~/.hermes`, model credentials, sessions, memory, and gateway/dashboard processes. Terminal, file, `execute_code`, and delegated subagent execution then create child Tenki sandboxes on demand. Give the supervisor Tenki credentials with `tenki login`, `TENKI_AUTH_TOKEN`, or `TENKI_API_KEY`; child sandboxes do not need host sudo passwords. @@ -452,22 +455,11 @@ If terminal commands fail immediately or the terminal tool is reported as disabl When in doubt, set `terminal.backend` back to `local` and verify that commands run there first. -### Remote-to-Host File Sync on Teardown +### Selected `.hermes` Sync for Remote Backends -For the **SSH**, **Modal**, **Daytona**, and **Tenki** backends (anywhere the agent's working tree lives on a different machine than the host running Hermes), Hermes tracks files the agent touched inside the remote sandbox and, on session teardown / sandbox cleanup, **syncs the modified files back to the host** under `~/.hermes/cache/remote-syncs//`. Tenki also supports opt-in selected `.hermes` credential/skill/cache sync with `terminal.tenki_sync_hermes_home: true`; it is disabled by default. +SSH, Modal, and Daytona synchronize selected files that Hermes itself needs in the remote environment: declared credential files, skills, and supported `~/.hermes/cache` entries. Tenki participates only when `terminal.tenki_sync_hermes_home: true`. Credential files are upload-only; sync-back never overwrites their host copies. -- Triggers on: session close, `/new`, `/reset`, gateway message timeout, `delegate_task` subagent completion when the child used a remote backend. -- Covers the whole tree the agent modified, not just files it explicitly opened. Additions, edits, and deletions are all captured. -- The remote sandbox may have been torn down by the time you go looking; the local `~/.hermes/cache/remote-syncs/…` copy is the authoritative record of what the agent changed. -- Large binary outputs (model checkpoints, raw datasets) are capped by size — the sync skips files over `file_sync_max_mb` (default `100`). Bump that if you expect bigger artifacts to come back. - -```yaml -terminal: - file_sync_max_mb: 100 # default — sync files up to 100 MB each - file_sync_enabled: true # default — set false to skip the sync entirely -``` - -This is how you recover results from ephemeral cloud sandboxes that get destroyed after the session ends, without having to tell the agent to explicitly `scp` or `modal volume put` every artifact. +This is **not** a working-tree backup. Hermes does not automatically collect arbitrary files created under the remote command working directory, and there is no `~/.hermes/cache/remote-syncs//` recovery copy. Before an ephemeral sandbox is cleaned up, explicitly download, commit, upload, or otherwise transfer every output you need. With Tenki's default non-persistent mode, cleanup terminates the sandbox. ### Docker Volume Mounts From d6f259903fbad02d81dcf6147d64b73a952e87d0 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 28 Jul 2026 08:14:32 -0700 Subject: [PATCH 09/12] fix(tools): keep live environment authoritative in path detection _terminal_env_type_for_task now reads the terminal env config before the active-environment lookup, because the environment registry's cache key is derived from env_type. That hoist put the config read inside the function's outer try, so any config failure aborted the whole lookup and returned "local" even when a container backend was registered -- silently resolving container paths against the host, which is the misrouting this module exists to prevent. Give the config read its own handler so it degrades to an empty mapping instead of pre-empting the live-environment lookup. Restores the precedence asserted by test_container_path_detection_uses_live_docker_environment, which was red on this branch. Co-Authored-By: Claude Opus 5 (1M context) --- tools/file_tools.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/file_tools.py b/tools/file_tools.py index d855c6348c748..2ac65ef77fe83 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -178,7 +178,16 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str: _select_active_environment, ) - cfg = _get_env_config() + # The registry cache key needs env_type, so the config read has to come + # first — but it must not pre-empt the live-environment lookup. A + # registered backend is authoritative on its own, and letting a config + # failure fall through to the outer handler would report "local" while + # a container env is active, silently routing container paths to the + # host (the misrouting this module exists to prevent). + try: + cfg = _get_env_config() + except Exception: + cfg = {} try: container_key = _resolve_environment_cache_key( task_id, From 2565a14231be8e0945ce9ab2a6d675a7d534ab4b Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 28 Jul 2026 08:14:39 -0700 Subject: [PATCH 10/12] refactor(tools): unify sudo command-word rewriting _rewrite_real_sudo_invocations and _count_real_sudo_invocations were the same shell tokenizer walk -- tracking command_start, skipping comments, handling the &&/||/;;/;|&() operators and leading env assignments -- with the counter differing only in dropping the output list. Two copies is two chances to drift on quote and comment handling, and the rewriter already returns the count the counter recomputes. Collapse the walk into _rewrite_sudo_command_words(command, replacement). Both public names survive as thin callers, so no call site changes. No behaviour change: the old and new implementations were differentially compared over roughly 67k generated command strings, covering quotes, escapes, comments, operators, env-assignment prefixes and subshells, with identical rewritten output and identical counts throughout. Co-Authored-By: Claude Opus 5 (1M context) --- tools/terminal_tool.py | 67 ++++++++---------------------------------- 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 79698a303ac37..dba05efee8a39 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -559,10 +559,12 @@ def _read_shell_token(command: str, start: int) -> tuple[str, int]: return command[start:i], i -def _rewrite_real_sudo_invocations(command: str) -> tuple[str, int]: - """Rewrite only real unquoted sudo command words, not plain text mentions. +def _rewrite_sudo_command_words(command: str, replacement: str) -> tuple[str, int]: + """Replace real unquoted sudo command words with *replacement*. - Returns the rewritten command and the number of sudo invocations rewritten. + Only command words are rewritten, so plain text mentions of sudo (arguments, + quoted strings, comments) are left alone. Returns the rewritten command and + the number of sudo command words replaced. """ out: list[str] = [] i = 0 @@ -609,7 +611,7 @@ def _rewrite_real_sudo_invocations(command: str) -> tuple[str, int]: token, next_i = _read_shell_token(command, i) if command_start and token == "sudo": - out.append("sudo -S -p ''") + out.append(replacement) sudo_count += 1 else: out.append(token) @@ -623,60 +625,17 @@ def _rewrite_real_sudo_invocations(command: str) -> tuple[str, int]: return "".join(out), sudo_count -def _count_real_sudo_invocations(command: str) -> int: - """Return how many real sudo command words appear in *command*. +def _rewrite_real_sudo_invocations(command: str) -> tuple[str, int]: + """Rewrite only real unquoted sudo command words, not plain text mentions. - Lightweight scan that reuses the same tokeniser as - ``_rewrite_real_sudo_invocations`` but skips the string-building, so it - is cheap to call from the result-processing path. + Returns the rewritten command and the number of sudo invocations rewritten. """ - count = 0 - i = 0 - n = len(command) - command_start = True - - while i < n: - ch = command[i] - - if ch.isspace(): - if ch == "\n": - command_start = True - i += 1 - continue - - if ch == "#" and command_start: - comment_end = command.find("\n", i) - if comment_end == -1: - break - i = comment_end - continue + return _rewrite_sudo_command_words(command, "sudo -S -p ''") - if command.startswith("&&", i) or command.startswith("||", i) or command.startswith(";;", i): - i += 2 - command_start = True - continue - if ch in ";|&(": - i += 1 - command_start = True - continue - - if ch == ")": - i += 1 - command_start = False - continue - - token, next_i = _read_shell_token(command, i) - if command_start and token == "sudo": - count += 1 - - if command_start and _looks_like_env_assignment(token): - command_start = True - else: - command_start = False - i = next_i - - return count +def _count_real_sudo_invocations(command: str) -> int: + """Return how many real sudo command words appear in *command*.""" + return _rewrite_real_sudo_invocations(command)[1] def _sudo_nopasswd_works() -> bool: From 5df47eef0c74a1ce0879da6563432f03976749fe Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 28 Jul 2026 08:14:50 -0700 Subject: [PATCH 11/12] refactor(tools): restructure Tenki recovery-state layer Deduplication pass over the Tenki snapshot and binding registry. No change to durability, lock ordering, or which errors propagate. - Adopt the shared _rewrite_sudo_command_words walker and drop Tenki's private third copy of it. - _mutate_store() replaces the resolve-path/lock/load/save preamble that was repeated across 11 registry mutators. - _RemoteBinding replaces an unnamed six-tuple return and the six parallel instance attributes that mirrored it. - One parameterised file-lock helper serves both the blocking snapshot-store lock and the non-blocking task-ownership lock, which had each hand-rolled the same fcntl/msvcrt branch. - _record_field() and _fail_ambiguous_lineage() collapse the dict-or-scalar unwrap and the set-flag-then-raise pattern. - Rename _load_json_store to _load_recovery_registry. It shadowed base._load_json_store with the opposite error semantics: base returns an empty mapping on an unreadable file, this one deliberately raises so a corrupt registry cannot erase the only recovery pointer. - Move the durable atomic-write primitive to base._atomic_save_json_durable so other backends can reuse it; Tenki keeps the uncertain-commit policy wrapper on top. Four sites were deliberately left on their existing code because routing them through a shared helper would have changed when a write becomes durable relative to a remote RPC: _queue_snapshot_retirement (its early return must not write), _retire_pending_snapshot_if_unreferenced (holds one lock across tombstone, remote delete and clear, which is the non-resurrection guarantee), _confirm_snapshot_store_durable (fsync-only, not load-mutate-save), and the ownership lock's file-close ownership. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tools/test_tenki_environment.py | 34 +- tools/environments/base.py | 115 ++++ tools/environments/tenki.py | 773 +++++++++++--------------- 3 files changed, 448 insertions(+), 474 deletions(-) diff --git a/tests/tools/test_tenki_environment.py b/tests/tools/test_tenki_environment.py index c645baa1fecda..708efea67230a 100644 --- a/tests/tools/test_tenki_environment.py +++ b/tests/tools/test_tenki_environment.py @@ -1391,7 +1391,7 @@ def test_tenki_unidentified_collision_is_durably_unresolvable( with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): TenkiEnvironment(task_id="persist", persistent_filesystem=True) - assert tenki_module._remote_binding_state("persist") == ( + assert tenki_module._remote_binding_state("persist") == tenki_module._RemoteBinding( None, None, False, @@ -1456,7 +1456,7 @@ def test_tenki_mixed_unmanaged_collision_preserves_known_ids( with pytest.raises(RuntimeError, match="unmanaged persistent sandbox"): TenkiEnvironment(task_id=task_id, persistent_filesystem=True) - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( known.id, None, False, @@ -1582,7 +1582,7 @@ def test_tenki_exact_binding_survives_list_omission_on_create_and_restart( ) remote = first._sandbox assert len(_FakeSandboxFactory.created_kwargs) == 1 - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( remote.id, remote.info.metadata["hermes_create_attempt"], True, @@ -2172,7 +2172,7 @@ def test_tenki_snapshot_pointer_updates_are_serialized_and_atomic( active_loads = 0 max_active_loads = 0 state_lock = threading.Lock() - original_load = tenki_module._load_json_store + original_load = tenki_module._load_recovery_registry def tracked_load(path): nonlocal active_loads, max_active_loads @@ -2186,7 +2186,7 @@ def tracked_load(path): with state_lock: active_loads -= 1 - monkeypatch.setattr(tenki_module, "_load_json_store", tracked_load) + monkeypatch.setattr(tenki_module, "_load_recovery_registry", tracked_load) start = threading.Barrier(3) def save(task_id, snapshot_id): @@ -2673,7 +2673,7 @@ def commit_then_timeout(self, **kwargs): assert new_remote.terminated is False attempt_id = new_remote.info.metadata["hermes_create_attempt"] assert tenki_module._get_create_attempt(task_id) is None - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( min(old_remote.id, new_remote.id), attempt_id, False, @@ -2819,10 +2819,8 @@ def test_tenki_known_conflict_clears_only_after_exact_remote_is_terminal( assert successor._sandbox is not existing assert len(_FakeSandboxFactory.created_kwargs) == 1 - assert tenki_module._remote_binding_state(task_id)[2:4] == ( - True, - False, - ) + _binding = tenki_module._remote_binding_state(task_id) + assert (_binding.validated, _binding.conflicted) == (True, False) successor.cleanup() @@ -3074,10 +3072,8 @@ def test_tenki_terminal_list_row_requires_authoritative_get_before_clear( assert env._sandbox is authoritative assert len(_FakeSandboxFactory.created_kwargs) == 0 - assert tenki_module._remote_binding_state(task_id)[2:4] == ( - True, - False, - ) + _binding = tenki_module._remote_binding_state(task_id) + assert (_binding.validated, _binding.conflicted) == (True, False) env.cleanup() @@ -3126,7 +3122,7 @@ def test_tenki_expired_attempt_never_adopts_unmanaged_match( ) assert tenki_module._get_create_attempt(task_id) is None - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( unmanaged.id, "expired-attempt", False, @@ -3190,7 +3186,7 @@ def test_tenki_hidden_exact_conflict_preserves_visible_known_id( with pytest.raises(RuntimeError, match="prior create is unresolved"): TenkiEnvironment(task_id=task_id, persistent_filesystem=True) - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( visible.id, "hidden-exact-attempt", False, @@ -3254,7 +3250,7 @@ def test_tenki_unvalidated_binding_requires_durable_expected_attempt( persistent_filesystem=True, ) - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( remote.id, "expected-attempt", False, @@ -3375,7 +3371,7 @@ def reveal_fork_after_create( created = _FakeSandboxFactory.sandboxes[0] assert tenki_module._get_create_attempt(task_id) is None - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( created.id, created.info.metadata["hermes_create_attempt"], False, @@ -3471,7 +3467,7 @@ def reveal_mixed_conflict(self, **_kwargs): TenkiEnvironment(task_id=task_id, persistent_filesystem=True) created = _FakeSandboxFactory.sandboxes[0] - assert tenki_module._remote_binding_state(task_id) == ( + assert tenki_module._remote_binding_state(task_id) == tenki_module._RemoteBinding( created.id, created.info.metadata["hermes_create_attempt"], False, diff --git a/tools/environments/base.py b/tools/environments/base.py index dd9be1db6c7bf..cc13112711321 100644 --- a/tools/environments/base.py +++ b/tools/environments/base.py @@ -13,6 +13,7 @@ import select import shlex import subprocess +import tempfile import threading import time import uuid @@ -272,6 +273,120 @@ def _save_json_store(path: Path, data: dict) -> None: path.write_text(json.dumps(data, indent=2), encoding="utf-8") +def _windows_replace_file_write_through(source: str, destination: Path) -> None: + """Atomically replace *destination* with Windows write-through semantics.""" + import ctypes + from ctypes import wintypes + + move_file_ex = ctypes.WinDLL( + "kernel32", + use_last_error=True, + ).MoveFileExW + move_file_ex.argtypes = ( + wintypes.LPCWSTR, + wintypes.LPCWSTR, + wintypes.DWORD, + ) + move_file_ex.restype = wintypes.BOOL + flags = 0x1 | 0x8 # MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH + if move_file_ex( + os.path.abspath(source), + os.path.abspath(destination), + flags, + ): + return + error_code = ctypes.get_last_error() + raise OSError( + error_code, + ctypes.FormatError(error_code), + str(destination), + ) + + +def _atomic_save_json_durable( + path: Path, + data: dict, + *, + subject: str = "file", + store_label: str = "This JSON store", + commit_uncertain_error: Callable[[str], BaseException] = OSError, + platform: str | None = None, + replace_write_through: Callable[[str, Path], None] | None = None, +) -> None: + """Write *data* to *path* as JSON with a crash-durable atomic commit. + + Mechanism only; the caller supplies the policy. Contents are fsynced before + the replace, and on POSIX the containing directory is fsynced afterwards so + the rename itself survives a host crash. A failure of either durability + step is an *uncertain commit*: the visible file is already the new one, but + a crash may roll the directory entry back. That case raises + *commit_uncertain_error* so a caller with recovery state can keep both the + old and the new referents alive. + + *platform* and *replace_write_through* exist so a caller can substitute its + own (testable) platform detection and Windows replace implementation. + """ + platform = platform if platform is not None else os.name + if replace_write_through is None: + replace_write_through = _windows_replace_file_write_through + path.parent.mkdir(parents=True, exist_ok=True) + temp_path: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as tmp: + json.dump(data, tmp, indent=2) + tmp.flush() + os.fsync(tmp.fileno()) + temp_path = tmp.name + if platform == "nt": + try: + replace_write_through(temp_path, path) + except OSError as exc: + raise commit_uncertain_error( + f"could not durably replace {subject} file {path}" + ) from exc + return + if platform != "posix": + raise RuntimeError( + f"{store_label} cannot prove rename durability on " + f"platform {platform!r}" + ) + + os.replace(temp_path, path) + # The file contents were fsynced before replace; fsync the containing + # directory too so the pointer rename itself survives a host crash. + # A POSIX failure is an *uncertain commit*: the visible file is new, + # but a crash may roll the directory entry back. The caller must keep + # every referent of both the old and the new state in that case. + dir_fd: int | None = None + try: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + dir_fd = os.open(path.parent, flags) + os.fsync(dir_fd) + except OSError as exc: + raise commit_uncertain_error( + f"could not fsync {subject} directory {path.parent}" + ) from exc + finally: + if dir_fd is not None: + try: + os.close(dir_fd) + except OSError: + pass + finally: + if temp_path and os.path.exists(temp_path): + try: + os.unlink(temp_path) + except OSError: + pass + + def _file_mtime_key(host_path: str) -> tuple[float, int] | None: """Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable.""" try: diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index 5a9aff9f6b341..b731b9e5ddac1 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -17,13 +17,16 @@ import uuid from contextlib import contextmanager from contextvars import copy_context +from dataclasses import dataclass, replace from pathlib import Path -from typing import Any +from typing import Any, NoReturn from hermes_constants import get_hermes_home from tools.environments.base import ( BaseEnvironment, + _atomic_save_json_durable, _ThreadedProcessHandle, + _windows_replace_file_write_through, ) from tools.environments.file_sync import ( FileSyncManager, @@ -80,12 +83,32 @@ class _SnapshotPointerConflict(RuntimeError): """A stale writer attempted to replace newer or retired recovery state.""" -def _load_json_store(path: Path) -> dict: +@dataclass(frozen=True) +class _RemoteBinding: + """One task's durable binding to its remote Tenki sandbox lineage. + + ``conflicted`` means a fork was positively observed and may never be + forgotten by a later omission-prone listing; ``unresolvable`` additionally + means at least one branch had no authoritative id, so no exact lookup can + ever prove it terminated. The default instance is "no binding recorded". + """ + + remote_id: str | None = None + attempt_id: str | None = None + validated: bool = False + conflicted: bool = False + conflict_ids: tuple[str, ...] = () + unresolvable: bool = False + + +def _load_recovery_registry(path: Path) -> dict: """Load Tenki recovery state, failing closed when it is unreadable. - Treating a malformed existing registry as empty would let the next - read-modify-write erase the only snapshot pointer, create attempt, or - exact remote binding. A missing file is the sole valid empty state. + Deliberately the opposite of :func:`base._load_json_store`, which returns + ``{}`` on any read error: treating a malformed existing registry as empty + would let the next read-modify-write erase the only snapshot pointer, + create attempt, or exact remote binding. A missing file is the sole valid + empty state. """ if not path.exists(): return {} @@ -118,33 +141,67 @@ def _task_ownership_lock_path(profile_home: Path, task_id: str) -> Path: return profile_home / "locks" / "tenki" / f"{task_hash}.lock" -def _acquire_task_ownership_lock(profile_home: Path, task_id: str): - """Acquire one profile/task lifetime lock or fail without remote mutation.""" - lock_path = _task_ownership_lock_path(profile_home, task_id) - lock_path.parent.mkdir(parents=True, exist_ok=True) - lock_file = lock_path.open("a+", encoding="utf-8") +def _lock_open_file(lock_file: Any, *, blocking: bool, requirement: str) -> None: + """Take one exclusive cross-process lock on *lock_file*'s first byte. + + *blocking* selects between waiting for the holder (the snapshot registry, + which must serialize pointer RMW) and failing fast (task ownership, which + must never wait on another Hermes process). Locking is mandatory on both + paths: without a kernel lock two processes can each report success while + silently losing one task's sole recovery pointer or forking its sandbox, + so an unsupported platform fails closed. + """ + if _fcntl is not None: + flags = _fcntl.LOCK_EX if blocking else _fcntl.LOCK_EX | _fcntl.LOCK_NB + _fcntl.flock(lock_file.fileno(), flags) + return + if _msvcrt is not None: + # Windows byte-range locks require a real byte at the current file + # position. Concurrent initializers may append more than one byte, + # but every process locks byte zero. + lock_file.seek(0, os.SEEK_END) + if lock_file.tell() == 0: + lock_file.write(" ") + lock_file.flush() + lock_file.seek(0) + _msvcrt.locking( + lock_file.fileno(), + _msvcrt.LK_LOCK if blocking else _msvcrt.LK_NBLCK, + 1, + ) + return + raise RuntimeError( + f"{requirement} requires fcntl or msvcrt cross-process file locking" + ) + + +def _unlock_open_file(lock_file: Any) -> None: + """Best-effort kernel unlock; closing the file stays with the caller.""" try: if _fcntl is not None: - _fcntl.flock( - lock_file.fileno(), - _fcntl.LOCK_EX | _fcntl.LOCK_NB, - ) + _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_UN) elif _msvcrt is not None: - lock_file.seek(0, os.SEEK_END) - if lock_file.tell() == 0: - lock_file.write(" ") - lock_file.flush() lock_file.seek(0) _msvcrt.locking( lock_file.fileno(), - _msvcrt.LK_NBLCK, + _msvcrt.LK_UNLCK, 1, ) - else: - raise RuntimeError( - "Tenki task ownership requires fcntl or msvcrt " - "cross-process file locking" - ) + except OSError: + pass + + +def _acquire_task_ownership_lock(profile_home: Path, task_id: str): + """Acquire one profile/task lifetime lock or fail without remote mutation.""" + lock_path = _task_ownership_lock_path(profile_home, task_id) + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock_file = lock_path.open("a+", encoding="utf-8") + try: + _lock_open_file( + lock_file, + blocking=False, + requirement="Tenki task ownership", + ) except (BlockingIOError, OSError) as exc: lock_file.close() raise RuntimeError( @@ -161,17 +218,7 @@ def _release_task_ownership_lock(lock_file: Any) -> None: if lock_file is None: return try: - if _fcntl is not None: - _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_UN) - elif _msvcrt is not None: - lock_file.seek(0) - _msvcrt.locking( - lock_file.fileno(), - _msvcrt.LK_UNLCK, - 1, - ) - except OSError: - pass + _unlock_open_file(lock_file) finally: lock_file.close() @@ -225,7 +272,7 @@ def _legacy_profile_tokens() -> tuple[str, ...]: def _load_snapshots(store_path: Path | None = None) -> dict: path = store_path or _snapshot_store_path() with _snapshot_store_lock(path): - return _load_json_store(path) + return _load_recovery_registry(path) def _save_snapshots(data: dict, store_path: Path | None = None) -> None: @@ -238,36 +285,6 @@ def _snapshot_platform() -> str: return os.name -def _windows_replace_file_write_through(source: str, destination: Path) -> None: - """Atomically replace *destination* with Windows write-through semantics.""" - import ctypes - from ctypes import wintypes - - move_file_ex = ctypes.WinDLL( - "kernel32", - use_last_error=True, - ).MoveFileExW - move_file_ex.argtypes = ( - wintypes.LPCWSTR, - wintypes.LPCWSTR, - wintypes.DWORD, - ) - move_file_ex.restype = wintypes.BOOL - flags = 0x1 | 0x8 # MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH - if move_file_ex( - os.path.abspath(source), - os.path.abspath(destination), - flags, - ): - return - error_code = ctypes.get_last_error() - raise OSError( - error_code, - ctypes.FormatError(error_code), - str(destination), - ) - - @contextmanager def _snapshot_store_lock(path: Path): """Serialize snapshot-pointer RMW in-process and across processes.""" @@ -280,113 +297,58 @@ def _snapshot_store_lock(path: Path): lock_file = lock_path.open("a+", encoding="utf-8") os_locked = False try: - if _fcntl is not None: - _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_EX) - os_locked = True - elif _msvcrt is not None: - # Windows byte-range locks require a real byte at the current - # file position. Concurrent initializers may append more than - # one byte, but every process locks byte zero. - lock_file.seek(0, os.SEEK_END) - if lock_file.tell() == 0: - lock_file.write(" ") - lock_file.flush() - lock_file.seek(0) - _msvcrt.locking( - lock_file.fileno(), - _msvcrt.LK_LOCK, - 1, - ) - os_locked = True - else: - # Pointer RMW without a kernel lock can report success in two - # processes while silently losing one task's sole recovery - # pointer. Persistent state must fail closed on unsupported - # platforms. - raise RuntimeError( - "Tenki snapshot registry requires fcntl or msvcrt " - "cross-process file locking" - ) + _lock_open_file( + lock_file, + blocking=True, + requirement="Tenki snapshot registry", + ) + os_locked = True yield finally: - if os_locked and _fcntl is not None: - try: - _fcntl.flock(lock_file.fileno(), _fcntl.LOCK_UN) - except OSError: - pass - elif os_locked and _msvcrt is not None: - try: - lock_file.seek(0) - _msvcrt.locking( - lock_file.fileno(), - _msvcrt.LK_UNLCK, - 1, - ) - except OSError: - pass + if os_locked: + _unlock_open_file(lock_file) lock_file.close() def _atomic_save_snapshots(path: Path, data: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temp_path: str | None = None - try: - with tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - suffix=".tmp", - delete=False, - ) as tmp: - import json - - json.dump(data, tmp, indent=2) - tmp.flush() - os.fsync(tmp.fileno()) - temp_path = tmp.name - platform = _snapshot_platform() - if platform == "nt": - try: - _windows_replace_file_write_through(temp_path, path) - except OSError as exc: - raise _SnapshotPointerCommitUncertain( - f"could not durably replace snapshot-pointer file {path}" - ) from exc - return - if platform != "posix": - raise RuntimeError( - "Tenki snapshot registry cannot prove rename durability on " - f"platform {platform!r}" - ) + """Apply Tenki's recovery-pointer policy to the durable write primitive. - os.replace(temp_path, path) - # The file contents were fsynced before replace; fsync the containing - # directory too so the pointer rename itself survives a host crash. - # A POSIX failure is an *uncertain commit*: the visible file is new, - # but a crash may roll the directory entry back. The caller must keep - # both remote snapshots and the live sandbox in that state. - dir_fd: int | None = None - try: - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - dir_fd = os.open(path.parent, flags) - os.fsync(dir_fd) - except OSError as exc: - raise _SnapshotPointerCommitUncertain( - f"could not fsync snapshot-pointer directory {path.parent}" - ) from exc - finally: - if dir_fd is not None: - try: - os.close(dir_fd) - except OSError: - pass - finally: - if temp_path and os.path.exists(temp_path): - try: - os.unlink(temp_path) - except OSError: - pass + The mechanism lives in ``base._atomic_save_json_durable``; this supplies the + Tenki-specific policy: an uncertain commit surfaces as + ``_SnapshotPointerCommitUncertain`` so callers keep both the old and the new + remote snapshot alive, and the platform/Windows-replace hooks stay resolved + from this module so they remain individually substitutable. + """ + _atomic_save_json_durable( + path, + data, + subject="snapshot-pointer", + store_label="Tenki snapshot registry", + commit_uncertain_error=_SnapshotPointerCommitUncertain, + platform=_snapshot_platform(), + replace_write_through=_windows_replace_file_write_through, + ) + + +@contextmanager +def _mutate_store(store_path: Path | None): + """Read-modify-write the recovery registry under its cross-process lock. + + Resolves the active profile's store path, takes the lock, fail-closed loads + the registry, yields it for mutation, and durably republishes it. The save + runs on every normal exit *including an early ``return``* from the body: + several callers deliberately republish unchanged state to re-establish + durability after an uncertain commit. A body that raises skips the save and + releases the lock, leaving the previous durable state visible. + + Callers whose no-op path must NOT write (or that need the save inside their + own ``try``) cannot use this and manage the lock themselves. + """ + path = store_path or _snapshot_store_path() + with _snapshot_store_lock(path): + snapshots = _load_recovery_registry(path) + yield snapshots + _atomic_save_snapshots(path, snapshots) def _snapshot_key(task_id: str) -> str: @@ -409,6 +371,16 @@ def _remote_binding_key(task_id: str) -> str: return f"{_REMOTE_BINDING_NAMESPACE}:{task_id}" +def _record_field(value: Any, field: str) -> Any: + """Read *field* from a registry record, tolerating the bare-string shape. + + Early unpublished builds wrote a plain id string where the current format + writes a dict, and a registry written by one of those builds must still be + comparable rather than silently read as absent. + """ + return value.get(field) if isinstance(value, dict) else value + + def _create_attempt_state( task_id: str, store_path: Path | None = None, @@ -431,29 +403,28 @@ def _get_create_attempt( task_id: str, store_path: Path | None = None, ) -> str | None: + # Test helper: no production caller. Kept as the readable projection the + # suite asserts against. return _create_attempt_state(task_id, store_path)[0] def _remote_binding_state( task_id: str, store_path: Path | None = None, -) -> tuple[ - str | None, - str | None, - bool, - bool, - tuple[str, ...], - bool, -]: +) -> _RemoteBinding: snapshots = _load_snapshots(store_path) value = snapshots.get(_remote_binding_key(task_id)) if isinstance(value, str) and value: # This unversioned shape was used only by unpublished development # candidates. It cannot prove how ownership was established, so never # auto-adopt it as a supported Hermes lineage. - return value, None, False, True, (value,), False + return _RemoteBinding( + remote_id=value, + conflicted=True, + conflict_ids=(value,), + ) if not isinstance(value, dict): - return None, None, False, False, (), False + return _RemoteBinding() remote_id = value.get("remote_id") attempt_id = value.get("attempt_id") conflict_ids = value.get("conflict_ids") @@ -473,13 +444,18 @@ def _remote_binding_state( attempt_id if isinstance(attempt_id, str) and attempt_id else None ) conflicted = bool(value.get("conflicted", False)) - return ( - parsed_remote_id, - parsed_attempt_id, - bool(value.get("validated", False)) and parsed_attempt_id is not None, - conflicted, - parsed_conflict_ids, - bool(value.get("unresolvable", conflicted and not parsed_conflict_ids)), + return _RemoteBinding( + remote_id=parsed_remote_id, + attempt_id=parsed_attempt_id, + validated=( + bool(value.get("validated", False)) + and parsed_attempt_id is not None + ), + conflicted=conflicted, + conflict_ids=parsed_conflict_ids, + unresolvable=bool( + value.get("unresolvable", conflicted and not parsed_conflict_ids) + ), ) @@ -487,7 +463,9 @@ def _get_remote_binding( task_id: str, store_path: Path | None = None, ) -> str | None: - return _remote_binding_state(task_id, store_path)[0] + # Test helper: no production caller. Kept as the readable projection the + # suite asserts against. + return _remote_binding_state(task_id, store_path).remote_id def _begin_create_attempt( @@ -497,17 +475,11 @@ def _begin_create_attempt( store_path: Path | None = None, ) -> None: """Durably journal one unique remote create before issuing its RPC.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: key = _create_attempt_key(task_id) existing = snapshots.get(key) if existing not in (None, attempt_id): - existing_id = ( - existing.get("attempt_id") - if isinstance(existing, dict) - else existing - ) + existing_id = _record_field(existing, "attempt_id") if existing_id != attempt_id: raise _SnapshotPointerConflict( f"task {task_id} already has unresolved create attempt " @@ -517,7 +489,6 @@ def _begin_create_attempt( "attempt_id": attempt_id, "expires_at": expires_at, } - _atomic_save_snapshots(path, snapshots) def _clear_create_attempt( @@ -526,28 +497,21 @@ def _clear_create_attempt( store_path: Path | None = None, ) -> None: """Durably clear exactly the create attempt whose remote is gone.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: key = _create_attempt_key(task_id) existing = snapshots.get(key) if existing is None: # A prior removal may be visible after an uncertain directory/ - # write-through commit. Re-publish that marker-free state so this - # retry establishes durability before ownership can be released. - _atomic_save_snapshots(path, snapshots) + # write-through commit. Returning here still re-publishes that + # marker-free state, so this retry establishes durability before + # ownership can be released. return - existing_id = ( - existing.get("attempt_id") - if isinstance(existing, dict) - else existing - ) + existing_id = _record_field(existing, "attempt_id") if existing_id != attempt_id: raise _SnapshotPointerConflict( f"create attempt advanced from {attempt_id} to {existing_id}" ) snapshots.pop(key, None) - _atomic_save_snapshots(path, snapshots) def _store_remote_binding( @@ -563,22 +527,22 @@ def _store_remote_binding( raise ValueError( "a validated Tenki binding requires its durable create attempt id" ) - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: attempt_key = _create_attempt_key(task_id) - existing_attempt = snapshots.get(attempt_key) - if isinstance(existing_attempt, dict): - existing_attempt = existing_attempt.get("attempt_id") + existing_attempt = _record_field( + snapshots.get(attempt_key), + "attempt_id", + ) if attempt_id is not None and existing_attempt != attempt_id: raise _SnapshotPointerConflict( f"create attempt advanced from {attempt_id} to " f"{existing_attempt}" ) binding_key = _remote_binding_key(task_id) - existing_binding = snapshots.get(binding_key) - if isinstance(existing_binding, dict): - existing_binding = existing_binding.get("remote_id") + existing_binding = _record_field( + snapshots.get(binding_key), + "remote_id", + ) if existing_binding not in (None, remote_id): raise _SnapshotPointerConflict( f"remote binding advanced from {remote_id} to " @@ -593,7 +557,6 @@ def _store_remote_binding( } if attempt_id is not None: snapshots.pop(attempt_key, None) - _atomic_save_snapshots(path, snapshots) def _replace_create_attempt_with_lineage_conflict( @@ -606,15 +569,11 @@ def _replace_create_attempt_with_lineage_conflict( ) -> None: """Atomically replace an uncertain create with a durable conflict.""" ids = sorted(set(remote_ids)) - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: attempt_key = _create_attempt_key(task_id) - existing_attempt = snapshots.get(attempt_key) - existing_attempt_id = ( - existing_attempt.get("attempt_id") - if isinstance(existing_attempt, dict) - else existing_attempt + existing_attempt_id = _record_field( + snapshots.get(attempt_key), + "attempt_id", ) if existing_attempt_id != attempt_id: raise _SnapshotPointerConflict( @@ -635,7 +594,6 @@ def _replace_create_attempt_with_lineage_conflict( "unresolvable": unresolvable, } snapshots.pop(attempt_key, None) - _atomic_save_snapshots(path, snapshots) def _mark_remote_binding_validated( @@ -644,16 +602,10 @@ def _mark_remote_binding_validated( store_path: Path | None = None, ) -> None: """Durably publish positive sole-lineage validation for a bound remote.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: key = _remote_binding_key(task_id) existing = snapshots.get(key) - existing_id = ( - existing.get("remote_id") - if isinstance(existing, dict) - else existing - ) + existing_id = _record_field(existing, "remote_id") if existing_id != remote_id: raise _SnapshotPointerConflict( f"remote binding advanced from {remote_id} to {existing_id}" @@ -668,7 +620,6 @@ def _mark_remote_binding_validated( existing = {"remote_id": remote_id, "attempt_id": None} existing["validated"] = True snapshots[key] = existing - _atomic_save_snapshots(path, snapshots) def _mark_remote_binding_conflicted( @@ -680,16 +631,10 @@ def _mark_remote_binding_conflicted( store_path: Path | None = None, ) -> None: """Permanently remember an observed fork despite later list omissions.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: key = _remote_binding_key(task_id) existing = snapshots.get(key) - existing_id = ( - existing.get("remote_id") - if isinstance(existing, dict) - else existing - ) + existing_id = _record_field(existing, "remote_id") if existing_id != remote_id: raise _SnapshotPointerConflict( f"remote binding advanced from {remote_id} to {existing_id}" @@ -706,7 +651,6 @@ def _mark_remote_binding_conflicted( conflict_record["conflict_ids"] = sorted(set(conflict_ids)) conflict_record["unresolvable"] = unresolvable snapshots[key] = conflict_record - _atomic_save_snapshots(path, snapshots) def _store_unmanaged_lineage_conflict( @@ -718,9 +662,7 @@ def _store_unmanaged_lineage_conflict( ) -> None: """Record visible task-name collisions without claiming their ownership.""" ids = sorted(set(remote_ids)) - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: if snapshots.get(_create_attempt_key(task_id)) is not None: raise _SnapshotPointerConflict( f"task {task_id} acquired a create attempt during collision check" @@ -742,7 +684,6 @@ def _store_unmanaged_lineage_conflict( "conflict_ids": ids, "unresolvable": unresolvable, } - _atomic_save_snapshots(path, snapshots) def _clear_remote_binding( @@ -750,22 +691,19 @@ def _clear_remote_binding( remote_id: str, store_path: Path | None = None, ) -> None: - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: key = _remote_binding_key(task_id) - existing = snapshots.get(key) - if isinstance(existing, dict): - existing = existing.get("remote_id") + existing = _record_field(snapshots.get(key), "remote_id") if existing is None: - _atomic_save_snapshots(path, snapshots) + # Returning still re-publishes this binding-free state, so an + # earlier removal left visible by an uncertain commit becomes + # durable before ownership can be released. return if existing != remote_id: raise _SnapshotPointerConflict( f"remote binding advanced from {remote_id} to {existing}" ) snapshots.pop(key, None) - _atomic_save_snapshots(path, snapshots) def _pending_snapshot_retirements( @@ -784,9 +722,11 @@ def _queue_snapshot_retirement( snapshot_id: str, store_path: Path | None = None, ) -> None: + # Deliberately NOT _mutate_store: an already-tombstoned snapshot must exit + # without writing at all, and that helper always republishes on return. path = store_path or _snapshot_store_path() with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + snapshots = _load_recovery_registry(path) if _snapshot_retired_key(snapshot_id) in snapshots: return snapshots[_snapshot_retirement_key(snapshot_id)] = snapshot_id @@ -799,15 +739,12 @@ def _queue_snapshot_pointer_retirement( store_path: Path | None = None, ) -> None: """Atomically detach one unusable pointer and journal its retirement.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: for key in (_snapshot_key(task_id), task_id): if snapshots.get(key) == snapshot_id: snapshots.pop(key, None) if _snapshot_retired_key(snapshot_id) not in snapshots: snapshots[_snapshot_retirement_key(snapshot_id)] = snapshot_id - _atomic_save_snapshots(path, snapshots) def _confirm_snapshot_store_durable( @@ -832,7 +769,7 @@ def _confirm_snapshot_store_durable( # MOVEFILE_WRITE_THROUGH. This also upgrades state left by an # uncertain prior replacement before a retirement retry can delete # a remote recovery copy. - _atomic_save_snapshots(path, _load_json_store(path)) + _atomic_save_snapshots(path, _load_recovery_registry(path)) return if platform != "posix": raise RuntimeError( @@ -867,37 +804,38 @@ def _store_snapshot( store_path: Path | None = None, ) -> str | None: """Atomically install *snapshot_id* and return its predecessor, if any.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) - key = _snapshot_key(task_id) - if _snapshot_retired_key(snapshot_id) in snapshots: - raise _SnapshotPointerConflict( - f"snapshot {snapshot_id} was already retired" - ) - previous = snapshots.get(key) - if previous is None: - previous = snapshots.get(task_id) - snapshots[key] = snapshot_id - snapshots.pop(task_id, None) - if ( - isinstance(previous, str) - and previous - and previous != snapshot_id - ): - # Journal the predecessor in the same atomic commit as the new - # pointer. A crash or transient delete failure can then retry - # retirement without ever forgetting the old remote snapshot. - snapshots[_snapshot_retirement_key(previous)] = previous - try: - _atomic_save_snapshots(path, snapshots) - except _SnapshotPointerCommitUncertain as exc: - exc.previous_snapshot_id = ( - previous if isinstance(previous, str) and previous else None - ) - exc.new_snapshot_id = snapshot_id - raise - return previous if isinstance(previous, str) and previous else None + previous: Any = None + # _mutate_store commits on exit from the ``with`` block, so the annotating + # handler has to wrap the block itself. Only the commit can raise this + # type, so the wider span does not widen what is caught. + try: + with _mutate_store(store_path) as snapshots: + key = _snapshot_key(task_id) + if _snapshot_retired_key(snapshot_id) in snapshots: + raise _SnapshotPointerConflict( + f"snapshot {snapshot_id} was already retired" + ) + previous = snapshots.get(key) + if previous is None: + previous = snapshots.get(task_id) + snapshots[key] = snapshot_id + snapshots.pop(task_id, None) + if ( + isinstance(previous, str) + and previous + and previous != snapshot_id + ): + # Journal the predecessor in the same atomic commit as the new + # pointer. A crash or transient delete failure can then retry + # retirement without ever forgetting the old remote snapshot. + snapshots[_snapshot_retirement_key(previous)] = previous + except _SnapshotPointerCommitUncertain as exc: + exc.previous_snapshot_id = ( + previous if isinstance(previous, str) and previous else None + ) + exc.new_snapshot_id = snapshot_id + raise + return previous if isinstance(previous, str) and previous else None def _migrate_snapshot_pointer( @@ -907,9 +845,7 @@ def _migrate_snapshot_pointer( store_path: Path | None = None, ) -> None: """CAS-migrate one legacy pointer without overwriting newer state.""" - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + with _mutate_store(store_path) as snapshots: if _snapshot_retired_key(snapshot_id) in snapshots: raise _SnapshotPointerConflict( f"legacy snapshot {snapshot_id} was already retired" @@ -941,7 +877,6 @@ def _migrate_snapshot_pointer( # Same-task legacy format is the plain task key. if snapshots.get(task_id) == snapshot_id: snapshots.pop(task_id, None) - _atomic_save_snapshots(path, snapshots) def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]: @@ -1066,69 +1001,6 @@ def _tenki_resource_value( return None -def _rewrite_sudo_noninteractive(command: str) -> tuple[str, int]: - """Add ``-n`` to real sudo invocations so Tenki never prompts.""" - from tools.terminal_tool import _looks_like_env_assignment, _read_shell_token - - out: list[str] = [] - i = 0 - n = len(command) - command_start = True - sudo_count = 0 - - while i < n: - ch = command[i] - - if ch.isspace(): - out.append(ch) - if ch == "\n": - command_start = True - i += 1 - continue - - if ch == "#" and command_start: - comment_end = command.find("\n", i) - if comment_end == -1: - out.append(command[i:]) - break - out.append(command[i:comment_end]) - i = comment_end - continue - - if command.startswith("&&", i) or command.startswith("||", i) or command.startswith(";;", i): - out.append(command[i:i + 2]) - i += 2 - command_start = True - continue - - if ch in ";|&(": - out.append(ch) - i += 1 - command_start = True - continue - - if ch == ")": - out.append(ch) - i += 1 - command_start = False - continue - - token, next_i = _read_shell_token(command, i) - if command_start and token == "sudo": - out.append("sudo -n") - sudo_count += 1 - else: - out.append(token) - - if command_start and _looks_like_env_assignment(token): - command_start = True - else: - command_start = False - i = next_i - - return "".join(out), sudo_count - - class TenkiEnvironment(BaseEnvironment): """Tenki sandbox backend. @@ -1182,12 +1054,7 @@ def __init__( self._create_outcome_uncertain = False self._create_attempt_id: str | None = None self._create_attempt_expires_at: float | None = None - self._remote_binding_id: str | None = None - self._remote_binding_attempt_id: str | None = None - self._remote_binding_validated = False - self._remote_binding_conflicted = False - self._remote_binding_conflict_ids: tuple[str, ...] = () - self._remote_binding_unresolvable = False + self._remote_binding = _RemoteBinding() self._create_lineage_ambiguous = False self._lock = threading.Lock() self._lifecycle_condition = threading.Condition(self._lock) @@ -1263,20 +1130,13 @@ def __init__( self._task_id, self._snapshot_store, ) - ( - self._remote_binding_id, - self._remote_binding_attempt_id, - self._remote_binding_validated, - self._remote_binding_conflicted, - self._remote_binding_conflict_ids, - self._remote_binding_unresolvable, - ) = _remote_binding_state( + self._remote_binding = _remote_binding_state( self._task_id, self._snapshot_store, ) if self._create_attempt_id is not None and ( - self._remote_binding_id is not None - or self._remote_binding_conflicted + self._remote_binding.remote_id is not None + or self._remote_binding.conflicted ): raise RuntimeError( "Tenki task has both an unresolved create and a remote " @@ -1611,6 +1471,23 @@ def _list_kwargs(self, client: Any) -> dict[str, Any]: kwargs["workspace_id"] = self._workspace_id return kwargs + def _fail_ambiguous_lineage( + self, + message: str, + cause: BaseException | None = None, + ) -> NoReturn: + """Mark this task's lineage unusable and fail closed in one step. + + The flag and the raise are inseparable: a raise that forgot the flag + would let ``_abort_failed_initialization`` release task ownership (and + ``cleanup`` terminate a sandbox) while two remote lineages may still be + live. Passing *cause* preserves explicit exception chaining. + """ + self._create_lineage_ambiguous = True + if cause is not None: + raise RuntimeError(message) from cause + raise RuntimeError(message) + def _assert_no_unmanaged_persistent_sandbox(self) -> None: """Refuse visible name collisions that have no local ownership state. @@ -1658,20 +1535,17 @@ def _assert_no_unmanaged_persistent_sandbox(self) -> None: except BaseException: self._create_lineage_ambiguous = True raise - self._remote_binding_id = ( - None - if unidentified_collision - else sorted(set(remote_ids))[0] - ) - self._remote_binding_attempt_id = None - self._remote_binding_validated = False - self._remote_binding_conflicted = True - self._remote_binding_conflict_ids = ( - tuple(sorted(set(remote_ids))) + self._remote_binding = _RemoteBinding( + remote_id=( + None + if unidentified_collision + else sorted(set(remote_ids))[0] + ), + conflicted=True, + conflict_ids=tuple(sorted(set(remote_ids))), + unresolvable=unidentified_collision, ) - self._remote_binding_unresolvable = unidentified_collision - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki found an unmanaged persistent sandbox collision for task " f"{self._task_id}; terminate the reported remote ids before retrying" ) @@ -1735,8 +1609,8 @@ def _ensure_sandbox(self) -> None: if self._cleanup_in_progress or self._cleanup_complete: raise RuntimeError("Tenki cleanup is in progress") if ( - self._remote_binding_id is not None - or self._remote_binding_conflicted + self._remote_binding.remote_id is not None + or self._remote_binding.conflicted ): self._resolve_remote_binding() if self._create_outcome_uncertain: @@ -1747,7 +1621,7 @@ def _ensure_sandbox(self) -> None: ) if self._sandbox is not None: sandbox = self._sandbox - if not self._remote_binding_validated: + if not self._remote_binding.validated: self._validate_created_lineage() if self._ensure_sandbox_ready(sandbox): self._after_sandbox_ownership_confirmed() @@ -1908,7 +1782,7 @@ def _create_sandbox_from_kwargs(self, kwargs: dict[str, Any]): def _validate_created_lineage(self) -> None: """Validate exact ownership; use list only to detect visible conflicts.""" sandbox = self._sandbox - expected_attempt_id = self._remote_binding_attempt_id + expected_attempt_id = self._remote_binding.attempt_id remote_id = self._sandbox_identity(sandbox) client = self._client get_sandbox = getattr(client, "get", None) @@ -1916,7 +1790,7 @@ def _validate_created_lineage(self) -> None: if ( expected_attempt_id is None or remote_id is None - or remote_id != self._remote_binding_id + or remote_id != self._remote_binding.remote_id or not self._sandbox_has_owned_identity( sandbox, expected_attempt_id, @@ -1924,8 +1798,7 @@ def _validate_created_lineage(self) -> None: or not callable(get_sandbox) or not callable(list_sandboxes) ): - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( f"Tenki cannot validate the new lineage for task {self._task_id}" ) try: @@ -1984,19 +1857,19 @@ def _validate_created_lineage(self) -> None: store_path=self._snapshot_store, ) except BaseException as exc: - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki detected multiple lineages but could not durably " - f"record the conflict for task {self._task_id}" - ) from exc - self._remote_binding_validated = False - self._remote_binding_conflicted = True - self._remote_binding_conflict_ids = tuple( - sorted(set(conflict_ids)) + f"record the conflict for task {self._task_id}", + exc, + ) + self._remote_binding = replace( + self._remote_binding, + validated=False, + conflicted=True, + conflict_ids=tuple(sorted(set(conflict_ids))), + unresolvable=unresolvable, ) - self._remote_binding_unresolvable = unresolvable - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki detected multiple active sandbox lineages for task " f"{self._task_id}; leaving every lineage untouched" ) @@ -2011,7 +1884,7 @@ def _validate_created_lineage(self) -> None: "Tenki could not durably record sole-lineage validation for " f"task {self._task_id}" ) from exc - self._remote_binding_validated = True + self._remote_binding = replace(self._remote_binding, validated=True) def _wait_created_sandbox_ready(self, sandbox: Any) -> None: """Wait only after the exact newly created Sandbox handle is owned.""" @@ -2299,7 +2172,7 @@ def _retire_pending_snapshot_if_unreferenced( """Reference-check, remote-delete, and tombstone under one store lock.""" path = self._snapshot_store with _snapshot_store_lock(path): - snapshots = _load_json_store(path) + snapshots = _load_recovery_registry(path) pending_key = _snapshot_retirement_key(snapshot_id) if snapshots.get(pending_key) != snapshot_id: return True @@ -2578,7 +2451,9 @@ def _prepare_command(self, command: str | None) -> tuple[str | None, str | None] # Tenki sandboxes should rely on their own sudoers policy. Do not ask # the user for a host sudo password, and do not send SUDO_PASSWORD to a # remote cloud sandbox. The default Tenki image supports NOPASSWD sudo. - transformed, sudo_count = _rewrite_sudo_noninteractive(command) + from tools.terminal_tool import _rewrite_sudo_command_words + + transformed, sudo_count = _rewrite_sudo_command_words(command, "sudo -n") if sudo_count == 0: return command, None if self._sudo_nopasswd_works(): @@ -2741,19 +2616,20 @@ def _bind_remote_sandbox( exc, ) return False - self._remote_binding_id = remote_id - self._remote_binding_attempt_id = attempt_id - self._remote_binding_validated = validated - self._remote_binding_conflicted = False - self._remote_binding_conflict_ids = () - self._remote_binding_unresolvable = False + self._remote_binding = _RemoteBinding( + remote_id=remote_id, + attempt_id=attempt_id, + validated=validated, + ) if attempt_id is not None: self._create_attempt_id = None self._create_attempt_expires_at = None return True def _clear_remote_binding_marker(self, sandbox: Any) -> bool: - remote_id = self._remote_binding_id or self._sandbox_identity(sandbox) + remote_id = ( + self._remote_binding.remote_id or self._sandbox_identity(sandbox) + ) if remote_id is None: return True try: @@ -2771,12 +2647,7 @@ def _clear_remote_binding_marker(self, sandbox: Any) -> bool: exc, ) return False - self._remote_binding_id = None - self._remote_binding_attempt_id = None - self._remote_binding_validated = False - self._remote_binding_conflicted = False - self._remote_binding_conflict_ids = () - self._remote_binding_unresolvable = False + self._remote_binding = _RemoteBinding() return True @staticmethod @@ -2795,63 +2666,54 @@ def _remote_definitively_absent(exc: BaseException) -> bool: def _resolve_remote_binding(self) -> None: """Resolve a durable task binding through authoritative Client.get.""" - remote_id = self._remote_binding_id + binding = self._remote_binding + remote_id = binding.remote_id if ( - (remote_id is None and not self._remote_binding_conflicted) + (remote_id is None and not binding.conflicted) or self._sandbox is not None ): return - if self._remote_binding_conflicted: - if ( - not self._remote_binding_conflict_ids - and not self._remote_binding_unresolvable - ): - self._create_lineage_ambiguous = True - raise RuntimeError( + if binding.conflicted: + if not binding.conflict_ids and not binding.unresolvable: + self._fail_ambiguous_lineage( "Tenki task has a malformed durable persistent-lineage " f"conflict for {self._task_id}" ) client = self._create_client() active_ids: list[str] = [] - for conflict_id in self._remote_binding_conflict_ids: + for conflict_id in binding.conflict_ids: try: candidate = client.get(conflict_id) except BaseException as exc: if self._remote_definitively_absent(exc): continue - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki could not resolve every remote in the durable " - f"lineage conflict for task {self._task_id}" - ) from exc + f"lineage conflict for task {self._task_id}", + exc, + ) if self._sandbox_identity(candidate) != conflict_id: - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki conflict lookup returned a different sandbox id" ) if self._sandbox_state(candidate) not in self._terminal_states: active_ids.append(conflict_id) if active_ids: - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki task has a durable persistent-lineage conflict; " f"active remote ids for {self._task_id}: " + ", ".join(active_ids) ) - if self._remote_binding_unresolvable: - self._create_lineage_ambiguous = True - known_ids = ", ".join( - self._remote_binding_conflict_ids - ) or "" - raise RuntimeError( + if binding.unresolvable: + known_ids = ", ".join(binding.conflict_ids) or "" + self._fail_ambiguous_lineage( "Tenki task has an unresolvable durable " f"persistent-lineage conflict for {self._task_id}; " f"known remote ids: {known_ids}; manual remote cleanup " "and local conflict removal are required" ) if not self._clear_remote_binding_marker(None): - self._create_lineage_ambiguous = True - raise RuntimeError( + self._fail_ambiguous_lineage( "Tenki could not clear a fully terminated lineage conflict " f"for task {self._task_id}" ) @@ -2889,7 +2751,7 @@ def _resolve_remote_binding(self) -> None: return if self._persistent: self._sandbox = sandbox - if not self._remote_binding_validated: + if not self._remote_binding.validated: self._validate_created_lineage() return @@ -2989,12 +2851,13 @@ def _persist_create_attempt_conflict( self._create_attempt_id = None self._create_attempt_expires_at = None self._create_outcome_uncertain = False - self._remote_binding_id = ids[0] if ids else None - self._remote_binding_attempt_id = attempt_id - self._remote_binding_validated = False - self._remote_binding_conflicted = True - self._remote_binding_conflict_ids = ids - self._remote_binding_unresolvable = unresolvable + self._remote_binding = _RemoteBinding( + remote_id=ids[0] if ids else None, + attempt_id=attempt_id, + conflicted=True, + conflict_ids=ids, + unresolvable=unresolvable, + ) self._create_lineage_ambiguous = True return True @@ -3353,7 +3216,7 @@ def cleanup(self, *, discard: bool = False): f"exist for task {self._task_id}" ) if sandbox is None: - if self._remote_binding_id is not None: + if self._remote_binding.remote_id is not None: self._resolve_remote_binding() sandbox = self._sandbox client = self._client From 506981091f420e2a5838aacb6c94b7e5e25e7b4a Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 29 Jul 2026 14:47:06 -0700 Subject: [PATCH 12/12] refactor(tools): quality pass over the Tenki backend Follow-up cleanups from a four-angle review (reuse, simplification, efficiency, altitude) of this branch. No behaviour change. - Delete dead code: TenkiEnvironment._max_duration (written, never read), _save_snapshots (no callers, and it bypassed _mutate_store's read-modify-write discipline while looking like the sanctioned write path), a _stdin_mode override restating BaseEnvironment's default, and the unused explicit/key parameters in tenki_config. - Hoist _normalize_forward_env_names into file_sync as normalize_forward_env_names, taking a setting_name for the warning text. docker and tenki carried verbatim copies of it. Rendered warnings are byte-identical to before. - Add TenkiEnvironment._dispose_remote for the repeated "walk the sandbox disposal methods, retrying each" block. Two of the five sites adopt it. The other three either clear the remote-binding marker inside the walk (so a failed clear deliberately falls through to the next method) or log once per failing method, and sharing them would change behaviour. Each exclusion is commented at the site. - Make base._atomic_save_json_durable's five keyword parameters required. Its single caller passed all five explicitly, so the defaults and their coalescing were unreachable. - Precompute the expected sandbox-name map in __init__. _sandbox_matches_task ran a regex substitution per profile/task candidate per listed sandbox -- measured around 205 per environment init against a 50-sandbox workspace listing, which is itself scanned up to three times per create. Now a dict lookup. Co-Authored-By: Claude Opus 5 (1M context) --- tools/environments/base.py | 21 ++--- tools/environments/docker.py | 31 ++----- tools/environments/file_sync.py | 37 ++++++++ tools/environments/tenki.py | 150 +++++++++++++++++--------------- tools/tenki_config.py | 7 +- 5 files changed, 135 insertions(+), 111 deletions(-) diff --git a/tools/environments/base.py b/tools/environments/base.py index cc13112711321..69708cdf62dc3 100644 --- a/tools/environments/base.py +++ b/tools/environments/base.py @@ -307,11 +307,11 @@ def _atomic_save_json_durable( path: Path, data: dict, *, - subject: str = "file", - store_label: str = "This JSON store", - commit_uncertain_error: Callable[[str], BaseException] = OSError, - platform: str | None = None, - replace_write_through: Callable[[str, Path], None] | None = None, + subject: str, + store_label: str, + commit_uncertain_error: Callable[[str], BaseException], + platform: str, + replace_write_through: Callable[[str, Path], None], ) -> None: """Write *data* to *path* as JSON with a crash-durable atomic commit. @@ -323,12 +323,13 @@ def _atomic_save_json_durable( *commit_uncertain_error* so a caller with recovery state can keep both the old and the new referents alive. - *platform* and *replace_write_through* exist so a caller can substitute its - own (testable) platform detection and Windows replace implementation. + Every policy input is required rather than defaulted: a caller that reaches + for this primitive is one with recovery state to protect, and silently + inheriting a generic uncertain-commit error or this module's own platform + detection would hide exactly the substitutions such a caller depends on. + *platform* and *replace_write_through* in particular let it supply its own + (testable) platform detection and Windows replace implementation. """ - platform = platform if platform is not None else os.name - if replace_write_through is None: - replace_write_through = _windows_replace_file_write_through path.parent.mkdir(parents=True, exist_ok=True) temp_path: str | None = None try: diff --git a/tools/environments/docker.py b/tools/environments/docker.py index def1ac157f197..8aa322a73890b 100644 --- a/tools/environments/docker.py +++ b/tools/environments/docker.py @@ -18,6 +18,7 @@ from typing import Optional from tools.environments.base import BaseEnvironment, _popen_bash +from tools.environments.file_sync import normalize_forward_env_names from tools.environments.local import ( _HERMES_PROVIDER_ENV_BLOCKLIST, _is_hermes_internal_secret, @@ -40,31 +41,6 @@ _EGRESS_LABEL_KEY = "hermes-egress" -def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]: - """Return a deduplicated list of valid environment variable names.""" - normalized: list[str] = [] - seen: set[str] = set() - - for item in forward_env or []: - if not isinstance(item, str): - logger.warning("Ignoring non-string docker_forward_env entry: %r", item) - continue - - key = item.strip() - if not key: - continue - if not _ENV_VAR_NAME_RE.match(key): - logger.warning("Ignoring invalid docker_forward_env entry: %r", item) - continue - if key in seen: - continue - - seen.add(key) - normalized.append(key) - - return normalized - - def _normalize_env_dict(env: dict | None) -> dict[str, str]: """Validate and normalize a docker_env dict to {str: str}. @@ -846,7 +822,10 @@ def __init__( self._persistent = persistent_filesystem self._persist_across_processes = persist_across_processes self._task_id = task_id - self._forward_env = _normalize_forward_env_names(forward_env) + self._forward_env = normalize_forward_env_names( + forward_env, + setting_name="docker_forward_env", + ) self._env = _normalize_env_dict(env) self._container_id: Optional[str] = None self._labels: dict[str, str] = {} diff --git a/tools/environments/file_sync.py b/tools/environments/file_sync.py index 2ae9cc5fd1621..26b504d19ed19 100644 --- a/tools/environments/file_sync.py +++ b/tools/environments/file_sync.py @@ -10,6 +10,7 @@ import logging import os import posixpath +import re import shlex import shutil import signal @@ -42,6 +43,42 @@ _SYNC_INTERVAL_SECONDS = 5.0 _FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC" +_ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def normalize_forward_env_names( + forward_env: list[str] | None, + *, + setting_name: str, +) -> list[str]: + """Return a deduplicated list of valid environment variable names. + + *setting_name* names the backend setting being validated (e.g. + ``docker_forward_env``) and only appears in the warning text. + """ + normalized: list[str] = [] + seen: set[str] = set() + + for item in forward_env or []: + if not isinstance(item, str): + logger.warning("Ignoring non-string %s entry: %r", setting_name, item) + continue + + key = item.strip() + if not key: + continue + if not _ENV_VAR_NAME_RE.match(key): + logger.warning("Ignoring invalid %s entry: %r", setting_name, item) + continue + if key in seen: + continue + + seen.add(key) + normalized.append(key) + + return normalized + + # Transport callbacks provided by each backend UploadFn = Callable[[str, str], None] # (host_path, remote_path) -> raises on failure BulkUploadFn = Callable[[list[tuple[str, str]]], None] # [(host_path, remote_path), ...] -> raises on failure diff --git a/tools/environments/tenki.py b/tools/environments/tenki.py index b731b9e5ddac1..16ae9c18d4bf5 100644 --- a/tools/environments/tenki.py +++ b/tools/environments/tenki.py @@ -32,6 +32,7 @@ FileSyncManager, _credential_host_paths, iter_sync_files, + normalize_forward_env_names, quoted_mkdir_command, quoted_rm_command, unique_parent_dirs, @@ -49,7 +50,6 @@ _CREATE_ATTEMPT_NAMESPACE = "create-attempt" _REMOTE_BINDING_NAMESPACE = "remote-binding" _CREATE_ATTEMPT_EXPIRY_GRACE = 3600 -_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _TENKI_CPU_RANGE = (1, 16) _TENKI_MEMORY_MB_RANGE = (128, 65_536) _TENKI_DISK_GB_RANGE = (5, 100) @@ -275,12 +275,6 @@ def _load_snapshots(store_path: Path | None = None) -> dict: return _load_recovery_registry(path) -def _save_snapshots(data: dict, store_path: Path | None = None) -> None: - path = store_path or _snapshot_store_path() - with _snapshot_store_lock(path): - _atomic_save_snapshots(path, data) - - def _snapshot_platform() -> str: return os.name @@ -879,25 +873,6 @@ def _migrate_snapshot_pointer( snapshots.pop(task_id, None) -def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]: - normalized: list[str] = [] - seen: set[str] = set() - for item in forward_env or []: - if not isinstance(item, str): - logger.warning("Ignoring non-string tenki_forward_env entry: %r", item) - continue - name = item.strip() - if not name: - continue - if not _ENV_NAME_RE.match(name): - logger.warning("Ignoring invalid tenki_forward_env entry: %r", item) - continue - if name not in seen: - normalized.append(name) - seen.add(name) - return normalized - - def _safe_name(value: str, *, fallback: str = "default", max_len: int = 48) -> str: safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", value or "").strip("-._") return (safe or fallback)[:max_len] @@ -1009,7 +984,6 @@ class TenkiEnvironment(BaseEnvironment): ``_ThreadedProcessHandle``. """ - _stdin_mode = "pipe" _snapshot_timeout = 60 _terminal_states = frozenset({"TERMINATING", "TERMINATED", "DELETED", "FAILED", "ERROR"}) @@ -1106,13 +1080,33 @@ def __init__( self._workspace_id = resolve_tenki_workspace_id(workspace_id) self._auth_token = resolve_tenki_auth_token() self._name_prefix = _safe_name(name_prefix, fallback="hermes", max_len=28) + # Every name this wrapper will ever answer to is fixed here: the prefix, + # the profile token, and the task id never change after construction. + # Precomputing turns the per-row match in _sandbox_matches_task into a + # dict lookup — a workspace listing is scanned up to three times per + # create, and re-deriving (and re-sanitizing) one name per legacy + # profile candidate per listed sandbox was hundreds of regex substs per + # environment init. First candidate wins on a name collision, matching + # the previous first-match-wins scan order. + self._canonical_sandbox_name = self._sandbox_name_for( + self._profile_token, + self._task_id, + ) + self._expected_sandbox_names: dict[str, tuple[str, str]] = {} + for candidate_identity in self._profile_task_candidates: + self._expected_sandbox_names.setdefault( + self._sandbox_name_for(*candidate_identity), + candidate_identity, + ) self._allow_inbound = allow_inbound self._allow_outbound = allow_outbound - self._max_duration = max_duration self._effective_max_duration = _positive_float(max_duration) or 3600 self._idle_timeout = idle_timeout self._pause_retention = pause_retention - self._forward_env = _normalize_forward_env_names(forward_env) + self._forward_env = normalize_forward_env_names( + forward_env, + setting_name="tenki_forward_env", + ) self._remote_home = "/home/tenki" # Hold this profile/task lock for the wrapper's full lifetime. It is # acquired before pointer discovery and remote listing/creation, so two @@ -1390,7 +1384,7 @@ def _create_client(self): def _sandbox_name(self) -> str: # The profile token namespaces the name so two profiles sharing one # Tenki account never collide on a name or reuse each other's sandbox. - return self._sandbox_name_for(self._profile_token, self._task_id) + return self._canonical_sandbox_name def _sandbox_name_for(self, profile_token: str, task_id: str) -> str: return f"{self._name_prefix}-{profile_token}-{_safe_name(task_id)}" @@ -1410,13 +1404,13 @@ def _sandbox_matches_task(self, sandbox: Any) -> bool: info = getattr(sandbox, "info", None) if not name and info is not None: name = getattr(info, "name", "") - matched_identity = next( - ( - (profile_token, task_id) - for profile_token, task_id in self._profile_task_candidates - if name == self._sandbox_name_for(profile_token, task_id) - ), - None, + # ``isinstance`` guard, not a behavior change: a non-``str`` name could + # never equal one of the generated names under the old scan either, and + # an unhashable one would raise inside the lookup. + matched_identity = ( + self._expected_sandbox_names.get(name) + if isinstance(name, str) + else None ) if matched_identity is None: return False @@ -1642,7 +1636,7 @@ def _ensure_sandbox(self) -> None: self._validate_created_lineage() self._wait_created_sandbox_ready(self._sandbox) self._after_sandbox_ownership_confirmed() - sandbox_id = getattr(self._sandbox, "id", None) or getattr(self._sandbox, "sandbox_id", None) + sandbox_id = self._sandbox_identity(self._sandbox) logger.info("Tenki: created sandbox %s for task %s", sandbox_id or "", self._task_id) def _after_sandbox_ownership_confirmed(self) -> None: @@ -2755,6 +2749,9 @@ def _resolve_remote_binding(self) -> None: self._validate_created_lineage() return + # Not shared with _dispose_remote: the binding clear runs inside the + # walk and gates it, so a successful terminate() whose clear failed + # deliberately falls through to close() for a second clear attempt. last_exc: BaseException | None = None for method_name in ("terminate", "close"): method = getattr(sandbox, method_name, None) @@ -2861,6 +2858,42 @@ def _persist_create_attempt_conflict( self._create_lineage_ambiguous = True return True + @staticmethod + def _dispose_remote( + sandbox: Any, + method_names: tuple[str, ...], + *, + delays: tuple[float, ...] = _TERMINATE_RETRY_DELAYS, + ) -> tuple[bool, BaseException | None]: + """Try each supported disposal method until one returns cleanly. + + Walks *method_names* in order, skipping names the SDK object does not + expose, and retries each one over *delays* before moving on. Returns + ``(disposed, last_exception)`` instead of raising so every caller keeps + its own failure policy — some log and continue, some quarantine + ownership. ``BaseException`` is caught deliberately: a disposal that + dies on ``KeyboardInterrupt`` must still let the caller decide whether + the remote is safe, and the exception is handed back rather than + swallowed. + + Callers that must run extra work (a durable binding clear) *inside* the + walk cannot use this — see ``cleanup`` and ``_resolve_remote_binding``. + """ + last_exc: BaseException | None = None + for method_name in method_names: + method = getattr(sandbox, method_name, None) + if not callable(method): + continue + for attempt in range(len(delays) + 1): + try: + method() + return True, last_exc + except BaseException as exc: + last_exc = exc + if attempt < len(delays): + time.sleep(delays[attempt]) + return False, last_exc + def _reconcile_uncertain_create(self) -> bool: """Resolve exactly one durable create attempt without touching siblings.""" attempt_id = self._create_attempt_id @@ -3028,23 +3061,7 @@ def _reconcile_uncertain_create(self) -> bool: self._create_outcome_uncertain = False return True - disposed = False - last_exc: BaseException | None = None - for method_name in ("terminate", "close"): - method = getattr(sandbox, method_name, None) - if not callable(method): - continue - for attempt in range(len(_TERMINATE_RETRY_DELAYS) + 1): - try: - method() - disposed = True - break - except BaseException as exc: - last_exc = exc - if attempt < len(_TERMINATE_RETRY_DELAYS): - time.sleep(_TERMINATE_RETRY_DELAYS[attempt]) - if disposed: - break + disposed, last_exc = self._dispose_remote(sandbox, ("terminate", "close")) if not disposed: logger.error( "Tenki: could not dispose exact uncertain create %s for task " @@ -3087,21 +3104,10 @@ def _abort_failed_initialization(self) -> None: if self._persistent else ("terminate", "close") ) - for method_name in method_names: - method = getattr(sandbox, method_name, None) - if not callable(method): - continue - for attempt in range(len(_TERMINATE_RETRY_DELAYS) + 1): - try: - method() - remote_safe = True - break - except BaseException as exc: - last_exc = exc - if attempt < len(_TERMINATE_RETRY_DELAYS): - time.sleep(_TERMINATE_RETRY_DELAYS[attempt]) - if remote_safe: - break + remote_safe, last_exc = self._dispose_remote( + sandbox, + method_names, + ) except BaseException as exc: # Rollback itself must never obscure the constructor failure or # release ownership in an unknown remote state. @@ -3271,6 +3277,10 @@ def cleanup(self, *, discard: bool = False): cleanup_succeeded = True return + # Not shared with _dispose_remote: the durable binding clear runs + # INSIDE the retry body, so a failed clear deliberately re-invokes + # terminate() on the next attempt. Hoisting the clear out of the + # loop would drop that retry. for method_name in ("terminate", "close"): method = getattr(sandbox, method_name, None) if not callable(method): diff --git a/tools/tenki_config.py b/tools/tenki_config.py index d624cb0508390..bfc08bcf56adb 100644 --- a/tools/tenki_config.py +++ b/tools/tenki_config.py @@ -90,7 +90,7 @@ def _first_string(data: dict[str, Any], keys: tuple[str, ...]) -> str: return "" -def _normalize_cli_auth_token(secret: str, key: str = "") -> str: +def _normalize_cli_auth_token(secret: str, key: str) -> str: """Return a Tenki SDK-compatible auth token from Tenki CLI config. Tenki CLI v0.6 stores its browser session cookie as a bare ``auth_token``. @@ -162,7 +162,7 @@ def resolve_tenki_workspace_id(explicit: str = "") -> str: return _first_string(load_tenki_cli_config(), ("current_workspace_id", "workspace_id", "workspace")) -def resolve_tenki_auth_token(explicit: str = "") -> str: +def resolve_tenki_auth_token() -> str: """Resolve a Tenki auth token/API key without logging or persisting it. Reads are profile-scope-aware (see :func:`_scoped_env`): under a @@ -170,9 +170,6 @@ def resolve_tenki_auth_token(explicit: str = "") -> str: machine ``tenki login`` credential is consulted only when no profile scope is authoritative. """ - explicit = _string(explicit) - if explicit: - return explicit for env_name in ("TENKI_AUTH_TOKEN", "TENKI_API_KEY"): value = _scoped_env(env_name) if value: