Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions orchestrator/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,16 +420,23 @@ def cmd_pipelines_list(args: argparse.Namespace) -> int:

def cmd_pipelines_create(args: argparse.Namespace) -> int:
"""Create a new pipeline."""
from routes.pipelines import _ensure_pipeline_work_ref
from state_store import get_state_store

repo_path = Path(args.repo_path) if args.repo_path else Path.cwd()
store = get_state_store(repo_path)

# Route through the same normalisation as the HTTP `create_pipeline`
# endpoint so a CLI-provisioned pipeline gets the ``<branch>/work``
# shape and slice integration branches at ``<branch>/slice-N`` can
# coexist as siblings (#2399).
branch = _ensure_pipeline_work_ref(args.branch or f"egg/issue-{args.issue}")

try:
pipeline = store.create_pipeline(
issue_number=args.issue,
repo=args.repo,
branch=args.branch or f"egg/issue-{args.issue}",
branch=branch,
)

if args.json:
Expand Down Expand Up @@ -956,7 +963,10 @@ def create_parser() -> argparse.ArgumentParser:
create_parser = pipelines_subparsers.add_parser("create", help="Create a pipeline")
create_parser.add_argument("--issue", type=int, required=True, help="Issue number")
create_parser.add_argument("--repo", required=True, help="Repository (owner/repo)")
create_parser.add_argument("--branch", help="Branch name (default: egg/issue-N)")
create_parser.add_argument(
"--branch",
help="Branch name (default: egg/issue-N; normalised to egg/issue-N/work)",
)
create_parser.add_argument("--repo-path", help="Repository path")
create_parser.add_argument("--json", action="store_true", help="Output as JSON")
create_parser.set_defaults(func=cmd_pipelines_create)
Expand Down
31 changes: 26 additions & 5 deletions orchestrator/concurrent_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,20 @@ def get_worktree_branch(
# cannot see in the slice PR's diff. We honour the
# pipeline's existing branch as the issue prefix when set,
# otherwise fall back to the issue-number / pipeline id.
#
# The pipeline tip is pushed to ``egg/<id>/work`` (#2399), so
# the slice integration branch lives as a sibling of ``/work``
# under ``egg/<id>/`` — strip the trailing ``/work`` from the
# pipeline branch to get the namespace root.
issue = self.pipeline.issue_number or self.pipeline.id
issue_branch = self.pipeline.branch or f"egg/issue-{issue}"
# Structural check (≥2 slashes, last segment ``work``) — see
# ``_slice_namespace_root`` in ``routes/pipelines.py`` for
# the matching helper. A degenerate single-segment input
# like ``egg/work`` is treated as the root itself rather
# than collapsing to ``egg``.
if issue_branch.count("/") >= 2 and issue_branch.rsplit("/", 1)[1] == "work":
issue_branch = issue_branch.rsplit("/", 1)[0]
normalised_slice = slice_id if slice_id.startswith("slice-") else f"slice-{slice_id}"
# Defense-in-depth: re-validate the normalised slice id
# shape before embedding it in a git ref. The contract-
Expand All @@ -298,17 +310,26 @@ def get_worktree_branch(
def get_slice_integration_branch(self, slice_id: str) -> str:
"""Return the shared integration branch for a slice's BRC.

Each slice has its own integration branch under the pipeline
branch — ``egg/issue-N/slice-M`` — that the per-role work
branches rebase onto. Roots base off the pipeline branch
directly; child slices base off their parent slice's
integration branch.
Each slice has its own integration branch as a sibling of the
pipeline tip under ``egg/<id>/`` — ``egg/issue-N/slice-M`` —
that the per-role work branches rebase onto. Roots base off the
pipeline branch directly (``egg/issue-N/work``); child slices
base off their parent slice's integration branch.

The pipeline tip is pushed to ``egg/<id>/work`` (#2399), so the
slice integration branch lives as a sibling of ``/work`` under
``egg/<id>/`` — strip the trailing ``/work`` from the pipeline
branch to get the namespace root.

The slice id is regex-validated for defense-in-depth (see
``get_worktree_branch``).
"""
issue = self.pipeline.issue_number or self.pipeline.id
issue_branch = self.pipeline.branch or f"egg/issue-{issue}"
# Structural check (≥2 slashes, last segment ``work``) — see
# ``_slice_namespace_root`` in ``routes/pipelines.py``.
if issue_branch.count("/") >= 2 and issue_branch.rsplit("/", 1)[1] == "work":
issue_branch = issue_branch.rsplit("/", 1)[0]
normalised_slice = slice_id if slice_id.startswith("slice-") else f"slice-{slice_id}"
import re

Expand Down
94 changes: 79 additions & 15 deletions orchestrator/kubernetes_spawner.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"""

import os
import re
import sys
import threading
import time
Expand Down Expand Up @@ -235,35 +236,69 @@ class KubernetesSpawner:

DEFAULT_SANDBOX_IMAGE = os.environ.get("EGG_SANDBOX_IMAGE", "egg:latest")
JOB_NAME_FORMAT = "egg-agent-{pipeline_id}-{role}"
JOB_NAME_FORMAT_SLICE = "egg-agent-{pipeline_id}-{slice_id}-{role}"

@classmethod
def _build_k8s_job_names(
cls,
pipeline_id: str,
agent_role: AgentRole,
slice_id: str | None = None,
) -> tuple[str, str]:
"""Build the two identifiers an agent Job is known by.

Returns a ``(job_name, actual_k8s_job_name)`` pair where:

- ``job_name`` is the unprefixed identifier used as the gateway
session ``container_id`` and in labels (e.g.
``egg-agent-issue-1962-task-planner``).
``egg-agent-issue-1962-task-planner`` or, for slice-scoped
spawns, ``egg-agent-issue-2261-v7-slice-2-coder``).
- ``actual_k8s_job_name`` is the real k8s Job name after
``KubernetesClient`` prepends ``JOB_PREFIX`` during
``create_container`` (e.g.
``egg-sandbox-egg-agent-issue-1962-task-planner``).
``create_container``.

Underscores in ``agent_role.value`` (``task_planner``,
``reviewer_refine``, …) are converted to hyphens because k8s
resource names are RFC-1123 labels and reject underscores.

Slice scope (#2403): when ``slice_id`` is supplied, it is
embedded between the pipeline id and the role so concurrent
slices in the same pipeline don't collide on a single Job name
(which would cause ``spawn_agent_job``'s pre-spawn cleanup to
delete the in-flight sibling slice's Job — see line 405).
"""
job_name = cls.JOB_NAME_FORMAT.format(
pipeline_id=pipeline_id,
role=agent_role.value.replace("_", "-"),
)
if slice_id:
job_name = cls.JOB_NAME_FORMAT_SLICE.format(
pipeline_id=pipeline_id,
slice_id=slice_id,
role=agent_role.value.replace("_", "-"),
)
else:
job_name = cls.JOB_NAME_FORMAT.format(
pipeline_id=pipeline_id,
role=agent_role.value.replace("_", "-"),
)
return job_name, f"{KubernetesClient.JOB_PREFIX}{job_name}"

@staticmethod
def _build_agent_worktree_id(
pipeline_id: str,
agent_role: AgentRole,
slice_id: str | None = None,
) -> str:
"""Build the per-agent worktree identifier.

The id is the gateway worktree key (``container_id`` for
``create_worktrees`` / ``delete_worktrees``) and the agent's
``CONTAINER_ID`` env var. For slice-scoped spawns it embeds the
slice id (#2403) so concurrent slices don't share a worktree —
otherwise slice-N's coder would inherit slice-(N-1)'s worktree
contents (or step on them mid-flight).
"""
if slice_id:
return f"{pipeline_id}-{slice_id}-{agent_role.value}"
return f"{pipeline_id}-{agent_role.value}"

def __init__(
self,
k8s_client: KubernetesClient | None = None,
Expand Down Expand Up @@ -368,6 +403,7 @@ def spawn_agent_job(
spawn_max_retries: int = DEFAULT_SPAWN_MAX_RETRIES,
spawn_retry_initial_backoff_seconds: float = (DEFAULT_SPAWN_RETRY_INITIAL_BACKOFF_SECONDS),
jira_ticket: str | None = None,
slice_id: str | None = None,
) -> SpawnedContainer:
"""Spawn a Kubernetes Job for an agent.

Expand Down Expand Up @@ -398,7 +434,9 @@ def spawn_agent_job(
Raises:
KubernetesSpawnError: If spawning fails
"""
job_name, actual_k8s_job_name = self._build_k8s_job_names(pipeline_id, agent_role)
job_name, actual_k8s_job_name = self._build_k8s_job_names(
pipeline_id, agent_role, slice_id=slice_id
)

# Clean up any existing Job with the same name.
try:
Expand Down Expand Up @@ -441,8 +479,16 @@ def spawn_agent_job(
host_uid = int(os.environ.get("HOST_UID", 1000))
host_gid = int(os.environ.get("HOST_GID", 1000))

# Per-agent worktree isolation: create a dedicated worktree
agent_worktree_id = f"{pipeline_id}-{agent_role.value}"
# Per-agent worktree isolation: create a dedicated worktree.
# Slice scope (#2403): concurrent slices in the same pipeline
# MUST get distinct worktree ids — otherwise slice-N's coder
# spawns onto slice-(N-1)'s already-mounted worktree (or
# races with it during cleanup). The id is also the agent's
# ``CONTAINER_ID`` env and the gateway worktree key, so the
# whole gateway / agent / orchestrator triangle agrees on it.
agent_worktree_id = self._build_agent_worktree_id(
pipeline_id, agent_role, slice_id=slice_id
)
worktree_created_this_call = False

if repos:
Expand Down Expand Up @@ -916,15 +962,17 @@ def cleanup_pipeline(
worktree_ids_to_clean.add(f"{pipeline_id}-{role_label}")

# Also scan filesystem for any per-agent worktrees. Only match
# entries that are either the pipeline-level worktree or a
# "{pipeline_id}-{role}" directory where {role} is a known
# AgentRole value. A naive `startswith(f"{pipeline_id}-")`
# entries that are either the pipeline-level worktree, a
# "{pipeline_id}-{role}" directory, or a slice-scoped
# "{pipeline_id}-slice-{N}-{role}" directory where {role} is a
# known AgentRole value (#2403). A naive `startswith(f"{pipeline_id}-")`
# collides with longer pipeline IDs that share the prefix — e.g.
# cleanup of `issue-1758` would match active worktrees of
# `issue-1758-worktree-fix-tester`, wiping another pipeline's
# state mid-phase (#1865).
if WORKTREE_BASE_DIR.exists():
valid_suffixes = {f"-{role.value}" for role in AgentRole}
valid_role_suffixes = {f"-{role.value}" for role in AgentRole}
slice_segment_re = re.compile(r"^-slice-[0-9]+(-.+)$")
try:
for entry in WORKTREE_BASE_DIR.iterdir():
if not entry.is_dir():
Expand All @@ -936,7 +984,15 @@ def cleanup_pipeline(
if not name.startswith(pipeline_id):
continue
suffix = name[len(pipeline_id) :]
if suffix in valid_suffixes:
if suffix in valid_role_suffixes:
worktree_ids_to_clean.add(name)
continue
# Slice-scoped: "{pipeline_id}-slice-{N}-{role}".
# The trailing "-{role}" inside the captured group
# is matched against the role allowlist so this
# branch can't sweep an unrelated sibling worktree.
slice_match = slice_segment_re.match(suffix)
if slice_match and slice_match.group(1) in valid_role_suffixes:
worktree_ids_to_clean.add(name)
except Exception as e:
logger.warning(
Expand Down Expand Up @@ -1325,6 +1381,7 @@ def create_concurrent_spawn_fn(
certs_volume: str | None = None, # noqa: ARG002 — Docker-era compat
spawn_max_retries: int = DEFAULT_SPAWN_MAX_RETRIES,
spawn_retry_initial_backoff_seconds: float = (DEFAULT_SPAWN_RETRY_INITIAL_BACKOFF_SECONDS),
slice_id: str | None = None,
):
"""Create a spawn callable compatible with ConcurrentPhaseExecutor.

Expand All @@ -1341,6 +1398,12 @@ def create_concurrent_spawn_fn(
sandbox_env: Base environment variables.
image: Container image override.
base_branch: Branch to base worktrees on.
slice_id: Optional slice scope (#2403). When supplied, every
spawn (including ``spawn_specific_roles`` retries) is
tagged with this slice so concurrent slices in the same
pipeline get distinct Job names and worktree ids. Without
this, slice-N spawning ``coder`` would delete slice-(N-1)'s
still-running ``coder`` Job during the pre-spawn cleanup.

Returns:
Callable suitable for ConcurrentPhaseExecutor.spawn_fn.
Expand Down Expand Up @@ -1368,6 +1431,7 @@ def _spawn(
command=command,
spawn_max_retries=spawn_max_retries,
spawn_retry_initial_backoff_seconds=(spawn_retry_initial_backoff_seconds),
slice_id=slice_id,
)

return _spawn
Expand Down
7 changes: 6 additions & 1 deletion orchestrator/routes/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,12 @@ def _branch_read_contract(
)
return None

branch = pipeline.branch or f"egg/{pipeline_id}"
# The pipeline tip is pushed to ``egg/<id>/work`` so slice integration
# branches can coexist as siblings — see
# :func:`routes.pipelines._ensure_pipeline_work_ref` for the rationale
# (#2399). The fallback shape mirrors the actual remote ref the
# contract was committed to.
branch = pipeline.branch or f"egg/{pipeline_id}/work"
return contract_store.load_contract_from_branch(identifier, store.repo_path, branch)


Expand Down
Loading
Loading