Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions contextual_orchestrator/cost_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,14 @@ def __init__(
)
else:
self.batch_backend = batch_backend
self._embedding_backend_is_default = embedding_batch_backend is None
self.embedding_batch_backend: EmbeddingBatchBackend = embedding_batch_backend or self._default_embedding_backend()
# job_id -> submitted BatchJob (so poll/retrieve can be driven by id)
self._batch_jobs: Dict[str, BatchJob] = {}
# embeddings batch state: job handle + submitted requests + cached doc,
# keyed by batch id so poll/retrieve is idempotent (usage recorded once).
self._embedding_jobs: Dict[str, BatchJob] = {}
self._embedding_job_backends: Dict[str, EmbeddingBatchBackend] = {}
self._embedding_requests: Dict[str, List[EmbeddingBatchRequest]] = {}
self._embedding_input_counts: Dict[str, int] = {}
self._embedding_part_counts: Dict[str, List[int]] = {}
Expand Down Expand Up @@ -487,8 +489,14 @@ def submit_embeddings_batch(
provider_name=provider_name,
attribution=shared_attribution,
)
job = self.embedding_batch_backend.submit(requests, metadata=metadata)
backend = (
self._default_embedding_backend()
if self._embedding_backend_is_default
else self.embedding_batch_backend
)
job = backend.submit(requests, metadata=metadata)
self._embedding_jobs[job.job_id] = job
self._embedding_job_backends[job.job_id] = backend
Comment thread
seonghobae marked this conversation as resolved.
Resolved
self._embedding_requests[job.job_id] = requests
self._embedding_input_counts[job.job_id] = len(inputs)
self._embedding_part_counts[job.job_id] = part_counts
Expand Down Expand Up @@ -702,7 +710,8 @@ def embeddings_batch_document(self, batch_id: str) -> Dict[str, Any]:
return cached

job = self._require_embedding_job(batch_id)
status = self.embedding_batch_backend.poll(job)
backend = self._embedding_job_backends.get(batch_id, self.embedding_batch_backend)
status = backend.poll(job)
if not status.get("is_complete"):
return {
"batch_id": batch_id,
Expand All @@ -711,7 +720,7 @@ def embeddings_batch_document(self, batch_id: str) -> Dict[str, Any]:
"embeddings": None,
}

items: List[EmbeddingBatchResultItem] = self.embedding_batch_backend.retrieve(job)
items: List[EmbeddingBatchResultItem] = backend.retrieve(job)
requests = self._embedding_requests.get(batch_id, [])
request_by_custom_id = {request.custom_id: request for request in requests}
input_count = self._embedding_input_counts.get(batch_id, len(requests))
Expand Down
163 changes: 130 additions & 33 deletions contextual_orchestrator/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ def _temperature_capability_rejection(exc: Exception) -> bool:
return False
try:
body = exc.read()
except http.client.IncompleteRead as read_error:
body = read_error.partial
except (OSError, ValueError):
body = b""
body_bytes = body if isinstance(body, bytes) else str(body).encode("utf-8")
Expand Down Expand Up @@ -220,6 +222,7 @@ def complete_structured(
response = self.orchestrator.client.proxy_send(agent, "chat/completions", payload)
output = ModelClient._response_content(agent, response)
usage = response.get("usage") if isinstance(response.get("usage"), dict) else None
self.orchestrator._record_in_flight_provider_usage(agent, usage, output)
return self._completion_payload(output, agent.id, usage, self.mode if mode is None else mode)

def _completion_payload(
Expand Down Expand Up @@ -313,6 +316,8 @@ def __post_init__(self) -> None:
raise TypeError("local_credential_key must be a string")
if scheme == "local" and not self.local_credential_key:
raise ValueError("local:// gateway URLs require local_credential_key")
if scheme == "local" and not _is_local_provider_url(self.base_url):
raise ValueError("local:// gateway URLs require a well-formed explicit loopback endpoint")
Comment thread
seonghobae marked this conversation as resolved.
Resolved
if self.local_credential_key and scheme != "local":
raise ValueError("local_credential_key requires a local:// gateway URL")
if not self.auth_scheme or type(self.auth_scheme) is not str:
Expand Down Expand Up @@ -456,7 +461,7 @@ def as_dict(self) -> dict[str, Any]:
# is a caller or configuration error and must not be retried.
TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504})
LOCAL_PROVIDER_SCHEMES = frozenset({"local"})
LOCAL_PROVIDER_HOSTS = frozenset({"127.0.0.1", "::1", "localhost", "host.docker.internal"})
LOCAL_PROVIDER_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})


def _is_local_provider_url(base_url: str) -> bool:
Expand Down Expand Up @@ -2018,14 +2023,15 @@ def close(self) -> None:
class _StateStore:
"""Minimal write-through sqlite persistence for orchestrator runtime state.

ponytail: one generic table, no ORM. Keyed kinds (workflow_run, evaluation_run)
ponytail: one generic table, no ORM. Keyed kinds (workflow_run, evaluation_run,
budget_meter)
upsert by key; stream kinds (analytics, audit) append. Stream rows grow unbounded
on disk while the in-memory deques stay capped — add pruning if db size matters.
Runtime values (kind, key, payload, limit) are always bound through SQLite
placeholders so persisted prompts and identifiers cannot become SQL syntax.
"""

_KEYED = {"workflow_run", "evaluation_run"}
_KEYED = {"workflow_run", "evaluation_run", "budget_meter"}
_CREATE_RECORDS_SQL = (
"CREATE TABLE IF NOT EXISTS records ("
"seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, key TEXT, payload TEXT NOT NULL)"
Expand Down Expand Up @@ -2169,6 +2175,9 @@ def __init__(
self._audit_events: deque[dict[str, Any]] = deque(maxlen=256)
self._run_order: deque[str] = deque(maxlen=128)
self._workflow_run_lock = threading.Lock()
self._budget_spend_lock = threading.Lock()
self._budget_spent_output_tokens = 0
self._budget_spent_cost_usd = 0.0
self._archived_spend: dict[str, Any] = {
"run_count": 0,
"estimated_prompt_tokens": 0,
Expand All @@ -2192,6 +2201,15 @@ def __init__(
self._commercial_report_cache_local = threading.local()
if self._store is not None:
self._reload_state()
restored = self.spend_analytics()["totals"]
self._budget_spent_output_tokens = max(
self._budget_spent_output_tokens,
restored["estimated_output_tokens"],
)
self._budget_spent_cost_usd = max(
self._budget_spent_cost_usd,
restored["estimated_cost_usd"] or 0.0,
)
Comment thread
seonghobae marked this conversation as resolved.
Resolved

def close(self) -> None:
"""Release optional durable resources owned by this orchestrator."""
Expand Down Expand Up @@ -2248,6 +2266,14 @@ def provider_readiness_report(
}

def _reload_state(self) -> None:
budget_meter = self._store.load("budget_meter")
if budget_meter:
self._budget_spent_output_tokens = int(
budget_meter[-1].get("spent_output_tokens", 0)
)
self._budget_spent_cost_usd = float(
budget_meter[-1].get("spent_cost_usd", 0.0)
)
for record in self._store.load("workflow_run"):
self._remember_workflow_run(record)
for evaluation in self._store.load("evaluation_run"):
Expand Down Expand Up @@ -2339,6 +2365,11 @@ def proxy_completion(
usage,
responses=endpoint.strip("/") == "responses",
)
self._record_in_flight_provider_usage(
agent,
passthrough_step.get("usage"),
passthrough_output,
)
self._persist_workflow_run(
{
"workflow_run_id": f"run_{uuid.uuid4().hex}",
Expand Down Expand Up @@ -2386,29 +2417,47 @@ def _orchestrated_provider_completion(
disabled_model = requested_model if requested_model is not None else final_agent.model
raise RuntimeError(f"requested model {disabled_model!r} is disabled")

with self._budget_spend_lock:
budget_before = (
self._budget_spent_output_tokens,
self._budget_spent_cost_usd,
)
workflow = self.conduct(messages, preserve_messages=True, judge=True)
in_flight_output_tokens = 0
in_flight_cost_usd = 0.0
workflow_output_tokens = 0
workflow_cost_usd = 0.0
model_by_agent = {agent.id: agent.model for agent in self.agents}
for row in workflow["trace"]:
metered_steps = list(workflow["trace"])
verification = workflow.get("verification")
if isinstance(verification, dict) and isinstance(
verification.get("judge_usage"), dict
):
metered_steps.append(
{
"agent_id": verification.get("judge_agent_id", "unknown"),
"usage": verification["judge_usage"],
"output": "",
}
)
for row in metered_steps:
output_tokens, _reported = _step_output_token_count(row)
in_flight_output_tokens += output_tokens
model = model_by_agent.get(row.get("served_agent_id") or row.get("agent_id"))
workflow_output_tokens += output_tokens
model = model_by_agent.get(
row.get("served_agent_id") or row.get("agent_id")
)
price = self.price_per_million.get(model) if model is not None else None
if price is not None:
in_flight_cost_usd += output_tokens / 1_000_000 * price
verification = workflow.get("verification")
verification_usage = verification.get("judge_usage") if isinstance(verification, dict) else None
if isinstance(verification_usage, dict):
judge_output_tokens, _reported = _step_output_token_count(
{"usage": verification_usage, "output": ""}
workflow_cost_usd += output_tokens / 1_000_000 * price
with self._budget_spend_lock:
recorded_output_tokens = self._budget_spent_output_tokens - budget_before[0]
recorded_cost_usd = self._budget_spent_cost_usd - budget_before[1]
self._budget_spent_output_tokens += max(
0,
workflow_output_tokens - recorded_output_tokens,
)
self._budget_spent_cost_usd += max(
0.0,
workflow_cost_usd - recorded_cost_usd,
)
in_flight_output_tokens += judge_output_tokens
judge_agent_id = verification.get("judge_agent_id")
judge_model = model_by_agent.get(judge_agent_id)
judge_price = self.price_per_million.get(judge_model) if judge_model is not None else None
if judge_price is not None:
in_flight_cost_usd += judge_output_tokens / 1_000_000 * judge_price
evidence = "\n\n".join(
f"Workflow step {row['id']} ({row['role']}):\n{row['output']}"
for row in workflow["trace"]
Expand Down Expand Up @@ -2438,10 +2487,7 @@ def _orchestrated_provider_completion(
)
upstream["model"] = final_agent.model
upstream["stream"] = False
self._raise_if_spend_budget_exceeded(
additional_output_tokens=in_flight_output_tokens,
additional_cost_usd=round(in_flight_cost_usd, 6),
)
self._raise_if_spend_budget_exceeded()
synthesis_started = time.perf_counter()
raw = self.client.proxy_send(final_agent, "responses", upstream)
synthesis_output = raw.get("output_text")
Expand Down Expand Up @@ -2487,10 +2533,7 @@ def _orchestrated_provider_completion(
upstream["model"] = final_agent.model
upstream["messages"] = synthesis_messages
upstream["stream"] = False
self._raise_if_spend_budget_exceeded(
additional_output_tokens=in_flight_output_tokens,
additional_cost_usd=round(in_flight_cost_usd, 6),
)
self._raise_if_spend_budget_exceeded()
synthesis_started = time.perf_counter()
raw = self.client.proxy_send(final_agent, "chat/completions", upstream)
synthesis_output = ""
Expand All @@ -2515,6 +2558,11 @@ def _orchestrated_provider_completion(
usage,
responses=response_request,
)
self._record_in_flight_provider_usage(
final_agent,
synthesis_step.get("usage"),
synthesis_output,
)
trace = [*workflow["trace"], synthesis_step]
workflow_run_id = f"run_{uuid.uuid4().hex}"
record = {
Expand Down Expand Up @@ -2599,6 +2647,7 @@ def stream_route(self, messages: list[ChatMessage], workflow_run_id: str | None
parts.append(delta)
yield delta
answer = "".join(parts)
self._record_in_flight_provider_usage(agent, None, answer)
record = {
"workflow_run_id": workflow_run_id or f"run_{uuid.uuid4().hex}",
"created_at": int(time.time()),
Expand Down Expand Up @@ -2749,6 +2798,11 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]:
}
if result.get("usage") is not None:
row["usage"] = result["usage"]
self._record_in_flight_provider_usage(
agent,
row.get("usage"),
result["content"],
)
record = {
"workflow_run_id": f"run_{uuid.uuid4().hex}",
"created_at": int(time.time()),
Expand Down Expand Up @@ -3106,6 +3160,8 @@ def conduct(
try:
steps = self._plan_generated(task)
plan_source = "generated"
except BudgetExceededError:
raise
except Exception: # noqa: BLE001 - invalid plans must not break the request
steps = self._plan(task)
plan_source = "template_fallback"
Expand Down Expand Up @@ -3235,10 +3291,15 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]:
"verifier step when correctness matters.\n"
f"Available agents:\n{pool}"
)
raw = self.client.chat(planner, [
{"role": "system", "content": system},
{"role": "user", "content": task},
])
raw, _served_id, _usage = self._invoke(
planner,
[
{"role": "system", "content": system},
{"role": "user", "content": task},
],
text=task,
role="thinker",
)
return self._parse_workflow_plan(raw)

def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]:
Expand Down Expand Up @@ -3363,13 +3424,17 @@ def _invoke(
last_error: Exception | None = None
for agent in candidates:
try:
self._raise_if_spend_budget_exceeded()
Comment thread
seonghobae marked this conversation as resolved.
Resolved
output = self.client.chat(agent, messages)
except BudgetExceededError:
raise
except Exception as exc: # noqa: BLE001 - one agent failing routes to the next
last_error = exc
self._record_failure(agent.id)
continue
self._record_success(agent.id)
usage = self.client.take_usage() if hasattr(self.client, "take_usage") else None
self._record_in_flight_provider_usage(agent, usage, output)
return output, agent.id, usage
if isinstance(last_error, ProviderResponseError):
raise last_error
Expand Down Expand Up @@ -3772,6 +3837,10 @@ def spend_analytics(self, price_per_million: dict[str, float] | None = None) ->
"estimated_cost_usd": cost,
})

with self._budget_spend_lock:
budget_output_tokens = self._budget_spent_output_tokens
budget_cost_usd = self._budget_spent_cost_usd

return {
"measurement_status": "local_runtime_estimate",
"source_note": (
Expand All @@ -3792,7 +3861,10 @@ def spend_analytics(self, price_per_million: dict[str, float] | None = None) ->
},
"by_model": rows,
"unpriced_models": unpriced,
"budget": self._budget_block(total_output_tokens, round(total_cost, 6) if prices else None),
"budget": self._budget_block(
budget_output_tokens,
round(budget_cost_usd, 6) if prices else None,
),
}

def _budget_block(self, spent_tokens: int, spent_cost: float | None) -> dict[str, Any]:
Expand Down Expand Up @@ -3820,6 +3892,31 @@ def budget_status(self) -> dict[str, Any]:
"""Current spend-budget state (limits, spent, remaining, exceeded)."""
return self.spend_analytics()["budget"]

def _record_in_flight_provider_usage(
self,
agent: ModelAgent,
usage: dict[str, Any] | None,
output: str,
) -> None:
"""Add one completed provider call to the process budget ledger."""
output_tokens, _reported = _step_output_token_count(
{"usage": usage, "output": output}
)
price = self.price_per_million.get(agent.model)
with self._budget_spend_lock:
self._budget_spent_output_tokens += output_tokens
if price is not None:
self._budget_spent_cost_usd += output_tokens / 1_000_000 * price
if self._store is not None:
self._store.save(
"budget_meter",
"process_budget",
{
"spent_output_tokens": self._budget_spent_output_tokens,
"spent_cost_usd": self._budget_spent_cost_usd,
},
)
Comment thread
seonghobae marked this conversation as resolved.
Resolved

def _raise_if_spend_budget_exceeded(
self,
*,
Expand Down
Loading
Loading