From 7b9f0f6c35be9401dfcd5dd287a63dd336e837cd Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 07:06:40 -0700 Subject: [PATCH 01/27] feat(foundry): add brokered responses mode Signed-off-by: Sertac Ozercan --- .github/workflows/ci.yml | 6 +- .gitignore | 6 + deploy/foundry/README.md | 46 + deploy/foundry/doctor.sh | 69 +- .../scripts/foundry_brokered_conformance.sh | 149 ++ .../local_brokered_conformance_container.sh | 157 ++ .../scripts/verify_brokered_transcript.py | 154 ++ docs/agent-abi.md | 30 + docs/foundry-hosted-brokered.md | 365 ++++ docs/runtime-capabilities.md | 20 +- pkg/agentkit/abi/render.go | 146 ++ pkg/agentkit/abi/render_test.go | 63 + pkg/agentkit/config/config_test.go | 504 ++++++ pkg/agentkit/config/specs.go | 4 +- pkg/agentkit/config/tool.go | 11 + pkg/agentkit/config/validate.go | 678 ++++++- pkg/agentkit/effective/agent.go | 120 ++ pkg/agentkit/effective/agent_test.go | 95 +- runtimes/common/README.md | 29 + .../common/agentkit_serve_common/brokered.py | 212 +++ .../common/agentkit_serve_common/config.py | 452 +++++ .../common/agentkit_serve_common/foundry.py | 1314 +++++++++++++- .../foundry_brokered_cli.py | 96 + .../foundry_conformance.py | 151 ++ .../foundry_model_loop.py | 210 +++ .../common/agentkit_serve_common/runtime.py | 29 + runtimes/common/pyproject.toml | 6 + .../approval_declined_payload.json | 7 + .../continuation_request.json | 11 + .../final_message_response.json | 25 + .../function_call_response.json | 19 + .../foundry_brokered/initial_request.json | 3 + ...iple_function_calls_unsupported_error.json | 6 + .../tool_execution_failure_payload.json | 7 + .../tool_policy_rejection_payload.json | 7 + .../unknown_call_id_error.json | 6 + .../unknown_previous_response_id_error.json | 6 + runtimes/common/tests/test_brokered_schema.py | 154 ++ .../common/tests/test_config_validation.py | 509 ++++++ .../tests/test_foundry_brokered_protocol.py | 1600 +++++++++++++++++ .../common/tests/test_foundry_conformance.py | 108 ++ .../common/tests/test_foundry_protocol.py | 155 ++ .../tests/test_foundry_transcript_verifier.py | 151 ++ test/foundry-brokered-agentkit/Dockerfile | 16 + test/foundry-brokered-agentkit/README.md | 47 + test/foundry-brokered-agentkit/agent.yaml | 22 + test/foundry-brokered-conformance/Dockerfile | 15 + test/foundry-brokered-conformance/README.md | 86 + .../foundry.agent.yaml.example | 11 + 49 files changed, 8057 insertions(+), 36 deletions(-) create mode 100755 deploy/foundry/scripts/foundry_brokered_conformance.sh create mode 100755 deploy/foundry/scripts/local_brokered_conformance_container.sh create mode 100755 deploy/foundry/scripts/verify_brokered_transcript.py create mode 100644 docs/foundry-hosted-brokered.md create mode 100644 runtimes/common/agentkit_serve_common/brokered.py create mode 100644 runtimes/common/agentkit_serve_common/foundry_brokered_cli.py create mode 100644 runtimes/common/agentkit_serve_common/foundry_conformance.py create mode 100644 runtimes/common/agentkit_serve_common/foundry_model_loop.py create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/approval_declined_payload.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/continuation_request.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/final_message_response.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/function_call_response.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/initial_request.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/multiple_function_calls_unsupported_error.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/tool_execution_failure_payload.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/tool_policy_rejection_payload.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/unknown_call_id_error.json create mode 100644 runtimes/common/tests/fixtures/foundry_brokered/unknown_previous_response_id_error.json create mode 100644 runtimes/common/tests/test_brokered_schema.py create mode 100644 runtimes/common/tests/test_foundry_brokered_protocol.py create mode 100644 runtimes/common/tests/test_foundry_conformance.py create mode 100644 runtimes/common/tests/test_foundry_transcript_verifier.py create mode 100644 test/foundry-brokered-agentkit/Dockerfile create mode 100644 test/foundry-brokered-agentkit/README.md create mode 100644 test/foundry-brokered-agentkit/agent.yaml create mode 100644 test/foundry-brokered-conformance/Dockerfile create mode 100644 test/foundry-brokered-conformance/README.md create mode 100644 test/foundry-brokered-conformance/foundry.agent.yaml.example diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdca002..8364e65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,10 +46,12 @@ jobs: deploy/foundry/search/setup.sh \ deploy/foundry/memory/setup.sh \ deploy/foundry/rbac/assign-agent-identity.sh \ - deploy/foundry/scripts/invoke_responses.sh; do + deploy/foundry/scripts/invoke_responses.sh \ + deploy/foundry/scripts/foundry_brokered_conformance.sh \ + deploy/foundry/scripts/local_brokered_conformance_container.sh; do bash -n "$script" done - python3 -m py_compile test/foundry-hosted-agent/foundry_live.py + python3 -m py_compile test/foundry-hosted-agent/foundry_live.py deploy/foundry/scripts/verify_brokered_transcript.py python3 - <<'PY' from pathlib import Path import re diff --git a/.gitignore b/.gitignore index 25cc9b9..6d15f72 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,9 @@ __pycache__/ # Local provider validation outputs deploy/foundry/**/output.env + +# Local Foundry brokered conformance transcripts (request/response evidence may contain endpoint-specific data). +foundry-brokered-transcript*/ +foundry-brokered-local-transcript*/ +agentkit-foundry-brokered.*/ +agentkit-foundry-brokered-local.*/ diff --git a/deploy/foundry/README.md b/deploy/foundry/README.md index 2fc651e..c7ede84 100644 --- a/deploy/foundry/README.md +++ b/deploy/foundry/README.md @@ -60,7 +60,53 @@ as a Foundry-hosted protocol image. the project/account role required for workload-identity model/tool calls (defaults to `Foundry User`). - `scripts/invoke_responses.sh` sends the minimal portable hosted Responses payload (`{"input":"..."}`), avoiding gateway-specific optional fields. +- `scripts/foundry_brokered_conformance.sh` runs the Phase A0 brokered + Responses function-call/continuation loop against a deployed `/responses` + endpoint and writes a sanitized transcript directory for review evidence. +- `scripts/local_brokered_conformance_container.sh` builds the conformance + container locally, runs it, and exercises the same transcript helper before an + image is pushed to Foundry. - `doctor.sh` checks the expected Foundry/project env and local CLI prerequisites. + Use `doctor.sh --brokered-conformance` before running the brokered transcript + helper to verify `AGENT_RESPONSES_ENDPOINT` and auth prerequisites. These scripts intentionally produce local `output.env` files that should not be committed with live subscription or endpoint values. + + +## Brokered conformance smoke + + +Before pushing the image, validate the packaged container locally: + +```sh +deploy/foundry/scripts/local_brokered_conformance_container.sh \ + --platform linux/amd64 \ + --tag agentkit-foundry-brokered-conformance:amd64-test \ + --port 18090 \ + --transcript-dir ./foundry-brokered-local-transcript +``` + +After deploying an image that serves +`agentkit_serve_common.foundry_conformance.create_foundry_conformance_app()`, run: + +```sh +export AGENT_RESPONSES_ENDPOINT="https:///responses" +# Optional: export AZURE_SUBSCRIPTION_ID="" to select an account. +deploy/foundry/doctor.sh --brokered-conformance +deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundry-brokered-transcript +``` + +To validate the production AgentKit brokered path locally instead of the +standalone SDK conformance app, see `test/foundry-brokered-agentkit/`. That +fixture uses `agentkit-foundry-brokered` and expects generated call IDs, so run +the transcript helper with `AGENTKIT_EXPECTED_CALL_ID=auto` and +`AGENTKIT_EXPECTED_CALL_ID_PREFIX=call_`. + +Alternatively set `AGENT_RESPONSES_BEARER_TOKEN` to use a pre-acquired token +instead of invoking `az account get-access-token`. If `AZURE_SUBSCRIPTION_ID` is +omitted, the helper uses the current `az` account. The script stores request and +response JSON files plus `summary.json`; do not include bearer tokens in the +transcript. Re-run +`python3 deploy/foundry/scripts/verify_brokered_transcript.py ` +to verify archived transcript evidence later. diff --git a/deploy/foundry/doctor.sh b/deploy/foundry/doctor.sh index 04b191a..79271d9 100755 --- a/deploy/foundry/doctor.sh +++ b/deploy/foundry/doctor.sh @@ -1,5 +1,25 @@ #!/usr/bin/env bash set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: doctor.sh [--brokered-conformance] + +Default mode checks the generic Foundry deployment helper prerequisites. +--brokered-conformance checks the env/tools needed to run + deploy/foundry/scripts/foundry_brokered_conformance.sh +against a deployed hosted-agent /responses endpoint. +EOF +} + +mode="default" +case "${1:-}" in + "") ;; + --brokered-conformance) mode="brokered-conformance" ;; + -h|--help) usage; exit 0 ;; + *) usage; exit 2 ;; +esac + missing=0 need() { if [[ -z "${!1:-}" ]]; then @@ -7,13 +27,50 @@ need() { missing=1 fi } -need FOUNDRY_PROJECT_ENDPOINT -if ! command -v az >/dev/null 2>&1; then - printf 'warning: az CLI not found; hosted resource/RBAC checks cannot run locally\n' >&2 -fi -if ! command -v azd >/dev/null 2>&1; then - printf 'warning: azd CLI not found; hosted-agent deploy/invoke checks cannot run locally\n' >&2 +need_command() { + if ! command -v "$1" >/dev/null 2>&1; then + printf '%s: %s\n' "$2" "$1" >&2 + if [[ "${3:-required}" == "required" ]]; then + missing=1 + fi + fi +} + +if [[ "$mode" == "brokered-conformance" ]]; then + need AGENT_RESPONSES_ENDPOINT + need_command curl "missing command" + need_command python3 "missing command" + if [[ -z "${AGENT_RESPONSES_BEARER_TOKEN:-}" ]]; then + need_command az "missing command" + if command -v az >/dev/null 2>&1; then + if [[ -n "${AZURE_SUBSCRIPTION_ID:-}" ]]; then + az account set --subscription "$AZURE_SUBSCRIPTION_ID" >/dev/null 2>&1 || missing=1 + fi + if ! az account show >/dev/null 2>&1; then + printf 'missing auth: set AGENT_RESPONSES_BEARER_TOKEN or run az login/select an account +' >&2 + missing=1 + fi + fi + fi + if [[ ! -x deploy/foundry/scripts/foundry_brokered_conformance.sh ]]; then + printf 'missing executable: deploy/foundry/scripts/foundry_brokered_conformance.sh\n' >&2 + missing=1 + fi + if [[ ! -f deploy/foundry/scripts/verify_brokered_transcript.py ]]; then + printf 'missing verifier: deploy/foundry/scripts/verify_brokered_transcript.py\n' >&2 + missing=1 + fi + if [[ "$missing" -ne 0 ]]; then + exit 2 + fi + printf 'foundry doctor: brokered conformance prerequisites checked\n' + exit 0 fi + +need FOUNDRY_PROJECT_ENDPOINT +need_command az "warning: az CLI not found; hosted resource/RBAC checks cannot run locally" optional +need_command azd "warning: azd CLI not found; hosted-agent deploy/invoke checks cannot run locally" optional if [[ "$missing" -ne 0 ]]; then exit 2 fi diff --git a/deploy/foundry/scripts/foundry_brokered_conformance.sh b/deploy/foundry/scripts/foundry_brokered_conformance.sh new file mode 100755 index 0000000..0cc1367 --- /dev/null +++ b/deploy/foundry/scripts/foundry_brokered_conformance.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: foundry_brokered_conformance.sh [prompt] [transcript-dir] + +Runs the Phase A0 hosted Responses brokered conformance loop against a deployed +Foundry hosted agent endpoint: + 1. POST an initial /responses request with no request-level tools. + 2. Assert the response contains the deterministic conformance_read function_call. + 3. POST a function_call_output continuation with previous_response_id. + 4. Assert the final response is a completed assistant message. + +Required environment: + AGENT_RESPONSES_ENDPOINT Full deployed /responses endpoint URL. + +Authentication, one of: + AGENT_RESPONSES_BEARER_TOKEN Pre-acquired bearer token for the endpoint. + AZURE_SUBSCRIPTION_ID Optional subscription to select before `az account get-access-token`. + If omitted, the current `az` account is used. + +Optional: + AGENTKIT_CONFORMANCE_OUTPUT Defaults to {"approved":true,"output":{"success":true}}. + AGENTKIT_EXPECTED_TOOL_NAME Defaults to conformance_read. + AGENTKIT_EXPECTED_ARGUMENTS Defaults to {"probe":true}. + AGENTKIT_EXPECTED_CALL_ID Defaults to call_conformance_1; set to auto for generated IDs. + AGENTKIT_EXPECTED_CALL_ID_PREFIX Optional required call_id prefix, e.g. call_. + AGENTKIT_CONTINUATION_PROOF Optional x-agentkit-brokered-continuation-proof header. +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +: "${AGENT_RESPONSES_ENDPOINT:?set AGENT_RESPONSES_ENDPOINT to the deployed /responses URL}" + +prompt="${1:-conformance_read}" +transcript_dir="${2:-$(mktemp -d "${TMPDIR:-/tmp}/agentkit-foundry-brokered.XXXXXX")}" +mkdir -p "$transcript_dir" + +if [[ -n "${AGENT_RESPONSES_BEARER_TOKEN:-}" ]]; then + token="$AGENT_RESPONSES_BEARER_TOKEN" +else + if [[ -n "${AZURE_SUBSCRIPTION_ID:-}" ]]; then + az account set --subscription "$AZURE_SUBSCRIPTION_ID" >/dev/null + else + az account show >/dev/null + fi + token="$(az account get-access-token --resource https://ai.azure.com --query accessToken -o tsv)" +fi + +conformance_output="${AGENTKIT_CONFORMANCE_OUTPUT:-{\"approved\":true,\"output\":{\"success\":true}}}" +expected_tool_name="${AGENTKIT_EXPECTED_TOOL_NAME:-conformance_read}" +expected_arguments="${AGENTKIT_EXPECTED_ARGUMENTS:-{\"probe\":true}}" +expected_call_id="${AGENTKIT_EXPECTED_CALL_ID:-call_conformance_1}" +expected_call_id_prefix="${AGENTKIT_EXPECTED_CALL_ID_PREFIX:-}" +initial_request="$transcript_dir/01-initial-request.json" +initial_response="$transcript_dir/02-initial-response.json" +continuation_request="$transcript_dir/03-continuation-request.json" +continuation_response="$transcript_dir/04-continuation-response.json" +summary_file="$transcript_dir/summary.json" + +PROMPT="$prompt" python3 - <<'PY' >"$initial_request" +import json +import os +print(json.dumps({"input": os.environ["PROMPT"]}, separators=(",", ":"))) +PY + +curl -fsS \ + -H "Authorization: Bearer ${token}" \ + -H 'content-type: application/json' \ + "$AGENT_RESPONSES_ENDPOINT" \ + -d "@$initial_request" >"$initial_response" + +read -r response_id call_id < <(EXPECTED_TOOL_NAME="$expected_tool_name" EXPECTED_ARGUMENTS="$expected_arguments" EXPECTED_CALL_ID="$expected_call_id" EXPECTED_CALL_ID_PREFIX="$expected_call_id_prefix" python3 - "$initial_response" <<'PY' +import json +import os +import sys +from pathlib import Path + +body = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +assert body.get("status") == "completed", body +response_id = body.get("id") +assert isinstance(response_id, str) and response_id.startswith("caresp_"), body +assert not response_id.startswith("resp_"), body +output = body.get("output") +assert isinstance(output, list) and len(output) == 1, body +call = output[0] +assert call.get("type") == "function_call", call +expected_tool_name = os.environ["EXPECTED_TOOL_NAME"] +expected_arguments = json.loads(os.environ["EXPECTED_ARGUMENTS"]) +expected_call_id = os.environ["EXPECTED_CALL_ID"] +expected_call_id_prefix = os.environ.get("EXPECTED_CALL_ID_PREFIX", "") +assert call.get("name") == expected_tool_name, call +call_id = call.get("call_id") +assert isinstance(call_id, str) and call_id, call +if expected_call_id != "auto": + assert call_id == expected_call_id, call +if expected_call_id_prefix: + assert call_id.startswith(expected_call_id_prefix), call +assert json.loads(call.get("arguments") or "{}") == expected_arguments, call +print(response_id, call_id) +PY +) + +PREVIOUS_RESPONSE_ID="$response_id" CALL_ID="$call_id" CONFORMANCE_OUTPUT="$conformance_output" python3 - <<'PY' >"$continuation_request" +import json +import os +# Validate the configured output is JSON before placing it in the Responses item. +json.loads(os.environ["CONFORMANCE_OUTPUT"]) +print(json.dumps({ + "previous_response_id": os.environ["PREVIOUS_RESPONSE_ID"], + "input": [{ + "type": "function_call_output", + "call_id": os.environ["CALL_ID"], + "output": os.environ["CONFORMANCE_OUTPUT"], + "status": "completed", + }], +}, separators=(",", ":"))) +PY + +continuation_headers=(-H "Authorization: Bearer ${token}" -H 'content-type: application/json') +if [[ -n "${AGENTKIT_CONTINUATION_PROOF:-}" ]]; then + continuation_headers+=(-H "x-agentkit-brokered-continuation-proof: ${AGENTKIT_CONTINUATION_PROOF}") +fi + +curl -fsS \ + "${continuation_headers[@]}" \ + "$AGENT_RESPONSES_ENDPOINT" \ + -d "@$continuation_request" >"$continuation_response" + +verifier_args=( + "$transcript_dir" + --expected-tool-name "$expected_tool_name" + --expected-arguments-json "$expected_arguments" + --expected-call-id "$expected_call_id" + --write-summary +) +if [[ -n "$expected_call_id_prefix" ]]; then + verifier_args+=(--expected-call-id-prefix "$expected_call_id_prefix") +fi +python3 deploy/foundry/scripts/verify_brokered_transcript.py "${verifier_args[@]}" >"$summary_file.tmp" +rm -f "$summary_file.tmp" + +echo "Foundry brokered conformance passed. Sanitized transcript: ${transcript_dir}" +cat "$summary_file" diff --git a/deploy/foundry/scripts/local_brokered_conformance_container.sh b/deploy/foundry/scripts/local_brokered_conformance_container.sh new file mode 100755 index 0000000..a33d1cc --- /dev/null +++ b/deploy/foundry/scripts/local_brokered_conformance_container.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: local_brokered_conformance_container.sh [--fixture sdk|agentkit] [--platform linux/amd64] [--tag TAG] [--port PORT] [--transcript-dir DIR] + +Builds and runs a local Foundry brokered Responses conformance container, then +validates the full function_call/function_call_output loop with +foundry_brokered_conformance.sh. This is local proof for packaged container paths +before pushing/deploying images to Foundry. + +Fixtures: + sdk Minimal Azure Responses SDK conformance app (default). + agentkit Production AgentKit create_foundry_app brokered-only path. + +Options: + --fixture NAME sdk or agentkit (default: sdk). + --platform PLATFORM Optional docker build/run platform, e.g. linux/amd64. + --tag TAG Image tag to build/run. Defaults per fixture. + --port PORT Local host port. Defaults per fixture. + --transcript-dir DIR Transcript directory. Defaults to a temp directory. + -h, --help Show this help. +EOF +} + +fixture="sdk" +platform="" +tag="" +port="" +transcript_dir="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --fixture) + fixture="${2:?--fixture requires a value}" + shift 2 + ;; + --platform) + platform="${2:?--platform requires a value}" + shift 2 + ;; + --tag) + tag="${2:?--tag requires a value}" + shift 2 + ;; + --port) + port="${2:?--port requires a value}" + shift 2 + ;; + --transcript-dir) + transcript_dir="${2:?--transcript-dir requires a value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + exit 2 + ;; + esac +done + +case "$fixture" in + sdk) + dockerfile="test/foundry-brokered-conformance/Dockerfile" + tag="${tag:-agentkit-foundry-brokered-conformance:local}" + port="${port:-18088}" + expected_call_id="call_conformance_1" + expected_call_id_prefix="" + continuation_proof="" + ;; + agentkit) + dockerfile="test/foundry-brokered-agentkit/Dockerfile" + tag="${tag:-agentkit-foundry-brokered:local}" + port="${port:-18092}" + expected_call_id="auto" + expected_call_id_prefix="call_" + continuation_proof="local-dev-proof" + ;; + *) + usage + exit 2 + ;; +esac + +if [[ -z "$transcript_dir" ]]; then + transcript_dir="$(mktemp -d "${TMPDIR:-/tmp}/agentkit-foundry-brokered-${fixture}.XXXXXX")" +fi + +for cmd in docker curl python3; do + if ! command -v "$cmd" >/dev/null 2>&1; then + printf 'missing command: %s\n' "$cmd" >&2 + exit 2 + fi +done + +name_suffix="$(printf '%s-%s-%s' "$fixture" "$tag" "$port" | tr -c 'A-Za-z0-9_.-' '-')" +container_name="agentkit-foundry-brokered-${name_suffix}" +run_args=() +if [[ -n "$platform" ]]; then + run_args+=(--platform "$platform") +fi +if [[ -n "$continuation_proof" ]]; then + run_args+=(-e "AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=${continuation_proof}") +fi + +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +cleanup + +build_args=(.) +if [[ -n "$platform" ]]; then + build_args=(--platform "$platform" "${build_args[@]}") +fi + +docker build "${build_args[@]}" -f "$dockerfile" -t "$tag" + +docker run -d --rm \ + "${run_args[@]}" \ + --name "$container_name" \ + -p "127.0.0.1:${port}:8088" \ + "$tag" >/dev/null + +ready_url="http://127.0.0.1:${port}/readiness" +for _ in $(seq 1 80); do + if curl -fsS "$ready_url" >/dev/null 2>&1; then + break + fi + sleep 0.5 +done +curl -fsS "$ready_url" >/dev/null + +helper_env=( + "AGENT_RESPONSES_ENDPOINT=http://127.0.0.1:${port}/responses" + "AGENT_RESPONSES_BEARER_TOKEN=local-dummy-token" + "AGENTKIT_EXPECTED_CALL_ID=${expected_call_id}" +) +if [[ -n "$expected_call_id_prefix" ]]; then + helper_env+=("AGENTKIT_EXPECTED_CALL_ID_PREFIX=${expected_call_id_prefix}") +fi +if [[ -n "$continuation_proof" ]]; then + helper_env+=("AGENTKIT_CONTINUATION_PROOF=${continuation_proof}") +fi + +env "${helper_env[@]}" deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read "$transcript_dir" + +image_info="$(docker image inspect "$tag" --format '{{.Id}} {{.Architecture}} {{.Os}}')" +printf 'Local Foundry brokered %s container passed.\n' "$fixture" +printf ' image: %s\n' "$tag" +printf ' image_info: %s\n' "$image_info" +printf ' transcript: %s\n' "$transcript_dir" diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py new file mode 100755 index 0000000..59f05c4 --- /dev/null +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Verify a Foundry brokered Responses conformance transcript. + +The transcript is produced by deploy/foundry/scripts/foundry_brokered_conformance.sh +and is intentionally token-free: it contains request/response JSON only. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +EXPECTED_FILES = ( + "01-initial-request.json", + "02-initial-response.json", + "03-continuation-request.json", + "04-continuation-response.json", +) + + +def _load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ValueError(f"missing transcript file: {path.name}") from exc + except json.JSONDecodeError as exc: + raise ValueError(f"{path.name} is not valid JSON: {exc}") from exc + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def _message_text(response: dict[str, Any]) -> str: + output = response.get("output") + _require(isinstance(output, list) and bool(output), "final response output must be a non-empty array") + message = output[0] + _require(isinstance(message, dict) and message.get("type") == "message", "final response output[0] must be a message") + content = message.get("content") + _require(isinstance(content, list) and bool(content), "final message content must be a non-empty array") + text = content[0].get("text") if isinstance(content[0], dict) else None + _require(isinstance(text, str) and bool(text), "final message must contain text") + return text + + +def verify_transcript( + transcript_dir: str | Path, + *, + expected_tool_name: str = "conformance_read", + expected_arguments_json: str = '{"probe":true}', + expected_call_id: str = "call_conformance_1", + expected_call_id_prefix: str | None = None, +) -> dict[str, Any]: + root = Path(transcript_dir) + expected_arguments = json.loads(expected_arguments_json) + initial_request = _load_json(root / "01-initial-request.json") + initial_response = _load_json(root / "02-initial-response.json") + continuation_request = _load_json(root / "03-continuation-request.json") + continuation_response = _load_json(root / "04-continuation-response.json") + + _require(isinstance(initial_request, dict), "initial request must be a JSON object") + _require("tools" not in initial_request, "initial request must not contain request-level tools") + _require("input" in initial_request, "initial request must contain input") + + _require(isinstance(initial_response, dict), "initial response must be a JSON object") + _require(initial_response.get("status") == "completed", "initial response status must be completed") + initial_response_id = initial_response.get("id") + _require(isinstance(initial_response_id, str) and initial_response_id.startswith("caresp_"), "initial response id must start with caresp_") + _require(not initial_response_id.startswith("resp_"), "initial response id must not use old resp_ format") + output = initial_response.get("output") + _require(isinstance(output, list) and len(output) == 1, "initial response output must contain exactly one item") + call = output[0] + _require(isinstance(call, dict), "initial response output[0] must be an object") + _require(call.get("type") == "function_call", "initial output item must be function_call") + function_name = call.get("name") + _require(function_name == expected_tool_name, f"function_call name must be {expected_tool_name}") + call_id = call.get("call_id") + _require(isinstance(call_id, str) and bool(call_id), "function_call call_id must be a non-empty string") + if expected_call_id != "auto": + _require(call_id == expected_call_id, f"function_call call_id must be {expected_call_id}") + if expected_call_id_prefix: + _require(call_id.startswith(expected_call_id_prefix), f"function_call call_id must start with {expected_call_id_prefix}") + arguments = call.get("arguments") + _require(isinstance(arguments, str), "function_call arguments must be a JSON string") + parsed_arguments = json.loads(arguments) + _require(parsed_arguments == expected_arguments, f"function_call arguments must be {expected_arguments}") + + _require(isinstance(continuation_request, dict), "continuation request must be a JSON object") + _require(continuation_request.get("previous_response_id") == initial_response_id, "continuation previous_response_id must match initial id") + continuation_input = continuation_request.get("input") + _require(isinstance(continuation_input, list) and len(continuation_input) == 1, "continuation input must contain exactly one item") + continuation_item = continuation_input[0] + _require(isinstance(continuation_item, dict), "continuation item must be an object") + _require(continuation_item.get("type") == "function_call_output", "continuation item must be function_call_output") + _require(continuation_item.get("call_id") == call_id, "continuation call_id must match function_call call_id") + continuation_output = continuation_item.get("output") + _require(isinstance(continuation_output, str), "continuation output must be a JSON string") + parsed_output = json.loads(continuation_output) + _require(isinstance(parsed_output, dict), "continuation output JSON must be an object") + + _require(isinstance(continuation_response, dict), "continuation response must be a JSON object") + _require(continuation_response.get("status") == "completed", "continuation response status must be completed") + _require(continuation_response.get("previous_response_id") == initial_response_id, "continuation response previous_response_id must match initial id") + continuation_response_id = continuation_response.get("id") + _require(isinstance(continuation_response_id, str) and continuation_response_id.startswith("caresp_"), "continuation response id must start with caresp_") + final_text = _message_text(continuation_response) + + return { + "initial_response_id": initial_response_id, + "continuation_response_id": continuation_response_id, + "function_call_name": function_name, + "call_id": call_id, + "arguments": parsed_arguments, + "final_text": final_text, + "transcript_files": list(EXPECTED_FILES), + } + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Verify a Foundry brokered conformance transcript directory.") + parser.add_argument("transcript_dir", help="directory containing 01/02/03/04 conformance transcript JSON files") + parser.add_argument("--expected-tool-name", default="conformance_read", help="expected function_call name") + parser.add_argument("--expected-arguments-json", default='{"probe":true}', help="expected function_call arguments JSON") + parser.add_argument("--expected-call-id", default="call_conformance_1", help="expected call_id, or 'auto' to only require a non-empty id") + parser.add_argument("--expected-call-id-prefix", default=None, help="optional required call_id prefix") + parser.add_argument("--write-summary", action="store_true", help="write summary.json in the transcript directory") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + try: + summary = verify_transcript( + args.transcript_dir, + expected_tool_name=args.expected_tool_name, + expected_arguments_json=args.expected_arguments_json, + expected_call_id=args.expected_call_id, + expected_call_id_prefix=args.expected_call_id_prefix, + ) + if args.write_summary: + Path(args.transcript_dir, "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + except Exception as exc: # noqa: BLE001 - CLI verifier should print concise evidence failures. + print(f"verify_brokered_transcript: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/agent-abi.md b/docs/agent-abi.md index 97fbe25..ec0a6a9 100644 --- a/docs/agent-abi.md +++ b/docs/agent-abi.md @@ -57,6 +57,20 @@ tools: type: bearer tokenEnv: TOOLBOX_TOKEN +# Static safe schemas for Foundry hosted Orka-brokered mode. These are schema-only; +# no execution URL, auth header, token, or Secret ref is allowed here. +brokeredTools: + - name: check-network-telemetry + description: Read sanitized optical telemetry. + brokeredClass: read + parameters: + type: object + properties: + site: + type: string + required: [site] + schemaDigest: sha256: + env: - name: REQUIRED_FOO required: true @@ -97,6 +111,7 @@ expose: | `model` | yes | Hosted OpenAI-compatible model connection metadata. | | `instructions` | yes | Fully-resolved system prompt scalar. | | `tools` | no | Owned MCP tools, either stdio or Streamable HTTP. | +| `brokeredTools` | no | Static safe Orka-brokered tool schemas for Foundry hosted Responses mode. | | `env` | no | Runtime env var requirements by name only. | | `context` | no | Provider-neutral context providers; runtime capability-gated. Filesystem skills paths must be pre-staged under `/agent/skills`; arbitrary build-context directories are not copied into the image. | | `observability` | no | Provider-neutral observability env names; runtime capability-gated. | @@ -147,6 +162,21 @@ Supported auth types: - `workload-identity-token` with opaque `audience`, only for runtimes that declare `workload-identity-token-auth`. +## Brokered tool schemas + +`brokeredTools` is used by Foundry hosted brokered Responses mode. It is +intentionally schema-only: the reader rejects execution URLs, auth/header/token +fields, Secret refs, unsafe parameter names, unknown brokered classes, malformed +JSON Schema, duplicate names, and owned-tool/brokered-tool name overlap. + +When `schemaDigest` is present, it must match the deterministic digest of the +safe model-facing schema fields. Generate it from Orka Tool CRDs during +deployment so stale or hand-edited schemas fail before a live run. Orka remains +the execution and policy authority even when AgentKit's static schema is valid. + +See `docs/foundry-hosted-brokered.md` for the hosted Responses continuation +lifecycle and state/scaling limits. + ## Reader contract The shared Python runtime core must: diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md new file mode 100644 index 0000000..f5e209f --- /dev/null +++ b/docs/foundry-hosted-brokered.md @@ -0,0 +1,365 @@ +# Foundry hosted brokered Responses mode + +AgentKit can expose a Foundry hosted `/responses` surface that pauses on an +Orka-brokered function call and resumes only after Orka returns a +`function_call_output`. This mode is for hosted agents that must let Orka remain +the sole policy, approval, credential, idempotency, and execution authority. + +## Security invariant + +Hosted AgentKit receives only safe brokered schemas: + +- tool name; +- description; +- brokered class: `read`, `write`, or `coordination`; +- JSON parameters schema; and +- optional `schemaDigest` drift metadata. + +Hosted AgentKit must never receive Orka Tool execution URLs, Kubernetes Secret +refs, auth headers, bearer tokens, downstream credentials, approval-bypass +metadata, or durable Orka control-plane credentials. Direct AgentKit-owned tools +are not allowed in the same v0 `agent.yaml` as `brokeredTools`; mixed owned-tool +and brokered-tool mode is intentionally deferred. `/invocations` is disabled when +`brokeredTools` are configured so it cannot bypass Orka. The container emits a +Responses `function_call` item and waits for Orka to execute and resume it. + +## Static schema configuration + +Foundry hosted-agent endpoint-scoped `/responses` calls do not accept dynamic +request-level `tools` in this mode. Instead, bake safe schemas into `agent.yaml`: + +```yaml +brokeredTools: + - name: check-network-telemetry + description: Read sanitized optical telemetry. + brokeredClass: read + parameters: + type: object + properties: + site: + type: string + required: [site] + schemaDigest: sha256: +``` + +The ABI loader rejects unsafe fields such as `url`, `headers`, `secretRef`, +`auth`, and `token`, rejects unknown brokered classes, requires a top-level JSON +Schema `type: object`, and validates `schemaDigest` when present. + +## Drift workflow + +Orka Tool CRDs remain the source of truth. Deployment tooling should export only +the safe model-facing subset into AgentKit `brokeredTools`. The shared helper +`agentkit_serve_common.brokered.generate_brokered_tools_from_orka_tool_crds` and +its CLI wrapper produce deterministic entries and compute `schemaDigest`; AgentKit +startup fails if a configured digest no longer matches the safe schema in +`agent.yaml`. Orka still validates every live call against current Tool CRDs at +execution time. + +Example export command: + +```sh +agentkit-brokered-tools ./orka-tools/*.yaml -o brokered-tools.generated.yaml +``` + +The output is an `agent.yaml` fragment shaped as: + +```yaml +brokeredTools: + - name: check-network-telemetry + description: Read telemetry. + brokeredClass: read + parameters: + type: object + schemaDigest: sha256:... +``` + +Use `--no-digest` only for ad-hoc demos where drift failure is not desired, and +`--bare` when another deployment templater owns the top-level `brokeredTools` key. + +## Responses lifecycle + +Initial request: + +```json +{"input":"please read telemetry"} +``` + +Deterministic local brokered mode requires the prompt to name exactly one +configured tool whenever multiple tools are configured, and for any non-conformance +single tool. Only the special `conformance_read` smoke-test schema may be selected +without an explicit tool-name mention. If no configured tool name is present when +explicit selection is required, the request is rejected with +`brokered_tool_selection_required` instead of silently choosing the wrong tool. A +real model-adapter implementation must replace this deterministic selection with +model-driven tool choice. + +Brokered response: + +```json +{ + "status": "completed", + "output": [ + { + "type": "function_call", + "call_id": "call__1", + "name": "conformance_read", + "arguments": "{\"probe\":true}", + "status": "completed" + } + ] +} +``` + +Continuation request: + +```json +{ + "previous_response_id": "", + "input": [ + { + "type": "function_call_output", + "call_id": "call__1", + "output": "{\"approved\":true,\"output\":{\"success\":true}}", + "status": "completed" + } + ] +} +``` + +Canonical approved payload exposed back to the model: + +```json +{"approved":true,"output":{"success":true}} +``` + +Canonical denied/error payload: + +```json +{ + "approved": false, + "error": { + "code": "approval_declined", + "message": "Human declined dispatch-work-order" + } +} +``` + +`function_call_output` is privileged continuation input. It is rejected unless a +known `previous_response_id` has a pending matching `call_id` **and** the request +uses the Orka-only continuation path. Configure +`AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` and have the Orka hosted-Responses +adapter send it as `X-AgentKit-Brokered-Continuation-Proof`; ordinary client +requests must not be able to submit tool results. Unknown response IDs, unknown +call IDs, orphan tool outputs, duplicate conflicts, malformed outputs, missing or +wrong continuation auth, and multiple tool outputs all return deterministic error +envelopes. An identical duplicate continuation from the broker path returns the +same final response idempotently. + +## Response IDs and state + +The adapter uses the Azure hosted Responses SDK ID generator when available, so +new response IDs use the hosted-compatible `caresp_...` form instead of the old +hand-rolled `resp_` form. + +`/readiness` fails with HTTP 503 when continuation auth is missing and reports +`continuationAuth: missing`; the adapter also refuses to start brokered +function-call responses until the proof is configured. + +By default the state backend is in-memory and suitable for deterministic local +smoke and single-replica demos only. Set +`AGENTKIT_FOUNDRY_RESPONSE_STATE_FILE=/path/to/state.json` to persist pending and +completed brokered response state as an atomically rewritten JSON file; with a +shared persistent volume and sticky/single-writer deployment this allows a known +`previous_response_id` to survive a container restart. Deployments without a +shared file or platform-managed store must pin one replica or use sticky routing; +otherwise a continuation that lands on a different/restarted container fails +safely with `unknown_previous_response_id`. Pending state expires after +`AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS` (default: 900 seconds); expired +continuations fail with `response_state_expired`. The store is bounded by +`AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING` (default: 128), and generated +brokered arguments are bounded by `AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES` +(default: 8192) before state is accepted. A platform-managed state backend is +still required before treating multi-replica production as fully supported. + +## Streaming + +The current route is non-streaming. If clients send `stream: true`, AgentKit +returns the same normal JSON response rather than SSE. This keeps azd/direct curl +smokes deterministic while making streaming support an explicit future step. + +## Troubleshooting + +- `invocations_disabled_in_brokered_mode`: call `/responses`; brokered mode disables `/invocations` to avoid direct tool bypass. +- `tools_unsupported`: remove request-level `tools`; use static `brokeredTools`. +- `tool_choice_unsupported`: brokered Foundry mode owns tool selection. +- `brokered_continuation_auth_required`: set `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` before accepting brokered continuations. +- `brokered_continuation_forbidden`: only Orka should send `X-AgentKit-Brokered-Continuation-Proof` with the configured proof. +- `missing_previous_response_id`: a `function_call_output` cannot start a new run. +- `unknown_previous_response_id`: state is missing, expired/purged, or routed to a + different replica. +- `response_pending_function_call_output`: finish the pending brokered tool call before sending a normal follow-up turn against that response. +- `brokered_tool_selection_required`: name exactly one configured brokered tool in deterministic mode when multiple schemas or any non-conformance single schema are configured. +- `brokered_response_state_full`: too many uncontinued brokered responses are pending. Completed entries are evicted before this error is returned. +- `brokered_arguments_too_large`: generated brokered call arguments exceeded the configured pending-state byte budget. +- `unknown_call_id`: the output did not match the pending function call. +- `conflicting_duplicate_continuation`: the same `call_id` was already completed + with different output. + +## SDK conformance spike app + +`agentkit_serve_common.foundry_conformance.create_foundry_conformance_app()` is a +small, production-independent hosted Responses app for Phase A0 smokes. It is +also packaged as the `agentkit-foundry-conformance` console script so the same +app can be used as a hosted container entrypoint. It uses +`azure-ai-agentserver-responses` for request parsing, SDK-assigned `caresp_...` +response IDs, response envelopes, event sequencing, and in-memory response state. +It returns the deterministic `conformance_read` function call on the first +`/responses` request and completes when resumed with the matching +`function_call_output`. + +Local runnable entrypoint check: + +```sh +uv run --directory runtimes/common --extra dev agentkit-foundry-conformance --dry-run +``` + +Container entrypoint example for the A0 spike image: + +```Dockerfile +ENTRYPOINT ["agentkit-foundry-conformance", "--host", "0.0.0.0", "--port", "8088"] +``` + +A minimal deployable container fixture lives in `test/foundry-brokered-conformance/`: + +```sh +docker buildx build --builder desktop-linux . \ + -f test/foundry-brokered-conformance/Dockerfile \ + --platform linux/amd64 \ + -t agentkit-foundry-brokered-conformance:test --load --provenance=false +``` + +Use `test/foundry-brokered-conformance/foundry.agent.yaml.example` as the hosted +agent manifest template; it advertises `responses` protocol version `2.0.0` for +this A0 spike. You can also run the transcript helper against a local container +by setting `AGENT_RESPONSES_ENDPOINT=http://127.0.0.1:18088/responses` and +`AGENT_RESPONSES_BEARER_TOKEN=local-dummy-token`. + +For a one-command local pre-deploy smoke of the SDK conformance image, run: + +```sh +deploy/foundry/scripts/local_brokered_conformance_container.sh \ + --fixture sdk \ + --platform linux/amd64 \ + --transcript-dir ./foundry-brokered-local-transcript +``` + +Local proof: + +```sh +uv run --directory runtimes/common --extra dev pytest -q tests/test_foundry_conformance.py +``` + +Live direct-endpoint proof after deployment: + +```sh +export AGENT_RESPONSES_ENDPOINT="https:///responses" +export AZURE_SUBSCRIPTION_ID="" +deploy/foundry/doctor.sh --brokered-conformance +deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundry-brokered-transcript +``` + +The script performs the initial `function_call` request, posts the matching +`function_call_output` continuation with `previous_response_id`, asserts SDK-style +`caresp_...` IDs, and saves request/response JSON plus `summary.json` as a +sanitized transcript. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py ` to verify archived transcript evidence later. This live transcript is still required before claiming A0 +completion; the local test only proves the SDK-hosted contract before deployment. + +## Production brokered-only fixture + +`test/foundry-brokered-agentkit/` packages the production AgentKit Foundry +brokered path, not the standalone SDK conformance app. It bakes a minimal +`agent.yaml` with static `brokeredTools` and runs: + +```sh +agentkit-foundry-brokered --config /agent/agent.yaml --host 0.0.0.0 --port 8088 +``` + +Local proof for this production adapter path: + +```sh +docker build . -f test/foundry-brokered-agentkit/Dockerfile -t agentkit-foundry-brokered:local +docker run --rm \ + -e AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=local-dev-proof \ + -p 127.0.0.1:18092:8088 \ + agentkit-foundry-brokered:local + +AGENT_RESPONSES_ENDPOINT=http://127.0.0.1:18092/responses \ +AGENT_RESPONSES_BEARER_TOKEN=local-dummy-token \ +AGENTKIT_CONTINUATION_PROOF=local-dev-proof \ +AGENTKIT_EXPECTED_CALL_ID=auto \ +AGENTKIT_EXPECTED_CALL_ID_PREFIX=call_ \ +deploy/foundry/scripts/foundry_brokered_conformance.sh \ + conformance_read ./foundry-brokered-agentkit-transcript +``` + +Use this fixture when you want to validate AgentKit's generated `call__1` +call IDs and continuation-proof enforcement before pushing a real brokered +AgentKit image. The all-in-one local helper supports this fixture too: + +```sh +deploy/foundry/scripts/local_brokered_conformance_container.sh \ + --fixture agentkit \ + --platform linux/amd64 \ + --transcript-dir ./foundry-brokered-agentkit-transcript +``` + +## Lower-level model-loop fallback + +Phase A4/A5 has an opt-in fallback when a high-level framework cannot prove +pause/resume: set `AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP=1` in Foundry brokered +mode. AgentKit then calls the configured OpenAI-compatible chat-completions +model directly with the static safe `brokeredTools` as function schemas. If the +model requests exactly one configured tool, AgentKit rewrites the model's tool +call id to a stable hosted Responses `call__` id and +returns a `function_call` output item for Orka. On the Orka-authenticated +`function_call_output` continuation, AgentKit resumes the model with a `tool` +message and returns the final assistant message. + +In this mode AgentKit-owned MCP/direct tools remain disabled; only the static +safe brokered schemas are model-visible. The first implementation intentionally +limits each turn to one brokered tool call and rejects unknown, multiple, or +repeated model tool calls deterministically. + +## Implementation status and evidence + +This section records the current AgentKit-side evidence against the implementation +plan. It is intentionally explicit about what is locally proven versus what still +requires deployed Foundry/Orka/Fibey state. + +| Plan area | Current AgentKit status | Evidence | Remaining gate | +|---|---|---|---| +| Golden hosted Responses fixtures | Implemented locally | `runtimes/common/tests/fixtures/foundry_brokered/*`, `tests/test_foundry_brokered_protocol.py` | Orka repo must consume/verify the same wire shapes. | +| A0 SDK hosted Responses spike | Local SDK app and container fixture implemented | `agentkit_serve_common.foundry_conformance`, `agentkit-foundry-conformance`, `test/foundry-brokered-conformance/`, `tests/test_foundry_conformance.py` | Deploy to Foundry and archive a verified live transcript with `deploy/foundry/scripts/foundry_brokered_conformance.sh`. | +| A1 hosted Responses lifecycle/state | Implemented for deterministic/local brokered mode; file-backed state available for single-writer/sticky deployments | `agentkit_serve_common.foundry.create_foundry_app`, `tests/test_foundry_brokered_protocol.py` | Live Foundry must accept generated response IDs and state/routing constraints must be chosen for deployment. | +| A2 static schemas and drift control | Implemented in Go writer/validator and Python runtime; export CLI added | `brokeredTools` ABI, `agentkit-brokered-tools`, `tests/test_config_validation.py`, `tests/test_brokered_schema.py`, Go config/ABI tests | Orka Tool CRDs must be exported during deployment and current digests verified before live runs. | +| A3 deterministic brokered runtime | Implemented for local/fake hosted protocol integration | deterministic `/responses` brokered path and tests | Live Orka deterministic read/write smoke still required. | +| A4 framework pause/resume decision | Lower-level OpenAI-compatible fallback implemented; high-level framework native hooks remain gated | `agentkit_serve_common.foundry_model_loop`, `AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP=1`, model-loop tests | Live model smoke for brokered read/write prompts. | +| A5 first real model adapter brokered mode | Fallback model loop can emit/resume brokered calls from static safe schemas | model-loop tests in `tests/test_foundry_brokered_protocol.py` | Deployed real model read and write prompts, including declined/policy/error outcomes. | +| A6 live Orka integration | Not proven in this repo state | Local AgentKit/Foundry side helpers exist | Deploy AgentKit and Orka hosted-Responses adapter; run brokered read/write approval smoke. | +| A7 Fibey | Not started; gates not satisfied | N/A | Requires A3/A5/A6 live gates first, then Fibey schemas/instructions/scenario. | +| A8 hardening/docs/review | Local docs/tests/autoreview complete for current patch | This doc, `docs/agent-abi.md`, `docs/runtime-capabilities.md`; full tests/lint; `$autoreview` clean | Record live transcript and Orka/Fibey validation evidence before final completion. | + +Local verification commands used for the current AgentKit patch: + +```sh +uv run --directory runtimes/common --extra dev pytest -q +go test ./... +make lint +git diff --check +.agents/skills/autoreview/scripts/autoreview +``` + +Completion must still be judged against live evidence: this local status matrix is +not a substitute for Foundry accepting `previous_response_id`, Orka brokering the +actual read/write calls, or Fibey completing the end-to-end scenario. diff --git a/docs/runtime-capabilities.md b/docs/runtime-capabilities.md index 8c80dc1..8fb6cfe 100644 --- a/docs/runtime-capabilities.md +++ b/docs/runtime-capabilities.md @@ -17,8 +17,11 @@ Current and reserved names: - `foundry-invocations-protocol` — Foundry hosted-agent `/readiness` + `/invocations` wrapper. - `foundry-responses-minimal` — current Foundry `/responses` wrapper: synchronous - and non-streaming. Do not treat this as full Responses parity for background, - streaming, polling, cancel, or durable response IDs. + and non-streaming. It supports a deterministic schema-only brokered function-call + loop when `agent.yaml` contains `brokeredTools`, using hosted-compatible + response IDs and a memory or optional file-backed continuation store. Do not + treat this as full Responses parity for background, streaming, polling, cancel, + or multi-replica platform-managed production state. - `orka-harness-v1` — observed-mode native Orka `orka.harness.v1` wire protocol over HTTP+SSE (`HealthResponse`, flat `CapabilitiesResponse`, `StartTurnRequest`, `StartTurnResponse`, and `HarnessEventFrame`). - `orka-observed-tools` — AgentKit-owned tools/MCP execute inside the runtime; Orka observes lifecycle/output frames and governs externally. @@ -50,8 +53,9 @@ side-effect governance. Context-provider schemas are capability-gated per runtime; the MAF adapter currently declares skills, search, and memory support. OTel export, local tool -approval enforcement, log-level observability, and Orka brokered-tool mode remain -gated until a runtime and protocol contract declare support. +approval enforcement, log-level observability, and native real-model Orka +brokered-tool adapters remain gated until a runtime and protocol contract declare +support. The shared runtime package now defines the neutral brokered-tool Interface types (`BrokeredToolDefinition`, `BrokeredToolCall`, `BrokeredToolResult`, @@ -61,7 +65,13 @@ read/write/coordination and `/continue` behind `AGENTKIT_ORKA_ENABLE_BROKERED_READ=1`, `AGENTKIT_ORKA_ENABLE_BROKERED_WRITE=1`, and `AGENTKIT_ORKA_ENABLE_BROKERED_COORDINATION=1`; default capabilities still -advertise observed mode only. Orka remains responsible for coordination policy, +advertise observed mode only. Foundry hosted `/responses` can also exercise a +deterministic brokered function-call loop from static `brokeredTools`. For +A4/A5 fallback validation, `AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP=1` enables a +lower-level OpenAI-compatible chat-completions loop that exposes static safe +brokered schemas as function tools, emits hosted Responses `function_call` +items, and resumes the model with Orka-provided `function_call_output`. Orka +remains responsible for coordination policy, quotas, child-task lineage, and namespace/agent authorization. Native framework adapter brokered hooks are still intentionally gated: today the brokered profiles are validated through the offline echo/conformance runtime, while real model diff --git a/pkg/agentkit/abi/render.go b/pkg/agentkit/abi/render.go index aa33b95..fd88fd1 100644 --- a/pkg/agentkit/abi/render.go +++ b/pkg/agentkit/abi/render.go @@ -3,6 +3,10 @@ package abi import ( + "encoding/json" + "strconv" + "strings" + "github.com/goccy/go-yaml" "github.com/sozercan/agentkit/pkg/agentkit/effective" ) @@ -19,6 +23,12 @@ const Path = "/agent/agent.yaml" // extra="forbid", so these structs MUST emit EXACTLY the keys documented there. // Field order here is the emit order. +type yamlNumber string + +func (n yamlNumber) MarshalYAML() ([]byte, error) { + return []byte(n), nil +} + type abiMetadata struct { Name string `yaml:"name"` } @@ -55,6 +65,14 @@ type abiTool struct { Env []string `yaml:"env,omitempty"` } +type abiBrokeredTool struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + BrokeredClass string `yaml:"brokeredClass"` + Parameters map[string]any `yaml:"parameters"` + SchemaDigest string `yaml:"schemaDigest,omitempty"` +} + type abiEnvVar struct { Name string `yaml:"name"` Required bool `yaml:"required,omitempty"` @@ -97,12 +115,128 @@ type abiAgent struct { Model abiModel `yaml:"model"` Instructions string `yaml:"instructions"` Tools []abiTool `yaml:"tools"` + BrokeredTools []abiBrokeredTool `yaml:"brokeredTools,omitempty"` Env []abiEnvVar `yaml:"env,omitempty"` Context *abiContext `yaml:"context,omitempty"` Observability *abiObservability `yaml:"observability,omitempty"` Expose abiExpose `yaml:"expose"` } +func expandJSONNumber(value string) string { + lower := strings.ToLower(value) + parts := strings.Split(lower, "e") + if len(parts) != 2 { + return value + } + exponent, err := strconv.Atoi(parts[1]) + if err != nil { + return value + } + mantissa := parts[0] + sign := "" + if strings.HasPrefix(mantissa, "-") || strings.HasPrefix(mantissa, "+") { + if mantissa[0] == '-' { + sign = "-" + } + mantissa = mantissa[1:] + } + fracLen := 0 + if dot := strings.IndexByte(mantissa, '.'); dot >= 0 { + fracLen = len(mantissa) - dot - 1 + mantissa = mantissa[:dot] + mantissa[dot+1:] + } + mantissa = strings.TrimLeft(mantissa, "0") + if mantissa == "" { + return "0" + } + decimalPos := len(mantissa) - fracLen + exponent + var out string + switch { + case decimalPos <= 0: + out = "0." + strings.Repeat("0", -decimalPos) + mantissa + case decimalPos >= len(mantissa): + out = mantissa + strings.Repeat("0", decimalPos-len(mantissa)) + default: + out = mantissa[:decimalPos] + "." + mantissa[decimalPos:] + } + if strings.Contains(out, ".") { + out = strings.TrimRight(out, "0") + out = strings.TrimRight(out, ".") + } + if out == "" || out == "-" { + return "0" + } + return sign + out +} + +func copyMap(in map[string]any) map[string]any { + if in == nil { + return nil + } + out := make(map[string]any, len(in)) + for k, v := range in { + out[k] = copyAny(v) + } + return out +} + +func copyAny(v any) any { + switch typed := v.(type) { + case map[string]any: + return copyMap(typed) + case map[string]string: + out := make(map[string]string, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case map[string]int: + out := make(map[string]int, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case map[string]float64: + out := make(map[string]any, len(typed)) + for key, value := range typed { + out[key] = yamlNumber(strconv.FormatFloat(value, 'f', -1, 64)) + } + return out + case map[string]bool: + out := make(map[string]bool, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = copyAny(item) + } + return out + case []string: + return append([]string(nil), typed...) + case []int: + return append([]int(nil), typed...) + case []float64: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = yamlNumber(strconv.FormatFloat(item, 'f', -1, 64)) + } + return out + case []bool: + return append([]bool(nil), typed...) + case float32: + return yamlNumber(strconv.FormatFloat(float64(typed), 'f', -1, 32)) + case float64: + return yamlNumber(strconv.FormatFloat(typed, 'f', -1, 64)) + case json.Number: + return yamlNumber(expandJSONNumber(typed.String())) + default: + return typed + } +} + // Render produces the baked /agent/agent.yaml from an effective Agent. The // output is byte-compatible with agentkit-serve's strict (extra=forbid) reader. func Render(agent effective.Agent) ([]byte, error) { @@ -145,6 +279,18 @@ func Render(agent effective.Agent) ([]byte, error) { } out.Tools = append(out.Tools, tool) } + for _, t := range agent.BrokeredTools { + out.BrokeredTools = append(out.BrokeredTools, abiBrokeredTool{ + Name: t.Name, + Description: t.Description, + BrokeredClass: t.BrokeredClass, + Parameters: copyMap(t.Parameters), + SchemaDigest: t.SchemaDigest, + }) + } + if len(out.BrokeredTools) == 0 { + out.BrokeredTools = nil + } for _, e := range agent.Env { out.Env = append(out.Env, abiEnvVar{Name: e.Name, Required: e.Required}) } diff --git a/pkg/agentkit/abi/render_test.go b/pkg/agentkit/abi/render_test.go index b02cc38..e4848fa 100644 --- a/pkg/agentkit/abi/render_test.go +++ b/pkg/agentkit/abi/render_test.go @@ -1,6 +1,7 @@ package abi import ( + "encoding/json" "os" "strings" "testing" @@ -18,6 +19,11 @@ const ( testInstructions = "Be helpful and cite sources." ) +const ( + jsonSchemaTypeKey = "type" + jsonSchemaMinimumKey = "minimum" +) + func sampleConfig() *config.AgentConfig { return &config.AgentConfig{ APIVersion: "v1alpha1", @@ -245,3 +251,60 @@ func TestRenderAgentYAMLIncludesModelWorkloadIdentityAuth(t *testing.T) { } } } + +func TestRenderAgentYAMLIncludesBrokeredTools(t *testing.T) { + cfg := sampleConfig() + cfg.Tools = nil + cfg.BrokeredTools = []config.BrokeredTool{{ + Name: "check-network-telemetry", + Description: "Read telemetry.", + BrokeredClass: config.BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: "object", + "properties": map[string]any{ + "site": map[string]any{jsonSchemaTypeKey: "string", jsonSchemaMinimumKey: 0.000001}, + "typedFloats": map[string]float64{jsonSchemaMinimumKey: 0.000001}, + "empty": map[string]any{}, + }, + }, + }} + out, err := Render(effective.FromConfig(cfg, testInstructions)) + if err != nil { + t.Fatalf("render error: %v", err) + } + s := string(out) + for _, want := range []string{"brokeredTools:", "name: check-network-telemetry", "description: Read telemetry.", "brokeredClass: read", "properties:", "site:", "minimum: 0.000001", "typedFloats:", "empty: {}"} { + if !strings.Contains(s, want) { + t.Fatalf("rendered agent.yaml missing %q\n---\n%s", want, s) + } + } + for _, never := range []string{"url:", "secretRef:", "headers:", "auth" + ":", "1e-06"} { + if strings.Contains(s, never) { + t.Fatalf("rendered brokered agent.yaml leaked %q\n---\n%s", never, s) + } + } +} + +func TestRenderAgentYAMLFormatsJSONNumberBrokeredSchemaValuesAsNumbers(t *testing.T) { + cfg := sampleConfig() + cfg.Tools = nil + cfg.BrokeredTools = []config.BrokeredTool{{ + Name: "numeric-tool", + Description: "Read numeric data.", + BrokeredClass: config.BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: "object", + "properties": map[string]any{ + "small": map[string]any{jsonSchemaTypeKey: "number", jsonSchemaMinimumKey: json.Number("1e-7")}, + }, + }, + }} + + out, err := Render(effective.FromConfig(cfg, testInstructions)) + if err != nil { + t.Fatalf("render error: %v", err) + } + if !strings.Contains(string(out), "minimum: 0.0000001") || strings.Contains(string(out), "1e-7") { + t.Fatalf("rendered agent.yaml did not preserve json.Number as fixed YAML number\n---\n%s", out) + } +} diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index e3a9aae..c874127 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -1,12 +1,20 @@ package config import ( + "encoding/json" "strings" "testing" "github.com/sozercan/agentkit/pkg/agentkit/runtimes" ) +const ( + safeLookupToolName = "safe_lookup" + jsonSchemaPropertiesKey = "properties" + brokeredSiteField = "site" + brokeredSafeDescription = "safe schema" +) + // TestKindProbeRejectsKindlessFile is the regression guard for the AIKit // silent-misparse bug (plan §5.1/§16.1): a file without `kind: Agent` must be a // loud load-time error, never a silently-empty config. @@ -445,6 +453,63 @@ expose: } } +func TestValidateRejectsUnsupportedBrokeredSchemaCompositionKeywords(t *testing.T) { + for _, keyword := range []string{"allOf", "anyOf", "oneOf", "$ref", "if", "then", "else", "contains", "propertyNames", "dependentSchemas", "patternProperties", "unevaluatedProperties", "uniqueItems"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + keyword: []any{}, + }, + }} + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "not supported") { + t.Fatalf("keyword %q: expected unsupported schema rejection, got: %v", keyword, err) + } + } +} + +func TestValidateAcceptsBrokeredPropertyNamedType(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + jsonSchemaTypeKey: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString}, + }, + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("property named type should validate: %v", err) + } +} + +func TestValidateRejectsInvalidBrokeredSchemaTypeValuesAndDefaults(t *testing.T) { + cases := []map[string]any{ + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: nil}}}, + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{"n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: "1"}}}, + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, "enum": []any{0, "ok"}}}}, + } + for _, schema := range cases { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: schema, + }} + if err := cfg.Validate(); err == nil { + t.Fatalf("expected invalid schema rejection for %#v", schema) + } + } +} + func TestValidateRejectsInvalidRemoteMCPToolShapes(t *testing.T) { cases := map[string]string{ //nolint:gosec // test YAML uses credential-looking field names/invalid examples, not real secrets "missing type": `tools: @@ -916,3 +981,442 @@ expose: t.Fatalf("expected logs support error, got: %v", verr) } } + +func TestBrokeredToolSchemaDigestMatchesPythonCanonicalJSON(t *testing.T) { + tool := BrokeredTool{ + Name: "check-network-telemetry", + Description: "São & R&D ", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, brokeredDigestDescriptionKey: "São & R&D "}, + }, + jsonSchemaRequiredKey: []any{brokeredSiteField}, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:7066de4e62dd1a6550701772aad901e1efcf5eb81a4f639252f82e6c7be8d4c1" { + t.Fatalf("digest = %q", digest) + } +} + +func TestBrokeredToolSchemaDigestNormalizesIntegerValuedFloatConstraints(t *testing.T) { + tool := BrokeredTool{ + Name: "numeric-tool", + Description: "Numeric constraints.", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "retries": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMinimumKey: 1.0}, + }, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:7ad9d43791e157981bcd65fd8452c9e71a64064875cc1330ced42d4956bf7d75" { + t.Fatalf("digest = %q", digest) + } +} + +func TestBrokeredToolSchemaDigestCanonicalizesNumericConstraints(t *testing.T) { + tool := BrokeredTool{ + Name: "num-tool", + Description: "Numeric schema", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMinimumKey: 1e-6}, + }, + jsonSchemaRequiredKey: []any{"n"}, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:83bf12180154a21f8ba19049687e24acee9ef430966af67a83721a10bf7eee50" { + t.Fatalf("digest = %q", digest) + } +} + +func TestBrokeredToolSchemaDigestCanonicalizesPositiveExponentNumbers(t *testing.T) { + tool := BrokeredTool{ + Name: "large-num-tool", + Description: "Large numeric schema", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMaximumKey: 1e20}, + }, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:51ebe9cdbb967453ba3ae9fb737566028814968d8121ba0d32825f7c8ffb5639" { + t.Fatalf("digest = %q", digest) + } +} + +func TestValidateAcceptsDigestForTypedNestedBrokeredSchemaMaps(t *testing.T) { + tool := BrokeredTool{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]string{jsonSchemaTypeKey: jsonSchemaTypeString}, + }, + }, + } + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + tool.SchemaDigest = digest + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{tool} + + if err := cfg.Validate(); err != nil { + t.Fatalf("valid typed nested schema with digest failed validation: %v", err) + } +} + +func TestBrokeredToolSchemaDigestPreservesLargeIntegers(t *testing.T) { + tool := BrokeredTool{ + Name: "large-int-tool", + Description: "Large integer schema", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, "const": int64(9007199254740993)}, + }, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:d0e703a2d84f12caa3275fbf8632ec8626ecc84fb27e187380ed61d54cef0e23" { + t.Fatalf("digest = %q", digest) + } +} + +func TestBrokeredToolSchemaDigestCanonicalizesJSONNumberExponentSpellings(t *testing.T) { + tool := BrokeredTool{ + Name: "json-number-tool", + Description: "JSON number schema", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMinimumKey: json.Number("1e-7")}, + }, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:f38729b2c519d7a92f3ddcbb7139271a0a6e25f515e180fa5a16737e5a0efcb4" { + t.Fatalf("digest = %q", digest) + } +} + +func TestBrokeredToolSchemaDigestCanonicalizesExponentNumberSpellings(t *testing.T) { + tool := BrokeredTool{ + Name: "exponent-num-tool", + Description: "Exponent numeric schema", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "small": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, "minimum": 1e-7}, + "large": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMaximumKey: 1e21}, + }, + }, + } + + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + if digest != "sha256:bb4fac58c3f65a33ed3c4ebfa10e5da9c3dc5a9250a1316f64d25e40e0e645e0" { + t.Fatalf("digest = %q", digest) + } +} + +func TestValidateAcceptsBrokeredToolsWithMatchingDigest(t *testing.T) { + tool := BrokeredTool{ + Name: "check-network-telemetry", + Description: "Read sanitized optical telemetry.", + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]any{"type": "string"}, + }, + jsonSchemaRequiredKey: []any{brokeredSiteField}, + }, + } + digest, err := BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest: %v", err) + } + tool.SchemaDigest = digest + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{tool} + + if err := cfg.Validate(); err != nil { + t.Fatalf("valid brokered tool failed validation: %v", err) + } +} + +func TestValidateRejectsUnsafeBrokeredDescriptions(t *testing.T) { + for _, description := range []string{"contains sk-secret", "execution at https://tool.default", "Bearer token required", "execution at tool.default.svc.cluster.local"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: description, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeObject}, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "description") { + t.Fatalf("description %q: expected unsafe description rejection, got: %v", description, err) + } + } +} + +func TestValidateRejectsTypedNestedUnsafeBrokeredSchemaValues(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]string{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaDefaultKey: "sk-not-a-real-secret"}, + }, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "secret-like") { + t.Fatalf("expected typed nested unsafe value rejection, got: %v", err) + } +} + +func TestValidateRejectsPrivateKeyBrokeredNamesAndStrings(t *testing.T) { + for _, params := range []map[string]any{ + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{"privateKey": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString}}}, + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaDefaultKey: "BEGIN PRIVATE KEY"}}}, + } { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: params, + }} + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "safe") && !strings.Contains(err.Error(), "secret-like") { + t.Fatalf("expected private key rejection, got: %v", err) + } + } +} + +func TestValidateRejectsUnsafeBrokeredSchemaStringValues(t *testing.T) { + for _, value := range []string{"see https://internal-tool", "Bearer abc", "Bearer: abc", "Bearer=abc", "authorization header", "tool.default.svc.cluster.local", "example ghp_not_real", "AWS key AKIAEXAMPLE"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, brokeredDigestDescriptionKey: value}, + }, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "secret-like") && !strings.Contains(err.Error(), "URL") { + t.Fatalf("value %q: expected unsafe schema string rejection, got: %v", value, err) + } + } +} + +func TestValidateRejectsCommonCredentialBrokeredParameterNames(t *testing.T) { + for _, field := range []string{"authentication", "authConfig", "clientSecret", "dbPassword", "passphrase", "pwd", "apiKey", credentialHeaderAPIKey, "baseUrl", "callbackURL", "apiEndpoint", "sessionCookie", "cookies"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + field: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString}, + }, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "not safe") { + t.Fatalf("field %q: expected unsafe brokered schema rejection, got: %v", field, err) + } + } +} + +func TestValidateRejectsCommonCredentialBrokeredRequiredNames(t *testing.T) { + for _, field := range []string{"authentication", "authConfig", "clientSecret", "dbPassword", "passphrase", "pwd", "apiKey", credentialHeaderAPIKey, "baseUrl", "callbackURL", "apiEndpoint", "sessionCookie", "cookies"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaRequiredKey: []any{field}, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "not safe") { + t.Fatalf("field %q: expected unsafe brokered required rejection, got: %v", field, err) + } + } +} + +func TestValidateAcceptsHarmlessBrokeredAuthorField(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "author": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString}, + }, + jsonSchemaRequiredKey: []any{"author"}, + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("safe author field should validate: %v", err) + } +} + +func TestValidateRejectsUnsafeBrokeredToolSchema(t *testing.T) { + for _, unsafeName := range []string{"token", "authHeader", "authorizationHeader", "httpHeaders", "accessKey", "clientSecretValue", "tokenValue", "apiSecretKey", brokeredUnsafeCookieKey, "subscriptionKey", "xFunctionsKey"} { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + unsafeName: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString}, + }, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "not safe") { + t.Fatalf("expected unsafe brokered schema rejection for %s, got: %v", unsafeName, err) + } + } +} + +func TestValidateRejectsSecretLiteralBrokeredSchemaStringValues(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaDefaultKey: "sk-not-a-real-secret"}, + }, + }, + }} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "secret-like material") { + t.Fatalf("expected secret literal brokered schema rejection, got: %v", err) + } +} + +func TestValidateRejectsBrokeredToolDigestMismatchAndNameOverlap(t *testing.T) { + cfg := validMinimalConfig() + cfg.Tools = []Tool{{Name: safeLookupToolName, Command: []string{"echo", "ok"}}} + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeObject}, + SchemaDigest: "sha256:" + strings.Repeat("0", 64), + }} + + err := cfg.Validate() + if err == nil { + t.Fatal("expected validation errors, got nil") + } + msg := err.Error() + for _, want := range []string{"tools and brokeredTools cannot be mixed", "schemaDigest does not match"} { + if !strings.Contains(msg, want) { + t.Fatalf("expected %q in error, got: %v", want, err) + } + } +} + +func TestStrictParseRejectsUnsafeBrokeredTopLevelField(t *testing.T) { + in := []byte(`apiVersion: v1alpha1 +kind: Agent +metadata: + name: brokered +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: hi +brokeredTools: + - name: safe_lookup + description: safe schema + brokeredClass: read + parameters: + type: object + url: http://tool.default.svc.cluster.local +expose: + openai: true +`) + _, err := NewFromBytes(in) + if err == nil || !strings.Contains(err.Error(), "url") { + t.Fatalf("expected strict parse rejection for unsafe field, got: %v", err) + } +} + +func validMinimalConfig() *AgentConfig { + cfg, err := NewFromBytes(agentBaseYAML("")) + if err != nil { + panic(err) + } + return cfg +} diff --git a/pkg/agentkit/config/specs.go b/pkg/agentkit/config/specs.go index 1f41692..a16c29c 100644 --- a/pkg/agentkit/config/specs.go +++ b/pkg/agentkit/config/specs.go @@ -104,8 +104,10 @@ type AgentConfig struct { // Instructions is the system prompt, authored inline (bare string or // {inline: ...}) or sourced from a file (plan §7 source union). Instructions Source `yaml:"instructions"` - // Tools are MCP servers. v0: stdio command servers (plan §5.2 ⚠). + // Tools are AgentKit-owned MCP servers. Tools []Tool `yaml:"tools,omitempty"` + // BrokeredTools are static safe Orka-brokered schemas for Foundry hosted Responses mode. + BrokeredTools []BrokeredTool `yaml:"brokeredTools,omitempty"` // Env declares runtime env var requirements by NAME only. Values are injected // by the deployment/runtime environment and never baked into the image. Env []EnvVar `yaml:"env,omitempty"` diff --git a/pkg/agentkit/config/tool.go b/pkg/agentkit/config/tool.go index 4277595..ed51ec0 100644 --- a/pkg/agentkit/config/tool.go +++ b/pkg/agentkit/config/tool.go @@ -56,6 +56,17 @@ type Tool struct { Env []string `yaml:"env,omitempty"` } +// BrokeredTool is a schema-only Orka-brokered tool declaration for hosted +// Foundry Responses mode. It deliberately excludes execution URLs, auth headers, +// Secret refs, and credentials; Orka remains the executor and policy authority. +type BrokeredTool struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + BrokeredClass string `yaml:"brokeredClass"` + Parameters map[string]any `yaml:"parameters"` + SchemaDigest string `yaml:"schemaDigest,omitempty"` +} + // variantsSet returns the names of the populated tool-source variants. func (t Tool) variantsSet() []string { var set []string diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index b7346d7..c2d7c64 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -1,10 +1,16 @@ package config import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" + "math" pathpkg "path" "sort" + "strconv" "strings" "github.com/sozercan/agentkit/pkg/agentkit/runtimes" @@ -26,6 +32,28 @@ const ( ApprovalNever = "never" ApprovalAuto = "auto" ApprovalAlways = "always" + + BrokeredClassRead = "read" + BrokeredClassWrite = "write" + BrokeredClassCoordination = "coordination" + + jsonSchemaTypeKey = "type" + jsonSchemaTypeObject = "object" + jsonSchemaTypeNumber = "number" + jsonSchemaTypeString = "string" + jsonSchemaTypeInteger = "integer" + jsonSchemaTypeArray = "array" + jsonSchemaTypeBoolean = "boolean" + jsonSchemaTypeNull = "null" + jsonSchemaDefaultKey = "default" + jsonSchemaRequiredKey = "required" + jsonSchemaDependentRequiredKey = "dependentRequired" + jsonSchemaMinimumKey = "minimum" + jsonSchemaMaximumKey = "maximum" + brokeredDigestDescriptionKey = "description" + brokeredDigestNumberKey = "\u0000agentkit_json_number" + brokeredUnsafeCookieKey = "cookie" + credentialHeaderAPIKey = "api-key" ) // Validate reports every problem with the config at once via errors.Join (plan @@ -128,6 +156,8 @@ func (c *AgentConfig) Validate() error { validateApproval(add, i, t) } + validateBrokeredTools(add, c.BrokeredTools, seen) + // --- env requirements --------------------------------------------------- seenEnv := map[string]bool{} for i, e := range c.Env { @@ -166,6 +196,643 @@ func (c *AgentConfig) Validate() error { return errors.Join(errs...) } +func validateBrokeredTools(add func(string, ...any), tools []BrokeredTool, ownedToolNames map[string]bool) { + if len(tools) > 0 && len(ownedToolNames) > 0 { + add("tools and brokeredTools cannot be mixed in v0; direct AgentKit-owned tools are disabled in brokered Foundry mode") + } + seen := map[string]bool{} + for i, tool := range tools { + path := fmt.Sprintf("brokeredTools[%d]", i) + if tool.Name == "" { + add("%s.name is required", path) + } else { + if !isBrokeredToolName(tool.Name) { + add("%s.name %q must match [A-Za-z0-9_-]{1,64}", path, tool.Name) + } + if seen[tool.Name] { + add("%s: duplicate brokered tool name %q", path, tool.Name) + } + if ownedToolNames[tool.Name] { + add("%s.name %q cannot be both owned and brokered", path, tool.Name) + } + seen[tool.Name] = true + } + if tool.Description == "" { + add("%s.description is required", path) + } else if hasUnsafeBrokeredText(tool.Description) { + add("%s.description must not contain URLs or secret-like material", path) + } + switch tool.BrokeredClass { + case BrokeredClassRead, BrokeredClassWrite, BrokeredClassCoordination: + case "": + add("%s.brokeredClass is required", path) + default: + add("%s.brokeredClass %q is not supported (expected read, write, or coordination)", path, tool.BrokeredClass) + } + validateBrokeredToolParameters(add, path+".parameters", tool.Parameters) + if tool.SchemaDigest != "" { + if !isSchemaDigest(tool.SchemaDigest) { + add("%s.schemaDigest must be sha256:<64 lowercase hex>", path) + } else if actual, err := BrokeredToolSchemaDigest(tool); err != nil { + add("%s.schemaDigest could not be checked: %v", path, err) + } else if tool.SchemaDigest != actual { + add("%s.schemaDigest does not match the safe schema", path) + } + } + } +} + +func validateBrokeredToolParameters(add func(string, ...any), path string, parameters map[string]any) { + if parameters == nil { + add("%s must be a JSON Schema object", path) + return + } + encoded, err := json.Marshal(parameters) + if err != nil { + add("%s must be JSON serializable: %v", path, err) + return + } + if len(encoded) > 64*1024 { + add("%s schema is too large", path) + } + var schema map[string]any + if err := json.Unmarshal(encoded, &schema); err != nil { + add("%s must be a JSON Schema object", path) + return + } + validateJSONSchemaSubset(add, path, schema) + if typ, _ := schema[jsonSchemaTypeKey].(string); typ != jsonSchemaTypeObject { + add("%s must set type: object", path) + } + rejectUnsafeBrokeredSchemaKeys(add, path, schema) + validateBrokeredSchemaValueConstraints(add, path, schema) +} + +func hasAnySchemaKey(schema map[string]any, keys ...string) bool { + for _, key := range keys { + if _, ok := schema[key]; ok { + return true + } + } + return false +} + +func validateJSONSchemaSubset(add func(string, ...any), path string, schema map[string]any) { + for _, key := range []string{"allOf", "anyOf", "oneOf", "not", "$ref", "if", "then", "else", "contains", "minContains", "maxContains", "propertyNames", "dependentSchemas", "patternProperties", "unevaluatedProperties", "unevaluatedItems", "prefixItems", "uniqueItems"} { + if _, ok := schema[key]; ok { + add("%s.%s is not supported for deterministic brokered tool schemas", path, key) + } + } + if value, ok := schema[jsonSchemaTypeKey]; ok { + validateJSONSchemaType(add, path, value) + } + if properties, ok := schema["properties"]; ok { + props, ok := properties.(map[string]any) + if !ok { + add("%s.properties must be an object", path) + } else { + for name, child := range props { + childSchema, ok := child.(map[string]any) + if !ok { + add("%s.properties.%s must be a JSON Schema object", path, name) + continue + } + validateJSONSchemaSubset(add, path+".properties."+name, childSchema) + } + } + } + if items, ok := schema["items"]; ok { + switch typed := items.(type) { + case map[string]any: + validateJSONSchemaSubset(add, path+".items", typed) + case []any: + add("%s.items array form is not supported for brokered tool schemas", path) + default: + add("%s.items must be an object", path) + } + } + if required, ok := schema[jsonSchemaRequiredKey]; ok && !isStringArray(required) { + add("%s.required must be a string array", path) + } + if dependentRequired, ok := schema[jsonSchemaDependentRequiredKey]; ok { + values, ok := dependentRequired.(map[string]any) + if !ok { + add("%s.dependentRequired must be an object", path) + } else { + for name, value := range values { + if !isStringArray(value) { + add("%s.dependentRequired.%s must be a string array", path, name) + } + } + } + } + if enumValue, ok := schema["enum"]; ok { + if _, ok := enumValue.([]any); !ok { + add("%s.enum must be an array", path) + } + } + if _, ok := schema["pattern"]; ok { + add("%s.pattern is not supported for deterministic brokered tool schemas", path) + } + if additional, ok := schema["additionalProperties"]; ok { + switch typed := additional.(type) { + case bool: + case map[string]any: + validateJSONSchemaSubset(add, path+".additionalProperties", typed) + default: + add("%s.additionalProperties must be a boolean or object", path) + } + } + if hasAnySchemaKey(schema, "enum", "const", "default") && hasAnySchemaKey(schema, jsonSchemaMinimumKey, "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties", "pattern") { + add("%s combines enum/const/default with constraints unsupported by deterministic brokered synthesis", path) + } + if _, ok := schema["multipleOf"]; ok { + add("%s.multipleOf is not supported for deterministic brokered tool schemas", path) + } + for _, key := range []string{jsonSchemaMinimumKey, jsonSchemaMaximumKey, "exclusiveMinimum", "exclusiveMaximum"} { + if value, ok := schema[key]; ok { + if _, ok := value.(float64); !ok { + add("%s.%s must be a number", path, key) + } + } + } + for _, key := range []string{"minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties"} { + if value, ok := schema[key]; ok { + number, ok := value.(float64) + if !ok || number < 0 || number != float64(int64(number)) { + add("%s.%s must be a non-negative integer", path, key) + } + } + } +} + +func validateJSONSchemaType(add func(string, ...any), path string, value any) { + if value == nil { + return + } + valid := func(v string) bool { + switch v { + case jsonSchemaTypeObject, jsonSchemaTypeString, jsonSchemaTypeInteger, jsonSchemaTypeNumber, jsonSchemaTypeBoolean, jsonSchemaTypeArray, jsonSchemaTypeNull: + return true + default: + return false + } + } + switch typed := value.(type) { + case string: + if !valid(typed) { + add("%s.type %q is not supported", path, typed) + } + case []any: + if len(typed) == 0 { + add("%s.type must not be empty", path) + } + for _, item := range typed { + text, ok := item.(string) + if !ok { + add("%s.type must contain only strings", path) + continue + } + if !valid(text) { + add("%s.type %q is not supported", path, text) + } + } + default: + add("%s.type must be a string or string array", path) + } +} + +func validateBrokeredSchemaValueConstraints(add func(string, ...any), path string, value any) { + schema, ok := value.(map[string]any) + if !ok { + return + } + types, ok := brokeredSchemaTypes(add, path, schema) + if ok && len(types) > 0 { + for _, keyword := range []string{"const", jsonSchemaDefaultKey} { + if child, exists := schema[keyword]; exists && !matchesAnySchemaType(child, types) { + add("%s.%s must match the declared JSON Schema type", path, keyword) + } + } + if enum, exists := schema["enum"]; exists { + items, ok := enum.([]any) + if !ok { + add("%s.enum must be an array", path) + } else { + for i, item := range items { + if !matchesAnySchemaType(item, types) { + add("%s.enum[%d] must match the declared JSON Schema type", path, i) + } + } + } + } + } + if properties, ok := schema["properties"].(map[string]any); ok { + for name, child := range properties { + if childSchema, ok := child.(map[string]any); ok { + validateBrokeredSchemaValueConstraints(add, path+".properties."+name, childSchema) + } + } + } + if items, ok := schema["items"].(map[string]any); ok { + validateBrokeredSchemaValueConstraints(add, path+".items", items) + } else if tupleItems, ok := schema["items"].([]any); ok { + for i, child := range tupleItems { + if childSchema, ok := child.(map[string]any); ok { + validateBrokeredSchemaValueConstraints(add, fmt.Sprintf("%s.items[%d]", path, i), childSchema) + } + } + } + if additional, ok := schema["additionalProperties"].(map[string]any); ok { + validateBrokeredSchemaValueConstraints(add, path+".additionalProperties", additional) + } +} + +func brokeredSchemaTypes(add func(string, ...any), path string, schema map[string]any) ([]string, bool) { + raw, exists := schema[jsonSchemaTypeKey] + if !exists { + return nil, true + } + switch typed := raw.(type) { + case string: + if !isSupportedBrokeredSchemaType(typed) { + add("%s.type %q is not supported", path, typed) + return nil, false + } + return []string{typed}, true + case []any: + out := make([]string, 0, len(typed)) + for _, item := range typed { + name, ok := item.(string) + if !ok || !isSupportedBrokeredSchemaType(name) { + add("%s.type must be a string or string array", path) + return nil, false + } + out = append(out, name) + } + return out, true + default: + add("%s.type must be a string or string array", path) + return nil, false + } +} + +func isSupportedBrokeredSchemaType(schemaType string) bool { + switch schemaType { + case "null", "boolean", "integer", jsonSchemaTypeNumber, jsonSchemaTypeString, "array", jsonSchemaTypeObject: + return true + default: + return false + } +} + +func matchesAnySchemaType(value any, types []string) bool { + for _, schemaType := range types { + if matchesSchemaType(value, schemaType) { + return true + } + } + return false +} + +func matchesSchemaType(value any, schemaType string) bool { + switch schemaType { + case jsonSchemaTypeNull: + return value == nil + case jsonSchemaTypeBoolean: + _, ok := value.(bool) + return ok + case jsonSchemaTypeInteger: + switch typed := value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return true + case float64: + return typed == math.Trunc(typed) + case json.Number: + _, err := typed.Int64() + return err == nil + default: + return false + } + case jsonSchemaTypeNumber: + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, json.Number: + return true + default: + return false + } + case jsonSchemaTypeString: + _, ok := value.(string) + return ok + case jsonSchemaTypeArray: + _, ok := value.([]any) + return ok + case jsonSchemaTypeObject: + _, ok := value.(map[string]any) + return ok + default: + return false + } +} + +func isStringArray(value any) bool { + switch typed := value.(type) { + case []string: + return true + case []any: + for _, item := range typed { + if _, ok := item.(string); !ok { + return false + } + } + return true + default: + return false + } +} + +func rejectUnsafeBrokeredSchemaKeys(add func(string, ...any), path string, value any) { + switch typed := value.(type) { + case map[string]any: + for key, child := range typed { + childPath := path + "." + key + if isUnsafeBrokeredKey(key) { + add("%s is not safe for brokered tool schemas", childPath) + } + if key == jsonSchemaRequiredKey || key == jsonSchemaDependentRequiredKey { + rejectUnsafeBrokeredPropertyNameValues(add, childPath, child) + } + rejectUnsafeBrokeredSchemaKeys(add, childPath, child) + } + case []any: + for i, child := range typed { + rejectUnsafeBrokeredSchemaKeys(add, fmt.Sprintf("%s[%d]", path, i), child) + } + case []map[string]any: + for i, child := range typed { + rejectUnsafeBrokeredSchemaKeys(add, fmt.Sprintf("%s[%d]", path, i), child) + } + case string: + if hasUnsafeBrokeredText(typed) { + add("%s contains URL or secret-like material", path) + } + } +} + +func rejectUnsafeBrokeredPropertyNameValues(add func(string, ...any), path string, value any) { + switch typed := value.(type) { + case string: + if isUnsafeBrokeredKey(typed) { + add("%s value %q is not safe for brokered tool schemas", path, typed) + } + case []any: + for i, child := range typed { + rejectUnsafeBrokeredPropertyNameValues(add, fmt.Sprintf("%s[%d]", path, i), child) + } + case []string: + for i, child := range typed { + rejectUnsafeBrokeredPropertyNameValues(add, fmt.Sprintf("%s[%d]", path, i), child) + } + case map[string]any: + for key, child := range typed { + childPath := path + "." + key + if isUnsafeBrokeredKey(key) { + add("%s is not safe for brokered tool schemas", childPath) + } + rejectUnsafeBrokeredPropertyNameValues(add, childPath, child) + } + } +} + +func isBrokeredToolName(value string) bool { + if len(value) == 0 || len(value) > 64 { + return false + } + for _, r := range value { + isAlpha := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') + isDigit := r >= '0' && r <= '9' + if !isAlpha && !isDigit && r != '_' && r != '-' { + return false + } + } + return true +} + +func isSchemaDigest(value string) bool { + if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") || value != strings.ToLower(value) { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func hasUnsafeBrokeredText(value string) bool { + lowered := strings.ToLower(value) + normalized := normalizeKey(lowered) + return containsSecretPrefix(value) || strings.Contains(value, "://") || strings.Contains(lowered, "bearer") || strings.Contains(lowered, "basic") || strings.Contains(lowered, "authorization") || strings.Contains(lowered, "secret") || strings.Contains(lowered, "token") || strings.Contains(lowered, "password") || strings.Contains(lowered, "passphrase") || strings.Contains(lowered, "pwd") || strings.Contains(lowered, "api key") || strings.Contains(lowered, "apikey") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "xapikey") || strings.Contains(normalized, "subscriptionkey") || strings.Contains(normalized, "xfunctionskey") || strings.Contains(lowered, brokeredUnsafeCookieKey) || strings.Contains(lowered, "set-cookie") || strings.Contains(lowered, "x-api-key") || strings.Contains(lowered, credentialHeaderAPIKey) || strings.Contains(lowered, "subscription-key") || strings.Contains(lowered, "x-functions-key") || strings.Contains(lowered, "ocp-apim-subscription-key") || strings.Contains(lowered, "private key") || strings.Contains(lowered, "privatekey") || strings.Contains(lowered, "key material") || strings.Contains(lowered, ".svc") || strings.Contains(lowered, "cluster.local") +} + +func isUnsafeBrokeredKey(value string) bool { + normalized := normalizeKey(value) + switch normalized { + case "auth", "authorization", "apikey", "bearer", brokeredUnsafeCookieKey, "credential", "credentials", "endpoint", "endpoints", "executionendpoint", "executionurl", "header", "headers", "ocpapimsubscriptionkey", "password", "proxyauthorization", "secret", "secretref", "setcookie", "subscriptionkey", "token", "tokens", "url", "urls", "xapikey", "xfunctionskey": + return true + } + authLike := (strings.HasPrefix(normalized, "auth") && !strings.HasPrefix(normalized, "author")) || strings.HasSuffix(normalized, "auth") + return authLike || strings.Contains(normalized, "authorization") || strings.Contains(normalized, "header") || strings.Contains(normalized, "url") || strings.Contains(normalized, "endpoint") || strings.Contains(normalized, brokeredUnsafeCookieKey) || strings.Contains(normalized, "secret") || strings.Contains(normalized, "token") || strings.Contains(normalized, "password") || strings.Contains(normalized, "passphrase") || strings.Contains(normalized, "pwd") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "accesskey") || strings.Contains(normalized, "privatekey") || strings.Contains(normalized, "keymaterial") || strings.Contains(normalized, "credential") || strings.Contains(normalized, "executionurl") || strings.Contains(normalized, "executionendpoint") +} + +func normalizeKey(value string) string { + var b strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + case r >= '0' && r <= '9': + b.WriteRune(r) + } + } + return b.String() +} + +func canonicalNumber(value float64, bitSize int) ([]byte, error) { + if math.IsNaN(value) || math.IsInf(value, 0) { + return nil, fmt.Errorf("JSON numbers must be finite") + } + return []byte(strconv.FormatFloat(value, 'f', -1, bitSize)), nil +} + +func canonicalJSONString(value string) ([]byte, error) { + var encoded bytes.Buffer + encoder := json.NewEncoder(&encoded) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(value); err != nil { + return nil, err + } + return bytes.TrimSuffix(encoded.Bytes(), []byte("\n")), nil +} + +func canonicalJSONNumberString(value string) (string, error) { + if !strings.ContainsAny(value, ".eE") { + return value, nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return "", err + } + if math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return "", fmt.Errorf("JSON numbers must be finite") + } + return strconv.FormatFloat(parsed, 'f', -1, 64), nil +} + +func canonicalJSON(value any) ([]byte, error) { + switch typed := value.(type) { + case nil: + return []byte("null"), nil + case bool: + if typed { + return []byte("true"), nil + } + return []byte("false"), nil + case string: + return canonicalJSONString(typed) + case int: + return []byte(strconv.FormatInt(int64(typed), 10)), nil + case int8: + return []byte(strconv.FormatInt(int64(typed), 10)), nil + case int16: + return []byte(strconv.FormatInt(int64(typed), 10)), nil + case int32: + return []byte(strconv.FormatInt(int64(typed), 10)), nil + case int64: + return []byte(strconv.FormatInt(typed, 10)), nil + case uint: + return []byte(strconv.FormatUint(uint64(typed), 10)), nil + case uint8: + return []byte(strconv.FormatUint(uint64(typed), 10)), nil + case uint16: + return []byte(strconv.FormatUint(uint64(typed), 10)), nil + case uint32: + return []byte(strconv.FormatUint(uint64(typed), 10)), nil + case uint64: + return []byte(strconv.FormatUint(typed, 10)), nil + case float32: + return canonicalNumber(float64(typed), 32) + case float64: + return canonicalNumber(typed, 64) + case json.Number: + formatted, err := canonicalJSONNumberString(typed.String()) + if err != nil { + return nil, err + } + return []byte(formatted), nil + case []any: + return canonicalJSONArray(typed) + case []string: + items := make([]any, len(typed)) + for i, item := range typed { + items[i] = item + } + return canonicalJSONArray(items) + case []int: + items := make([]any, len(typed)) + for i, item := range typed { + items[i] = item + } + return canonicalJSONArray(items) + case []float64: + items := make([]any, len(typed)) + for i, item := range typed { + items[i] = item + } + return canonicalJSONArray(items) + case []bool: + items := make([]any, len(typed)) + for i, item := range typed { + items[i] = item + } + return canonicalJSONArray(items) + case map[string]any: + return canonicalJSONObject(typed) + default: + return nil, fmt.Errorf("unsupported JSON value %T", value) + } +} + +func canonicalJSONArray(items []any) ([]byte, error) { + var out bytes.Buffer + out.WriteByte('[') + for i, item := range items { + if i > 0 { + out.WriteByte(',') + } + encoded, err := canonicalJSON(item) + if err != nil { + return nil, err + } + out.Write(encoded) + } + out.WriteByte(']') + return out.Bytes(), nil +} + +func canonicalJSONObject(object map[string]any) ([]byte, error) { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + sort.Strings(keys) + var out bytes.Buffer + out.WriteByte('{') + for i, key := range keys { + if i > 0 { + out.WriteByte(',') + } + encodedKey, err := canonicalJSONString(key) + if err != nil { + return nil, err + } + encodedValue, err := canonicalJSON(object[key]) + if err != nil { + return nil, err + } + out.Write(encodedKey) + out.WriteByte(':') + out.Write(encodedValue) + } + out.WriteByte('}') + return out.Bytes(), nil +} + +// BrokeredToolSchemaDigest digests the exact safe schema surface AgentKit exposes to a model. +func BrokeredToolSchemaDigest(tool BrokeredTool) (string, error) { + payload := map[string]any{ + "name": tool.Name, + brokeredDigestDescriptionKey: tool.Description, + "brokeredClass": tool.BrokeredClass, + "parameters": tool.Parameters, + } + encodedPayload, err := json.Marshal(payload) + if err != nil { + return "", err + } + var normalizedPayload map[string]any + decoder := json.NewDecoder(bytes.NewReader(encodedPayload)) + decoder.UseNumber() + if err := decoder.Decode(&normalizedPayload); err != nil { + return "", err + } + canonical, err := canonicalJSON(normalizedPayload) + if err != nil { + return "", err + } + sum := sha256.Sum256(canonical) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + func validateContext(add func(string, ...any), ctx Context, tools []Tool) { seen := map[string]bool{} toolByName := map[string]Tool{} @@ -454,7 +1121,7 @@ func isHTTPHeaderName(v string) bool { func isCredentialHeaderName(name string) bool { switch strings.ToLower(name) { - case "authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", "api-key", "ocp-apim-subscription-key", "subscription-key", "x-functions-key": + case "authorization", "proxy-authorization", brokeredUnsafeCookieKey, "set-cookie", "x-api-key", credentialHeaderAPIKey, "ocp-apim-subscription-key", "subscription-key", "x-functions-key": return true default: return false @@ -470,6 +1137,15 @@ func hasSecretPrefix(v string) bool { return false } +func containsSecretPrefix(v string) bool { + for _, p := range []string{"sk-", "sk_", "ghp_", "github_pat_", "xoxb-", "AKIA"} { + if strings.Contains(v, p) { + return true + } + } + return false +} + // looksLikeSecretLiteral heuristically flags a value that appears to be a secret // rather than an env var NAME. Env var names are uppercase letters, digits, and // underscores; common secret prefixes (sk-, etc.) and lowercase/punctuation are diff --git a/pkg/agentkit/effective/agent.go b/pkg/agentkit/effective/agent.go index 1eb9466..559ebed 100644 --- a/pkg/agentkit/effective/agent.go +++ b/pkg/agentkit/effective/agent.go @@ -3,6 +3,8 @@ package effective import ( + "reflect" + "github.com/sozercan/agentkit/pkg/agentkit/config" "github.com/sozercan/agentkit/pkg/agentkit/runtimes" "github.com/sozercan/agentkit/pkg/utils" @@ -19,6 +21,7 @@ type Agent struct { Model config.Model Instructions string Tools []config.Tool + BrokeredTools []config.BrokeredTool Env []config.EnvVar Context config.Context Observability config.Observability @@ -49,6 +52,7 @@ func FromConfig(cfg *config.AgentConfig, instructions string) Agent { Model: cfg.Model, Instructions: instructions, Tools: copyTools(cfg.Tools), + BrokeredTools: copyBrokeredTools(cfg.BrokeredTools), Env: copyEnvVars(cfg.Env), Context: copyContext(cfg.Context), Observability: cfg.Observability, @@ -85,6 +89,122 @@ func copyTools(in []config.Tool) []config.Tool { return out } +func copyBrokeredTools(in []config.BrokeredTool) []config.BrokeredTool { + if len(in) == 0 { + return nil + } + out := make([]config.BrokeredTool, len(in)) + for i, tool := range in { + out[i] = tool + out[i].Parameters = copyMap(tool.Parameters) + } + return out +} + +func copyMap(in map[string]any) map[string]any { + if in == nil { + return nil + } + out := make(map[string]any, len(in)) + for k, v := range in { + out[k] = copyAny(v) + } + return out +} + +func copyAny(v any) any { + if v == nil { + return nil + } + switch typed := v.(type) { + case map[string]any: + return copyMap(typed) + case map[string]string: + out := make(map[string]string, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case map[string]int: + out := make(map[string]int, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case map[string]float64: + out := make(map[string]float64, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case map[string]bool: + out := make(map[string]bool, len(typed)) + for key, value := range typed { + out[key] = value + } + return out + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = copyAny(item) + } + return out + case []string: + return append([]string(nil), typed...) + case []int: + return append([]int(nil), typed...) + case []float64: + return append([]float64(nil), typed...) + case []bool: + return append([]bool(nil), typed...) + default: + return copyReflectValue(v) + } +} + +func copyReflectValue(v any) any { + value := reflect.ValueOf(v) + switch value.Kind() { + case reflect.Map: + out := reflect.MakeMapWithSize(value.Type(), value.Len()) + iter := value.MapRange() + for iter.Next() { + copied := copyAny(iter.Value().Interface()) + copiedValue := reflect.ValueOf(copied) + if copied == nil { + copiedValue = reflect.Zero(value.Type().Elem()) + } else if !copiedValue.Type().AssignableTo(value.Type().Elem()) { + if copiedValue.Type().ConvertibleTo(value.Type().Elem()) { + copiedValue = copiedValue.Convert(value.Type().Elem()) + } else { + copiedValue = iter.Value() + } + } + out.SetMapIndex(iter.Key(), copiedValue) + } + return out.Interface() + case reflect.Slice: + out := reflect.MakeSlice(value.Type(), value.Len(), value.Len()) + for i := 0; i < value.Len(); i++ { + copied := copyAny(value.Index(i).Interface()) + copiedValue := reflect.ValueOf(copied) + if copied == nil { + copiedValue = reflect.Zero(value.Type().Elem()) + } else if !copiedValue.Type().AssignableTo(value.Type().Elem()) { + if copiedValue.Type().ConvertibleTo(value.Type().Elem()) { + copiedValue = copiedValue.Convert(value.Type().Elem()) + } else { + copiedValue = value.Index(i) + } + } + out.Index(i).Set(copiedValue) + } + return out.Interface() + default: + return v + } +} + func copyEnvVars(in []config.EnvVar) []config.EnvVar { if len(in) == 0 { return nil diff --git a/pkg/agentkit/effective/agent_test.go b/pkg/agentkit/effective/agent_test.go index d828dff..1b0deae 100644 --- a/pkg/agentkit/effective/agent_test.go +++ b/pkg/agentkit/effective/agent_test.go @@ -72,7 +72,7 @@ func TestFromConfigCopiesMutableFields(t *testing.T) { cfg := baseConfig() agent := FromConfig(cfg, "prompt") - cfg.Metadata.Labels["team"] = "mutated" + cfg.Metadata.Labels["team"] = mutatedValue cfg.Tools[0].Command[0] = "mutated" cfg.Tools[0].Env[0] = mutatedValue cfg.Tools[0].Headers[0].Name = mutatedValue @@ -98,3 +98,96 @@ func TestFromConfigCopiesMutableFields(t *testing.T) { t.Fatalf("agent env was not copied: %q", got) } } + +const ( + testSiteField = "site" + testSchemaTypeKey = "type" + testSchemaTypeString = "string" +) + +func TestFromConfigCopiesBrokeredTools(t *testing.T) { + cfg := baseConfig() + cfg.BrokeredTools = []config.BrokeredTool{{ + Name: "check-network-telemetry", + Description: "Read telemetry.", + BrokeredClass: config.BrokeredClassRead, + Parameters: map[string]any{ + testSchemaTypeKey: "object", + "properties": map[string]any{ + testSiteField: map[string]any{testSchemaTypeKey: testSchemaTypeString}, + "typed": map[string]string{testSchemaTypeKey: testSchemaTypeString}, + "generic": map[string][]string{"enum": {"a", "b"}}, + "tuple": []map[string]any{{testSchemaTypeKey: testSchemaTypeString}}, + "empty": map[string]any{}, + }, + "required": []string{testSiteField}, + }, + }} + + agent := FromConfig(cfg, "prompt") + cfg.BrokeredTools[0].Parameters[testSchemaTypeKey] = mutatedValue + mutatedProperties, ok := cfg.BrokeredTools[0].Parameters["properties"].(map[string]any) + if !ok { + t.Fatalf("brokered tool properties had unexpected type: %#v", cfg.BrokeredTools[0].Parameters["properties"]) + } + mutatedSite, ok := mutatedProperties[testSiteField].(map[string]any) + if !ok { + t.Fatalf("brokered tool site property had unexpected type: %#v", mutatedProperties["site"]) + } + mutatedSite[testSchemaTypeKey] = mutatedValue + mutatedTyped, ok := mutatedProperties["typed"].(map[string]string) + if !ok { + t.Fatalf("brokered typed property had unexpected type: %#v", mutatedProperties["typed"]) + } + mutatedTyped[testSchemaTypeKey] = mutatedValue + mutatedRequired, ok := cfg.BrokeredTools[0].Parameters["required"].([]string) + if !ok { + t.Fatalf("brokered tool required slice had unexpected type: %#v", cfg.BrokeredTools[0].Parameters["required"]) + } + mutatedRequired[0] = mutatedValue + mutatedGeneric, ok := mutatedProperties["generic"].(map[string][]string) + if !ok { + t.Fatalf("brokered generic property had unexpected type: %#v", mutatedProperties["generic"]) + } + mutatedGeneric["enum"][0] = mutatedValue + mutatedTuple, ok := mutatedProperties["tuple"].([]map[string]any) + if !ok { + t.Fatalf("brokered tuple property had unexpected type: %#v", mutatedProperties["tuple"]) + } + mutatedTuple[0][testSchemaTypeKey] = mutatedValue + + if got := agent.BrokeredTools[0].Parameters[testSchemaTypeKey]; got != "object" { + t.Fatalf("brokered tool parameters were not copied: %q", got) + } + properties, ok := agent.BrokeredTools[0].Parameters["properties"].(map[string]any) + if !ok { + t.Fatalf("brokered tool properties had unexpected type: %#v", agent.BrokeredTools[0].Parameters["properties"]) + } + site, ok := properties[testSiteField].(map[string]any) + if !ok { + t.Fatalf("brokered tool site property had unexpected type: %#v", properties["site"]) + } + if got := site[testSchemaTypeKey]; got != "string" { + t.Fatalf("nested brokered tool parameters were not copied: %q", got) + } + typed, ok := properties["typed"].(map[string]string) + if !ok || typed[testSchemaTypeKey] != testSchemaTypeString { + t.Fatalf("typed brokered tool schema map was not copied: %#v", properties["typed"]) + } + empty, ok := properties["empty"].(map[string]any) + if !ok || empty == nil || len(empty) != 0 { + t.Fatalf("empty brokered tool schema map was not preserved: %#v", properties["empty"]) + } + generic, ok := properties["generic"].(map[string][]string) + if !ok || generic["enum"][0] != "a" { + t.Fatalf("generic typed brokered schema map was not copied: %#v", properties["generic"]) + } + tuple, ok := properties["tuple"].([]map[string]any) + if !ok || tuple[0][testSchemaTypeKey] != testSchemaTypeString { + t.Fatalf("typed brokered schema slice was not copied: %#v", properties["tuple"]) + } + required, ok := agent.BrokeredTools[0].Parameters["required"].([]string) + if !ok || len(required) != 1 || required[0] != testSiteField { + t.Fatalf("brokered tool required slice was not copied: %#v", agent.BrokeredTools[0].Parameters["required"]) + } +} diff --git a/runtimes/common/README.md b/runtimes/common/README.md index a125ce4..a8b738b 100644 --- a/runtimes/common/README.md +++ b/runtimes/common/README.md @@ -49,3 +49,32 @@ A new single-agent adapter should provide: Adapters remain separate images with separate framework dependencies, while this package is installed into each image as the shared façade/runtime core. + + +## Brokered tool schema export + +`agentkit-serve-common` includes a small deployment helper for Foundry hosted +Orka-brokered mode: + +```sh +agentkit-brokered-tools ./orka-tools/*.yaml -o brokered-tools.generated.yaml +``` + +It reads Orka Tool CRD YAML/JSON documents and writes a safe `brokeredTools:` +`agent.yaml` fragment containing only name, description, brokered class, JSON +parameters schema, and optional schema digest. Execution URLs, auth headers, +Secret refs, tokens, and other credential-shaped schema fields are rejected or +omitted before the fragment is model-visible. + + +## Foundry brokered conformance app + +The common package also installs `agentkit-foundry-conformance`, a tiny +Azure Responses SDK app for Phase A0 hosted brokered smokes. It serves +`/readiness` and `/responses`, emits a deterministic `conformance_read` +`function_call`, and completes after a matching `function_call_output` +continuation. + +```sh +agentkit-foundry-conformance --host 0.0.0.0 --port 8088 +``` diff --git a/runtimes/common/agentkit_serve_common/brokered.py b/runtimes/common/agentkit_serve_common/brokered.py new file mode 100644 index 0000000..1188ed2 --- /dev/null +++ b/runtimes/common/agentkit_serve_common/brokered.py @@ -0,0 +1,212 @@ +"""Helpers for safe Orka-brokered tool schemas. + +The hosted Foundry `/responses` endpoint cannot receive request-level tool +schemas, so brokered mode uses static schema-only declarations baked into +`agent.yaml`. This module keeps that schema surface small and deterministic: +only name, description, brokered class, JSON parameters schema, and an optional +schema digest may cross into hosted AgentKit. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +import yaml + +from .config import AgentSpec, BrokeredToolSpec, brokered_tool_schema_digest +from .runtime import BrokeredToolDefinition + + +def brokered_tool_definitions(spec: AgentSpec) -> list[BrokeredToolDefinition]: + """Return runtime-safe brokered tool definitions from an AgentSpec.""" + + return [ + BrokeredToolDefinition( + name=tool.name, + description=tool.description, + brokered_class=tool.brokered_class, + parameters=dict(tool.parameters), + schema_digest=tool.schema_digest, + ) + for tool in spec.brokered_tools + ] + + +def brokered_tool_config_entry( + *, + name: str, + description: str, + brokered_class: str, + parameters: Mapping[str, Any], + include_digest: bool = True, +) -> dict[str, Any]: + """Build one validated `agent.yaml` brokeredTools entry.""" + + payload: dict[str, Any] = { + "name": name, + "description": description, + "brokeredClass": brokered_class, + "parameters": dict(parameters), + } + if include_digest: + payload["schemaDigest"] = brokered_tool_schema_digest( + name=name, + description=description, + brokered_class=brokered_class, + parameters=dict(parameters), + ) + validated = BrokeredToolSpec.model_validate(payload) + out: dict[str, Any] = { + "name": validated.name, + "description": validated.description, + "brokeredClass": validated.brokered_class, + "parameters": validated.parameters, + } + if validated.schema_digest is not None: + out["schemaDigest"] = validated.schema_digest + return out + + +def _nested_get(mapping: Mapping[str, Any], *path: str) -> Any: + current: Any = mapping + for key in path: + if not isinstance(current, Mapping): + return None + current = current.get(key) + return current + + +def _first_present(*values: Any) -> Any: + for value in values: + if value not in (None, ""): + return value + return None + + +def generate_brokered_tools_from_orka_tool_crds( + documents: Iterable[Mapping[str, Any]], + *, + include_digest: bool = True, +) -> list[dict[str, Any]]: + """Generate deterministic safe AgentKit brokeredTools config from Orka Tool CRDs. + + The Orka Tool CRD remains the source of truth. This helper deliberately + extracts only the safe model-facing subset and validates it with the same + `BrokeredToolSpec` model used by `agent.yaml` loading. + """ + + entries: list[dict[str, Any]] = [] + for idx, document in enumerate(documents): + if not isinstance(document, Mapping) or not document: + continue + kind = str(document.get("kind") or "") + if kind and kind.lower() != "tool": + continue + metadata = document.get("metadata") if isinstance(document.get("metadata"), Mapping) else {} + spec = document.get("spec") if isinstance(document.get("spec"), Mapping) else {} + name = _first_present(spec.get("name"), metadata.get("name")) + if not isinstance(name, str): + raise ValueError(f"Tool CRD document {idx} is missing metadata.name") + description = _first_present(spec.get("description"), spec.get("summary"), name) + if not isinstance(description, str): + raise ValueError(f"Tool CRD {name!r} description must be a string") + brokered_class = _first_present( + spec.get("brokeredClass"), + spec.get("brokered_class"), + spec.get("class"), + _nested_get(spec, "brokered", "class"), + "read", + ) + if not isinstance(brokered_class, str): + raise ValueError(f"Tool CRD {name!r} brokered class must be a string") + parameters = _first_present( + spec.get("parameters"), + spec.get("inputSchema"), + spec.get("input_schema"), + spec.get("schema"), + _nested_get(spec, "input", "schema"), + ) + if parameters is None: + parameters = {"type": "object"} + if not isinstance(parameters, Mapping): + raise ValueError(f"Tool CRD {name!r} parameters schema must be an object") + entries.append( + brokered_tool_config_entry( + name=name, + description=description, + brokered_class=brokered_class, + parameters=parameters, + include_digest=include_digest, + ) + ) + return sorted(entries, key=lambda item: item["name"]) + + +def load_orka_tool_crd_file(path: str | Path, *, include_digest: bool = True) -> list[dict[str, Any]]: + """Load Tool CRD YAML/JSON documents and return safe brokeredTools entries.""" + + raw = Path(path).read_text(encoding="utf-8") + docs = [doc for doc in yaml.safe_load_all(raw) if isinstance(doc, Mapping)] + return generate_brokered_tools_from_orka_tool_crds(docs, include_digest=include_digest) + + +def load_orka_tool_crd_files(paths: Sequence[str | Path], *, include_digest: bool = True) -> list[dict[str, Any]]: + """Load one or more Tool CRD files and merge deterministic safe entries.""" + + documents: list[Mapping[str, Any]] = [] + for path in paths: + raw = Path(path).read_text(encoding="utf-8") + documents.extend(doc for doc in yaml.safe_load_all(raw) if isinstance(doc, Mapping)) + entries = generate_brokered_tools_from_orka_tool_crds(documents, include_digest=include_digest) + seen: set[str] = set() + duplicates: set[str] = set() + for entry in entries: + name = str(entry["name"]) + if name in seen: + duplicates.add(name) + seen.add(name) + if duplicates: + names = ", ".join(sorted(duplicates)) + raise ValueError(f"duplicate brokered tool names across input files: {names}") + return entries + + +def render_brokered_tools_yaml(entries: list[dict[str, Any]], *, bare: bool = False) -> str: + """Render safe brokered tool entries as deterministic YAML.""" + + payload: Any = entries if bare else {"brokeredTools": entries} + return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True) + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="agentkit-brokered-tools", + description="Export safe AgentKit brokeredTools YAML from Orka Tool CRD YAML/JSON files.", + ) + parser.add_argument("paths", nargs="+", help="Orka Tool CRD YAML/JSON file(s) to export") + parser.add_argument("--no-digest", action="store_true", help="omit schemaDigest fields") + parser.add_argument("--bare", action="store_true", help="emit only the brokeredTools list, not a top-level brokeredTools key") + parser.add_argument("--output", "-o", help="write output YAML to this file instead of stdout") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + try: + entries = load_orka_tool_crd_files(args.paths, include_digest=not args.no_digest) + rendered = render_brokered_tools_yaml(entries, bare=args.bare) + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + except Exception as exc: # noqa: BLE001 - CLI must print concise validation failures. + print(f"agentkit-brokered-tools: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() tests and console script. + raise SystemExit(main()) diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index 1ed7cff..f65a726 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -8,11 +8,16 @@ from __future__ import annotations +import hashlib +import json +import math +from decimal import Decimal import os import posixpath import re import sys from pathlib import Path +from typing import Any, Literal, Mapping import yaml from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator @@ -22,6 +27,7 @@ _PROVIDER_OPENAI_COMPATIBLE = "openai-compatible" _ENV_NAME_RE = re.compile(r"^[A-Z0-9_]+$") _HEADER_NAME_RE = re.compile(r"^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$") +_BROKERED_TOOL_NAME_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") _SECRET_VALUE_PREFIXES = ("sk-", "sk_", "ghp_", "github_pat_", "xoxb-", "AKIA") _TOOL_TYPE_MCP = "mcp" _TRANSPORT_STDIO = "stdio" @@ -45,6 +51,38 @@ "subscription-key", "x-functions-key", } +_BROKERED_CLASSES = {"read", "write", "coordination"} +_BROKERED_SCHEMA_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_MAX_BROKERED_SCHEMA_BYTES = 64 * 1024 +_UNSAFE_BROKERED_FIELD_NAMES = { + "auth", + "authorization", + "apikey", + "bearer", + "cookie", + "credential", + "credentials", + "endpoint", + "endpoints", + "executionendpoint", + "executionurl", + "header", + "headers", + "ocpapimsubscriptionkey", + "password", + "proxyauthorization", + "secret", + "secretref", + "setcookie", + "subscriptionkey", + "token", + "tokens", + "url", + "urls", + "xapikey", + "xfunctionskey", +} + class _Strict(BaseModel): @@ -68,6 +106,260 @@ def _looks_like_secret_literal(value: str) -> bool: return value.startswith(_SECRET_VALUE_PREFIXES) +def _contains_secret_prefix(value: str) -> bool: + return any(prefix in value for prefix in _SECRET_VALUE_PREFIXES) + + +def _unsafe_brokered_text(value: str) -> bool: + lowered = value.lower() + normalized = re.sub(r"[^a-z0-9]", "", lowered) + return ( + (_looks_like_secret_literal(value) or _contains_secret_prefix(value)) + or "://" in value + or re.search(r"\bbearer\b", lowered) is not None + or re.search(r"\bbasic\b", lowered) is not None + or "authorization" in lowered + or "secret" in lowered + or "token" in lowered + or "password" in lowered + or "passphrase" in lowered + or "pwd" in lowered + or "api key" in lowered + or "apikey" in lowered + or "apikey" in normalized + or "xapikey" in normalized + or "subscriptionkey" in normalized + or "xfunctionskey" in normalized + or "cookie" in lowered + or "set-cookie" in lowered + or "x-api-key" in lowered + or "api-key" in lowered + or "subscription-key" in lowered + or "x-functions-key" in lowered + or "ocp-apim-subscription-key" in lowered + or "private key" in lowered + or "privatekey" in lowered + or "key material" in lowered + or ".svc" in lowered + or "cluster.local" in lowered + ) + + +def _canonical_number(value: int | float) -> str: + if isinstance(value, bool): + raise TypeError("boolean is not a JSON number") + if isinstance(value, int): + return str(value) + if not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + decimal = Decimal(str(value)) + if decimal == decimal.to_integral_value(): + return format(decimal.to_integral_value(), "f") + out = format(decimal, "f") + if "." in out: + out = out.rstrip("0").rstrip(".") + return out + + +def _canonical_json(value: Any) -> str: + """Return a deterministic JSON representation for digests and drift checks.""" + + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)) and not isinstance(value, bool): + return _canonical_number(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + if isinstance(value, list): + return "[" + ",".join(_canonical_json(item) for item in value) + "]" + if isinstance(value, Mapping): + parts: list[str] = [] + for key in sorted(value): + if not isinstance(key, str): + raise TypeError("JSON object keys must be strings") + parts.append(json.dumps(key, ensure_ascii=False, separators=(",", ":")) + ":" + _canonical_json(value[key])) + return "{" + ",".join(parts) + "}" + raise TypeError(f"unsupported JSON value {type(value).__name__}") + + +def brokered_tool_schema_digest( + *, + name: str, + description: str, + brokered_class: str, + parameters: Mapping[str, Any], +) -> str: + """Digest the exact safe schema surface AgentKit exposes to a model.""" + + payload = { + "name": name, + "description": description, + "brokeredClass": brokered_class, + "parameters": parameters, + } + return "sha256:" + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + + +def _unsafe_brokered_key(value: str) -> str | None: + normalized = re.sub(r"[^A-Za-z0-9]", "", value).lower() + if normalized in _UNSAFE_BROKERED_FIELD_NAMES: + return value + auth_like = (normalized.startswith("auth") and not normalized.startswith("author")) or normalized.endswith("auth") + if ( + auth_like + or "authorization" in normalized + or "header" in normalized + or "url" in normalized + or "endpoint" in normalized + or "cookie" in normalized + or "secret" in normalized + or "token" in normalized + or "password" in normalized + or "passphrase" in normalized + or "pwd" in normalized + or "apikey" in normalized + or "accesskey" in normalized + or "privatekey" in normalized + or "keymaterial" in normalized + ): + return value + if "credential" in normalized or "executionurl" in normalized or "executionendpoint" in normalized: + return value + return None + + +def _reject_unsafe_brokered_schema_keys(value: Any, *, path: str) -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + if not isinstance(key, str): + raise ValueError(f"{path} keys must be strings") + unsafe = _unsafe_brokered_key(key) + if unsafe is not None: + raise ValueError(f"{path}.{unsafe} is not safe for brokered tool schemas") + if key in {"required", "dependentRequired"}: + _reject_unsafe_brokered_property_name_values(child, path=f"{path}.{key}") + _reject_unsafe_brokered_schema_keys(child, path=f"{path}.{key}") + elif isinstance(value, list): + for idx, child in enumerate(value): + _reject_unsafe_brokered_schema_keys(child, path=f"{path}[{idx}]") + elif isinstance(value, str) and _unsafe_brokered_text(value): + raise ValueError(f"{path} contains URL or secret-like material") + + + +_JSON_SCHEMA_TYPES = {"object", "string", "integer", "number", "boolean", "array", "null"} +_NUMERIC_SCHEMA_KEYS = {"minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"} +_INTEGER_SCHEMA_KEYS = {"minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties"} + + +def _validate_json_schema_subset(schema: Any, *, path: str) -> None: + if not isinstance(schema, Mapping): + raise ValueError(f"{path} must be a JSON Schema object") + for unsupported_key in ( + "allOf", + "anyOf", + "oneOf", + "not", + "$ref", + "if", + "then", + "else", + "contains", + "minContains", + "maxContains", + "propertyNames", + "dependentSchemas", + "patternProperties", + "unevaluatedProperties", + "unevaluatedItems", + "prefixItems", + "uniqueItems", + ): + if unsupported_key in schema: + raise ValueError(f"{path}.{unsupported_key} is not supported for deterministic brokered tool schemas") + schema_type = schema.get("type") + if schema_type is not None: + if isinstance(schema_type, str): + if schema_type not in _JSON_SCHEMA_TYPES: + raise ValueError(f"{path}.type {schema_type!r} is not supported") + elif isinstance(schema_type, list) and schema_type and all(isinstance(item, str) for item in schema_type): + unsupported = [item for item in schema_type if item not in _JSON_SCHEMA_TYPES] + if unsupported: + raise ValueError(f"{path}.type contains unsupported value {unsupported[0]!r}") + else: + raise ValueError(f"{path}.type must be a string or string array") + if "properties" in schema: + properties = schema["properties"] + if not isinstance(properties, Mapping): + raise ValueError(f"{path}.properties must be an object") + for name, child in properties.items(): + if not isinstance(name, str): + raise ValueError(f"{path}.properties keys must be strings") + _validate_json_schema_subset(child, path=f"{path}.properties.{name}") + if "items" in schema: + items = schema["items"] + if isinstance(items, Mapping): + _validate_json_schema_subset(items, path=f"{path}.items") + elif isinstance(items, list): + raise ValueError(f"{path}.items array form is not supported for brokered tool schemas") + else: + raise ValueError(f"{path}.items must be an object") + if "required" in schema: + required = schema["required"] + if not isinstance(required, list) or not all(isinstance(item, str) for item in required): + raise ValueError(f"{path}.required must be a string array") + if "dependentRequired" in schema: + dependent_required = schema["dependentRequired"] + if not isinstance(dependent_required, Mapping): + raise ValueError(f"{path}.dependentRequired must be an object") + for key, value in dependent_required.items(): + if not isinstance(key, str) or not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{path}.dependentRequired values must be string arrays") + if "enum" in schema and not isinstance(schema["enum"], list): + raise ValueError(f"{path}.enum must be an array") + if "pattern" in schema: + raise ValueError(f"{path}.pattern is not supported for deterministic brokered tool schemas") + if "additionalProperties" in schema: + additional_properties = schema["additionalProperties"] + if not isinstance(additional_properties, (bool, dict)): + raise ValueError(f"{path}.additionalProperties must be a boolean or object") + if isinstance(additional_properties, dict): + _validate_json_schema_subset(additional_properties, path=f"{path}.additionalProperties") + constraint_keys = _NUMERIC_SCHEMA_KEYS | _INTEGER_SCHEMA_KEYS | {"pattern"} + if any(key in schema for key in ("enum", "const", "default")) and any(key in schema for key in constraint_keys): + raise ValueError(f"{path} combines enum/const/default with constraints unsupported by deterministic brokered synthesis") + if "multipleOf" in schema: + raise ValueError(f"{path}.multipleOf is not supported for deterministic brokered tool schemas") + for key in _NUMERIC_SCHEMA_KEYS: + if key in schema: + value = schema[key] + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{path}.{key} must be a number") + for key in _INTEGER_SCHEMA_KEYS: + if key in schema: + value = schema[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"{path}.{key} must be a non-negative integer") + +def _reject_unsafe_brokered_property_name_values(value: Any, *, path: str) -> None: + if isinstance(value, str): + unsafe = _unsafe_brokered_key(value) + if unsafe is not None: + raise ValueError(f"{path} value {unsafe!r} is not safe for brokered tool schemas") + elif isinstance(value, list): + for idx, item in enumerate(value): + _reject_unsafe_brokered_property_name_values(item, path=f"{path}[{idx}]") + elif isinstance(value, Mapping): + for key, child in value.items(): + if isinstance(key, str): + unsafe = _unsafe_brokered_key(key) + if unsafe is not None: + raise ValueError(f"{path}.{unsafe} is not safe for brokered tool schemas") + _reject_unsafe_brokered_property_name_values(child, path=f"{path}.{key}") + + class Metadata(_Strict): name: str = Field(min_length=1) @@ -309,6 +601,144 @@ class ContextSpec(_Strict): providers: list[ContextProviderSpec] = Field(default_factory=list) +def _schema_types(schema: Mapping[str, Any]) -> list[str]: + if "type" not in schema: + return [] + raw = schema.get("type") + if isinstance(raw, str): + return [raw] + if isinstance(raw, list) and all(isinstance(item, str) for item in raw): + return list(raw) + raise ValueError("brokered tool JSON Schema type must be a string or string array") + + +def _value_matches_schema_type(value: Any, schema_type: str) -> bool: + if schema_type == "null": + return value is None + if schema_type == "boolean": + return isinstance(value, bool) + if schema_type == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if schema_type == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if schema_type == "string": + return isinstance(value, str) + if schema_type == "array": + return isinstance(value, list) + if schema_type == "object": + return isinstance(value, Mapping) + raise ValueError(f"unsupported brokered tool JSON Schema type {schema_type!r}") + + +def _validate_schema_value_constraints(value: Any, *, path: str) -> None: + if not isinstance(value, Mapping): + return + types = _schema_types(value) + if types: + for keyword in ("const", "default"): + if keyword in value and not any(_value_matches_schema_type(value[keyword], schema_type) for schema_type in types): + raise ValueError(f"{path}.{keyword} must match the declared JSON Schema type") + enum = value.get("enum") + if enum is not None: + if not isinstance(enum, list): + raise ValueError(f"{path}.enum must be an array") + for idx, item in enumerate(enum): + if not any(_value_matches_schema_type(item, schema_type) for schema_type in types): + raise ValueError(f"{path}.enum[{idx}] must match the declared JSON Schema type") + properties = value.get("properties") + if isinstance(properties, Mapping): + for name, child in properties.items(): + if isinstance(child, Mapping): + _validate_schema_value_constraints(child, path=f"{path}.properties.{name}") + items = value.get("items") + if isinstance(items, Mapping): + _validate_schema_value_constraints(items, path=f"{path}.items") + elif isinstance(items, list): + for idx, child in enumerate(items): + if isinstance(child, Mapping): + _validate_schema_value_constraints(child, path=f"{path}.items[{idx}]") + additional = value.get("additionalProperties") + if isinstance(additional, Mapping): + _validate_schema_value_constraints(additional, path=f"{path}.additionalProperties") + + +class BrokeredToolSpec(_Strict): + """Static, safe Orka-brokered tool schema exposed to hosted Foundry models.""" + + name: str = Field(min_length=1) + description: str = Field(min_length=1) + brokered_class: Literal["read", "write", "coordination"] = Field(alias="brokeredClass") + parameters: dict[str, Any] + schema_digest: str | None = Field(default=None, alias="schemaDigest") + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def _reject_unsafe_top_level_fields(cls, data: Any) -> Any: + if isinstance(data, Mapping): + for key in data: + if isinstance(key, str) and _unsafe_brokered_key(key) is not None and key not in {"schemaDigest"}: + raise ValueError(f"brokered tool field {key!r} is unsafe") + return data + + @field_validator("description") + @classmethod + def _safe_description(cls, value: str) -> str: + if _unsafe_brokered_text(value): + raise ValueError("brokered tool description must not contain URLs or secret-like material") + return value + + @field_validator("name") + @classmethod + def _valid_name(cls, value: str) -> str: + if not _BROKERED_TOOL_NAME_RE.fullmatch(value): + raise ValueError("brokered tool name must match [A-Za-z0-9_-]{1,64}") + return value + + @field_validator("parameters") + @classmethod + def _valid_json_schema(cls, value: dict[str, Any]) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("brokered tool parameters must be a JSON Schema object") + try: + encoded = _canonical_json(value) + except (TypeError, ValueError) as exc: + raise ValueError("brokered tool parameters must be JSON serializable") from exc + if len(encoded.encode("utf-8")) > _MAX_BROKERED_SCHEMA_BYTES: + raise ValueError("brokered tool parameters schema is too large") + cloned = json.loads(encoded) + if cloned.get("type") != "object": + raise ValueError("brokered tool parameters schema must set type: object") + _validate_json_schema_subset(cloned, path="brokeredTools[].parameters") + _reject_unsafe_brokered_schema_keys(cloned, path="brokeredTools[].parameters") + _validate_schema_value_constraints(cloned, path="brokeredTools[].parameters") + return cloned + + @field_validator("schema_digest") + @classmethod + def _valid_schema_digest(cls, value: str | None) -> str | None: + if value is None: + return value + normalized = value.lower() + if value != normalized or not _BROKERED_SCHEMA_DIGEST_RE.fullmatch(value): + raise ValueError("brokered tool schemaDigest must be sha256:<64 lowercase hex>") + return value + + @model_validator(mode="after") + def _schema_digest_matches(self) -> "BrokeredToolSpec": + if self.schema_digest is not None: + actual = brokered_tool_schema_digest( + name=self.name, + description=self.description, + brokered_class=self.brokered_class, + parameters=self.parameters, + ) + if self.schema_digest != actual: + raise ValueError("brokered tool schemaDigest does not match the safe schema") + return self + + class ObservabilityOTelSpec(_Strict): endpoint_env: str | None = Field(default=None, alias="endpointEnv") @@ -370,6 +800,7 @@ class AgentSpec(_Strict): # Fully-resolved system prompt scalar (writer resolves inline|file -> string). instructions: str tools: list[ToolSpec] = Field(default_factory=list) + brokered_tools: list[BrokeredToolSpec] = Field(default_factory=list, alias="brokeredTools") env: list[EnvVarSpec] = Field(default_factory=list) context: ContextSpec = Field(default_factory=ContextSpec) observability: ObservabilitySpec = Field(default_factory=ObservabilitySpec) @@ -391,9 +822,30 @@ def _unique_env_names(cls, value: list[EnvVarSpec]) -> list[EnvVarSpec]: raise ValueError(f"duplicate env var declarations: {names}") return value + @field_validator("brokered_tools") + @classmethod + def _unique_brokered_tool_names(cls, value: list[BrokeredToolSpec]) -> list[BrokeredToolSpec]: + seen: set[str] = set() + duplicates: list[str] = [] + for entry in value: + if entry.name in seen: + duplicates.append(entry.name) + seen.add(entry.name) + if duplicates: + names = ", ".join(sorted(set(duplicates))) + raise ValueError(f"duplicate brokered tool declarations: {names}") + return value + @model_validator(mode="after") def _valid_context_tool_refs(self) -> "AgentSpec": tools = {tool.name: tool for tool in self.tools} + owned_tool_names = set(tools) + brokered_names = {tool.name for tool in self.brokered_tools} + if owned_tool_names and brokered_names: + raise ValueError("tools and brokeredTools cannot be mixed in v0; direct AgentKit-owned tools are disabled in brokered Foundry mode") + overlap = owned_tool_names & brokered_names + if overlap: + raise ValueError(f"tool names cannot be both owned and brokered: {', '.join(sorted(overlap))}") for provider in self.context.providers: if provider.type == _CONTEXT_TYPE_SKILLS and provider.source == _CONTEXT_SOURCE_MCP: if not provider.tool_ref: diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index a6c5022..a67bff9 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -1,28 +1,89 @@ """Foundry Hosted Agent protocol adapters over the AgentKit RuntimeSession seam. The adapter intentionally stays provider-light: it exposes the container HTTP -contract expected by Foundry Hosted Agents (``/readiness``, ``/invocations`` and a -minimal non-streaming ``/responses``) while reusing the same ``RuntimeFactory`` / -``RunRequest`` seam as the native OpenAI facade. +contract expected by Foundry Hosted Agents (``/readiness``, ``/invocations`` and +``/responses``) while reusing the same ``RuntimeFactory`` / ``RunRequest`` seam +as the native OpenAI facade. + +When ``agent.yaml`` contains static ``brokeredTools`` declarations, the hosted +``/responses`` route enters a deterministic brokered function-call loop. The +container emits Responses ``function_call`` output items but never executes the +Orka-governed tool locally. A later continuation must provide a matching +``function_call_output`` for the same ``previous_response_id`` and pending +``call_id``. """ from __future__ import annotations +import asyncio import json +import logging +import math import os +import re import time import uuid +from decimal import Decimal, InvalidOperation +from pathlib import Path from contextlib import asynccontextmanager -from typing import Any +from dataclasses import dataclass, field +from typing import Any, Mapping from fastapi import Depends, FastAPI, Request from fastapi.responses import JSONResponse, Response -from .config import AgentSpec +from .brokered import brokered_tool_definitions +from .config import AgentSpec, _unsafe_brokered_key, _unsafe_brokered_text +from .foundry_model_loop import BrokeredChatModelLoop, ModelLoopFinal, ModelLoopToolRequest from .conversation import FORWARDED_ROLES, ConversationTurn, RunRequest -from .runtime import AgentRunError, RuntimeFactory, RunResult +from .runtime import AgentRunError, BrokeredToolDefinition, RunResult, RuntimeFactory from .server import make_auth_dependency +logger = logging.getLogger(__name__) + +try: # Prefer the official hosted Responses SDK ID format/state-compatible prefix. + from azure.ai.agentserver.responses._id_generator import IdGenerator as _AzureResponsesIdGenerator +except Exception: # pragma: no cover - dependency is declared; fallback is for source-tree imports only. + _AzureResponsesIdGenerator = None + +_DEFAULT_STATE_TTL_SECONDS = 15 * 60 +_DEFAULT_MAX_PENDING_RESPONSES = 128 +_DEFAULT_MAX_ARGUMENT_BYTES = 8192 +_MAX_SYNTHETIC_ARRAY_ITEMS = 32 +_MAX_SYNTHETIC_STRING_LENGTH = 4096 +_STATE_TTL_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS" +_MAX_PENDING_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING" +_MAX_ARGUMENT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES" +_CONTINUATION_PROOF_ENV = "AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF" +_CONTINUATION_PROOF_HEADER = "x-agentkit-brokered-continuation-proof" +_MODEL_LOOP_ENV = "AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP" +_STATE_FILE_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_FILE" + + +def _new_response_id(previous_response_id: str | None = None) -> str: + if _AzureResponsesIdGenerator is not None: + try: + return _AzureResponsesIdGenerator.new_response_id(previous_response_id or "") + except TypeError: # Older/newer SDKs may expose this as a zero-argument helper. + return _AzureResponsesIdGenerator.new_response_id() + return f"caresp_{uuid.uuid4().hex}{uuid.uuid4().hex[:18]}" + + +def _new_message_id(response_id: str) -> str: + if _AzureResponsesIdGenerator is not None: + message_id = getattr(_AzureResponsesIdGenerator, "new_message_item_id", None) + if callable(message_id): + return message_id(response_id) + return f"msg_{uuid.uuid4().hex}" + + +def _new_function_call_id(response_id: str) -> str: + if _AzureResponsesIdGenerator is not None: + function_call_id = getattr(_AzureResponsesIdGenerator, "new_function_call_item_id", None) + if callable(function_call_id): + return function_call_id(response_id) + return f"fc_{uuid.uuid4().hex}" + def _usage(result: RunResult) -> dict[str, int]: usage = result.usage or {} @@ -33,11 +94,11 @@ def _usage(result: RunResult) -> dict[str, int]: } -def _responses_usage(result: RunResult) -> dict[str, int]: - usage = result.usage or {} - input_tokens = int(usage.get("input_tokens", usage.get("prompt_tokens", 0)) or 0) - output_tokens = int(usage.get("output_tokens", usage.get("completion_tokens", 0)) or 0) - total_tokens = int(usage.get("total_tokens", input_tokens + output_tokens) or 0) +def _responses_usage(result: RunResult | None = None, usage: Mapping[str, int] | None = None) -> dict[str, int]: + raw = dict(usage or (result.usage if result is not None else {}) or {}) + input_tokens = int(raw.get("input_tokens", raw.get("prompt_tokens", 0)) or 0) + output_tokens = int(raw.get("output_tokens", raw.get("completion_tokens", 0)) or 0) + total_tokens = int(raw.get("total_tokens", input_tokens + output_tokens) or 0) return { "input_tokens": input_tokens, "output_tokens": output_tokens, @@ -45,6 +106,21 @@ def _responses_usage(result: RunResult) -> dict[str, int]: } +def _combine_usage(*usages: Mapping[str, int] | None) -> dict[str, int]: + prompt_tokens = 0 + completion_tokens = 0 + for usage in usages: + if not usage: + continue + prompt_tokens += int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) + completion_tokens += int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + def _error(message: str, status: int = 400, code: str | None = None) -> JSONResponse: return JSONResponse( {"error": {"message": message, "code": code}}, @@ -131,10 +207,10 @@ def _responses_input_to_run_request(value: Any, *, session_id: str | None) -> Ru return RunRequest(prompt=json.dumps(value, separators=(",", ":"), sort_keys=True), session_id=session_id) -def _responses_payload(spec: AgentSpec, result: RunResult) -> dict[str, Any]: - response_id = f"resp_{uuid.uuid4().hex}" - message_id = f"msg_{uuid.uuid4().hex}" - return { +def _responses_payload(spec: AgentSpec, result: RunResult, *, previous_response_id: str | None = None) -> dict[str, Any]: + response_id = _new_response_id(previous_response_id) + message_id = _new_message_id(response_id) + payload: dict[str, Any] = { "id": response_id, "object": "response", "created_at": int(time.time()), @@ -153,22 +229,1062 @@ def _responses_payload(spec: AgentSpec, result: RunResult) -> dict[str, Any]: "annotations": [], } ], + "response_id": response_id, } ], "usage": _responses_usage(result), } + if previous_response_id: + payload["previous_response_id"] = previous_response_id + return payload + + +@dataclass(frozen=True) +class _PendingCall: + call_id: str + item_id: str + tool: BrokeredToolDefinition + arguments: dict[str, Any] + + +@dataclass +class _HostedResponseState: + response_id: str + session_id: str | None + pending_calls: dict[str, _PendingCall] + expires_at: float + status: str = "pending" + accepted_outputs: dict[str, str] = field(default_factory=dict) + final_payload: dict[str, Any] | None = None + model_messages: list[dict[str, Any]] | None = None + initial_usage: dict[str, int] = field(default_factory=dict) + + +class _StateExpired(KeyError): + pass + + +class _StateStoreFull(Exception): + pass + + +def _tool_to_state_payload(tool: BrokeredToolDefinition) -> dict[str, Any]: + return { + "name": tool.name, + "description": tool.description, + "brokeredClass": tool.brokered_class, + "parameters": dict(tool.parameters), + "schemaDigest": tool.schema_digest, + } + + +def _tool_from_state_payload(data: Mapping[str, Any]) -> BrokeredToolDefinition: + brokered_class = data.get("brokeredClass") + if brokered_class not in {"read", "write", "coordination"}: + raise ValueError("stored brokered tool class is invalid") + parameters = data.get("parameters", {}) + if not isinstance(parameters, Mapping): + raise ValueError("stored brokered tool parameters must be an object") + schema_digest = data.get("schemaDigest") + return BrokeredToolDefinition( + name=str(data.get("name") or ""), + description=str(data.get("description") or ""), + brokered_class=brokered_class, # type: ignore[arg-type] + parameters=dict(parameters), + schema_digest=str(schema_digest) if schema_digest else None, + ) + + +def _pending_call_to_state_payload(call: _PendingCall) -> dict[str, Any]: + return { + "callID": call.call_id, + "itemID": call.item_id, + "tool": _tool_to_state_payload(call.tool), + "arguments": dict(call.arguments), + } + + +def _pending_call_from_state_payload(data: Mapping[str, Any]) -> _PendingCall: + tool = data.get("tool") + if not isinstance(tool, Mapping): + raise ValueError("stored pending call tool must be an object") + arguments = data.get("arguments", {}) + if not isinstance(arguments, Mapping): + raise ValueError("stored pending call arguments must be an object") + return _PendingCall( + call_id=str(data.get("callID") or ""), + item_id=str(data.get("itemID") or ""), + tool=_tool_from_state_payload(tool), + arguments=dict(arguments), + ) + + +def _state_to_payload(state: _HostedResponseState) -> dict[str, Any]: + return { + "responseID": state.response_id, + "sessionID": state.session_id, + "pendingCalls": {call_id: _pending_call_to_state_payload(call) for call_id, call in state.pending_calls.items()}, + "expiresAt": state.expires_at, + "status": state.status, + "acceptedOutputs": dict(state.accepted_outputs), + "finalPayload": state.final_payload, + "modelMessages": state.model_messages, + "initialUsage": dict(state.initial_usage), + } + + +def _state_from_payload(data: Mapping[str, Any]) -> _HostedResponseState: + pending_calls_raw = data.get("pendingCalls", {}) + if not isinstance(pending_calls_raw, Mapping): + raise ValueError("stored pendingCalls must be an object") + accepted_outputs = data.get("acceptedOutputs", {}) + if not isinstance(accepted_outputs, Mapping): + raise ValueError("stored acceptedOutputs must be an object") + final_payload = data.get("finalPayload") + model_messages = data.get("modelMessages") + initial_usage = data.get("initialUsage", {}) + if final_payload is not None and not isinstance(final_payload, dict): + raise ValueError("stored finalPayload must be an object") + if model_messages is not None and not isinstance(model_messages, list): + raise ValueError("stored modelMessages must be an array") + if not isinstance(initial_usage, Mapping): + raise ValueError("stored initialUsage must be an object") + status = str(data.get("status") or "pending") + accepted = {str(key): str(value) for key, value in accepted_outputs.items()} + if final_payload is None and accepted: + # A persisted accepted output without a final payload means the process + # stopped between accepting the continuation and completing the resume. + # Clear it on load so Orka can retry the same continuation instead of + # being stuck behind duplicate_continuation_in_progress until TTL expiry. + status = "pending" + accepted = {} + return _HostedResponseState( + response_id=str(data.get("responseID") or ""), + session_id=str(data["sessionID"]) if data.get("sessionID") is not None else None, + pending_calls={str(call_id): _pending_call_from_state_payload(call) for call_id, call in pending_calls_raw.items() if isinstance(call, Mapping)}, + expires_at=float(data.get("expiresAt") or 0), + status=status, + accepted_outputs=accepted, + final_payload=final_payload, + model_messages=model_messages, + initial_usage={str(key): int(value or 0) for key, value in initial_usage.items()}, + ) + + +class _FoundryResponseStateStore: + """Continuation store for hosted Responses brokered calls. + + Without a file path this is in-memory only. With a file path, the store + persists pending/final response state using atomic JSON writes so a restarted + single-replica/sticky deployment can resume known response IDs. + """ + + def __init__(self, ttl_seconds: float, max_entries: int, state_file: str | Path | None = None) -> None: + self.ttl_seconds = ttl_seconds + self.max_entries = max_entries + self.state_file = Path(state_file) if state_file else None + self._states: dict[str, _HostedResponseState] = {} + self._load() + + @property + def backend_name(self) -> str: + return "file" if self.state_file else "memory" + + def add(self, state: _HostedResponseState) -> None: + self.purge_expired() + if len(self._states) >= self.max_entries and state.response_id not in self._states: + self.evict_completed_to_capacity(reserve_slots=1) + if len(self._states) >= self.max_entries and state.response_id not in self._states: + raise _StateStoreFull("too many pending brokered responses") + self._states[state.response_id] = state + self._persist() + + def save(self, state: _HostedResponseState) -> None: + if state.response_id in self._states: + self._states[state.response_id] = state + self._persist() + + def evict_completed_to_capacity(self, *, reserve_slots: int = 0) -> None: + target = max(self.max_entries - reserve_slots, 0) + if len(self._states) <= target: + return + completed = sorted( + (entry for entry in self._states.values() if entry.status != "pending"), + key=lambda entry: entry.expires_at, + ) + changed = False + for entry in completed: + self._states.pop(entry.response_id, None) + changed = True + if len(self._states) <= target: + break + if changed: + self._persist() + + def get(self, response_id: str) -> _HostedResponseState: + state = self._states.get(response_id) + if state is None: + raise KeyError(response_id) + if state.expires_at <= time.time(): + self._states.pop(response_id, None) + self._persist() + raise _StateExpired(response_id) + self.purge_expired() + return state + + def purge_expired(self) -> None: + now = time.time() + expired = [response_id for response_id, state in self._states.items() if state.expires_at <= now] + for response_id in expired: + self._states.pop(response_id, None) + if expired: + self._persist() + + def _load(self) -> None: + if self.state_file is None or not self.state_file.exists(): + return + try: + data = json.loads(self.state_file.read_text(encoding="utf-8")) + states = data.get("states", {}) if isinstance(data, Mapping) else {} + if not isinstance(states, Mapping): + raise ValueError("Foundry response state file states must be an object") + self._states = {str(response_id): _state_from_payload(state) for response_id, state in states.items() if isinstance(state, Mapping)} + self.purge_expired() + except (OSError, json.JSONDecodeError, ValueError) as exc: + logger.warning("ignoring invalid Foundry response state file %s: %s", self.state_file, exc) + self._states = {} + + def _persist(self) -> None: + if self.state_file is None: + return + self.state_file.parent.mkdir(parents=True, exist_ok=True) + payload = {"states": {response_id: _state_to_payload(state) for response_id, state in self._states.items()}} + tmp = self.state_file.with_name(f".{self.state_file.name}.tmp") + data = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(tmp, flags, 0o600) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + except Exception: + try: + tmp.unlink(missing_ok=True) + finally: + raise + os.chmod(tmp, 0o600) + tmp.replace(self.state_file) + os.chmod(self.state_file, 0o600) + + +def _state_ttl_seconds(value: float | None = None) -> float: + if value is not None: + return max(float(value), 0.0) + raw = os.environ.get(_STATE_TTL_ENV) + if not raw: + return float(_DEFAULT_STATE_TTL_SECONDS) + try: + parsed = float(raw) + except ValueError: + return float(_DEFAULT_STATE_TTL_SECONDS) + return max(parsed, 0.0) + + +def _positive_int_setting(value: int | None, *, env_name: str, default: int) -> int: + if value is not None: + return max(int(value), 1) + raw = os.environ.get(env_name) + if not raw: + return default + try: + parsed = int(raw) + except ValueError: + return default + return parsed if parsed > 0 else default + + +def _max_pending_responses(value: int | None = None) -> int: + return _positive_int_setting(value, env_name=_MAX_PENDING_ENV, default=_DEFAULT_MAX_PENDING_RESPONSES) + + +def _max_argument_bytes(value: int | None = None) -> int: + return _positive_int_setting(value, env_name=_MAX_ARGUMENT_BYTES_ENV, default=_DEFAULT_MAX_ARGUMENT_BYTES) + + +def _brokered_model_loop_enabled(value: bool | None = None) -> bool: + if value is not None: + return value + return os.environ.get(_MODEL_LOOP_ENV, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _response_state_file(value: str | Path | None = None) -> str | Path | None: + if value is not None: + return value + raw = os.environ.get(_STATE_FILE_ENV) + return raw.strip() if raw and raw.strip() else None + + +def _function_call_outputs_from_input(input_value: Any) -> list[dict[str, Any]]: + items: list[Any] + if isinstance(input_value, dict): + items = [input_value] + elif isinstance(input_value, list): + items = input_value + else: + return [] + outputs: list[dict[str, Any]] = [] + for item in items: + if isinstance(item, dict) and item.get("type") == "function_call_output": + outputs.append(item) + return outputs + + +def _reject_nonfinite_json_values(value: Any, *, path: str = "value") -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"{path} must be finite") + if isinstance(value, Mapping): + for key, child in value.items(): + _reject_nonfinite_json_values(child, path=f"{path}.{key}") + elif isinstance(value, list): + for idx, child in enumerate(value): + _reject_nonfinite_json_values(child, path=f"{path}[{idx}]") + + +def _json_object_from_output(output: Any) -> dict[str, Any]: + if isinstance(output, str): + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + raise ValueError("function_call_output.output must be a JSON object string") from exc + else: + parsed = output + if not isinstance(parsed, dict): + raise ValueError("function_call_output.output must be a JSON object") + approved = parsed.get("approved") + if not isinstance(approved, bool): + raise ValueError("function_call_output.output.approved must be a boolean") + if approved: + tool_output = parsed.get("output", {}) + if tool_output is not None and not isinstance(tool_output, dict): + raise ValueError("approved function_call_output.output.output must be an object") + else: + error = parsed.get("error", {}) + if error is not None and not isinstance(error, dict): + raise ValueError("denied function_call_output.output.error must be an object") + _reject_nonfinite_json_values(parsed, path="function_call_output.output") + return json.loads(json.dumps(parsed, allow_nan=False, separators=(",", ":"), sort_keys=True)) + + +def _canonical_output_json(output: dict[str, Any]) -> str: + _reject_nonfinite_json_values(output) + return json.dumps(output, allow_nan=False, separators=(",", ":"), sort_keys=True) + + +def _first_typed_value(schema: Mapping[str, Any], key: str, expected_type: type) -> Any: + if key in schema and isinstance(schema[key], expected_type): + return schema[key] + return None + + +def _enum_value(schema: Mapping[str, Any], expected_type: type) -> Any: + values = schema.get("enum") + if isinstance(values, list): + for value in values: + if isinstance(value, expected_type): + return value + return None + + +def _required_property_names(schema: Mapping[str, Any]) -> list[str]: + names: list[str] = [] + required = schema.get("required") + if isinstance(required, list): + names.extend(name for name in required if isinstance(name, str)) + + dependent_required = schema.get("dependentRequired") + min_properties = schema.get("minProperties") + properties = schema.get("properties") + + changed = True + while changed: + changed = False + if isinstance(dependent_required, Mapping): + present = set(names) + for trigger, dependent_names in dependent_required.items(): + if trigger not in present or not isinstance(dependent_names, list): + continue + for dependent_name in dependent_names: + if isinstance(dependent_name, str) and dependent_name not in present: + names.append(dependent_name) + present.add(dependent_name) + changed = True + if isinstance(min_properties, int) and not isinstance(min_properties, bool) and isinstance(properties, Mapping): + for name in properties: + if len(names) >= min_properties: + break + if isinstance(name, str) and name not in names: + names.append(name) + changed = True + break + + if isinstance(min_properties, int) and not isinstance(min_properties, bool) and len(names) < min_properties: + raise AgentRunError( + "brokered tool schema has minProperties that cannot be synthesized", + status=400, + code="UnsupportedBrokeredSchema", + ) + return names + + +def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> Any: + if not isinstance(schema, Mapping): + return run_request.prompt + if "const" in schema: + return schema["const"] + if "default" in schema: + return schema["default"] + enum_values = schema.get("enum") + if isinstance(enum_values, list) and enum_values: + return enum_values[0] + + schema_type = schema.get("type") + if isinstance(schema_type, list): + schema_type = next((item for item in schema_type if item != "null"), schema_type[0] if schema_type else None) + + if schema_type == "null": + return None + + if schema_type == "boolean": + for key in ("const", "default"): + value = _first_typed_value(schema, key, bool) + if value is not None: + return value + value = _enum_value(schema, bool) + return value if value is not None else True + + if schema_type == "integer": + for key in ("const", "default"): + value = _first_typed_value(schema, key, int) + if value is not None and not isinstance(value, bool): + return value + value = _enum_value(schema, int) + if value is not None and not isinstance(value, bool): + return value + lower_value = schema.get("minimum") + if isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) and math.isfinite(float(lower_value)): + lower = math.ceil(float(lower_value)) + else: + exclusive_lower = schema.get("exclusiveMinimum") + if isinstance(exclusive_lower, (int, float)) and not isinstance(exclusive_lower, bool) and math.isfinite(float(exclusive_lower)): + lower = math.floor(float(exclusive_lower)) + 1 + else: + lower = 0 + upper_value = schema.get("maximum") + if isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) and math.isfinite(float(upper_value)): + upper = math.floor(float(upper_value)) + else: + exclusive_upper = schema.get("exclusiveMaximum") + if isinstance(exclusive_upper, (int, float)) and not isinstance(exclusive_upper, bool) and math.isfinite(float(exclusive_upper)): + upper = math.ceil(float(exclusive_upper)) - 1 + else: + upper = None + if upper is not None and lower > upper: + if "minimum" in schema or "exclusiveMinimum" in schema: + raise AgentRunError( + f"brokered tool schema for {name!r} has incompatible integer bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) + lower = upper + multiple_of = schema.get("multipleOf") + if isinstance(multiple_of, int) and not isinstance(multiple_of, bool) and multiple_of > 0: + remainder = lower % multiple_of + candidate = lower if remainder == 0 else lower + (multiple_of - remainder) + if upper is not None and candidate > upper: + raise AgentRunError( + f"brokered tool schema for {name!r} has no integer multipleOf value in bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) + return candidate + if multiple_of is not None: + raise AgentRunError( + f"brokered tool schema for {name!r} has unsupported integer multipleOf", + status=400, + code="UnsupportedBrokeredSchema", + ) + return lower + + if schema_type == "number": + for key in ("const", "default"): + value = schema.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return value + value = _enum_value(schema, (int, float)) + if value is not None and not isinstance(value, bool): + return value + lower_value = schema.get("minimum") + lower_open = False + if not isinstance(lower_value, (int, float)) or isinstance(lower_value, bool): + lower_value = schema.get("exclusiveMinimum") + lower_open = isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) + upper_value = schema.get("maximum") + upper_open = False + if not isinstance(upper_value, (int, float)) or isinstance(upper_value, bool): + upper_value = schema.get("exclusiveMaximum") + upper_open = isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) + has_lower = isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) + has_upper = isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) + if has_lower and has_upper: + if lower_value > upper_value or (lower_value == upper_value and (lower_open or upper_open)): + raise AgentRunError( + f"brokered tool schema for {name!r} has incompatible numeric bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) + if lower_value == upper_value: + return lower_value + return (lower_value + upper_value) / 2 + if has_lower: + candidate = lower_value + (1 if lower_open else 0) + elif has_upper: + candidate = upper_value - (1 if upper_open else 0) + else: + candidate = 0 + multiple_of = schema.get("multipleOf") + if isinstance(multiple_of, (int, float)) and not isinstance(multiple_of, bool) and multiple_of > 0: + candidate = math.ceil(candidate / multiple_of) * multiple_of + if has_upper and (candidate > upper_value or (candidate == upper_value and upper_open)): + raise AgentRunError( + f"brokered tool schema for {name!r} has no numeric multipleOf value in bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) + elif multiple_of is not None: + raise AgentRunError( + f"brokered tool schema for {name!r} has unsupported numeric multipleOf", + status=400, + code="UnsupportedBrokeredSchema", + ) + return candidate + + if schema_type == "array": + for key in ("const", "default"): + value = _first_typed_value(schema, key, list) + if value is not None: + return value + value = _enum_value(schema, list) + if value is not None: + return value + min_items = schema.get("minItems", 0) + if isinstance(min_items, int) and min_items > 0: + if min_items > _MAX_SYNTHETIC_ARRAY_ITEMS: + raise AgentRunError( + f"brokered tool schema for {name!r} has minItems too large for deterministic synthesis", + status=413, + code="brokered_arguments_too_large", + ) + item_schema = schema.get("items", {}) + return [_sample_argument_value(name, item_schema, run_request) for _ in range(min_items)] + return [] + + if schema_type == "object": + for key in ("const", "default"): + value = _first_typed_value(schema, key, dict) + if value is not None: + return value + nested: dict[str, Any] = {} + properties = schema.get("properties") if isinstance(schema.get("properties"), Mapping) else {} + for child_name in _required_property_names(schema): + nested[child_name] = _sample_argument_value(child_name, properties.get(child_name, {}), run_request) + return nested + + for key in ("const", "default"): + value = _first_typed_value(schema, key, str) + if value is not None: + return value + value = _enum_value(schema, str) + if value is not None: + return value + if schema.get("pattern"): + raise AgentRunError( + f"brokered tool schema for {name!r} uses pattern without const/default/enum", + status=400, + code="UnsupportedBrokeredSchema", + ) + sample = run_request.prompt if name in {"prompt", "site"} else name + min_length = schema.get("minLength", 0) + if isinstance(min_length, int) and min_length > _MAX_SYNTHETIC_STRING_LENGTH: + raise AgentRunError( + f"brokered tool schema for {name!r} has minLength too large for deterministic synthesis", + status=413, + code="brokered_arguments_too_large", + ) + if isinstance(min_length, int) and len(sample) < min_length: + sample = sample + ("x" * (min_length - len(sample))) + max_length = schema.get("maxLength") + if isinstance(max_length, int) and max_length >= 0 and len(sample) > max_length: + if isinstance(min_length, int) and min_length > max_length: + raise AgentRunError( + f"brokered tool schema for {name!r} has incompatible minLength/maxLength", + status=400, + code="UnsupportedBrokeredSchema", + ) + sample = sample[:max_length] + return sample + + +def _schema_has_literal_value(schema: Any) -> bool: + return isinstance(schema, Mapping) and ("const" in schema or "default" in schema or (isinstance(schema.get("enum"), list) and len(schema.get("enum")) == 1) or schema.get("type") == "null") + + +def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: RunRequest) -> dict[str, Any]: + parameters = tool.parameters if isinstance(tool.parameters, Mapping) else {} + for key in ("const", "default"): + literal = parameters.get(key) + if isinstance(literal, Mapping): + return dict(literal) + enum = parameters.get("enum") + if isinstance(enum, list): + for item in enum: + if isinstance(item, Mapping): + return dict(item) + properties = parameters.get("properties") if isinstance(parameters.get("properties"), Mapping) else {} + required_names = _required_property_names(parameters) + if tool.brokered_class != "read": + for name in required_names: + if not _schema_has_literal_value(properties.get(name, {})): + raise AgentRunError( + f"brokered {tool.brokered_class} tool {tool.name!r} requires non-literal argument {name!r}; deterministic mode refuses to synthesize side-effecting arguments", + status=400, + code="UnsupportedBrokeredSchema", + ) + arguments: dict[str, Any] = {} + for name in required_names: + arguments[name] = _sample_argument_value(name, properties.get(name, {}), run_request) + if tool.name == "conformance_read" and "probe" in properties and "probe" not in arguments: + arguments["probe"] = True + if not arguments and "prompt" in properties: + arguments["prompt"] = _sample_argument_value("prompt", properties["prompt"], run_request) + return arguments + + +def _prompt_mentions_tool(prompt: str, tool_name: str) -> bool: + pattern = r"(? BrokeredToolDefinition | None: + matches = [tool for tool in tools if _prompt_mentions_tool(run_request.prompt, tool.name)] + if len(matches) == 1: + return matches[0] + if len(tools) == 1 and tools[0].name == "conformance_read": + return tools[0] + return None + + + + +def _validate_model_brokered_arguments(value: Any, *, path: str = "arguments") -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + if not isinstance(key, str): + raise AgentRunError("model brokered tool argument keys must be strings", status=400, code="InvalidToolArguments") + if _unsafe_brokered_key(key) is not None: + raise AgentRunError( + f"model brokered tool argument {path}.{key} is not safe", + status=400, + code="UnsafeBrokeredArguments", + ) + _validate_model_brokered_arguments(child, path=f"{path}.{key}") + elif isinstance(value, list): + for idx, child in enumerate(value): + _validate_model_brokered_arguments(child, path=f"{path}[{idx}]") + elif isinstance(value, float) and not math.isfinite(value): + raise AgentRunError( + f"model brokered tool argument {path} must be finite", + status=400, + code="InvalidToolArguments", + ) + elif isinstance(value, str) and _unsafe_brokered_text(value): + raise AgentRunError( + f"model brokered tool argument {path} contains unsafe text", + status=400, + code="UnsafeBrokeredArguments", + ) + + + + +def _schema_types(schema: Mapping[str, Any]) -> list[str]: + schema_type = schema.get("type") + if isinstance(schema_type, str): + return [schema_type] + if isinstance(schema_type, list): + return [item for item in schema_type if isinstance(item, str)] + return [] + + +def _decimal_json_number(value: int | float) -> Decimal: + try: + decimal = Decimal(str(value)) + except InvalidOperation as exc: + raise AgentRunError("model brokered tool argument is not a finite JSON number", status=400, code="InvalidToolArguments") from exc + if not decimal.is_finite(): + raise AgentRunError("model brokered tool argument is not a finite JSON number", status=400, code="InvalidToolArguments") + return decimal + + +def _json_type_equal(left: Any, right: Any) -> bool: + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left == right + if left is None or right is None: + return left is None and right is None + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + return _decimal_json_number(left) == _decimal_json_number(right) + if isinstance(left, str) or isinstance(right, str): + return isinstance(left, str) and isinstance(right, str) and left == right + if isinstance(left, list) or isinstance(right, list): + return ( + isinstance(left, list) + and isinstance(right, list) + and len(left) == len(right) + and all(_json_type_equal(a, b) for a, b in zip(left, right)) + ) + if isinstance(left, Mapping) or isinstance(right, Mapping): + return ( + isinstance(left, Mapping) + and isinstance(right, Mapping) + and set(left) == set(right) + and all(_json_type_equal(left[key], right[key]) for key in left) + ) + return left == right + + +def _value_matches_type(value: Any, schema_type: str) -> bool: + if schema_type == "null": + return value is None + if schema_type == "boolean": + return isinstance(value, bool) + if schema_type == "integer": + return (isinstance(value, int) and not isinstance(value, bool)) or ( + isinstance(value, float) and math.isfinite(value) and value.is_integer() + ) + if schema_type == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if schema_type == "string": + return isinstance(value, str) + if schema_type == "array": + return isinstance(value, list) + if schema_type == "object": + return isinstance(value, Mapping) + return True + + +def _validate_model_argument_against_schema(value: Any, schema: Any, *, path: str) -> None: + if not isinstance(schema, Mapping): + return + if "const" in schema and not _json_type_equal(value, schema["const"]): + raise AgentRunError(f"model brokered tool argument {path} does not match const", status=400, code="InvalidToolArguments") + enum_values = schema.get("enum") + if isinstance(enum_values, list) and not any(_json_type_equal(value, enum_value) for enum_value in enum_values): + raise AgentRunError(f"model brokered tool argument {path} is not in enum", status=400, code="InvalidToolArguments") + types = _schema_types(schema) + if types and not any(_value_matches_type(value, schema_type) for schema_type in types): + raise AgentRunError(f"model brokered tool argument {path} has wrong type", status=400, code="InvalidToolArguments") + + if isinstance(value, str): + min_length = schema.get("minLength") + if isinstance(min_length, int) and len(value) < min_length: + raise AgentRunError(f"model brokered tool argument {path} is too short", status=400, code="InvalidToolArguments") + max_length = schema.get("maxLength") + if isinstance(max_length, int) and len(value) > max_length: + raise AgentRunError(f"model brokered tool argument {path} is too long", status=400, code="InvalidToolArguments") + pattern = schema.get("pattern") + if isinstance(pattern, str): + try: + matches = re.search(pattern, value) is not None + except re.error as exc: + raise AgentRunError( + f"brokered tool schema pattern for {path} is not supported by the model-loop validator", + status=400, + code="UnsupportedBrokeredSchema", + ) from exc + if not matches: + raise AgentRunError(f"model brokered tool argument {path} does not match pattern", status=400, code="InvalidToolArguments") + + if isinstance(value, int) and not isinstance(value, bool) or isinstance(value, float): + number = _decimal_json_number(value) + for key, compare in (("minimum", lambda a, b: a >= b), ("exclusiveMinimum", lambda a, b: a > b), ("maximum", lambda a, b: a <= b), ("exclusiveMaximum", lambda a, b: a < b)): + bound = schema.get(key) + if isinstance(bound, (int, float)) and not isinstance(bound, bool) and not compare(number, _decimal_json_number(bound)): + raise AgentRunError(f"model brokered tool argument {path} violates {key}", status=400, code="InvalidToolArguments") + multiple_of = schema.get("multipleOf") + if isinstance(multiple_of, (int, float)) and not isinstance(multiple_of, bool) and multiple_of > 0: + divisor = _decimal_json_number(multiple_of) + if divisor == 0 or number % divisor != 0: + raise AgentRunError(f"model brokered tool argument {path} violates multipleOf", status=400, code="InvalidToolArguments") + + if isinstance(value, list): + min_items = schema.get("minItems") + if isinstance(min_items, int) and len(value) < min_items: + raise AgentRunError(f"model brokered tool argument {path} has too few items", status=400, code="InvalidToolArguments") + max_items = schema.get("maxItems") + if isinstance(max_items, int) and len(value) > max_items: + raise AgentRunError(f"model brokered tool argument {path} has too many items", status=400, code="InvalidToolArguments") + item_schema = schema.get("items") + if isinstance(item_schema, Mapping): + for idx, item in enumerate(value): + _validate_model_argument_against_schema(item, item_schema, path=f"{path}[{idx}]") + + if isinstance(value, Mapping): + required = schema.get("required") + if isinstance(required, list): + for name in required: + if isinstance(name, str) and name not in value: + raise AgentRunError(f"model brokered tool argument {path}.{name} is required", status=400, code="InvalidToolArguments") + dependent_required = schema.get("dependentRequired") + if isinstance(dependent_required, Mapping): + for trigger, dependent_names in dependent_required.items(): + if trigger not in value or not isinstance(dependent_names, list): + continue + for dependent_name in dependent_names: + if isinstance(dependent_name, str) and dependent_name not in value: + raise AgentRunError(f"model brokered tool argument {path}.{dependent_name} is required", status=400, code="InvalidToolArguments") + min_properties = schema.get("minProperties") + if isinstance(min_properties, int) and len(value) < min_properties: + raise AgentRunError(f"model brokered tool argument {path} has too few properties", status=400, code="InvalidToolArguments") + max_properties = schema.get("maxProperties") + if isinstance(max_properties, int) and len(value) > max_properties: + raise AgentRunError(f"model brokered tool argument {path} has too many properties", status=400, code="InvalidToolArguments") + properties = schema.get("properties") if isinstance(schema.get("properties"), Mapping) else {} + additional = schema.get("additionalProperties") + if additional is False: + for key in value: + if key not in properties: + raise AgentRunError(f"model brokered tool argument {path}.{key} is not declared", status=400, code="InvalidToolArguments") + elif isinstance(additional, Mapping): + for key, child in value.items(): + if key not in properties: + _validate_model_argument_against_schema(child, additional, path=f"{path}.{key}") + for key, child_schema in properties.items(): + if isinstance(key, str) and key in value: + _validate_model_argument_against_schema(value[key], child_schema, path=f"{path}.{key}") + + +def _validate_model_arguments_for_tool(arguments: Mapping[str, Any], tool: BrokeredToolDefinition) -> None: + _validate_model_argument_against_schema(dict(arguments), tool.parameters, path="arguments") + + +def _function_call_response_payload( + spec: AgentSpec, + *, + response_id: str, + call: _PendingCall, + previous_response_id: str | None = None, + usage: Mapping[str, int] | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "id": response_id, + "object": "response", + "created_at": int(time.time()), + "status": "completed", + "model": spec.model.name, + "output": [ + { + "id": call.item_id, + "type": "function_call", + "call_id": call.call_id, + "name": call.tool.name, + "arguments": _canonical_output_json(call.arguments), + "status": "completed", + "response_id": response_id, + } + ], + "usage": _responses_usage(usage=usage), + } + if previous_response_id: + payload["previous_response_id"] = previous_response_id + return payload + + +def _final_text_from_tool_output(call: _PendingCall, output: dict[str, Any]) -> str: + if not output.get("approved"): + error = output.get("error") if isinstance(output.get("error"), dict) else {} + code = str(error.get("code") or "brokered_tool_denied") + message = str(error.get("message") or "brokered tool was not performed") + return f"Brokered tool {call.tool.name} was not performed: {code}: {message}" + tool_output = output.get("output") if isinstance(output.get("output"), dict) else {} + return f"Brokered tool {call.tool.name} completed with output: {_canonical_output_json(tool_output)}" + + +async def _handle_brokered_continuation( + *, + spec: AgentSpec, + store: _FoundryResponseStateStore, + previous_response_id: str | None, + input_value: Any, + continuation_proof: str | None, + request: Request, + model_loop: BrokeredChatModelLoop | None = None, +) -> JSONResponse: + if not continuation_proof: + return _error( + "brokered continuation proof is not configured; refusing function_call_output", + status=503, + code="brokered_continuation_auth_required", + ) + provided_proof = request.headers.get(_CONTINUATION_PROOF_HEADER) + if provided_proof != continuation_proof: + return _error( + "function_call_output is restricted to the Orka broker continuation path", + status=403, + code="brokered_continuation_forbidden", + ) + outputs = _function_call_outputs_from_input(input_value) + if not outputs: + return _error( + "Responses continuation requires a function_call_output input item", + status=400, + code="missing_function_call_output", + ) + if previous_response_id is None or not str(previous_response_id).strip(): + return _error( + "function_call_output requires previous_response_id", + status=400, + code="missing_previous_response_id", + ) + if len(outputs) != 1: + return _error( + "multiple function_call_output items are not supported by this deterministic brokered adapter", + status=400, + code="multiple_tool_outputs_unsupported", + ) + try: + state = store.get(str(previous_response_id)) + except _StateExpired: + return _error("previous_response_id state has expired", status=410, code="response_state_expired") + except KeyError: + return _error("unknown previous_response_id", status=404, code="unknown_previous_response_id") + + item = outputs[0] + call_id = item.get("call_id") + if not isinstance(call_id, str) or not call_id: + return _error("function_call_output.call_id is required", status=400, code="missing_call_id") + call = state.pending_calls.get(call_id) + if call is None: + return _error("unknown function_call_output call_id", status=400, code="unknown_call_id") + try: + parsed_output = _json_object_from_output(item.get("output")) + except ValueError as exc: + return _error(str(exc), status=400, code="invalid_function_call_output") + output_json = _canonical_output_json(parsed_output) + + existing_output = state.accepted_outputs.get(call_id) + if existing_output is not None: + if existing_output == output_json and state.final_payload is not None: + return JSONResponse(state.final_payload) + if existing_output == output_json: + return _error( + "matching function_call_output is already being processed", + status=409, + code="duplicate_continuation_in_progress", + ) + return _error( + "conflicting duplicate function_call_output for call_id", + status=409, + code="conflicting_duplicate_continuation", + ) + if state.status != "pending": + return _error("previous response is not pending a tool result", status=409, code="response_not_pending") + + state.accepted_outputs[call_id] = output_json + store.save(state) + if state.model_messages is not None and model_loop is not None: + state.status = "resuming" + store.save(state) + try: + model_result = await model_loop.resume(state.model_messages, call_id=call_id, output=output_json) + except AgentRunError as exc: + state.accepted_outputs.pop(call_id, None) + state.status = "pending" + store.save(state) + if exc.status >= 500: + logger.warning("brokered model-loop resume failed: %s", exc) + return _error("model resume failed", status=exc.status, code="ModelResumeError") + return _error(str(exc), status=exc.status, code=exc.code) + except asyncio.CancelledError: + state.accepted_outputs.pop(call_id, None) + state.status = "pending" + store.save(state) + raise + except Exception as exc: # noqa: BLE001 - reset continuation state before surfacing unexpected model failures. + logger.exception("brokered model-loop resume failed unexpectedly") + state.accepted_outputs.pop(call_id, None) + state.status = "pending" + store.save(state) + return _error("model resume failed", status=502, code="ModelResumeError") + if not isinstance(model_result, ModelLoopFinal): + state.accepted_outputs.pop(call_id, None) + state.status = "pending" + store.save(state) + return _error( + "model requested another brokered tool after resume", + status=400, + code="tool_loop_limit_exceeded", + ) + result = RunResult(text=model_result.text, usage=_combine_usage(state.initial_usage, model_result.usage)) + else: + result = RunResult(text=_final_text_from_tool_output(call, parsed_output)) + final_payload = _responses_payload(spec, result, previous_response_id=state.response_id) + state.status = "completed" + state.final_payload = final_payload + store.save(state) + store.evict_completed_to_capacity() + return JSONResponse(final_payload) def create_foundry_app( spec: AgentSpec, factory: RuntimeFactory, auth_token: str | None = None, + *, + state_ttl_seconds: float | None = None, + brokered_continuation_proof: str | None = None, + max_pending_responses: int | None = None, + max_brokered_argument_bytes: int | None = None, + brokered_model_loop_enabled: bool | None = None, + brokered_model_http_client: Any | None = None, + response_state_file: str | Path | None = None, ) -> FastAPI: """Create a Foundry-compatible wrapper app for one AgentKit runtime.""" - runtime = factory.build_runtime(spec) + brokered_tools = brokered_tool_definitions(spec) + runtime = None if brokered_tools else factory.build_runtime(spec) + continuation_proof = brokered_continuation_proof or os.environ.get(_CONTINUATION_PROOF_ENV) or None + max_argument_bytes = _max_argument_bytes(max_brokered_argument_bytes) + response_states = _FoundryResponseStateStore( + ttl_seconds=_state_ttl_seconds(state_ttl_seconds), + max_entries=_max_pending_responses(max_pending_responses), + state_file=_response_state_file(response_state_file) if brokered_tools else None, + ) + model_loop = ( + BrokeredChatModelLoop(spec, brokered_tools, http_client=brokered_model_http_client) + if brokered_tools and _brokered_model_loop_enabled(brokered_model_loop_enabled) + else None + ) @asynccontextmanager async def lifespan(app: FastAPI): + if runtime is None: + yield + return async with runtime: app.state.runtime = runtime yield @@ -178,10 +1294,31 @@ async def lifespan(app: FastAPI): @app.get("/readiness") async def readiness(): - return {"ready": True} + body: dict[str, Any] = {"ready": True} + if brokered_tools: + body["foundryResponses"] = { + "brokeredTools": len(brokered_tools), + "ownedToolsDisabled": len(spec.tools), + "stateBackend": response_states.backend_name, + "stateTtlSeconds": response_states.ttl_seconds, + "stateMaxPending": response_states.max_entries, + "continuationAuth": "configured" if continuation_proof else "missing", + "runtime": "model-loop" if model_loop is not None else "deterministic", + "scaling": "single-replica-or-sticky-routing-required", + } + if not continuation_proof: + body["ready"] = False + return JSONResponse(body, status_code=503) + return body @app.post("/invocations", dependencies=[auth]) async def invocations(request: Request): + if brokered_tools: + return _error( + "Foundry /invocations is disabled when brokeredTools are configured; use /responses so Orka can broker tools", + status=400, + code="invocations_disabled_in_brokered_mode", + ) try: data = await request.json() except json.JSONDecodeError: @@ -218,19 +1355,44 @@ async def responses(request: Request): # completed response instead of failing readiness/e2e checks. if data.get("tools"): return _error( - "request-supplied Responses tools are not allowed; this agent owns its tools", + "request-supplied Responses tools are not allowed; hosted brokered mode uses static safe schemas", status=400, code="tools_unsupported", ) - if data.get("tool_choice") not in (None, "", "none", "auto"): + tool_choice = data.get("tool_choice") + if tool_choice not in (None, "", "none", "auto") or (brokered_tools and tool_choice == "none"): return _error( - "request-supplied Responses tool_choice is not allowed; this agent owns its tools", + "request-supplied Responses tool_choice is not allowed; hosted brokered mode owns tool selection", status=400, code="tool_choice_unsupported", ) if "input" not in data: return _error("Missing 'input' in request", status=400, code="missing_input") + previous_response_id = data.get("previous_response_id") + function_outputs = _function_call_outputs_from_input(data["input"]) + if brokered_tools and function_outputs: + return await _handle_brokered_continuation( + spec=spec, + store=response_states, + previous_response_id=previous_response_id if isinstance(previous_response_id, str) else None, + input_value=data["input"], + continuation_proof=continuation_proof, + request=request, + model_loop=model_loop, + ) + if brokered_tools and isinstance(previous_response_id, str) and previous_response_id: + try: + previous_state = response_states.get(previous_response_id) + except (KeyError, _StateExpired): + previous_state = None + if previous_state is not None and previous_state.status == "pending": + return _error( + "previous_response_id is pending a brokered function_call_output", + status=409, + code="response_pending_function_call_output", + ) + try: run_request = _responses_input_to_run_request( data["input"], @@ -239,6 +1401,118 @@ async def responses(request: Request): except ValueError as exc: return _error(str(exc), status=400, code="invalid_input") + if brokered_tools: + if not continuation_proof: + return _error( + "brokered continuation proof is not configured; refusing to start an uncontinuable brokered response", + status=503, + code="brokered_continuation_auth_required", + ) + previous_response_id_for_output = previous_response_id if isinstance(previous_response_id, str) and previous_response_id else None + if model_loop is not None: + response_id = _new_response_id(previous_response_id_for_output) + call_id = f"call_{response_id}_1" + try: + model_result = await model_loop.start(run_request, call_id=call_id) + except AgentRunError as exc: + return _error(str(exc), status=exc.status, code=exc.code) + if isinstance(model_result, ModelLoopFinal): + return JSONResponse(_responses_payload(spec, RunResult(text=model_result.text, usage=model_result.usage), previous_response_id=previous_response_id_for_output)) + tool = {tool.name: tool for tool in brokered_tools}.get(model_result.name) + if tool is None: + return _error("model requested unknown brokered tool", status=400, code="unknown_brokered_tool") + try: + _validate_model_brokered_arguments(model_result.arguments) + _validate_model_arguments_for_tool(model_result.arguments, tool) + except AgentRunError as exc: + return _error(str(exc), status=exc.status, code=exc.code) + if len(_canonical_output_json(model_result.arguments).encode("utf-8")) > max_argument_bytes: + return _error( + "brokered function_call arguments are too large for pending state", + status=413, + code="brokered_arguments_too_large", + ) + call = _PendingCall( + call_id=call_id, + item_id=_new_function_call_id(response_id), + tool=tool, + arguments=model_result.arguments, + ) + state = _HostedResponseState( + response_id=response_id, + session_id=run_request.session_id, + pending_calls={call_id: call}, + expires_at=time.time() + response_states.ttl_seconds, + model_messages=model_result.messages, + initial_usage=dict(model_result.usage), + ) + try: + response_states.add(state) + except _StateStoreFull: + return _error( + "too many pending brokered responses", + status=429, + code="brokered_response_state_full", + ) + return JSONResponse( + _function_call_response_payload( + spec, + response_id=response_id, + call=call, + previous_response_id=previous_response_id_for_output, + usage=model_result.usage, + ) + ) + tool = _select_brokered_tool(brokered_tools, run_request) + if tool is None: + return _error( + "prompt must name exactly one configured brokered tool in deterministic brokered mode", + status=400, + code="brokered_tool_selection_required", + ) + response_id = _new_response_id(previous_response_id_for_output) + call_id = f"call_{response_id}_1" + try: + arguments = _deterministic_tool_arguments(tool, run_request) + _validate_model_brokered_arguments(arguments) + _validate_model_arguments_for_tool(arguments, tool) + except AgentRunError as exc: + return _error(str(exc), status=exc.status, code=exc.code) + if len(_canonical_output_json(arguments).encode("utf-8")) > max_argument_bytes: + return _error( + "brokered function_call arguments are too large for pending state", + status=413, + code="brokered_arguments_too_large", + ) + call = _PendingCall( + call_id=call_id, + item_id=_new_function_call_id(response_id), + tool=tool, + arguments=arguments, + ) + state = _HostedResponseState( + response_id=response_id, + session_id=run_request.session_id, + pending_calls={call_id: call}, + expires_at=time.time() + response_states.ttl_seconds, + ) + try: + response_states.add(state) + except _StateStoreFull: + return _error( + "too many pending brokered responses", + status=429, + code="brokered_response_state_full", + ) + return JSONResponse( + _function_call_response_payload( + spec, + response_id=response_id, + call=call, + previous_response_id=previous_response_id_for_output, + ) + ) + try: result = await request.app.state.runtime.run(run_request) except AgentRunError as exc: diff --git a/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py new file mode 100644 index 0000000..62cf38b --- /dev/null +++ b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py @@ -0,0 +1,96 @@ +"""Brokered-only Foundry hosted Responses entrypoint. + +This entrypoint is intentionally small: it loads a baked ``agent.yaml`` with +static ``brokeredTools`` and serves the shared Foundry `/responses` brokered +adapter without constructing a framework runtime. It is useful for deterministic +Foundry/Orka brokered smokes and for the lower-level model-loop fallback where +AgentKit-owned direct tools must stay disabled. +""" + +from __future__ import annotations + +import argparse +import json +import os +from types import TracebackType +from typing import Sequence + +import uvicorn + +from .config import AgentSpec, load_or_exit +from .foundry import create_foundry_app +from .runtime import AgentRunError, RunResult +from .conversation import RunRequest + +DEFAULT_CONFIG_PATH = "/agent/agent.yaml" +DEFAULT_PORT = 8088 + + +class _NoDirectRuntime: + async def __aenter__(self): + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + return None + + async def run(self, request: RunRequest) -> RunResult: # noqa: ARG002 - direct run is intentionally disabled. + raise AgentRunError( + "direct runtime execution is disabled in Foundry brokered-only mode", + status=400, + code="DirectRuntimeDisabled", + ) + + +class _NoDirectFactory: + def build_runtime(self, spec: AgentSpec): # noqa: ARG002 - spec-independent guard runtime. + return _NoDirectRuntime() + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="agentkit-foundry-brokered", + description="Serve a brokered-only AgentKit Foundry /responses endpoint from agent.yaml.", + ) + parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help=f"path to agent.yaml (default: {DEFAULT_CONFIG_PATH})") + parser.add_argument("--host", default=os.environ.get("AGENTKIT_BIND", "0.0.0.0"), help="host interface to bind") + parser.add_argument("--port", type=int, default=int(os.environ.get("AGENTKIT_PORT", os.environ.get("PORT", str(DEFAULT_PORT)))), help="port to bind") + parser.add_argument("--dry-run", action="store_true", help="load config and print selected serving metadata without binding") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + spec = load_or_exit(args.config) + if not spec.brokered_tools: + raise SystemExit("agentkit-foundry-brokered: agent.yaml must declare at least one brokeredTools entry") + auth_token = os.environ.get("AGENTKIT_AUTH_TOKEN") or None + app = create_foundry_app(spec, _NoDirectFactory(), auth_token=auth_token) + if args.dry_run: + print( + json.dumps( + { + "host": args.host, + "port": args.port, + "agent": spec.metadata.name, + "brokeredTools": [tool.name for tool in spec.brokered_tools], + "auth": "configured" if auth_token else "none", + "continuationProof": "configured" if os.environ.get("AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF") else "missing", + }, + sort_keys=True, + ) + ) + return 0 + uvicorn.run(app, host=args.host, port=args.port, log_level="info", access_log=True) + return 0 + + +__all__ = ["main"] + + +if __name__ == "__main__": # pragma: no cover - exercised by console script/dry-run tests. + raise SystemExit(main()) diff --git a/runtimes/common/agentkit_serve_common/foundry_conformance.py b/runtimes/common/agentkit_serve_common/foundry_conformance.py new file mode 100644 index 0000000..bf24623 --- /dev/null +++ b/runtimes/common/agentkit_serve_common/foundry_conformance.py @@ -0,0 +1,151 @@ +"""Tiny Azure Responses SDK conformance app for Foundry hosted brokered spikes. + +This module is intentionally separate from the production Foundry adapter. It is +used to prove the hosted Responses lifecycle can carry a deterministic +function_call/function_call_output loop with SDK-assigned response IDs and state. +""" + +from __future__ import annotations + +import argparse +import json +import os +from typing import Any, Sequence + +from starlette.responses import JSONResponse +from starlette.routing import Route + +from azure.ai.agentserver.responses import ( + InMemoryResponseProvider, + ResponseEventStream, + ResponsesAgentServerHost, + get_input_expanded, +) + +_CONFORMANCE_CALL_ID = "call_conformance_1" +_CONFORMANCE_TOOL_NAME = "conformance_read" +_CONFORMANCE_ARGUMENTS = '{"probe":true}' + + +def _input_items(request: Any) -> list[dict[str, Any]]: + return [dict(item) for item in get_input_expanded(request)] + + +def _function_call_outputs(request: Any) -> list[dict[str, Any]]: + return [item for item in _input_items(request) if item.get("type") == "function_call_output"] + + +def _request_tools(request: Any) -> Any: + tools = getattr(request, "tools", None) + if tools is None and hasattr(request, "get"): + tools = request.get("tools") + return tools + + +def create_foundry_conformance_app(*, model: str = "agentkit-foundry-conformance") -> ResponsesAgentServerHost: + """Create a minimal Responses SDK app for A0 Foundry function-call smokes.""" + + store = InMemoryResponseProvider() + app = ResponsesAgentServerHost(store=store, configure_observability=None) + pending: dict[str, set[str]] = {} + + async def readiness(_request): # noqa: ANN001 - Starlette passes Request. + return JSONResponse( + { + "ready": True, + "protocols": {"responses": "2.0.0"}, + "implementation": "azure-ai-agentserver-responses", + } + ) + + app.router.routes.insert(0, Route("/readiness", readiness, methods=["GET"])) + + @app.response_handler + async def response_handler(request, context, cancellation_signal): # noqa: ANN001 - SDK-defined handler types. + stream = ResponseEventStream(response_id=context.response_id, model=model, request=request) + yield stream.emit_created() + yield stream.emit_in_progress() + + if _request_tools(request): + yield stream.emit_failed( + code="tools_unsupported", + message="request-level tools are not allowed for hosted brokered conformance", + ) + return + + outputs = _function_call_outputs(request) + if outputs: + previous_response_id = getattr(request, "previous_response_id", None) + if not previous_response_id: + yield stream.emit_failed( + code="missing_previous_response_id", + message="function_call_output requires previous_response_id", + ) + return + pending_calls = pending.get(str(previous_response_id)) + if pending_calls is None: + yield stream.emit_failed( + code="unknown_previous_response_id", + message="unknown previous_response_id", + ) + return + if len(outputs) != 1: + yield stream.emit_failed( + code="multiple_tool_outputs_unsupported", + message="multiple function_call_output items are not supported", + ) + return + call_id = outputs[0].get("call_id") + if call_id not in pending_calls: + yield stream.emit_failed(code="unknown_call_id", message="unknown function_call_output call_id") + return + pending.pop(str(previous_response_id), None) + output = outputs[0].get("output", "") + try: + parsed = json.loads(output) if isinstance(output, str) else output + except json.JSONDecodeError: + parsed = output + for event in stream.output_item_message(f"conformance complete: {json.dumps(parsed, sort_keys=True)}"): + yield event + yield stream.emit_completed() + return + + pending[context.response_id] = {_CONFORMANCE_CALL_ID} + for event in stream.output_item_function_call( + name=_CONFORMANCE_TOOL_NAME, + call_id=_CONFORMANCE_CALL_ID, + arguments=_CONFORMANCE_ARGUMENTS, + ): + yield event + yield stream.emit_completed() + + return app + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="agentkit-foundry-conformance", + description="Run the tiny Azure Responses SDK conformance app for Foundry brokered A0 smokes.", + ) + parser.add_argument("--host", default=os.environ.get("AGENTKIT_BIND", "0.0.0.0"), help="host interface to bind") + parser.add_argument("--port", type=int, default=int(os.environ.get("AGENTKIT_PORT", os.environ.get("PORT", "8088"))), help="port to bind") + parser.add_argument("--model", default=os.environ.get("AGENTKIT_FOUNDRY_CONFORMANCE_MODEL", "agentkit-foundry-conformance"), help="model name to stamp in response envelopes") + parser.add_argument("--dry-run", action="store_true", help="validate arguments and print the selected bind/model without serving") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + app = create_foundry_conformance_app(model=args.model) + if args.dry_run: + print(json.dumps({"host": args.host, "port": args.port, "model": args.model, "protocols": {"responses": "2.0.0"}}, sort_keys=True)) + return 0 + app.run(host=args.host, port=args.port) + return 0 + + +__all__ = ["create_foundry_conformance_app", "main"] + + +if __name__ == "__main__": # pragma: no cover - exercised by console script/dry-run tests. + raise SystemExit(main()) diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py new file mode 100644 index 0000000..cc07384 --- /dev/null +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -0,0 +1,210 @@ +"""Lower-level OpenAI-compatible brokered tool loop for Foundry Responses mode. + +This is the Phase A4 fallback path: when high-level frameworks cannot suspend and +resume externally brokered tool calls, AgentKit can drive a minimal model loop +itself. The loop exposes only static safe brokered schemas to the model, converts +one model tool request into a hosted Responses function_call, and later resumes +with Orka's function_call_output to obtain the final assistant message. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Mapping, Sequence + +import httpx + +from .adapter_support import AgentBuildError, NO_AUTH_API_KEY, resolve_api_key +from .config import AgentSpec +from .conversation import FORWARDED_ROLES, RunRequest +from .runtime import AgentRunError, BrokeredToolDefinition + + +@dataclass(frozen=True) +class ModelLoopFinal: + text: str + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelLoopToolRequest: + name: str + arguments: dict[str, Any] + messages: list[dict[str, Any]] + usage: dict[str, int] = field(default_factory=dict) + + +class BrokeredChatModelLoop: + """Explicit one-tool brokered model loop over OpenAI Chat Completions.""" + + def __init__( + self, + spec: AgentSpec, + tools: Sequence[BrokeredToolDefinition], + *, + http_client: httpx.AsyncClient | None = None, + max_output_chars: int = 64_000, + ) -> None: + self.spec = spec + self.tools = list(tools) + self.http_client = http_client + self.max_output_chars = max_output_chars + self.tools_by_name = {tool.name: tool for tool in self.tools} + + async def start(self, request: RunRequest, *, call_id: str) -> ModelLoopFinal | ModelLoopToolRequest: + messages = self._initial_messages(request) + data = await self._chat(messages, tools=self._tool_payloads()) + message = _choice_message(data) + usage = _usage(data) + tool_calls = message.get("tool_calls") + if not tool_calls: + return ModelLoopFinal(text=_message_text(message), usage=usage) + if not isinstance(tool_calls, list) or len(tool_calls) != 1: + raise AgentRunError( + "model requested multiple brokered tools; deterministic brokered mode supports one call per turn", + status=400, + code="multiple_tool_calls_unsupported", + ) + call = tool_calls[0] + if not isinstance(call, Mapping) or call.get("type") != "function": + raise AgentRunError("model returned an unsupported tool call", status=400, code="unsupported_tool_call") + function = call.get("function") + if not isinstance(function, Mapping): + raise AgentRunError("model tool call is missing function payload", status=400, code="invalid_tool_call") + name = function.get("name") + if not isinstance(name, str) or name not in self.tools_by_name: + raise AgentRunError(f"model requested unknown brokered tool {name!r}", status=400, code="unknown_brokered_tool") + raw_arguments = function.get("arguments", "{}") + arguments = _parse_arguments(raw_arguments) + argument_text = json.dumps(arguments, separators=(",", ":"), sort_keys=True) + assistant_message = { + "role": "assistant", + "content": message.get("content"), + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": argument_text}, + } + ], + } + return ModelLoopToolRequest(name=name, arguments=arguments, messages=[*messages, assistant_message], usage=usage) + + async def resume(self, messages: Sequence[Mapping[str, Any]], *, call_id: str, output: str) -> ModelLoopFinal: + if len(output) > self.max_output_chars: + raise AgentRunError("brokered tool output is too large for model resume", status=413, code="brokered_output_too_large") + resumed = [dict(message) for message in messages] + resumed.append({"role": "tool", "tool_call_id": call_id, "content": output}) + data = await self._chat(resumed, tools=[]) + message = _choice_message(data) + if message.get("tool_calls"): + raise AgentRunError("model requested another brokered tool after resume", status=400, code="tool_loop_limit_exceeded") + return ModelLoopFinal(text=_message_text(message), usage=_usage(data)) + + def _initial_messages(self, request: RunRequest) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + if self.spec.instructions: + messages.append({"role": "system", "content": self.spec.instructions}) + for turn in request.history: + if turn.role in FORWARDED_ROLES and turn.text: + messages.append({"role": turn.role, "content": turn.text}) + messages.append({"role": "user", "content": request.prompt}) + return messages + + def _tool_payloads(self) -> list[dict[str, Any]]: + payloads: list[dict[str, Any]] = [] + for tool in self.tools: + description = f"Brokered class: {tool.brokered_class}. {tool.description}".strip() + payloads.append( + { + "type": "function", + "function": { + "name": tool.name, + "description": description, + "parameters": dict(tool.parameters), + }, + } + ) + return payloads + + async def _chat(self, messages: Sequence[Mapping[str, Any]], *, tools: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + payload: dict[str, Any] = {"model": self.spec.model.name, "messages": list(messages)} + if tools: + payload["tools"] = list(tools) + payload["tool_choice"] = "auto" + client = self.http_client + close_client = False + if client is None: + try: + api_key = resolve_api_key(self.spec) + except AgentBuildError as exc: + raise AgentRunError(str(exc), status=400, code="ModelAuthMissing") from exc + headers = {} + if api_key != NO_AUTH_API_KEY: + headers["Authorization"] = f"Bearer {api_key}" + client = httpx.AsyncClient(headers=headers, timeout=60) + close_client = True + try: + response = await client.post(_chat_completions_url(self.spec.model.base_url), json=payload) + response.raise_for_status() + data = response.json() + except httpx.HTTPStatusError as exc: + raise AgentRunError(str(exc), status=exc.response.status_code, code="ModelHTTPError") from exc + except Exception as exc: # noqa: BLE001 - normalize transport/model failures. + raise AgentRunError(str(exc), status=502, code=exc.__class__.__name__) from exc + finally: + if close_client: + await client.aclose() + if not isinstance(data, dict): + raise AgentRunError("model response must be a JSON object", status=502, code="InvalidModelResponse") + return data + + +def _chat_completions_url(base_url: str) -> str: + root = base_url.rstrip("/") + if root.endswith("/chat/completions"): + return root + return f"{root}/chat/completions" + + +def _choice_message(data: Mapping[str, Any]) -> Mapping[str, Any]: + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + raise AgentRunError("model response did not include choices", status=502, code="InvalidModelResponse") + choice = choices[0] + if not isinstance(choice, Mapping): + raise AgentRunError("model response choice must be an object", status=502, code="InvalidModelResponse") + message = choice.get("message") + if not isinstance(message, Mapping): + raise AgentRunError("model response choice did not include a message", status=502, code="InvalidModelResponse") + return message + + +def _message_text(message: Mapping[str, Any]) -> str: + content = message.get("content", "") + return content if isinstance(content, str) else str(content or "") + + +def _parse_arguments(raw: Any) -> dict[str, Any]: + if isinstance(raw, str): + try: + parsed = json.loads(raw or "{}") + except json.JSONDecodeError as exc: + raise AgentRunError("model tool arguments must be valid JSON", status=400, code="InvalidToolArguments") from exc + else: + parsed = raw + if not isinstance(parsed, dict): + raise AgentRunError("model tool arguments must be a JSON object", status=400, code="InvalidToolArguments") + return parsed + + +def _usage(data: Mapping[str, Any]) -> dict[str, int]: + usage = data.get("usage") if isinstance(data.get("usage"), Mapping) else {} + prompt_tokens = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) + completion_tokens = int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0) + total_tokens = int(usage.get("total_tokens", prompt_tokens + completion_tokens) or 0) + return {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + + +__all__ = ["BrokeredChatModelLoop", "ModelLoopFinal", "ModelLoopToolRequest"] diff --git a/runtimes/common/agentkit_serve_common/runtime.py b/runtimes/common/agentkit_serve_common/runtime.py index aa2fc7d..1d35edd 100644 --- a/runtimes/common/agentkit_serve_common/runtime.py +++ b/runtimes/common/agentkit_serve_common/runtime.py @@ -65,6 +65,34 @@ class RunResult: usage: dict[str, int] = field(default_factory=dict) +@dataclass(frozen=True) +class RuntimeMessageCompleted: + """A Responses-compatible runtime result containing final assistant text.""" + + text: str + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RuntimeToolCallRequested: + """A Responses-compatible runtime pause requesting brokered tool execution.""" + + tool_call_id: str + name: str + arguments: Mapping[str, Any] + brokered_class: Literal["read", "write", "coordination"] + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RuntimeFailed: + """A deterministic runtime failure result for hosted protocol adapters.""" + + message: str + status: int = 502 + code: str = "RuntimeFailed" + + @dataclass(frozen=True) class BrokeredToolDefinition: """Safe tool schema a brokered runtime may request through Orka. @@ -78,6 +106,7 @@ class BrokeredToolDefinition: description: str brokered_class: Literal["read", "write", "coordination"] parameters: Mapping[str, Any] = field(default_factory=dict) + schema_digest: str | None = None @dataclass(frozen=True) diff --git a/runtimes/common/pyproject.toml b/runtimes/common/pyproject.toml index f40d3a1..4d414df 100644 --- a/runtimes/common/pyproject.toml +++ b/runtimes/common/pyproject.toml @@ -20,8 +20,14 @@ dependencies = [ "pydantic>=2.7", "pyyaml>=6.0", "httpx>=0.28", + "azure-ai-agentserver-responses>=1.0.0b8", ] +[project.scripts] +agentkit-brokered-tools = "agentkit_serve_common.brokered:main" +agentkit-foundry-brokered = "agentkit_serve_common.foundry_brokered_cli:main" +agentkit-foundry-conformance = "agentkit_serve_common.foundry_conformance:main" + [project.optional-dependencies] # Starlette/FastAPI TestClient currently imports the httpx2 compatibility package. dev = ["pytest>=8.0", "httpx2>=0.1"] diff --git a/runtimes/common/tests/fixtures/foundry_brokered/approval_declined_payload.json b/runtimes/common/tests/fixtures/foundry_brokered/approval_declined_payload.json new file mode 100644 index 0000000..facb8ed --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/approval_declined_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "approval_declined", + "message": "Human declined dispatch-work-order" + } +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/continuation_request.json b/runtimes/common/tests/fixtures/foundry_brokered/continuation_request.json new file mode 100644 index 0000000..e5a8e91 --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/continuation_request.json @@ -0,0 +1,11 @@ +{ + "previous_response_id": "caresp_test", + "input": [ + { + "type": "function_call_output", + "call_id": "call_caresp_test_1", + "output": "{\"approved\":true,\"output\":{\"success\":true}}", + "status": "completed" + } + ] +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/final_message_response.json b/runtimes/common/tests/fixtures/foundry_brokered/final_message_response.json new file mode 100644 index 0000000..6d4786f --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/final_message_response.json @@ -0,0 +1,25 @@ +{ + "id": "caresp_final", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_final", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Brokered tool conformance_read completed with output: {\"success\":true}", + "annotations": [] + } + ], + "response_id": "caresp_final" + } + ], + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "previous_response_id": "caresp_test" +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/function_call_response.json b/runtimes/common/tests/fixtures/foundry_brokered/function_call_response.json new file mode 100644 index 0000000..adc0d9d --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/function_call_response.json @@ -0,0 +1,19 @@ +{ + "id": "caresp_test", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "fc_test", + "type": "function_call", + "call_id": "call_caresp_test_1", + "name": "conformance_read", + "arguments": "{\"probe\":true}", + "status": "completed", + "response_id": "caresp_test" + } + ], + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/initial_request.json b/runtimes/common/tests/fixtures/foundry_brokered/initial_request.json new file mode 100644 index 0000000..02c4727 --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/initial_request.json @@ -0,0 +1,3 @@ +{ + "input": "please read telemetry" +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/multiple_function_calls_unsupported_error.json b/runtimes/common/tests/fixtures/foundry_brokered/multiple_function_calls_unsupported_error.json new file mode 100644 index 0000000..2839d63 --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/multiple_function_calls_unsupported_error.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "multiple function_call_output items are not supported by this deterministic brokered adapter", + "code": "multiple_tool_outputs_unsupported" + } +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/tool_execution_failure_payload.json b/runtimes/common/tests/fixtures/foundry_brokered/tool_execution_failure_payload.json new file mode 100644 index 0000000..b0955fe --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/tool_execution_failure_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "tool_execution_failed", + "message": "downstream timed out" + } +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/tool_policy_rejection_payload.json b/runtimes/common/tests/fixtures/foundry_brokered/tool_policy_rejection_payload.json new file mode 100644 index 0000000..d32368f --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/tool_policy_rejection_payload.json @@ -0,0 +1,7 @@ +{ + "approved": false, + "error": { + "code": "tool_policy_rejected", + "message": "writes are disabled" + } +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/unknown_call_id_error.json b/runtimes/common/tests/fixtures/foundry_brokered/unknown_call_id_error.json new file mode 100644 index 0000000..00ed6d3 --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/unknown_call_id_error.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "unknown function_call_output call_id", + "code": "unknown_call_id" + } +} diff --git a/runtimes/common/tests/fixtures/foundry_brokered/unknown_previous_response_id_error.json b/runtimes/common/tests/fixtures/foundry_brokered/unknown_previous_response_id_error.json new file mode 100644 index 0000000..ea6f178 --- /dev/null +++ b/runtimes/common/tests/fixtures/foundry_brokered/unknown_previous_response_id_error.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "unknown previous_response_id", + "code": "unknown_previous_response_id" + } +} diff --git a/runtimes/common/tests/test_brokered_schema.py b/runtimes/common/tests/test_brokered_schema.py new file mode 100644 index 0000000..7072999 --- /dev/null +++ b/runtimes/common/tests/test_brokered_schema.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import yaml + +from agentkit_serve_common.brokered import ( + brokered_tool_definitions, + generate_brokered_tools_from_orka_tool_crds, + load_orka_tool_crd_files, + main, + render_brokered_tools_yaml, +) +from agentkit_serve_common.config import AgentSpec, brokered_tool_schema_digest +from agentkit_serve_common.runtime import BrokeredToolDefinition + + +def _tool_docs() -> list[dict]: + return [ + { + "apiVersion": "orka.example/v1", + "kind": "Tool", + "metadata": {"name": "dispatch-work-order"}, + "spec": { + "description": "Dispatch a field tech.", + "brokeredClass": "write", + "parameters": {"type": "object", "properties": {"incident": {"type": "string"}}, "required": ["incident"]}, + "url": "http://tool.default.svc.cluster.local", + "secretRef": {"name": "do-not-export"}, + }, + }, + { + "apiVersion": "orka.example/v1", + "kind": "Tool", + "metadata": {"name": "check-network-telemetry"}, + "spec": { + "description": "Read telemetry.", + "brokeredClass": "read", + "inputSchema": {"type": "object", "properties": {"site": {"type": "string"}}}, + "headers": {"Authorization": "do-not-export"}, + }, + }, + ] + + +def test_generate_brokered_tools_from_orka_tool_crds_is_deterministic_and_schema_only(): + generated = generate_brokered_tools_from_orka_tool_crds(_tool_docs()) + + assert [tool["name"] for tool in generated] == ["check-network-telemetry", "dispatch-work-order"] + assert all(set(tool) == {"name", "description", "brokeredClass", "parameters", "schemaDigest"} for tool in generated) + assert generated[0]["schemaDigest"] == brokered_tool_schema_digest( + name="check-network-telemetry", + description="Read telemetry.", + brokered_class="read", + parameters={"properties": {"site": {"type": "string"}}, "type": "object"}, + ) + + +def test_generate_brokered_tools_can_omit_digest(): + generated = generate_brokered_tools_from_orka_tool_crds(_tool_docs(), include_digest=False) + + assert all("schemaDigest" not in entry for entry in generated) + + +def test_render_brokered_tools_yaml_outputs_agent_yaml_fragment(): + generated = generate_brokered_tools_from_orka_tool_crds(_tool_docs(), include_digest=False) + + rendered = render_brokered_tools_yaml(generated) + + parsed = yaml.safe_load(rendered) + assert list(parsed) == ["brokeredTools"] + assert [entry["name"] for entry in parsed["brokeredTools"]] == ["check-network-telemetry", "dispatch-work-order"] + assert "url" not in rendered + assert "secretRef" not in rendered + assert "Authorization" not in rendered + + +def test_load_orka_tool_crd_files_rejects_duplicate_exported_names(tmp_path): + first = tmp_path / "first.yaml" + second = tmp_path / "second.yaml" + first.write_text(yaml.safe_dump(_tool_docs()[0]), encoding="utf-8") + second.write_text(yaml.safe_dump(_tool_docs()[0]), encoding="utf-8") + + try: + load_orka_tool_crd_files([first, second]) + except ValueError as exc: + assert "duplicate brokered tool names" in str(exc) + else: # pragma: no cover - assertion path. + raise AssertionError("expected duplicate brokered tool names to fail") + + +def test_brokered_tools_export_cli_writes_safe_fragment(tmp_path, capsys): + src = tmp_path / "tools.yaml" + out = tmp_path / "brokered-tools.yaml" + src.write_text("---\n" + yaml.safe_dump(_tool_docs()[0]) + "---\n" + yaml.safe_dump(_tool_docs()[1]), encoding="utf-8") + + code = main([str(src), "--no-digest", "--output", str(out)]) + + assert code == 0 + assert capsys.readouterr().out == "" + parsed = yaml.safe_load(out.read_text(encoding="utf-8")) + assert [entry["name"] for entry in parsed["brokeredTools"]] == ["check-network-telemetry", "dispatch-work-order"] + assert all("schemaDigest" not in entry for entry in parsed["brokeredTools"]) + + +def test_brokered_tools_export_cli_reports_validation_errors(tmp_path, capsys): + src = tmp_path / "bad.yaml" + src.write_text( + yaml.safe_dump( + { + "kind": "Tool", + "metadata": {"name": "bad"}, + "spec": { + "description": "bad", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"tokenValue": {"type": "string"}}}, + }, + } + ), + encoding="utf-8", + ) + + code = main([str(src)]) + + assert code == 2 + assert "agentkit-brokered-tools:" in capsys.readouterr().err + + +def test_brokered_tool_definitions_preserve_only_safe_runtime_fields(): + spec = AgentSpec.model_validate( + { + "abiVersion": "v0", + "metadata": {"name": "brokered-test"}, + "model": {"provider": "openai-compatible", "baseURL": "https://api.openai.com/v1", "name": "gpt-4o-mini"}, + "instructions": "Be helpful.", + "tools": [], + "brokeredTools": [ + { + "name": "check-network-telemetry", + "description": "Read telemetry.", + "brokeredClass": "read", + "parameters": {"type": "object"}, + } + ], + "expose": {"openai": True, "port": 8080}, + } + ) + + assert brokered_tool_definitions(spec) == [ + BrokeredToolDefinition( + name="check-network-telemetry", + description="Read telemetry.", + brokered_class="read", + parameters={"type": "object"}, + ) + ] diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index 3b94c9a..21e60fd 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -435,3 +435,512 @@ def test_load_rejects_unsupported_otel_observability(tmp_path): load(_write_spec(tmp_path, spec_dict)) assert "observability.otel.endpointEnv is not supported" in str(exc.value) + + +def test_load_accepts_static_brokered_tools_with_matching_digest(tmp_path): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = {"type": "object", "properties": {"site": {"type": "string"}}, "required": ["site"]} + digest = brokered_tool_schema_digest( + name="check-network-telemetry", + description="Read sanitized optical telemetry.", + brokered_class="read", + parameters=parameters, + ) + spec_dict = deepcopy(_BASE_SPEC) + spec_dict["tools"] = [] + spec_dict["brokeredTools"] = [ + { + "name": "check-network-telemetry", + "description": "Read sanitized optical telemetry.", + "brokeredClass": "read", + "parameters": parameters, + "schemaDigest": digest, + } + ] + + spec = load(_write_spec(tmp_path, spec_dict)) + + assert spec.brokered_tools[0].name == "check-network-telemetry" + assert spec.brokered_tools[0].brokered_class == "read" + assert spec.brokered_tools[0].schema_digest == digest + + +@pytest.mark.parametrize("description", ["contains sk-secret", "execution at https://tool.default", "Bearer token required"]) +def test_load_rejects_unsafe_brokered_tool_descriptions(tmp_path, description: str): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": description, + "brokeredClass": "read", + "parameters": {"type": "object"}, + } + ], + ), + ) + assert "brokeredTools.0.description" in msg + assert "secret-like" in msg or "URLs" in msg + + +@pytest.mark.parametrize("field", ["url", "headers", "secretRef", "auth", "token"]) +def test_load_rejects_unsafe_brokered_tool_fields(tmp_path, field: str): + def mutate(spec): + spec["tools"] = [] + spec["brokeredTools"] = [ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object"}, + field: "should-not-cross", + } + ] + + msg = _invalid_message(tmp_path, mutate) + assert "brokeredTools.0" in msg + assert "unsafe" in msg or "Extra inputs" in msg + + +@pytest.mark.parametrize("unsafe_name", ["token", "authHeader", "authorizationHeader", "httpHeaders", "accessKey", "clientSecretValue", "tokenValue", "apiSecretKey", "cookie", "subscriptionKey", "xFunctionsKey"]) +def test_load_rejects_unsafe_brokered_parameter_names(tmp_path, unsafe_name: str): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {unsafe_name: {"type": "string"}}}, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + assert "not safe" in msg + + +@pytest.mark.parametrize("value", ["see https://internal-tool", "Bearer abc", "authorization header", "Cookie: session=abc", "x-api-key: abc", "api_key=abc", "x_api_key: abc", "format password", "enter passphrase"]) +def test_load_rejects_unsafe_brokered_schema_string_values(tmp_path, value: str): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string", "description": value}}}, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + assert "secret-like" in msg or "URL" in msg + + +def test_load_rejects_private_key_brokered_parameter_names_and_strings(tmp_path): + name_msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"privateKey": {"type": "string"}}}, + } + ], + ), + ) + text_msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string", "default": "BEGIN PRIVATE KEY"}}}, + } + ], + ), + ) + assert "not safe" in name_msg + assert "secret-like" in text_msg or "URL" in text_msg + + +@pytest.mark.parametrize("field", ["authentication", "authConfig", "clientSecret", "dbPassword", "passphrase", "pwd", "apiKey", "api-key", "baseUrl", "callbackURL", "apiEndpoint", "sessionCookie", "cookies"]) +def test_load_rejects_common_credential_brokered_parameter_names(tmp_path, field: str): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {field: {"type": "string"}}}, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + assert "not safe" in msg + + +@pytest.mark.parametrize("field", ["clientSecret", "dbPassword", "apiKey", "api-key"]) +def test_load_rejects_common_credential_brokered_required_names(tmp_path, field: str): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "required": [field]}, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + assert "not safe" in msg + + +def test_load_rejects_secret_literals_inside_brokered_schema_strings(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string", "default": "sk-not-a-real-secret"}}}, + } + ], + ), + ) + assert "secret-like material" in msg + + +def test_load_rejects_unsafe_text_literals_inside_brokered_schema_strings(tmp_path): + for unsafe_value in ["Bearer abc", "Bearer: abc", "Bearer=abc", "https://internal.example", "http://tool.default.svc.cluster.local", "tool.default.svc.cluster.local", "example ghp_not_real", "AWS key AKIAEXAMPLE"]: + msg = _invalid_message( + tmp_path, + lambda spec, unsafe_value=unsafe_value: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string", "default": unsafe_value}}}, + } + ], + ), + ) + assert "URL or secret-like material" in msg + + +def test_load_rejects_brokered_enum_combined_with_constraints_for_deterministic_synthesis(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"n": {"type": "integer", "minimum": 2, "enum": [1, 2]}}}, + } + ], + ), + ) + assert "enum/const/default" in msg + + +def test_load_rejects_brokered_tool_names_over_model_function_limit(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "a" * 65, + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object"}, + } + ], + ), + ) + assert "[A-Za-z0-9_-]{1,64}" in msg + + +def test_load_rejects_unsupported_brokered_json_schema_composition_keywords(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "allOf": [{"required": ["site"]}]}, + } + ], + ), + ) + assert "allOf" in msg + + +def test_load_rejects_unsupported_brokered_schema_pattern(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string", "pattern": "["}}}, + } + ], + ), + ) + assert "pattern" in msg + + +@pytest.mark.parametrize("bad_parameters", [ + {"type": "object", "properties": None}, + {"type": "object", "properties": {"site": {"type": "string", "enum": None}}}, + {"type": "object", "properties": {"site": {"type": "string", "pattern": None}}}, +]) +def test_load_rejects_explicit_null_brokered_json_schema_keywords(tmp_path, bad_parameters: dict): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": bad_parameters, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + + +@pytest.mark.parametrize("bad_child", [{"type": 123}, {"type": "strnig"}, {"items": "bad"}, {"items": [{"type": "number"}]}, {"multipleOf": 2}, {"uniqueItems": True}, {"minLength": -1}]) +def test_load_rejects_malformed_nested_brokered_json_schema(tmp_path, bad_child: dict): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": bad_child}}, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + + +@pytest.mark.parametrize( + "schema", + [ + {"type": "object", "properties": {"site": {"type": None}}}, + {"type": "object", "properties": {"n": {"type": "integer", "default": "1"}}}, + {"type": "object", "properties": {"site": {"type": "string", "enum": [0, "ok"]}}}, + ], +) +def test_load_rejects_invalid_brokered_schema_type_values_and_defaults(tmp_path, schema: dict): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": schema, + } + ], + ), + ) + assert "brokeredTools.0.parameters" in msg + + +def test_load_rejects_unknown_brokered_class_and_malformed_schema(tmp_path): + unknown_class = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "admin", + "parameters": {"type": "object"}, + } + ], + ), + ) + bad_schema = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "array"}, + } + ], + ), + ) + + assert "brokeredTools.0.brokeredClass" in unknown_class + assert "brokeredTools.0.parameters" in bad_schema + assert "type: object" in bad_schema + + +def test_load_rejects_brokered_schema_digest_mismatch(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object"}, + "schemaDigest": "sha256:" + "0" * 64, + } + ], + ), + ) + assert "brokeredTools.0" in msg + assert "schemaDigest does not match" in msg + + +def test_load_rejects_owned_and_brokered_tool_name_overlap(tmp_path): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + brokeredTools=[ + { + "name": "fetch", + "description": "safe schema", + "brokeredClass": "read", + "parameters": {"type": "object"}, + } + ] + ), + ) + assert "tools and brokeredTools cannot be mixed" in msg + + +def test_brokered_tool_schema_digest_uses_utf8_canonical_json(): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = { + "type": "object", + "properties": {"site": {"type": "string", "description": "São & R&D "}}, + "required": ["site"], + } + + assert brokered_tool_schema_digest( + name="check-network-telemetry", + description="São & R&D ", + brokered_class="read", + parameters=parameters, + ) == "sha256:7066de4e62dd1a6550701772aad901e1efcf5eb81a4f639252f82e6c7be8d4c1" + + +def test_brokered_tool_schema_digest_normalizes_integer_valued_floats(): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = {"type": "object", "properties": {"retries": {"type": "number", "minimum": 1.0}}} + + assert brokered_tool_schema_digest( + name="numeric-tool", + description="Numeric constraints.", + brokered_class="read", + parameters=parameters, + ) == "sha256:7ad9d43791e157981bcd65fd8452c9e71a64064875cc1330ced42d4956bf7d75" + + +def test_brokered_tool_schema_digest_canonicalizes_numeric_constraints_cross_language(): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = { + "type": "object", + "properties": {"n": {"type": "number", "minimum": 1e-6}}, + "required": ["n"], + } + + assert brokered_tool_schema_digest( + name="num-tool", + description="Numeric schema", + brokered_class="read", + parameters=parameters, + ) == "sha256:83bf12180154a21f8ba19049687e24acee9ef430966af67a83721a10bf7eee50" + + +def test_brokered_tool_schema_digest_canonicalizes_positive_exponent_numbers(): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = {"type": "object", "properties": {"n": {"type": "number", "maximum": 1e20}}} + + assert brokered_tool_schema_digest( + name="large-num-tool", + description="Large numeric schema", + brokered_class="read", + parameters=parameters, + ) == "sha256:51ebe9cdbb967453ba3ae9fb737566028814968d8121ba0d32825f7c8ffb5639" + + +def test_brokered_tool_schema_digest_canonicalizes_exponent_number_spellings(): + from agentkit_serve_common.config import brokered_tool_schema_digest + + parameters = { + "type": "object", + "properties": { + "small": {"type": "number", "minimum": 1e-7}, + "large": {"type": "number", "maximum": 1e21}, + }, + } + + assert brokered_tool_schema_digest( + name="exponent-num-tool", + description="Exponent numeric schema", + brokered_class="read", + parameters=parameters, + ) == "sha256:bb4fac58c3f65a33ed3c4ebfa10e5da9c3dc5a9250a1316f64d25e40e0e645e0" diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py new file mode 100644 index 0000000..d1a1ec4 --- /dev/null +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -0,0 +1,1600 @@ +from __future__ import annotations + +import json +import time +from copy import deepcopy +from types import TracebackType +from typing import Any + +from fastapi.testclient import TestClient +import httpx + +from agentkit_serve_common.config import AgentSpec +from agentkit_serve_common.conversation import RunRequest +from agentkit_serve_common.foundry import create_foundry_app +from agentkit_serve_common.runtime import RunResult, RuntimeSession + + +CONTINUATION_PROOF = "test-orka-continuation-proof" +CONTINUATION_AUTH = {"x-agentkit-brokered-continuation-proof": CONTINUATION_PROOF} + + +def _app(spec: AgentSpec | None = None, factory: NoDirectRunFactory | None = None, **kwargs: Any): + return create_foundry_app( + spec or _spec(), + factory or NoDirectRunFactory(), + brokered_continuation_proof=CONTINUATION_PROOF, + **kwargs, + ) + + +def _spec(*, tool_name: str = "conformance_read", brokered_class: str = "read") -> AgentSpec: + return AgentSpec.model_validate( + { + "abiVersion": "v0", + "metadata": {"name": "foundry-brokered-test"}, + "model": { + "provider": "openai-compatible", + "baseURL": "https://api.openai.com/v1", + "name": "gpt-4o-mini", + }, + "instructions": "Be helpful.", + "tools": [], + "brokeredTools": [ + { + "name": tool_name, + "description": "Safe deterministic conformance tool.", + "brokeredClass": brokered_class, + "parameters": {"type": "object", "properties": {"probe": {"type": "boolean"}}}, + } + ], + "expose": {"openai": True, "port": 8080}, + } + ) + + +def _multi_tool_spec() -> AgentSpec: + data = _spec().model_dump(by_alias=True) + data["brokeredTools"] = [ + { + "name": "check-network-telemetry", + "description": "Read telemetry.", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string"}}, "required": ["site"]}, + }, + { + "name": "get-active-incidents", + "description": "Read active incidents.", + "brokeredClass": "read", + "parameters": {"type": "object"}, + }, + ] + return AgentSpec.model_validate(data) + + +class NoDirectRunRuntime: + def __init__(self) -> None: + self.run_requests: list[RunRequest] = [] + + async def __aenter__(self) -> RuntimeSession: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + return None + + async def run(self, request: RunRequest) -> RunResult: + self.run_requests.append(request) + raise AssertionError("Foundry brokered /responses must not execute direct AgentKit-owned tools") + + +class NoDirectRunFactory: + def __init__(self) -> None: + self.runtime = NoDirectRunRuntime() + + def build_runtime(self, spec: AgentSpec) -> RuntimeSession: + return self.runtime + + +def _start(client: TestClient, prompt: str = "please read telemetry") -> dict[str, Any]: + resp = client.post("/responses", json={"input": prompt}) + assert resp.status_code == 200, resp.text + return resp.json() + + +def _continuation(response_id: str, call_id: str, payload: dict[str, Any]) -> dict[str, Any]: + return { + "previous_response_id": response_id, + "input": [ + { + "type": "function_call_output", + "call_id": call_id, + "output": json.dumps(payload, separators=(",", ":"), sort_keys=True), + "status": "completed", + } + ], + } + + +def _call(body: dict[str, Any]) -> dict[str, Any]: + assert body["status"] == "completed" + assert body["id"].startswith("caresp_") + output = body["output"] + assert len(output) == 1 + assert output[0]["type"] == "function_call" + return output[0] + + +def _message_text(body: dict[str, Any]) -> str: + assert body["status"] == "completed" + message = body["output"][0] + assert message["type"] == "message" + return message["content"][0]["text"] + + +def test_foundry_brokered_requires_continuation_proof_for_readiness_and_initial_call(): + app = create_foundry_app(_spec(), NoDirectRunFactory()) + + with TestClient(app) as client: + readiness = client.get("/readiness") + initial = client.post("/responses", json={"input": "please read telemetry"}) + + assert readiness.status_code == 503 + assert readiness.json()["ready"] is False + assert readiness.json()["foundryResponses"]["continuationAuth"] == "missing" + assert initial.status_code == 503 + assert initial.json()["error"]["code"] == "brokered_continuation_auth_required" + + +def test_foundry_brokered_initial_response_emits_static_function_call_without_direct_execution(): + factory = NoDirectRunFactory() + app = _app(_spec(), factory) + + with TestClient(app) as client: + readiness = client.get("/readiness") + body = _start(client) + + assert readiness.status_code == 200 + assert readiness.json()["foundryResponses"] == { + "brokeredTools": 1, + "ownedToolsDisabled": 0, + "stateBackend": "memory", + "stateTtlSeconds": 900.0, + "stateMaxPending": 128, + "continuationAuth": "configured", + "runtime": "deterministic", + "scaling": "single-replica-or-sticky-routing-required", + } + call = _call(body) + assert call["name"] == "conformance_read" + assert call["call_id"] == f"call_{body['id']}_1" + assert json.loads(call["arguments"]) == {"probe": True} + assert factory.runtime.run_requests == [] + + +def test_foundry_brokered_deterministic_arguments_reject_unsafe_prompt_text(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string"}}, + "required": ["site"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "check-network-telemetry https://internal.example"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsafeBrokeredArguments" + + +def test_foundry_brokered_synthesizes_arguments_from_required_schema_fields(): + spec = _spec(tool_name="check-network-telemetry") + tool = spec.brokered_tools[0] + tool.parameters["required"] = ["site"] + app = _app(spec) + + with TestClient(app) as client: + body = _start(client, "please call check-network-telemetry") + + call = _call(body) + assert call["name"] == "check-network-telemetry" + assert json.loads(call["arguments"]) == {"site": "please call check-network-telemetry"} + + +def test_foundry_brokered_normal_followup_with_previous_response_id_is_not_treated_as_tool_output(): + app = _app() + + with TestClient(app) as client: + resp = client.post("/responses", json={"previous_response_id": "caresp_completed_elsewhere", "input": "next question"}) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["previous_response_id"] == "caresp_completed_elsewhere" + call = _call(body) + assert call["type"] == "function_call" + + +def test_foundry_brokered_rejects_normal_followup_while_previous_response_is_pending_tool_output(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + resp = client.post("/responses", json={"previous_response_id": initial["id"], "input": "next question"}) + + assert resp.status_code == 409 + assert resp.json()["error"]["code"] == "response_pending_function_call_output" + + +def test_foundry_brokered_pending_state_store_is_bounded(): + app = _app(max_pending_responses=1) + + with TestClient(app) as client: + first = _start(client) + second = client.post("/responses", json={"input": "another pending request"}) + + assert _call(first) + assert second.status_code == 429 + assert second.json()["error"]["code"] == "brokered_response_state_full" + + +def test_foundry_brokered_completed_state_is_evicted_before_rejecting_new_pending_state(): + app = _app(max_pending_responses=1) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + completed = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}), + ) + next_initial = client.post("/responses", json={"input": "another pending request"}) + + assert completed.status_code == 200, completed.text + assert next_initial.status_code == 200, next_initial.text + assert _call(next_initial.json()) + + +def test_foundry_brokered_arguments_honor_typeless_const_and_enum_values(): + spec = _spec(tool_name="typeless-constraints") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "probe": {"const": True}, + "mode": {"enum": ["safe"]}, + }, + "required": ["probe", "mode"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "typeless-constraints"}) + + assert body.status_code == 200, body.text + assert json.loads(_call(body.json())["arguments"]) == {"probe": True, "mode": "safe"} + + +def test_foundry_brokered_min_properties_runs_dependent_required_closure(): + spec = _spec(tool_name="minprops-dependent") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "a": {"type": "string", "const": "a"}, + "b": {"type": "string", "const": "b"}, + }, + "minProperties": 1, + "dependentRequired": {"a": ["b"]}, + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "minprops-dependent"}) + + assert body.status_code == 200, body.text + assert json.loads(_call(body.json())["arguments"]) == {"a": "a", "b": "b"} + + +def test_foundry_brokered_arguments_honor_dependent_required_and_null_type(): + spec = _spec(tool_name="dependent-null") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "region": {"type": "string", "const": "west"}, + "site": {"type": "string", "const": "sfo"}, + "marker": {"type": "null"}, + }, + "required": ["region", "marker"], + "dependentRequired": {"region": ["site"]}, + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "dependent-null"}) + + assert body.status_code == 200, body.text + assert json.loads(_call(body.json())["arguments"]) == {"region": "west", "site": "sfo", "marker": None} + + +def test_foundry_brokered_integer_arguments_honor_float_bounds(): + spec = _spec(tool_name="bounded-integer") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1.0}, + "after": {"type": "integer", "exclusiveMinimum": 0.5}, + "below": {"type": "integer", "exclusiveMaximum": 4.5}, + }, + "required": ["count", "after", "below"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "bounded-integer"}) + + assert body.status_code == 200, body.text + assert json.loads(_call(body.json())["arguments"]) == {"count": 1, "after": 1, "below": 0} + + +def test_foundry_brokered_synthesizes_arguments_that_honor_basic_constraints(): + spec = _spec(tool_name="bounded-lookup") + tool = spec.brokered_tools[0] + tool.parameters = { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "mode": {"type": "string", "enum": ["safe"]}, + "label": {"type": "string", "minLength": 5, "maxLength": 7}, + }, + "required": ["count", "mode", "label"], + } + app = _app(spec) + + with TestClient(app) as client: + body = _start(client, "please call bounded-lookup") + + assert json.loads(_call(body)["arguments"]) == {"count": 1, "mode": "safe", "label": "label"} + + +def test_foundry_brokered_rejects_schema_constraints_it_cannot_synthesize(): + spec = _spec(tool_name="pattern-lookup") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string", "pattern": "^[A-Z]+$"}}, + "required": ["site"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "pattern-lookup sfo"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_numeric_argument_synthesis_respects_upper_bounds(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "count": {"type": "integer", "maximum": -1}, + "ratio": {"type": "number", "exclusiveMaximum": 0}, + }, + "required": ["count", "ratio"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + + assert json.loads(_call(body)["arguments"]) == {"count": -1, "ratio": -1} + + +def test_foundry_brokered_optional_prompt_argument_is_synthesized_through_schema(): + spec = _spec(tool_name="prompt-tool") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"prompt": {"type": "string", "const": "fixed"}}, + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "please call prompt-tool with arbitrary text"}).json() + + assert json.loads(_call(body)["arguments"]) == {"prompt": "fixed"} + + +def test_foundry_brokered_deterministic_arguments_honor_root_const_object_schema(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = {"type": "object", "const": {"site": "iad"}} + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + + assert json.loads(_call(body)["arguments"]) == {"site": "iad"} + + +def test_foundry_brokered_validates_deterministic_arguments_before_emitting_call(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = {"type": "object", "required": ["site"], "additionalProperties": False} + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_argument_synthesis_honors_dependent_required_and_min_properties(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "site": {"type": "string"}, + "region": {"type": "string", "default": "west"}, + "extra": {"type": "boolean"}, + }, + "required": ["site"], + "dependentRequired": {"site": ["region"]}, + "minProperties": 3, + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + + assert json.loads(_call(body)["arguments"]) == {"site": "check-network-telemetry", "region": "west", "extra": True} + + +def test_foundry_brokered_integer_synthesis_honors_multiple_of(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"n": {"type": "integer", "minimum": 1, "multipleOf": 2}}, + "required": ["n"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + + assert json.loads(_call(body)["arguments"]) == {"n": 2} + + +def test_foundry_brokered_number_synthesis_uses_midpoint_for_fractional_exclusive_range(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"ratio": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}}, + "required": ["ratio"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + + assert json.loads(_call(body)["arguments"]) == {"ratio": 0.5} + + +def test_foundry_brokered_rejects_unbounded_schema_synthesis_before_allocating(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string", "minLength": 1_000_000_000}}, + "required": ["site"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert resp.status_code == 413 + assert resp.json()["error"]["code"] == "brokered_arguments_too_large" + + +def test_foundry_brokered_tool_selection_requires_token_boundary_match(): + data = _multi_tool_spec().model_dump(by_alias=True) + data["brokeredTools"] = [ + {"name": "read", "description": "read", "brokeredClass": "read", "parameters": {"type": "object"}}, + {"name": "read_telemetry", "description": "read telemetry", "brokeredClass": "read", "parameters": {"type": "object"}}, + ] + app = _app(AgentSpec.model_validate(data)) + + with TestClient(app) as client: + telemetry = client.post("/responses", json={"input": "please call read_telemetry"}) + unrelated = client.post("/responses", json={"input": "already done"}) + + assert telemetry.status_code == 200, telemetry.text + assert _call(telemetry.json())["name"] == "read_telemetry" + assert unrelated.status_code == 400 + assert unrelated.json()["error"]["code"] == "brokered_tool_selection_required" + + +def test_foundry_brokered_rejects_schema_that_would_generate_huge_arguments_before_allocating(): + spec = _spec(tool_name="huge-args") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "items": {"type": "array", "minItems": 1000000000, "items": {"type": "string"}}, + "label": {"type": "string", "minLength": 1000000000}, + }, + "required": ["items", "label"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "huge-args"}) + + assert resp.status_code == 413 + assert resp.json()["error"]["code"] == "brokered_arguments_too_large" + + +def test_foundry_brokered_rejects_arguments_that_exceed_state_budget(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters["required"] = ["site"] + app = _app(spec, max_brokered_argument_bytes=8) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "check-network-telemetry: a prompt that is too large for the site argument"}) + + assert resp.status_code == 413 + assert resp.json()["error"]["code"] == "brokered_arguments_too_large" + + +def test_foundry_brokered_single_read_tool_requires_explicit_name_except_conformance(): + app = _app(_spec(tool_name="check-network-telemetry")) + + with TestClient(app) as client: + unrelated = client.post("/responses", json={"input": "my password should stay in chat"}) + explicit = client.post("/responses", json={"input": "please call check-network-telemetry"}) + + assert unrelated.status_code == 400 + assert unrelated.json()["error"]["code"] == "brokered_tool_selection_required" + assert explicit.status_code == 200, explicit.text + assert _call(explicit.json())["name"] == "check-network-telemetry" + + +def test_foundry_brokered_single_write_tool_requires_explicit_tool_name(): + app = _app(_spec(tool_name="dispatch-work-order", brokered_class="write")) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "hello"}) + explicit = client.post("/responses", json={"input": "please call dispatch-work-order"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "brokered_tool_selection_required" + assert explicit.status_code == 200, explicit.text + assert _call(explicit.json())["name"] == "dispatch-work-order" + + +def test_foundry_brokered_selects_named_tool_when_multiple_schemas_are_configured(): + app = _app(_multi_tool_spec()) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "please call get-active-incidents"}) + + assert resp.status_code == 200, resp.text + assert _call(resp.json())["name"] == "get-active-incidents" + + +def test_foundry_brokered_rejects_ambiguous_multi_tool_prompt(): + app = _app(_multi_tool_spec()) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "please inspect the network"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "brokered_tool_selection_required" + + +def test_foundry_brokered_rejects_nonfinite_function_call_output_values(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": '{"approved":true,"output":{"value":NaN}}', + } + ], + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_function_call_output" + + +def test_foundry_brokered_continuation_accepts_matching_tool_output_and_completes(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + cont = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}), + ) + + assert cont.status_code == 200, cont.text + final = cont.json() + assert final["previous_response_id"] == initial["id"] + assert final["id"].startswith("caresp_") + assert final["id"] != initial["id"] + assert _message_text(final) == 'Brokered tool conformance_read completed with output: {"success":true}' + + +def test_foundry_brokered_rejects_function_call_output_without_orka_continuation_auth(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + resp = client.post( + "/responses", + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}), + ) + + assert resp.status_code == 403 + assert resp.json()["error"]["code"] == "brokered_continuation_forbidden" + + +def test_foundry_brokered_rejects_orphan_function_call_output_without_previous_response_id(): + app = _app() + + with TestClient(app) as client: + resp = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "input": [ + { + "type": "function_call_output", + "call_id": "call_missing", + "output": '{"approved":true,"output":{}}', + } + ] + }, + ) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "missing_previous_response_id" + + +def test_foundry_brokered_rejects_unknown_previous_response_id(): + app = _app() + + with TestClient(app) as client: + resp = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation("caresp_unknown", "call_unknown", {"approved": True, "output": {}}), + ) + + assert resp.status_code == 404 + assert resp.json()["error"]["code"] == "unknown_previous_response_id" + + +def test_foundry_brokered_rejects_unknown_call_id(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + resp = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], "call_other", {"approved": True, "output": {}}), + ) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "unknown_call_id" + + +def test_foundry_brokered_duplicate_continuation_is_idempotent_but_conflicts_are_rejected(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + payload = _continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}) + first = client.post("/responses", json=payload, headers=CONTINUATION_AUTH) + duplicate = client.post("/responses", json=payload, headers=CONTINUATION_AUTH) + conflicting_payload = deepcopy(payload) + conflicting_payload["input"][0]["output"] = '{"approved":true,"output":{"success":false}}' + conflict = client.post("/responses", json=conflicting_payload, headers=CONTINUATION_AUTH) + + assert first.status_code == 200 + assert duplicate.status_code == 200 + assert duplicate.json() == first.json() + assert conflict.status_code == 409 + assert conflict.json()["error"]["code"] == "conflicting_duplicate_continuation" + + +def test_foundry_brokered_file_state_survives_restart_for_deterministic_continuation(tmp_path): + state_file = tmp_path / "foundry-state.json" + + with TestClient(_app(response_state_file=state_file)) as client: + initial = _start(client) + call = _call(initial) + readiness = client.get("/readiness") + + assert readiness.json()["foundryResponses"]["stateBackend"] == "file" + assert state_file.exists() + + with TestClient(_app(response_state_file=state_file)) as client: + final = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}), + ) + + assert final.status_code == 200, final.text + assert _message_text(final.json()) == 'Brokered tool conformance_read completed with output: {"success":true}' + + with TestClient(_app(response_state_file=state_file)) as client: + duplicate = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"success": True}}), + ) + + assert duplicate.status_code == 200 + assert duplicate.json() == final.json() + + +def test_foundry_brokered_file_state_survives_restart_for_model_loop_continuation(tmp_path): + state_file = tmp_path / "foundry-model-state.json" + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters["required"] = ["site"] + first_fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":"sfo"}'}, + } + ], + } + ) + ] + ) + + with TestClient(_model_loop_app(spec, first_fake, response_state_file=state_file)) as client: + initial = client.post("/responses", json={"input": "call check-network-telemetry"}).json() + call = _call(initial) + + second_fake = _FakeChatTransport([_chat_response({"role": "assistant", "content": "Restart resume worked."})]) + with TestClient(_model_loop_app(spec, second_fake, response_state_file=state_file)) as client: + final = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"status": "ok"}}), + ) + + assert final.status_code == 200, final.text + assert _message_text(final.json()) == "Restart resume worked." + assert second_fake.requests[0]["messages"][-1]["tool_call_id"] == call["call_id"] + + +def test_foundry_brokered_rejects_expired_response_state(): + app = _app(state_ttl_seconds=0) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + time.sleep(0.01) + resp = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {}}), + ) + + assert resp.status_code == 410 + assert resp.json()["error"]["code"] == "response_state_expired" + + +def test_foundry_brokered_refuses_to_synthesize_nonliteral_write_arguments(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"incident": {"type": "string"}}, + "required": ["incident"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_refuses_multi_value_enum_write_arguments(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"operation": {"type": "string", "enum": ["delete", "create"]}}, + "required": ["operation"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_allows_single_value_enum_write_arguments(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"operation": {"type": "string", "enum": ["create"]}}, + "required": ["operation"], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert resp.status_code == 200 + assert json.loads(_call(resp.json())["arguments"]) == {"operation": "create"} + + +def test_foundry_brokered_allows_literal_write_arguments_for_conformance(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"incident": {"type": "string", "const": "INC-1"}}, + "required": ["incident"], + } + app = _app(spec) + + with TestClient(app) as client: + body = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert body.status_code == 200, body.text + assert json.loads(_call(body.json())["arguments"]) == {"incident": "INC-1"} + + +def test_foundry_brokered_decline_policy_rejection_and_execution_error_are_truthful_final_answers(): + cases = [ + ( + {"approved": False, "error": {"code": "approval_declined", "message": "Human declined dispatch-work-order"}}, + "approval_declined: Human declined dispatch-work-order", + ), + ( + {"approved": False, "error": {"code": "tool_policy_rejected", "message": "writes are disabled"}}, + "tool_policy_rejected: writes are disabled", + ), + ( + {"approved": False, "error": {"code": "tool_execution_failed", "message": "downstream timed out"}}, + "tool_execution_failed: downstream timed out", + ), + ] + + for payload, expected in cases: + app = _app(_spec(tool_name="dispatch-work-order", brokered_class="write")) + with TestClient(app) as client: + initial = _start(client, "please call dispatch-work-order") + call = _call(initial) + resp = client.post("/responses", json=_continuation(initial["id"], call["call_id"], payload), headers=CONTINUATION_AUTH) + assert resp.status_code == 200, resp.text + assert _message_text(resp.json()) == f"Brokered tool dispatch-work-order was not performed: {expected}" + + +def test_foundry_brokered_rejects_multiple_tool_outputs_deterministically(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + request = _continuation(initial["id"], call["call_id"], {"approved": True, "output": {}}) + request["input"].append(dict(request["input"][0])) + resp = client.post("/responses", json=request, headers=CONTINUATION_AUTH) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "multiple_tool_outputs_unsupported" + + + +def test_foundry_brokered_disables_invocations_direct_runtime_bypass(): + app = _app() + + with TestClient(app) as client: + resp = client.post("/invocations", json={"message": "bypass"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "invocations_disabled_in_brokered_mode" + +class _FakeChatTransport: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = list(responses) + self.requests: list[dict[str, Any]] = [] + + def handler(self, request: httpx.Request) -> httpx.Response: + self.requests.append(json.loads(request.content.decode("utf-8"))) + if not self.responses: + return httpx.Response(500, json={"error": "unexpected extra model call"}) + return httpx.Response(200, json=self.responses.pop(0)) + + +def _chat_response(message: dict[str, Any], *, prompt_tokens: int = 1, completion_tokens: int = 1) -> dict[str, Any]: + return { + "choices": [{"message": message}], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + } + + +def _model_loop_app(spec: AgentSpec, fake: _FakeChatTransport, **kwargs: Any): + client = httpx.AsyncClient(transport=httpx.MockTransport(fake.handler)) + return _app(spec, brokered_model_loop_enabled=True, brokered_model_http_client=client, **kwargs) + + +def test_foundry_brokered_model_loop_emits_model_requested_tool_and_resumes_to_final_answer(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters["required"] = ["site"] + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":"sfo"}'}, + } + ], + }, + prompt_tokens=2, + completion_tokens=3, + ), + _chat_response({"role": "assistant", "content": "Telemetry is healthy."}, prompt_tokens=5, completion_tokens=7), + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "Check SFO telemetry"}) + call = _call(initial.json()) + final = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial.json()["id"], call["call_id"], {"approved": True, "output": {"status": "healthy"}}), + ) + + assert initial.status_code == 200, initial.text + assert call["name"] == "check-network-telemetry" + assert call["call_id"] == f"call_{initial.json()['id']}_1" + assert json.loads(call["arguments"]) == {"site": "sfo"} + assert initial.json()["usage"] == {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5} + assert final.status_code == 200, final.text + assert _message_text(final.json()) == "Telemetry is healthy." + assert final.json()["usage"] == {"input_tokens": 7, "output_tokens": 10, "total_tokens": 17} + assert fake.requests[0]["tools"][0]["function"]["name"] == "check-network-telemetry" + assert fake.requests[0]["tools"][0]["function"]["description"].startswith("Brokered class: read.") + assert fake.requests[1]["messages"][-1]["role"] == "tool" + assert fake.requests[1]["messages"][-1]["tool_call_id"] == call["call_id"] + assert "tools" not in fake.requests[1] + + +def test_foundry_brokered_invalid_file_state_starts_with_empty_store(tmp_path): + state_file = tmp_path / "responses-state.json" + state_file.write_text("{not valid json", encoding="utf-8") + app = _app(response_state_file=state_file) + + with TestClient(app) as client: + response = client.get("/readiness") + initial = client.post("/responses", json={"input": "conformance_read"}) + + assert response.status_code == 200 + assert initial.status_code == 200, initial.text + assert _call(initial.json())["name"] == "conformance_read" + + +def test_foundry_brokered_file_state_is_written_with_private_permissions(tmp_path): + state_file = tmp_path / "responses-state.json" + app = _app(response_state_file=state_file) + + with TestClient(app) as client: + _start(client) + + assert state_file.exists() + assert state_file.stat().st_mode & 0o777 == 0o600 + + +def test_foundry_brokered_file_state_recovers_unfinalized_accepted_continuation_after_restart(tmp_path): + state_file = tmp_path / "responses-state.json" + spec = _spec(tool_name="check-network-telemetry") + fake_first = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{}'}, + } + ], + } + ) + ] + ) + first_app = _app(spec, brokered_model_loop_enabled=True, brokered_model_http_client=httpx.AsyncClient(transport=httpx.MockTransport(fake_first.handler)), response_state_file=state_file) + + with TestClient(first_app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + + payload = _continuation(initial.json()["id"], call["call_id"], {"approved": True, "output": {"ok": True}}) + state_data = json.loads(state_file.read_text(encoding="utf-8")) + state = state_data["states"][initial.json()["id"]] + state["acceptedOutputs"] = {call["call_id"]: payload["input"][0]["output"]} + state["status"] = "resuming" + state["finalPayload"] = None + state_file.write_text(json.dumps(state_data, separators=(",", ":"), sort_keys=True), encoding="utf-8") + + fake_second = _FakeChatTransport([_chat_response({"role": "assistant", "content": "Recovered."})]) + second_app = _app(spec, brokered_model_loop_enabled=True, brokered_model_http_client=httpx.AsyncClient(transport=httpx.MockTransport(fake_second.handler)), response_state_file=state_file) + with TestClient(second_app) as client: + retried = client.post("/responses", headers=CONTINUATION_AUTH, json=payload) + + assert retried.status_code == 200, retried.text + assert _message_text(retried.json()) == "Recovered." + + +def test_foundry_brokered_model_loop_accepts_integer_arguments_encoded_as_integral_float(): + spec = _spec(tool_name="retry-tool") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"retries": {"type": "integer"}}, + "required": ["retries"], + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "retry-tool", "arguments": '{"retries":1.0}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call retry-tool"}) + + assert response.status_code == 200, response.text + assert json.loads(_call(response.json())["arguments"]) == {"retries": 1.0} + + +def test_foundry_brokered_model_loop_unexpected_resume_failure_can_be_retried(): + spec = _spec(tool_name="check-network-telemetry") + + class FlakyResumeTransport: + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self.resume_attempts = 0 + + def handler(self, request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content.decode("utf-8")) + self.requests.append(payload) + if len(self.requests) == 1: + return httpx.Response( + 200, + json=_chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{}'}, + } + ], + } + ), + ) + self.resume_attempts += 1 + if self.resume_attempts == 1: + raise RuntimeError("transient resume failure") + return httpx.Response(200, json=_chat_response({"role": "assistant", "content": "Recovered after retry."})) + + fake = FlakyResumeTransport() + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + payload = _continuation(initial.json()["id"], call["call_id"], {"approved": True, "output": {"ok": True}}) + failed = client.post("/responses", headers=CONTINUATION_AUTH, json=payload) + retried = client.post("/responses", headers=CONTINUATION_AUTH, json=payload) + + assert failed.status_code == 502 + assert failed.json()["error"] == {"message": "model resume failed", "code": "ModelResumeError"} + assert retried.status_code == 200, retried.text + assert _message_text(retried.json()) == "Recovered after retry." + + +def test_foundry_brokered_model_loop_failed_resume_can_be_retried(): + spec = _spec(tool_name="check-network-telemetry") + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{}'}, + } + ], + } + ), + {"choices": []}, + _chat_response({"role": "assistant", "content": "Retry worked."}), + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + payload = _continuation(initial.json()["id"], call["call_id"], {"approved": True, "output": {"ok": True}}) + failed = client.post("/responses", headers=CONTINUATION_AUTH, json=payload) + retried = client.post("/responses", headers=CONTINUATION_AUTH, json=payload) + + assert failed.status_code == 502 + assert retried.status_code == 200, retried.text + assert _message_text(retried.json()) == "Retry worked." + + +def test_foundry_brokered_model_loop_can_return_final_message_without_tool_call(): + fake = _FakeChatTransport([_chat_response({"role": "assistant", "content": "No tool needed."})]) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "Say hello"}) + + assert response.status_code == 200, response.text + assert _message_text(response.json()) == "No tool needed." + assert fake.requests[0]["tool_choice"] == "auto" + + +def test_foundry_brokered_model_loop_rejects_unsupported_pattern_deterministically(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string", "pattern": "\\p{L}+"}}, + "required": ["site"], + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":"sfo"}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_model_loop_validates_additional_properties_schema(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string"}}, + "additionalProperties": {"type": "string"}, + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":"sfo","count":1}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_unknown_model_tool_request(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "unknown", "arguments": "{}"}, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call unknown"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "unknown_brokered_tool" + + +def test_foundry_brokered_model_loop_validates_schema_valued_additional_properties(): + spec = _spec(tool_name="flex-tool") + spec.brokered_tools[0].parameters = { + "type": "object", + "additionalProperties": {"type": "integer"}, + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "flex-tool", "arguments": '{"safe":"not-int"}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call flex-tool"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_nonfinite_model_arguments(): + spec = _spec(tool_name="check-network-telemetry") + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"value":NaN}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_uses_type_strict_const_and_enum_matching(): + for property_schema, arguments in [ + ({"const": 1}, '{"value":true}'), + ({"enum": [0]}, '{"value":false}'), + ]: + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"value": property_schema}, + "required": ["value"], + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": arguments}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_arguments_that_violate_declared_schema(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"site": {"type": "string"}}, + "required": ["site"], + "additionalProperties": False, + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":123,"extra":"nope"}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_validates_large_integer_bounds_exactly(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"count": {"type": "integer", "maximum": 9007199254740992}}, + "required": ["count"], + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"count":9007199254740993}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_unsafe_model_generated_arguments(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"site":"sfo","tokenValue":"ghp_not_real"}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsafeBrokeredArguments" + + +def test_foundry_brokered_rejects_tool_choice_none_instead_of_ignoring_it(): + app = _app() + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "hi", "tool_choice": "none"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "tool_choice_unsupported" + + +def test_foundry_brokered_rejects_request_supplied_tools_even_when_static_tools_exist(): + app = _app() + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "hi", "tools": [{"type": "function"}]}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "tools_unsupported" + + +def _fixture(name: str) -> dict[str, Any]: + path = __import__("pathlib").Path(__file__).parent / "fixtures" / "foundry_brokered" / name + return json.loads(path.read_text(encoding="utf-8")) + + +def _normalize_initial_response(body: dict[str, Any]) -> dict[str, Any]: + normalized = deepcopy(body) + actual_response_id = normalized["id"] + normalized["id"] = "caresp_test" + normalized["created_at"] = 0 + item = normalized["output"][0] + item["id"] = "fc_test" + item["call_id"] = item["call_id"].replace(actual_response_id, "caresp_test") + item["response_id"] = "caresp_test" + return normalized + + +def _normalize_final_response(body: dict[str, Any], *, previous_response_id: str) -> dict[str, Any]: + normalized = deepcopy(body) + actual_response_id = normalized["id"] + normalized["id"] = "caresp_final" + normalized["created_at"] = 0 + normalized["previous_response_id"] = "caresp_test" + item = normalized["output"][0] + item["id"] = "msg_final" + item["response_id"] = "caresp_final" + assert previous_response_id + assert actual_response_id + return normalized + + +def _materialize_continuation(fixture: dict[str, Any], *, response_id: str, call_id: str) -> dict[str, Any]: + materialized = deepcopy(fixture) + materialized["previous_response_id"] = response_id + materialized["input"][0]["call_id"] = call_id + return materialized + + +def test_foundry_brokered_golden_fixtures_pin_function_call_loop_and_errors(): + app = _app() + + with TestClient(app) as client: + initial = client.post("/responses", json=_fixture("initial_request.json")) + assert initial.status_code == 200, initial.text + initial_body = initial.json() + call = _call(initial_body) + assert _normalize_initial_response(initial_body) == _fixture("function_call_response.json") + + continuation = _materialize_continuation( + _fixture("continuation_request.json"), + response_id=initial_body["id"], + call_id=call["call_id"], + ) + final = client.post("/responses", json=continuation, headers=CONTINUATION_AUTH) + assert final.status_code == 200, final.text + assert _normalize_final_response(final.json(), previous_response_id=initial_body["id"]) == _fixture("final_message_response.json") + + unknown_prev = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_materialize_continuation( + _fixture("continuation_request.json"), + response_id="caresp_unknown", + call_id=call["call_id"], + ), + ) + assert unknown_prev.status_code == 404 + assert unknown_prev.json() == _fixture("unknown_previous_response_id_error.json") + + app_for_errors = _app() + with TestClient(app_for_errors) as client: + initial_body = _start(client) + call = _call(initial_body) + unknown_call = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_materialize_continuation( + _fixture("continuation_request.json"), + response_id=initial_body["id"], + call_id="call_unknown", + ), + ) + assert unknown_call.status_code == 400 + assert unknown_call.json() == _fixture("unknown_call_id_error.json") + + multiple = _materialize_continuation( + _fixture("continuation_request.json"), + response_id=initial_body["id"], + call_id=call["call_id"], + ) + multiple["input"].append(dict(multiple["input"][0])) + multiple_resp = client.post("/responses", json=multiple, headers=CONTINUATION_AUTH) + assert multiple_resp.status_code == 400 + assert multiple_resp.json() == _fixture("multiple_function_calls_unsupported_error.json") diff --git a/runtimes/common/tests/test_foundry_conformance.py b/runtimes/common/tests/test_foundry_conformance.py new file mode 100644 index 0000000..485ea10 --- /dev/null +++ b/runtimes/common/tests/test_foundry_conformance.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient +from azure.ai.agentserver.responses._id_generator import IdGenerator + +from agentkit_serve_common.foundry_conformance import create_foundry_conformance_app + + +def _function_output(previous_response_id: str | None, call_id: str = "call_conformance_1") -> dict: + payload = { + "input": [ + { + "type": "function_call_output", + "call_id": call_id, + "output": '{"approved":true,"output":{"success":true}}', + "status": "completed", + } + ] + } + if previous_response_id is not None: + payload["previous_response_id"] = previous_response_id + return payload + + +def test_foundry_conformance_sdk_function_call_loop_uses_platform_ids(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + readiness = client.get("/readiness") + initial = client.post("/responses", json={"input": "hello"}) + initial_body = initial.json() + call = initial_body["output"][0] + final = client.post("/responses", json=_function_output(initial_body["id"])) + + assert readiness.status_code == 200 + assert readiness.json()["protocols"] == {"responses": "2.0.0"} + assert initial.status_code == 200, initial.text + assert initial_body["id"].startswith("caresp_") + assert not initial_body["id"].startswith("resp_") + assert call == { + "type": "function_call", + "id": call["id"], + "call_id": "call_conformance_1", + "name": "conformance_read", + "arguments": '{"probe":true}', + "status": "completed", + "response_id": initial_body["id"], + "agent_reference": None, + } + assert final.status_code == 200, final.text + final_body = final.json() + assert final_body["previous_response_id"] == initial_body["id"] + assert final_body["id"].startswith("caresp_") + assert final_body["output"][0]["type"] == "message" + assert "success" in final_body["output"][0]["content"][0]["text"] + + +def test_foundry_conformance_sdk_rejects_orphan_and_unknown_continuations(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + missing_previous = client.post("/responses", json=_function_output(None)) + unknown_previous = client.post("/responses", json=_function_output(IdGenerator.new_response_id())) + initial = client.post("/responses", json={"input": "hello"}).json() + unknown_call = client.post("/responses", json=_function_output(initial["id"], call_id="call_other")) + + assert missing_previous.status_code == 200 + assert missing_previous.json()["status"] == "failed" + assert missing_previous.json()["error"]["code"] == "missing_previous_response_id" + assert unknown_previous.status_code == 200 + assert unknown_previous.json()["status"] == "failed" + assert unknown_previous.json()["error"]["code"] == "unknown_previous_response_id" + assert unknown_call.status_code == 200 + assert unknown_call.json()["status"] == "failed" + assert unknown_call.json()["error"]["code"] == "unknown_call_id" + + +def test_foundry_conformance_sdk_rejects_request_level_tools(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + response = client.post( + "/responses", + json={ + "input": "hello", + "tools": [{"type": "function", "name": "unsafe", "parameters": {"type": "object"}}], + }, + ) + + assert response.status_code == 200 + assert response.json()["status"] == "failed" + assert response.json()["error"]["code"] == "tools_unsupported" + + +def test_foundry_conformance_console_dry_run(capsys): + from agentkit_serve_common.foundry_conformance import main + + assert main(["--host", "127.0.0.1", "--port", "18088", "--model", "conformance-model", "--dry-run"]) == 0 + + body = json.loads(capsys.readouterr().out) + assert body == { + "host": "127.0.0.1", + "model": "conformance-model", + "port": 18088, + "protocols": {"responses": "2.0.0"}, + } diff --git a/runtimes/common/tests/test_foundry_protocol.py b/runtimes/common/tests/test_foundry_protocol.py index 3bf0f76..91a0c65 100644 --- a/runtimes/common/tests/test_foundry_protocol.py +++ b/runtimes/common/tests/test_foundry_protocol.py @@ -76,6 +76,19 @@ def test_foundry_invocations_and_responses_protocols(): assert body["usage"] == {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3} +def test_foundry_non_brokered_ignores_brokered_state_file_env(monkeypatch, tmp_path): + state_file = tmp_path / "corrupt-state.json" + state_file.write_text("not json", encoding="utf-8") + monkeypatch.setenv("AGENTKIT_FOUNDRY_RESPONSE_STATE_FILE", str(state_file)) + + app = create_foundry_app(_spec(), EchoFactory()) + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "hi"}) + + assert resp.status_code == 200 + assert resp.json()["output"][0]["content"][0]["text"] == "echo: hi" + + def test_foundry_responses_tolerates_stream_flag_with_non_streaming_response(): app = create_foundry_app(_spec(), EchoFactory()) with TestClient(app) as client: @@ -147,6 +160,82 @@ def test_foundry_responses_preserves_message_history_for_list_input(): ] +def test_foundry_non_brokered_responses_previous_response_id_does_not_force_continuation(): + factory = EchoFactory() + app = create_foundry_app(_spec(), factory) + + with TestClient(app) as client: + resp = client.post("/responses", json={"previous_response_id": "caresp_prior", "input": "next prompt"}) + + assert resp.status_code == 200 + assert resp.json()["output"][0]["content"][0]["text"] == "echo: next prompt" + assert factory.runtime.requests[0].prompt == "next prompt" + + +def test_foundry_non_brokered_function_call_output_is_not_routed_to_brokered_state_machine(): + factory = EchoFactory() + app = create_foundry_app(_spec(), factory) + + with TestClient(app) as client: + resp = client.post( + "/responses", + json={ + "previous_response_id": "caresp_prior", + "input": [ + { + "type": "function_call_output", + "call_id": "call_prior_1", + "output": '{"approved":true,"output":{"ok":true}}', + } + ], + }, + ) + + assert resp.status_code == 200 + assert resp.json()["output"][0]["content"][0]["text"].startswith("echo: ") + assert "function_call_output" in factory.runtime.requests[0].prompt + + +def test_foundry_non_brokered_function_call_output_input_stays_on_runtime_path(): + factory = EchoFactory() + app = create_foundry_app(_spec(), factory) + + with TestClient(app) as client: + resp = client.post( + "/responses", + json={ + "previous_response_id": "caresp_prior", + "input": [{"type": "function_call_output", "call_id": "call_1", "output": "{}"}], + }, + ) + + assert resp.status_code == 200 + assert "function_call_output" in factory.runtime.requests[0].prompt + assert resp.json()["output"][0]["content"][0]["text"].startswith("echo:") + + +def test_foundry_response_id_generator_tolerates_zero_arg_sdk(monkeypatch): + from agentkit_serve_common import foundry + + class ZeroArgIdGenerator: + @staticmethod + def new_response_id(): + return "caresp_zero_arg" + + @staticmethod + def new_message_item_id(response_id: str): + return f"msg_{response_id}" + + monkeypatch.setattr(foundry, "_AzureResponsesIdGenerator", ZeroArgIdGenerator) + app = create_foundry_app(_spec(), EchoFactory()) + + with TestClient(app) as client: + resp = client.post("/responses", json={"previous_response_id": "caresp_previous", "input": "hi"}) + + assert resp.status_code == 200 + assert resp.json()["id"] == "caresp_zero_arg" + + def test_foundry_responses_rejects_request_supplied_tools(): app = create_foundry_app(_spec(), EchoFactory()) with TestClient(app) as client: @@ -185,3 +274,69 @@ def test_foundry_protocol_uses_platform_session_env_fallback(monkeypatch): assert resp.status_code == 200 assert factory.runtime.requests[0].session_id == "platform-session" + + +def test_foundry_brokered_cli_dry_run_loads_static_brokered_agent(tmp_path, capsys): + from agentkit_serve_common.foundry_brokered_cli import main + + config = tmp_path / "agent.yaml" + config.write_text( + """abiVersion: v0 +metadata: + name: brokered-cli +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Broker tools. +tools: [] +brokeredTools: + - name: conformance_read + description: Read conformance data. + brokeredClass: read + parameters: + type: object + properties: + probe: + type: boolean +expose: + openai: true + port: 8088 +""", + encoding="utf-8", + ) + + assert main(["--config", str(config), "--dry-run"]) == 0 + + output = capsys.readouterr().out + assert '"agent": "brokered-cli"' in output + assert '"brokeredTools": ["conformance_read"]' in output + + +def test_foundry_brokered_cli_rejects_agents_without_brokered_tools(tmp_path): + from agentkit_serve_common.foundry_brokered_cli import main + + config = tmp_path / "agent.yaml" + config.write_text( + """abiVersion: v0 +metadata: + name: not-brokered +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Be helpful. +tools: [] +expose: + openai: true + port: 8088 +""", + encoding="utf-8", + ) + + try: + main(["--config", str(config), "--dry-run"]) + except SystemExit as exc: + assert "brokeredTools" in str(exc) + else: # pragma: no cover - assertion path. + raise AssertionError("expected missing brokeredTools to fail") diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py new file mode 100644 index 0000000..17d41cd --- /dev/null +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from agentkit_serve_common.foundry_conformance import create_foundry_conformance_app + + +def _load_verifier(): + repo = Path(__file__).resolve().parents[3] + path = repo / "deploy" / "foundry" / "scripts" / "verify_brokered_transcript.py" + spec = importlib.util.spec_from_file_location("verify_brokered_transcript", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _write_transcript(tmp_path: Path) -> Path: + app = create_foundry_conformance_app() + with TestClient(app) as client: + initial_request = {"input": "conformance_read"} + initial_response = client.post("/responses", json=initial_request).json() + continuation_request = { + "previous_response_id": initial_response["id"], + "input": [ + { + "type": "function_call_output", + "call_id": initial_response["output"][0]["call_id"], + "output": '{"approved":true,"output":{"success":true}}', + "status": "completed", + } + ], + } + continuation_response = client.post("/responses", json=continuation_request).json() + + files = { + "01-initial-request.json": initial_request, + "02-initial-response.json": initial_response, + "03-continuation-request.json": continuation_request, + "04-continuation-response.json": continuation_response, + } + for name, payload in files.items(): + (tmp_path / name).write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + return tmp_path + + +def test_verify_brokered_transcript_accepts_conformance_loop(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + + summary = verifier.verify_transcript(transcript) + + assert summary["initial_response_id"].startswith("caresp_") + assert summary["continuation_response_id"].startswith("caresp_") + assert summary["call_id"] == "call_conformance_1" + assert "success" in summary["final_text"] + + +def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + initial = json.loads((transcript / "02-initial-response.json").read_text(encoding="utf-8")) + initial["id"] = "resp_old" + (transcript / "02-initial-response.json").write_text(json.dumps(initial), encoding="utf-8") + + try: + verifier.verify_transcript(transcript) + except ValueError as exc: + assert "caresp_" in str(exc) + else: # pragma: no cover - assertion path. + raise AssertionError("expected old response id to fail") + + +def test_verify_brokered_transcript_cli_writes_summary(tmp_path, capsys): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + + assert verifier.main([str(transcript), "--write-summary"]) == 0 + + output = json.loads(capsys.readouterr().out) + written = json.loads((transcript / "summary.json").read_text(encoding="utf-8")) + assert output == written + assert written["call_id"] == "call_conformance_1" + + +def test_verify_brokered_transcript_accepts_agentkit_generated_call_id(tmp_path): + from agentkit_serve_common.config import AgentSpec + from agentkit_serve_common.foundry import create_foundry_app + + class Factory: + def build_runtime(self, spec): # noqa: ANN001 + raise AssertionError("brokered Foundry mode must not build direct runtime") + + spec = AgentSpec.model_validate( + { + "abiVersion": "v0", + "metadata": {"name": "agentkit-brokered"}, + "model": {"provider": "openai-compatible", "baseURL": "https://api.openai.com/v1", "name": "gpt-4o-mini"}, + "instructions": "Be helpful.", + "tools": [], + "brokeredTools": [ + { + "name": "conformance_read", + "description": "Read conformance data.", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"probe": {"type": "boolean"}}}, + } + ], + "expose": {"openai": True, "port": 8080}, + } + ) + app = create_foundry_app(spec, Factory(), brokered_continuation_proof="proof") + initial_request = {"input": "conformance_read"} + with TestClient(app) as client: + initial_response = client.post("/responses", json=initial_request).json() + continuation_request = { + "previous_response_id": initial_response["id"], + "input": [ + { + "type": "function_call_output", + "call_id": initial_response["output"][0]["call_id"], + "output": '{"approved":true,"output":{"success":true}}', + "status": "completed", + } + ], + } + continuation_response = client.post( + "/responses", + headers={"x-agentkit-brokered-continuation-proof": "proof"}, + json=continuation_request, + ).json() + for name, payload in { + "01-initial-request.json": initial_request, + "02-initial-response.json": initial_response, + "03-continuation-request.json": continuation_request, + "04-continuation-response.json": continuation_response, + }.items(): + (tmp_path / name).write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + + summary = _load_verifier().verify_transcript( + tmp_path, + expected_call_id="auto", + expected_call_id_prefix="call_", + ) + + assert summary["call_id"].startswith("call_") + assert summary["arguments"] == {"probe": True} diff --git a/test/foundry-brokered-agentkit/Dockerfile b/test/foundry-brokered-agentkit/Dockerfile new file mode 100644 index 0000000..6bc0d54 --- /dev/null +++ b/test/foundry-brokered-agentkit/Dockerfile @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1 +# Minimal production AgentKit Foundry brokered-only image with static brokeredTools. +# Build from the repository root so runtimes/common is available in context. +FROM python:3.12-slim + +ENV PORT=8088 \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/agentkit +COPY runtimes/common /tmp/agentkit-serve-common +RUN pip install --no-cache-dir /tmp/agentkit-serve-common \ + && rm -rf /tmp/agentkit-serve-common +COPY test/foundry-brokered-agentkit/agent.yaml /agent/agent.yaml + +EXPOSE 8088 +ENTRYPOINT ["agentkit-foundry-brokered", "--host", "0.0.0.0", "--port", "8088", "--config", "/agent/agent.yaml"] diff --git a/test/foundry-brokered-agentkit/README.md b/test/foundry-brokered-agentkit/README.md new file mode 100644 index 0000000..f74d40a --- /dev/null +++ b/test/foundry-brokered-agentkit/README.md @@ -0,0 +1,47 @@ +# Production AgentKit Foundry brokered-only fixture + +This fixture packages the shared production `create_foundry_app` brokered path, +not the separate SDK conformance spike app. It uses a static `/agent/agent.yaml` +with a safe `conformance_read` brokered schema and the brokered-only entrypoint: + +```sh +agentkit-foundry-brokered --config /agent/agent.yaml --host 0.0.0.0 --port 8088 +``` + +Set `AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF` at runtime so transcript +validation can exercise the Orka-only continuation proof header. For real +deployments, inject this value from runtime configuration or a secret. + +Build and validate locally: + +```sh +docker build . -f test/foundry-brokered-agentkit/Dockerfile \ + -t agentkit-foundry-brokered:local + +docker run --rm \ + -e AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=local-dev-proof \ + -p 127.0.0.1:18092:8088 \ + agentkit-foundry-brokered:local +``` + +Then run: + +```sh +AGENT_RESPONSES_ENDPOINT=http://127.0.0.1:18092/responses \ +AGENT_RESPONSES_BEARER_TOKEN=local-dummy-token \ +AGENTKIT_CONTINUATION_PROOF=local-dev-proof \ +AGENTKIT_EXPECTED_CALL_ID=auto \ +AGENTKIT_EXPECTED_CALL_ID_PREFIX=call_ \ +deploy/foundry/scripts/foundry_brokered_conformance.sh \ + conformance_read ./foundry-brokered-agentkit-transcript +``` + + +Or run the all-in-one local build/run/transcript smoke from the repository root: + +```sh +deploy/foundry/scripts/local_brokered_conformance_container.sh \ + --fixture agentkit \ + --platform linux/amd64 \ + --transcript-dir ./foundry-brokered-agentkit-transcript +``` diff --git a/test/foundry-brokered-agentkit/agent.yaml b/test/foundry-brokered-agentkit/agent.yaml new file mode 100644 index 0000000..734eac5 --- /dev/null +++ b/test/foundry-brokered-agentkit/agent.yaml @@ -0,0 +1,22 @@ +abiVersion: v0 +metadata: + name: agentkit-foundry-brokered-local +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: | + Broker all tool use through Orka. Never claim a tool completed unless the brokered tool output confirms it. +tools: [] +brokeredTools: + - name: conformance_read + description: Read conformance data. + brokeredClass: read + parameters: + type: object + properties: + probe: + type: boolean +expose: + openai: true + port: 8088 diff --git a/test/foundry-brokered-conformance/Dockerfile b/test/foundry-brokered-conformance/Dockerfile new file mode 100644 index 0000000..0e8518a --- /dev/null +++ b/test/foundry-brokered-conformance/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +# Minimal Phase A0 Foundry hosted Responses brokered conformance image. +# Build from the repository root so runtimes/common is available in context. +FROM python:3.12-slim + +ENV PORT=8088 \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/agentkit +COPY runtimes/common /tmp/agentkit-serve-common +RUN pip install --no-cache-dir /tmp/agentkit-serve-common \ + && rm -rf /tmp/agentkit-serve-common + +EXPOSE 8088 +ENTRYPOINT ["agentkit-foundry-conformance", "--host", "0.0.0.0", "--port", "8088"] diff --git a/test/foundry-brokered-conformance/README.md b/test/foundry-brokered-conformance/README.md new file mode 100644 index 0000000..9683a4e --- /dev/null +++ b/test/foundry-brokered-conformance/README.md @@ -0,0 +1,86 @@ +# Foundry brokered Responses conformance image + +This fixture packages the Phase A0 SDK conformance app as a minimal Foundry +hosted-agent container. It is intentionally separate from production +`foundry.py`: its only job is to prove that a deployed Foundry hosted container +can return a Responses `function_call` item and later consume a matching +`function_call_output` continuation using SDK/platform response IDs. + +The container entrypoint is: + +```sh +agentkit-foundry-conformance --host 0.0.0.0 --port 8088 +``` + +It serves: + +- `GET /readiness` +- `POST /responses` + +It does **not** accept request-level `tools`. + +## Build locally + +From the repository root: + +```sh +docker buildx build --builder desktop-linux . \ + -f test/foundry-brokered-conformance/Dockerfile \ + --platform linux/amd64 \ + -t agentkit-foundry-brokered-conformance:test --load --provenance=false +``` + +## Validate locally + +```sh +docker run --rm --platform linux/amd64 -p 127.0.0.1:18088:8088 \ + agentkit-foundry-brokered-conformance:test +``` + +In another terminal: + +```sh +curl -fsS http://127.0.0.1:18088/readiness +curl -fsS -H 'content-type: application/json' \ + http://127.0.0.1:18088/responses \ + -d '{"input":"conformance_read"}' +``` + +The first `/responses` call should return one `function_call` named +`conformance_read` with `call_id: call_conformance_1` and a `caresp_...` +response id. + +To exercise the full initial/continuation loop locally with the same transcript +helper used for live validation: + +```sh +AGENT_RESPONSES_ENDPOINT=http://127.0.0.1:18088/responses \ +AGENT_RESPONSES_BEARER_TOKEN=local-dummy-token \ +deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundry-brokered-local-transcript +``` + +Or run the all-in-one local build/run/transcript smoke from the repository root: + +```sh +deploy/foundry/scripts/local_brokered_conformance_container.sh \ + --fixture sdk \ + --platform linux/amd64 \ + --transcript-dir ./foundry-brokered-local-transcript +``` + +## Deploy to Foundry + +1. Push the image to a registry reachable by Foundry. +2. Copy `foundry.agent.yaml.example` to the azd hosted-agent project as + `agent.yaml` and set `image:` to the pushed tag. +3. Run `azd provision` and `azd deploy`. +4. Run the live transcript helper from the repository root: + +```sh +export AGENT_RESPONSES_ENDPOINT="https:///responses" +# Optional: export AZURE_SUBSCRIPTION_ID="" to select an account. +deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundry-brokered-transcript +``` + +The helper writes request/response JSON plus `summary.json`. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py ` to verify an archived transcript later. Keep bearer tokens +out of transcripts. diff --git a/test/foundry-brokered-conformance/foundry.agent.yaml.example b/test/foundry-brokered-conformance/foundry.agent.yaml.example new file mode 100644 index 0000000..aa9833a --- /dev/null +++ b/test/foundry-brokered-conformance/foundry.agent.yaml.example @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml +# Copy this to agent.yaml inside an azd project and set image to the pushed tag. +kind: hosted +name: agentkit-foundry-brokered-conformance +image: docker.io/YOUR_ORG/agentkit-foundry-brokered-conformance:TAG +protocols: + - protocol: responses + version: 2.0.0 +resources: + cpu: "0.25" + memory: 0.5Gi From 268a67c200a5cd661afd3dae0bb12ae8cc22a4a3 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 09:30:57 -0700 Subject: [PATCH 02/27] fix(foundry): harden brokered response auth Signed-off-by: Sertac Ozercan --- .../local_brokered_conformance_container.sh | 3 +++ .../common/agentkit_serve_common/foundry.py | 2 +- .../foundry_brokered_cli.py | 10 ++++++++++ .../foundry_model_loop.py | 19 ++++++++++++++----- .../common/tests/test_foundry_conformance.py | 4 +--- test/foundry-brokered-agentkit/README.md | 1 + 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/deploy/foundry/scripts/local_brokered_conformance_container.sh b/deploy/foundry/scripts/local_brokered_conformance_container.sh index a33d1cc..89b684b 100755 --- a/deploy/foundry/scripts/local_brokered_conformance_container.sh +++ b/deploy/foundry/scripts/local_brokered_conformance_container.sh @@ -103,6 +103,9 @@ run_args=() if [[ -n "$platform" ]]; then run_args+=(--platform "$platform") fi +if [[ "$fixture" == "agentkit" ]]; then + run_args+=(-e "AGENTKIT_AUTH_TOKEN=local-dummy-token") +fi if [[ -n "$continuation_proof" ]]; then run_args+=(-e "AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=${continuation_proof}") fi diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index a67bff9..2d40c76 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -409,7 +409,7 @@ def evict_completed_to_capacity(self, *, reserve_slots: int = 0) -> None: if len(self._states) <= target: return completed = sorted( - (entry for entry in self._states.values() if entry.status != "pending"), + (entry for entry in self._states.values() if entry.status == "completed" and entry.final_payload is not None), key=lambda entry: entry.expires_at, ) changed = False diff --git a/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py index 62cf38b..77d36e4 100644 --- a/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py +++ b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py @@ -24,6 +24,11 @@ DEFAULT_CONFIG_PATH = "/agent/agent.yaml" DEFAULT_PORT = 8088 +_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", "::ffff:127.0.0.1"} + + +def _is_loopback(host: str) -> bool: + return host.strip().lower() in _LOOPBACK_HOSTS class _NoDirectRuntime: @@ -69,6 +74,11 @@ def main(argv: Sequence[str] | None = None) -> int: if not spec.brokered_tools: raise SystemExit("agentkit-foundry-brokered: agent.yaml must declare at least one brokeredTools entry") auth_token = os.environ.get("AGENTKIT_AUTH_TOKEN") or None + if not args.dry_run and not _is_loopback(args.host) and not auth_token: + raise SystemExit( + f"agentkit-foundry-brokered: refusing to bind {args.host!r} without AGENTKIT_AUTH_TOKEN; " + "set a bearer token or bind 127.0.0.1 for local-only use" + ) app = create_foundry_app(spec, _NoDirectFactory(), auth_token=auth_token) if args.dry_run: print( diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index cc07384..cf1ba68 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -9,13 +9,15 @@ from __future__ import annotations +import asyncio import json +import os from dataclasses import dataclass, field from typing import Any, Mapping, Sequence import httpx -from .adapter_support import AgentBuildError, NO_AUTH_API_KEY, resolve_api_key +from .adapter_support import AgentBuildError, NO_AUTH_API_KEY, resolve_api_key, resolve_workload_identity_token from .config import AgentSpec from .conversation import FORWARDED_ROLES, RunRequest from .runtime import AgentRunError, BrokeredToolDefinition @@ -136,13 +138,20 @@ async def _chat(self, messages: Sequence[Mapping[str, Any]], *, tools: Sequence[ client = self.http_client close_client = False if client is None: + headers: dict[str, str] = {} try: - api_key = resolve_api_key(self.spec) + auth = self.spec.model.auth + if auth is not None and auth.type == "workload-identity-token": + token = os.environ.get("AGENTKIT_MODEL_WORKLOAD_IDENTITY_TOKEN") + if not token: + token = await asyncio.to_thread(resolve_workload_identity_token, auth.audience or "") + headers["Authorization"] = f"Bearer {token}" + else: + api_key = resolve_api_key(self.spec) + if api_key != NO_AUTH_API_KEY: + headers["Authorization"] = f"Bearer {api_key}" except AgentBuildError as exc: raise AgentRunError(str(exc), status=400, code="ModelAuthMissing") from exc - headers = {} - if api_key != NO_AUTH_API_KEY: - headers["Authorization"] = f"Bearer {api_key}" client = httpx.AsyncClient(headers=headers, timeout=60) close_client = True try: diff --git a/runtimes/common/tests/test_foundry_conformance.py b/runtimes/common/tests/test_foundry_conformance.py index 485ea10..7147fab 100644 --- a/runtimes/common/tests/test_foundry_conformance.py +++ b/runtimes/common/tests/test_foundry_conformance.py @@ -3,8 +3,6 @@ import json from fastapi.testclient import TestClient -from azure.ai.agentserver.responses._id_generator import IdGenerator - from agentkit_serve_common.foundry_conformance import create_foundry_conformance_app @@ -62,7 +60,7 @@ def test_foundry_conformance_sdk_rejects_orphan_and_unknown_continuations(): with TestClient(app) as client: missing_previous = client.post("/responses", json=_function_output(None)) - unknown_previous = client.post("/responses", json=_function_output(IdGenerator.new_response_id())) + unknown_previous = client.post("/responses", json=_function_output("caresp_0123456789abcdef00ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef")) initial = client.post("/responses", json={"input": "hello"}).json() unknown_call = client.post("/responses", json=_function_output(initial["id"], call_id="call_other")) diff --git a/test/foundry-brokered-agentkit/README.md b/test/foundry-brokered-agentkit/README.md index f74d40a..883ae5f 100644 --- a/test/foundry-brokered-agentkit/README.md +++ b/test/foundry-brokered-agentkit/README.md @@ -19,6 +19,7 @@ docker build . -f test/foundry-brokered-agentkit/Dockerfile \ -t agentkit-foundry-brokered:local docker run --rm \ + -e AGENTKIT_AUTH_TOKEN=local-dummy-token \ -e AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=local-dev-proof \ -p 127.0.0.1:18092:8088 \ agentkit-foundry-brokered:local From bd9641c7addd59e65dc1c7211a160af8ad82d537 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 09:47:25 -0700 Subject: [PATCH 03/27] fix(config): align brokered auth text validation Signed-off-by: Sertac Ozercan --- pkg/agentkit/config/config_test.go | 13 +++++++++ pkg/agentkit/config/validate.go | 24 ++++++++++++++++- .../common/agentkit_serve_common/runtime.py | 27 ------------------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index c874127..708d47b 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -1345,6 +1345,19 @@ func TestValidateRejectsUnsafeBrokeredToolSchema(t *testing.T) { } } +func TestHasUnsafeBrokeredTextMatchesAuthSchemesAsWords(t *testing.T) { + for _, safe := range []string{"This tool basically reads telemetry", "Read crossbearer telemetry labels"} { + if hasUnsafeBrokeredText(safe) { + t.Fatalf("expected %q to be safe brokered text", safe) + } + } + for _, unsafe := range []string{"Use bearer auth", "Basic authentication required"} { + if !hasUnsafeBrokeredText(unsafe) { + t.Fatalf("expected %q to be unsafe brokered text", unsafe) + } + } +} + func TestValidateRejectsSecretLiteralBrokeredSchemaStringValues(t *testing.T) { cfg := validMinimalConfig() cfg.BrokeredTools = []BrokeredTool{{ diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index c2d7c64..2c7cb51 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -629,7 +629,29 @@ func isSchemaDigest(value string) bool { func hasUnsafeBrokeredText(value string) bool { lowered := strings.ToLower(value) normalized := normalizeKey(lowered) - return containsSecretPrefix(value) || strings.Contains(value, "://") || strings.Contains(lowered, "bearer") || strings.Contains(lowered, "basic") || strings.Contains(lowered, "authorization") || strings.Contains(lowered, "secret") || strings.Contains(lowered, "token") || strings.Contains(lowered, "password") || strings.Contains(lowered, "passphrase") || strings.Contains(lowered, "pwd") || strings.Contains(lowered, "api key") || strings.Contains(lowered, "apikey") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "xapikey") || strings.Contains(normalized, "subscriptionkey") || strings.Contains(normalized, "xfunctionskey") || strings.Contains(lowered, brokeredUnsafeCookieKey) || strings.Contains(lowered, "set-cookie") || strings.Contains(lowered, "x-api-key") || strings.Contains(lowered, credentialHeaderAPIKey) || strings.Contains(lowered, "subscription-key") || strings.Contains(lowered, "x-functions-key") || strings.Contains(lowered, "ocp-apim-subscription-key") || strings.Contains(lowered, "private key") || strings.Contains(lowered, "privatekey") || strings.Contains(lowered, "key material") || strings.Contains(lowered, ".svc") || strings.Contains(lowered, "cluster.local") + return containsSecretPrefix(value) || strings.Contains(value, "://") || containsBrokeredWord(lowered, "bearer") || containsBrokeredWord(lowered, "basic") || strings.Contains(lowered, "authorization") || strings.Contains(lowered, "secret") || strings.Contains(lowered, "token") || strings.Contains(lowered, "password") || strings.Contains(lowered, "passphrase") || strings.Contains(lowered, "pwd") || strings.Contains(lowered, "api key") || strings.Contains(lowered, "apikey") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "xapikey") || strings.Contains(normalized, "subscriptionkey") || strings.Contains(normalized, "xfunctionskey") || strings.Contains(lowered, brokeredUnsafeCookieKey) || strings.Contains(lowered, "set-cookie") || strings.Contains(lowered, "x-api-key") || strings.Contains(lowered, credentialHeaderAPIKey) || strings.Contains(lowered, "subscription-key") || strings.Contains(lowered, "x-functions-key") || strings.Contains(lowered, "ocp-apim-subscription-key") || strings.Contains(lowered, "private key") || strings.Contains(lowered, "privatekey") || strings.Contains(lowered, "key material") || strings.Contains(lowered, ".svc") || strings.Contains(lowered, "cluster.local") +} + +func containsBrokeredWord(value string, word string) bool { + start := 0 + for { + idx := strings.Index(value[start:], word) + if idx < 0 { + return false + } + idx += start + after := idx + len(word) + beforeOK := idx == 0 || !isBrokeredWordByte(value[idx-1]) + afterOK := after == len(value) || !isBrokeredWordByte(value[after]) + if beforeOK && afterOK { + return true + } + start = after + } +} + +func isBrokeredWordByte(value byte) bool { + return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') || (value >= '0' && value <= '9') || value == '_' } func isUnsafeBrokeredKey(value string) bool { diff --git a/runtimes/common/agentkit_serve_common/runtime.py b/runtimes/common/agentkit_serve_common/runtime.py index 1d35edd..046c141 100644 --- a/runtimes/common/agentkit_serve_common/runtime.py +++ b/runtimes/common/agentkit_serve_common/runtime.py @@ -65,33 +65,6 @@ class RunResult: usage: dict[str, int] = field(default_factory=dict) -@dataclass(frozen=True) -class RuntimeMessageCompleted: - """A Responses-compatible runtime result containing final assistant text.""" - - text: str - usage: dict[str, int] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RuntimeToolCallRequested: - """A Responses-compatible runtime pause requesting brokered tool execution.""" - - tool_call_id: str - name: str - arguments: Mapping[str, Any] - brokered_class: Literal["read", "write", "coordination"] - usage: dict[str, int] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RuntimeFailed: - """A deterministic runtime failure result for hosted protocol adapters.""" - - message: str - status: int = 502 - code: str = "RuntimeFailed" - @dataclass(frozen=True) class BrokeredToolDefinition: From ba926555826fa1fd5a9589c444bd379c659d796b Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 09:55:05 -0700 Subject: [PATCH 04/27] fix(foundry): reject lossy model-loop floats Signed-off-by: Sertac Ozercan --- .../foundry_model_loop.py | 24 +++++++++++++- .../tests/test_foundry_brokered_protocol.py | 33 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index cf1ba68..7b1011a 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -13,6 +13,7 @@ import json import os from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation from typing import Any, Mapping, Sequence import httpx @@ -198,7 +199,9 @@ def _message_text(message: Mapping[str, Any]) -> str: def _parse_arguments(raw: Any) -> dict[str, Any]: if isinstance(raw, str): try: - parsed = json.loads(raw or "{}") + parsed = json.loads(raw or "{}", parse_float=_parse_json_float, parse_constant=_reject_json_constant) + except AgentRunError: + raise except json.JSONDecodeError as exc: raise AgentRunError("model tool arguments must be valid JSON", status=400, code="InvalidToolArguments") from exc else: @@ -208,6 +211,25 @@ def _parse_arguments(raw: Any) -> dict[str, Any]: return parsed +def _parse_json_float(raw: str) -> float: + try: + decimal = Decimal(raw) + except InvalidOperation as exc: + raise AgentRunError("model tool arguments must contain valid JSON numbers", status=400, code="InvalidToolArguments") from exc + parsed = float(decimal) + if not Decimal(str(parsed)) == decimal: + raise AgentRunError( + "model tool arguments contain a number that cannot be represented exactly", + status=400, + code="InvalidToolArguments", + ) + return parsed + + +def _reject_json_constant(raw: str) -> None: + raise AgentRunError(f"model tool arguments contain non-finite number {raw}", status=400, code="InvalidToolArguments") + + def _usage(data: Mapping[str, Any]) -> dict[str, int]: usage = data.get("usage") if isinstance(data.get("usage"), Mapping) else {} prompt_tokens = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index d1a1ec4..33fb514 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1457,6 +1457,39 @@ def test_foundry_brokered_model_loop_validates_large_integer_bounds_exactly(): assert response.json()["error"]["code"] == "InvalidToolArguments" +def test_foundry_brokered_model_loop_rejects_float_arguments_that_cannot_round_trip_exactly(): + spec = _spec(tool_name="check-network-telemetry") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"count": {"type": "integer", "maximum": 9007199254740992}}, + "required": ["count"], + } + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": '{"count":9007199254740993.0}'}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "call check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + def test_foundry_brokered_model_loop_rejects_unsafe_model_generated_arguments(): fake = _FakeChatTransport( [ From 4ad34a970480716a9e1387b64f131a8d7f07794e Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Thu, 9 Jul 2026 09:56:17 -0700 Subject: [PATCH 05/27] fix(foundry): tidy doctor auth warning Signed-off-by: Sertac Ozercan --- deploy/foundry/doctor.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/foundry/doctor.sh b/deploy/foundry/doctor.sh index 79271d9..9594713 100755 --- a/deploy/foundry/doctor.sh +++ b/deploy/foundry/doctor.sh @@ -47,8 +47,7 @@ if [[ "$mode" == "brokered-conformance" ]]; then az account set --subscription "$AZURE_SUBSCRIPTION_ID" >/dev/null 2>&1 || missing=1 fi if ! az account show >/dev/null 2>&1; then - printf 'missing auth: set AGENT_RESPONSES_BEARER_TOKEN or run az login/select an account -' >&2 + printf 'missing auth: set AGENT_RESPONSES_BEARER_TOKEN or run az login/select an account\n' >&2 missing=1 fi fi From 9a7185809ff4e07311e0ee908e7202c7447d2123 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 18:37:06 -0700 Subject: [PATCH 06/27] fix(foundry): address brokered review feedback Signed-off-by: Sertac Ozercan --- docs/foundry-hosted-brokered.md | 5 + runtimes/common/README.md | 5 + .../common/agentkit_serve_common/brokered.py | 89 ++++--- .../common/agentkit_serve_common/foundry.py | 14 +- .../common/agentkit_serve_common/runtime.py | 1 - runtimes/common/tests/test_brokered_schema.py | 236 +++++++++++++++++- .../tests/test_foundry_brokered_protocol.py | 90 +++++++ 7 files changed, 384 insertions(+), 56 deletions(-) diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md index f5e209f..383c355 100644 --- a/docs/foundry-hosted-brokered.md +++ b/docs/foundry-hosted-brokered.md @@ -56,6 +56,11 @@ startup fails if a configured digest no longer matches the safe schema in `agent.yaml`. Orka still validates every live call against current Tool CRDs at execution time. +The exporter accepts canonical `core.orka.ai/v1alpha1` `Tool` resources and +reads the governed class from `spec.brokeredToolClass`. Tools without that field +are not brokered and are omitted; exporting an input set with no brokered tools +fails instead of silently downgrading a tool to `read`. + Example export command: ```sh diff --git a/runtimes/common/README.md b/runtimes/common/README.md index a8b738b..7de57fc 100644 --- a/runtimes/common/README.md +++ b/runtimes/common/README.md @@ -66,6 +66,11 @@ parameters schema, and optional schema digest. Execution URLs, auth headers, Secret refs, tokens, and other credential-shaped schema fields are rejected or omitted before the fragment is model-visible. +Inputs must use the canonical `core.orka.ai/v1alpha1` `Tool` shape. The exporter +reads `spec.brokeredToolClass`; unclassified tools are not brokered and are +skipped, and an input set with no classified tools fails rather than defaulting +their class to `read`. + ## Foundry brokered conformance app diff --git a/runtimes/common/agentkit_serve_common/brokered.py b/runtimes/common/agentkit_serve_common/brokered.py index 1188ed2..f2d26fb 100644 --- a/runtimes/common/agentkit_serve_common/brokered.py +++ b/runtimes/common/agentkit_serve_common/brokered.py @@ -19,6 +19,10 @@ from .config import AgentSpec, BrokeredToolSpec, brokered_tool_schema_digest from .runtime import BrokeredToolDefinition +_ORKA_TOOL_API_VERSION = "core.orka.ai/v1alpha1" +_ORKA_TOOL_KIND = "Tool" +_ORKA_BROKERED_TOOL_CLASSES = ("read", "write", "coordination") + def brokered_tool_definitions(spec: AgentSpec) -> list[BrokeredToolDefinition]: """Return runtime-safe brokered tool definitions from an AgentSpec.""" @@ -70,65 +74,56 @@ def brokered_tool_config_entry( return out -def _nested_get(mapping: Mapping[str, Any], *path: str) -> Any: - current: Any = mapping - for key in path: - if not isinstance(current, Mapping): - return None - current = current.get(key) - return current - - -def _first_present(*values: Any) -> Any: - for value in values: - if value not in (None, ""): - return value - return None - - def generate_brokered_tools_from_orka_tool_crds( - documents: Iterable[Mapping[str, Any]], + documents: Iterable[Any], *, include_digest: bool = True, ) -> list[dict[str, Any]]: """Generate deterministic safe AgentKit brokeredTools config from Orka Tool CRDs. - The Orka Tool CRD remains the source of truth. This helper deliberately - extracts only the safe model-facing subset and validates it with the same - `BrokeredToolSpec` model used by `agent.yaml` loading. + The Orka Tool CRD remains the source of truth. Only the canonical + `core.orka.ai/v1alpha1` `Tool` shape is accepted. Tools without + `spec.brokeredToolClass` are not brokered and are skipped; an input set with + no brokered tools is rejected. Safe model-facing fields are validated with + the same `BrokeredToolSpec` model used by `agent.yaml` loading. """ entries: list[dict[str, Any]] = [] for idx, document in enumerate(documents): - if not isinstance(document, Mapping) or not document: + if document is None: continue - kind = str(document.get("kind") or "") - if kind and kind.lower() != "tool": + if not isinstance(document, Mapping): + raise ValueError(f"Orka Tool CRD document {idx} must be an object") + api_version = document.get("apiVersion") + kind = document.get("kind") + if api_version != _ORKA_TOOL_API_VERSION or kind != _ORKA_TOOL_KIND: + raise ValueError( + f"unsupported Orka Tool GVK in document {idx}: apiVersion={api_version!r}, kind={kind!r}; " + f"expected apiVersion={_ORKA_TOOL_API_VERSION!r}, kind={_ORKA_TOOL_KIND!r}" + ) + metadata = document.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError(f"Orka Tool CRD document {idx} metadata must be an object") + spec = document.get("spec") + if not isinstance(spec, Mapping): + raise ValueError(f"Orka Tool CRD document {idx} spec must be an object") + if "brokeredToolClass" not in spec: continue - metadata = document.get("metadata") if isinstance(document.get("metadata"), Mapping) else {} - spec = document.get("spec") if isinstance(document.get("spec"), Mapping) else {} - name = _first_present(spec.get("name"), metadata.get("name")) + name = metadata.get("name") if not isinstance(name, str): raise ValueError(f"Tool CRD document {idx} is missing metadata.name") - description = _first_present(spec.get("description"), spec.get("summary"), name) + description = spec.get("description") if not isinstance(description, str): raise ValueError(f"Tool CRD {name!r} description must be a string") - brokered_class = _first_present( - spec.get("brokeredClass"), - spec.get("brokered_class"), - spec.get("class"), - _nested_get(spec, "brokered", "class"), - "read", - ) + brokered_class = spec["brokeredToolClass"] if not isinstance(brokered_class, str): - raise ValueError(f"Tool CRD {name!r} brokered class must be a string") - parameters = _first_present( - spec.get("parameters"), - spec.get("inputSchema"), - spec.get("input_schema"), - spec.get("schema"), - _nested_get(spec, "input", "schema"), - ) + raise ValueError(f"Tool CRD {name!r} spec.brokeredToolClass must be a string") + if brokered_class not in _ORKA_BROKERED_TOOL_CLASSES: + supported = ", ".join(_ORKA_BROKERED_TOOL_CLASSES) + raise ValueError( + f"Tool CRD {name!r} has unsupported spec.brokeredToolClass {brokered_class!r}; expected {supported}" + ) + parameters = spec.get("parameters") if parameters is None: parameters = {"type": "object"} if not isinstance(parameters, Mapping): @@ -142,6 +137,10 @@ def generate_brokered_tools_from_orka_tool_crds( include_digest=include_digest, ) ) + if not entries: + raise ValueError( + "no brokered Orka Tool CRDs found; set spec.brokeredToolClass to read, write, or coordination" + ) return sorted(entries, key=lambda item: item["name"]) @@ -149,17 +148,17 @@ def load_orka_tool_crd_file(path: str | Path, *, include_digest: bool = True) -> """Load Tool CRD YAML/JSON documents and return safe brokeredTools entries.""" raw = Path(path).read_text(encoding="utf-8") - docs = [doc for doc in yaml.safe_load_all(raw) if isinstance(doc, Mapping)] + docs = [doc for doc in yaml.safe_load_all(raw) if doc is not None] return generate_brokered_tools_from_orka_tool_crds(docs, include_digest=include_digest) def load_orka_tool_crd_files(paths: Sequence[str | Path], *, include_digest: bool = True) -> list[dict[str, Any]]: """Load one or more Tool CRD files and merge deterministic safe entries.""" - documents: list[Mapping[str, Any]] = [] + documents: list[Any] = [] for path in paths: raw = Path(path).read_text(encoding="utf-8") - documents.extend(doc for doc in yaml.safe_load_all(raw) if isinstance(doc, Mapping)) + documents.extend(doc for doc in yaml.safe_load_all(raw) if doc is not None) entries = generate_brokered_tools_from_orka_tool_crds(documents, include_digest=include_digest) seen: set[str] = set() duplicates: set[str] = set() diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 2d40c76..f08d4a5 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -853,6 +853,12 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run return dict(literal) enum = parameters.get("enum") if isinstance(enum, list): + if tool.brokered_class != "read" and len(enum) > 1: + raise AgentRunError( + f"brokered {tool.brokered_class} tool {tool.name!r} has multiple root enum payloads; deterministic mode refuses to choose side-effecting arguments", + status=400, + code="UnsupportedBrokeredSchema", + ) for item in enum: if isinstance(item, Mapping): return dict(item) @@ -872,6 +878,12 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run if tool.name == "conformance_read" and "probe" in properties and "probe" not in arguments: arguments["probe"] = True if not arguments and "prompt" in properties: + if tool.brokered_class != "read" and not _schema_has_literal_value(properties["prompt"]): + raise AgentRunError( + f"brokered {tool.brokered_class} tool {tool.name!r} has a non-literal optional prompt; deterministic mode refuses to synthesize side-effecting arguments", + status=400, + code="UnsupportedBrokeredSchema", + ) arguments["prompt"] = _sample_argument_value("prompt", properties["prompt"], run_request) return arguments @@ -1386,7 +1398,7 @@ async def responses(request: Request): previous_state = response_states.get(previous_response_id) except (KeyError, _StateExpired): previous_state = None - if previous_state is not None and previous_state.status == "pending": + if previous_state is not None and previous_state.status in {"pending", "resuming"}: return _error( "previous_response_id is pending a brokered function_call_output", status=409, diff --git a/runtimes/common/agentkit_serve_common/runtime.py b/runtimes/common/agentkit_serve_common/runtime.py index 046c141..c906061 100644 --- a/runtimes/common/agentkit_serve_common/runtime.py +++ b/runtimes/common/agentkit_serve_common/runtime.py @@ -65,7 +65,6 @@ class RunResult: usage: dict[str, int] = field(default_factory=dict) - @dataclass(frozen=True) class BrokeredToolDefinition: """Safe tool schema a brokered runtime may request through Orka. diff --git a/runtimes/common/tests/test_brokered_schema.py b/runtimes/common/tests/test_brokered_schema.py index 7072999..d3c2399 100644 --- a/runtimes/common/tests/test_brokered_schema.py +++ b/runtimes/common/tests/test_brokered_schema.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pytest import yaml from agentkit_serve_common.brokered import ( @@ -16,30 +17,222 @@ def _tool_docs() -> list[dict]: return [ { - "apiVersion": "orka.example/v1", + "apiVersion": "core.orka.ai/v1alpha1", "kind": "Tool", "metadata": {"name": "dispatch-work-order"}, "spec": { "description": "Dispatch a field tech.", - "brokeredClass": "write", + "brokeredToolClass": "write", "parameters": {"type": "object", "properties": {"incident": {"type": "string"}}, "required": ["incident"]}, - "url": "http://tool.default.svc.cluster.local", - "secretRef": {"name": "do-not-export"}, + "http": { + "url": "http://tool.default.svc.cluster.local", + "method": "POST", + "authSecretRef": {"name": "do-not-export", "key": "token"}, + }, }, }, { - "apiVersion": "orka.example/v1", + "apiVersion": "core.orka.ai/v1alpha1", "kind": "Tool", "metadata": {"name": "check-network-telemetry"}, "spec": { "description": "Read telemetry.", - "brokeredClass": "read", - "inputSchema": {"type": "object", "properties": {"site": {"type": "string"}}}, - "headers": {"Authorization": "do-not-export"}, + "brokeredToolClass": "read", + "parameters": {"type": "object", "properties": {"site": {"type": "string"}}}, + "http": { + "url": "http://tool.default.svc.cluster.local", + "method": "POST", + "headers": {"Authorization": "do-not-export"}, + }, + }, + }, + ] + + +def test_generate_brokered_tools_uses_canonical_orka_brokered_tool_class(): + documents = [ + { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "read-telemetry"}, + "spec": { + "description": "Read telemetry.", + "brokeredToolClass": "read", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/read", "method": "POST"}, + }, + }, + { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "dispatch-work-order"}, + "spec": { + "description": "Dispatch a work order.", + "brokeredToolClass": "write", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/write", "method": "POST"}, + }, + }, + { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "coordinate-response"}, + "spec": { + "description": "Coordinate incident response.", + "brokeredToolClass": "coordination", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/coordinate", "method": "POST"}, + }, + }, + ] + + generated = generate_brokered_tools_from_orka_tool_crds(documents, include_digest=False) + + assert [(tool["name"], tool["brokeredClass"]) for tool in generated] == [ + ("coordinate-response", "coordination"), + ("dispatch-work-order", "write"), + ("read-telemetry", "read"), + ] + + +def test_generate_brokered_tools_rejects_unsupported_orka_api_version(): + document = { + "apiVersion": "core.orka.ai/v1", + "kind": "Tool", + "metadata": {"name": "read-telemetry"}, + "spec": { + "description": "Read telemetry.", + "brokeredToolClass": "read", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/read", "method": "POST"}, + }, + } + + with pytest.raises(ValueError, match=r"unsupported Orka Tool GVK.*core\.orka\.ai/v1alpha1.*Tool"): + generate_brokered_tools_from_orka_tool_crds([document]) + + +def test_generate_brokered_tools_rejects_unsupported_orka_kind(): + document = { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "AgentRuntime", + "metadata": {"name": "runtime"}, + "spec": {}, + } + + with pytest.raises(ValueError, match=r"unsupported Orka Tool GVK.*AgentRuntime.*Tool"): + generate_brokered_tools_from_orka_tool_crds([document]) + + +def test_generate_brokered_tools_rejects_non_object_documents(): + valid_tool = { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "read-telemetry"}, + "spec": { + "description": "Read telemetry.", + "brokeredToolClass": "read", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/read", "method": "POST"}, + }, + } + + with pytest.raises(ValueError, match=r"document 0 must be an object"): + generate_brokered_tools_from_orka_tool_crds(["not-a-resource", valid_tool]) + + +def test_generate_brokered_tools_skips_unclassified_orka_tools(): + documents = [ + { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "local-only-tool"}, + "spec": { + "description": "Run only inside Orka.", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/local", "method": "POST"}, + }, + }, + { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "read-telemetry"}, + "spec": { + "description": "Read telemetry.", + "brokeredToolClass": "read", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/read", "method": "POST"}, }, }, ] + generated = generate_brokered_tools_from_orka_tool_crds(documents, include_digest=False) + + assert [(tool["name"], tool["brokeredClass"]) for tool in generated] == [("read-telemetry", "read")] + + +def test_generate_brokered_tools_rejects_unknown_canonical_brokered_tool_class(): + document = { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "admin-tool"}, + "spec": { + "description": "Attempt an unsupported operation.", + "brokeredToolClass": "admin", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/admin", "method": "POST"}, + }, + } + + with pytest.raises(ValueError, match=r"spec\.brokeredToolClass.*read.*write.*coordination"): + generate_brokered_tools_from_orka_tool_crds([document]) + + +def test_generate_brokered_tools_rejects_input_without_brokered_tools(): + document = { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "local-only-tool"}, + "spec": { + "description": "Run only inside Orka.", + "brokeredClass": "read", + "parameters": {"type": "object"}, + "http": {"url": "http://tools.default.svc/local", "method": "POST"}, + }, + } + + with pytest.raises(ValueError, match=r"no brokered Orka Tool CRDs.*spec\.brokeredToolClass"): + generate_brokered_tools_from_orka_tool_crds([document]) + + +def test_generate_brokered_tools_never_lets_legacy_aliases_override_canonical_fields(): + document = { + "apiVersion": "core.orka.ai/v1alpha1", + "kind": "Tool", + "metadata": {"name": "canonical-tool"}, + "spec": { + "name": "legacy-tool", + "description": "Canonical description.", + "summary": "Legacy description.", + "brokeredToolClass": "write", + "brokeredClass": "read", + "parameters": {"type": "object", "properties": {"canonical": {"type": "string"}}}, + "inputSchema": {"type": "object", "properties": {"legacy": {"type": "string"}}}, + "http": {"url": "http://tools.default.svc/canonical", "method": "POST"}, + }, + } + + generated = generate_brokered_tools_from_orka_tool_crds([document], include_digest=False) + + assert generated == [ + { + "name": "canonical-tool", + "description": "Canonical description.", + "brokeredClass": "write", + "parameters": {"properties": {"canonical": {"type": "string"}}, "type": "object"}, + } + ] + def test_generate_brokered_tools_from_orka_tool_crds_is_deterministic_and_schema_only(): generated = generate_brokered_tools_from_orka_tool_crds(_tool_docs()) @@ -87,6 +280,14 @@ def test_load_orka_tool_crd_files_rejects_duplicate_exported_names(tmp_path): raise AssertionError("expected duplicate brokered tool names to fail") +def test_load_orka_tool_crd_files_does_not_silently_drop_invalid_documents(tmp_path): + src = tmp_path / "tools.yaml" + src.write_text("---\nnot-a-resource\n---\n" + yaml.safe_dump(_tool_docs()[0]), encoding="utf-8") + + with pytest.raises(ValueError, match=r"document 0 must be an object"): + load_orka_tool_crd_files([src]) + + def test_brokered_tools_export_cli_writes_safe_fragment(tmp_path, capsys): src = tmp_path / "tools.yaml" out = tmp_path / "brokered-tools.yaml" @@ -101,16 +302,33 @@ def test_brokered_tools_export_cli_writes_safe_fragment(tmp_path, capsys): assert all("schemaDigest" not in entry for entry in parsed["brokeredTools"]) +def test_brokered_tools_export_cli_rejects_inputs_without_brokered_tools(tmp_path, capsys): + src = tmp_path / "tools.yaml" + out = tmp_path / "brokered-tools.yaml" + document = _tool_docs()[0] + document["spec"].pop("brokeredToolClass") + src.write_text(yaml.safe_dump(document), encoding="utf-8") + + code = main([str(src), "--output", str(out)]) + + captured = capsys.readouterr() + assert code == 2 + assert captured.out == "" + assert "no brokered Orka Tool CRDs" in captured.err + assert not out.exists() + + def test_brokered_tools_export_cli_reports_validation_errors(tmp_path, capsys): src = tmp_path / "bad.yaml" src.write_text( yaml.safe_dump( { + "apiVersion": "core.orka.ai/v1alpha1", "kind": "Tool", "metadata": {"name": "bad"}, "spec": { "description": "bad", - "brokeredClass": "read", + "brokeredToolClass": "read", "parameters": {"type": "object", "properties": {"tokenValue": {"type": "string"}}}, }, } diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 33fb514..1909818 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -8,7 +8,9 @@ from fastapi.testclient import TestClient import httpx +import pytest +from agentkit_serve_common import foundry as foundry_module from agentkit_serve_common.config import AgentSpec from agentkit_serve_common.conversation import RunRequest from agentkit_serve_common.foundry import create_foundry_app @@ -230,6 +232,29 @@ def test_foundry_brokered_rejects_normal_followup_while_previous_response_is_pen assert resp.json()["error"]["code"] == "response_pending_function_call_output" +def test_foundry_brokered_rejects_normal_followup_while_previous_response_is_resuming(monkeypatch): + stores: list[Any] = [] + original_store = foundry_module._FoundryResponseStateStore + + def capture_store(*args: Any, **kwargs: Any) -> Any: + store = original_store(*args, **kwargs) + stores.append(store) + return store + + monkeypatch.setattr(foundry_module, "_FoundryResponseStateStore", capture_store) + app = _app() + + with TestClient(app) as client: + initial = _start(client) + state = stores[0].get(initial["id"]) + state.status = "resuming" + stores[0].save(state) + resp = client.post("/responses", json={"previous_response_id": initial["id"], "input": "next question"}) + + assert resp.status_code == 409 + assert resp.json()["error"]["code"] == "response_pending_function_call_output" + + def test_foundry_brokered_pending_state_store_is_bounded(): app = _app(max_pending_responses=1) @@ -844,6 +869,40 @@ def test_foundry_brokered_refuses_multi_value_enum_write_arguments(): assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" +@pytest.mark.parametrize("brokered_class", ["write", "coordination"]) +def test_foundry_brokered_refuses_multi_value_root_enum_side_effecting_arguments(brokered_class: str): + spec = _spec(tool_name="dispatch-work-order", brokered_class=brokered_class) + spec.brokered_tools[0].parameters = { + "type": "object", + "enum": [ + {"operation": "delete"}, + {"operation": "create"}, + ], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_allows_single_value_root_enum_write_arguments(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "enum": [{"operation": "create"}], + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order"}) + + assert resp.status_code == 200 + assert json.loads(_call(resp.json())["arguments"]) == {"operation": "create"} + + def test_foundry_brokered_allows_single_value_enum_write_arguments(): spec = _spec(tool_name="dispatch-work-order", brokered_class="write") spec.brokered_tools[0].parameters = { @@ -860,6 +919,37 @@ def test_foundry_brokered_allows_single_value_enum_write_arguments(): assert json.loads(_call(resp.json())["arguments"]) == {"operation": "create"} +@pytest.mark.parametrize("brokered_class", ["write", "coordination"]) +def test_foundry_brokered_refuses_nonliteral_optional_prompt_for_side_effecting_tools(brokered_class: str): + spec = _spec(tool_name="dispatch-work-order", brokered_class=brokered_class) + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"prompt": {"type": "string"}}, + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order with arbitrary payload"}) + + assert resp.status_code == 400 + assert resp.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_allows_literal_optional_prompt_for_write_tools(): + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"prompt": {"type": "string", "const": "fixed"}}, + } + app = _app(spec) + + with TestClient(app) as client: + resp = client.post("/responses", json={"input": "dispatch-work-order with arbitrary payload"}) + + assert resp.status_code == 200 + assert json.loads(_call(resp.json())["arguments"]) == {"prompt": "fixed"} + + def test_foundry_brokered_allows_literal_write_arguments_for_conformance(): spec = _spec(tool_name="dispatch-work-order", brokered_class="write") spec.brokered_tools[0].parameters = { From 82a98ae9480a875b8c378126bc2f889e65bbb934 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 19:04:21 -0700 Subject: [PATCH 07/27] fix(foundry): address follow-up review feedback Signed-off-by: Sertac Ozercan --- docs/foundry-hosted-brokered.md | 5 ++-- pkg/agentkit/config/config_test.go | 26 ++++++++++++++++++- pkg/agentkit/config/validate.go | 12 ++++++--- runtimes/common/README.md | 5 +++- .../common/agentkit_serve_common/config.py | 8 ++++-- .../common/agentkit_serve_common/foundry.py | 2 +- runtimes/common/pyproject.toml | 6 +++-- .../common/tests/test_config_validation.py | 25 ++++++++++++++++++ .../common/tests/test_foundry_conformance.py | 13 ++++++++++ test/foundry-brokered-conformance/Dockerfile | 6 ++++- 10 files changed, 94 insertions(+), 14 deletions(-) diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md index 383c355..a00fc72 100644 --- a/docs/foundry-hosted-brokered.md +++ b/docs/foundry-hosted-brokered.md @@ -226,7 +226,7 @@ It returns the deterministic `conformance_read` function call on the first Local runnable entrypoint check: ```sh -uv run --directory runtimes/common --extra dev agentkit-foundry-conformance --dry-run +uv run --directory runtimes/common --extra dev --extra foundry-conformance agentkit-foundry-conformance --dry-run ``` Container entrypoint example for the A0 spike image: @@ -262,7 +262,7 @@ deploy/foundry/scripts/local_brokered_conformance_container.sh \ Local proof: ```sh -uv run --directory runtimes/common --extra dev pytest -q tests/test_foundry_conformance.py +uv run --directory runtimes/common --extra dev --extra foundry-conformance pytest -q tests/test_foundry_conformance.py ``` Live direct-endpoint proof after deployment: @@ -295,6 +295,7 @@ Local proof for this production adapter path: ```sh docker build . -f test/foundry-brokered-agentkit/Dockerfile -t agentkit-foundry-brokered:local docker run --rm \ + -e AGENTKIT_AUTH_TOKEN=local-dummy-token \ -e AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF=local-dev-proof \ -p 127.0.0.1:18092:8088 \ agentkit-foundry-brokered:local diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index 708d47b..972dc32 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -494,7 +494,7 @@ func TestValidateRejectsInvalidBrokeredSchemaTypeValuesAndDefaults(t *testing.T) cases := []map[string]any{ {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: nil}}}, {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{"n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: "1"}}}, - {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, "enum": []any{0, "ok"}}}}, + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaEnumKey: []any{0, "ok"}}}}, } for _, schema := range cases { cfg := validMinimalConfig() @@ -510,6 +510,30 @@ func TestValidateRejectsInvalidBrokeredSchemaTypeValuesAndDefaults(t *testing.T) } } +func TestValidateRejectsEmptyBrokeredSchemaEnums(t *testing.T) { + cases := []map[string]any{ + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaEnumKey: []any{}}, + { + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaEnumKey: []any{}}, + }, + }, + } + for _, schema := range cases { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: schema, + }} + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "enum must contain at least one value") { + t.Fatalf("expected empty enum rejection for %#v, got: %v", schema, err) + } + } +} + func TestValidateRejectsInvalidRemoteMCPToolShapes(t *testing.T) { cases := map[string]string{ //nolint:gosec // test YAML uses credential-looking field names/invalid examples, not real secrets "missing type": `tools: diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index 2c7cb51..e344adb 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -45,6 +45,7 @@ const ( jsonSchemaTypeArray = "array" jsonSchemaTypeBoolean = "boolean" jsonSchemaTypeNull = "null" + jsonSchemaEnumKey = "enum" jsonSchemaDefaultKey = "default" jsonSchemaRequiredKey = "required" jsonSchemaDependentRequiredKey = "dependentRequired" @@ -326,9 +327,12 @@ func validateJSONSchemaSubset(add func(string, ...any), path string, schema map[ } } } - if enumValue, ok := schema["enum"]; ok { - if _, ok := enumValue.([]any); !ok { + if enumValue, ok := schema[jsonSchemaEnumKey]; ok { + items, ok := enumValue.([]any) + if !ok { add("%s.enum must be an array", path) + } else if len(items) == 0 { + add("%s.enum must contain at least one value", path) } } if _, ok := schema["pattern"]; ok { @@ -343,7 +347,7 @@ func validateJSONSchemaSubset(add func(string, ...any), path string, schema map[ add("%s.additionalProperties must be a boolean or object", path) } } - if hasAnySchemaKey(schema, "enum", "const", "default") && hasAnySchemaKey(schema, jsonSchemaMinimumKey, "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties", "pattern") { + if hasAnySchemaKey(schema, jsonSchemaEnumKey, "const", "default") && hasAnySchemaKey(schema, jsonSchemaMinimumKey, "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties", "pattern") { add("%s combines enum/const/default with constraints unsupported by deterministic brokered synthesis", path) } if _, ok := schema["multipleOf"]; ok { @@ -414,7 +418,7 @@ func validateBrokeredSchemaValueConstraints(add func(string, ...any), path strin add("%s.%s must match the declared JSON Schema type", path, keyword) } } - if enum, exists := schema["enum"]; exists { + if enum, exists := schema[jsonSchemaEnumKey]; exists { items, ok := enum.([]any) if !ok { add("%s.enum must be an array", path) diff --git a/runtimes/common/README.md b/runtimes/common/README.md index 7de57fc..3c9816c 100644 --- a/runtimes/common/README.md +++ b/runtimes/common/README.md @@ -80,6 +80,9 @@ Azure Responses SDK app for Phase A0 hosted brokered smokes. It serves `function_call`, and completes after a matching `function_call_output` continuation. +Install the optional `foundry-conformance` extra when using this SDK-backed +entrypoint; normal runtime adapter images install the common package without it. + ```sh -agentkit-foundry-conformance --host 0.0.0.0 --port 8088 +uv run --extra foundry-conformance agentkit-foundry-conformance --host 0.0.0.0 --port 8088 ``` diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index f65a726..a4c77d5 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -317,8 +317,12 @@ def _validate_json_schema_subset(schema: Any, *, path: str) -> None: for key, value in dependent_required.items(): if not isinstance(key, str) or not isinstance(value, list) or not all(isinstance(item, str) for item in value): raise ValueError(f"{path}.dependentRequired values must be string arrays") - if "enum" in schema and not isinstance(schema["enum"], list): - raise ValueError(f"{path}.enum must be an array") + if "enum" in schema: + enum_values = schema["enum"] + if not isinstance(enum_values, list): + raise ValueError(f"{path}.enum must be an array") + if not enum_values: + raise ValueError(f"{path}.enum must contain at least one value") if "pattern" in schema: raise ValueError(f"{path}.pattern is not supported for deterministic brokered tool schemas") if "additionalProperties" in schema: diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index f08d4a5..f502090 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -43,7 +43,7 @@ try: # Prefer the official hosted Responses SDK ID format/state-compatible prefix. from azure.ai.agentserver.responses._id_generator import IdGenerator as _AzureResponsesIdGenerator -except Exception: # pragma: no cover - dependency is declared; fallback is for source-tree imports only. +except Exception: # pragma: no cover - the SDK is optional outside conformance installs. _AzureResponsesIdGenerator = None _DEFAULT_STATE_TTL_SECONDS = 15 * 60 diff --git a/runtimes/common/pyproject.toml b/runtimes/common/pyproject.toml index 4d414df..b011082 100644 --- a/runtimes/common/pyproject.toml +++ b/runtimes/common/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "pydantic>=2.7", "pyyaml>=6.0", "httpx>=0.28", - "azure-ai-agentserver-responses>=1.0.0b8", ] [project.scripts] @@ -29,8 +28,11 @@ agentkit-foundry-brokered = "agentkit_serve_common.foundry_brokered_cli:main" agentkit-foundry-conformance = "agentkit_serve_common.foundry_conformance:main" [project.optional-dependencies] +foundry-conformance = ["azure-ai-agentserver-responses>=1.0.0b8"] # Starlette/FastAPI TestClient currently imports the httpx2 compatibility package. -dev = ["pytest>=8.0", "httpx2>=0.1"] +# Keep the Foundry SDK here as well because the common test suite exercises the +# optional conformance app. Runtime adapter images install the base package only. +dev = ["pytest>=8.0", "httpx2>=0.1", "azure-ai-agentserver-responses>=1.0.0b8"] [tool.hatch.build.targets.wheel] packages = ["agentkit_serve_common"] diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index 21e60fd..ad01574 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -750,6 +750,31 @@ def test_load_rejects_explicit_null_brokered_json_schema_keywords(tmp_path, bad_ assert "brokeredTools.0.parameters" in msg +@pytest.mark.parametrize( + "bad_parameters", + [ + {"type": "object", "enum": []}, + {"type": "object", "properties": {"site": {"type": "string", "enum": []}}}, + ], +) +def test_load_rejects_empty_brokered_json_schema_enums(tmp_path, bad_parameters: dict): + msg = _invalid_message( + tmp_path, + lambda spec: spec.update( + tools=[], + brokeredTools=[ + { + "name": "safe_lookup", + "description": "safe schema", + "brokeredClass": "read", + "parameters": bad_parameters, + } + ], + ), + ) + assert "enum must contain at least one value" in msg + + @pytest.mark.parametrize("bad_child", [{"type": 123}, {"type": "strnig"}, {"items": "bad"}, {"items": [{"type": "number"}]}, {"multipleOf": 2}, {"uniqueItems": True}, {"minLength": -1}]) def test_load_rejects_malformed_nested_brokered_json_schema(tmp_path, bad_child: dict): msg = _invalid_message( diff --git a/runtimes/common/tests/test_foundry_conformance.py b/runtimes/common/tests/test_foundry_conformance.py index 7147fab..8c2fae7 100644 --- a/runtimes/common/tests/test_foundry_conformance.py +++ b/runtimes/common/tests/test_foundry_conformance.py @@ -1,11 +1,24 @@ from __future__ import annotations import json +import tomllib +from pathlib import Path from fastapi.testclient import TestClient from agentkit_serve_common.foundry_conformance import create_foundry_conformance_app +def test_foundry_conformance_sdk_dependency_is_optional(): + project = tomllib.loads((Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8"))["project"] + sdk_dependency = "azure-ai-agentserver-responses" + + assert not any(dependency.startswith(sdk_dependency) for dependency in project["dependencies"]) + assert any( + dependency.startswith(sdk_dependency) + for dependency in project["optional-dependencies"]["foundry-conformance"] + ) + + def _function_output(previous_response_id: str | None, call_id: str = "call_conformance_1") -> dict: payload = { "input": [ diff --git a/test/foundry-brokered-conformance/Dockerfile b/test/foundry-brokered-conformance/Dockerfile index 0e8518a..75bf8ab 100644 --- a/test/foundry-brokered-conformance/Dockerfile +++ b/test/foundry-brokered-conformance/Dockerfile @@ -8,7 +8,11 @@ ENV PORT=8088 \ WORKDIR /opt/agentkit COPY runtimes/common /tmp/agentkit-serve-common -RUN pip install --no-cache-dir /tmp/agentkit-serve-common \ +# Keep the Azure Responses SDK out of generic runtime images; this dedicated +# conformance fixture opts into it explicitly. +RUN cd /tmp/agentkit-serve-common \ + && pip install --no-cache-dir '.[foundry-conformance]' \ + && cd /opt/agentkit \ && rm -rf /tmp/agentkit-serve-common EXPOSE 8088 From 528e8ffbf367a9d899420190768204a159a4e51f Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 19:28:24 -0700 Subject: [PATCH 08/27] fix(foundry): bound brokered continuation outputs Signed-off-by: Sertac Ozercan --- docs/foundry-hosted-brokered.md | 8 +- .../common/agentkit_serve_common/foundry.py | 29 +++- .../foundry_model_loop.py | 6 +- .../tests/test_foundry_brokered_protocol.py | 124 ++++++++++++++++++ 4 files changed, 159 insertions(+), 8 deletions(-) diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md index a00fc72..4f484e0 100644 --- a/docs/foundry-hosted-brokered.md +++ b/docs/foundry-hosted-brokered.md @@ -184,8 +184,11 @@ safely with `unknown_previous_response_id`. Pending state expires after continuations fail with `response_state_expired`. The store is bounded by `AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING` (default: 128), and generated brokered arguments are bounded by `AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES` -(default: 8192) before state is accepted. A platform-managed state backend is -still required before treating multi-replica production as fully supported. +(default: 8192) before state is accepted. Brokered continuation outputs are +bounded by `AGENTKIT_FOUNDRY_BROKERED_MAX_OUTPUT_BYTES` (default: 65536) before +they are persisted, replayed, embedded in deterministic responses, or sent back +through the model loop. A platform-managed state backend is still required +before treating multi-replica production as fully supported. ## Streaming @@ -207,6 +210,7 @@ smokes deterministic while making streaming support an explicit future step. - `brokered_tool_selection_required`: name exactly one configured brokered tool in deterministic mode when multiple schemas or any non-conformance single schema are configured. - `brokered_response_state_full`: too many uncontinued brokered responses are pending. Completed entries are evicted before this error is returned. - `brokered_arguments_too_large`: generated brokered call arguments exceeded the configured pending-state byte budget. +- `brokered_output_too_large`: a brokered `function_call_output` exceeded the configured output byte budget. - `unknown_call_id`: the output did not match the pending function call. - `conflicting_duplicate_continuation`: the same `call_id` was already completed with different output. diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index f502090..44bfe6a 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -49,11 +49,13 @@ _DEFAULT_STATE_TTL_SECONDS = 15 * 60 _DEFAULT_MAX_PENDING_RESPONSES = 128 _DEFAULT_MAX_ARGUMENT_BYTES = 8192 +_DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024 _MAX_SYNTHETIC_ARRAY_ITEMS = 32 _MAX_SYNTHETIC_STRING_LENGTH = 4096 _STATE_TTL_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS" _MAX_PENDING_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING" _MAX_ARGUMENT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES" +_MAX_OUTPUT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_OUTPUT_BYTES" _CONTINUATION_PROOF_ENV = "AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF" _CONTINUATION_PROOF_HEADER = "x-agentkit-brokered-continuation-proof" _MODEL_LOOP_ENV = "AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP" @@ -486,7 +488,8 @@ def _persist(self) -> None: def _state_ttl_seconds(value: float | None = None) -> float: if value is not None: - return max(float(value), 0.0) + parsed = float(value) + return max(parsed, 0.0) if math.isfinite(parsed) else float(_DEFAULT_STATE_TTL_SECONDS) raw = os.environ.get(_STATE_TTL_ENV) if not raw: return float(_DEFAULT_STATE_TTL_SECONDS) @@ -494,7 +497,7 @@ def _state_ttl_seconds(value: float | None = None) -> float: parsed = float(raw) except ValueError: return float(_DEFAULT_STATE_TTL_SECONDS) - return max(parsed, 0.0) + return max(parsed, 0.0) if math.isfinite(parsed) else float(_DEFAULT_STATE_TTL_SECONDS) def _positive_int_setting(value: int | None, *, env_name: str, default: int) -> int: @@ -518,6 +521,10 @@ def _max_argument_bytes(value: int | None = None) -> int: return _positive_int_setting(value, env_name=_MAX_ARGUMENT_BYTES_ENV, default=_DEFAULT_MAX_ARGUMENT_BYTES) +def _max_output_bytes(value: int | None = None) -> int: + return _positive_int_setting(value, env_name=_MAX_OUTPUT_BYTES_ENV, default=_DEFAULT_MAX_OUTPUT_BYTES) + + def _brokered_model_loop_enabled(value: bool | None = None) -> bool: if value is not None: return value @@ -1145,6 +1152,7 @@ async def _handle_brokered_continuation( input_value: Any, continuation_proof: str | None, request: Request, + max_output_bytes: int, model_loop: BrokeredChatModelLoop | None = None, ) -> JSONResponse: if not continuation_proof: @@ -1198,6 +1206,13 @@ async def _handle_brokered_continuation( except ValueError as exc: return _error(str(exc), status=400, code="invalid_function_call_output") output_json = _canonical_output_json(parsed_output) + output_size = len(output_json.encode("utf-8")) + if output_size > max_output_bytes: + return _error( + "brokered function_call_output is too large", + status=413, + code="brokered_output_too_large", + ) existing_output = state.accepted_outputs.get(call_id) if existing_output is not None: @@ -1272,6 +1287,7 @@ def create_foundry_app( brokered_continuation_proof: str | None = None, max_pending_responses: int | None = None, max_brokered_argument_bytes: int | None = None, + max_brokered_output_bytes: int | None = None, brokered_model_loop_enabled: bool | None = None, brokered_model_http_client: Any | None = None, response_state_file: str | Path | None = None, @@ -1281,13 +1297,19 @@ def create_foundry_app( runtime = None if brokered_tools else factory.build_runtime(spec) continuation_proof = brokered_continuation_proof or os.environ.get(_CONTINUATION_PROOF_ENV) or None max_argument_bytes = _max_argument_bytes(max_brokered_argument_bytes) + max_output_bytes = _max_output_bytes(max_brokered_output_bytes) response_states = _FoundryResponseStateStore( ttl_seconds=_state_ttl_seconds(state_ttl_seconds), max_entries=_max_pending_responses(max_pending_responses), state_file=_response_state_file(response_state_file) if brokered_tools else None, ) model_loop = ( - BrokeredChatModelLoop(spec, brokered_tools, http_client=brokered_model_http_client) + BrokeredChatModelLoop( + spec, + brokered_tools, + http_client=brokered_model_http_client, + max_output_bytes=max_output_bytes, + ) if brokered_tools and _brokered_model_loop_enabled(brokered_model_loop_enabled) else None ) @@ -1391,6 +1413,7 @@ async def responses(request: Request): input_value=data["input"], continuation_proof=continuation_proof, request=request, + max_output_bytes=max_output_bytes, model_loop=model_loop, ) if brokered_tools and isinstance(previous_response_id, str) and previous_response_id: diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index 7b1011a..dca61dd 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -47,12 +47,12 @@ def __init__( tools: Sequence[BrokeredToolDefinition], *, http_client: httpx.AsyncClient | None = None, - max_output_chars: int = 64_000, + max_output_bytes: int = 64 * 1024, ) -> None: self.spec = spec self.tools = list(tools) self.http_client = http_client - self.max_output_chars = max_output_chars + self.max_output_bytes = max_output_bytes self.tools_by_name = {tool.name: tool for tool in self.tools} async def start(self, request: RunRequest, *, call_id: str) -> ModelLoopFinal | ModelLoopToolRequest: @@ -95,7 +95,7 @@ async def start(self, request: RunRequest, *, call_id: str) -> ModelLoopFinal | return ModelLoopToolRequest(name=name, arguments=arguments, messages=[*messages, assistant_message], usage=usage) async def resume(self, messages: Sequence[Mapping[str, Any]], *, call_id: str, output: str) -> ModelLoopFinal: - if len(output) > self.max_output_chars: + if len(output.encode("utf-8")) > self.max_output_bytes: raise AgentRunError("brokered tool output is too large for model resume", status=413, code="brokered_output_too_large") resumed = [dict(message) for message in messages] resumed.append({"role": "tool", "tool_call_id": call_id, "content": output}) diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 1909818..f95e4eb 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -178,6 +178,29 @@ def test_foundry_brokered_initial_response_emits_static_function_call_without_di assert factory.runtime.run_requests == [] +@pytest.mark.parametrize("ttl_seconds", [float("nan"), float("inf"), float("-inf")]) +def test_foundry_brokered_nonfinite_explicit_state_ttl_uses_default(ttl_seconds: float): + app = _app(state_ttl_seconds=ttl_seconds) + + with TestClient(app) as client: + readiness = client.get("/readiness") + + assert readiness.status_code == 200 + assert readiness.json()["foundryResponses"]["stateTtlSeconds"] == 900.0 + + +@pytest.mark.parametrize("raw_ttl", ["nan", "inf", "-inf"]) +def test_foundry_brokered_nonfinite_state_ttl_env_uses_default(monkeypatch, raw_ttl: str): + monkeypatch.setenv("AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS", raw_ttl) + app = _app() + + with TestClient(app) as client: + readiness = client.get("/readiness") + + assert readiness.status_code == 200 + assert readiness.json()["foundryResponses"]["stateTtlSeconds"] == 900.0 + + def test_foundry_brokered_deterministic_arguments_reject_unsafe_prompt_text(): spec = _spec(tool_name="check-network-telemetry") spec.brokered_tools[0].parameters = { @@ -643,6 +666,39 @@ def test_foundry_brokered_rejects_nonfinite_function_call_output_values(): assert response.json()["error"]["code"] == "invalid_function_call_output" +def test_foundry_brokered_rejects_oversized_function_call_output_before_state_change(tmp_path): + state_file = tmp_path / "responses-state.json" + app = _app(response_state_file=state_file, max_brokered_output_bytes=128) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + persisted_before = state_file.read_bytes() + oversized = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation( + initial["id"], + call["call_id"], + {"approved": True, "output": {"blob": "x" * 256}}, + ), + ) + persisted_after_rejection = state_file.read_bytes() + accepted = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + + assert oversized.status_code == 413 + assert oversized.json()["error"] == { + "message": "brokered function_call_output is too large", + "code": "brokered_output_too_large", + } + assert persisted_after_rejection == persisted_before + assert accepted.status_code == 200, accepted.text + + def test_foundry_brokered_continuation_accepts_matching_tool_output_and_completes(): app = _app() @@ -749,6 +805,27 @@ def test_foundry_brokered_duplicate_continuation_is_idempotent_but_conflicts_are assert conflict.json()["error"]["code"] == "conflicting_duplicate_continuation" +def test_foundry_brokered_rejects_oversized_persisted_replay_after_limit_is_lowered(tmp_path): + state_file = tmp_path / "responses-state.json" + + with TestClient(_app(response_state_file=state_file, max_brokered_output_bytes=1024)) as client: + initial = _start(client) + call = _call(initial) + payload = _continuation( + initial["id"], + call["call_id"], + {"approved": True, "output": {"blob": "x" * 256}}, + ) + completed = client.post("/responses", json=payload, headers=CONTINUATION_AUTH) + + with TestClient(_app(response_state_file=state_file, max_brokered_output_bytes=128)) as client: + replay = client.post("/responses", json=payload, headers=CONTINUATION_AUTH) + + assert completed.status_code == 200, completed.text + assert replay.status_code == 413 + assert replay.json()["error"]["code"] == "brokered_output_too_large" + + def test_foundry_brokered_file_state_survives_restart_for_deterministic_continuation(tmp_path): state_file = tmp_path / "foundry-state.json" @@ -1093,6 +1170,53 @@ def test_foundry_brokered_model_loop_emits_model_requested_tool_and_resumes_to_f assert "tools" not in fake.requests[1] +def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_state_change(tmp_path): + state_file = tmp_path / "responses-state.json" + spec = _spec(tool_name="check-network-telemetry") + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ) + ] + ) + app = _model_loop_app( + spec, + fake, + response_state_file=state_file, + max_brokered_output_bytes=128, + ) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + persisted_before = state_file.read_bytes() + oversized = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation( + initial.json()["id"], + call["call_id"], + {"approved": True, "output": {"blob": "x" * 256}}, + ), + ) + + assert oversized.status_code == 413 + assert oversized.json()["error"]["code"] == "brokered_output_too_large" + assert state_file.read_bytes() == persisted_before + assert len(fake.requests) == 1 + + def test_foundry_brokered_invalid_file_state_starts_with_empty_store(tmp_path): state_file = tmp_path / "responses-state.json" state_file.write_text("{not valid json", encoding="utf-8") From 021c83d80a2f5a3bd74e0401933094a8c3c95695 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 19:39:30 -0700 Subject: [PATCH 09/27] fix(foundry): reject lossy tool output numbers Signed-off-by: Sertac Ozercan --- .../common/agentkit_serve_common/foundry.py | 21 ++++++++- .../tests/test_foundry_brokered_protocol.py | 45 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 44bfe6a..12cc6be 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -564,10 +564,29 @@ def _reject_nonfinite_json_values(value: Any, *, path: str = "value") -> None: _reject_nonfinite_json_values(child, path=f"{path}[{idx}]") +def _parse_output_float(raw: str) -> float: + try: + decimal = Decimal(raw) + except InvalidOperation as exc: + raise ValueError("function_call_output.output must contain valid JSON numbers") from exc + parsed = float(decimal) + if not math.isfinite(parsed) or Decimal(str(parsed)) != decimal: + raise ValueError("function_call_output.output contains a number that cannot be represented exactly") + return parsed + + +def _reject_output_constant(raw: str) -> None: + raise ValueError(f"function_call_output.output contains non-finite number {raw}") + + def _json_object_from_output(output: Any) -> dict[str, Any]: if isinstance(output, str): try: - parsed = json.loads(output) + parsed = json.loads( + output, + parse_float=_parse_output_float, + parse_constant=_reject_output_constant, + ) except json.JSONDecodeError as exc: raise ValueError("function_call_output.output must be a JSON object string") from exc else: diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index f95e4eb..9367fdc 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -666,6 +666,51 @@ def test_foundry_brokered_rejects_nonfinite_function_call_output_values(): assert response.json()["error"]["code"] == "invalid_function_call_output" +def test_foundry_brokered_rejects_lossy_function_call_output_float_before_state_change(tmp_path): + state_file = tmp_path / "responses-state.json" + app = _app(response_state_file=state_file) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + persisted_before = state_file.read_bytes() + lossy = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": '{"approved":true,"output":{"id":9007199254740993.0}}', + } + ], + }, + ) + persisted_after_rejection = state_file.read_bytes() + exact = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": '{"approved":true,"output":{"id":9007199254740992.0}}', + } + ], + }, + ) + + assert lossy.status_code == 400 + assert lossy.json()["error"]["code"] == "invalid_function_call_output" + assert persisted_after_rejection == persisted_before + assert exact.status_code == 200, exact.text + assert _message_text(exact.json()).endswith('{"id":9007199254740992.0}') + + def test_foundry_brokered_rejects_oversized_function_call_output_before_state_change(tmp_path): state_file = tmp_path / "responses-state.json" app = _app(response_state_file=state_file, max_brokered_output_bytes=128) From 6e61abb46ff1afb51f255d18f8a859421d7d8ac0 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 19:51:19 -0700 Subject: [PATCH 10/27] fix(brokered): harden validation and fixtures Signed-off-by: Sertac Ozercan --- pkg/agentkit/config/config_test.go | 22 ++++++++++ pkg/agentkit/config/validate.go | 2 +- .../foundry_model_loop.py | 41 +++++++++++++++++-- .../tests/test_foundry_brokered_protocol.py | 26 ++++++++++++ test/foundry-brokered-agentkit/Dockerfile | 3 ++ test/foundry-brokered-conformance/Dockerfile | 5 ++- 6 files changed, 93 insertions(+), 6 deletions(-) diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index 972dc32..00a32d0 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -534,6 +534,28 @@ func TestValidateRejectsEmptyBrokeredSchemaEnums(t *testing.T) { } } +func TestValidateAcceptsLargeBrokeredIntegerConstraintsWithoutInt64Narrowing(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + "minProperties": 1e19, + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("large integral JSON Schema constraint should validate: %v", err) + } + + cfg.BrokeredTools[0].Parameters["minProperties"] = 1.5 + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must be a non-negative integer") { + t.Fatalf("fractional JSON Schema constraint should fail validation, got: %v", err) + } +} + func TestValidateRejectsInvalidRemoteMCPToolShapes(t *testing.T) { cases := map[string]string{ //nolint:gosec // test YAML uses credential-looking field names/invalid examples, not real secrets "missing type": `tools: diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index e344adb..6bf8c46 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -363,7 +363,7 @@ func validateJSONSchemaSubset(add func(string, ...any), path string, schema map[ for _, key := range []string{"minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties"} { if value, ok := schema[key]; ok { number, ok := value.(float64) - if !ok || number < 0 || number != float64(int64(number)) { + if !ok || math.IsNaN(number) || math.IsInf(number, 0) || number < 0 || math.Trunc(number) != number { add("%s.%s must be a non-negative integer", path, key) } } diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index dca61dd..3e849ab 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -11,6 +11,7 @@ import asyncio import json +import math import os from dataclasses import dataclass, field from decimal import Decimal, InvalidOperation @@ -230,12 +231,44 @@ def _reject_json_constant(raw: str) -> None: raise AgentRunError(f"model tool arguments contain non-finite number {raw}", status=400, code="InvalidToolArguments") +def _usage_token_count(value: Any) -> int: + if value is None or isinstance(value, bool): + if value is None: + return 0 + raise AgentRunError( + "model response usage must contain non-negative integer token counts", + status=502, + code="InvalidModelResponse", + ) + if isinstance(value, float) and (not math.isfinite(value) or not value.is_integer()): + raise AgentRunError( + "model response usage must contain non-negative integer token counts", + status=502, + code="InvalidModelResponse", + ) + try: + count = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise AgentRunError( + "model response usage must contain non-negative integer token counts", + status=502, + code="InvalidModelResponse", + ) from exc + if count < 0: + raise AgentRunError( + "model response usage must contain non-negative integer token counts", + status=502, + code="InvalidModelResponse", + ) + return count + + def _usage(data: Mapping[str, Any]) -> dict[str, int]: usage = data.get("usage") if isinstance(data.get("usage"), Mapping) else {} - prompt_tokens = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) - completion_tokens = int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0) - total_tokens = int(usage.get("total_tokens", prompt_tokens + completion_tokens) or 0) - return {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + prompt_count = _usage_token_count(usage.get("prompt_tokens", usage.get("input_tokens", 0))) + completion_count = _usage_token_count(usage.get("completion_tokens", usage.get("output_tokens", 0))) + total_count = _usage_token_count(usage.get("total_tokens", prompt_count + completion_count)) + return {"prompt_tokens": prompt_count, "completion_tokens": completion_count, "total_tokens": total_count} __all__ = ["BrokeredChatModelLoop", "ModelLoopFinal", "ModelLoopToolRequest"] diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 9367fdc..c0b8509 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1166,6 +1166,32 @@ def _model_loop_app(spec: AgentSpec, fake: _FakeChatTransport, **kwargs: Any): return _app(spec, brokered_model_loop_enabled=True, brokered_model_http_client=client, **kwargs) +@pytest.mark.parametrize( + "usage", + [ + {"prompt_tokens": {"unexpected": 1}}, + {"completion_tokens": "not-a-number"}, + {"total_tokens": 1.5}, + ], +) +def test_foundry_brokered_model_loop_normalizes_malformed_usage(usage: dict[str, Any]): + fake = _FakeChatTransport( + [ + { + "choices": [{"message": {"role": "assistant", "content": "done"}}], + "usage": usage, + } + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "InvalidModelResponse" + + def test_foundry_brokered_model_loop_emits_model_requested_tool_and_resumes_to_final_answer(): spec = _spec(tool_name="check-network-telemetry") spec.brokered_tools[0].parameters["required"] = ["site"] diff --git a/test/foundry-brokered-agentkit/Dockerfile b/test/foundry-brokered-agentkit/Dockerfile index 6bc0d54..0bb71de 100644 --- a/test/foundry-brokered-agentkit/Dockerfile +++ b/test/foundry-brokered-agentkit/Dockerfile @@ -11,6 +11,9 @@ COPY runtimes/common /tmp/agentkit-serve-common RUN pip install --no-cache-dir /tmp/agentkit-serve-common \ && rm -rf /tmp/agentkit-serve-common COPY test/foundry-brokered-agentkit/agent.yaml /agent/agent.yaml +RUN useradd --uid 1000 --create-home --shell /usr/sbin/nologin agentkit \ + && chown -R 1000:1000 /agent /opt/agentkit EXPOSE 8088 +USER 1000 ENTRYPOINT ["agentkit-foundry-brokered", "--host", "0.0.0.0", "--port", "8088", "--config", "/agent/agent.yaml"] diff --git a/test/foundry-brokered-conformance/Dockerfile b/test/foundry-brokered-conformance/Dockerfile index 75bf8ab..7bc751c 100644 --- a/test/foundry-brokered-conformance/Dockerfile +++ b/test/foundry-brokered-conformance/Dockerfile @@ -13,7 +13,10 @@ COPY runtimes/common /tmp/agentkit-serve-common RUN cd /tmp/agentkit-serve-common \ && pip install --no-cache-dir '.[foundry-conformance]' \ && cd /opt/agentkit \ - && rm -rf /tmp/agentkit-serve-common + && rm -rf /tmp/agentkit-serve-common \ + && useradd --uid 1000 --create-home --shell /usr/sbin/nologin agentkit \ + && chown -R 1000:1000 /opt/agentkit EXPOSE 8088 +USER 1000 ENTRYPOINT ["agentkit-foundry-conformance", "--host", "0.0.0.0", "--port", "8088"] From 5ce2d5b44a5e0da767539238f080e10e474a5ea4 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 20:01:21 -0700 Subject: [PATCH 11/27] fix(brokered): preserve schema and transcript fidelity Signed-off-by: Sertac Ozercan --- .../scripts/verify_brokered_transcript.py | 1 + pkg/agentkit/abi/render.go | 17 +++++-- pkg/agentkit/abi/render_test.go | 45 ++++++++++++++++--- .../common/agentkit_serve_common/config.py | 4 +- .../common/agentkit_serve_common/foundry.py | 21 +++++---- .../common/tests/test_config_validation.py | 29 ++++++++++++ .../tests/test_foundry_brokered_protocol.py | 28 ++++++++++++ .../tests/test_foundry_transcript_verifier.py | 13 ++++++ 8 files changed, 136 insertions(+), 22 deletions(-) diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index 59f05c4..e32f650 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -107,6 +107,7 @@ def verify_transcript( _require(continuation_response.get("previous_response_id") == initial_response_id, "continuation response previous_response_id must match initial id") continuation_response_id = continuation_response.get("id") _require(isinstance(continuation_response_id, str) and continuation_response_id.startswith("caresp_"), "continuation response id must start with caresp_") + _require(continuation_response_id != initial_response_id, "continuation response id must differ from initial response id") final_text = _message_text(continuation_response) return { diff --git a/pkg/agentkit/abi/render.go b/pkg/agentkit/abi/render.go index fd88fd1..f86f472 100644 --- a/pkg/agentkit/abi/render.go +++ b/pkg/agentkit/abi/render.go @@ -4,6 +4,7 @@ package abi import ( "encoding/json" + "math" "strconv" "strings" @@ -29,6 +30,14 @@ func (n yamlNumber) MarshalYAML() ([]byte, error) { return []byte(n), nil } +func yamlFloat(value float64, bitSize int) yamlNumber { + rendered := strconv.FormatFloat(value, 'f', -1, bitSize) + if value == 0 && math.Signbit(value) { + rendered = "-0.0" + } + return yamlNumber(rendered) +} + type abiMetadata struct { Name string `yaml:"name"` } @@ -199,7 +208,7 @@ func copyAny(v any) any { case map[string]float64: out := make(map[string]any, len(typed)) for key, value := range typed { - out[key] = yamlNumber(strconv.FormatFloat(value, 'f', -1, 64)) + out[key] = yamlFloat(value, 64) } return out case map[string]bool: @@ -221,15 +230,15 @@ func copyAny(v any) any { case []float64: out := make([]any, len(typed)) for i, item := range typed { - out[i] = yamlNumber(strconv.FormatFloat(item, 'f', -1, 64)) + out[i] = yamlFloat(item, 64) } return out case []bool: return append([]bool(nil), typed...) case float32: - return yamlNumber(strconv.FormatFloat(float64(typed), 'f', -1, 32)) + return yamlFloat(float64(typed), 32) case float64: - return yamlNumber(strconv.FormatFloat(typed, 'f', -1, 64)) + return yamlFloat(typed, 64) case json.Number: return yamlNumber(expandJSONNumber(typed.String())) default: diff --git a/pkg/agentkit/abi/render_test.go b/pkg/agentkit/abi/render_test.go index e4848fa..9149f7c 100644 --- a/pkg/agentkit/abi/render_test.go +++ b/pkg/agentkit/abi/render_test.go @@ -2,6 +2,7 @@ package abi import ( "encoding/json" + "math" "os" "strings" "testing" @@ -20,8 +21,10 @@ const ( ) const ( - jsonSchemaTypeKey = "type" - jsonSchemaMinimumKey = "minimum" + jsonSchemaTypeKey = "type" + jsonSchemaTypeObject = "object" + jsonSchemaPropertiesKey = "properties" + jsonSchemaMinimumKey = "minimum" ) func sampleConfig() *config.AgentConfig { @@ -260,8 +263,8 @@ func TestRenderAgentYAMLIncludesBrokeredTools(t *testing.T) { Description: "Read telemetry.", BrokeredClass: config.BrokeredClassRead, Parameters: map[string]any{ - jsonSchemaTypeKey: "object", - "properties": map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ "site": map[string]any{jsonSchemaTypeKey: "string", jsonSchemaMinimumKey: 0.000001}, "typedFloats": map[string]float64{jsonSchemaMinimumKey: 0.000001}, "empty": map[string]any{}, @@ -293,8 +296,8 @@ func TestRenderAgentYAMLFormatsJSONNumberBrokeredSchemaValuesAsNumbers(t *testin Description: "Read numeric data.", BrokeredClass: config.BrokeredClassRead, Parameters: map[string]any{ - jsonSchemaTypeKey: "object", - "properties": map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ "small": map[string]any{jsonSchemaTypeKey: "number", jsonSchemaMinimumKey: json.Number("1e-7")}, }, }, @@ -308,3 +311,33 @@ func TestRenderAgentYAMLFormatsJSONNumberBrokeredSchemaValuesAsNumbers(t *testin t.Fatalf("rendered agent.yaml did not preserve json.Number as fixed YAML number\n---\n%s", out) } } + +func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) { + cfg := sampleConfig() + cfg.Tools = nil + tool := config.BrokeredTool{ + Name: "negative-zero-tool", + Description: "Preserve negative zero.", + BrokeredClass: config.BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "offset": map[string]any{jsonSchemaTypeKey: "number", "default": math.Copysign(0, -1)}, + }, + }, + } + digest, err := config.BrokeredToolSchemaDigest(tool) + if err != nil { + t.Fatalf("digest error: %v", err) + } + tool.SchemaDigest = digest + cfg.BrokeredTools = []config.BrokeredTool{tool} + + out, err := Render(effective.FromConfig(cfg, testInstructions)) + if err != nil { + t.Fatalf("render error: %v", err) + } + if !strings.Contains(string(out), "default: -0.0") { + t.Fatalf("rendered agent.yaml did not preserve negative zero as a float\n---\n%s", out) + } +} diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index a4c77d5..c2c9e1d 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -622,7 +622,9 @@ def _value_matches_schema_type(value: Any, schema_type: str) -> bool: if schema_type == "boolean": return isinstance(value, bool) if schema_type == "integer": - return isinstance(value, int) and not isinstance(value, bool) + return (isinstance(value, int) and not isinstance(value, bool)) or ( + isinstance(value, float) and math.isfinite(value) and value.is_integer() + ) if schema_type == "number": return isinstance(value, (int, float)) and not isinstance(value, bool) if schema_type == "string": diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 12cc6be..25ee6db 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -580,17 +580,16 @@ def _reject_output_constant(raw: str) -> None: def _json_object_from_output(output: Any) -> dict[str, Any]: - if isinstance(output, str): - try: - parsed = json.loads( - output, - parse_float=_parse_output_float, - parse_constant=_reject_output_constant, - ) - except json.JSONDecodeError as exc: - raise ValueError("function_call_output.output must be a JSON object string") from exc - else: - parsed = output + if not isinstance(output, str): + raise ValueError("function_call_output.output must be a JSON object string") + try: + parsed = json.loads( + output, + parse_float=_parse_output_float, + parse_constant=_reject_output_constant, + ) + except json.JSONDecodeError as exc: + raise ValueError("function_call_output.output must be a JSON object string") from exc if not isinstance(parsed, dict): raise ValueError("function_call_output.output must be a JSON object") approved = parsed.get("approved") diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index ad01574..01246c3 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -820,6 +820,35 @@ def test_load_rejects_invalid_brokered_schema_type_values_and_defaults(tmp_path, assert "brokeredTools.0.parameters" in msg +def test_load_accepts_integral_float_values_for_integer_brokered_schema(tmp_path): + spec_dict = deepcopy(_BASE_SPEC) + spec_dict.update( + tools=[], + brokeredTools=[ + { + "name": "integer_values", + "description": "integer schema values", + "brokeredClass": "read", + "parameters": { + "type": "object", + "properties": { + "withDefault": {"type": "integer", "default": 1.0}, + "withConst": {"type": "integer", "const": 2.0}, + "withEnum": {"type": "integer", "enum": [3.0]}, + }, + }, + } + ], + ) + + spec = load(_write_spec(tmp_path, spec_dict)) + + properties = spec.brokered_tools[0].parameters["properties"] + assert properties["withDefault"]["default"] == 1.0 + assert properties["withConst"]["const"] == 2.0 + assert properties["withEnum"]["enum"] == [3.0] + + def test_load_rejects_unknown_brokered_class_and_malformed_schema(tmp_path): unknown_class = _invalid_message( tmp_path, diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index c0b8509..8e1e897 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -711,6 +711,34 @@ def test_foundry_brokered_rejects_lossy_function_call_output_float_before_state_ assert _message_text(exact.json()).endswith('{"id":9007199254740992.0}') +def test_foundry_brokered_rejects_object_valued_function_call_output_before_state_change(tmp_path): + state_file = tmp_path / "responses-state.json" + app = _app(response_state_file=state_file) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + persisted_before = state_file.read_bytes() + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": {"approved": True, "output": {"id": 9007199254740993.0}}, + } + ], + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_function_call_output" + assert state_file.read_bytes() == persisted_before + + def test_foundry_brokered_rejects_oversized_function_call_output_before_state_change(tmp_path): state_file = tmp_path / "responses-state.json" app = _app(response_state_file=state_file, max_brokered_output_bytes=128) diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index 17d41cd..e207ddc 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -5,6 +5,7 @@ from pathlib import Path from fastapi.testclient import TestClient +import pytest from agentkit_serve_common.foundry_conformance import create_foundry_conformance_app @@ -75,6 +76,18 @@ def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): raise AssertionError("expected old response id to fail") +def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + initial = json.loads((transcript / "02-initial-response.json").read_text(encoding="utf-8")) + continuation = json.loads((transcript / "04-continuation-response.json").read_text(encoding="utf-8")) + continuation["id"] = initial["id"] + (transcript / "04-continuation-response.json").write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="must differ from initial response id"): + verifier.verify_transcript(transcript) + + def test_verify_brokered_transcript_cli_writes_summary(tmp_path, capsys): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From 6c0234b3334ffb69d18cc71b0c80f09192736bd6 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 20:18:22 -0700 Subject: [PATCH 12/27] fix(config): preserve brokered schema numbers Signed-off-by: Sertac Ozercan --- pkg/agentkit/config/config_test.go | 46 +++++++ pkg/agentkit/config/validate.go | 113 ++++++++++++++++-- .../common/agentkit_serve_common/config.py | 7 +- .../common/tests/test_config_validation.py | 1 + 4 files changed, 155 insertions(+), 12 deletions(-) diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index 00a32d0..a44dc62 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -494,6 +494,7 @@ func TestValidateRejectsInvalidBrokeredSchemaTypeValuesAndDefaults(t *testing.T) cases := []map[string]any{ {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: nil}}}, {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{"n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: "1"}}}, + {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{"n": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: 9007199254740993.0}}}, {jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{brokeredSiteField: map[string]any{jsonSchemaTypeKey: jsonSchemaTypeString, jsonSchemaEnumKey: []any{0, "ok"}}}}, } for _, schema := range cases { @@ -556,6 +557,51 @@ func TestValidateAcceptsLargeBrokeredIntegerConstraintsWithoutInt64Narrowing(t * } } +func TestValidatePreservesLargeIntegerSchemaValuesBeforeTypeChecking(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "n": map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeInteger, + jsonSchemaDefaultKey: int64(9007199254740993), + jsonSchemaEnumKey: []any{int64(9007199254740993)}, + }, + }, + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("large integer schema values should validate without float normalization: %v", err) + } +} + +func TestValidateMeasuresBrokeredSchemaSizeWithCanonicalJSON(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + brokeredDigestDescriptionKey: strings.Repeat("<", 11_000), + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("canonical schema below the byte limit should validate despite json.Marshal HTML escaping: %v", err) + } + + cfg.BrokeredTools[0].Parameters[brokeredDigestDescriptionKey] = strings.Repeat("a", 64*1024) + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "schema is too large") { + t.Fatalf("canonical schema above the byte limit should fail validation, got: %v", err) + } +} + func TestValidateRejectsInvalidRemoteMCPToolShapes(t *testing.T) { cases := map[string]string{ //nolint:gosec // test YAML uses credential-looking field names/invalid examples, not real secrets "missing type": `tools: diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index 6bf8c46..8b7af89 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -9,6 +9,7 @@ import ( "fmt" "math" pathpkg "path" + "reflect" "sort" "strconv" "strings" @@ -55,6 +56,7 @@ const ( brokeredDigestNumberKey = "\u0000agentkit_json_number" brokeredUnsafeCookieKey = "cookie" credentialHeaderAPIKey = "api-key" + maxExactJSONFloatInteger = float64(1<<53 - 1) ) // Validate reports every problem with the config at once via errors.Join (plan @@ -248,19 +250,24 @@ func validateBrokeredToolParameters(add func(string, ...any), path string, param add("%s must be a JSON Schema object", path) return } - encoded, err := json.Marshal(parameters) - if err != nil { + if _, err := json.Marshal(parameters); err != nil { add("%s must be JSON serializable: %v", path, err) return } - if len(encoded) > 64*1024 { - add("%s schema is too large", path) - } - var schema map[string]any - if err := json.Unmarshal(encoded, &schema); err != nil { + normalized, ok := normalizeJSONContainers(parameters).(map[string]any) + if !ok { add("%s must be a JSON Schema object", path) return } + canonical, err := canonicalJSON(normalized) + if err != nil { + add("%s must be JSON serializable: %v", path, err) + return + } + if len(canonical) > 64*1024 { + add("%s schema is too large", path) + } + schema := normalized validateJSONSchemaSubset(add, path, schema) if typ, _ := schema[jsonSchemaTypeKey].(string); typ != jsonSchemaTypeObject { add("%s must set type: object", path) @@ -269,6 +276,42 @@ func validateBrokeredToolParameters(add func(string, ...any), path string, param validateBrokeredSchemaValueConstraints(add, path, schema) } +func normalizeJSONContainers(value any) any { + if value == nil { + return nil + } + rv := reflect.ValueOf(value) + for rv.Kind() == reflect.Interface || rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return nil + } + rv = rv.Elem() + } + switch rv.Kind() { + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return value + } + out := make(map[string]any, rv.Len()) + iter := rv.MapRange() + for iter.Next() { + out[iter.Key().String()] = normalizeJSONContainers(iter.Value().Interface()) + } + return out + case reflect.Slice, reflect.Array: + if rv.Type().Elem().Kind() == reflect.Uint8 { + return value + } + out := make([]any, rv.Len()) + for i := 0; i < rv.Len(); i++ { + out[i] = normalizeJSONContainers(rv.Index(i).Interface()) + } + return out + default: + return rv.Interface() + } +} + func hasAnySchemaKey(schema map[string]any, keys ...string) bool { for _, key := range keys { if _, ok := schema[key]; ok { @@ -355,21 +398,69 @@ func validateJSONSchemaSubset(add func(string, ...any), path string, schema map[ } for _, key := range []string{jsonSchemaMinimumKey, jsonSchemaMaximumKey, "exclusiveMinimum", "exclusiveMaximum"} { if value, ok := schema[key]; ok { - if _, ok := value.(float64); !ok { + if !isFiniteJSONNumber(value) { add("%s.%s must be a number", path, key) } } } for _, key := range []string{"minLength", "maxLength", "minItems", "maxItems", "minProperties", "maxProperties"} { if value, ok := schema[key]; ok { - number, ok := value.(float64) - if !ok || math.IsNaN(number) || math.IsInf(number, 0) || number < 0 || math.Trunc(number) != number { + if !isNonNegativeJSONInteger(value) { add("%s.%s must be a non-negative integer", path, key) } } } } +func isFiniteJSONNumber(value any) bool { + switch typed := value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return true + case float32: + return !math.IsNaN(float64(typed)) && !math.IsInf(float64(typed), 0) + case float64: + return !math.IsNaN(typed) && !math.IsInf(typed, 0) + case json.Number: + number, err := typed.Float64() + return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) + default: + return false + } +} + +func isNonNegativeJSONInteger(value any) bool { + switch typed := value.(type) { + case int: + return typed >= 0 + case int8: + return typed >= 0 + case int16: + return typed >= 0 + case int32: + return typed >= 0 + case int64: + return typed >= 0 + case uint, uint8, uint16, uint32, uint64: + return true + case float32: + number := float64(typed) + return !math.IsNaN(number) && !math.IsInf(number, 0) && number >= 0 && math.Trunc(number) == number + case float64: + return !math.IsNaN(typed) && !math.IsInf(typed, 0) && typed >= 0 && math.Trunc(typed) == typed + case json.Number: + if integer, err := strconv.ParseInt(typed.String(), 10, 64); err == nil { + return integer >= 0 + } + if _, err := strconv.ParseUint(typed.String(), 10, 64); err == nil { + return true + } + number, err := typed.Float64() + return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) && number >= 0 && math.Trunc(number) == number + default: + return false + } +} + func validateJSONSchemaType(add func(string, ...any), path string, value any) { if value == nil { return @@ -511,7 +602,7 @@ func matchesSchemaType(value any, schemaType string) bool { case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: return true case float64: - return typed == math.Trunc(typed) + return !math.IsNaN(typed) && !math.IsInf(typed, 0) && typed == math.Trunc(typed) && math.Abs(typed) <= maxExactJSONFloatInteger case json.Number: _, err := typed.Int64() return err == nil diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index c2c9e1d..1272599 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -54,6 +54,7 @@ _BROKERED_CLASSES = {"read", "write", "coordination"} _BROKERED_SCHEMA_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") _MAX_BROKERED_SCHEMA_BYTES = 64 * 1024 +_MAX_EXACT_JSON_FLOAT_INTEGER = (1 << 53) - 1 _UNSAFE_BROKERED_FIELD_NAMES = { "auth", "authorization", @@ -623,7 +624,10 @@ def _value_matches_schema_type(value: Any, schema_type: str) -> bool: return isinstance(value, bool) if schema_type == "integer": return (isinstance(value, int) and not isinstance(value, bool)) or ( - isinstance(value, float) and math.isfinite(value) and value.is_integer() + isinstance(value, float) + and math.isfinite(value) + and value.is_integer() + and abs(value) <= _MAX_EXACT_JSON_FLOAT_INTEGER ) if schema_type == "number": return isinstance(value, (int, float)) and not isinstance(value, bool) @@ -707,6 +711,7 @@ def _valid_name(cls, value: str) -> str: def _valid_json_schema(cls, value: dict[str, Any]) -> dict[str, Any]: if not isinstance(value, dict): raise ValueError("brokered tool parameters must be a JSON Schema object") + _validate_schema_value_constraints(value, path="brokeredTools[].parameters") try: encoded = _canonical_json(value) except (TypeError, ValueError) as exc: diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index 01246c3..07f8c58 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -799,6 +799,7 @@ def test_load_rejects_malformed_nested_brokered_json_schema(tmp_path, bad_child: [ {"type": "object", "properties": {"site": {"type": None}}}, {"type": "object", "properties": {"n": {"type": "integer", "default": "1"}}}, + {"type": "object", "properties": {"n": {"type": "integer", "default": 9007199254740993.0}}}, {"type": "object", "properties": {"site": {"type": "string", "enum": [0, "ok"]}}}, ], ) From 14cad45196caa1cf7edc423c2bd30c610f9aa051 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 20:28:27 -0700 Subject: [PATCH 13/27] fix(brokered): normalize usage and schema edge cases Signed-off-by: Sertac Ozercan --- pkg/agentkit/config/config_test.go | 28 +++++++ pkg/agentkit/config/validate.go | 36 ++++++++- .../common/agentkit_serve_common/foundry.py | 16 +++- .../tests/test_foundry_brokered_protocol.py | 80 +++++++++++++++++++ 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index a44dc62..475dc32 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -580,6 +580,34 @@ func TestValidatePreservesLargeIntegerSchemaValuesBeforeTypeChecking(t *testing. } } +func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing.T) { + type count int64 + namedSchema := map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: count(1)} + properties := map[string]any{ + "named": namedSchema, + "number": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: json.Number("1.0")}, + } + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: properties, + }, + }} + + if err := cfg.Validate(); err != nil { + t.Fatalf("JSON-serializable named and json.Number integer values should validate: %v", err) + } + + namedSchema[jsonSchemaDefaultKey] = map[string]any(nil) + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { + t.Fatalf("typed nil map must retain JSON null semantics, got: %v", err) + } +} + func TestValidateMeasuresBrokeredSchemaSizeWithCanonicalJSON(t *testing.T) { cfg := validMinimalConfig() cfg.BrokeredTools = []BrokeredTool{{ diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index 8b7af89..84da247 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -280,6 +280,9 @@ func normalizeJSONContainers(value any) any { if value == nil { return nil } + if number, ok := value.(json.Number); ok { + return number + } rv := reflect.ValueOf(value) for rv.Kind() == reflect.Interface || rv.Kind() == reflect.Pointer { if rv.IsNil() { @@ -289,6 +292,9 @@ func normalizeJSONContainers(value any) any { } switch rv.Kind() { case reflect.Map: + if rv.IsNil() { + return nil + } if rv.Type().Key().Kind() != reflect.String { return value } @@ -298,7 +304,10 @@ func normalizeJSONContainers(value any) any { out[iter.Key().String()] = normalizeJSONContainers(iter.Value().Interface()) } return out - case reflect.Slice, reflect.Array: + case reflect.Slice: + if rv.IsNil() { + return nil + } if rv.Type().Elem().Kind() == reflect.Uint8 { return value } @@ -307,6 +316,24 @@ func normalizeJSONContainers(value any) any { out[i] = normalizeJSONContainers(rv.Index(i).Interface()) } return out + case reflect.Array: + out := make([]any, rv.Len()) + for i := 0; i < rv.Len(); i++ { + out[i] = normalizeJSONContainers(rv.Index(i).Interface()) + } + return out + case reflect.Bool: + return rv.Bool() + case reflect.String: + return rv.String() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() + case reflect.Float32: + return float32(rv.Float()) + case reflect.Float64: + return rv.Float() default: return rv.Interface() } @@ -604,8 +631,11 @@ func matchesSchemaType(value any, schemaType string) bool { case float64: return !math.IsNaN(typed) && !math.IsInf(typed, 0) && typed == math.Trunc(typed) && math.Abs(typed) <= maxExactJSONFloatInteger case json.Number: - _, err := typed.Int64() - return err == nil + if _, err := typed.Int64(); err == nil { + return true + } + number, err := typed.Float64() + return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) && number == math.Trunc(number) && math.Abs(number) <= maxExactJSONFloatInteger default: return false } diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 25ee6db..12fb3f6 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -111,15 +111,19 @@ def _responses_usage(result: RunResult | None = None, usage: Mapping[str, int] | def _combine_usage(*usages: Mapping[str, int] | None) -> dict[str, int]: prompt_tokens = 0 completion_tokens = 0 + total_tokens = 0 for usage in usages: if not usage: continue - prompt_tokens += int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) - completion_tokens += int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0) + prompt_count = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0) + completion_count = int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0) + prompt_tokens += prompt_count + completion_tokens += completion_count + total_tokens += int(usage.get("total_tokens", prompt_count + completion_count) or 0) return { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, + "total_tokens": total_tokens, } @@ -596,10 +600,16 @@ def _json_object_from_output(output: Any) -> dict[str, Any]: if not isinstance(approved, bool): raise ValueError("function_call_output.output.approved must be a boolean") if approved: + unexpected = set(parsed) - {"approved", "output"} + if unexpected: + raise ValueError("approved function_call_output.output contains unsupported fields") tool_output = parsed.get("output", {}) if tool_output is not None and not isinstance(tool_output, dict): raise ValueError("approved function_call_output.output.output must be an object") else: + unexpected = set(parsed) - {"approved", "error"} + if unexpected: + raise ValueError("denied function_call_output.output contains unsupported fields") error = parsed.get("error", {}) if error is not None and not isinstance(error, dict): raise ValueError("denied function_call_output.output.error must be an object") diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 8e1e897..2a97ccf 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1269,6 +1269,86 @@ def test_foundry_brokered_model_loop_emits_model_requested_tool_and_resumes_to_f assert "tools" not in fake.requests[1] +def test_foundry_brokered_model_loop_preserves_total_only_usage_across_resume(): + spec = _spec(tool_name="check-network-telemetry") + initial_model_response = _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ) + initial_model_response["usage"] = {"total_tokens": 5} + final_model_response = _chat_response({"role": "assistant", "content": "done"}) + final_model_response["usage"] = {"total_tokens": 7} + app = _model_loop_app(spec, _FakeChatTransport([initial_model_response, final_model_response])) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + final = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial.json()["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + + assert initial.json()["usage"] == {"input_tokens": 0, "output_tokens": 0, "total_tokens": 5} + assert final.status_code == 200, final.text + assert final.json()["usage"] == {"input_tokens": 0, "output_tokens": 0, "total_tokens": 12} + + +def test_foundry_brokered_model_loop_rejects_noncanonical_denied_output_before_resume(tmp_path): + state_file = tmp_path / "responses-state.json" + spec = _spec(tool_name="dispatch-work-order", brokered_class="write") + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "model_generated_call_id", + "type": "function", + "function": {"name": "dispatch-work-order", "arguments": "{}"}, + } + ], + } + ) + ] + ) + app = _model_loop_app(spec, fake, response_state_file=state_file) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "dispatch-work-order"}) + call = _call(initial.json()) + persisted_before = state_file.read_bytes() + denied = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation( + initial.json()["id"], + call["call_id"], + { + "approved": False, + "error": {"code": "approval_declined", "message": "denied"}, + "output": {"sensitiveDiagnostic": "must-not-reach-model"}, + }, + ), + ) + + assert denied.status_code == 400 + assert denied.json()["error"]["code"] == "invalid_function_call_output" + assert state_file.read_bytes() == persisted_before + assert len(fake.requests) == 1 + + def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_state_change(tmp_path): state_file = tmp_path / "responses-state.json" spec = _spec(tool_name="check-network-telemetry") From a0a9729312462abacc90d0d334e299b39edef94e Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 21:04:01 -0700 Subject: [PATCH 14/27] fix(brokered): preserve numeric ABI fidelity Signed-off-by: Sertac Ozercan --- pkg/agentkit/abi/render.go | 16 ++- pkg/agentkit/abi/render_test.go | 13 ++- pkg/agentkit/config/config_test.go | 45 ++++++++- pkg/agentkit/config/validate.go | 97 +++++++++++++++---- .../common/agentkit_serve_common/foundry.py | 38 ++++---- .../tests/test_foundry_brokered_protocol.py | 25 +++++ 6 files changed, 189 insertions(+), 45 deletions(-) diff --git a/pkg/agentkit/abi/render.go b/pkg/agentkit/abi/render.go index f86f472..75ba5f8 100644 --- a/pkg/agentkit/abi/render.go +++ b/pkg/agentkit/abi/render.go @@ -5,6 +5,7 @@ package abi import ( "encoding/json" "math" + "math/big" "strconv" "strings" @@ -19,6 +20,8 @@ const Version = "v0" // Path is where the rendered agent.yaml is baked into the agent image. const Path = "/agent/agent.yaml" +const yamlNegativeZero = "-0.0" + // The following types are the WRITER half of the frozen agent.yaml ABI // (docs/agent-abi.md). agentkit-serve reads this file with pydantic // extra="forbid", so these structs MUST emit EXACTLY the keys documented there. @@ -33,7 +36,7 @@ func (n yamlNumber) MarshalYAML() ([]byte, error) { func yamlFloat(value float64, bitSize int) yamlNumber { rendered := strconv.FormatFloat(value, 'f', -1, bitSize) if value == 0 && math.Signbit(value) { - rendered = "-0.0" + rendered = yamlNegativeZero } return yamlNumber(rendered) } @@ -133,6 +136,14 @@ type abiAgent struct { func expandJSONNumber(value string) string { lower := strings.ToLower(value) + if json.Valid([]byte(lower)) { + if number, ok := new(big.Rat).SetString(lower); ok && number.IsInt() { + if number.Sign() == 0 && strings.HasPrefix(lower, "-") { + return yamlNegativeZero + } + return number.Num().String() + } + } parts := strings.Split(lower, "e") if len(parts) != 2 { return value @@ -156,6 +167,9 @@ func expandJSONNumber(value string) string { } mantissa = strings.TrimLeft(mantissa, "0") if mantissa == "" { + if sign == "-" { + return yamlNegativeZero + } return "0" } decimalPos := len(mantissa) - fracLen + exponent diff --git a/pkg/agentkit/abi/render_test.go b/pkg/agentkit/abi/render_test.go index 9149f7c..060878d 100644 --- a/pkg/agentkit/abi/render_test.go +++ b/pkg/agentkit/abi/render_test.go @@ -23,8 +23,10 @@ const ( const ( jsonSchemaTypeKey = "type" jsonSchemaTypeObject = "object" + jsonSchemaTypeNumber = "number" jsonSchemaPropertiesKey = "properties" jsonSchemaMinimumKey = "minimum" + jsonSchemaDefaultKey = "default" ) func sampleConfig() *config.AgentConfig { @@ -298,7 +300,7 @@ func TestRenderAgentYAMLFormatsJSONNumberBrokeredSchemaValuesAsNumbers(t *testin Parameters: map[string]any{ jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{ - "small": map[string]any{jsonSchemaTypeKey: "number", jsonSchemaMinimumKey: json.Number("1e-7")}, + "small": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaMinimumKey: json.Number("1e-7")}, }, }, }} @@ -322,7 +324,9 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) Parameters: map[string]any{ jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{ - "offset": map[string]any{jsonSchemaTypeKey: "number", "default": math.Copysign(0, -1)}, + "offset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: math.Copysign(0, -1)}, + "jsonOffset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: json.Number("-0e0")}, + "largeInteger": map[string]any{jsonSchemaTypeKey: "integer", jsonSchemaDefaultKey: json.Number("9007199254740995.0")}, }, }, } @@ -337,7 +341,10 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) if err != nil { t.Fatalf("render error: %v", err) } - if !strings.Contains(string(out), "default: -0.0") { + if strings.Count(string(out), "default: -0.0") != 2 { t.Fatalf("rendered agent.yaml did not preserve negative zero as a float\n---\n%s", out) } + if !strings.Contains(string(out), "default: 9007199254740995") || strings.Contains(string(out), "9007199254740995.0") { + t.Fatalf("rendered agent.yaml did not preserve a large integral decimal as an integer\n---\n%s", out) + } } diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index 475dc32..de22c45 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -582,10 +582,15 @@ func TestValidatePreservesLargeIntegerSchemaValuesBeforeTypeChecking(t *testing. func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing.T) { type count int64 + pointerNumber := json.Number("2.0") namedSchema := map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: count(1)} + numberSchema := map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: json.Number("1.0")} + fractionalSchema := map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: json.Number("0.1")} properties := map[string]any{ - "named": namedSchema, - "number": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: json.Number("1.0")}, + "fractional": fractionalSchema, + "named": namedSchema, + "number": numberSchema, + "pointer": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeInteger, jsonSchemaDefaultKey: &pointerNumber}, } cfg := validMinimalConfig() cfg.BrokeredTools = []BrokeredTool{{ @@ -602,6 +607,29 @@ func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing t.Fatalf("JSON-serializable named and json.Number integer values should validate: %v", err) } + numberSchema[jsonSchemaDefaultKey] = json.Number("1.00000000000000001") + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { + t.Fatalf("non-integral json.Number must fail integer validation, got: %v", err) + } + numberSchema[jsonSchemaDefaultKey] = json.Number("100000000000000000001.0") + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { + t.Fatalf("large float-form json.Number must fail integer validation, got: %v", err) + } + numberSchema[jsonSchemaDefaultKey] = json.Number("100000000000000000001") + if err := cfg.Validate(); err != nil { + t.Fatalf("large lexical integer json.Number should validate exactly: %v", err) + } + numberSchema[jsonSchemaDefaultKey] = json.Number("1/1") + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must be JSON serializable") { + t.Fatalf("non-JSON json.Number syntax must fail serialization validation, got: %v", err) + } + numberSchema[jsonSchemaDefaultKey] = json.Number("1.0") + fractionalSchema[jsonSchemaDefaultKey] = json.Number("1e-400") + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { + t.Fatalf("underflowing json.Number must fail number validation, got: %v", err) + } + fractionalSchema[jsonSchemaDefaultKey] = json.Number("0.1") + namedSchema[jsonSchemaDefaultKey] = map[string]any(nil) if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { t.Fatalf("typed nil map must retain JSON null semantics, got: %v", err) @@ -1125,6 +1153,19 @@ func TestBrokeredToolSchemaDigestMatchesPythonCanonicalJSON(t *testing.T) { } } +func TestCanonicalJSONStringMatchesPythonForUnicodeLineSeparators(t *testing.T) { + value := "line\u2028paragraph\u2029literal\\u2028" + + encoded, err := canonicalJSONString(value) + if err != nil { + t.Fatalf("canonicalJSONString error: %v", err) + } + want := "\"line\u2028paragraph\u2029literal\\\\u2028\"" + if string(encoded) != want { + t.Fatalf("canonicalJSONString = %q, want %q", encoded, want) + } +} + func TestBrokeredToolSchemaDigestNormalizesIntegerValuedFloatConstraints(t *testing.T) { tool := BrokeredTool{ Name: "numeric-tool", diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index 84da247..bfc119a 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "math" + "math/big" pathpkg "path" "reflect" "sort" @@ -280,9 +281,6 @@ func normalizeJSONContainers(value any) any { if value == nil { return nil } - if number, ok := value.(json.Number); ok { - return number - } rv := reflect.ValueOf(value) for rv.Kind() == reflect.Interface || rv.Kind() == reflect.Pointer { if rv.IsNil() { @@ -290,6 +288,9 @@ func normalizeJSONContainers(value any) any { } rv = rv.Elem() } + if number, ok := rv.Interface().(json.Number); ok { + return number + } switch rv.Kind() { case reflect.Map: if rv.IsNil() { @@ -448,8 +449,8 @@ func isFiniteJSONNumber(value any) bool { case float64: return !math.IsNaN(typed) && !math.IsInf(typed, 0) case json.Number: - number, err := typed.Float64() - return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) + number, ok := parseJSONNumber(typed) + return ok && jsonNumberIsRepresentable(typed, number) default: return false } @@ -475,19 +476,38 @@ func isNonNegativeJSONInteger(value any) bool { case float64: return !math.IsNaN(typed) && !math.IsInf(typed, 0) && typed >= 0 && math.Trunc(typed) == typed case json.Number: - if integer, err := strconv.ParseInt(typed.String(), 10, 64); err == nil { - return integer >= 0 - } - if _, err := strconv.ParseUint(typed.String(), 10, 64); err == nil { - return true - } - number, err := typed.Float64() - return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) && number >= 0 && math.Trunc(number) == number + number, ok := parseJSONNumber(typed) + return ok && number.Sign() >= 0 && jsonNumberIsExactInteger(typed, number) default: return false } } +func parseJSONNumber(value json.Number) (*big.Rat, bool) { + raw := value.String() + if !json.Valid([]byte(raw)) { + return nil, false + } + return new(big.Rat).SetString(raw) +} + +func jsonNumberIsExactInteger(value json.Number, number *big.Rat) bool { + return number.IsInt() && jsonNumberIsRepresentable(value, number) +} + +func jsonNumberIsRepresentable(value json.Number, number *big.Rat) bool { + raw := value.String() + if !strings.ContainsAny(raw, ".eE") { + return true + } + parsed, err := strconv.ParseFloat(raw, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return false + } + roundTrip, ok := new(big.Rat).SetString(strconv.FormatFloat(parsed, 'g', -1, 64)) + return ok && roundTrip.Cmp(number) == 0 +} + func validateJSONSchemaType(add func(string, ...any), path string, value any) { if value == nil { return @@ -631,18 +651,22 @@ func matchesSchemaType(value any, schemaType string) bool { case float64: return !math.IsNaN(typed) && !math.IsInf(typed, 0) && typed == math.Trunc(typed) && math.Abs(typed) <= maxExactJSONFloatInteger case json.Number: - if _, err := typed.Int64(); err == nil { - return true - } - number, err := typed.Float64() - return err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) && number == math.Trunc(number) && math.Abs(number) <= maxExactJSONFloatInteger + number, ok := parseJSONNumber(typed) + return ok && jsonNumberIsExactInteger(typed, number) default: return false } case jsonSchemaTypeNumber: - switch value.(type) { - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, json.Number: + switch typed := value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: return true + case float32: + return !math.IsNaN(float64(typed)) && !math.IsInf(float64(typed), 0) + case float64: + return !math.IsNaN(typed) && !math.IsInf(typed, 0) + case json.Number: + number, ok := parseJSONNumber(typed) + return ok && jsonNumberIsRepresentable(typed, number) default: return false } @@ -818,7 +842,38 @@ func canonicalJSONString(value string) ([]byte, error) { if err := encoder.Encode(value); err != nil { return nil, err } - return bytes.TrimSuffix(encoded.Bytes(), []byte("\n")), nil + return unescapeJSONLineSeparators(bytes.TrimSuffix(encoded.Bytes(), []byte("\n"))), nil +} + +func unescapeJSONLineSeparators(encoded []byte) []byte { + out := make([]byte, 0, len(encoded)) + for i := 0; i < len(encoded); { + if encoded[i] != '\\' { + out = append(out, encoded[i]) + i++ + continue + } + start := i + for i < len(encoded) && encoded[i] == '\\' { + i++ + } + runLength := i - start + if runLength%2 == 1 && i+5 <= len(encoded) && encoded[i] == 'u' { + escape := string(encoded[i : i+5]) + if escape == "u2028" || escape == "u2029" { + out = append(out, encoded[start:i-1]...) + if escape == "u2028" { + out = append(out, []byte("\u2028")...) + } else { + out = append(out, []byte("\u2029")...) + } + i += 5 + continue + } + } + out = append(out, encoded[start:i]...) + } + return out } func canonicalJSONNumberString(value string) (string, error) { diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 12fb3f6..e71fab0 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -637,6 +637,18 @@ def _enum_value(schema: Mapping[str, Any], expected_type: type) -> Any: return None +def _integer_schema_bound(value: Any, *, lower: bool, exclusive: bool) -> int | None: + if isinstance(value, int) and not isinstance(value, bool): + if exclusive: + return value + 1 if lower else value - 1 + return value + if isinstance(value, float) and math.isfinite(value): + if lower: + return math.floor(value) + 1 if exclusive else math.ceil(value) + return math.ceil(value) - 1 if exclusive else math.floor(value) + return None + + def _required_property_names(schema: Mapping[str, Any]) -> list[str]: names: list[str] = [] required = schema.get("required") @@ -712,24 +724,14 @@ def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> A value = _enum_value(schema, int) if value is not None and not isinstance(value, bool): return value - lower_value = schema.get("minimum") - if isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) and math.isfinite(float(lower_value)): - lower = math.ceil(float(lower_value)) - else: - exclusive_lower = schema.get("exclusiveMinimum") - if isinstance(exclusive_lower, (int, float)) and not isinstance(exclusive_lower, bool) and math.isfinite(float(exclusive_lower)): - lower = math.floor(float(exclusive_lower)) + 1 - else: - lower = 0 - upper_value = schema.get("maximum") - if isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) and math.isfinite(float(upper_value)): - upper = math.floor(float(upper_value)) - else: - exclusive_upper = schema.get("exclusiveMaximum") - if isinstance(exclusive_upper, (int, float)) and not isinstance(exclusive_upper, bool) and math.isfinite(float(exclusive_upper)): - upper = math.ceil(float(exclusive_upper)) - 1 - else: - upper = None + lower = _integer_schema_bound(schema.get("minimum"), lower=True, exclusive=False) + if lower is None: + lower = _integer_schema_bound(schema.get("exclusiveMinimum"), lower=True, exclusive=True) + if lower is None: + lower = 0 + upper = _integer_schema_bound(schema.get("maximum"), lower=False, exclusive=False) + if upper is None: + upper = _integer_schema_bound(schema.get("exclusiveMaximum"), lower=False, exclusive=True) if upper is not None and lower > upper: if "minimum" in schema or "exclusiveMinimum" in schema: raise AgentRunError( diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 2a97ccf..eb6f2f2 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -388,6 +388,31 @@ def test_foundry_brokered_integer_arguments_honor_float_bounds(): assert json.loads(_call(body.json())["arguments"]) == {"count": 1, "after": 1, "below": 0} +def test_foundry_brokered_integer_arguments_preserve_large_integer_bounds(): + huge = 10**100 + spec = _spec(tool_name="large-integer-bounds") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "minimum": {"type": "integer", "minimum": huge}, + "exclusiveMinimum": {"type": "integer", "exclusiveMinimum": huge}, + "exclusiveMaximum": {"type": "integer", "exclusiveMaximum": -huge}, + }, + "required": ["minimum", "exclusiveMinimum", "exclusiveMaximum"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "large-integer-bounds"}) + + assert response.status_code == 200, response.text + assert json.loads(_call(response.json())["arguments"]) == { + "minimum": huge, + "exclusiveMinimum": huge + 1, + "exclusiveMaximum": -huge - 1, + } + + def test_foundry_brokered_synthesizes_arguments_that_honor_basic_constraints(): spec = _spec(tool_name="bounded-lookup") tool = spec.brokered_tools[0] From 67e69b0e7d8a3798286fac94dac7f1062b06f3db Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 21:10:45 -0700 Subject: [PATCH 15/27] fix(foundry): reject decoded model arguments Signed-off-by: Sertac Ozercan --- .../foundry_model_loop.py | 17 +++++------ .../common/tests/test_config_validation.py | 30 +++++++++++++++++++ .../tests/test_foundry_brokered_protocol.py | 30 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index 3e849ab..e46e534 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -198,15 +198,14 @@ def _message_text(message: Mapping[str, Any]) -> str: def _parse_arguments(raw: Any) -> dict[str, Any]: - if isinstance(raw, str): - try: - parsed = json.loads(raw or "{}", parse_float=_parse_json_float, parse_constant=_reject_json_constant) - except AgentRunError: - raise - except json.JSONDecodeError as exc: - raise AgentRunError("model tool arguments must be valid JSON", status=400, code="InvalidToolArguments") from exc - else: - parsed = raw + if not isinstance(raw, str): + raise AgentRunError("model tool arguments must be a JSON object string", status=400, code="InvalidToolArguments") + try: + parsed = json.loads(raw or "{}", parse_float=_parse_json_float, parse_constant=_reject_json_constant) + except AgentRunError: + raise + except json.JSONDecodeError as exc: + raise AgentRunError("model tool arguments must be valid JSON", status=400, code="InvalidToolArguments") from exc if not isinstance(parsed, dict): raise AgentRunError("model tool arguments must be a JSON object", status=400, code="InvalidToolArguments") return parsed diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index 07f8c58..2ca7a57 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -850,6 +850,36 @@ def test_load_accepts_integral_float_values_for_integer_brokered_schema(tmp_path assert properties["withEnum"]["enum"] == [3.0] +def test_load_accepts_integral_float_integer_schema_constraints(tmp_path): + spec_dict = deepcopy(_BASE_SPEC) + spec_dict.update( + tools=[], + brokeredTools=[ + { + "name": "integer_constraints", + "description": "integer schema constraints", + "brokeredClass": "read", + "parameters": { + "type": "object", + "properties": { + "values": { + "type": "array", + "minItems": 1.0, + "maxItems": 2.0, + } + }, + }, + } + ], + ) + + spec = load(_write_spec(tmp_path, spec_dict)) + + values = spec.brokered_tools[0].parameters["properties"]["values"] + assert values["minItems"] == 1 + assert values["maxItems"] == 2 + + def test_load_rejects_unknown_brokered_class_and_malformed_schema(tmp_path): unknown_class = _invalid_message( tmp_path, diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index eb6f2f2..7611b0d 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1711,6 +1711,36 @@ def test_foundry_brokered_model_loop_rejects_unknown_model_tool_request(): assert response.json()["error"]["code"] == "unknown_brokered_tool" +def test_foundry_brokered_model_loop_rejects_object_valued_tool_arguments(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": {"id": 9007199254740993.0}, + }, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + def test_foundry_brokered_model_loop_validates_schema_valued_additional_properties(): spec = _spec(tool_name="flex-tool") spec.brokered_tools[0].parameters = { From 10ae43a6f6bb4a7cef5bc1520205b2d25c88f4c6 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 21:15:37 -0700 Subject: [PATCH 16/27] fix(config): preserve negative zero schema values Signed-off-by: Sertac Ozercan --- .../common/agentkit_serve_common/config.py | 8 ++++- .../common/tests/test_config_validation.py | 35 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index 1272599..18fa4dd 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -185,6 +185,12 @@ def _canonical_json(value: Any) -> str: raise TypeError(f"unsupported JSON value {type(value).__name__}") +def _parse_canonical_int(value: str) -> int | float: + if value == "-0": + return -0.0 + return int(value) + + def brokered_tool_schema_digest( *, name: str, @@ -718,7 +724,7 @@ def _valid_json_schema(cls, value: dict[str, Any]) -> dict[str, Any]: raise ValueError("brokered tool parameters must be JSON serializable") from exc if len(encoded.encode("utf-8")) > _MAX_BROKERED_SCHEMA_BYTES: raise ValueError("brokered tool parameters schema is too large") - cloned = json.loads(encoded) + cloned = json.loads(encoded, parse_int=_parse_canonical_int) if cloned.get("type") != "object": raise ValueError("brokered tool parameters schema must set type: object") _validate_json_schema_subset(cloned, path="brokeredTools[].parameters") diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index 2ca7a57..de66ac0 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -1,11 +1,12 @@ from __future__ import annotations +import math from copy import deepcopy import pytest import yaml -from agentkit_serve_common.config import ConfigError, load, load_or_exit, validate_required_env +from agentkit_serve_common.config import ConfigError, brokered_tool_schema_digest, load, load_or_exit, validate_required_env _BASE_SPEC = { @@ -850,6 +851,38 @@ def test_load_accepts_integral_float_values_for_integer_brokered_schema(tmp_path assert properties["withEnum"]["enum"] == [3.0] +def test_load_preserves_negative_zero_for_brokered_schema_digest(tmp_path): + parameters = { + "type": "object", + "properties": {"offset": {"type": "number", "default": -0.0}}, + } + spec_dict = deepcopy(_BASE_SPEC) + spec_dict.update( + tools=[], + brokeredTools=[ + { + "name": "negative_zero", + "description": "preserve negative zero", + "brokeredClass": "read", + "parameters": parameters, + "schemaDigest": brokered_tool_schema_digest( + name="negative_zero", + description="preserve negative zero", + brokered_class="read", + parameters=parameters, + ), + } + ], + ) + + spec = load(_write_spec(tmp_path, spec_dict)) + + default = spec.brokered_tools[0].parameters["properties"]["offset"]["default"] + assert isinstance(default, float) + assert default == 0.0 + assert math.copysign(1.0, default) == -1.0 + + def test_load_accepts_integral_float_integer_schema_constraints(tmp_path): spec_dict = deepcopy(_BASE_SPEC) spec_dict.update( From 85ffae8e1623ae4e54b960fdd36ad9a43551ea42 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sat, 11 Jul 2026 21:23:12 -0700 Subject: [PATCH 17/27] fix(foundry): harden fallback and fixtures Signed-off-by: Sertac Ozercan --- .../scripts/foundry_brokered_conformance.sh | 1 + pkg/agentkit/abi/render.go | 58 ++++++++++++++++++- pkg/agentkit/abi/render_test.go | 4 ++ .../foundry_brokered_cli.py | 2 +- .../foundry_model_loop.py | 5 +- .../tests/test_foundry_brokered_protocol.py | 31 ++++++++++ .../common/tests/test_foundry_protocol.py | 11 +++- .../tests/test_foundry_transcript_verifier.py | 7 +++ 8 files changed, 111 insertions(+), 8 deletions(-) diff --git a/deploy/foundry/scripts/foundry_brokered_conformance.sh b/deploy/foundry/scripts/foundry_brokered_conformance.sh index 0cc1367..4b73433 100755 --- a/deploy/foundry/scripts/foundry_brokered_conformance.sh +++ b/deploy/foundry/scripts/foundry_brokered_conformance.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +umask 077 usage() { cat >&2 <<'EOF' diff --git a/pkg/agentkit/abi/render.go b/pkg/agentkit/abi/render.go index 75ba5f8..3136d7b 100644 --- a/pkg/agentkit/abi/render.go +++ b/pkg/agentkit/abi/render.go @@ -6,6 +6,7 @@ import ( "encoding/json" "math" "math/big" + "reflect" "strconv" "strings" @@ -256,7 +257,62 @@ func copyAny(v any) any { case json.Number: return yamlNumber(expandJSONNumber(typed.String())) default: - return typed + return copyReflectedJSON(typed) + } +} + +func copyReflectedJSON(value any) any { + if value == nil { + return nil + } + rv := reflect.ValueOf(value) + for rv.Kind() == reflect.Interface || rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return nil + } + rv = rv.Elem() + } + if number, ok := rv.Interface().(json.Number); ok { + return yamlNumber(expandJSONNumber(number.String())) + } + switch rv.Kind() { + case reflect.Map: + if rv.IsNil() { + return nil + } + if rv.Type().Key().Kind() != reflect.String { + return value + } + out := make(map[string]any, rv.Len()) + iter := rv.MapRange() + for iter.Next() { + out[iter.Key().String()] = copyAny(iter.Value().Interface()) + } + return out + case reflect.Slice: + if rv.IsNil() { + return nil + } + if rv.Type().Elem().Kind() == reflect.Uint8 { + return value + } + out := make([]any, rv.Len()) + for i := 0; i < rv.Len(); i++ { + out[i] = copyAny(rv.Index(i).Interface()) + } + return out + case reflect.Array: + out := make([]any, rv.Len()) + for i := 0; i < rv.Len(); i++ { + out[i] = copyAny(rv.Index(i).Interface()) + } + return out + case reflect.Float32: + return yamlFloat(rv.Float(), 32) + case reflect.Float64: + return yamlFloat(rv.Float(), 64) + default: + return rv.Interface() } } diff --git a/pkg/agentkit/abi/render_test.go b/pkg/agentkit/abi/render_test.go index 060878d..840f75b 100644 --- a/pkg/agentkit/abi/render_test.go +++ b/pkg/agentkit/abi/render_test.go @@ -327,6 +327,7 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) "offset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: math.Copysign(0, -1)}, "jsonOffset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: json.Number("-0e0")}, "largeInteger": map[string]any{jsonSchemaTypeKey: "integer", jsonSchemaDefaultKey: json.Number("9007199254740995.0")}, + "typedNested": map[string][]float64{"enum": {math.Copysign(0, -1)}}, }, }, } @@ -347,4 +348,7 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) if !strings.Contains(string(out), "default: 9007199254740995") || strings.Contains(string(out), "9007199254740995.0") { t.Fatalf("rendered agent.yaml did not preserve a large integral decimal as an integer\n---\n%s", out) } + if strings.Count(string(out), yamlNegativeZero) != 3 { + t.Fatalf("rendered agent.yaml did not normalize negative zero in typed nested containers\n---\n%s", out) + } } diff --git a/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py index 77d36e4..19060a2 100644 --- a/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py +++ b/runtimes/common/agentkit_serve_common/foundry_brokered_cli.py @@ -79,7 +79,6 @@ def main(argv: Sequence[str] | None = None) -> int: f"agentkit-foundry-brokered: refusing to bind {args.host!r} without AGENTKIT_AUTH_TOKEN; " "set a bearer token or bind 127.0.0.1 for local-only use" ) - app = create_foundry_app(spec, _NoDirectFactory(), auth_token=auth_token) if args.dry_run: print( json.dumps( @@ -95,6 +94,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) ) return 0 + app = create_foundry_app(spec, _NoDirectFactory(), auth_token=auth_token) uvicorn.run(app, host=args.host, port=args.port, log_level="info", access_log=True) return 0 diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index e46e534..0dce6df 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -19,7 +19,7 @@ import httpx -from .adapter_support import AgentBuildError, NO_AUTH_API_KEY, resolve_api_key, resolve_workload_identity_token +from .adapter_support import AgentBuildError, resolve_api_key, resolve_workload_identity_token from .config import AgentSpec from .conversation import FORWARDED_ROLES, RunRequest from .runtime import AgentRunError, BrokeredToolDefinition @@ -150,8 +150,7 @@ async def _chat(self, messages: Sequence[Mapping[str, Any]], *, tools: Sequence[ headers["Authorization"] = f"Bearer {token}" else: api_key = resolve_api_key(self.spec) - if api_key != NO_AUTH_API_KEY: - headers["Authorization"] = f"Bearer {api_key}" + headers["Authorization"] = f"Bearer {api_key}" except AgentBuildError as exc: raise AgentRunError(str(exc), status=400, code="ModelAuthMissing") from exc client = httpx.AsyncClient(headers=headers, timeout=60) diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 7611b0d..dc9bbd4 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -11,6 +11,8 @@ import pytest from agentkit_serve_common import foundry as foundry_module +from agentkit_serve_common import foundry_model_loop as foundry_model_loop_module +from agentkit_serve_common.adapter_support import NO_AUTH_API_KEY from agentkit_serve_common.config import AgentSpec from agentkit_serve_common.conversation import RunRequest from agentkit_serve_common.foundry import create_foundry_app @@ -1219,6 +1221,35 @@ def _model_loop_app(spec: AgentSpec, fake: _FakeChatTransport, **kwargs: Any): return _app(spec, brokered_model_loop_enabled=True, brokered_model_http_client=client, **kwargs) +def test_foundry_brokered_model_loop_sends_placeholder_api_key_when_auth_is_omitted(monkeypatch): + captured_headers: dict[str, str] = {} + + class FakeClient: + def __init__(self, *, headers: dict[str, str], timeout: int) -> None: + assert timeout == 60 + captured_headers.update(headers) + + async def post(self, url: str, *, json: dict[str, Any]) -> httpx.Response: + request = httpx.Request("POST", url, json=json) + return httpx.Response( + 200, + request=request, + json=_chat_response({"role": "assistant", "content": "done"}), + ) + + async def aclose(self) -> None: + return None + + monkeypatch.setattr(foundry_model_loop_module.httpx, "AsyncClient", FakeClient) + app = _app(_spec(tool_name="check-network-telemetry"), brokered_model_loop_enabled=True) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 200, response.text + assert captured_headers["Authorization"] == f"Bearer {NO_AUTH_API_KEY}" + + @pytest.mark.parametrize( "usage", [ diff --git a/runtimes/common/tests/test_foundry_protocol.py b/runtimes/common/tests/test_foundry_protocol.py index 91a0c65..d9541ce 100644 --- a/runtimes/common/tests/test_foundry_protocol.py +++ b/runtimes/common/tests/test_foundry_protocol.py @@ -276,8 +276,13 @@ def test_foundry_protocol_uses_platform_session_env_fallback(monkeypatch): assert factory.runtime.requests[0].session_id == "platform-session" -def test_foundry_brokered_cli_dry_run_loads_static_brokered_agent(tmp_path, capsys): - from agentkit_serve_common.foundry_brokered_cli import main +def test_foundry_brokered_cli_dry_run_loads_static_brokered_agent(tmp_path, capsys, monkeypatch): + from agentkit_serve_common import foundry_brokered_cli + + def fail_if_app_is_constructed(*args, **kwargs): # noqa: ANN002, ANN003 + raise AssertionError("dry-run must not construct the Foundry app or open response state") + + monkeypatch.setattr(foundry_brokered_cli, "create_foundry_app", fail_if_app_is_constructed) config = tmp_path / "agent.yaml" config.write_text( @@ -306,7 +311,7 @@ def test_foundry_brokered_cli_dry_run_loads_static_brokered_agent(tmp_path, caps encoding="utf-8", ) - assert main(["--config", str(config), "--dry-run"]) == 0 + assert foundry_brokered_cli.main(["--config", str(config), "--dry-run"]) == 0 output = capsys.readouterr().out assert '"agent": "brokered-cli"' in output diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index e207ddc..46c6b38 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -61,6 +61,13 @@ def test_verify_brokered_transcript_accepts_conformance_loop(tmp_path): assert "success" in summary["final_text"] +def test_foundry_brokered_conformance_script_uses_private_file_umask(): + script = Path(__file__).parents[3] / "deploy" / "foundry" / "scripts" / "foundry_brokered_conformance.sh" + lines = script.read_text(encoding="utf-8").splitlines() + + assert "umask 077" in lines[:5] + + def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From 7400cd2835bf52ac083c94a5322002699130cd18 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 00:59:25 -0700 Subject: [PATCH 18/27] fix(brokered): harden synthesis and deep copies Signed-off-by: Sertac Ozercan --- .../scripts/verify_brokered_transcript.py | 2 +- pkg/agentkit/effective/agent.go | 148 ++++++++-- pkg/agentkit/effective/agent_test.go | 99 ++++++- .../common/agentkit_serve_common/foundry.py | 267 ++++++++++++++---- .../tests/test_foundry_brokered_protocol.py | 128 ++++++++- .../tests/test_foundry_transcript_verifier.py | 20 ++ 6 files changed, 577 insertions(+), 87 deletions(-) diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index e32f650..d989d52 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -37,7 +37,7 @@ def _require(condition: bool, message: str) -> None: def _message_text(response: dict[str, Any]) -> str: output = response.get("output") - _require(isinstance(output, list) and bool(output), "final response output must be a non-empty array") + _require(isinstance(output, list) and len(output) == 1, "final response output must contain exactly one item") message = output[0] _require(isinstance(message, dict) and message.get("type") == "message", "final response output[0] must be a message") content = message.get("content") diff --git a/pkg/agentkit/effective/agent.go b/pkg/agentkit/effective/agent.go index 559ebed..26e491a 100644 --- a/pkg/agentkit/effective/agent.go +++ b/pkg/agentkit/effective/agent.go @@ -3,6 +3,8 @@ package effective import ( + "bytes" + "encoding/json" "reflect" "github.com/sozercan/agentkit/pkg/agentkit/config" @@ -120,43 +122,78 @@ func copyAny(v any) any { case map[string]any: return copyMap(typed) case map[string]string: + if typed == nil { + return typed + } out := make(map[string]string, len(typed)) for key, value := range typed { out[key] = value } return out case map[string]int: + if typed == nil { + return typed + } out := make(map[string]int, len(typed)) for key, value := range typed { out[key] = value } return out case map[string]float64: + if typed == nil { + return typed + } out := make(map[string]float64, len(typed)) for key, value := range typed { out[key] = value } return out case map[string]bool: + if typed == nil { + return typed + } out := make(map[string]bool, len(typed)) for key, value := range typed { out[key] = value } return out case []any: + if typed == nil { + return typed + } out := make([]any, len(typed)) for i, item := range typed { out[i] = copyAny(item) } return out case []string: - return append([]string(nil), typed...) + if typed == nil { + return typed + } + out := make([]string, len(typed)) + copy(out, typed) + return out case []int: - return append([]int(nil), typed...) + if typed == nil { + return typed + } + out := make([]int, len(typed)) + copy(out, typed) + return out case []float64: - return append([]float64(nil), typed...) + if typed == nil { + return typed + } + out := make([]float64, len(typed)) + copy(out, typed) + return out case []bool: - return append([]bool(nil), typed...) + if typed == nil { + return typed + } + out := make([]bool, len(typed)) + copy(out, typed) + return out default: return copyReflectValue(v) } @@ -165,46 +202,105 @@ func copyAny(v any) any { func copyReflectValue(v any) any { value := reflect.ValueOf(v) switch value.Kind() { + case reflect.Interface: + if value.IsNil() { + return nil + } + return copyAny(value.Elem().Interface()) + case reflect.Pointer: + if value.IsNil() { + return reflect.Zero(value.Type()).Interface() + } + if normalized, ok := copyJSONNormalized(value.Interface()); ok { + return normalized + } + out := reflect.New(value.Type().Elem()) + copied := copyAny(value.Elem().Interface()) + out.Elem().Set(copiedReflectValue(copied, value.Type().Elem(), value.Elem())) + return out.Interface() case reflect.Map: + if value.IsNil() { + return reflect.Zero(value.Type()).Interface() + } out := reflect.MakeMapWithSize(value.Type(), value.Len()) iter := value.MapRange() for iter.Next() { copied := copyAny(iter.Value().Interface()) - copiedValue := reflect.ValueOf(copied) - if copied == nil { - copiedValue = reflect.Zero(value.Type().Elem()) - } else if !copiedValue.Type().AssignableTo(value.Type().Elem()) { - if copiedValue.Type().ConvertibleTo(value.Type().Elem()) { - copiedValue = copiedValue.Convert(value.Type().Elem()) - } else { - copiedValue = iter.Value() - } - } - out.SetMapIndex(iter.Key(), copiedValue) + out.SetMapIndex(iter.Key(), copiedReflectValue(copied, value.Type().Elem(), iter.Value())) } return out.Interface() case reflect.Slice: + if value.IsNil() { + return reflect.Zero(value.Type()).Interface() + } out := reflect.MakeSlice(value.Type(), value.Len(), value.Len()) for i := 0; i < value.Len(); i++ { copied := copyAny(value.Index(i).Interface()) - copiedValue := reflect.ValueOf(copied) - if copied == nil { - copiedValue = reflect.Zero(value.Type().Elem()) - } else if !copiedValue.Type().AssignableTo(value.Type().Elem()) { - if copiedValue.Type().ConvertibleTo(value.Type().Elem()) { - copiedValue = copiedValue.Convert(value.Type().Elem()) - } else { - copiedValue = value.Index(i) - } - } - out.Index(i).Set(copiedValue) + out.Index(i).Set(copiedReflectValue(copied, value.Type().Elem(), value.Index(i))) } return out.Interface() + case reflect.Array: + out := reflect.New(value.Type()).Elem() + for i := 0; i < value.Len(); i++ { + copied := copyAny(value.Index(i).Interface()) + out.Index(i).Set(copiedReflectValue(copied, value.Type().Elem(), value.Index(i))) + } + return out.Interface() + case reflect.Struct: + if normalized, ok := copyJSONNormalized(value.Interface()); ok { + return normalized + } + return value.Interface() default: return v } } +func copyJSONNormalized(value any) (any, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, false + } + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + var out any + if err := decoder.Decode(&out); err != nil { + return nil, false + } + return out, true +} + +func copiedReflectValue(copied any, targetType reflect.Type, fallback reflect.Value) reflect.Value { + if copied == nil { + return reflect.Zero(targetType) + } + value := reflect.ValueOf(copied) + if value.Type().AssignableTo(targetType) { + return value + } + if value.Type().ConvertibleTo(targetType) { + return value.Convert(targetType) + } + if rehydrated, ok := jsonNormalizedToType(copied, targetType); ok { + return rehydrated + } + return fallback +} + +func jsonNormalizedToType(value any, targetType reflect.Type) (reflect.Value, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return reflect.Value{}, false + } + target := reflect.New(targetType) + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(target.Interface()); err != nil { + return reflect.Value{}, false + } + return target.Elem(), true +} + func copyEnvVars(in []config.EnvVar) []config.EnvVar { if len(in) == 0 { return nil diff --git a/pkg/agentkit/effective/agent_test.go b/pkg/agentkit/effective/agent_test.go index 1b0deae..caf8d55 100644 --- a/pkg/agentkit/effective/agent_test.go +++ b/pkg/agentkit/effective/agent_test.go @@ -1,7 +1,10 @@ package effective import ( + "encoding/json" + "math/big" "testing" + "time" "github.com/sozercan/agentkit/pkg/agentkit/config" "github.com/sozercan/agentkit/pkg/agentkit/runtimes" @@ -106,7 +109,22 @@ const ( ) func TestFromConfigCopiesBrokeredTools(t *testing.T) { + type schemaValue struct { + Type string + Enum []string + Value any + } cfg := baseConfig() + pointerNumber := json.Number("1.0") + nilMap := map[string]string(nil) + nilSlice := []string(nil) + emptySlice := []string{} + structPointer := &schemaValue{Type: testSchemaTypeString, Enum: []string{"a", "b"}, Value: int64(9007199254740993)} + typedStructs := []schemaValue{{Type: testSchemaTypeString, Enum: []string{"a", "b"}, Value: int64(9007199254740993)}} + typedPointers := []*schemaValue{{Type: testSchemaTypeString, Enum: []string{"a", "b"}, Value: int64(9007199254740993)}} + timestamp := time.Date(2026, time.July, 11, 12, 0, 0, 0, time.UTC) + bigInteger := big.NewInt(123) + largePointer := &map[string]any{"value": int64(9007199254740993)} cfg.BrokeredTools = []config.BrokeredTool{{ Name: "check-network-telemetry", Description: "Read telemetry.", @@ -114,11 +132,22 @@ func TestFromConfigCopiesBrokeredTools(t *testing.T) { Parameters: map[string]any{ testSchemaTypeKey: "object", "properties": map[string]any{ - testSiteField: map[string]any{testSchemaTypeKey: testSchemaTypeString}, - "typed": map[string]string{testSchemaTypeKey: testSchemaTypeString}, - "generic": map[string][]string{"enum": {"a", "b"}}, - "tuple": []map[string]any{{testSchemaTypeKey: testSchemaTypeString}}, - "empty": map[string]any{}, + testSiteField: map[string]any{testSchemaTypeKey: testSchemaTypeString}, + "typed": map[string]string{testSchemaTypeKey: testSchemaTypeString}, + "generic": map[string][]string{"enum": {"a", "b"}}, + "nilArray": [1]map[string]string{nil}, + "nilPointer": &nilMap, + "nilSlice": &nilSlice, + "emptySlice": &emptySlice, + "pointer": &pointerNumber, + "struct": structPointer, + "typedStructs": typedStructs, + "typedPointers": typedPointers, + "timestamp": ×tamp, + "bigInteger": bigInteger, + "largePointer": largePointer, + "tuple": []map[string]any{{testSchemaTypeKey: testSchemaTypeString}}, + "empty": map[string]any{}, }, "required": []string{testSiteField}, }, @@ -155,6 +184,13 @@ func TestFromConfigCopiesBrokeredTools(t *testing.T) { t.Fatalf("brokered tuple property had unexpected type: %#v", mutatedProperties["tuple"]) } mutatedTuple[0][testSchemaTypeKey] = mutatedValue + pointerNumber = json.Number("2.0") + structPointer.Enum[0] = mutatedValue + typedStructs[0].Enum[0] = mutatedValue + typedPointers[0].Enum[0] = mutatedValue + timestamp = time.Time{} + bigInteger.SetInt64(456) + (*largePointer)["value"] = int64(1) if got := agent.BrokeredTools[0].Parameters[testSchemaTypeKey]; got != "object" { t.Fatalf("brokered tool parameters were not copied: %q", got) @@ -186,6 +222,59 @@ func TestFromConfigCopiesBrokeredTools(t *testing.T) { if !ok || tuple[0][testSchemaTypeKey] != testSchemaTypeString { t.Fatalf("typed brokered schema slice was not copied: %#v", properties["tuple"]) } + pointer, ok := properties["pointer"].(json.Number) + if !ok || pointer.String() != "1.0" { + t.Fatalf("pointer-valued brokered schema value was not deep-copied: %#v", properties["pointer"]) + } + if properties["nilPointer"] != nil { + t.Fatalf("pointer to nil typed map was not preserved: %#v", properties["nilPointer"]) + } + nilArray, ok := properties["nilArray"].([1]map[string]string) + if !ok || nilArray[0] != nil { + t.Fatalf("nil typed map in array was not preserved: %#v", properties["nilArray"]) + } + if properties["nilSlice"] != nil { + t.Fatalf("pointer to nil typed slice was not preserved: %#v", properties["nilSlice"]) + } + emptySliceCopy, ok := properties["emptySlice"].([]any) + if !ok || emptySliceCopy == nil || len(emptySliceCopy) != 0 { + t.Fatalf("pointer to non-nil empty typed slice was not preserved: %#v", properties["emptySlice"]) + } + structCopy, ok := properties["struct"].(map[string]any) + structEnum, enumOK := structCopy["Enum"].([]any) + structValue, valueOK := structCopy["Value"].(json.Number) + if !ok || !enumOK || !valueOK || structCopy["Type"] != testSchemaTypeString || structEnum[0] != "a" || structValue.String() != "9007199254740993" { + t.Fatalf("pointer to struct with slice field was not deep-copied: %#v", properties["struct"]) + } + typedStructCopies, ok := properties["typedStructs"].([]schemaValue) + if !ok || len(typedStructCopies) != 1 { + t.Fatalf("typed struct slice had unexpected shape: %#v", properties["typedStructs"]) + } + typedStructValue, typedStructValueOK := typedStructCopies[0].Value.(json.Number) + if !typedStructValueOK || typedStructCopies[0].Enum[0] != "a" || typedStructValue.String() != "9007199254740993" { + t.Fatalf("typed struct slice was not deep-copied: %#v", properties["typedStructs"]) + } + typedPointerCopies, ok := properties["typedPointers"].([]*schemaValue) + if !ok || len(typedPointerCopies) != 1 || typedPointerCopies[0] == nil { + t.Fatalf("typed pointer slice had unexpected shape: %#v", properties["typedPointers"]) + } + typedPointerValue, typedPointerValueOK := typedPointerCopies[0].Value.(json.Number) + if !typedPointerValueOK || typedPointerCopies[0] == typedPointers[0] || typedPointerCopies[0].Enum[0] != "a" || typedPointerValue.String() != "9007199254740993" { + t.Fatalf("typed pointer slice was not deep-copied: %#v", properties["typedPointers"]) + } + timestampCopy, ok := properties["timestamp"].(string) + if !ok || timestampCopy != "2026-07-11T12:00:00Z" { + t.Fatalf("pointer to opaque struct was not preserved: %#v", properties["timestamp"]) + } + bigIntegerCopy, ok := properties["bigInteger"].(json.Number) + if !ok || bigIntegerCopy.String() != "123" { + t.Fatalf("pointer to mutable opaque struct was not deep-copied: %#v", properties["bigInteger"]) + } + largeCopy, ok := properties["largePointer"].(map[string]any) + largeValue, valueOK := largeCopy["value"].(json.Number) + if !ok || !valueOK || largeValue.String() != "9007199254740993" { + t.Fatalf("pointer-held large integer lost precision during copy: %#v", properties["largePointer"]) + } required, ok := agent.BrokeredTools[0].Parameters["required"].([]string) if !ok || len(required) != 1 || required[0] != testSiteField { t.Fatalf("brokered tool required slice was not copied: %#v", agent.BrokeredTools[0].Parameters["required"]) diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index e71fab0..0ffff8c 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -24,6 +24,7 @@ import time import uuid from decimal import Decimal, InvalidOperation +from fractions import Fraction from pathlib import Path from contextlib import asynccontextmanager from dataclasses import dataclass, field @@ -649,6 +650,118 @@ def _integer_schema_bound(value: Any, *, lower: bool, exclusive: bool) -> int | return None +def _effective_numeric_bounds(schema: Mapping[str, Any]) -> tuple[Fraction | None, bool, Fraction | None, bool]: + lower_value: Fraction | None = None + lower_open = False + for key, exclusive in (("minimum", False), ("exclusiveMinimum", True)): + raw = schema.get(key) + if not isinstance(raw, (int, float)) or isinstance(raw, bool): + continue + if isinstance(raw, float) and not math.isfinite(raw): + raise AgentRunError("brokered tool schema has a non-finite numeric bound", status=400, code="UnsupportedBrokeredSchema") + try: + candidate = Fraction(str(raw)) + except (ValueError, ZeroDivisionError) as exc: + raise AgentRunError("brokered tool schema has an invalid numeric bound", status=400, code="UnsupportedBrokeredSchema") from exc + if lower_value is None or candidate > lower_value: + lower_value = candidate + lower_open = exclusive + elif candidate == lower_value and exclusive: + lower_open = True + + upper_value: Fraction | None = None + upper_open = False + for key, exclusive in (("maximum", False), ("exclusiveMaximum", True)): + raw = schema.get(key) + if not isinstance(raw, (int, float)) or isinstance(raw, bool): + continue + if isinstance(raw, float) and not math.isfinite(raw): + raise AgentRunError("brokered tool schema has a non-finite numeric bound", status=400, code="UnsupportedBrokeredSchema") + try: + candidate = Fraction(str(raw)) + except (ValueError, ZeroDivisionError) as exc: + raise AgentRunError("brokered tool schema has an invalid numeric bound", status=400, code="UnsupportedBrokeredSchema") from exc + if upper_value is None or candidate < upper_value: + upper_value = candidate + upper_open = exclusive + elif candidate == upper_value and exclusive: + upper_open = True + return lower_value, lower_open, upper_value, upper_open + + +def _fraction_floor(value: Fraction) -> int: + return value.numerator // value.denominator + + +def _fraction_ceil(value: Fraction) -> int: + return -((-value.numerator) // value.denominator) + + +def _fraction_json_candidate(value: Fraction, *, name: str, require_exact: bool) -> int | float: + if value.denominator == 1: + integer_text = str(value.numerator) + try: + compact_float = float(value.numerator) + except OverflowError: + compact_float = math.inf + if math.isfinite(compact_float) and Fraction(str(compact_float)) == value and len(str(compact_float)) < len(integer_text): + return compact_float + return value.numerator + try: + candidate = float(value) + except OverflowError as exc: + raise AgentRunError( + f"brokered tool schema for {name!r} has no representable numeric value in bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) from exc + if not math.isfinite(candidate) or (require_exact and Fraction(str(candidate)) != value): + raise AgentRunError( + f"brokered tool schema for {name!r} has no representable numeric value in bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) + return candidate + + +def _fraction_in_numeric_bounds( + candidate: Fraction, + *, + lower_value: Fraction | None, + lower_open: bool, + upper_value: Fraction | None, + upper_open: bool, +) -> bool: + if lower_value is not None and (candidate < lower_value or (candidate == lower_value and lower_open)): + return False + if upper_value is not None and (candidate > upper_value or (candidate == upper_value and upper_open)): + return False + return True + + +def _integer_candidate_from_bounds( + *, + lower_value: Fraction | None, + lower_open: bool, + upper_value: Fraction | None, + upper_open: bool, + step: int = 1, +) -> int: + if lower_value is not None: + quotient = lower_value / step + multiplier = _fraction_ceil(quotient) + if lower_open and quotient.denominator == 1: + multiplier += 1 + return multiplier * step + if upper_value is not None: + quotient = upper_value / step + multiplier = _fraction_floor(quotient) + if upper_open and quotient.denominator == 1: + multiplier -= 1 + return multiplier * step + return 0 + + def _required_property_names(schema: Mapping[str, Any]) -> list[str]: names: list[str] = [] required = schema.get("required") @@ -693,6 +806,12 @@ def _required_property_names(schema: Mapping[str, Any]) -> list[str]: def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> Any: if not isinstance(schema, Mapping): return run_request.prompt + if "multipleOf" in schema: + raise AgentRunError( + f"brokered tool schema for {name!r} has unsupported numeric multipleOf", + status=400, + code="UnsupportedBrokeredSchema", + ) if "const" in schema: return schema["const"] if "default" in schema: @@ -724,16 +843,26 @@ def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> A value = _enum_value(schema, int) if value is not None and not isinstance(value, bool): return value - lower = _integer_schema_bound(schema.get("minimum"), lower=True, exclusive=False) - if lower is None: - lower = _integer_schema_bound(schema.get("exclusiveMinimum"), lower=True, exclusive=True) - if lower is None: - lower = 0 - upper = _integer_schema_bound(schema.get("maximum"), lower=False, exclusive=False) - if upper is None: - upper = _integer_schema_bound(schema.get("exclusiveMaximum"), lower=False, exclusive=True) + lower_candidates = [ + candidate + for candidate in ( + _integer_schema_bound(schema.get("minimum"), lower=True, exclusive=False), + _integer_schema_bound(schema.get("exclusiveMinimum"), lower=True, exclusive=True), + ) + if candidate is not None + ] + upper_candidates = [ + candidate + for candidate in ( + _integer_schema_bound(schema.get("maximum"), lower=False, exclusive=False), + _integer_schema_bound(schema.get("exclusiveMaximum"), lower=False, exclusive=True), + ) + if candidate is not None + ] + lower = max(lower_candidates, default=0) + upper = min(upper_candidates) if upper_candidates else None if upper is not None and lower > upper: - if "minimum" in schema or "exclusiveMinimum" in schema: + if lower_candidates: raise AgentRunError( f"brokered tool schema for {name!r} has incompatible integer bounds", status=400, @@ -741,16 +870,6 @@ def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> A ) lower = upper multiple_of = schema.get("multipleOf") - if isinstance(multiple_of, int) and not isinstance(multiple_of, bool) and multiple_of > 0: - remainder = lower % multiple_of - candidate = lower if remainder == 0 else lower + (multiple_of - remainder) - if upper is not None and candidate > upper: - raise AgentRunError( - f"brokered tool schema for {name!r} has no integer multipleOf value in bounds", - status=400, - code="UnsupportedBrokeredSchema", - ) - return candidate if multiple_of is not None: raise AgentRunError( f"brokered tool schema for {name!r} has unsupported integer multipleOf", @@ -767,50 +886,94 @@ def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> A value = _enum_value(schema, (int, float)) if value is not None and not isinstance(value, bool): return value - lower_value = schema.get("minimum") - lower_open = False - if not isinstance(lower_value, (int, float)) or isinstance(lower_value, bool): - lower_value = schema.get("exclusiveMinimum") - lower_open = isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) - upper_value = schema.get("maximum") - upper_open = False - if not isinstance(upper_value, (int, float)) or isinstance(upper_value, bool): - upper_value = schema.get("exclusiveMaximum") - upper_open = isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) - has_lower = isinstance(lower_value, (int, float)) and not isinstance(lower_value, bool) - has_upper = isinstance(upper_value, (int, float)) and not isinstance(upper_value, bool) - if has_lower and has_upper: - if lower_value > upper_value or (lower_value == upper_value and (lower_open or upper_open)): + lower_fraction, lower_open, upper_fraction, upper_open = _effective_numeric_bounds(schema) + has_lower = lower_fraction is not None + has_upper = upper_fraction is not None + if lower_fraction is not None and upper_fraction is not None: + if lower_fraction > upper_fraction or (lower_fraction == upper_fraction and (lower_open or upper_open)): raise AgentRunError( f"brokered tool schema for {name!r} has incompatible numeric bounds", status=400, code="UnsupportedBrokeredSchema", ) - if lower_value == upper_value: - return lower_value - return (lower_value + upper_value) / 2 - if has_lower: - candidate = lower_value + (1 if lower_open else 0) - elif has_upper: - candidate = upper_value - (1 if upper_open else 0) - else: - candidate = 0 + if lower_fraction == upper_fraction: + return _fraction_json_candidate(lower_fraction, name=name, require_exact=True) + multiple_of = schema.get("multipleOf") - if isinstance(multiple_of, (int, float)) and not isinstance(multiple_of, bool) and multiple_of > 0: - candidate = math.ceil(candidate / multiple_of) * multiple_of - if has_upper and (candidate > upper_value or (candidate == upper_value and upper_open)): - raise AgentRunError( - f"brokered tool schema for {name!r} has no numeric multipleOf value in bounds", - status=400, - code="UnsupportedBrokeredSchema", - ) - elif multiple_of is not None: + if multiple_of is not None: raise AgentRunError( f"brokered tool schema for {name!r} has unsupported numeric multipleOf", status=400, code="UnsupportedBrokeredSchema", ) - return candidate + + candidates: list[int | float] = [] + + def add_candidate(candidate: int | float) -> None: + if isinstance(candidate, float) and not math.isfinite(candidate): + return + if _fraction_in_numeric_bounds( + Fraction(str(candidate)), + lower_value=lower_fraction, + lower_open=lower_open, + upper_value=upper_fraction, + upper_open=upper_open, + ): + candidates.append(candidate) + + add_candidate(0) + + for boundary, is_open in ( + (lower_fraction, lower_open), + (upper_fraction, upper_open), + ): + if boundary is None or is_open: + continue + try: + add_candidate(_fraction_json_candidate(boundary, name=name, require_exact=True)) + except AgentRunError: + pass + + for boundary, is_open, direction in ( + (lower_fraction, lower_open, math.inf), + (upper_fraction, upper_open, -math.inf), + ): + if boundary is None or not is_open: + continue + try: + boundary_float = float(boundary) + except OverflowError: + continue + if math.isfinite(boundary_float): + add_candidate(math.nextafter(boundary_float, direction)) + + integer_candidate = _integer_candidate_from_bounds( + lower_value=lower_fraction, + lower_open=lower_open, + upper_value=upper_fraction, + upper_open=upper_open, + step=1, + ) + add_candidate(integer_candidate) + + if lower_fraction is not None and upper_fraction is not None: + midpoint = (lower_fraction + upper_fraction) / 2 + try: + add_candidate(_fraction_json_candidate(midpoint, name=name, require_exact=False)) + except AgentRunError: + pass + + if candidates: + return min( + candidates, + key=lambda candidate: len(json.dumps(candidate, allow_nan=False, separators=(",", ":"))), + ) + + raise AgentRunError( + f"brokered tool schema for {name!r} has no representable numeric value in bounds", + status=400, + code="UnsupportedBrokeredSchema", + ) if schema_type == "array": for key in ("const", "default"): diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index dc9bbd4..0065a46 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math import time from copy import deepcopy from types import TracebackType @@ -527,7 +528,7 @@ def test_foundry_brokered_argument_synthesis_honors_dependent_required_and_min_p assert json.loads(_call(body)["arguments"]) == {"site": "check-network-telemetry", "region": "west", "extra": True} -def test_foundry_brokered_integer_synthesis_honors_multiple_of(): +def test_foundry_brokered_integer_synthesis_rejects_unsupported_multiple_of(): spec = _spec(tool_name="check-network-telemetry") spec.brokered_tools[0].parameters = { "type": "object", @@ -537,9 +538,10 @@ def test_foundry_brokered_integer_synthesis_honors_multiple_of(): app = _app(spec) with TestClient(app) as client: - body = client.post("/responses", json={"input": "check-network-telemetry"}).json() + response = client.post("/responses", json={"input": "check-network-telemetry"}) - assert json.loads(_call(body)["arguments"]) == {"n": 2} + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsupportedBrokeredSchema" def test_foundry_brokered_number_synthesis_uses_midpoint_for_fractional_exclusive_range(): @@ -557,6 +559,126 @@ def test_foundry_brokered_number_synthesis_uses_midpoint_for_fractional_exclusiv assert json.loads(_call(body)["arguments"]) == {"ratio": 0.5} +def test_foundry_brokered_number_synthesis_handles_large_finite_bounds_without_overflow(): + spec = _spec(tool_name="large-number-bounds") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "midpoint": {"type": "number", "minimum": 1e308, "maximum": 1.1e308}, + "above": {"type": "number", "exclusiveMinimum": 1e308}, + "below": {"type": "number", "exclusiveMaximum": -1e308}, + "exactInteger": {"type": "number", "minimum": 9007199254740993, "maximum": 9007199254740994}, + "roundedMidpoint": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 0.9999999999999999}, + "narrowValue": {"type": "number", "exclusiveMinimum": 1.0, "maximum": 1.0000000000000002}, + }, + "required": ["midpoint", "above", "below", "exactInteger", "roundedMidpoint", "narrowValue"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "large-number-bounds"}) + + assert response.status_code == 200, response.text + arguments = json.loads(_call(response.json())["arguments"]) + assert arguments["midpoint"] == 1e308 + assert arguments["above"] == math.nextafter(1e308, math.inf) + assert arguments["below"] == math.nextafter(-1e308, -math.inf) + assert arguments["exactInteger"] == 9007199254740993 + assert 0 < arguments["roundedMidpoint"] < 0.9999999999999999 + assert arguments["narrowValue"] == 1.0000000000000002 + + +def test_foundry_brokered_number_synthesis_rejects_unsupported_multiple_of(): + spec = _spec(tool_name="number-multiple") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"value": {"type": "number", "minimum": 0, "multipleOf": 0.5}}, + "required": ["value"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "number-multiple"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_number_synthesis_combines_all_declared_bounds(): + spec = _spec(tool_name="combined-number-bounds") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "value": {"type": "number", "minimum": 0, "exclusiveMinimum": 0, "maximum": 1}, + }, + "required": ["value"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "combined-number-bounds"}) + + assert response.status_code == 200, response.text + assert json.loads(_call(response.json())["arguments"]) == {"value": 1} + + +@pytest.mark.parametrize("bound", [float("nan"), float("inf"), float("-inf")]) +def test_foundry_brokered_number_synthesis_rejects_nonfinite_bounds(bound: float): + spec = _spec(tool_name="nonfinite-number-bound") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"value": {"type": "number", "minimum": bound}}, + "required": ["value"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "nonfinite-number-bound"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsupportedBrokeredSchema" + + +def test_foundry_brokered_number_synthesis_prefers_compact_zero_within_wide_bounds(): + spec = _spec(tool_name="compact-number-bounds") + properties = { + f"value{index}": {"type": "number", "minimum": -1e308, "maximum": 1e308} + for index in range(27) + } + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": properties, + "required": list(properties), + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "compact-number-bounds"}) + + assert response.status_code == 200, response.text + assert json.loads(_call(response.json())["arguments"]) == {name: 0 for name in properties} + + +def test_foundry_brokered_number_synthesis_prefers_compact_float_boundary_over_large_integer(): + spec = _spec(tool_name="compact-large-number-bounds") + properties = { + f"value{index}": {"type": "number", "minimum": 1e308, "maximum": 1.1e308} + for index in range(27) + } + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": properties, + "required": list(properties), + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "compact-large-number-bounds"}) + + assert response.status_code == 200, response.text + assert json.loads(_call(response.json())["arguments"]) == {name: 1e308 for name in properties} + + def test_foundry_brokered_rejects_unbounded_schema_synthesis_before_allocating(): spec = _spec(tool_name="check-network-telemetry") spec.brokered_tools[0].parameters = { diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index 46c6b38..92bdbd4 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -95,6 +95,26 @@ def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_ verifier.verify_transcript(transcript) +def test_verify_brokered_transcript_rejects_extra_final_output_items(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation = json.loads((transcript / "04-continuation-response.json").read_text(encoding="utf-8")) + continuation["output"].append( + { + "type": "function_call", + "id": "fc_unexpected", + "call_id": "call_unexpected", + "name": "unexpected", + "arguments": "{}", + "status": "completed", + } + ) + (transcript / "04-continuation-response.json").write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="must contain exactly one item"): + verifier.verify_transcript(transcript) + + def test_verify_brokered_transcript_cli_writes_summary(tmp_path, capsys): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From e6d3b2587d9387249d6303577170366822ba1898 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 01:27:21 -0700 Subject: [PATCH 19/27] fix(brokered): close conformance and continuation gaps Signed-off-by: Sertac Ozercan --- .../scripts/foundry_brokered_conformance.sh | 34 ++++++--- docs/agent-abi.md | 27 ++++---- pkg/agentkit/config/config_test.go | 33 +++++++++ pkg/agentkit/config/validate.go | 8 +++ .../common/agentkit_serve_common/foundry.py | 13 +++- .../tests/test_foundry_brokered_protocol.py | 69 +++++++++++++++++++ .../tests/test_foundry_transcript_verifier.py | 9 +++ 7 files changed, 168 insertions(+), 25 deletions(-) diff --git a/deploy/foundry/scripts/foundry_brokered_conformance.sh b/deploy/foundry/scripts/foundry_brokered_conformance.sh index 4b73433..e3a5012 100755 --- a/deploy/foundry/scripts/foundry_brokered_conformance.sh +++ b/deploy/foundry/scripts/foundry_brokered_conformance.sh @@ -36,6 +36,17 @@ if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then exit 0 fi +write_curl_header() { + local header="$1" + if [[ "$header" == *$'\r'* || "$header" == *$'\n'* ]]; then + echo "refusing curl header containing a newline" >&2 + return 1 + fi + header="${header//\\/\\\\}" + header="${header//\"/\\\"}" + printf 'header = "%s"\n' "$header" +} + : "${AGENT_RESPONSES_ENDPOINT:?set AGENT_RESPONSES_ENDPOINT to the deployed /responses URL}" prompt="${1:-conformance_read}" @@ -70,11 +81,13 @@ import os print(json.dumps({"input": os.environ["PROMPT"]}, separators=(",", ":"))) PY -curl -fsS \ - -H "Authorization: Bearer ${token}" \ +initial_curl_config="$(write_curl_header "Authorization: Bearer ${token}")" +printf '%s\n' "$initial_curl_config" | curl -fsS \ + --config - \ -H 'content-type: application/json' \ "$AGENT_RESPONSES_ENDPOINT" \ -d "@$initial_request" >"$initial_response" +unset initial_curl_config read -r response_id call_id < <(EXPECTED_TOOL_NAME="$expected_tool_name" EXPECTED_ARGUMENTS="$expected_arguments" EXPECTED_CALL_ID="$expected_call_id" EXPECTED_CALL_ID_PREFIX="$expected_call_id_prefix" python3 - "$initial_response" <<'PY' import json @@ -123,15 +136,18 @@ print(json.dumps({ }, separators=(",", ":"))) PY -continuation_headers=(-H "Authorization: Bearer ${token}" -H 'content-type: application/json') -if [[ -n "${AGENTKIT_CONTINUATION_PROOF:-}" ]]; then - continuation_headers+=(-H "x-agentkit-brokered-continuation-proof: ${AGENTKIT_CONTINUATION_PROOF}") -fi - -curl -fsS \ - "${continuation_headers[@]}" \ +continuation_curl_config="$({ + write_curl_header "Authorization: Bearer ${token}" + if [[ -n "${AGENTKIT_CONTINUATION_PROOF:-}" ]]; then + write_curl_header "x-agentkit-brokered-continuation-proof: ${AGENTKIT_CONTINUATION_PROOF}" + fi +})" +printf '%s\n' "$continuation_curl_config" | curl -fsS \ + --config - \ + -H 'content-type: application/json' \ "$AGENT_RESPONSES_ENDPOINT" \ -d "@$continuation_request" >"$continuation_response" +unset continuation_curl_config verifier_args=( "$transcript_dir" diff --git a/docs/agent-abi.md b/docs/agent-abi.md index ec0a6a9..499bb9d 100644 --- a/docs/agent-abi.md +++ b/docs/agent-abi.md @@ -57,19 +57,20 @@ tools: type: bearer tokenEnv: TOOLBOX_TOKEN -# Static safe schemas for Foundry hosted Orka-brokered mode. These are schema-only; -# no execution URL, auth header, token, or Secret ref is allowed here. -brokeredTools: - - name: check-network-telemetry - description: Read sanitized optical telemetry. - brokeredClass: read - parameters: - type: object - properties: - site: - type: string - required: [site] - schemaDigest: sha256: +# Alternative to `tools` above for Foundry hosted Orka-brokered mode. v0 does +# not allow owned `tools` and `brokeredTools` together, so remove/comment the +# `tools` block before enabling this schema-only block. +# brokeredTools: +# - name: check-network-telemetry +# description: Read sanitized optical telemetry. +# brokeredClass: read +# parameters: +# type: object +# properties: +# site: +# type: string +# required: [site] +# schemaDigest: sha256: env: - name: REQUIRED_FOO diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index de22c45..8548b36 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -1153,6 +1153,39 @@ func TestBrokeredToolSchemaDigestMatchesPythonCanonicalJSON(t *testing.T) { } } +func TestBrokeredToolSchemaDigestPreservesIntegralJSONNumberDecimals(t *testing.T) { + decimalTool := BrokeredTool{ + Name: "numeric-tool", + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassRead, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "id": map[string]any{jsonSchemaDefaultKey: json.Number("9007199254740995.0")}, + }, + }, + } + integerTool := decimalTool + integerTool.Parameters = map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "id": map[string]any{jsonSchemaDefaultKey: int64(9007199254740995)}, + }, + } + + decimalDigest, err := BrokeredToolSchemaDigest(decimalTool) + if err != nil { + t.Fatalf("decimal digest error: %v", err) + } + integerDigest, err := BrokeredToolSchemaDigest(integerTool) + if err != nil { + t.Fatalf("integer digest error: %v", err) + } + if decimalDigest != integerDigest { + t.Fatalf("integral decimal digest = %s, integer digest = %s", decimalDigest, integerDigest) + } +} + func TestCanonicalJSONStringMatchesPythonForUnicodeLineSeparators(t *testing.T) { value := "line\u2028paragraph\u2029literal\\u2028" diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index bfc119a..d31a183 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -877,6 +877,14 @@ func unescapeJSONLineSeparators(encoded []byte) []byte { } func canonicalJSONNumberString(value string) (string, error) { + if json.Valid([]byte(value)) { + if number, ok := new(big.Rat).SetString(value); ok && number.IsInt() { + if number.Sign() == 0 && strings.HasPrefix(value, "-") { + return "-0", nil + } + return number.Num().String(), nil + } + } if !strings.ContainsAny(value, ".eE") { return value, nil } diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 0ffff8c..f6e652a 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -604,8 +604,7 @@ def _json_object_from_output(output: Any) -> dict[str, Any]: unexpected = set(parsed) - {"approved", "output"} if unexpected: raise ValueError("approved function_call_output.output contains unsupported fields") - tool_output = parsed.get("output", {}) - if tool_output is not None and not isinstance(tool_output, dict): + if "output" not in parsed or not isinstance(parsed["output"], dict): raise ValueError("approved function_call_output.output.output must be an object") else: unexpected = set(parsed) - {"approved", "error"} @@ -1422,6 +1421,12 @@ async def _handle_brokered_continuation( status=409, code="conflicting_duplicate_continuation", ) + if state.model_messages is not None and model_loop is None: + return _error( + "brokered model-loop continuation is unavailable for this pending response", + status=503, + code="brokered_model_loop_unavailable", + ) if state.status != "pending": return _error("previous response is not pending a tool result", status=409, code="response_not_pending") @@ -1612,7 +1617,9 @@ async def responses(request: Request): if brokered_tools and isinstance(previous_response_id, str) and previous_response_id: try: previous_state = response_states.get(previous_response_id) - except (KeyError, _StateExpired): + except _StateExpired: + return _error("previous_response_id state has expired", status=410, code="response_state_expired") + except KeyError: previous_state = None if previous_state is not None and previous_state.status in {"pending", "resuming"}: return _error( diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 0065a46..c302cb0 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1119,6 +1119,43 @@ def test_foundry_brokered_file_state_survives_restart_for_model_loop_continuatio assert second_fake.requests[0]["messages"][-1]["tool_call_id"] == call["call_id"] +def test_foundry_brokered_rejects_persisted_model_loop_continuation_when_loop_is_disabled(tmp_path): + state_file = tmp_path / "foundry-model-state.json" + spec = _spec(tool_name="check-network-telemetry") + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ) + ] + ) + with TestClient(_model_loop_app(spec, fake, response_state_file=state_file)) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}).json() + call = _call(initial) + + persisted_before = state_file.read_bytes() + with TestClient(_app(spec, response_state_file=state_file, brokered_model_loop_enabled=False)) as client: + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + + assert response.status_code == 503 + assert response.json()["error"]["code"] == "brokered_model_loop_unavailable" + assert state_file.read_bytes() == persisted_before + + def test_foundry_brokered_rejects_expired_response_state(): app = _app(state_ttl_seconds=0) @@ -1136,6 +1173,21 @@ def test_foundry_brokered_rejects_expired_response_state(): assert resp.json()["error"]["code"] == "response_state_expired" +def test_foundry_brokered_rejects_normal_followup_to_expired_pending_response(): + app = _app(state_ttl_seconds=0) + + with TestClient(app) as client: + initial = _start(client) + time.sleep(0.01) + response = client.post( + "/responses", + json={"previous_response_id": initial["id"], "input": "next question"}, + ) + + assert response.status_code == 410 + assert response.json()["error"]["code"] == "response_state_expired" + + def test_foundry_brokered_refuses_to_synthesize_nonliteral_write_arguments(): spec = _spec(tool_name="dispatch-work-order", brokered_class="write") spec.brokered_tools[0].parameters = { @@ -1527,6 +1579,23 @@ def test_foundry_brokered_model_loop_rejects_noncanonical_denied_output_before_r assert len(fake.requests) == 1 +@pytest.mark.parametrize("payload", [{"approved": True}, {"approved": True, "output": None}]) +def test_foundry_brokered_rejects_approved_continuation_without_object_output(payload: dict[str, Any]): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], payload), + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_function_call_output" + + def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_state_change(tmp_path): state_file = tmp_path / "responses-state.json" spec = _spec(tool_name="check-network-telemetry") diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index 92bdbd4..ba5f696 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -68,6 +68,15 @@ def test_foundry_brokered_conformance_script_uses_private_file_umask(): assert "umask 077" in lines[:5] +def test_foundry_brokered_conformance_script_keeps_sensitive_headers_out_of_curl_argv(): + script = Path(__file__).parents[3] / "deploy" / "foundry" / "scripts" / "foundry_brokered_conformance.sh" + text = script.read_text(encoding="utf-8") + + assert text.count("--config -") == 2 + assert '-H "Authorization: Bearer ${token}"' not in text + assert '-H "x-agentkit-brokered-continuation-proof:' not in text + + def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From 75957e68b5d0fee694e71693415c7077e66aead4 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 01:48:03 -0700 Subject: [PATCH 20/27] fix(foundry): harden persisted continuation state Signed-off-by: Sertac Ozercan --- .../common/agentkit_serve_common/foundry.py | 28 ++++-- .../tests/test_foundry_brokered_protocol.py | 89 +++++++++++++++++-- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index f6e652a..5378f86 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -452,14 +452,17 @@ def _load(self) -> None: return try: data = json.loads(self.state_file.read_text(encoding="utf-8")) - states = data.get("states", {}) if isinstance(data, Mapping) else {} + if not isinstance(data, Mapping): + raise ValueError("Foundry response state file root must be an object") + states = data.get("states", {}) if not isinstance(states, Mapping): raise ValueError("Foundry response state file states must be an object") - self._states = {str(response_id): _state_from_payload(state) for response_id, state in states.items() if isinstance(state, Mapping)} + if not all(isinstance(state, Mapping) for state in states.values()): + raise ValueError("Foundry response state entries must be objects") + self._states = {str(response_id): _state_from_payload(state) for response_id, state in states.items()} self.purge_expired() except (OSError, json.JSONDecodeError, ValueError) as exc: - logger.warning("ignoring invalid Foundry response state file %s: %s", self.state_file, exc) - self._states = {} + raise RuntimeError(f"invalid Foundry response state file {self.state_file}: {exc}") from exc def _persist(self) -> None: if self.state_file is None: @@ -610,8 +613,7 @@ def _json_object_from_output(output: Any) -> dict[str, Any]: unexpected = set(parsed) - {"approved", "error"} if unexpected: raise ValueError("denied function_call_output.output contains unsupported fields") - error = parsed.get("error", {}) - if error is not None and not isinstance(error, dict): + if "error" not in parsed or not isinstance(parsed["error"], dict): raise ValueError("denied function_call_output.output.error must be an object") _reject_nonfinite_json_values(parsed, path="function_call_output.output") return json.loads(json.dumps(parsed, allow_nan=False, separators=(",", ":"), sort_keys=True)) @@ -1393,8 +1395,19 @@ async def _handle_brokered_continuation( call = state.pending_calls.get(call_id) if call is None: return _error("unknown function_call_output call_id", status=400, code="unknown_call_id") + raw_output = item.get("output") + if isinstance(raw_output, str): + try: + if len(raw_output) > max_output_bytes or len(raw_output.encode("utf-8")) > max_output_bytes: + return _error( + "brokered function_call_output is too large", + status=413, + code="brokered_output_too_large", + ) + except UnicodeEncodeError as exc: + return _error(str(exc), status=400, code="invalid_function_call_output") try: - parsed_output = _json_object_from_output(item.get("output")) + parsed_output = _json_object_from_output(raw_output) except ValueError as exc: return _error(str(exc), status=400, code="invalid_function_call_output") output_json = _canonical_output_json(parsed_output) @@ -1431,6 +1444,7 @@ async def _handle_brokered_continuation( return _error("previous response is not pending a tool result", status=409, code="response_not_pending") state.accepted_outputs[call_id] = output_json + state.expires_at = time.time() + store.ttl_seconds store.save(state) if state.model_messages is not None and model_loop is not None: state.status = "resuming" diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index c302cb0..9d9fec1 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -921,6 +921,35 @@ def test_foundry_brokered_rejects_oversized_function_call_output_before_state_ch assert accepted.status_code == 200, accepted.text +def test_foundry_brokered_rejects_oversized_raw_output_before_json_parsing(monkeypatch): + app = _app(max_brokered_output_bytes=64) + + def fail_if_parsed(_output: Any) -> dict[str, Any]: + raise AssertionError("oversized raw output must be rejected before JSON parsing") + + monkeypatch.setattr(foundry_module, "_json_object_from_output", fail_if_parsed) + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": '{"approved":true,"output":{"blob":"' + ("x" * 128) + '"}}', + } + ], + }, + ) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_output_too_large" + + def test_foundry_brokered_continuation_accepts_matching_tool_output_and_completes(): app = _app() @@ -941,6 +970,34 @@ def test_foundry_brokered_continuation_accepts_matching_tool_output_and_complete assert _message_text(final) == 'Brokered tool conformance_read completed with output: {"success":true}' +def test_foundry_brokered_accepting_continuation_refreshes_state_expiry(monkeypatch): + stores: list[Any] = [] + original_store = foundry_module._FoundryResponseStateStore + + def capture_store(*args: Any, **kwargs: Any) -> Any: + store = original_store(*args, **kwargs) + stores.append(store) + return store + + monkeypatch.setattr(foundry_module, "_FoundryResponseStateStore", capture_store) + app = _app(state_ttl_seconds=60) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + state = stores[0].get(initial["id"]) + state.expires_at = time.time() + 1 + stores[0].save(state) + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + + assert response.status_code == 200, response.text + assert state.expires_at > time.time() + 50 + + def test_foundry_brokered_rejects_function_call_output_without_orka_continuation_auth(): app = _app() @@ -1596,6 +1653,23 @@ def test_foundry_brokered_rejects_approved_continuation_without_object_output(pa assert response.json()["error"]["code"] == "invalid_function_call_output" +@pytest.mark.parametrize("payload", [{"approved": False}, {"approved": False, "error": None}]) +def test_foundry_brokered_rejects_denied_continuation_without_error_object(payload: dict[str, Any]): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], payload), + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_function_call_output" + + def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_state_change(tmp_path): state_file = tmp_path / "responses-state.json" spec = _spec(tool_name="check-network-telemetry") @@ -1643,18 +1717,15 @@ def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_s assert len(fake.requests) == 1 -def test_foundry_brokered_invalid_file_state_starts_with_empty_store(tmp_path): +@pytest.mark.parametrize("raw_state", ["{not valid json", "[]"]) +def test_foundry_brokered_invalid_file_state_fails_startup_without_overwriting(tmp_path, raw_state: str): state_file = tmp_path / "responses-state.json" - state_file.write_text("{not valid json", encoding="utf-8") - app = _app(response_state_file=state_file) + state_file.write_text(raw_state, encoding="utf-8") - with TestClient(app) as client: - response = client.get("/readiness") - initial = client.post("/responses", json={"input": "conformance_read"}) + with pytest.raises(RuntimeError, match="invalid Foundry response state file"): + _app(response_state_file=state_file) - assert response.status_code == 200 - assert initial.status_code == 200, initial.text - assert _call(initial.json())["name"] == "conformance_read" + assert state_file.read_text(encoding="utf-8") == raw_state def test_foundry_brokered_file_state_is_written_with_private_permissions(tmp_path): From 0b33dc253b126ec703e3eae7232502d7d3e91a51 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 02:54:40 -0700 Subject: [PATCH 21/27] fix(conformance): harden transcript evidence Signed-off-by: Sertac Ozercan --- .../scripts/foundry_brokered_conformance.sh | 6 ++ .../scripts/verify_brokered_transcript.py | 81 +++++++++++++++++-- docs/foundry-hosted-brokered.md | 6 ++ pkg/agentkit/abi/render.go | 30 +++++++ pkg/agentkit/abi/render_test.go | 13 ++- .../tests/test_foundry_transcript_verifier.py | 47 +++++++++++ 6 files changed, 174 insertions(+), 9 deletions(-) diff --git a/deploy/foundry/scripts/foundry_brokered_conformance.sh b/deploy/foundry/scripts/foundry_brokered_conformance.sh index e3a5012..75daf36 100755 --- a/deploy/foundry/scripts/foundry_brokered_conformance.sh +++ b/deploy/foundry/scripts/foundry_brokered_conformance.sh @@ -74,6 +74,9 @@ initial_response="$transcript_dir/02-initial-response.json" continuation_request="$transcript_dir/03-continuation-request.json" continuation_response="$transcript_dir/04-continuation-response.json" summary_file="$transcript_dir/summary.json" +expected_output_file="$transcript_dir/.expected-output.json" +trap 'rm -f -- "$expected_output_file"' EXIT +printf '%s' "$conformance_output" >"$expected_output_file" PROMPT="$prompt" python3 - <<'PY' >"$initial_request" import json @@ -153,6 +156,7 @@ verifier_args=( "$transcript_dir" --expected-tool-name "$expected_tool_name" --expected-arguments-json "$expected_arguments" + --expected-output-file "$expected_output_file" --expected-call-id "$expected_call_id" --write-summary ) @@ -161,6 +165,8 @@ if [[ -n "$expected_call_id_prefix" ]]; then fi python3 deploy/foundry/scripts/verify_brokered_transcript.py "${verifier_args[@]}" >"$summary_file.tmp" rm -f "$summary_file.tmp" +rm -f "$expected_output_file" +trap - EXIT echo "Foundry brokered conformance passed. Sanitized transcript: ${transcript_dir}" cat "$summary_file" diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index d989d52..ac31ed0 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -10,6 +10,7 @@ import argparse import json import sys +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any @@ -19,6 +20,7 @@ "03-continuation-request.json", "04-continuation-response.json", ) +_MAX_SUMMARY_INTEGER_DIGITS = 4096 def _load_json(path: Path) -> Any: @@ -35,6 +37,64 @@ def _require(condition: bool, message: str) -> None: raise ValueError(message) +def _reject_json_constant(raw: str) -> None: + raise ValueError(f"non-finite JSON number {raw} is not allowed") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise ValueError(f"duplicate JSON object key {key!r} is not allowed") + out[key] = value + return out + + +def _parse_json_lossless(raw: str) -> Any: + return json.loads( + raw, + parse_int=Decimal, + parse_float=Decimal, + parse_constant=_reject_json_constant, + object_pairs_hook=_reject_duplicate_keys, + ) + + +def _strict_json_equal(left: Any, right: Any) -> bool: + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left == right + if isinstance(left, (int, float, Decimal)) and isinstance(right, (int, float, Decimal)): + try: + return Decimal(str(left)) == Decimal(str(right)) + except InvalidOperation: + return False + if type(left) is not type(right): + return False + if isinstance(left, dict): + return left.keys() == right.keys() and all(_strict_json_equal(left[key], right[key]) for key in left) + if isinstance(left, list): + return len(left) == len(right) and all(_strict_json_equal(a, b) for a, b in zip(left, right)) + return left == right + + +def _json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {key: _json_compatible(child) for key, child in value.items()} + if isinstance(value, list): + return [_json_compatible(child) for child in value] + if isinstance(value, Decimal): + if value == value.to_integral_value(): + digits = max(value.adjusted() + 1, len(value.as_tuple().digits)) if value else 1 + if digits > _MAX_SUMMARY_INTEGER_DIGITS: + raise ValueError("numeric argument is too large to include safely in the summary") + return int(value) + candidate = float(value) + if Decimal(str(candidate)) == value: + return candidate + return str(value) + return value + + def _message_text(response: dict[str, Any]) -> str: output = response.get("output") _require(isinstance(output, list) and len(output) == 1, "final response output must contain exactly one item") @@ -52,11 +112,13 @@ def verify_transcript( *, expected_tool_name: str = "conformance_read", expected_arguments_json: str = '{"probe":true}', + expected_output_json: str = '{"approved":true,"output":{"success":true}}', expected_call_id: str = "call_conformance_1", expected_call_id_prefix: str | None = None, ) -> dict[str, Any]: root = Path(transcript_dir) - expected_arguments = json.loads(expected_arguments_json) + expected_arguments = _parse_json_lossless(expected_arguments_json) + expected_output = _parse_json_lossless(expected_output_json) initial_request = _load_json(root / "01-initial-request.json") initial_response = _load_json(root / "02-initial-response.json") continuation_request = _load_json(root / "03-continuation-request.json") @@ -86,8 +148,8 @@ def verify_transcript( _require(call_id.startswith(expected_call_id_prefix), f"function_call call_id must start with {expected_call_id_prefix}") arguments = call.get("arguments") _require(isinstance(arguments, str), "function_call arguments must be a JSON string") - parsed_arguments = json.loads(arguments) - _require(parsed_arguments == expected_arguments, f"function_call arguments must be {expected_arguments}") + parsed_arguments = _parse_json_lossless(arguments) + _require(_strict_json_equal(parsed_arguments, expected_arguments), f"function_call arguments must be {expected_arguments}") _require(isinstance(continuation_request, dict), "continuation request must be a JSON object") _require(continuation_request.get("previous_response_id") == initial_response_id, "continuation previous_response_id must match initial id") @@ -99,8 +161,9 @@ def verify_transcript( _require(continuation_item.get("call_id") == call_id, "continuation call_id must match function_call call_id") continuation_output = continuation_item.get("output") _require(isinstance(continuation_output, str), "continuation output must be a JSON string") - parsed_output = json.loads(continuation_output) + parsed_output = _parse_json_lossless(continuation_output) _require(isinstance(parsed_output, dict), "continuation output JSON must be an object") + _require(_strict_json_equal(parsed_output, expected_output), "continuation output did not match expected JSON") _require(isinstance(continuation_response, dict), "continuation response must be a JSON object") _require(continuation_response.get("status") == "completed", "continuation response status must be completed") @@ -115,7 +178,7 @@ def verify_transcript( "continuation_response_id": continuation_response_id, "function_call_name": function_name, "call_id": call_id, - "arguments": parsed_arguments, + "arguments": _json_compatible(parsed_arguments), "final_text": final_text, "transcript_files": list(EXPECTED_FILES), } @@ -126,6 +189,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("transcript_dir", help="directory containing 01/02/03/04 conformance transcript JSON files") parser.add_argument("--expected-tool-name", default="conformance_read", help="expected function_call name") parser.add_argument("--expected-arguments-json", default='{"probe":true}', help="expected function_call arguments JSON") + parser.add_argument("--expected-output-json", default='{"approved":true,"output":{"success":true}}', help="expected function_call_output JSON") + parser.add_argument("--expected-output-file", default=None, help="private file containing expected function_call_output JSON") parser.add_argument("--expected-call-id", default="call_conformance_1", help="expected call_id, or 'auto' to only require a non-empty id") parser.add_argument("--expected-call-id-prefix", default=None, help="optional required call_id prefix") parser.add_argument("--write-summary", action="store_true", help="write summary.json in the transcript directory") @@ -135,10 +200,16 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) try: + expected_output_json = ( + Path(args.expected_output_file).read_text(encoding="utf-8") + if args.expected_output_file + else args.expected_output_json + ) summary = verify_transcript( args.transcript_dir, expected_tool_name=args.expected_tool_name, expected_arguments_json=args.expected_arguments_json, + expected_output_json=expected_output_json, expected_call_id=args.expected_call_id, expected_call_id_prefix=args.expected_call_id_prefix, ) diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md index 4f484e0..aed4eef 100644 --- a/docs/foundry-hosted-brokered.md +++ b/docs/foundry-hosted-brokered.md @@ -190,6 +190,12 @@ they are persisted, replayed, embedded in deterministic responses, or sent back through the model loop. A platform-managed state backend is still required before treating multi-replica production as fully supported. +The file is sensitive runtime state, not harmless metadata. In model-loop mode +it includes model messages such as system instructions, conversation history, +and user prompts, in addition to brokered arguments/outputs and cached final +payloads. Store it on access-controlled storage and apply an appropriate +retention/deletion policy. + ## Streaming The current route is non-streaming. If clients send `stream: true`, AgentKit diff --git a/pkg/agentkit/abi/render.go b/pkg/agentkit/abi/render.go index 3136d7b..05e0358 100644 --- a/pkg/agentkit/abi/render.go +++ b/pkg/agentkit/abi/render.go @@ -207,48 +207,78 @@ func copyMap(in map[string]any) map[string]any { func copyAny(v any) any { switch typed := v.(type) { case map[string]any: + if typed == nil { + return nil + } return copyMap(typed) case map[string]string: + if typed == nil { + return nil + } out := make(map[string]string, len(typed)) for key, value := range typed { out[key] = value } return out case map[string]int: + if typed == nil { + return nil + } out := make(map[string]int, len(typed)) for key, value := range typed { out[key] = value } return out case map[string]float64: + if typed == nil { + return nil + } out := make(map[string]any, len(typed)) for key, value := range typed { out[key] = yamlFloat(value, 64) } return out case map[string]bool: + if typed == nil { + return nil + } out := make(map[string]bool, len(typed)) for key, value := range typed { out[key] = value } return out case []any: + if typed == nil { + return nil + } out := make([]any, len(typed)) for i, item := range typed { out[i] = copyAny(item) } return out case []string: + if typed == nil { + return nil + } return append([]string(nil), typed...) case []int: + if typed == nil { + return nil + } return append([]int(nil), typed...) case []float64: + if typed == nil { + return nil + } out := make([]any, len(typed)) for i, item := range typed { out[i] = yamlFloat(item, 64) } return out case []bool: + if typed == nil { + return nil + } return append([]bool(nil), typed...) case float32: return yamlFloat(float64(typed), 32) diff --git a/pkg/agentkit/abi/render_test.go b/pkg/agentkit/abi/render_test.go index 840f75b..2d409db 100644 --- a/pkg/agentkit/abi/render_test.go +++ b/pkg/agentkit/abi/render_test.go @@ -324,10 +324,12 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) Parameters: map[string]any{ jsonSchemaTypeKey: jsonSchemaTypeObject, jsonSchemaPropertiesKey: map[string]any{ - "offset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: math.Copysign(0, -1)}, - "jsonOffset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: json.Number("-0e0")}, - "largeInteger": map[string]any{jsonSchemaTypeKey: "integer", jsonSchemaDefaultKey: json.Number("9007199254740995.0")}, - "typedNested": map[string][]float64{"enum": {math.Copysign(0, -1)}}, + "offset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: math.Copysign(0, -1)}, + "jsonOffset": map[string]any{jsonSchemaTypeKey: jsonSchemaTypeNumber, jsonSchemaDefaultKey: json.Number("-0e0")}, + "largeInteger": map[string]any{jsonSchemaTypeKey: "integer", jsonSchemaDefaultKey: json.Number("9007199254740995.0")}, + "typedNested": map[string][]float64{"enum": {math.Copysign(0, -1)}}, + "typedNil": map[string]any{jsonSchemaDefaultKey: map[string]string(nil)}, + "typedNilSlice": map[string]any{jsonSchemaDefaultKey: []string(nil)}, }, }, } @@ -351,4 +353,7 @@ func TestRenderAgentYAMLPreservesNegativeZeroBrokeredSchemaFloats(t *testing.T) if strings.Count(string(out), yamlNegativeZero) != 3 { t.Fatalf("rendered agent.yaml did not normalize negative zero in typed nested containers\n---\n%s", out) } + if strings.Count(string(out), "default: null") != 2 { + t.Fatalf("rendered agent.yaml did not preserve typed nil containers as null\n---\n%s", out) + } } diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index ba5f696..59eb7e9 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -75,6 +75,8 @@ def test_foundry_brokered_conformance_script_keeps_sensitive_headers_out_of_curl assert text.count("--config -") == 2 assert '-H "Authorization: Bearer ${token}"' not in text assert '-H "x-agentkit-brokered-continuation-proof:' not in text + assert '--expected-output-file "$expected_output_file"' in text + assert '--expected-output-json "$conformance_output"' not in text def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): @@ -124,6 +126,51 @@ def test_verify_brokered_transcript_rejects_extra_final_output_items(tmp_path): verifier.verify_transcript(transcript) +def test_verify_brokered_transcript_rejects_unexpected_continuation_output(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation = json.loads((transcript / "03-continuation-request.json").read_text(encoding="utf-8")) + continuation["input"][0]["output"] = '{"approved":false,"error":{"code":"denied"}}' + (transcript / "03-continuation-request.json").write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="continuation output did not match expected JSON"): + verifier.verify_transcript(transcript) + + +def test_verify_brokered_transcript_compares_output_json_types_strictly(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation = json.loads((transcript / "03-continuation-request.json").read_text(encoding="utf-8")) + continuation["input"][0]["output"] = '{"approved":1,"output":{"success":1}}' + (transcript / "03-continuation-request.json").write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="continuation output did not match expected JSON"): + verifier.verify_transcript(transcript) + + +def test_verify_brokered_transcript_json_comparison_distinguishes_booleans_but_normalizes_numbers(): + verifier = _load_verifier() + + assert verifier._strict_json_equal({"limit": 1}, {"limit": 1.0}) + assert not verifier._strict_json_equal({"approved": True}, {"approved": 1}) + assert not verifier._strict_json_equal( + verifier._parse_json_lossless('{"value":9007199254740992.0}'), + verifier._parse_json_lossless('{"value":9007199254740993.0}'), + ) + compatible = verifier._json_compatible(verifier._parse_json_lossless('{"count":9007199254740993,"ratio":0.5}')) + assert compatible == {"count": 9007199254740993, "ratio": 0.5} + json.dumps(compatible) + + +def test_verify_brokered_transcript_rejects_duplicate_json_keys_and_huge_summary_integers(): + verifier = _load_verifier() + + with pytest.raises(ValueError, match="duplicate JSON object key"): + verifier._parse_json_lossless('{"probe":false,"probe":true}') + with pytest.raises(ValueError, match="too large to include safely"): + verifier._json_compatible(verifier._parse_json_lossless("1e1000000")) + + def test_verify_brokered_transcript_cli_writes_summary(tmp_path, capsys): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From ba81cd7ed3ee45c6f59acb34a3301f30cf3568f6 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 03:11:19 -0700 Subject: [PATCH 22/27] fix(foundry): distinguish expired continuation states Signed-off-by: Sertac Ozercan --- .../common/agentkit_serve_common/foundry.py | 19 +++++-- .../tests/test_foundry_brokered_protocol.py | 53 +++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 5378f86..758611b 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -268,7 +268,9 @@ class _HostedResponseState: class _StateExpired(KeyError): - pass + def __init__(self, response_id: str, state: _HostedResponseState) -> None: + super().__init__(response_id) + self.state = state class _StateStoreFull(Exception): @@ -435,7 +437,7 @@ def get(self, response_id: str) -> _HostedResponseState: if state.expires_at <= time.time(): self._states.pop(response_id, None) self._persist() - raise _StateExpired(response_id) + raise _StateExpired(response_id, state) self.purge_expired() return state @@ -1375,12 +1377,19 @@ async def _handle_brokered_continuation( status=400, code="missing_previous_response_id", ) + input_items = input_value if isinstance(input_value, list) else [input_value] if isinstance(input_value, dict) else [] if len(outputs) != 1: return _error( "multiple function_call_output items are not supported by this deterministic brokered adapter", status=400, code="multiple_tool_outputs_unsupported", ) + if len(input_items) != 1: + return _error( + "continuation input must contain exactly one function_call_output item", + status=400, + code="multiple_tool_outputs_unsupported", + ) try: state = store.get(str(previous_response_id)) except _StateExpired: @@ -1631,8 +1640,10 @@ async def responses(request: Request): if brokered_tools and isinstance(previous_response_id, str) and previous_response_id: try: previous_state = response_states.get(previous_response_id) - except _StateExpired: - return _error("previous_response_id state has expired", status=410, code="response_state_expired") + except _StateExpired as exc: + if exc.state.status in {"pending", "resuming"}: + return _error("previous_response_id state has expired", status=410, code="response_state_expired") + previous_state = None except KeyError: previous_state = None if previous_state is not None and previous_state.status in {"pending", "resuming"}: diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 9d9fec1..b642872 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1245,6 +1245,39 @@ def test_foundry_brokered_rejects_normal_followup_to_expired_pending_response(): assert response.json()["error"]["code"] == "response_state_expired" +def test_foundry_brokered_allows_normal_followup_after_completed_state_expires(monkeypatch): + stores: list[Any] = [] + original_store = foundry_module._FoundryResponseStateStore + + def capture_store(*args: Any, **kwargs: Any) -> Any: + store = original_store(*args, **kwargs) + stores.append(store) + return store + + monkeypatch.setattr(foundry_module, "_FoundryResponseStateStore", capture_store) + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + completed = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + state = stores[0].get(initial["id"]) + state.expires_at = time.time() - 1 + stores[0].save(state) + followup = client.post( + "/responses", + json={"previous_response_id": initial["id"], "input": "next question"}, + ) + + assert completed.status_code == 200, completed.text + assert followup.status_code == 200, followup.text + assert _call(followup.json()) + + def test_foundry_brokered_refuses_to_synthesize_nonliteral_write_arguments(): spec = _spec(tool_name="dispatch-work-order", brokered_class="write") spec.brokered_tools[0].parameters = { @@ -1415,6 +1448,26 @@ def test_foundry_brokered_rejects_multiple_tool_outputs_deterministically(): +def test_foundry_brokered_rejects_continuation_with_extra_non_output_item(): + app = _app() + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + request = _continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}) + request["input"].append({"type": "message", "role": "user", "content": "ignored"}) + rejected = client.post("/responses", json=request, headers=CONTINUATION_AUTH) + accepted = client.post( + "/responses", + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + headers=CONTINUATION_AUTH, + ) + + assert rejected.status_code == 400 + assert rejected.json()["error"]["code"] == "multiple_tool_outputs_unsupported" + assert accepted.status_code == 200, accepted.text + + def test_foundry_brokered_disables_invocations_direct_runtime_bypass(): app = _app() From 42006c664c5e0a7f85a4508f97d307a05aae7260 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 03:17:23 -0700 Subject: [PATCH 23/27] fix(foundry): reject ambiguous broker output Signed-off-by: Sertac Ozercan --- .../common/agentkit_serve_common/foundry.py | 10 +++++++ .../tests/test_foundry_brokered_protocol.py | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 758611b..add39c0 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -589,6 +589,15 @@ def _reject_output_constant(raw: str) -> None: raise ValueError(f"function_call_output.output contains non-finite number {raw}") +def _reject_duplicate_output_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise ValueError(f"function_call_output.output contains duplicate key {key!r}") + out[key] = value + return out + + def _json_object_from_output(output: Any) -> dict[str, Any]: if not isinstance(output, str): raise ValueError("function_call_output.output must be a JSON object string") @@ -597,6 +606,7 @@ def _json_object_from_output(output: Any) -> dict[str, Any]: output, parse_float=_parse_output_float, parse_constant=_reject_output_constant, + object_pairs_hook=_reject_duplicate_output_keys, ) except json.JSONDecodeError as exc: raise ValueError("function_call_output.output must be a JSON object string") from exc diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index b642872..dcce51b 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -860,6 +860,34 @@ def test_foundry_brokered_rejects_lossy_function_call_output_float_before_state_ assert _message_text(exact.json()).endswith('{"id":9007199254740992.0}') +def test_foundry_brokered_rejects_duplicate_keys_in_function_call_output(tmp_path): + state_file = tmp_path / "responses-state.json" + app = _app(response_state_file=state_file) + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + persisted_before = state_file.read_bytes() + response = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json={ + "previous_response_id": initial["id"], + "input": [ + { + "type": "function_call_output", + "call_id": call["call_id"], + "output": '{"approved":false,"approved":true,"output":{}}', + } + ], + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_function_call_output" + assert state_file.read_bytes() == persisted_before + + def test_foundry_brokered_rejects_object_valued_function_call_output_before_state_change(tmp_path): state_file = tmp_path / "responses-state.json" app = _app(response_state_file=state_file) From e1b3db1c8d018332dbdc63717df33b0bc32bf305 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 03:36:51 -0700 Subject: [PATCH 24/27] fix(conformance): validate complete broker exchange Signed-off-by: Sertac Ozercan --- .../scripts/foundry_brokered_conformance.sh | 31 +++++++++++++++++-- .../scripts/verify_brokered_transcript.py | 15 +++++++-- .../foundry_model_loop.py | 2 +- .../tests/test_foundry_brokered_protocol.py | 30 ++++++++++++++++++ .../tests/test_foundry_transcript_verifier.py | 21 +++++++++++++ 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/deploy/foundry/scripts/foundry_brokered_conformance.sh b/deploy/foundry/scripts/foundry_brokered_conformance.sh index 75daf36..0c4f29e 100755 --- a/deploy/foundry/scripts/foundry_brokered_conformance.sh +++ b/deploy/foundry/scripts/foundry_brokered_conformance.sh @@ -27,6 +27,7 @@ Optional: AGENTKIT_EXPECTED_ARGUMENTS Defaults to {"probe":true}. AGENTKIT_EXPECTED_CALL_ID Defaults to call_conformance_1; set to auto for generated IDs. AGENTKIT_EXPECTED_CALL_ID_PREFIX Optional required call_id prefix, e.g. call_. + AGENTKIT_EXPECTED_FINAL_TEXT Optional exact final assistant text; defaults are derived for SDK/AgentKit fixtures. AGENTKIT_CONTINUATION_PROOF Optional x-agentkit-brokered-continuation-proof header. EOF } @@ -69,14 +70,39 @@ expected_tool_name="${AGENTKIT_EXPECTED_TOOL_NAME:-conformance_read}" expected_arguments="${AGENTKIT_EXPECTED_ARGUMENTS:-{\"probe\":true}}" expected_call_id="${AGENTKIT_EXPECTED_CALL_ID:-call_conformance_1}" expected_call_id_prefix="${AGENTKIT_EXPECTED_CALL_ID_PREFIX:-}" +expected_final_text="${AGENTKIT_EXPECTED_FINAL_TEXT:-}" +if [[ -z "$expected_final_text" ]]; then + expected_final_text="$( + EXPECTED_CALL_ID="$expected_call_id" EXPECTED_TOOL_NAME="$expected_tool_name" CONFORMANCE_OUTPUT="$conformance_output" python3 - <<'PY' +import json +import os + +call_id = os.environ["EXPECTED_CALL_ID"] +tool_name = os.environ["EXPECTED_TOOL_NAME"] +payload = json.loads(os.environ["CONFORMANCE_OUTPUT"]) +if call_id == "call_conformance_1": + print(f"conformance complete: {json.dumps(payload, sort_keys=True)}") +elif payload.get("approved") is True: + output = payload.get("output") if isinstance(payload.get("output"), dict) else {} + print(f"Brokered tool {tool_name} completed with output: {json.dumps(output, separators=(',', ':'), sort_keys=True)}") +else: + error = payload.get("error") if isinstance(payload.get("error"), dict) else {} + code = str(error.get("code") or "brokered_tool_denied") + message = str(error.get("message") or "brokered tool was not performed") + print(f"Brokered tool {tool_name} was not performed: {code}: {message}") +PY + )" +fi initial_request="$transcript_dir/01-initial-request.json" initial_response="$transcript_dir/02-initial-response.json" continuation_request="$transcript_dir/03-continuation-request.json" continuation_response="$transcript_dir/04-continuation-response.json" summary_file="$transcript_dir/summary.json" expected_output_file="$transcript_dir/.expected-output.json" -trap 'rm -f -- "$expected_output_file"' EXIT +expected_final_text_file="$transcript_dir/.expected-final-text.txt" +trap 'rm -f -- "$expected_output_file" "$expected_final_text_file"' EXIT printf '%s' "$conformance_output" >"$expected_output_file" +printf '%s' "$expected_final_text" >"$expected_final_text_file" PROMPT="$prompt" python3 - <<'PY' >"$initial_request" import json @@ -157,6 +183,7 @@ verifier_args=( --expected-tool-name "$expected_tool_name" --expected-arguments-json "$expected_arguments" --expected-output-file "$expected_output_file" + --expected-final-text-file "$expected_final_text_file" --expected-call-id "$expected_call_id" --write-summary ) @@ -165,7 +192,7 @@ if [[ -n "$expected_call_id_prefix" ]]; then fi python3 deploy/foundry/scripts/verify_brokered_transcript.py "${verifier_args[@]}" >"$summary_file.tmp" rm -f "$summary_file.tmp" -rm -f "$expected_output_file" +rm -f "$expected_output_file" "$expected_final_text_file" trap - EXIT echo "Foundry brokered conformance passed. Sanitized transcript: ${transcript_dir}" diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index ac31ed0..54a677e 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -25,10 +25,10 @@ def _load_json(path: Path) -> Any: try: - return json.loads(path.read_text(encoding="utf-8")) + return _parse_json_lossless(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"missing transcript file: {path.name}") from exc - except json.JSONDecodeError as exc: + except (json.JSONDecodeError, ValueError, RecursionError) as exc: raise ValueError(f"{path.name} is not valid JSON: {exc}") from exc @@ -113,6 +113,7 @@ def verify_transcript( expected_tool_name: str = "conformance_read", expected_arguments_json: str = '{"probe":true}', expected_output_json: str = '{"approved":true,"output":{"success":true}}', + expected_final_text: str | None = None, expected_call_id: str = "call_conformance_1", expected_call_id_prefix: str | None = None, ) -> dict[str, Any]: @@ -172,6 +173,8 @@ def verify_transcript( _require(isinstance(continuation_response_id, str) and continuation_response_id.startswith("caresp_"), "continuation response id must start with caresp_") _require(continuation_response_id != initial_response_id, "continuation response id must differ from initial response id") final_text = _message_text(continuation_response) + if expected_final_text is not None: + _require(final_text == expected_final_text, "final message text did not match expected conformance result") return { "initial_response_id": initial_response_id, @@ -191,6 +194,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--expected-arguments-json", default='{"probe":true}', help="expected function_call arguments JSON") parser.add_argument("--expected-output-json", default='{"approved":true,"output":{"success":true}}', help="expected function_call_output JSON") parser.add_argument("--expected-output-file", default=None, help="private file containing expected function_call_output JSON") + parser.add_argument("--expected-final-text", default=None, help="expected final assistant text") + parser.add_argument("--expected-final-text-file", default=None, help="private file containing expected final assistant text") parser.add_argument("--expected-call-id", default="call_conformance_1", help="expected call_id, or 'auto' to only require a non-empty id") parser.add_argument("--expected-call-id-prefix", default=None, help="optional required call_id prefix") parser.add_argument("--write-summary", action="store_true", help="write summary.json in the transcript directory") @@ -205,11 +210,17 @@ def main(argv: list[str] | None = None) -> int: if args.expected_output_file else args.expected_output_json ) + expected_final_text = ( + Path(args.expected_final_text_file).read_text(encoding="utf-8") + if args.expected_final_text_file + else args.expected_final_text + ) summary = verify_transcript( args.transcript_dir, expected_tool_name=args.expected_tool_name, expected_arguments_json=args.expected_arguments_json, expected_output_json=expected_output_json, + expected_final_text=expected_final_text, expected_call_id=args.expected_call_id, expected_call_id_prefix=args.expected_call_id_prefix, ) diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index 0dce6df..9dc0e26 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -203,7 +203,7 @@ def _parse_arguments(raw: Any) -> dict[str, Any]: parsed = json.loads(raw or "{}", parse_float=_parse_json_float, parse_constant=_reject_json_constant) except AgentRunError: raise - except json.JSONDecodeError as exc: + except (json.JSONDecodeError, ValueError, RecursionError) as exc: raise AgentRunError("model tool arguments must be valid JSON", status=400, code="InvalidToolArguments") from exc if not isinstance(parsed, dict): raise AgentRunError("model tool arguments must be a JSON object", status=400, code="InvalidToolArguments") diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index dcce51b..2ff26d1 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -2115,6 +2115,36 @@ def test_foundry_brokered_model_loop_rejects_object_valued_tool_arguments(): assert response.json()["error"]["code"] == "InvalidToolArguments" +def test_foundry_brokered_model_loop_normalizes_json_parser_failures(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": '{"value":' + ("9" * 5000) + "}", + }, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + def test_foundry_brokered_model_loop_validates_schema_valued_additional_properties(): spec = _spec(tool_name="flex-tool") spec.brokered_tools[0].parameters = { diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index 59eb7e9..6d68a78 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -77,6 +77,7 @@ def test_foundry_brokered_conformance_script_keeps_sensitive_headers_out_of_curl assert '-H "x-agentkit-brokered-continuation-proof:' not in text assert '--expected-output-file "$expected_output_file"' in text assert '--expected-output-json "$conformance_output"' not in text + assert '--expected-final-text-file "$expected_final_text_file"' in text def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): @@ -94,6 +95,18 @@ def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): raise AssertionError("expected old response id to fail") +def test_verify_brokered_transcript_rejects_duplicate_keys_in_top_level_files(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + initial_path = transcript / "02-initial-response.json" + initial = json.loads(initial_path.read_text(encoding="utf-8")) + encoded = json.dumps(initial, separators=(",", ":")) + initial_path.write_text(encoded[:-1] + ',"status":"failed"}', encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate JSON object key"): + verifier.verify_transcript(transcript) + + def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) @@ -126,6 +139,14 @@ def test_verify_brokered_transcript_rejects_extra_final_output_items(tmp_path): verifier.verify_transcript(transcript) +def test_verify_brokered_transcript_rejects_unexpected_final_text(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + + with pytest.raises(ValueError, match="final message text did not match"): + verifier.verify_transcript(transcript, expected_final_text="unrelated response") + + def test_verify_brokered_transcript_rejects_unexpected_continuation_output(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From f0a3d72ecdfbc47ca9133c215fabfd115eefca5d Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Sun, 12 Jul 2026 08:19:35 -0700 Subject: [PATCH 25/27] fix(foundry): close brokered validation gaps Signed-off-by: Sertac Ozercan --- pkg/agentkit/config/config_test.go | 28 ++- pkg/agentkit/config/validate.go | 24 +- .../common/agentkit_serve_common/brokered.py | 44 +++- runtimes/common/agentkit_serve_common/cli.py | 5 + .../common/agentkit_serve_common/foundry.py | 24 +- .../foundry_conformance.py | 126 ++++++++++- .../foundry_model_loop.py | 64 +++++- runtimes/common/tests/test_brokered_schema.py | 50 +++++ runtimes/common/tests/test_cli_protocol.py | 26 +++ .../tests/test_foundry_brokered_protocol.py | 206 ++++++++++++++++++ .../common/tests/test_foundry_conformance.py | 85 ++++++++ 11 files changed, 653 insertions(+), 29 deletions(-) diff --git a/pkg/agentkit/config/config_test.go b/pkg/agentkit/config/config_test.go index 8548b36..047ed26 100644 --- a/pkg/agentkit/config/config_test.go +++ b/pkg/agentkit/config/config_test.go @@ -608,8 +608,8 @@ func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing } numberSchema[jsonSchemaDefaultKey] = json.Number("1.00000000000000001") - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { - t.Fatalf("non-integral json.Number must fail integer validation, got: %v", err) + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "cannot be represented exactly") { + t.Fatalf("lossy non-integral json.Number must fail exact representation validation, got: %v", err) } numberSchema[jsonSchemaDefaultKey] = json.Number("100000000000000000001.0") if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { @@ -625,8 +625,8 @@ func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing } numberSchema[jsonSchemaDefaultKey] = json.Number("1.0") fractionalSchema[jsonSchemaDefaultKey] = json.Number("1e-400") - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "must match the declared JSON Schema type") { - t.Fatalf("underflowing json.Number must fail number validation, got: %v", err) + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "cannot be represented exactly") { + t.Fatalf("underflowing json.Number must fail exact representation validation, got: %v", err) } fractionalSchema[jsonSchemaDefaultKey] = json.Number("0.1") @@ -636,6 +636,26 @@ func TestValidateNormalizesJSONContainersWithoutChangingScalarMeaning(t *testing } } +func TestValidateRejectsLossyTypelessJSONNumberSchemaValues(t *testing.T) { + cfg := validMinimalConfig() + cfg.BrokeredTools = []BrokeredTool{{ + Name: safeLookupToolName, + Description: brokeredSafeDescription, + BrokeredClass: BrokeredClassWrite, + Parameters: map[string]any{ + jsonSchemaTypeKey: jsonSchemaTypeObject, + jsonSchemaPropertiesKey: map[string]any{ + "ratio": map[string]any{jsonSchemaDefaultKey: json.Number("0.100000000000000005")}, + }, + }, + }} + + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "cannot be represented exactly") { + t.Fatalf("lossy typeless json.Number must fail validation, got: %v", err) + } +} + func TestValidateMeasuresBrokeredSchemaSizeWithCanonicalJSON(t *testing.T) { cfg := validMinimalConfig() cfg.BrokeredTools = []BrokeredTool{{ diff --git a/pkg/agentkit/config/validate.go b/pkg/agentkit/config/validate.go index d31a183..1de618a 100644 --- a/pkg/agentkit/config/validate.go +++ b/pkg/agentkit/config/validate.go @@ -877,16 +877,18 @@ func unescapeJSONLineSeparators(encoded []byte) []byte { } func canonicalJSONNumberString(value string) (string, error) { - if json.Valid([]byte(value)) { - if number, ok := new(big.Rat).SetString(value); ok && number.IsInt() { - if number.Sign() == 0 && strings.HasPrefix(value, "-") { - return "-0", nil - } - return number.Num().String(), nil - } + if !json.Valid([]byte(value)) { + return "", fmt.Errorf("invalid JSON number %q", value) } - if !strings.ContainsAny(value, ".eE") { - return value, nil + number, ok := new(big.Rat).SetString(value) + if !ok { + return "", fmt.Errorf("invalid JSON number %q", value) + } + if number.IsInt() { + if number.Sign() == 0 && strings.HasPrefix(value, "-") { + return "-0", nil + } + return number.Num().String(), nil } parsed, err := strconv.ParseFloat(value, 64) if err != nil { @@ -895,6 +897,10 @@ func canonicalJSONNumberString(value string) (string, error) { if math.IsNaN(parsed) || math.IsInf(parsed, 0) { return "", fmt.Errorf("JSON numbers must be finite") } + roundTrip, ok := new(big.Rat).SetString(strconv.FormatFloat(parsed, 'g', -1, 64)) + if !ok || roundTrip.Cmp(number) != 0 { + return "", fmt.Errorf("JSON number %q cannot be represented exactly", value) + } return strconv.FormatFloat(parsed, 'f', -1, 64), nil } diff --git a/runtimes/common/agentkit_serve_common/brokered.py b/runtimes/common/agentkit_serve_common/brokered.py index f2d26fb..d4670ea 100644 --- a/runtimes/common/agentkit_serve_common/brokered.py +++ b/runtimes/common/agentkit_serve_common/brokered.py @@ -10,7 +10,9 @@ from __future__ import annotations import argparse +import math import sys +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Iterable, Mapping, Sequence @@ -22,6 +24,44 @@ _ORKA_TOOL_API_VERSION = "core.orka.ai/v1alpha1" _ORKA_TOOL_KIND = "Tool" _ORKA_BROKERED_TOOL_CLASSES = ("read", "write", "coordination") +_MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS = 4096 + + +class _LosslessToolCRDLoader(yaml.SafeLoader): + """Safe YAML loader that never silently rounds CRD numeric scalars.""" + + +def _construct_lossless_tool_crd_float(loader: yaml.SafeLoader, node: yaml.Node) -> int | float: + raw = loader.construct_scalar(node).replace("_", "").lower() + if ":" in raw: + raise ValueError(f"YAML float literal {raw!r} uses unsupported sexagesimal notation") + try: + decimal = Decimal(raw) + except InvalidOperation as exc: + raise ValueError(f"YAML float literal {raw!r} is invalid") from exc + if not decimal.is_finite(): + raise ValueError(f"YAML float literal {raw!r} must be finite") + if decimal.is_zero() and decimal.is_signed(): + return -0.0 + if decimal == decimal.to_integral_value(): + digits = max(decimal.adjusted() + 1, len(decimal.as_tuple().digits)) if decimal else 1 + if digits > _MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS: + raise ValueError(f"YAML float literal {raw!r} expands to an integer that is too large") + return int(decimal) + try: + candidate = float(decimal) + except (OverflowError, ValueError) as exc: + raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") from exc + if not math.isfinite(candidate) or Decimal(str(candidate)) != decimal: + raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") + return candidate + + +_LosslessToolCRDLoader.add_constructor("tag:yaml.org,2002:float", _construct_lossless_tool_crd_float) + + +def _load_orka_tool_crd_documents(raw: str) -> list[Any]: + return [doc for doc in yaml.load_all(raw, Loader=_LosslessToolCRDLoader) if doc is not None] def brokered_tool_definitions(spec: AgentSpec) -> list[BrokeredToolDefinition]: @@ -148,7 +188,7 @@ def load_orka_tool_crd_file(path: str | Path, *, include_digest: bool = True) -> """Load Tool CRD YAML/JSON documents and return safe brokeredTools entries.""" raw = Path(path).read_text(encoding="utf-8") - docs = [doc for doc in yaml.safe_load_all(raw) if doc is not None] + docs = _load_orka_tool_crd_documents(raw) return generate_brokered_tools_from_orka_tool_crds(docs, include_digest=include_digest) @@ -158,7 +198,7 @@ def load_orka_tool_crd_files(paths: Sequence[str | Path], *, include_digest: boo documents: list[Any] = [] for path in paths: raw = Path(path).read_text(encoding="utf-8") - documents.extend(doc for doc in yaml.safe_load_all(raw) if doc is not None) + documents.extend(_load_orka_tool_crd_documents(raw)) entries = generate_brokered_tools_from_orka_tool_crds(documents, include_digest=include_digest) seen: set[str] = set() duplicates: set[str] = set() diff --git a/runtimes/common/agentkit_serve_common/cli.py b/runtimes/common/agentkit_serve_common/cli.py index b5a1891..6274353 100644 --- a/runtimes/common/agentkit_serve_common/cli.py +++ b/runtimes/common/agentkit_serve_common/cli.py @@ -131,6 +131,11 @@ def run(factory: RuntimeFactory, argv: list[str] | None = None) -> None: # builds a runtime session. os.environ["AGENTKIT_PROTOCOL"] = protocol spec = _load_spec_or_exit(args.config, protocol) + if spec.brokered_tools and protocol != "foundry": + _fail( + "brokeredTools require AGENTKIT_PROTOCOL=foundry (or --protocol foundry); " + f"the {protocol!r} protocol cannot broker Foundry Responses tool calls" + ) # --- resolve bind/port ------------------------------------------------ bind = os.environ.get("AGENTKIT_BIND", "127.0.0.1").strip() diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index add39c0..47d6410 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -53,6 +53,7 @@ _DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024 _MAX_SYNTHETIC_ARRAY_ITEMS = 32 _MAX_SYNTHETIC_STRING_LENGTH = 4096 +_MAX_SYNTHETIC_VALUES = 256 _STATE_TTL_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS" _MAX_PENDING_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING" _MAX_ARGUMENT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES" @@ -816,7 +817,16 @@ def _required_property_names(schema: Mapping[str, Any]) -> list[str]: return names -def _sample_argument_value(name: str, schema: Any, run_request: RunRequest) -> Any: +def _sample_argument_value(name: str, schema: Any, run_request: RunRequest, *, budget: list[int] | None = None) -> Any: + if budget is None: + budget = [_MAX_SYNTHETIC_VALUES] + if budget[0] <= 0: + raise AgentRunError( + f"brokered tool schema for {name!r} exceeds deterministic synthesis value budget", + status=413, + code="brokered_arguments_too_large", + ) + budget[0] -= 1 if not isinstance(schema, Mapping): return run_request.prompt if "multipleOf" in schema: @@ -998,14 +1008,14 @@ def add_candidate(candidate: int | float) -> None: return value min_items = schema.get("minItems", 0) if isinstance(min_items, int) and min_items > 0: - if min_items > _MAX_SYNTHETIC_ARRAY_ITEMS: + if min_items > _MAX_SYNTHETIC_ARRAY_ITEMS or min_items > budget[0]: raise AgentRunError( f"brokered tool schema for {name!r} has minItems too large for deterministic synthesis", status=413, code="brokered_arguments_too_large", ) item_schema = schema.get("items", {}) - return [_sample_argument_value(name, item_schema, run_request) for _ in range(min_items)] + return [_sample_argument_value(name, item_schema, run_request, budget=budget) for _ in range(min_items)] return [] if schema_type == "object": @@ -1016,7 +1026,7 @@ def add_candidate(candidate: int | float) -> None: nested: dict[str, Any] = {} properties = schema.get("properties") if isinstance(schema.get("properties"), Mapping) else {} for child_name in _required_property_names(schema): - nested[child_name] = _sample_argument_value(child_name, properties.get(child_name, {}), run_request) + nested[child_name] = _sample_argument_value(child_name, properties.get(child_name, {}), run_request, budget=budget) return nested for key in ("const", "default"): @@ -1086,8 +1096,9 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run code="UnsupportedBrokeredSchema", ) arguments: dict[str, Any] = {} + synthesis_budget = [_MAX_SYNTHETIC_VALUES] for name in required_names: - arguments[name] = _sample_argument_value(name, properties.get(name, {}), run_request) + arguments[name] = _sample_argument_value(name, properties.get(name, {}), run_request, budget=synthesis_budget) if tool.name == "conformance_read" and "probe" in properties and "probe" not in arguments: arguments["probe"] = True if not arguments and "prompt" in properties: @@ -1097,7 +1108,7 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run status=400, code="UnsupportedBrokeredSchema", ) - arguments["prompt"] = _sample_argument_value("prompt", properties["prompt"], run_request) + arguments["prompt"] = _sample_argument_value("prompt", properties["prompt"], run_request, budget=synthesis_budget) return arguments @@ -1539,6 +1550,7 @@ def create_foundry_app( spec, brokered_tools, http_client=brokered_model_http_client, + max_argument_bytes=max_argument_bytes, max_output_bytes=max_output_bytes, ) if brokered_tools and _brokered_model_loop_enabled(brokered_model_loop_enabled) diff --git a/runtimes/common/agentkit_serve_common/foundry_conformance.py b/runtimes/common/agentkit_serve_common/foundry_conformance.py index bf24623..c3e0dd0 100644 --- a/runtimes/common/agentkit_serve_common/foundry_conformance.py +++ b/runtimes/common/agentkit_serve_common/foundry_conformance.py @@ -10,10 +10,12 @@ import argparse import json import os +import time from typing import Any, Sequence from starlette.responses import JSONResponse from starlette.routing import Route +from azure.ai.agentserver.core._request_id import REQUEST_ID_STATE_KEY from azure.ai.agentserver.responses import ( InMemoryResponseProvider, @@ -27,6 +29,90 @@ _CONFORMANCE_ARGUMENTS = '{"probe":true}' +class _ForceNonStoredResponsesMiddleware: + def __init__(self, app, *, max_body_bytes: int, session_id: str) -> None: # noqa: ANN001 + self.app = app + self.max_body_bytes = max(int(max_body_bytes), 1) + self.session_id = session_id + + async def __call__(self, scope, receive, send) -> None: # noqa: ANN001 + if scope.get("type") != "http" or scope.get("method") != "POST" or scope.get("path", "").rstrip("/") != "/responses": + await self.app(scope, receive, send) + return + + chunks: list[bytes] = [] + total_bytes = 0 + more_body = True + while more_body: + message = await receive() + if message.get("type") != "http.request": + await self.app(scope, receive, send) + return + chunk = message.get("body", b"") + total_bytes += len(chunk) + if total_bytes > self.max_body_bytes: + await self._send_error(scope, send, status=413, code="request_body_too_large", message="request body too large") + return + chunks.append(chunk) + more_body = bool(message.get("more_body")) + body = b"".join(chunks) + try: + payload = json.loads(body) + if isinstance(payload, dict): + if payload.get("background") is True: + await self._send_error( + scope, + send, + status=400, + code="background_unsupported", + message="background responses are not supported by the conformance fixture", + ) + return + if "store" in payload and payload["store"] is not None and not isinstance(payload["store"], bool): + rewritten = body + else: + payload["store"] = False + rewritten = json.dumps(payload, separators=(",", ":")).encode("utf-8") + else: + rewritten = body + except (json.JSONDecodeError, UnicodeDecodeError, ValueError, TypeError, RecursionError): + rewritten = body + + sent = False + original_receive = receive + + async def replay_receive(): + nonlocal sent + if not sent: + sent = True + return {"type": "http.request", "body": rewritten, "more_body": False} + return await original_receive() + + rewritten_scope = dict(scope) + headers = [(name, value) for name, value in scope.get("headers", []) if name.lower() != b"content-length"] + headers.append((b"content-length", str(len(rewritten)).encode("ascii"))) + rewritten_scope["headers"] = headers + await self.app(rewritten_scope, replay_receive, send) + + async def _send_error(self, scope, send, *, status: int, code: str, message: str) -> None: # noqa: ANN001 + state = scope.get("state") if isinstance(scope.get("state"), dict) else {} + request_id = str(state.get(REQUEST_ID_STATE_KEY) or "") + error: dict[str, Any] = {"code": code, "message": message, "type": "invalid_request_error"} + if request_id: + error["additionalInfo"] = {"request_id": request_id} + body = json.dumps({"error": error}, separators=(",", ":")).encode("utf-8") + response_headers = [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode("ascii")), + (b"x-platform-error-source", b"user"), + ] + if self.session_id: + response_headers.append((b"x-agent-session-id", self.session_id.encode("utf-8"))) + await send({"type": "http.response.start", "status": status, "headers": response_headers}) + await send({"type": "http.response.body", "body": body}) + + + def _input_items(request: Any) -> list[dict[str, Any]]: return [dict(item) for item in get_input_expanded(request)] @@ -42,12 +128,30 @@ def _request_tools(request: Any) -> Any: return tools -def create_foundry_conformance_app(*, model: str = "agentkit-foundry-conformance") -> ResponsesAgentServerHost: +def create_foundry_conformance_app( + *, + model: str = "agentkit-foundry-conformance", + pending_ttl_seconds: float = 15 * 60, + max_pending_responses: int = 128, + max_request_body_bytes: int = 1024 * 1024, +) -> ResponsesAgentServerHost: """Create a minimal Responses SDK app for A0 Foundry function-call smokes.""" + pending: dict[str, tuple[set[str], float]] = {} store = InMemoryResponseProvider() app = ResponsesAgentServerHost(store=store, configure_observability=None) - pending: dict[str, set[str]] = {} + app.add_middleware( + _ForceNonStoredResponsesMiddleware, + max_body_bytes=max_request_body_bytes, + session_id=os.environ.get("FOUNDRY_AGENT_SESSION_ID", "").strip(), + ) + app.user_middleware.append(app.user_middleware.pop(0)) + app.state.conformance_store = store + + def purge_expired() -> None: + now = time.time() + for response_id in [response_id for response_id, (_calls, expires_at) in pending.items() if expires_at <= now]: + pending.pop(response_id, None) async def readiness(_request): # noqa: ANN001 - Starlette passes Request. return JSONResponse( @@ -55,6 +159,10 @@ async def readiness(_request): # noqa: ANN001 - Starlette passes Request. "ready": True, "protocols": {"responses": "2.0.0"}, "implementation": "azure-ai-agentserver-responses", + "pendingStateTtlSeconds": pending_ttl_seconds, + "pendingStateMax": max_pending_responses, + "requestBodyMaxBytes": max_request_body_bytes, + "background": False, } ) @@ -73,6 +181,7 @@ async def response_handler(request, context, cancellation_signal): # noqa: ANN0 ) return + purge_expired() outputs = _function_call_outputs(request) if outputs: previous_response_id = getattr(request, "previous_response_id", None) @@ -82,13 +191,14 @@ async def response_handler(request, context, cancellation_signal): # noqa: ANN0 message="function_call_output requires previous_response_id", ) return - pending_calls = pending.get(str(previous_response_id)) - if pending_calls is None: + pending_state = pending.get(str(previous_response_id)) + if pending_state is None: yield stream.emit_failed( code="unknown_previous_response_id", message="unknown previous_response_id", ) return + pending_calls, _expires_at = pending_state if len(outputs) != 1: yield stream.emit_failed( code="multiple_tool_outputs_unsupported", @@ -110,7 +220,13 @@ async def response_handler(request, context, cancellation_signal): # noqa: ANN0 yield stream.emit_completed() return - pending[context.response_id] = {_CONFORMANCE_CALL_ID} + if len(pending) >= max_pending_responses: + yield stream.emit_failed( + code="brokered_response_state_full", + message="too many pending brokered conformance responses", + ) + return + pending[context.response_id] = ({_CONFORMANCE_CALL_ID}, time.time() + max(pending_ttl_seconds, 0.0)) for event in stream.output_item_function_call( name=_CONFORMANCE_TOOL_NAME, call_id=_CONFORMANCE_CALL_ID, diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index 9dc0e26..bab480d 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -24,6 +24,8 @@ from .conversation import FORWARDED_ROLES, RunRequest from .runtime import AgentRunError, BrokeredToolDefinition +_MAX_ARGUMENT_DEPTH = 128 + @dataclass(frozen=True) class ModelLoopFinal: @@ -48,11 +50,13 @@ def __init__( tools: Sequence[BrokeredToolDefinition], *, http_client: httpx.AsyncClient | None = None, + max_argument_bytes: int = 8192, max_output_bytes: int = 64 * 1024, ) -> None: self.spec = spec self.tools = list(tools) self.http_client = http_client + self.max_argument_bytes = max_argument_bytes self.max_output_bytes = max_output_bytes self.tools_by_name = {tool.name: tool for tool in self.tools} @@ -80,7 +84,18 @@ async def start(self, request: RunRequest, *, call_id: str) -> ModelLoopFinal | if not isinstance(name, str) or name not in self.tools_by_name: raise AgentRunError(f"model requested unknown brokered tool {name!r}", status=400, code="unknown_brokered_tool") raw_arguments = function.get("arguments", "{}") + if isinstance(raw_arguments, str): + try: + if len(raw_arguments) > self.max_argument_bytes or len(raw_arguments.encode("utf-8")) > self.max_argument_bytes: + raise AgentRunError( + "model brokered tool arguments are too large", + status=413, + code="brokered_arguments_too_large", + ) + except UnicodeEncodeError as exc: + raise AgentRunError("model tool arguments must contain valid Unicode", status=400, code="InvalidToolArguments") from exc arguments = _parse_arguments(raw_arguments) + _validate_json_unicode(arguments) argument_text = json.dumps(arguments, separators=(",", ":"), sort_keys=True) assistant_message = { "role": "assistant", @@ -192,15 +207,29 @@ def _choice_message(data: Mapping[str, Any]) -> Mapping[str, Any]: def _message_text(message: Mapping[str, Any]) -> str: - content = message.get("content", "") - return content if isinstance(content, str) else str(content or "") + content = message.get("content") + if isinstance(content, str): + return content + refusal = message.get("refusal") + if content is None and isinstance(refusal, str): + return refusal + raise AgentRunError( + "model response final assistant content must be a string", + status=502, + code="InvalidModelResponse", + ) def _parse_arguments(raw: Any) -> dict[str, Any]: if not isinstance(raw, str): raise AgentRunError("model tool arguments must be a JSON object string", status=400, code="InvalidToolArguments") try: - parsed = json.loads(raw or "{}", parse_float=_parse_json_float, parse_constant=_reject_json_constant) + parsed = json.loads( + raw or "{}", + parse_float=_parse_json_float, + parse_constant=_reject_json_constant, + object_pairs_hook=_reject_duplicate_argument_keys, + ) except AgentRunError: raise except (json.JSONDecodeError, ValueError, RecursionError) as exc: @@ -210,6 +239,35 @@ def _parse_arguments(raw: Any) -> dict[str, Any]: return parsed +def _reject_duplicate_argument_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise AgentRunError(f"model tool arguments contain duplicate key {key!r}", status=400, code="InvalidToolArguments") + out[key] = value + return out + + +def _validate_json_unicode(value: Any, *, path: str = "arguments") -> None: + pending: list[tuple[Any, str, int]] = [(value, path, 0)] + while pending: + current, current_path, depth = pending.pop() + if depth > _MAX_ARGUMENT_DEPTH: + raise AgentRunError("model tool arguments are nested too deeply", status=400, code="InvalidToolArguments") + if isinstance(current, str): + try: + current.encode("utf-8") + except UnicodeEncodeError as exc: + raise AgentRunError(f"model tool arguments contain invalid Unicode at {current_path}", status=400, code="InvalidToolArguments") from exc + elif isinstance(current, Mapping): + for key, child in current.items(): + pending.append((child, f"{current_path}[{key!r}]", depth + 1)) + pending.append((key, f"{current_path}.", depth + 1)) + elif isinstance(current, list): + for index, child in enumerate(current): + pending.append((child, f"{current_path}[{index}]", depth + 1)) + + def _parse_json_float(raw: str) -> float: try: decimal = Decimal(raw) diff --git a/runtimes/common/tests/test_brokered_schema.py b/runtimes/common/tests/test_brokered_schema.py index d3c2399..5af923a 100644 --- a/runtimes/common/tests/test_brokered_schema.py +++ b/runtimes/common/tests/test_brokered_schema.py @@ -266,6 +266,56 @@ def test_render_brokered_tools_yaml_outputs_agent_yaml_fragment(): assert "Authorization" not in rendered +def test_load_orka_tool_crd_files_preserves_high_precision_integral_float(tmp_path): + src = tmp_path / "precise.yaml" + src.write_text( + """apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: precise-number +spec: + description: Preserve a precise number. + brokeredToolClass: read + parameters: + type: object + properties: + identifier: + type: integer + default: 9007199254740993.0 +""", + encoding="utf-8", + ) + + [entry] = load_orka_tool_crd_files([src], include_digest=False) + + default = entry["parameters"]["properties"]["identifier"]["default"] + assert default == 9007199254740993 + assert isinstance(default, int) + + +def test_load_orka_tool_crd_files_rejects_lossy_fractional_float(tmp_path): + src = tmp_path / "lossy.yaml" + src.write_text( + """apiVersion: core.orka.ai/v1alpha1 +kind: Tool +metadata: + name: lossy-number +spec: + description: Reject a lossy number. + brokeredToolClass: write + parameters: + type: object + properties: + ratio: + default: 0.100000000000000005 +""", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="cannot be represented exactly"): + load_orka_tool_crd_files([src], include_digest=False) + + def test_load_orka_tool_crd_files_rejects_duplicate_exported_names(tmp_path): first = tmp_path / "first.yaml" second = tmp_path / "second.yaml" diff --git a/runtimes/common/tests/test_cli_protocol.py b/runtimes/common/tests/test_cli_protocol.py index beeb65f..793ce4f 100644 --- a/runtimes/common/tests/test_cli_protocol.py +++ b/runtimes/common/tests/test_cli_protocol.py @@ -27,6 +27,19 @@ def _spec(port: int = 8080) -> AgentSpec: ) +def _brokered_spec() -> AgentSpec: + data = _spec().model_dump(by_alias=True) + data["brokeredTools"] = [ + { + "name": "check-network-telemetry", + "description": "Read telemetry.", + "brokeredClass": "read", + "parameters": {"type": "object"}, + } + ] + return AgentSpec.model_validate(data) + + class Runtime: async def __aenter__(self) -> RuntimeSession: return self @@ -63,6 +76,19 @@ def test_cli_protocol_flag_selects_foundry_and_default_foundry_port(monkeypatch) assert any(getattr(route, "path", None) == "/readiness" for route in captured["app"].routes) +def test_cli_rejects_brokered_tools_on_default_openai_protocol(monkeypatch, capsys): + monkeypatch.setattr(cli, "load_or_exit", lambda path: _brokered_spec()) + monkeypatch.setattr(cli.uvicorn, "run", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("server must not start"))) + monkeypatch.delenv("AGENTKIT_PROTOCOL", raising=False) + monkeypatch.delenv("AGENTKIT_AUTH_TOKEN", raising=False) + + with pytest.raises(SystemExit) as exc: + cli.run(Factory(), ["--config", "agent.yaml"]) + + assert exc.value.code == 2 + assert "brokeredTools require AGENTKIT_PROTOCOL=foundry" in capsys.readouterr().err + + def test_cli_protocol_env_selects_orka_and_requires_auth_token(monkeypatch): monkeypatch.setattr(cli, "load", lambda path: _spec()) monkeypatch.setenv("AGENTKIT_PROTOCOL", "orka") diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 2ff26d1..6b9ae72 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -695,6 +695,59 @@ def test_foundry_brokered_rejects_unbounded_schema_synthesis_before_allocating() assert resp.json()["error"]["code"] == "brokered_arguments_too_large" +def test_foundry_brokered_bounds_nested_array_synthesis_before_recursive_allocation(): + spec = _spec(tool_name="nested-array") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "values": { + "type": "array", + "minItems": 32, + "items": { + "type": "array", + "minItems": 32, + "items": {"type": "string"}, + }, + } + }, + "required": ["values"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "nested-array"}) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_arguments_too_large" + + +def test_foundry_brokered_counts_wide_object_members_against_synthesis_budget(): + item_properties = {f"field{index}": {"type": "string"} for index in range(32)} + spec = _spec(tool_name="wide-object-array") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": { + "values": { + "type": "array", + "minItems": 32, + "items": { + "type": "object", + "properties": item_properties, + "required": list(item_properties), + }, + } + }, + "required": ["values"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "wide-object-array"}) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_arguments_too_large" + + def test_foundry_brokered_tool_selection_requires_token_boundary_match(): data = _multi_tool_spec().model_dump(by_alias=True) data["brokeredTools"] = [ @@ -1992,6 +2045,31 @@ def test_foundry_brokered_model_loop_can_return_final_message_without_tool_call( assert fake.requests[0]["tool_choice"] == "auto" +def test_foundry_brokered_model_loop_returns_assistant_refusal_without_tool_call(): + fake = _FakeChatTransport( + [_chat_response({"role": "assistant", "content": None, "refusal": "I cannot help with that."})] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "Do something unsafe"}) + + assert response.status_code == 200, response.text + assert _message_text(response.json()) == "I cannot help with that." + + +@pytest.mark.parametrize("content", [None, {"text": "not supported"}, ["not supported"]]) +def test_foundry_brokered_model_loop_rejects_non_string_final_content(content: Any): + fake = _FakeChatTransport([_chat_response({"role": "assistant", "content": content})]) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "Say hello"}) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "InvalidModelResponse" + + def test_foundry_brokered_model_loop_rejects_unsupported_pattern_deterministically(): spec = _spec(tool_name="check-network-telemetry") spec.brokered_tools[0].parameters = { @@ -2115,6 +2193,134 @@ def test_foundry_brokered_model_loop_rejects_object_valued_tool_arguments(): assert response.json()["error"]["code"] == "InvalidToolArguments" +def test_foundry_brokered_model_loop_rejects_duplicate_tool_argument_keys(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": '{"site":"sfo","site":"sea"}', + }, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_decoded_surrogate_arguments(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": '{"site":"\\ud800"}', + }, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_rejects_excessively_nested_arguments(): + nested = '"leaf"' + for _ in range(200): + nested = f"[{nested}]" + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": '{"site":' + nested + "}", + }, + } + ], + } + ) + ] + ) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "InvalidToolArguments" + + +def test_foundry_brokered_model_loop_bounds_raw_arguments_before_parsing(monkeypatch): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": { + "name": "check-network-telemetry", + "arguments": '{"blob":"' + ("x" * 128) + '"}', + }, + } + ], + } + ) + ] + ) + + def fail_if_parsed(_raw: Any) -> dict[str, Any]: + raise AssertionError("oversized model arguments must be rejected before parsing") + + monkeypatch.setattr(foundry_model_loop_module, "_parse_arguments", fail_if_parsed) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake, max_brokered_argument_bytes=64) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_arguments_too_large" + + def test_foundry_brokered_model_loop_normalizes_json_parser_failures(): fake = _FakeChatTransport( [ diff --git a/runtimes/common/tests/test_foundry_conformance.py b/runtimes/common/tests/test_foundry_conformance.py index 8c2fae7..7bd62f3 100644 --- a/runtimes/common/tests/test_foundry_conformance.py +++ b/runtimes/common/tests/test_foundry_conformance.py @@ -105,6 +105,91 @@ def test_foundry_conformance_sdk_rejects_request_level_tools(): assert response.json()["error"]["code"] == "tools_unsupported" +def test_foundry_conformance_sdk_bounds_pending_response_state(): + app = create_foundry_conformance_app(max_pending_responses=1) + + with TestClient(app) as client: + first = client.post("/responses", json={"input": "first", "store": True}).json() + second = client.post("/responses", json={"input": "second"}).json() + completed = client.post( + "/responses", + json=_function_output(first["id"], first["output"][0]["call_id"]), + ).json() + third = client.post("/responses", json={"input": "third"}).json() + + assert second["status"] == "failed" + assert second["error"]["code"] == "brokered_response_state_full" + assert completed["status"] == "completed" + assert third["output"][0]["type"] == "function_call" + assert len(app.state.conformance_store._entries) == 0 + assert len(app.state.conformance_store._item_store) == 0 + assert len(app.state.conformance_store._stream_events) == 0 + + +def test_foundry_conformance_sdk_bounds_request_body_before_rewrite(monkeypatch): + monkeypatch.setenv("FOUNDRY_AGENT_SESSION_ID", "hosted-session") + app = create_foundry_conformance_app(max_request_body_bytes=64) + + with TestClient(app) as client: + response = client.post( + "/responses", + content=json.dumps({"input": "x" * 128}), + headers={"content-type": "application/json", "x-request-id": "oversized-request"}, + ) + generated_id_response = client.post( + "/responses", + content=json.dumps({"input": "x" * 128}), + headers={"content-type": "application/json"}, + ) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "request_body_too_large" + assert response.json()["error"]["type"] == "invalid_request_error" + assert response.json()["error"]["additionalInfo"]["request_id"] == "oversized-request" + assert response.headers["x-request-id"] == "oversized-request" + assert "azure-ai-agentserver-responses" in response.headers["x-platform-server"] + assert response.headers["x-platform-error-source"] == "user" + assert response.headers["x-agent-session-id"] == "hosted-session" + assert generated_id_response.json()["error"]["additionalInfo"]["request_id"] == generated_id_response.headers["x-request-id"] + + +def test_foundry_conformance_sdk_explicitly_rejects_background_mode(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + response = client.post( + "/responses", + json={"input": "background", "background": True, "store": True}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "background_unsupported" + assert response.headers["x-platform-error-source"] == "user" + assert len(app.state.conformance_store._entries) == 0 + + +def test_foundry_conformance_sdk_preserves_invalid_store_for_protocol_validation(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "invalid store", "store": "not-a-boolean"}) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert len(app.state.conformance_store._entries) == 0 + + +def test_foundry_conformance_sdk_rewrites_nullable_store_to_non_stored(): + app = create_foundry_conformance_app() + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "nullable store", "store": None}) + + assert response.status_code == 200 + assert response.json()["status"] == "completed" + assert len(app.state.conformance_store._entries) == 0 + + def test_foundry_conformance_console_dry_run(capsys): from agentkit_serve_common.foundry_conformance import main From 10addc89f55cc6380c4e6cf81c0b0c6951849380 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 09:19:07 -0700 Subject: [PATCH 26/27] fix(foundry): harden brokered response validation Signed-off-by: Sertac Ozercan --- .../scripts/verify_brokered_transcript.py | 8 +- .../common/agentkit_serve_common/brokered.py | 39 +---- .../common/agentkit_serve_common/config.py | 18 ++- .../common/agentkit_serve_common/foundry.py | 145 +++++++++++------- .../foundry_model_loop.py | 43 +++--- .../agentkit_serve_common/yaml_support.py | 59 +++++++ .../common/tests/test_config_validation.py | 134 +++++++++++++++- .../tests/test_foundry_brokered_protocol.py | 128 +++++++++++++++- .../tests/test_foundry_transcript_verifier.py | 45 ++++++ 9 files changed, 496 insertions(+), 123 deletions(-) create mode 100644 runtimes/common/agentkit_serve_common/yaml_support.py diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index 54a677e..56b81cb 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -95,11 +95,14 @@ def _json_compatible(value: Any) -> Any: return value -def _message_text(response: dict[str, Any]) -> str: +def _message_text(response: dict[str, Any], *, response_id: str) -> str: output = response.get("output") _require(isinstance(output, list) and len(output) == 1, "final response output must contain exactly one item") message = output[0] _require(isinstance(message, dict) and message.get("type") == "message", "final response output[0] must be a message") + _require(message.get("response_id") == response_id, "final message response_id must match continuation response id") + _require(message.get("role") == "assistant", "final message role must be assistant") + _require(message.get("status") == "completed", "final message status must be completed") content = message.get("content") _require(isinstance(content, list) and bool(content), "final message content must be a non-empty array") text = content[0].get("text") if isinstance(content[0], dict) else None @@ -139,6 +142,7 @@ def verify_transcript( call = output[0] _require(isinstance(call, dict), "initial response output[0] must be an object") _require(call.get("type") == "function_call", "initial output item must be function_call") + _require(call.get("response_id") == initial_response_id, "function_call response_id must match initial response id") function_name = call.get("name") _require(function_name == expected_tool_name, f"function_call name must be {expected_tool_name}") call_id = call.get("call_id") @@ -172,7 +176,7 @@ def verify_transcript( continuation_response_id = continuation_response.get("id") _require(isinstance(continuation_response_id, str) and continuation_response_id.startswith("caresp_"), "continuation response id must start with caresp_") _require(continuation_response_id != initial_response_id, "continuation response id must differ from initial response id") - final_text = _message_text(continuation_response) + final_text = _message_text(continuation_response, response_id=continuation_response_id) if expected_final_text is not None: _require(final_text == expected_final_text, "final message text did not match expected conformance result") diff --git a/runtimes/common/agentkit_serve_common/brokered.py b/runtimes/common/agentkit_serve_common/brokered.py index d4670ea..e3a4fee 100644 --- a/runtimes/common/agentkit_serve_common/brokered.py +++ b/runtimes/common/agentkit_serve_common/brokered.py @@ -10,9 +10,7 @@ from __future__ import annotations import argparse -import math import sys -from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Iterable, Mapping, Sequence @@ -20,48 +18,15 @@ from .config import AgentSpec, BrokeredToolSpec, brokered_tool_schema_digest from .runtime import BrokeredToolDefinition +from .yaml_support import safe_load_all_lossless _ORKA_TOOL_API_VERSION = "core.orka.ai/v1alpha1" _ORKA_TOOL_KIND = "Tool" _ORKA_BROKERED_TOOL_CLASSES = ("read", "write", "coordination") -_MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS = 4096 - - -class _LosslessToolCRDLoader(yaml.SafeLoader): - """Safe YAML loader that never silently rounds CRD numeric scalars.""" - - -def _construct_lossless_tool_crd_float(loader: yaml.SafeLoader, node: yaml.Node) -> int | float: - raw = loader.construct_scalar(node).replace("_", "").lower() - if ":" in raw: - raise ValueError(f"YAML float literal {raw!r} uses unsupported sexagesimal notation") - try: - decimal = Decimal(raw) - except InvalidOperation as exc: - raise ValueError(f"YAML float literal {raw!r} is invalid") from exc - if not decimal.is_finite(): - raise ValueError(f"YAML float literal {raw!r} must be finite") - if decimal.is_zero() and decimal.is_signed(): - return -0.0 - if decimal == decimal.to_integral_value(): - digits = max(decimal.adjusted() + 1, len(decimal.as_tuple().digits)) if decimal else 1 - if digits > _MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS: - raise ValueError(f"YAML float literal {raw!r} expands to an integer that is too large") - return int(decimal) - try: - candidate = float(decimal) - except (OverflowError, ValueError) as exc: - raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") from exc - if not math.isfinite(candidate) or Decimal(str(candidate)) != decimal: - raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") - return candidate - - -_LosslessToolCRDLoader.add_constructor("tag:yaml.org,2002:float", _construct_lossless_tool_crd_float) def _load_orka_tool_crd_documents(raw: str) -> list[Any]: - return [doc for doc in yaml.load_all(raw, Loader=_LosslessToolCRDLoader) if doc is not None] + return [doc for doc in safe_load_all_lossless(raw) if doc is not None] def brokered_tool_definitions(spec: AgentSpec) -> list[BrokeredToolDefinition]: diff --git a/runtimes/common/agentkit_serve_common/config.py b/runtimes/common/agentkit_serve_common/config.py index 18fa4dd..15db8c4 100644 --- a/runtimes/common/agentkit_serve_common/config.py +++ b/runtimes/common/agentkit_serve_common/config.py @@ -22,6 +22,8 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from .yaml_support import safe_load_lossless + # The ABI schema version this reader understands (agent-abi.md: ``abiVersion: v0``). ABI_VERSION = "v0" _PROVIDER_OPENAI_COMPATIBLE = "openai-compatible" @@ -717,12 +719,16 @@ def _valid_name(cls, value: str) -> str: def _valid_json_schema(cls, value: dict[str, Any]) -> dict[str, Any]: if not isinstance(value, dict): raise ValueError("brokered tool parameters must be a JSON Schema object") - _validate_schema_value_constraints(value, path="brokeredTools[].parameters") + try: + _validate_schema_value_constraints(value, path="brokeredTools[].parameters") + except RecursionError as exc: + raise ValueError("brokered tool parameters must be an acyclic JSON-serializable object") from exc try: encoded = _canonical_json(value) - except (TypeError, ValueError) as exc: - raise ValueError("brokered tool parameters must be JSON serializable") from exc - if len(encoded.encode("utf-8")) > _MAX_BROKERED_SCHEMA_BYTES: + encoded_bytes = encoded.encode("utf-8") + except (TypeError, ValueError, RecursionError, UnicodeEncodeError) as exc: + raise ValueError("brokered tool parameters must be JSON serializable UTF-8") from exc + if len(encoded_bytes) > _MAX_BROKERED_SCHEMA_BYTES: raise ValueError("brokered tool parameters schema is too large") cloned = json.loads(encoded, parse_int=_parse_canonical_int) if cloned.get("type") != "object": @@ -894,8 +900,8 @@ def load(path: str | Path) -> AgentSpec: raise ConfigError(f"cannot read agent config {p}: {exc}") from exc try: - data = yaml.safe_load(raw) - except yaml.YAMLError as exc: + data = safe_load_lossless(raw) + except (yaml.YAMLError, ValueError, RecursionError) as exc: raise ConfigError(f"agent config {p} is not valid YAML: {exc}") from exc if not isinstance(data, dict): diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 47d6410..644a868 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -54,6 +54,7 @@ _MAX_SYNTHETIC_ARRAY_ITEMS = 32 _MAX_SYNTHETIC_STRING_LENGTH = 4096 _MAX_SYNTHETIC_VALUES = 256 +_MAX_OUTPUT_DEPTH = 128 _STATE_TTL_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_TTL_SECONDS" _MAX_PENDING_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING" _MAX_ARGUMENT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES" @@ -394,18 +395,41 @@ def __init__(self, ttl_seconds: float, max_entries: int, state_file: str | Path self.max_entries = max_entries self.state_file = Path(state_file) if state_file else None self._states: dict[str, _HostedResponseState] = {} + self._reservations: set[str] = set() self._load() @property def backend_name(self) -> str: return "file" if self.state_file else "memory" - def add(self, state: _HostedResponseState) -> None: + def reserve(self, response_id: str) -> None: self.purge_expired() - if len(self._states) >= self.max_entries and state.response_id not in self._states: - self.evict_completed_to_capacity(reserve_slots=1) - if len(self._states) >= self.max_entries and state.response_id not in self._states: + if response_id in self._states or response_id in self._reservations: + return + non_evictable = sum( + 1 + for state in self._states.values() + if state.status != "completed" or state.final_payload is None + ) + if non_evictable + len(self._reservations) >= self.max_entries: raise _StateStoreFull("too many pending brokered responses") + self._reservations.add(response_id) + + def release_reservation(self, response_id: str) -> None: + self._reservations.discard(response_id) + + def add(self, state: _HostedResponseState) -> None: + self.purge_expired() + if state.response_id in self._states: + self._reservations.discard(state.response_id) + else: + reserved = state.response_id in self._reservations + self._reservations.discard(state.response_id) + self.evict_completed_to_capacity(reserve_slots=len(self._reservations) + 1) + if len(self._states) + len(self._reservations) >= self.max_entries: + if reserved: + self._reservations.add(state.response_id) + raise _StateStoreFull("too many pending brokered responses") self._states[state.response_id] = state self._persist() @@ -565,14 +589,19 @@ def _function_call_outputs_from_input(input_value: Any) -> list[dict[str, Any]]: def _reject_nonfinite_json_values(value: Any, *, path: str = "value") -> None: - if isinstance(value, float) and not math.isfinite(value): - raise ValueError(f"{path} must be finite") - if isinstance(value, Mapping): - for key, child in value.items(): - _reject_nonfinite_json_values(child, path=f"{path}.{key}") - elif isinstance(value, list): - for idx, child in enumerate(value): - _reject_nonfinite_json_values(child, path=f"{path}[{idx}]") + pending: list[tuple[Any, str, int]] = [(value, path, 0)] + while pending: + current, current_path, depth = pending.pop() + if depth > _MAX_OUTPUT_DEPTH: + raise ValueError(f"{path} is nested too deeply") + if isinstance(current, float) and not math.isfinite(current): + raise ValueError(f"{current_path} must be finite") + if isinstance(current, Mapping): + for key, child in current.items(): + pending.append((child, f"{current_path}.{key}", depth + 1)) + elif isinstance(current, list): + for idx, child in enumerate(current): + pending.append((child, f"{current_path}[{idx}]", depth + 1)) def _parse_output_float(raw: str) -> float: @@ -609,8 +638,8 @@ def _json_object_from_output(output: Any) -> dict[str, Any]: parse_constant=_reject_output_constant, object_pairs_hook=_reject_duplicate_output_keys, ) - except json.JSONDecodeError as exc: - raise ValueError("function_call_output.output must be a JSON object string") from exc + except (json.JSONDecodeError, RecursionError) as exc: + raise ValueError("function_call_output.output must be a JSON object string with bounded nesting") from exc if not isinstance(parsed, dict): raise ValueError("function_call_output.output must be a JSON object") approved = parsed.get("approved") @@ -1695,56 +1724,60 @@ async def responses(request: Request): response_id = _new_response_id(previous_response_id_for_output) call_id = f"call_{response_id}_1" try: - model_result = await model_loop.start(run_request, call_id=call_id) - except AgentRunError as exc: - return _error(str(exc), status=exc.status, code=exc.code) - if isinstance(model_result, ModelLoopFinal): - return JSONResponse(_responses_payload(spec, RunResult(text=model_result.text, usage=model_result.usage), previous_response_id=previous_response_id_for_output)) - tool = {tool.name: tool for tool in brokered_tools}.get(model_result.name) - if tool is None: - return _error("model requested unknown brokered tool", status=400, code="unknown_brokered_tool") - try: - _validate_model_brokered_arguments(model_result.arguments) - _validate_model_arguments_for_tool(model_result.arguments, tool) - except AgentRunError as exc: - return _error(str(exc), status=exc.status, code=exc.code) - if len(_canonical_output_json(model_result.arguments).encode("utf-8")) > max_argument_bytes: - return _error( - "brokered function_call arguments are too large for pending state", - status=413, - code="brokered_arguments_too_large", - ) - call = _PendingCall( - call_id=call_id, - item_id=_new_function_call_id(response_id), - tool=tool, - arguments=model_result.arguments, - ) - state = _HostedResponseState( - response_id=response_id, - session_id=run_request.session_id, - pending_calls={call_id: call}, - expires_at=time.time() + response_states.ttl_seconds, - model_messages=model_result.messages, - initial_usage=dict(model_result.usage), - ) - try: - response_states.add(state) + response_states.reserve(response_id) except _StateStoreFull: return _error( "too many pending brokered responses", status=429, code="brokered_response_state_full", ) - return JSONResponse( - _function_call_response_payload( - spec, + try: + try: + model_result = await model_loop.start(run_request, call_id=call_id) + except AgentRunError as exc: + return _error(str(exc), status=exc.status, code=exc.code) + if isinstance(model_result, ModelLoopFinal): + return JSONResponse(_responses_payload(spec, RunResult(text=model_result.text, usage=model_result.usage), previous_response_id=previous_response_id_for_output)) + tool = {tool.name: tool for tool in brokered_tools}.get(model_result.name) + if tool is None: + return _error("model requested unknown brokered tool", status=400, code="unknown_brokered_tool") + try: + _validate_model_brokered_arguments(model_result.arguments) + _validate_model_arguments_for_tool(model_result.arguments, tool) + except AgentRunError as exc: + return _error(str(exc), status=exc.status, code=exc.code) + if len(_canonical_output_json(model_result.arguments).encode("utf-8")) > max_argument_bytes: + return _error( + "brokered function_call arguments are too large for pending state", + status=413, + code="brokered_arguments_too_large", + ) + call = _PendingCall( + call_id=call_id, + item_id=_new_function_call_id(response_id), + tool=tool, + arguments=model_result.arguments, + ) + state = _HostedResponseState( response_id=response_id, - call=call, - previous_response_id=previous_response_id_for_output, - usage=model_result.usage, + session_id=run_request.session_id, + pending_calls={call_id: call}, + expires_at=time.time() + response_states.ttl_seconds, + model_messages=model_result.messages, + initial_usage=dict(model_result.usage), ) - ) + response_states.add(state) + return JSONResponse( + _function_call_response_payload( + spec, + response_id=response_id, + call=call, + previous_response_id=previous_response_id_for_output, + usage=model_result.usage, + ) + ) + finally: + response_states.release_reservation(response_id) tool = _select_brokered_tool(brokered_tools, run_request) if tool is None: return _error( diff --git a/runtimes/common/agentkit_serve_common/foundry_model_loop.py b/runtimes/common/agentkit_serve_common/foundry_model_loop.py index bab480d..82c94dc 100644 --- a/runtimes/common/agentkit_serve_common/foundry_model_loop.py +++ b/runtimes/common/agentkit_serve_common/foundry_model_loop.py @@ -209,15 +209,25 @@ def _choice_message(data: Mapping[str, Any]) -> Mapping[str, Any]: def _message_text(message: Mapping[str, Any]) -> str: content = message.get("content") if isinstance(content, str): - return content - refusal = message.get("refusal") - if content is None and isinstance(refusal, str): - return refusal - raise AgentRunError( - "model response final assistant content must be a string", - status=502, - code="InvalidModelResponse", - ) + text = content + else: + refusal = message.get("refusal") + if content is not None or not isinstance(refusal, str): + raise AgentRunError( + "model response final assistant content must be a string", + status=502, + code="InvalidModelResponse", + ) + text = refusal + try: + text.encode("utf-8") + except UnicodeEncodeError as exc: + raise AgentRunError( + "model response final assistant content must contain valid Unicode", + status=502, + code="InvalidModelResponse", + ) from exc + return text def _parse_arguments(raw: Any) -> dict[str, Any]: @@ -288,9 +298,9 @@ def _reject_json_constant(raw: str) -> None: def _usage_token_count(value: Any) -> int: - if value is None or isinstance(value, bool): - if value is None: - return 0 + if value is None: + return 0 + if isinstance(value, bool) or not isinstance(value, (int, float)): raise AgentRunError( "model response usage must contain non-negative integer token counts", status=502, @@ -302,14 +312,7 @@ def _usage_token_count(value: Any) -> int: status=502, code="InvalidModelResponse", ) - try: - count = int(value) - except (TypeError, ValueError, OverflowError) as exc: - raise AgentRunError( - "model response usage must contain non-negative integer token counts", - status=502, - code="InvalidModelResponse", - ) from exc + count = int(value) if count < 0: raise AgentRunError( "model response usage must contain non-negative integer token counts", diff --git a/runtimes/common/agentkit_serve_common/yaml_support.py b/runtimes/common/agentkit_serve_common/yaml_support.py new file mode 100644 index 0000000..afffeab --- /dev/null +++ b/runtimes/common/agentkit_serve_common/yaml_support.py @@ -0,0 +1,59 @@ +"""Lossless safe YAML loading for JSON-compatible numeric configuration.""" + +from __future__ import annotations + +import math +from decimal import Decimal, InvalidOperation +from typing import Any, Iterator + +import yaml + +_MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS = 4096 +_MAX_EXACT_JSON_FLOAT_INTEGER = (1 << 53) - 1 + + +class _LosslessSafeLoader(yaml.SafeLoader): + """SafeLoader variant that never silently rounds YAML float scalars.""" + + +def _construct_lossless_float(loader: yaml.SafeLoader, node: yaml.Node) -> int | float: + raw = loader.construct_scalar(node).replace("_", "").lower() + if ":" in raw: + raise ValueError(f"YAML float literal {raw!r} uses unsupported sexagesimal notation") + try: + decimal = Decimal(raw) + except InvalidOperation as exc: + raise ValueError(f"YAML float literal {raw!r} is invalid") from exc + if not decimal.is_finite(): + raise ValueError(f"YAML float literal {raw!r} must be finite") + if decimal.is_zero() and decimal.is_signed(): + return -0.0 + if decimal == decimal.to_integral_value(): + digits = max(decimal.adjusted() + 1, len(decimal.as_tuple().digits)) if decimal else 1 + if digits > _MAX_EXACT_YAML_FLOAT_INTEGER_DIGITS: + raise ValueError(f"YAML float literal {raw!r} expands to an integer that is too large") + if abs(decimal) <= _MAX_EXACT_JSON_FLOAT_INTEGER: + return float(decimal) + return int(decimal) + try: + candidate = float(decimal) + except (OverflowError, ValueError) as exc: + raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") from exc + if not math.isfinite(candidate) or Decimal(str(candidate)) != decimal: + raise ValueError(f"YAML float literal {raw!r} cannot be represented exactly") + return candidate + + +_LosslessSafeLoader.add_constructor("tag:yaml.org,2002:float", _construct_lossless_float) + + +def safe_load_lossless(raw: str) -> Any: + """Load one YAML document without lossy float coercion.""" + + return yaml.load(raw, Loader=_LosslessSafeLoader) + + +def safe_load_all_lossless(raw: str) -> Iterator[Any]: + """Load YAML documents without lossy float coercion.""" + + return yaml.load_all(raw, Loader=_LosslessSafeLoader) diff --git a/runtimes/common/tests/test_config_validation.py b/runtimes/common/tests/test_config_validation.py index de66ac0..47fc4cf 100644 --- a/runtimes/common/tests/test_config_validation.py +++ b/runtimes/common/tests/test_config_validation.py @@ -800,7 +800,6 @@ def test_load_rejects_malformed_nested_brokered_json_schema(tmp_path, bad_child: [ {"type": "object", "properties": {"site": {"type": None}}}, {"type": "object", "properties": {"n": {"type": "integer", "default": "1"}}}, - {"type": "object", "properties": {"n": {"type": "integer", "default": 9007199254740993.0}}}, {"type": "object", "properties": {"site": {"type": "string", "enum": [0, "ok"]}}}, ], ) @@ -822,6 +821,139 @@ def test_load_rejects_invalid_brokered_schema_type_values_and_defaults(tmp_path, assert "brokeredTools.0.parameters" in msg +def test_load_preserves_high_precision_integral_brokered_yaml_number(tmp_path): + path = tmp_path / "agent.yaml" + path.write_text( + """abiVersion: v0 +metadata: + name: precise-agent +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Be precise. +tools: [] +brokeredTools: + - name: precise_lookup + description: Preserve precise identifiers. + brokeredClass: read + parameters: + type: object + properties: + identifier: + type: integer + default: 9007199254740993.0 +expose: + openai: true + port: 8080 +""", + encoding="utf-8", + ) + + spec = load(path) + + default = spec.brokered_tools[0].parameters["properties"]["identifier"]["default"] + assert default == 9007199254740993 + assert isinstance(default, int) + + +def test_load_rejects_lossy_fractional_brokered_yaml_number(tmp_path): + path = tmp_path / "agent.yaml" + path.write_text( + """abiVersion: v0 +metadata: + name: lossy-agent +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Be precise. +tools: [] +brokeredTools: + - name: lossy_lookup + description: Reject lossy ratios. + brokeredClass: read + parameters: + type: object + properties: + ratio: + type: number + default: 0.100000000000000005 +expose: + openai: true + port: 8080 +""", + encoding="utf-8", + ) + + with pytest.raises(ConfigError, match="cannot be represented exactly"): + load(path) + + +def test_load_normalizes_cyclic_brokered_schema_to_config_error(tmp_path): + path = tmp_path / "agent.yaml" + path.write_text( + """abiVersion: v0 +metadata: + name: cyclic-agent +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Be safe. +tools: [] +brokeredTools: + - name: cyclic_lookup + description: Reject cyclic schemas. + brokeredClass: read + parameters: &schema + type: object + properties: + nested: *schema +expose: + openai: true + port: 8080 +""", + encoding="utf-8", + ) + + with pytest.raises(ConfigError, match="acyclic JSON-serializable"): + load(path) + + +def test_load_normalizes_surrogate_brokered_schema_to_config_error(tmp_path): + path = tmp_path / "agent.yaml" + path.write_text( + """abiVersion: v0 +metadata: + name: surrogate-agent +model: + provider: openai-compatible + baseURL: https://api.openai.com/v1 + name: gpt-4o-mini +instructions: Be safe. +tools: [] +brokeredTools: + - name: surrogate_lookup + description: Reject invalid Unicode. + brokeredClass: read + parameters: + type: object + properties: + text: + type: string + default: "\\ud800" +expose: + openai: true + port: 8080 +""", + encoding="utf-8", + ) + + with pytest.raises(ConfigError, match="JSON serializable UTF-8"): + load(path) + + def test_load_accepts_integral_float_values_for_integer_brokered_schema(tmp_path): spec_dict = deepcopy(_BASE_SPEC) spec_dict.update( diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 6b9ae72..861b40f 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -1567,7 +1567,8 @@ def handler(self, request: httpx.Request) -> httpx.Response: self.requests.append(json.loads(request.content.decode("utf-8"))) if not self.responses: return httpx.Response(500, json={"error": "unexpected extra model call"}) - return httpx.Response(200, json=self.responses.pop(0)) + body = json.dumps(self.responses.pop(0), separators=(",", ":")).encode("utf-8") + return httpx.Response(200, content=body, headers={"content-type": "application/json"}) def _chat_response(message: dict[str, Any], *, prompt_tokens: int = 1, completion_tokens: int = 1) -> dict[str, Any]: @@ -1620,6 +1621,7 @@ async def aclose(self) -> None: [ {"prompt_tokens": {"unexpected": 1}}, {"completion_tokens": "not-a-number"}, + {"prompt_tokens": "12"}, {"total_tokens": 1.5}, ], ) @@ -1804,6 +1806,112 @@ def test_foundry_brokered_rejects_denied_continuation_without_error_object(paylo assert response.json()["error"]["code"] == "invalid_function_call_output" +def test_foundry_brokered_rejects_deep_continuation_output_without_consuming_state(): + app = _app() + nested: dict[str, Any] = {} + for _ in range(200): + nested = {"nested": nested} + + with TestClient(app) as client: + initial = _start(client) + call = _call(initial) + rejected = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": nested}), + ) + accepted = client.post( + "/responses", + headers=CONTINUATION_AUTH, + json=_continuation(initial["id"], call["call_id"], {"approved": True, "output": {"ok": True}}), + ) + + assert rejected.status_code == 400 + assert rejected.json()["error"]["code"] == "invalid_function_call_output" + assert accepted.status_code == 200, accepted.text + + +def test_foundry_brokered_model_loop_reserves_capacity_before_model_call(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ), + _chat_response({"role": "assistant", "content": "must not be called"}), + ] + ) + app = _model_loop_app( + _spec(tool_name="check-network-telemetry"), + fake, + max_pending_responses=1, + ) + + with TestClient(app) as client: + first = client.post("/responses", json={"input": "check-network-telemetry"}) + second = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert first.status_code == 200, first.text + assert _call(first.json()) + assert second.status_code == 429 + assert second.json()["error"]["code"] == "brokered_response_state_full" + assert len(fake.requests) == 1 + + +def test_foundry_brokered_model_loop_unused_reservation_preserves_completed_replay(): + fake = _FakeChatTransport( + [ + _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ), + _chat_response({"role": "assistant", "content": "First response completed."}), + _chat_response({"role": "assistant", "content": "No tool needed."}), + ] + ) + app = _model_loop_app( + _spec(tool_name="check-network-telemetry"), + fake, + max_pending_responses=1, + ) + + with TestClient(app) as client: + initial = client.post("/responses", json={"input": "check-network-telemetry"}) + call = _call(initial.json()) + continuation = _continuation( + initial.json()["id"], + call["call_id"], + {"approved": True, "output": {"ok": True}}, + ) + completed = client.post("/responses", headers=CONTINUATION_AUTH, json=continuation) + direct = client.post("/responses", json={"input": "answer without a tool"}) + replayed = client.post("/responses", headers=CONTINUATION_AUTH, json=continuation) + + assert completed.status_code == 200, completed.text + assert direct.status_code == 200, direct.text + assert replayed.status_code == 200, replayed.text + assert replayed.json() == completed.json() + assert len(fake.requests) == 3 + + def test_foundry_brokered_model_loop_rejects_oversized_output_before_resume_or_state_change(tmp_path): state_file = tmp_path / "responses-state.json" spec = _spec(tool_name="check-network-telemetry") @@ -2058,6 +2166,24 @@ def test_foundry_brokered_model_loop_returns_assistant_refusal_without_tool_call assert _message_text(response.json()) == "I cannot help with that." +@pytest.mark.parametrize( + "message", + [ + {"role": "assistant", "content": "\ud800"}, + {"role": "assistant", "content": None, "refusal": "\ud800"}, + ], +) +def test_foundry_brokered_model_loop_rejects_surrogate_final_text(message: dict[str, Any]): + fake = _FakeChatTransport([_chat_response(message)]) + app = _model_loop_app(_spec(tool_name="check-network-telemetry"), fake) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "Say hello"}) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "InvalidModelResponse" + + @pytest.mark.parametrize("content", [None, {"text": "not supported"}, ["not supported"]]) def test_foundry_brokered_model_loop_rejects_non_string_final_content(content: Any): fake = _FakeChatTransport([_chat_response({"role": "assistant", "content": content})]) diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index 6d68a78..b1e0497 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -107,6 +107,51 @@ def test_verify_brokered_transcript_rejects_duplicate_keys_in_top_level_files(tm verifier.verify_transcript(transcript) +def test_verify_brokered_transcript_rejects_mismatched_function_call_response_id(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + initial_path = transcript / "02-initial-response.json" + initial = json.loads(initial_path.read_text(encoding="utf-8")) + initial["output"][0]["response_id"] = "caresp_other" + initial_path.write_text(json.dumps(initial), encoding="utf-8") + + with pytest.raises(ValueError, match="function_call response_id must match"): + verifier.verify_transcript(transcript) + + +def test_verify_brokered_transcript_rejects_mismatched_final_message_response_id(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation_path = transcript / "04-continuation-response.json" + continuation = json.loads(continuation_path.read_text(encoding="utf-8")) + continuation["output"][0]["response_id"] = "caresp_other" + continuation_path.write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="final message response_id must match"): + verifier.verify_transcript(transcript) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("role", "user", "final message role must be assistant"), + ("status", "in_progress", "final message status must be completed"), + ], +) +def test_verify_brokered_transcript_rejects_noncompleted_assistant_message( + tmp_path, field: str, value: str, message: str +): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation_path = transcript / "04-continuation-response.json" + continuation = json.loads(continuation_path.read_text(encoding="utf-8")) + continuation["output"][0][field] = value + continuation_path.write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match=message): + verifier.verify_transcript(transcript) + + def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) From 52309dd5b10eb4e237f8cef55c8a836a9afa58ad Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Mon, 13 Jul 2026 10:45:52 -0700 Subject: [PATCH 27/27] fix(foundry): bound brokered state and transcript data Signed-off-by: Sertac Ozercan --- deploy/foundry/README.md | 2 +- .../scripts/verify_brokered_transcript.py | 13 +- docs/foundry-hosted-brokered.md | 2 +- .../common/agentkit_serve_common/foundry.py | 142 +++++++++++++-- .../tests/test_foundry_brokered_protocol.py | 165 ++++++++++++++++++ .../common/tests/test_foundry_protocol.py | 16 ++ .../tests/test_foundry_transcript_verifier.py | 73 ++++++-- test/foundry-brokered-conformance/README.md | 2 +- 8 files changed, 383 insertions(+), 32 deletions(-) diff --git a/deploy/foundry/README.md b/deploy/foundry/README.md index c7ede84..8de4609 100644 --- a/deploy/foundry/README.md +++ b/deploy/foundry/README.md @@ -108,5 +108,5 @@ instead of invoking `az account get-access-token`. If `AZURE_SUBSCRIPTION_ID` is omitted, the helper uses the current `az` account. The script stores request and response JSON files plus `summary.json`; do not include bearer tokens in the transcript. Re-run -`python3 deploy/foundry/scripts/verify_brokered_transcript.py ` +`python3 deploy/foundry/scripts/verify_brokered_transcript.py --expected-final-text ''` to verify archived transcript evidence later. diff --git a/deploy/foundry/scripts/verify_brokered_transcript.py b/deploy/foundry/scripts/verify_brokered_transcript.py index 56b81cb..9fbd5d4 100755 --- a/deploy/foundry/scripts/verify_brokered_transcript.py +++ b/deploy/foundry/scripts/verify_brokered_transcript.py @@ -104,8 +104,10 @@ def _message_text(response: dict[str, Any], *, response_id: str) -> str: _require(message.get("role") == "assistant", "final message role must be assistant") _require(message.get("status") == "completed", "final message status must be completed") content = message.get("content") - _require(isinstance(content, list) and bool(content), "final message content must be a non-empty array") - text = content[0].get("text") if isinstance(content[0], dict) else None + _require(isinstance(content, list) and len(content) == 1, "final message content must contain exactly one item") + content_item = content[0] + _require(isinstance(content_item, dict) and content_item.get("type") == "output_text", "final message content[0] must be output_text") + text = content_item.get("text") _require(isinstance(text, str) and bool(text), "final message must contain text") return text @@ -120,6 +122,7 @@ def verify_transcript( expected_call_id: str = "call_conformance_1", expected_call_id_prefix: str | None = None, ) -> dict[str, Any]: + _require(expected_final_text is not None, "expected final text is required for transcript verification") root = Path(transcript_dir) expected_arguments = _parse_json_lossless(expected_arguments_json) expected_output = _parse_json_lossless(expected_output_json) @@ -142,6 +145,7 @@ def verify_transcript( call = output[0] _require(isinstance(call, dict), "initial response output[0] must be an object") _require(call.get("type") == "function_call", "initial output item must be function_call") + _require(call.get("status") == "completed", "function_call status must be completed") _require(call.get("response_id") == initial_response_id, "function_call response_id must match initial response id") function_name = call.get("name") _require(function_name == expected_tool_name, f"function_call name must be {expected_tool_name}") @@ -198,8 +202,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--expected-arguments-json", default='{"probe":true}', help="expected function_call arguments JSON") parser.add_argument("--expected-output-json", default='{"approved":true,"output":{"success":true}}', help="expected function_call_output JSON") parser.add_argument("--expected-output-file", default=None, help="private file containing expected function_call_output JSON") - parser.add_argument("--expected-final-text", default=None, help="expected final assistant text") - parser.add_argument("--expected-final-text-file", default=None, help="private file containing expected final assistant text") + final_text_group = parser.add_mutually_exclusive_group(required=True) + final_text_group.add_argument("--expected-final-text", help="expected final assistant text") + final_text_group.add_argument("--expected-final-text-file", help="private file containing expected final assistant text") parser.add_argument("--expected-call-id", default="call_conformance_1", help="expected call_id, or 'auto' to only require a non-empty id") parser.add_argument("--expected-call-id-prefix", default=None, help="optional required call_id prefix") parser.add_argument("--write-summary", action="store_true", help="write summary.json in the transcript directory") diff --git a/docs/foundry-hosted-brokered.md b/docs/foundry-hosted-brokered.md index aed4eef..beb7fed 100644 --- a/docs/foundry-hosted-brokered.md +++ b/docs/foundry-hosted-brokered.md @@ -287,7 +287,7 @@ deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundr The script performs the initial `function_call` request, posts the matching `function_call_output` continuation with `previous_response_id`, asserts SDK-style `caresp_...` IDs, and saves request/response JSON plus `summary.json` as a -sanitized transcript. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py ` to verify archived transcript evidence later. This live transcript is still required before claiming A0 +sanitized transcript. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py --expected-final-text ''` to verify archived transcript evidence later. This live transcript is still required before claiming A0 completion; the local test only proves the SDK-hosted contract before deployment. ## Production brokered-only fixture diff --git a/runtimes/common/agentkit_serve_common/foundry.py b/runtimes/common/agentkit_serve_common/foundry.py index 644a868..e20b522 100644 --- a/runtimes/common/agentkit_serve_common/foundry.py +++ b/runtimes/common/agentkit_serve_common/foundry.py @@ -51,6 +51,8 @@ _DEFAULT_MAX_PENDING_RESPONSES = 128 _DEFAULT_MAX_ARGUMENT_BYTES = 8192 _DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024 +_DEFAULT_MAX_REQUEST_BODY_BYTES = 1024 * 1024 +_DEFAULT_MAX_MODEL_MESSAGES_BYTES = 1024 * 1024 _MAX_SYNTHETIC_ARRAY_ITEMS = 32 _MAX_SYNTHETIC_STRING_LENGTH = 4096 _MAX_SYNTHETIC_VALUES = 256 @@ -59,6 +61,8 @@ _MAX_PENDING_ENV = "AGENTKIT_FOUNDRY_RESPONSE_STATE_MAX_PENDING" _MAX_ARGUMENT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_ARGUMENT_BYTES" _MAX_OUTPUT_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_OUTPUT_BYTES" +_MAX_REQUEST_BODY_BYTES_ENV = "AGENTKIT_FOUNDRY_MAX_REQUEST_BODY_BYTES" +_MAX_MODEL_MESSAGES_BYTES_ENV = "AGENTKIT_FOUNDRY_BROKERED_MAX_MODEL_MESSAGES_BYTES" _CONTINUATION_PROOF_ENV = "AGENTKIT_FOUNDRY_BROKERED_CONTINUATION_PROOF" _CONTINUATION_PROOF_HEADER = "x-agentkit-brokered-continuation-proof" _MODEL_LOOP_ENV = "AGENTKIT_FOUNDRY_BROKERED_MODEL_LOOP" @@ -330,6 +334,15 @@ def _pending_call_from_state_payload(data: Mapping[str, Any]) -> _PendingCall: ) +def _persistence_json_bytes(value: Any) -> bytes: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + def _state_to_payload(state: _HostedResponseState) -> dict[str, Any]: return { "responseID": state.response_id, @@ -361,6 +374,9 @@ def _state_from_payload(data: Mapping[str, Any]) -> _HostedResponseState: if not isinstance(initial_usage, Mapping): raise ValueError("stored initialUsage must be an object") status = str(data.get("status") or "pending") + expires_at = float(data.get("expiresAt") or 0) + if not math.isfinite(expires_at): + raise ValueError("stored expiresAt must be finite") accepted = {str(key): str(value) for key, value in accepted_outputs.items()} if final_payload is None and accepted: # A persisted accepted output without a final payload means the process @@ -373,7 +389,7 @@ def _state_from_payload(data: Mapping[str, Any]) -> _HostedResponseState: response_id=str(data.get("responseID") or ""), session_id=str(data["sessionID"]) if data.get("sessionID") is not None else None, pending_calls={str(call_id): _pending_call_from_state_payload(call) for call_id, call in pending_calls_raw.items() if isinstance(call, Mapping)}, - expires_at=float(data.get("expiresAt") or 0), + expires_at=expires_at, status=status, accepted_outputs=accepted, final_payload=final_payload, @@ -497,7 +513,7 @@ def _persist(self) -> None: self.state_file.parent.mkdir(parents=True, exist_ok=True) payload = {"states": {response_id: _state_to_payload(state) for response_id, state in self._states.items()}} tmp = self.state_file.with_name(f".{self.state_file.name}.tmp") - data = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + data = _persistence_json_bytes(payload) try: tmp.unlink(missing_ok=True) except OSError: @@ -560,6 +576,37 @@ def _max_output_bytes(value: int | None = None) -> int: return _positive_int_setting(value, env_name=_MAX_OUTPUT_BYTES_ENV, default=_DEFAULT_MAX_OUTPUT_BYTES) +def _max_request_body_bytes(value: int | None = None) -> int: + return _positive_int_setting( + value, + env_name=_MAX_REQUEST_BODY_BYTES_ENV, + default=_DEFAULT_MAX_REQUEST_BODY_BYTES, + ) + + +def _max_model_messages_bytes(value: int | None = None) -> int: + return _positive_int_setting( + value, + env_name=_MAX_MODEL_MESSAGES_BYTES_ENV, + default=_DEFAULT_MAX_MODEL_MESSAGES_BYTES, + ) + + +class _RequestBodyTooLarge(Exception): + pass + + +async def _bounded_request_body(request: Request, *, max_bytes: int) -> bytes: + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > max_bytes: + raise _RequestBodyTooLarge + chunks.append(chunk) + return b"".join(chunks) + + def _brokered_model_loop_enabled(value: bool | None = None) -> bool: if value is not None: return value @@ -846,7 +893,37 @@ def _required_property_names(schema: Mapping[str, Any]) -> list[str]: return names -def _sample_argument_value(name: str, schema: Any, run_request: RunRequest, *, budget: list[int] | None = None) -> Any: +def _bounded_synthetic_literal(name: str, value: Any, *, depth: int) -> Any: + pending: list[tuple[Any, int]] = [(value, depth)] + while pending: + current, current_depth = pending.pop() + if current_depth > _MAX_OUTPUT_DEPTH: + raise AgentRunError( + f"brokered tool schema for {name!r} exceeds deterministic synthesis depth limit", + status=413, + code="brokered_arguments_too_large", + ) + if isinstance(current, Mapping): + pending.extend((child, current_depth + 1) for child in current.values()) + elif isinstance(current, list): + pending.extend((child, current_depth + 1) for child in current) + return value + + +def _sample_argument_value( + name: str, + schema: Any, + run_request: RunRequest, + *, + budget: list[int] | None = None, + depth: int = 1, +) -> Any: + if depth > _MAX_OUTPUT_DEPTH: + raise AgentRunError( + f"brokered tool schema for {name!r} exceeds deterministic synthesis depth limit", + status=413, + code="brokered_arguments_too_large", + ) if budget is None: budget = [_MAX_SYNTHETIC_VALUES] if budget[0] <= 0: @@ -865,12 +942,12 @@ def _sample_argument_value(name: str, schema: Any, run_request: RunRequest, *, b code="UnsupportedBrokeredSchema", ) if "const" in schema: - return schema["const"] + return _bounded_synthetic_literal(name, schema["const"], depth=depth) if "default" in schema: - return schema["default"] + return _bounded_synthetic_literal(name, schema["default"], depth=depth) enum_values = schema.get("enum") if isinstance(enum_values, list) and enum_values: - return enum_values[0] + return _bounded_synthetic_literal(name, enum_values[0], depth=depth) schema_type = schema.get("type") if isinstance(schema_type, list): @@ -1044,7 +1121,10 @@ def add_candidate(candidate: int | float) -> None: code="brokered_arguments_too_large", ) item_schema = schema.get("items", {}) - return [_sample_argument_value(name, item_schema, run_request, budget=budget) for _ in range(min_items)] + return [ + _sample_argument_value(name, item_schema, run_request, budget=budget, depth=depth + 1) + for _ in range(min_items) + ] return [] if schema_type == "object": @@ -1055,7 +1135,13 @@ def add_candidate(candidate: int | float) -> None: nested: dict[str, Any] = {} properties = schema.get("properties") if isinstance(schema.get("properties"), Mapping) else {} for child_name in _required_property_names(schema): - nested[child_name] = _sample_argument_value(child_name, properties.get(child_name, {}), run_request, budget=budget) + nested[child_name] = _sample_argument_value( + child_name, + properties.get(child_name, {}), + run_request, + budget=budget, + depth=depth + 1, + ) return nested for key in ("const", "default"): @@ -1102,7 +1188,7 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run for key in ("const", "default"): literal = parameters.get(key) if isinstance(literal, Mapping): - return dict(literal) + return dict(_bounded_synthetic_literal(tool.name, literal, depth=0)) enum = parameters.get("enum") if isinstance(enum, list): if tool.brokered_class != "read" and len(enum) > 1: @@ -1113,7 +1199,7 @@ def _deterministic_tool_arguments(tool: BrokeredToolDefinition, run_request: Run ) for item in enum: if isinstance(item, Mapping): - return dict(item) + return dict(_bounded_synthetic_literal(tool.name, item, depth=0)) properties = parameters.get("properties") if isinstance(parameters.get("properties"), Mapping) else {} required_names = _required_property_names(parameters) if tool.brokered_class != "read": @@ -1559,6 +1645,8 @@ def create_foundry_app( max_pending_responses: int | None = None, max_brokered_argument_bytes: int | None = None, max_brokered_output_bytes: int | None = None, + max_request_body_bytes: int | None = None, + max_model_messages_bytes: int | None = None, brokered_model_loop_enabled: bool | None = None, brokered_model_http_client: Any | None = None, response_state_file: str | Path | None = None, @@ -1569,6 +1657,8 @@ def create_foundry_app( continuation_proof = brokered_continuation_proof or os.environ.get(_CONTINUATION_PROOF_ENV) or None max_argument_bytes = _max_argument_bytes(max_brokered_argument_bytes) max_output_bytes = _max_output_bytes(max_brokered_output_bytes) + request_body_limit = _max_request_body_bytes(max_request_body_bytes) + model_messages_limit = _max_model_messages_bytes(max_model_messages_bytes) response_states = _FoundryResponseStateStore( ttl_seconds=_state_ttl_seconds(state_ttl_seconds), max_entries=_max_pending_responses(max_pending_responses), @@ -1626,8 +1716,12 @@ async def invocations(request: Request): code="invocations_disabled_in_brokered_mode", ) try: - data = await request.json() - except json.JSONDecodeError: + raw_body = await _bounded_request_body(request, max_bytes=request_body_limit) + except _RequestBodyTooLarge: + return Response("Request body is too large", status_code=413) + try: + data = json.loads(raw_body) + except (json.JSONDecodeError, UnicodeDecodeError, RecursionError): return Response("Request body must be JSON", status_code=400) if not isinstance(data, dict): @@ -1650,8 +1744,12 @@ async def invocations(request: Request): @app.post("/responses", dependencies=[auth]) async def responses(request: Request): try: - data = await request.json() - except json.JSONDecodeError: + raw_body = await _bounded_request_body(request, max_bytes=request_body_limit) + except _RequestBodyTooLarge: + return _error("Request body is too large", status=413, code="request_body_too_large") + try: + data = json.loads(raw_body) + except (json.JSONDecodeError, UnicodeDecodeError, RecursionError): return _error("Request body must be JSON", status=400, code="invalid_json") if not isinstance(data, dict): @@ -1752,6 +1850,22 @@ async def responses(request: Request): status=413, code="brokered_arguments_too_large", ) + try: + model_messages_bytes = len( + _persistence_json_bytes(model_result.messages) + ) + except (TypeError, ValueError, RecursionError, UnicodeEncodeError) as exc: + return _error( + f"model loop messages are invalid: {exc}", + status=502, + code="InvalidModelResponse", + ) + if model_messages_bytes > model_messages_limit: + return _error( + "model loop messages are too large for pending state", + status=413, + code="brokered_model_messages_too_large", + ) call = _PendingCall( call_id=call_id, item_id=_new_function_call_id(response_id), diff --git a/runtimes/common/tests/test_foundry_brokered_protocol.py b/runtimes/common/tests/test_foundry_brokered_protocol.py index 861b40f..51a8d1f 100644 --- a/runtimes/common/tests/test_foundry_brokered_protocol.py +++ b/runtimes/common/tests/test_foundry_brokered_protocol.py @@ -748,6 +748,75 @@ def test_foundry_brokered_counts_wide_object_members_against_synthesis_budget(): assert response.json()["error"]["code"] == "brokered_arguments_too_large" +def test_foundry_brokered_rejects_overly_deep_deterministic_synthesis(): + child: dict[str, Any] = {"type": "string", "default": "leaf"} + for _ in range(129): + child = { + "type": "object", + "properties": {"child": child}, + "required": ["child"], + } + spec = _spec(tool_name="deep-synthesis") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"root": child}, + "required": ["root"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "deep-synthesis"}) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_arguments_too_large" + + +def test_foundry_brokered_rejects_overly_deep_root_literal_synthesis(): + literal: dict[str, Any] = {} + for _ in range(129): + literal = {"nested": literal} + spec = _spec(tool_name="deep-root-literal") + spec.brokered_tools[0].parameters = { + "type": "object", + "const": literal, + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "deep-root-literal"}) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "brokered_arguments_too_large" + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + ({"const": 2.0}, 2.0), + ({"default": 1.0}, 1.0), + ({"enum": [3.0]}, 3.0), + ], +) +def test_foundry_brokered_honors_integral_float_integer_literals( + literal: dict[str, Any], expected: float +): + spec = _spec(tool_name="integer-literal") + spec.brokered_tools[0].parameters = { + "type": "object", + "properties": {"count": {"type": "integer", **literal}}, + "required": ["count"], + } + app = _app(spec) + + with TestClient(app) as client: + response = client.post("/responses", json={"input": "integer-literal"}) + + assert response.status_code == 200, response.text + count = json.loads(_call(response.json())["arguments"])["count"] + assert count == expected + assert isinstance(count, float) + + def test_foundry_brokered_tool_selection_requires_token_boundary_match(): data = _multi_tool_spec().model_dump(by_alias=True) data["brokeredTools"] = [ @@ -1831,6 +1900,85 @@ def test_foundry_brokered_rejects_deep_continuation_output_without_consuming_sta assert accepted.status_code == 200, accepted.text +def test_foundry_brokered_bounds_request_body_before_model_call(): + fake = _FakeChatTransport([_chat_response({"role": "assistant", "content": "must not run"})]) + app = _model_loop_app( + _spec(tool_name="check-network-telemetry"), + fake, + max_request_body_bytes=64, + ) + + with TestClient(app) as client: + response = client.post( + "/responses", + content=json.dumps({"input": "x" * 128}), + headers={"content-type": "application/json"}, + ) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "request_body_too_large" + assert fake.requests == [] + + +def test_foundry_brokered_model_message_size_matches_persistence_encoding(): + messages = [{"role": "user", "content": "😀" * 8}] + + measured = foundry_module._persistence_json_bytes(messages) + persisted = json.dumps( + messages, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + utf8_compact = json.dumps( + messages, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + assert measured == persisted + assert len(measured) > len(utf8_compact) + + +def test_foundry_brokered_bounds_persisted_model_messages_and_releases_reservation(): + tool_response = _chat_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_model", + "type": "function", + "function": {"name": "check-network-telemetry", "arguments": "{}"}, + } + ], + } + ) + fake = _FakeChatTransport([tool_response, deepcopy(tool_response)]) + app = _model_loop_app( + _spec(tool_name="check-network-telemetry"), + fake, + max_request_body_bytes=4096, + max_model_messages_bytes=512, + max_pending_responses=1, + ) + + with TestClient(app) as client: + oversized = client.post( + "/responses", + json={"input": "check-network-telemetry " + ("x" * 1024)}, + ) + accepted = client.post("/responses", json={"input": "check-network-telemetry"}) + + assert oversized.status_code == 413 + assert oversized.json()["error"]["code"] == "brokered_model_messages_too_large" + assert accepted.status_code == 200, accepted.text + assert _call(accepted.json()) + assert len(fake.requests) == 2 + + def test_foundry_brokered_model_loop_reserves_capacity_before_model_call(): fake = _FakeChatTransport( [ @@ -1970,6 +2118,23 @@ def test_foundry_brokered_invalid_file_state_fails_startup_without_overwriting(t assert state_file.read_text(encoding="utf-8") == raw_state +@pytest.mark.parametrize("expires_at", [float("nan"), float("inf"), float("-inf")]) +def test_foundry_brokered_rejects_nonfinite_persisted_expiry(tmp_path, expires_at: float): + state_file = tmp_path / "responses-state.json" + with TestClient(_app(response_state_file=state_file)) as client: + _start(client) + persisted = json.loads(state_file.read_text(encoding="utf-8")) + state = next(iter(persisted["states"].values())) + state["expiresAt"] = expires_at + raw_state = json.dumps(persisted, separators=(",", ":")) + state_file.write_text(raw_state, encoding="utf-8") + + with pytest.raises(RuntimeError, match="stored expiresAt must be finite"): + _app(response_state_file=state_file) + + assert state_file.read_text(encoding="utf-8") == raw_state + + def test_foundry_brokered_file_state_is_written_with_private_permissions(tmp_path): state_file = tmp_path / "responses-state.json" app = _app(response_state_file=state_file) diff --git a/runtimes/common/tests/test_foundry_protocol.py b/runtimes/common/tests/test_foundry_protocol.py index d9541ce..c6133e7 100644 --- a/runtimes/common/tests/test_foundry_protocol.py +++ b/runtimes/common/tests/test_foundry_protocol.py @@ -98,6 +98,22 @@ def test_foundry_responses_tolerates_stream_flag_with_non_streaming_response(): assert resp.json()["output"][0]["content"][0]["text"] == "echo: hi" +def test_foundry_invocations_enforces_request_body_limit_before_runtime(): + factory = EchoFactory() + app = create_foundry_app(_spec(), factory, max_request_body_bytes=64) + + with TestClient(app) as client: + response = client.post( + "/invocations", + content='{"message":"' + ("x" * 128) + '"}', + headers={"content-type": "application/json"}, + ) + + assert response.status_code == 413 + assert "too large" in response.text + assert factory.runtime.requests == [] + + def test_foundry_protocols_reject_non_object_json(): app = create_foundry_app(_spec(), EchoFactory()) with TestClient(app) as client: diff --git a/runtimes/common/tests/test_foundry_transcript_verifier.py b/runtimes/common/tests/test_foundry_transcript_verifier.py index b1e0497..1f7932e 100644 --- a/runtimes/common/tests/test_foundry_transcript_verifier.py +++ b/runtimes/common/tests/test_foundry_transcript_verifier.py @@ -20,6 +20,15 @@ def _load_verifier(): return module +_EXPECTED_SDK_FINAL_TEXT = 'conformance complete: {"approved": true, "output": {"success": true}}' +_EXPECTED_AGENTKIT_FINAL_TEXT = 'Brokered tool conformance_read completed with output: {"success":true}' + + +def _verify(verifier, transcript, **kwargs): # noqa: ANN001 + kwargs.setdefault("expected_final_text", _EXPECTED_SDK_FINAL_TEXT) + return verifier.verify_transcript(transcript, **kwargs) + + def _write_transcript(tmp_path: Path) -> Path: app = create_foundry_conformance_app() with TestClient(app) as client: @@ -53,7 +62,7 @@ def test_verify_brokered_transcript_accepts_conformance_loop(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) - summary = verifier.verify_transcript(transcript) + summary = _verify(verifier, transcript) assert summary["initial_response_id"].startswith("caresp_") assert summary["continuation_response_id"].startswith("caresp_") @@ -88,7 +97,7 @@ def test_verify_brokered_transcript_rejects_old_response_ids(tmp_path): (transcript / "02-initial-response.json").write_text(json.dumps(initial), encoding="utf-8") try: - verifier.verify_transcript(transcript) + _verify(verifier, transcript) except ValueError as exc: assert "caresp_" in str(exc) else: # pragma: no cover - assertion path. @@ -104,7 +113,7 @@ def test_verify_brokered_transcript_rejects_duplicate_keys_in_top_level_files(tm initial_path.write_text(encoded[:-1] + ',"status":"failed"}', encoding="utf-8") with pytest.raises(ValueError, match="duplicate JSON object key"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_rejects_mismatched_function_call_response_id(tmp_path): @@ -116,7 +125,7 @@ def test_verify_brokered_transcript_rejects_mismatched_function_call_response_id initial_path.write_text(json.dumps(initial), encoding="utf-8") with pytest.raises(ValueError, match="function_call response_id must match"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_rejects_mismatched_final_message_response_id(tmp_path): @@ -128,7 +137,7 @@ def test_verify_brokered_transcript_rejects_mismatched_final_message_response_id continuation_path.write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match="final message response_id must match"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) @pytest.mark.parametrize( @@ -149,9 +158,48 @@ def test_verify_brokered_transcript_rejects_noncompleted_assistant_message( continuation_path.write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match=message): + _verify(verifier, transcript) + + +def test_verify_brokered_transcript_requires_expected_final_text(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + + with pytest.raises(ValueError, match="expected final text is required"): verifier.verify_transcript(transcript) +def test_verify_brokered_transcript_rejects_incomplete_function_call_item(tmp_path): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + initial_path = transcript / "02-initial-response.json" + initial = json.loads(initial_path.read_text(encoding="utf-8")) + initial["output"][0]["status"] = "in_progress" + initial_path.write_text(json.dumps(initial), encoding="utf-8") + + with pytest.raises(ValueError, match="function_call status must be completed"): + _verify(verifier, transcript) + + +@pytest.mark.parametrize( + "content", + [ + [{"type": "output_text", "text": "ok"}, {"type": "output_text", "text": "extra"}], + [{"type": "input_text", "text": "wrong type"}], + ], +) +def test_verify_brokered_transcript_rejects_noncanonical_final_content(tmp_path, content: list[dict]): + verifier = _load_verifier() + transcript = _write_transcript(tmp_path) + continuation_path = transcript / "04-continuation-response.json" + continuation = json.loads(continuation_path.read_text(encoding="utf-8")) + continuation["output"][0]["content"] = content + continuation_path.write_text(json.dumps(continuation), encoding="utf-8") + + with pytest.raises(ValueError, match="exactly one item|must be output_text"): + _verify(verifier, transcript) + + def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_path): verifier = _load_verifier() transcript = _write_transcript(tmp_path) @@ -161,7 +209,7 @@ def test_verify_brokered_transcript_rejects_reused_continuation_response_id(tmp_ (transcript / "04-continuation-response.json").write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match="must differ from initial response id"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_rejects_extra_final_output_items(tmp_path): @@ -181,7 +229,7 @@ def test_verify_brokered_transcript_rejects_extra_final_output_items(tmp_path): (transcript / "04-continuation-response.json").write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match="must contain exactly one item"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_rejects_unexpected_final_text(tmp_path): @@ -189,7 +237,7 @@ def test_verify_brokered_transcript_rejects_unexpected_final_text(tmp_path): transcript = _write_transcript(tmp_path) with pytest.raises(ValueError, match="final message text did not match"): - verifier.verify_transcript(transcript, expected_final_text="unrelated response") + _verify(verifier, transcript, expected_final_text="unrelated response") def test_verify_brokered_transcript_rejects_unexpected_continuation_output(tmp_path): @@ -200,7 +248,7 @@ def test_verify_brokered_transcript_rejects_unexpected_continuation_output(tmp_p (transcript / "03-continuation-request.json").write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match="continuation output did not match expected JSON"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_compares_output_json_types_strictly(tmp_path): @@ -211,7 +259,7 @@ def test_verify_brokered_transcript_compares_output_json_types_strictly(tmp_path (transcript / "03-continuation-request.json").write_text(json.dumps(continuation), encoding="utf-8") with pytest.raises(ValueError, match="continuation output did not match expected JSON"): - verifier.verify_transcript(transcript) + _verify(verifier, transcript) def test_verify_brokered_transcript_json_comparison_distinguishes_booleans_but_normalizes_numbers(): @@ -241,7 +289,9 @@ def test_verify_brokered_transcript_cli_writes_summary(tmp_path, capsys): verifier = _load_verifier() transcript = _write_transcript(tmp_path) - assert verifier.main([str(transcript), "--write-summary"]) == 0 + assert verifier.main( + [str(transcript), "--write-summary", "--expected-final-text", _EXPECTED_SDK_FINAL_TEXT] + ) == 0 output = json.loads(capsys.readouterr().out) written = json.loads((transcript / "summary.json").read_text(encoding="utf-8")) @@ -305,6 +355,7 @@ def build_runtime(self, spec): # noqa: ANN001 summary = _load_verifier().verify_transcript( tmp_path, + expected_final_text=_EXPECTED_AGENTKIT_FINAL_TEXT, expected_call_id="auto", expected_call_id_prefix="call_", ) diff --git a/test/foundry-brokered-conformance/README.md b/test/foundry-brokered-conformance/README.md index 9683a4e..5b1938e 100644 --- a/test/foundry-brokered-conformance/README.md +++ b/test/foundry-brokered-conformance/README.md @@ -82,5 +82,5 @@ export AGENT_RESPONSES_ENDPOINT="https:///responses" deploy/foundry/scripts/foundry_brokered_conformance.sh conformance_read ./foundry-brokered-transcript ``` -The helper writes request/response JSON plus `summary.json`. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py ` to verify an archived transcript later. Keep bearer tokens +The helper writes request/response JSON plus `summary.json`. Re-run `python3 deploy/foundry/scripts/verify_brokered_transcript.py --expected-final-text ''` to verify an archived transcript later. Keep bearer tokens out of transcripts.