diff --git a/.fork-only b/.fork-only new file mode 100644 index 00000000000..8b9bbeb5a4f --- /dev/null +++ b/.fork-only @@ -0,0 +1,5 @@ +# Files that exist only on this fork and must not be pushed to upstream. +# Used by the global pre-push hook to block accidental inclusion in PRs. +.fork-only +.github/workflows/sync-upstream.yml +.github/workflows/mirror-to-enjimi.yml diff --git a/.github/scripts/nightly-alert-issue.sh b/.github/scripts/nightly-alert-issue.sh new file mode 100755 index 00000000000..38f93756b85 --- /dev/null +++ b/.github/scripts/nightly-alert-issue.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Create, update, or close a GitHub issue for a scheduled nightly workflow. +# Required env: +# GH_TOKEN, REPO, ALERT_WORKFLOW_NAME, ALERT_ISSUE_TITLE, ALERT_RESULT +# Optional env: +# ALERT_RUN_ID, ALERT_RUN_URL, ALERT_SHA, MAX_EXCERPT_LINES + +require_env() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + echo "Required environment variable ${name} is not set" >&2 + exit 2 + fi +} + +strip_ansi() { + # Keep this sed expression POSIX-ish for the GitHub ubuntu runner. + sed -E $'s/\x1B\[[0-9;?]*[ -/]*[@-~]//g' +} + +truncate_lines() { + local max_chars="$1" + awk -v max_chars="$max_chars" ' + length($0) > max_chars { print substr($0, 1, max_chars) "... [line truncated]"; next } + { print } + ' +} + +truncate_file_chars() { + local input_file="$1" + local output_file="$2" + local max_chars="$3" + local label="${4:-content}" + awk -v max_chars="$max_chars" -v label="$label" ' + BEGIN { used = 0 } + { + line = $0 "\n" + line_len = length(line) + if (used + line_len > max_chars) { + remaining = max_chars - used + if (remaining > 0) { + printf "%s", substr(line, 1, remaining) + } + printf "\n... [%s truncated to %s characters]\n", label, max_chars + exit + } + printf "%s", line + used += line_len + } + ' "$input_file" > "$output_file" +} + +extract_failure_excerpt() { + local log_file="$1" + local output_file="$2" + local max_lines="${3:-${MAX_EXCERPT_LINES:-220}}" + local max_line_chars="${MAX_EXCERPT_LINE_CHARS:-2000}" + local max_excerpt_chars="${MAX_EXCERPT_CHARS:-50000}" + local cleaned_log raw_excerpt line_limited_excerpt + cleaned_log="$(mktemp)" + raw_excerpt="$(mktemp)" + line_limited_excerpt="$(mktemp)" + + if [[ ! -s "$log_file" ]]; then + echo "No failed-job logs were available from GitHub Actions." > "$output_file" + return 0 + fi + + strip_ansi < "$log_file" > "$cleaned_log" + + # Prefer high-signal lines from pytest, Rust/cargo, GitHub Actions, runner + # infrastructure, and common timeout/disk failures. Keep the excerpt bounded so + # repeated nightly failures do not create unreadably large issue comments. + grep -nEi \ + '(^|[[:space:]])(FAILED|ERROR|FAILURES|failures:|test result: FAILED|panicked at|thread .+ panicked|No space left|timed out|timeout|Process completed with exit code|::error|Error:|Traceback|AssertionError|short test summary info|cargo (test|nextest|insta)|pytest|failed to|could not|cannot|killed|segmentation fault|signal:)' \ + "$cleaned_log" \ + | head -n "$max_lines" \ + > "$raw_excerpt" || true + + if [[ ! -s "$raw_excerpt" ]]; then + { + echo "No high-signal failure lines matched; showing the last 120 failed-log lines instead." + echo + tail -n 120 "$cleaned_log" + } > "$raw_excerpt" + fi + + truncate_lines "$max_line_chars" < "$raw_excerpt" > "$line_limited_excerpt" + truncate_file_chars "$line_limited_excerpt" "$output_file" "$max_excerpt_chars" "excerpt" + + rm -f "$cleaned_log" "$raw_excerpt" "$line_limited_excerpt" +} + +find_open_issue() { + local title="$1" + gh issue list \ + --repo "$REPO" \ + --state open \ + --search "${title} in:title" \ + --json number,title \ + --jq 'map(select(.title == env.ALERT_ISSUE_TITLE))[0].number // empty' +} + +# Collect PRs merged into the alert branch since the previous successful run of +# this workflow, so the failure issue can @-mention authors who landed in the +# blame window. All API calls are best-effort: if any step fails (no prior +# green run, missing GroOT pulls endpoint, transient API error) the function +# writes a one-line note to its output file and returns 0 so the rest of the +# alert body renders normally. +collect_merged_prs() { + local output_file="$1" + local branch="${ALERT_BRANCH:-${GITHUB_REF_NAME:-main}}" + local current_sha="${ALERT_SHA}" + local max_prs="${ALERT_MAX_MERGED_PRS:-30}" + + : > "$output_file" + + local last_good_sha + last_good_sha="$( + gh run list \ + --repo "$REPO" \ + --workflow "$ALERT_WORKFLOW_NAME" \ + --branch "$branch" \ + --status success \ + --limit 1 \ + --json headSha \ + --jq '.[0].headSha // empty' \ + 2>/dev/null + )" || true + + if [[ -z "$last_good_sha" || "$last_good_sha" == "$current_sha" ]]; then + echo "_No prior successful \`${ALERT_WORKFLOW_NAME}\` run on \`${branch}\` to attribute against._" > "$output_file" + return 0 + fi + + local commits + commits="$( + gh api "repos/${REPO}/compare/${last_good_sha}...${current_sha}" \ + --jq '.commits[].sha' \ + 2>/dev/null + )" || true + + if [[ -z "$commits" ]]; then + echo "_No new commits between \`${last_good_sha:0:7}\` (last green) and \`${current_sha:0:7}\` (this run)._" > "$output_file" + return 0 + fi + + local pr_rows + pr_rows="$(mktemp)" + while IFS= read -r sha; do + [[ -z "$sha" ]] && continue + gh api "repos/${REPO}/commits/${sha}/pulls" \ + --jq '.[] | select(.merged_at != null) | "\(.number)\t\(.user.login)\t\(.title)"' \ + 2>/dev/null \ + >> "$pr_rows" || true + done <<< "$commits" + + if [[ ! -s "$pr_rows" ]]; then + { + echo "Commits since last green at \`${last_good_sha:0:7}\` (no associated merged PRs found):" + echo + while IFS= read -r sha; do + [[ -z "$sha" ]] && continue + echo "- \`${sha:0:7}\`" + done <<< "$commits" + } > "$output_file" + rm -f "$pr_rows" + return 0 + fi + + local total_prs + total_prs="$(sort -k1,1n "$pr_rows" | awk -F'\t' '!seen[$1]++' | wc -l | tr -d ' ')" + { + echo "PRs merged into \`${branch}\` between \`${last_good_sha:0:7}\` (last green) and \`${current_sha:0:7}\` (this run):" + echo + sort -k1,1n "$pr_rows" \ + | awk -F'\t' '!seen[$1]++' \ + | head -n "$max_prs" \ + | while IFS=$'\t' read -r number author title; do + echo "- #${number} ${title} — @${author}" + done + if [[ "$total_prs" -gt "$max_prs" ]]; then + echo + echo "_Showing first ${max_prs} of ${total_prs} PRs in the window._" + fi + } > "$output_file" + rm -f "$pr_rows" +} + +write_failure_body() { + local body_file="$1" + local jobs_file="$2" + local excerpt_file="$3" + local log_error_file="$4" + local merged_prs_file="${5:-}" + + { + echo "❌ ${ALERT_WORKFLOW_NAME} scheduled run failed." + echo + echo "- Workflow: ${ALERT_WORKFLOW_NAME}" + echo "- Result: ${ALERT_RESULT}" + echo "- Run: ${ALERT_RUN_URL}" + echo "- Commit: ${ALERT_SHA}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT:-1}" + echo "- Reported at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo + echo "## Failed jobs" + if [[ -s "$jobs_file" ]]; then + cat "$jobs_file" + else + echo "No failed job metadata was available from the Actions API. Check the run link above." + fi + echo + if [[ -s "$log_error_file" ]]; then + echo "## Log retrieval notes" + echo + echo "The alert job could not retrieve full failed-job logs with \`gh run view --log-failed\`. The run link and failed job links above are still authoritative." + echo + echo '```text' + head -n 40 "$log_error_file" | sed 's/```/` ` `/g' + echo '```' + echo + fi + echo "## Failure excerpt" + echo + echo '```text' + sed 's/```/` ` `/g' "$excerpt_file" + echo '```' + echo + if [[ -n "$merged_prs_file" && -s "$merged_prs_file" ]]; then + echo "## Merged since last green" + echo + cat "$merged_prs_file" + echo + fi + echo "This issue is updated in place on repeated failures to avoid notification spam. If the next scheduled run passes, the nightly alert job will close this issue automatically." + } > "$body_file" +} + +write_recovery_body() { + local body_file="$1" + { + echo "✅ ${ALERT_WORKFLOW_NAME} recovered on the latest scheduled run." + echo + echo "- Run: ${ALERT_RUN_URL}" + echo "- Commit: ${ALERT_SHA}" + echo "- Reported at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + } > "$body_file" +} + +main() { + require_env GH_TOKEN + require_env REPO + require_env ALERT_WORKFLOW_NAME + require_env ALERT_ISSUE_TITLE + require_env ALERT_RESULT + + export ALERT_ISSUE_TITLE + + ALERT_RUN_ID="${ALERT_RUN_ID:-${GITHUB_RUN_ID:-}}" + ALERT_RUN_URL="${ALERT_RUN_URL:-${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-$REPO}/actions/runs/${ALERT_RUN_ID}}" + ALERT_SHA="${ALERT_SHA:-${GITHUB_SHA:-unknown}}" + + if [[ -z "$ALERT_RUN_ID" ]]; then + echo "ALERT_RUN_ID or GITHUB_RUN_ID is required" >&2 + exit 2 + fi + + local issue_number + issue_number="$(find_open_issue "$ALERT_ISSUE_TITLE")" + + if [[ "$ALERT_RESULT" == "success" ]]; then + if [[ -n "$issue_number" ]]; then + local recovery_body + recovery_body="$(mktemp)" + write_recovery_body "$recovery_body" + gh issue close "$issue_number" --repo "$REPO" --comment "$(cat "$recovery_body")" + rm -f "$recovery_body" + else + echo "${ALERT_WORKFLOW_NAME} succeeded and no open alert issue exists." + fi + exit 0 + fi + + local tmp_dir log_file log_error failed_jobs excerpt body merged_prs + tmp_dir="$(mktemp -d)" + log_file="${tmp_dir}/failed.log" + log_error="${tmp_dir}/failed-log-error.txt" + failed_jobs="${tmp_dir}/failed-jobs.md" + excerpt="${tmp_dir}/excerpt.txt" + merged_prs="${tmp_dir}/merged-prs.md" + body="${tmp_dir}/issue.md" + + if ! gh run view "$ALERT_RUN_ID" --repo "$REPO" --log-failed > "$log_file" 2> "$log_error"; then + echo "Warning: failed to retrieve failed-job logs for run ${ALERT_RUN_ID}" >&2 + fi + + gh api "repos/${REPO}/actions/runs/${ALERT_RUN_ID}/jobs?per_page=100" \ + --jq '.jobs[] | select(.status == "completed" and .conclusion != "success" and .conclusion != "skipped") | "- \(.name) (`\(.conclusion // "unknown")`): \(.html_url)"' \ + > "$failed_jobs" || true + + collect_merged_prs "$merged_prs" + + extract_failure_excerpt "$log_file" "$excerpt" "${MAX_EXCERPT_LINES:-220}" + write_failure_body "$body" "$failed_jobs" "$excerpt" "$log_error" "$merged_prs" + truncate_file_chars "$body" "${body}.bounded" "${MAX_ISSUE_BODY_CHARS:-60000}" "issue body" + mv "${body}.bounded" "$body" + + if [[ -n "$issue_number" ]]; then + gh issue edit "$issue_number" --repo "$REPO" --body-file "$body" + echo "Updated existing nightly alert issue #${issue_number}." + else + gh issue create --repo "$REPO" --title "$ALERT_ISSUE_TITLE" --body-file "$body" + fi + + rm -rf "$tmp_dir" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/.github/scripts/nightly-alert-issue.test.sh b/.github/scripts/nightly-alert-issue.test.sh new file mode 100755 index 00000000000..593c0b6bfed --- /dev/null +++ b/.github/scripts/nightly-alert-issue.test.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=nightly-alert-issue.sh +source "${SCRIPT_DIR}/nightly-alert-issue.sh" + +assert_contains() { + local file="$1" + local needle="$2" + if ! grep -Fq "$needle" "$file"; then + echo "Expected ${file} to contain: ${needle}" >&2 + echo "--- ${file}" >&2 + cat "$file" >&2 + exit 1 + fi +} + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +cat > "${workdir}/failed.log" <<'LOG' +Build step noise +E2E (features) Run E2E tests (features) FAILED tests/e2e/scenarios/test_tool_approval.py::test_approval_flow - AssertionError: expected approval dialog +Rust tests Run Tests thread 'runtime::manager::tests::stop_thread_works' panicked at crates/ironclaw_engine/src/runtime/manager.rs:123:5 +Coverage Generate coverage Error: No space left on device +Runner Complete job Process completed with exit code 101. +LOG + +extract_failure_excerpt "${workdir}/failed.log" "${workdir}/excerpt.txt" 20 +assert_contains "${workdir}/excerpt.txt" "FAILED tests/e2e/scenarios/test_tool_approval.py::test_approval_flow" +assert_contains "${workdir}/excerpt.txt" "panicked at crates/ironclaw_engine/src/runtime/manager.rs" +assert_contains "${workdir}/excerpt.txt" "No space left on device" +assert_contains "${workdir}/excerpt.txt" "Process completed with exit code 101" + +cat > "${workdir}/quiet.log" <<'LOG' +ordinary line one +ordinary line two +LOG + +extract_failure_excerpt "${workdir}/quiet.log" "${workdir}/fallback.txt" 20 +assert_contains "${workdir}/fallback.txt" "No high-signal failure lines matched" +assert_contains "${workdir}/fallback.txt" "ordinary line two" + +python3 - <<'PY' "${workdir}/long.log" +from pathlib import Path +import sys +Path(sys.argv[1]).write_text('ERROR: ' + ('x' * 70000) + '\n') +PY +MAX_EXCERPT_LINE_CHARS=120 MAX_EXCERPT_CHARS=300 extract_failure_excerpt "${workdir}/long.log" "${workdir}/long-excerpt.txt" 20 +assert_contains "${workdir}/long-excerpt.txt" "[line truncated]" +long_excerpt_bytes="$(wc -c < "${workdir}/long-excerpt.txt" | tr -d ' ')" +if [[ "${long_excerpt_bytes}" -gt 360 ]]; then + echo "Expected long excerpt to stay bounded, got ${long_excerpt_bytes} bytes" >&2 + cat "${workdir}/long-excerpt.txt" >&2 + exit 1 +fi + +cat > "${workdir}/many.log" <<'LOG' +ERROR: first failure line with enough content to keep +ERROR: second failure line with enough content to keep +ERROR: third failure line with enough content to keep +ERROR: fourth failure line with enough content to keep +ERROR: fifth failure line with enough content to keep +LOG +MAX_EXCERPT_LINE_CHARS=1000 MAX_EXCERPT_CHARS=120 extract_failure_excerpt "${workdir}/many.log" "${workdir}/many-excerpt.txt" 20 +assert_contains "${workdir}/many-excerpt.txt" "[excerpt truncated to 120 characters]" +many_excerpt_bytes="$(wc -c < "${workdir}/many-excerpt.txt" | tr -d ' ')" +if [[ "${many_excerpt_bytes}" -gt 180 ]]; then + echo "Expected many-line excerpt to stay bounded, got ${many_excerpt_bytes} bytes" >&2 + cat "${workdir}/many-excerpt.txt" >&2 + exit 1 +fi + +cat > "${workdir}/jobs.md" <<'JOBS' +- E2E (features) (`failure`): https://github.example/jobs/1 +JOBS +cat > "${workdir}/log-error.txt" <<'ERR' +HTTP 404: logs are not ready yet +ERR + +ALERT_WORKFLOW_NAME="Nightly E2E" \ +ALERT_RESULT="failure" \ +ALERT_RUN_URL="https://github.example/runs/1" \ +ALERT_SHA="abc123" \ +write_failure_body "${workdir}/body.md" "${workdir}/jobs.md" "${workdir}/excerpt.txt" "${workdir}/log-error.txt" +assert_contains "${workdir}/body.md" "Log retrieval notes" +assert_contains "${workdir}/body.md" "HTTP 404: logs are not ready yet" +assert_contains "${workdir}/body.md" "updated in place on repeated failures" +truncate_file_chars "${workdir}/body.md" "${workdir}/body-bounded.md" 240 "issue body" +assert_contains "${workdir}/body-bounded.md" "[issue body truncated to 240 characters]" + +# write_failure_body renders the "Merged since last green" section when given +# a non-empty merged-PRs file, and omits it cleanly when not. +cat > "${workdir}/merged-prs.md" <<'PRS' +- #4242 fix(engine): tighten thread shutdown — @alice +- #4243 chore(deps): bump tokio — @bob +PRS +ALERT_WORKFLOW_NAME="Nightly E2E" \ +ALERT_RESULT="failure" \ +ALERT_RUN_URL="https://github.example/runs/1" \ +ALERT_SHA="abc123" \ +write_failure_body "${workdir}/body-with-prs.md" "${workdir}/jobs.md" "${workdir}/excerpt.txt" "${workdir}/log-error.txt" "${workdir}/merged-prs.md" +assert_contains "${workdir}/body-with-prs.md" "## Merged since last green" +assert_contains "${workdir}/body-with-prs.md" "@alice" +assert_contains "${workdir}/body-with-prs.md" "#4243" +if grep -Fq "## Merged since last green" "${workdir}/body.md"; then + echo "Body without merged-prs file must not render the merged-PRs section" >&2 + cat "${workdir}/body.md" >&2 + exit 1 +fi + +mkdir -p "${workdir}/bin" +cat > "${workdir}/bin/gh" <<'GH' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "${FAKE_GH_CALLS}" +case "${1:-} ${2:-}" in + "issue list") + if [[ "${FAKE_EXISTING_ISSUE:-}" == "1" ]]; then + echo "42" + fi + ;; + "run view") + echo "FAILED tests/e2e/scenarios/test_chat.py::test_chat - AssertionError" + ;; + "run list") + if [[ "${FAKE_LAST_GREEN_SHA:-}" != "" ]]; then + echo "${FAKE_LAST_GREEN_SHA}" + fi + ;; + "api repos/example/repo/actions/runs/99/jobs?per_page=100") + echo "- E2E (core) (\`failure\`): https://github.example/jobs/1" + ;; + "api repos/example/repo/compare/deadbee...abc123") + echo "sha111" + echo "sha222" + ;; + "api repos/example/repo/commits/sha111/pulls") + printf '%s\t%s\t%s\n' "4242" "alice" "fix(engine): tighten thread shutdown" + ;; + "api repos/example/repo/commits/sha222/pulls") + printf '%s\t%s\t%s\n' "4243" "bob" "chore(deps): bump tokio" + # Same PR appearing on multiple commits must dedupe. + printf '%s\t%s\t%s\n' "4242" "alice" "fix(engine): tighten thread shutdown" + ;; + "issue edit"|"issue create"|"issue close") + ;; + *) + echo "unexpected gh call: $*" >&2 + exit 1 + ;; +esac +GH +chmod +x "${workdir}/bin/gh" + +run_alert_script() { + local result="$1" + PATH="${workdir}/bin:${PATH}" \ + FAKE_GH_CALLS="${workdir}/gh-calls.txt" \ + FAKE_EXISTING_ISSUE="${FAKE_EXISTING_ISSUE:-}" \ + FAKE_LAST_GREEN_SHA="${FAKE_LAST_GREEN_SHA:-}" \ + GH_TOKEN="token" \ + REPO="example/repo" \ + GITHUB_RUN_ID="99" \ + GITHUB_RUN_ATTEMPT="1" \ + GITHUB_SERVER_URL="https://github.example" \ + GITHUB_REPOSITORY="example/repo" \ + GITHUB_SHA="abc123" \ + ALERT_BRANCH="main" \ + ALERT_WORKFLOW_NAME="Nightly E2E" \ + ALERT_ISSUE_TITLE="Nightly E2E failed" \ + ALERT_RESULT="$result" \ + "${SCRIPT_DIR}/nightly-alert-issue.sh" +} + +: > "${workdir}/gh-calls.txt" +FAKE_EXISTING_ISSUE="1" FAKE_LAST_GREEN_SHA="deadbee" run_alert_script "failure" +assert_contains "${workdir}/gh-calls.txt" "issue edit 42" +assert_contains "${workdir}/gh-calls.txt" "compare/deadbee...abc123" +assert_contains "${workdir}/gh-calls.txt" "commits/sha111/pulls" +if grep -Fq "issue comment" "${workdir}/gh-calls.txt"; then + echo "Repeated failures must update the issue body, not add comment spam" >&2 + cat "${workdir}/gh-calls.txt" >&2 + exit 1 +fi + +# When no prior green run is recorded the script must still post the alert, +# but with the no-baseline note instead of crashing. +: > "${workdir}/gh-calls.txt" +FAKE_EXISTING_ISSUE="1" FAKE_LAST_GREEN_SHA="" run_alert_script "failure" +assert_contains "${workdir}/gh-calls.txt" "issue edit 42" +if grep -Fq "compare/" "${workdir}/gh-calls.txt"; then + echo "Compare API must not be called when no prior green run is found" >&2 + cat "${workdir}/gh-calls.txt" >&2 + exit 1 +fi + +# Direct unit test for collect_merged_prs: happy path with two commits and a +# duplicate PR row across commits — output must dedupe and tag both authors. +ALERT_WORKFLOW_NAME="Nightly E2E" \ +REPO="example/repo" \ +ALERT_BRANCH="main" \ +ALERT_SHA="abc123" \ +PATH="${workdir}/bin:${PATH}" \ +FAKE_GH_CALLS="${workdir}/collect-calls.txt" \ +FAKE_LAST_GREEN_SHA="deadbee" \ +collect_merged_prs "${workdir}/merged-prs-collected.md" +assert_contains "${workdir}/merged-prs-collected.md" "between \`deadbee\` (last green) and \`abc123\` (this run)" +assert_contains "${workdir}/merged-prs-collected.md" "#4242" +assert_contains "${workdir}/merged-prs-collected.md" "@alice" +assert_contains "${workdir}/merged-prs-collected.md" "#4243" +assert_contains "${workdir}/merged-prs-collected.md" "@bob" +duplicate_alice_count="$(grep -Fc "@alice" "${workdir}/merged-prs-collected.md")" +if [[ "${duplicate_alice_count}" -ne 1 ]]; then + echo "Expected @alice to appear exactly once after dedupe, got ${duplicate_alice_count}" >&2 + cat "${workdir}/merged-prs-collected.md" >&2 + exit 1 +fi + +# collect_merged_prs with no prior green run produces a graceful note. +ALERT_WORKFLOW_NAME="Nightly E2E" \ +REPO="example/repo" \ +ALERT_BRANCH="main" \ +ALERT_SHA="abc123" \ +PATH="${workdir}/bin:${PATH}" \ +FAKE_GH_CALLS="${workdir}/collect-calls-empty.txt" \ +FAKE_LAST_GREEN_SHA="" \ +collect_merged_prs "${workdir}/merged-prs-empty.md" +assert_contains "${workdir}/merged-prs-empty.md" "No prior successful" + +: > "${workdir}/gh-calls.txt" +FAKE_EXISTING_ISSUE="1" run_alert_script "success" +assert_contains "${workdir}/gh-calls.txt" "issue close 42" +if grep -Fq "issue comment" "${workdir}/gh-calls.txt"; then + echo "Recovery should close with one close comment, not a separate comment" >&2 + cat "${workdir}/gh-calls.txt" >&2 + exit 1 +fi + +echo "nightly-alert-issue tests passed" diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 5946d3911a1..8e80289c124 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -70,7 +70,7 @@ jobs: FULL='[{"name":"all-features","flags":"--all-features"},{"name":"default","flags":""},{"name":"libsql-only","flags":"--no-default-features --features libsql"}]' SLIM='[{"name":"all-features","flags":"--all-features"}]' - if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "merge_group" ]; then echo "matrix=${SLIM}" >> "$GITHUB_OUTPUT" else echo "matrix=${FULL}" >> "$GITHUB_OUTPUT" @@ -153,7 +153,7 @@ jobs: clippy-windows: name: Clippy Windows (${{ matrix.name }}) needs: [changes, clippy-matrix] - if: needs.changes.outputs.has_code == 'true' && github.event_name != 'pull_request' + if: needs.changes.outputs.has_code == 'true' && github.event_name == 'push' runs-on: windows-latest strategy: fail-fast: false diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3deab5ff394..426bcdb5f35 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -38,6 +38,9 @@ jobs: coverage: name: Coverage (${{ matrix.name }}) runs-on: ubuntu-latest + env: + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_TEST_DEBUG: 0 permissions: id-token: write contents: read diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 04eae26e028..1f750a9e5f7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,6 +27,9 @@ on: required: false type: string default: "" + # Rolling :staging image tags from main branch (hourly) + schedule: + - cron: '0 * * * *' env: IMAGE_NAME: nearaidev/ironclaw @@ -44,6 +47,7 @@ jobs: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: + ref: ${{ github.event_name == 'schedule' && 'main' || '' }} persist-credentials: false - name: Resolve source git commit @@ -64,28 +68,20 @@ jobs: echo "version=${VERSION}" >> "$GITHUB_OUTPUT" echo "Detected version: ${VERSION}" - - name: Validate tag override - if: inputs.tag != '' - env: - INPUT_TAG: ${{ inputs.tag }} - run: | - if [[ ! "${INPUT_TAG}" =~ ^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$ ]]; then - echo "::error::Input tag '${INPUT_TAG}' does not match Docker tag grammar" - exit 1 - fi - if [[ "${INPUT_TAG}" == "staging" ]]; then - echo "::error::The staging tag has been retired. Use a different tag or rely on release tags." - exit 1 - fi - - name: Determine tags id: tags env: VERSION: ${{ steps.version.outputs.version }} + EVENT_NAME: ${{ github.event_name }} IS_RELEASE_BUILD: ${{ inputs.release && 'true' || 'false' }} INPUT_TAG: ${{ inputs.tag }} SOURCE_SHA: ${{ steps.source_sha.outputs.sha }} run: | + if [[ -n "${INPUT_TAG}" && ! "${INPUT_TAG}" =~ ^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$ ]]; then + echo "::error::Input tag '${INPUT_TAG}' does not match Docker tag grammar" + exit 1 + fi + SHA="sha-${SOURCE_SHA::7}" echo "sha_tag=${SHA}" >> "$GITHUB_OUTPUT" @@ -97,20 +93,32 @@ jobs: WORKER_TAGS="${WORKER_IMAGE_NAME}:${VERSION}" WORKER_TAGS="${WORKER_TAGS},${WORKER_IMAGE_NAME}:latest" WORKER_TAGS="${WORKER_TAGS},${WORKER_IMAGE_NAME}:${SHA}" + elif [[ "${EVENT_NAME}" == "schedule" ]]; then + # Hourly staging: :staging + :sha-xxx + TAGS="${IMAGE_NAME}:staging" + TAGS="${TAGS},${IMAGE_NAME}:${SHA}" + WORKER_TAGS="${WORKER_IMAGE_NAME}:staging" + WORKER_TAGS="${WORKER_TAGS},${WORKER_IMAGE_NAME}:${SHA}" else # Manual dispatch: :sha-xxx only TAGS="${IMAGE_NAME}:${SHA}" WORKER_TAGS="${WORKER_IMAGE_NAME}:${SHA}" fi + # Manual override adds an extra tag (e.g. "staging") if [[ -n "${INPUT_TAG}" ]]; then TAGS="${TAGS},${IMAGE_NAME}:${INPUT_TAG}" WORKER_TAGS="${WORKER_TAGS},${WORKER_IMAGE_NAME}:${INPUT_TAG}" fi - echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" echo "worker_tags=${WORKER_TAGS}" >> "$GITHUB_OUTPUT" - echo "target=runtime" >> "$GITHUB_OUTPUT" + + # Staging builds get pre-bundled WASM extensions + if [[ "${EVENT_NAME}" == "schedule" || "${INPUT_TAG}" == "staging" ]]; then + echo "target=runtime-staging" >> "$GITHUB_OUTPUT" + else + echo "target=runtime" >> "$GITHUB_OUTPUT" + fi - name: Log in to Docker Hub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 @@ -118,10 +126,40 @@ jobs: username: ${{ vars.DOCKER_REGISTRY_USER }} password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + - name: Check if current git commit is already built + id: check + if: steps.tags.outputs.target == 'runtime-staging' + env: + SOURCE_SHA: ${{ steps.source_sha.outputs.sha }} + run: | + CURRENT_BUILT_SHA="" + WORKER_BUILT_SHA="" + + if docker pull "${IMAGE_NAME}:staging" > /dev/null 2>&1; then + CURRENT_BUILT_SHA=$(docker inspect --format='{{index .Config.Labels "ironclaw.git.sha"}}' "${IMAGE_NAME}:staging" 2>/dev/null || echo "") + fi + if docker pull "${WORKER_IMAGE_NAME}:staging" > /dev/null 2>&1; then + WORKER_BUILT_SHA=$(docker inspect --format='{{index .Config.Labels "ironclaw.git.sha"}}' "${WORKER_IMAGE_NAME}:staging" 2>/dev/null || echo "") + fi + + echo "Current built commit (ironclaw): ${CURRENT_BUILT_SHA:-}" + echo "Current built commit (ironclaw-worker): ${WORKER_BUILT_SHA:-}" + echo "Current source commit: ${SOURCE_SHA}" + + if [[ -n "${CURRENT_BUILT_SHA}" && "${CURRENT_BUILT_SHA}" == "${SOURCE_SHA}" && -n "${WORKER_BUILT_SHA}" && "${WORKER_BUILT_SHA}" == "${SOURCE_SHA}" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "Current commit already built for both images — skipping." + else + echo "skip=false" >> "$GITHUB_OUTPUT" + echo "At least one image is missing or out of date — proceeding." + fi + - name: Set up Docker Buildx + if: steps.check.outputs.skip != 'true' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build and push (ironclaw) + if: steps.check.outputs.skip != 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . @@ -135,6 +173,7 @@ jobs: cache-to: type=gha,mode=max - name: Build and push (ironclaw-worker) + if: steps.check.outputs.skip != 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . @@ -149,6 +188,7 @@ jobs: - name: Create releases-manager app token id: app-token + if: steps.check.outputs.skip != 'true' continue-on-error: true uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 with: @@ -158,14 +198,13 @@ jobs: repositories: ironclaw-dind - name: Trigger ironclaw-dind Build & Push - if: > - steps.app-token.outcome == 'success' && - github.event_name == 'workflow_call' && - steps.version.outputs.version != '' + if: steps.app-token.outcome == 'success' && steps.check.outputs.skip != 'true' continue-on-error: true env: GH_TOKEN: ${{ steps.app-token.outputs.token }} + EVENT_NAME: ${{ github.event_name }} IS_RELEASE_BUILD: ${{ inputs.release && 'true' || 'false' }} + INPUT_TAG: ${{ inputs.tag }} VERSION: ${{ steps.version.outputs.version }} run: | if [[ "${IS_RELEASE_BUILD}" == "true" && -n "${VERSION}" ]]; then @@ -173,23 +212,45 @@ jobs: --method POST \ -f event_type="ironclaw_image_published" \ -f client_payload[version]="${VERSION}" + elif [[ "${EVENT_NAME}" == "schedule" ]] || [[ "${INPUT_TAG}" == "staging" ]]; then + gh api repos/nearai/ironclaw-dind/dispatches \ + --method POST \ + -f event_type="ironclaw_image_published" fi - name: Summary + if: steps.check.outputs.skip != 'true' + env: + TAGS: ${{ steps.tags.outputs.tags }} + WORKER_TAGS: ${{ steps.tags.outputs.worker_tags }} + VERSION: ${{ steps.version.outputs.version }} + SOURCE_SHA: ${{ steps.source_sha.outputs.sha }} run: | { echo "## Docker Images" echo "" echo "**ironclaw:**" echo '```' - echo "${{ steps.tags.outputs.tags }}" | tr ',' '\n' + echo "${TAGS}" | tr ',' '\n' echo '```' echo "" echo "**ironclaw-worker:**" echo '```' - echo "${{ steps.tags.outputs.worker_tags }}" | tr ',' '\n' + echo "${WORKER_TAGS}" | tr ',' '\n' echo '```' echo "" - echo "- version: \`${{ steps.version.outputs.version }}\`" - echo "- sha: \`${{ steps.source_sha.outputs.sha }}\`" + echo "- version: \`${VERSION}\`" + echo "- sha: \`${SOURCE_SHA}\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Summary (skipped) + if: steps.check.outputs.skip == 'true' + env: + SOURCE_SHA: ${{ steps.source_sha.outputs.sha }} + run: | + { + echo "## Docker Images — skipped" + echo "" + echo "Current commit already built for \`${IMAGE_NAME}:staging\`." + echo "- sha: \`${SOURCE_SHA}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 967e33cc27b..2f8a9ebd1d2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -11,8 +11,6 @@ on: required: false type: string default: full - schedule: - - cron: "0 6 * * 1" # Weekly Monday 6 AM UTC workflow_dispatch: permissions: @@ -32,7 +30,7 @@ jobs: - id: set run: | SMOKE='[{"group":"core","files":"tests/e2e/scenarios/test_connection.py tests/e2e/scenarios/test_chat.py tests/e2e/scenarios/test_sse_reconnect.py tests/e2e/scenarios/test_html_injection.py tests/e2e/scenarios/test_csp.py"}]' - FULL='[{"group":"core","files":"tests/e2e/scenarios/test_connection.py tests/e2e/scenarios/test_chat.py tests/e2e/scenarios/test_sse_reconnect.py tests/e2e/scenarios/test_html_injection.py tests/e2e/scenarios/test_csp.py"},{"group":"features","files":"tests/e2e/scenarios/test_skills.py tests/e2e/scenarios/test_tool_approval.py tests/e2e/scenarios/test_webhook.py"},{"group":"extensions","files":"tests/e2e/scenarios/test_extensions.py tests/e2e/scenarios/test_extension_oauth.py tests/e2e/scenarios/test_oauth_url_parameters.py tests/e2e/scenarios/test_telegram_token_validation.py tests/e2e/scenarios/test_telegram_hot_activation.py tests/e2e/scenarios/test_wasm_lifecycle.py tests/e2e/scenarios/test_tool_execution.py tests/e2e/scenarios/test_agent_loop_recovery.py tests/e2e/scenarios/test_pairing.py tests/e2e/scenarios/test_mcp_auth_flow.py tests/e2e/scenarios/test_oauth_credential_fallback.py tests/e2e/scenarios/test_routine_oauth_credential_injection.py"},{"group":"routines","files":"tests/e2e/scenarios/test_owner_scope.py tests/e2e/scenarios/test_routine_event_batch.py"},{"group":"web-regressions","files":"tests/e2e/scenarios/test_auth_no_duplicate_response.py tests/e2e/scenarios/test_message_persistence.py tests/e2e/scenarios/test_pending_user_messages.py"},{"group":"v2-engine","files":"tests/e2e/scenarios/test_v2_engine_auth_flow.py tests/e2e/scenarios/test_v2_engine_approval_flow.py tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py tests/e2e/scenarios/test_v2_tool_activate_surface.py tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py tests/e2e/scenarios/test_v2_thread_visibility.py"}]' + FULL='[{"group":"core","files":"tests/e2e/scenarios/test_connection.py tests/e2e/scenarios/test_chat.py tests/e2e/scenarios/test_sse_reconnect.py tests/e2e/scenarios/test_html_injection.py tests/e2e/scenarios/test_csp.py"},{"group":"features","files":"tests/e2e/scenarios/test_skills.py tests/e2e/scenarios/test_tool_approval.py tests/e2e/scenarios/test_webhook.py"},{"group":"extensions","files":"tests/e2e/scenarios/test_extensions.py tests/e2e/scenarios/test_extension_oauth.py tests/e2e/scenarios/test_oauth_url_parameters.py tests/e2e/scenarios/test_telegram_token_validation.py tests/e2e/scenarios/test_telegram_hot_activation.py tests/e2e/scenarios/test_wasm_lifecycle.py tests/e2e/scenarios/test_tool_execution.py tests/e2e/scenarios/test_agent_loop_recovery.py tests/e2e/scenarios/test_pairing.py tests/e2e/scenarios/test_mcp_auth_flow.py tests/e2e/scenarios/test_oauth_credential_fallback.py tests/e2e/scenarios/test_routine_oauth_credential_injection.py tests/e2e/scenarios/test_telegram_pairing_chat_claim.py"},{"group":"routines","files":"tests/e2e/scenarios/test_owner_scope.py tests/e2e/scenarios/test_routine_event_batch.py"},{"group":"web-regressions","files":"tests/e2e/scenarios/test_auth_no_duplicate_response.py tests/e2e/scenarios/test_message_persistence.py tests/e2e/scenarios/test_pending_user_messages.py"},{"group":"v2-engine","files":"tests/e2e/scenarios/test_v2_engine_auth_flow.py tests/e2e/scenarios/test_v2_engine_approval_flow.py tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py tests/e2e/scenarios/test_v2_tool_activate_surface.py tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py tests/e2e/scenarios/test_v2_thread_visibility.py"}]' if [ "${{ inputs.mode || 'full' }}" = "smoke" ]; then echo "matrix=${SMOKE}" >> "$GITHUB_OUTPUT" @@ -74,7 +72,7 @@ jobs: name: E2E (${{ matrix.group }}) needs: [matrix-config, build] runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -126,3 +124,4 @@ jobs: echo "One or more E2E jobs failed" exit 1 fi + diff --git a/.github/workflows/live-canary.yml b/.github/workflows/live-canary.yml index e83a9793a0f..bc6d0f77c23 100644 --- a/.github/workflows/live-canary.yml +++ b/.github/workflows/live-canary.yml @@ -504,6 +504,19 @@ jobs: LLM_BACKEND: anthropic ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_MODEL: ${{ vars.LIVE_ANTHROPIC_MODEL || 'claude-sonnet-4-6' }} + # Real third-party credentials forwarded into the persona harness. + # The harness reads these env vars per + # tests/e2e_live_personas.rs::PERSONA_CREDENTIALS — present → + # real API call lands; absent → dummy fallback satisfies the + # auth pre-flight and the agent recovers via workspace tools. + # Add each secret in repo settings (Settings → Secrets and + # variables → Actions) if you want the lane to exercise that + # integration end-to-end. + LIVE_CANARY_GITHUB_TOKEN: ${{ secrets.LIVE_CANARY_GITHUB_TOKEN }} + LIVE_CANARY_GOOGLE_OAUTH_TOKEN: ${{ secrets.LIVE_CANARY_GOOGLE_OAUTH_TOKEN }} + LIVE_CANARY_SLACK_BOT_TOKEN: ${{ secrets.LIVE_CANARY_SLACK_BOT_TOKEN }} + LIVE_CANARY_TELEGRAM_BOT_TOKEN: ${{ secrets.LIVE_CANARY_TELEGRAM_BOT_TOKEN }} + LIVE_CANARY_COMPOSIO_API_KEY: ${{ secrets.LIVE_CANARY_COMPOSIO_API_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -550,6 +563,21 @@ jobs: COMMAND_TIMEOUT: 60m STRICT_ARTIFACT_SCRUB: "true" steps: + - name: Preflight disk space + # The only self-hosted runner the canary uses. Nothing in this + # workflow cleans ~/.cargo, ~/.rustup, or runner _diag between + # jobs, so a wedged disk shows up as opaque mid-build failures + # we can't see (test-output.log isn't uploaded). Fail loudly + # with a clear marker instead — at least 10 GiB free on /. + run: | + df -h / + avail_kb=$(df --output=avail / | tail -1 | tr -d ' ') + min_kb=$((10 * 1024 * 1024)) + if [ "$avail_kb" -lt "$min_kb" ]; then + echo "::error::Self-hosted runner low on disk: ${avail_kb} KiB available, need ${min_kb} KiB (~10 GiB)." + exit 1 + fi + echo "Disk OK: ${avail_kb} KiB available on /." - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false @@ -570,11 +598,16 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: live-canary-private-oauth-summary + # results.json is emitted by scripts/live-canary/emit_results_json.py + # and contains only test names + outcomes + bounded panic messages + # — no URLs, tokens, or payloads — so it is scrub-safe to upload + # even under STRICT_ARTIFACT_SCRUB=true. path: | artifacts/live-canary/**/summary.md artifacts/live-canary/**/env-summary.txt artifacts/live-canary/**/trace-fixture-status.txt artifacts/live-canary/**/scrub-matches.txt + artifacts/live-canary/**/results.json retention-days: 7 provider-matrix: diff --git a/.github/workflows/mirror-to-enjimi.yml b/.github/workflows/mirror-to-enjimi.yml new file mode 100644 index 00000000000..5cc61a1c8e7 --- /dev/null +++ b/.github/workflows/mirror-to-enjimi.yml @@ -0,0 +1,23 @@ +name: Mirror to enjimi + +on: + push: + branches: + - native-matrix-channel-pilot + +jobs: + mirror: + if: github.repository == 'theredspoon/ironclaw' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Push to enjimi + env: + ENJIMI_PAT: ${{ secrets.ENJIMI_PAT }} + run: | + git remote add enjimi https://x-access-token:${ENJIMI_PAT}@github.com/enjimi/ironclaw.git + git push enjimi HEAD:refs/heads/main diff --git a/.github/workflows/nearai-bench-tests.yml b/.github/workflows/nearai-bench-tests.yml new file mode 100644 index 00000000000..9bfcc93abc8 --- /dev/null +++ b/.github/workflows/nearai-bench-tests.yml @@ -0,0 +1,154 @@ +# Regression coverage for the /benchmark slash-command dispatcher. +# +# The dispatcher in nearai-bench.yml parses comment bodies with a sed +# regex and gates work on a whole-word `startsWith` guard. Both surfaces +# gate paid LLM CI — silent breakage in either would let unintended +# bodies through (or block valid ones). This workflow runs the same +# regex + guard logic against fixture inputs so a change to the +# dispatcher's parsing logic can't ship without surfacing the diff. +# +# Filed in response to serrrfirat's Medium/tests finding on #3808. + +name: nearai-bench dispatcher tests + +on: + push: + branches: [main] + paths: + - '.github/workflows/nearai-bench.yml' + - '.github/workflows/nearai-bench-tests.yml' + pull_request: + paths: + - '.github/workflows/nearai-bench.yml' + - '.github/workflows/nearai-bench-tests.yml' + +permissions: + contents: read + +jobs: + parser: + name: parser regex + guard fixtures + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Run dispatcher parser fixtures + run: | + set -euo pipefail + + # ─── Parser under test ─────────────────────────────────────── + # Keep in sync with nearai-bench.yml's `parse` step. Both + # surfaces (this test + the dispatcher) live in the same path + # filter so a change to one without the other fails CI. + parse_suite() { + printf '%s' "$1" | sed -nE 's@^/benchmark[[:space:]]+([A-Za-z0-9_-]+)([[:space:]]+--model[[:space:]]+[A-Za-z0-9_./:-]+)?[[:space:]]*$@\1@p' + } + parse_model() { + printf '%s' "$1" | sed -nE 's@^/benchmark[[:space:]]+[A-Za-z0-9_-]+[[:space:]]+--model[[:space:]]+([A-Za-z0-9_./:-]+)[[:space:]]*$@\1@p' + } + + # Whole-word guard from the workflow-level `if:` clause. + # GitHub Actions: startsWith(body, '/benchmark ') || body == '/benchmark' + guard_match() { + [[ "$1" == "/benchmark "* ]] || [[ "$1" == "/benchmark" ]] + } + + fail=0 + + assert_parse() { + local label="$1" body="$2" want_suite="$3" want_model="$4" + local got_suite got_model + got_suite=$(parse_suite "$body") + got_model=$(parse_model "$body") + if [ "$got_suite" = "$want_suite" ] && [ "$got_model" = "$want_model" ]; then + printf ' PASS %s\n' "$label" + else + printf ' FAIL %s\n' "$label" + printf ' body: %q\n' "$body" + printf ' want: suite=%q model=%q\n' "$want_suite" "$want_model" + printf ' got: suite=%q model=%q\n' "$got_suite" "$got_model" + fail=1 + fi + } + + assert_guard() { + local label="$1" body="$2" want="$3" # "match" | "skip" + local got="skip" + if guard_match "$body"; then got="match"; fi + if [ "$got" = "$want" ]; then + printf ' PASS %s\n' "$label" + else + printf ' FAIL %s\n' "$label" + printf ' body: %q\n' "$body" + printf ' want %s, got %s\n' "$want" "$got" + fail=1 + fi + } + + # ─── Parser fixtures ───────────────────────────────────────── + echo "=== parser regex ===" + # Valid: authorized + parseable + assert_parse 'smoke suite, no model' \ + '/benchmark ironclaw-smoke' \ + 'ironclaw-smoke' '' + assert_parse 'full suite, no model' \ + '/benchmark ironclaw' \ + 'ironclaw' '' + assert_parse 'smoke + lowercase OpenRouter model' \ + '/benchmark ironclaw-smoke --model qwen/qwen3.5-35b-a3b' \ + 'ironclaw-smoke' 'qwen/qwen3.5-35b-a3b' + assert_parse 'smoke + mixed-case HF-style model' \ + '/benchmark ironclaw-smoke --model Qwen/Qwen3.5-122B-A10B' \ + 'ironclaw-smoke' 'Qwen/Qwen3.5-122B-A10B' + assert_parse 'three-segment model id' \ + '/benchmark ironclaw --model openrouter/anthropic/claude-sonnet-4' \ + 'ironclaw' 'openrouter/anthropic/claude-sonnet-4' + assert_parse 'trailing whitespace ok' \ + '/benchmark ironclaw-smoke ' \ + 'ironclaw-smoke' '' + + # Invalid: parser must reject (both outputs empty) + assert_parse 'empty command' \ + '/benchmark' '' '' + assert_parse 'extra positional token' \ + '/benchmark ironclaw-smoke garbage' '' '' + assert_parse 'two positional tokens' \ + '/benchmark foo bar' '' '' + assert_parse '--model flag without value' \ + '/benchmark ironclaw-smoke --model' '' '' + assert_parse 'unrelated comment' \ + 'looks good to me' '' '' + assert_parse 'starts with /benchmark but in a quote' \ + '> /benchmark ironclaw-smoke' '' '' + + # ─── Guard fixtures ────────────────────────────────────────── + echo + echo "=== whole-word startsWith guard ===" + assert_guard 'bare command' \ + '/benchmark' 'match' + assert_guard 'command with suite' \ + '/benchmark ironclaw-smoke' 'match' + assert_guard 'command with suite + model' \ + '/benchmark ironclaw-smoke --model qwen/qwen3.5-35b-a3b' 'match' + + assert_guard 'prefix collision: /benchmarking-something' \ + '/benchmarking-something' 'skip' + assert_guard 'prefix collision: /benchmark-foo' \ + '/benchmark-foo' 'skip' + assert_guard 'prefix collision: /benchmarkfoo' \ + '/benchmarkfoo' 'skip' + assert_guard 'unrelated comment' \ + 'looks good to me' 'skip' + assert_guard 'markdown-quoted command' \ + '> /benchmark ironclaw-smoke' 'skip' + + echo + if [ "$fail" = "1" ]; then + echo "::error::dispatcher parser regression — see failures above" + exit 1 + fi + echo "all dispatcher parser fixtures passed" diff --git a/.github/workflows/nearai-bench.yml b/.github/workflows/nearai-bench.yml new file mode 100644 index 00000000000..4209bf703b4 --- /dev/null +++ b/.github/workflows/nearai-bench.yml @@ -0,0 +1,201 @@ +# Slash-command dispatcher for nearai-bench on PR comments. A maintainer +# (write/maintain/admin permission) comments `/benchmark ` +# (optionally `--model `) on a PR; this workflow validates auth, +# parses the command, resolves the PR head SHA, reacts with 🚀, then +# delegates to nearai/benchmarks's reusable workflow which builds the +# harness against the PR's ironclaw SHA, runs the suite, compares against +# a baseline, and posts a markdown comment back here. +# +# Setup / usage docs: +# https://github.com/nearai/benchmarks/blob/main/docs/ci/README.md +# +# Requires `OPENROUTER_API_KEY` as an org-level (or repo-level) secret. +# Only that one secret is forwarded — `secrets: inherit` is intentionally +# NOT used (supply-chain hardening). + +name: nearai-bench + +on: + issue_comment: + types: [created] + +# Minimum surface for what this workflow actually does. The called +# reusable workflow (in nearai/benchmarks) declares its own ceiling +# (pull-requests:write + issues:write); the effective permissions in the +# called job are the intersection of these two, so a hostile change in +# the benchmarks repo can't acquire writes we didn't grant here. +# +# `pull-requests: write` (not just `issues: write`) is required because +# GitHub's reactions endpoint for PR-issue-comments rejects an +# `issues: write`-only GITHUB_TOKEN with `403 Resource not accessible by +# integration`, despite the docs at +# https://docs.github.com/rest/reactions/reactions#create-reaction-for-an-issue-comment +# listing only `issues: write`. Observed in +# https://github.com/nearai/ironclaw/actions/runs/26197185339 (the +# canary on PR #3834 that found this bug). +permissions: + pull-requests: write # gh pr view + gh pr comment + reactions on PR comments + issues: write # belt-and-suspenders; also covers parse-error comment + contents: read + +jobs: + parse: + # Tightened from `startsWith(body, '/benchmark')` to whole-word + # match — `/benchmarking-something` no longer enters the parse path. + if: >- + github.event.issue.pull_request != null + && ( + startsWith(github.event.comment.body, '/benchmark ') + || github.event.comment.body == '/benchmark' + ) + runs-on: ubuntu-latest + outputs: + suite: ${{ steps.parse.outputs.suite }} + model: ${{ steps.parse.outputs.model }} + head_sha: ${{ steps.parse.outputs.head_sha }} + authorized: ${{ steps.auth.outputs.authorized }} + steps: + - name: Check commenter has write permission + id: auth + # SHA-pinned to v7.0.1 to match the rest of this repo's workflows. + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + with: + script: | + // `getCollaboratorPermissionLevel` throws 404 for users who + // aren't collaborators at all (external contributors), and we + // want to silently drop those rather than fail the workflow + // with a noisy red run. Rethrow anything else. + let permission = 'none'; + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: context.payload.comment.user.login, + }); + permission = data.permission; + } catch (err) { + if (err.status !== 404) throw err; + // 404 → not a collaborator → unauthorized. + } + const ok = ['admin', 'write', 'maintain'].includes(permission); + core.setOutput('authorized', ok ? 'yes' : 'no'); + if (!ok) { + core.notice(`Comment author @${context.payload.comment.user.login} is not authorized to trigger benchmarks (permission: ${permission}).`); + } + + - name: Parse command + id: parse + if: steps.auth.outputs.authorized == 'yes' + env: + BODY: ${{ github.event.comment.body }} + PR: ${{ github.event.issue.number }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Pinned reviewed commit on nearai/benchmarks main. Must match + # the SHA in the `bench` job's `uses:` below — bump in both + # places together. Used here to validate the requested suite + # exists at the same SHA we'd actually run against. + BENCH_PIN: 67effacd8c8a7f6f43b422e30a1810822b9d6d5f + run: | + set -euo pipefail + # Grammar: /benchmark [--model ] + # Suite name: [A-Za-z0-9_-]+ + # Model ID: [A-Za-z0-9_./:-]+ (allows openrouter-style "vendor/model") + suite=$(printf '%s' "$BODY" | sed -nE 's@^/benchmark[[:space:]]+([A-Za-z0-9_-]+)([[:space:]]+--model[[:space:]]+[A-Za-z0-9_./:-]+)?[[:space:]]*$@\1@p') + model=$(printf '%s' "$BODY" | sed -nE 's@^/benchmark[[:space:]]+[A-Za-z0-9_-]+[[:space:]]+--model[[:space:]]+([A-Za-z0-9_./:-]+)[[:space:]]*$@\1@p') + if [ -z "$suite" ]; then + # User-input error — post a helpful reply and exit cleanly + # (exit 0 keeps the Actions tab green; bad syntax isn't infra + # failure). + gh pr comment "$PR" --repo "$REPO" --body $'Could not parse `/benchmark` command.\n\nUsage: `/benchmark [--model ]`\n\nExample: `/benchmark ironclaw-smoke`' + exit 0 + fi + # Pre-dispatch suite check: refuse unknown suite names before + # we acknowledge with 🚀 or spend any LLM credit. Cheap GitHub + # API lookup against the pinned benchmarks SHA so the user gets + # the same answer the dispatched workflow would. + if ! gh api "repos/nearai/benchmarks/contents/suites/${suite}.toml?ref=${BENCH_PIN}" --silent >/dev/null 2>&1; then + available=$(gh api "repos/nearai/benchmarks/contents/suites?ref=${BENCH_PIN}" \ + --jq '[.[] | select(.name | endswith(".toml")) | .name | rtrimstr(".toml")] | sort | join(", ")') + gh pr comment "$PR" --repo "$REPO" --body "Unknown suite \`${suite}\`. Available: ${available}" + exit 0 + fi + sha=$(gh pr view "$PR" --repo "$REPO" --json headRefOid -q .headRefOid) + echo "suite=$suite" >> "$GITHUB_OUTPUT" + echo "model=$model" >> "$GITHUB_OUTPUT" + echo "head_sha=$sha" >> "$GITHUB_OUTPUT" + + - name: Acknowledge with 🚀 reaction + if: steps.auth.outputs.authorized == 'yes' && steps.parse.outputs.suite != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + run: | + gh api "/repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -F content=rocket + + - name: Post "started" comment with link to dispatcher run + # Closes the gap between the 🚀 reaction and the eventual result + # comment (~5–10 min later from the called workflow). Gives the + # maintainer a clickable link straight to the Actions run. + if: steps.auth.outputs.authorized == 'yes' && steps.parse.outputs.suite != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + SUITE: ${{ steps.parse.outputs.suite }} + MODEL: ${{ steps.parse.outputs.model }} + HEAD_SHA: ${{ steps.parse.outputs.head_sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + short_sha="${HEAD_SHA:0:10}" + model_part="" + if [ -n "$MODEL" ]; then + model_part=" (model \`$MODEL\`)" + fi + gh pr comment "$PR" --repo "$REPO" --body \ + "🧪 Started \`$SUITE\`${model_part} against ironclaw \`$short_sha\` — [watch run]($RUN_URL)." + + bench: + needs: parse + if: needs.parse.outputs.authorized == 'yes' && needs.parse.outputs.suite != '' + # Job-scoped writes so the called workflow can post the result + # comment + toggle the trigger-comment reaction. Called workflows + # can only DOWNGRADE the caller's GITHUB_TOKEN, so the workflow-level + # `pull-requests: read` (which the parse job uses for `gh pr view`) + # would otherwise leave the called workflow without write to publish + # results. The parse job stays on read; only this authorized, + # post-auth job grants writes. + permissions: + contents: read + pull-requests: write + issues: write + id-token: write # the called workflow uses OIDC to assume an + # AWS role for S3 persistence; without this it + # fails to start. + # Concurrency lives on the authorized job, not the workflow root — + # otherwise an unauthorized commenter typing `/benchmark whatever` + # would acquire the per-PR group and cancel an in-flight maintainer + # run before failing auth. + concurrency: + group: nearai-bench-${{ github.event.issue.number }} + cancel-in-progress: true + # Tracks nearai/benchmarks main. Renderer / workflow changes there + # take effect on the next /benchmark without needing a deliberate + # SHA bump here. Both repos are under the same org with the same + # write-access set, so the supply-chain delta vs. a pinned SHA is + # small enough to trade for not stalling out every renderer change. + uses: nearai/benchmarks/.github/workflows/bench-pr-reusable.yml@main + with: + ironclaw-rev: ${{ needs.parse.outputs.head_sha }} + pr-repo: ${{ github.repository }} + pr-number: ${{ github.event.issue.number }} + suite-name: ${{ needs.parse.outputs.suite }} + model: ${{ needs.parse.outputs.model }} + trigger-comment-id: ${{ github.event.comment.id }} + # Scope inherited secrets to only what the reusable workflow needs. + # Replaces `secrets: inherit`, which gave the called workflow access + # to every secret available here. + secrets: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} diff --git a/.github/workflows/nightly-deep-ci.yml b/.github/workflows/nightly-deep-ci.yml new file mode 100644 index 00000000000..ab457ac0b9a --- /dev/null +++ b/.github/workflows/nightly-deep-ci.yml @@ -0,0 +1,59 @@ +name: Nightly Deep CI + +# This workflow intentionally reuses the canonical Run Tests workflow instead +# of duplicating job definitions here. With event_name == workflow_call, +# .github/workflows/test.yml runs the deterministic deep suite: +# - full Rust test matrix: all-features, default, libsql-only +# - heavy runtime integration tests +# - Telegram integration partitions +# - replay snapshot gate +# - Telegram and Slack channel tests +# - WASM/WIT compatibility checks +# - Windows compile matrix +# - benchmark compilation +# +# Docker is intentionally excluded here because QA/release Docker images are +# owned by the separate Docker pipeline and need release-team alignment. +# Full browser E2E is scheduled separately by .github/workflows/e2e.yml. + +on: + schedule: + - cron: "0 4 * * *" # Daily deterministic deep CI at 04:00 UTC + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-deep-ci-${{ github.ref }} + cancel-in-progress: false + +jobs: + deterministic-deep-tests: + name: Deterministic Deep Tests + uses: ./.github/workflows/test.yml + with: + ref: ${{ github.sha }} + include_docker: false + + nightly-alert: + name: Nightly Deep CI Alert + runs-on: ubuntu-latest + if: always() && github.event_name == 'schedule' + permissions: + actions: read + contents: read + issues: write + needs: [deterministic-deep-tests] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - name: Report nightly deep CI result + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ALERT_WORKFLOW_NAME: Nightly Deep CI + ALERT_ISSUE_TITLE: Nightly Deep CI failed + ALERT_RESULT: ${{ needs.deterministic-deep-tests.result }} + run: .github/scripts/nightly-alert-issue.sh diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml new file mode 100644 index 00000000000..a4ce3004f64 --- /dev/null +++ b/.github/workflows/nightly-e2e.yml @@ -0,0 +1,46 @@ +name: Nightly E2E + +# Scheduled wrapper around the reusable E2E workflow. Keep alerting here rather +# than inside e2e.yml so callers like Run Tests can reference e2e.yml without +# inheriting schedule-only issue-writing permissions. +on: + schedule: + - cron: "0 3 * * *" # Nightly full E2E at 03:00 UTC + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-e2e-${{ github.ref }} + cancel-in-progress: false + +jobs: + e2e: + name: Full E2E + uses: ./.github/workflows/e2e.yml + with: + ref: ${{ github.sha }} + mode: full + + nightly-alert: + name: Nightly E2E Alert + runs-on: ubuntu-latest + if: always() && github.event_name == 'schedule' + permissions: + actions: read + contents: read + issues: write + needs: [e2e] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - name: Report nightly E2E result + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ALERT_WORKFLOW_NAME: Nightly E2E + ALERT_ISSUE_TITLE: Nightly E2E failed + ALERT_RESULT: ${{ needs.e2e.result }} + run: .github/scripts/nightly-alert-issue.sh diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml new file mode 100644 index 00000000000..a06497f82ff --- /dev/null +++ b/.github/workflows/reborn-e2e.yml @@ -0,0 +1,140 @@ +name: Reborn E2E + +on: + workflow_call: + inputs: + ref: + description: Commit SHA or ref to test + required: false + type: string + workflow_dispatch: + pull_request: + branches: + - main + paths: + - "crates/ironclaw_*/**" + - "docs/reborn/**" + - "scripts/reborn-e2e-rust.sh" + - "tests/e2e/**" + - "build.rs" + - "providers.json" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/reborn-e2e.yml" + push: + branches: + - main + paths: + - "crates/ironclaw_*/**" + - "docs/reborn/**" + - "scripts/reborn-e2e-rust.sh" + - "tests/e2e/**" + - "build.rs" + - "providers.json" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/reborn-e2e.yml" + +permissions: + contents: read + +concurrency: + group: reborn-e2e-${{ github.event_name }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + rust-reborn: + name: Rust Reborn (${{ matrix.group }}) + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + group: + - architecture + - runtimes + - substrates + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Restore Rust cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: reborn-e2e-${{ matrix.group }} + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Run deterministic Reborn Rust gate + env: + CARGO_TEST_ARGS: "-- --nocapture" + run: scripts/reborn-e2e-rust.sh ${{ matrix.group }} + + gateway-smoke: + name: Reborn gateway smoke + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Restore Rust cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: reborn-e2e-gateway-smoke + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build ironclaw with libSQL + run: cargo build --no-default-features --features libsql + + - name: Install Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run Reborn gateway smoke + run: pytest tests/e2e/scenarios/test_reborn_gateway_smoke.py -v --timeout=120 + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: reborn-e2e-screenshots + path: tests/e2e/screenshots/ + if-no-files-found: ignore + + reborn-e2e: + name: Reborn E2E + runs-on: ubuntu-latest + if: always() + needs: + - rust-reborn + - gateway-smoke + steps: + - name: Check Reborn E2E jobs + run: | + if [[ "${{ needs.rust-reborn.result }}" != "success" ]]; then + echo "Rust Reborn E2E failed" + exit 1 + fi + if [[ "${{ needs.gateway-smoke.result }}" != "success" ]]; then + echo "Reborn gateway smoke failed" + exit 1 + fi diff --git a/.github/workflows/regression-test-check.yml b/.github/workflows/regression-test-check.yml index 6ce58b01218..b2359211e26 100644 --- a/.github/workflows/regression-test-check.yml +++ b/.github/workflows/regression-test-check.yml @@ -21,25 +21,29 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Fetch PR head and base + - name: Fetch PR base env: BASE_REF: ${{ github.event.pull_request.base.ref }} - PR_NUMBER: ${{ github.event.pull_request.number }} run: | git fetch origin -- "$BASE_REF" - git fetch origin -- "pull/${PR_NUMBER}/head:pr-head" - name: Check for regression tests env: PR_TITLE: ${{ github.event.pull_request.title }} PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail BASE_REF="origin/${PR_BASE_REF}" - # Use the actual PR head, not the merge commit that actions/checkout checks out - HEAD_REF="pr-head" + # Use the PR head SHA from the event payload directly. The previous + # approach fetched `refs/pull/N/head`, which GitHub occasionally + # fails to publish for in-repo PRs (hit on #3324, blocking this + # workflow before any of its actual checks could run). The SHA is + # reachable locally because actions/checkout (fetch-depth: 0) + # already pulled the merge commit, whose second parent is this SHA. + HEAD_REF="$PR_HEAD_SHA" # --- 1. Is this a fix PR? Check title first, then commit messages --- IS_FIX=false diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 00000000000..37dede013d0 --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,69 @@ +name: Sync upstream + +on: + schedule: + - cron: "0 * * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + if: github.repository == 'theredspoon/ironclaw' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Fetch upstream + run: | + git remote add upstream https://github.com/nearai/ironclaw.git + git fetch upstream main --no-tags + + - name: Check if upstream has new commits + id: check + run: | + if git merge-base --is-ancestor upstream/main HEAD; then + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Attempt fast-forward merge + if: steps.check.outputs.skip == 'false' + id: ff + run: | + if git merge --ff-only upstream/main; then + echo "result=fast-forward" >> "$GITHUB_OUTPUT" + else + echo "result=diverged" >> "$GITHUB_OUTPUT" + fi + + - name: Push if fast-forwarded + if: steps.ff.outputs.result == 'fast-forward' + run: git push origin main --tags + + - name: Open PR if diverged + if: steps.ff.outputs.result == 'diverged' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git reset --hard HEAD + branch="sync/upstream-$(date +%Y%m%d)" + git checkout -b "$branch" + git merge upstream/main --no-edit || true + git push -u origin "$branch" + if git diff --quiet HEAD; then + gh pr create \ + --title "sync: merge upstream/main $(date +%Y-%m-%d)" \ + --body "Automated sync from nearai/ironclaw. Fast-forward was not possible — manual review required." + else + gh pr create \ + --title "sync: merge upstream/main $(date +%Y-%m-%d) (conflicts)" \ + --body "Automated sync from nearai/ironclaw. There are merge conflicts that need manual resolution." + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57bc51cccf7..9ccf13f7386 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,11 @@ on: description: Commit SHA or ref to test required: false type: string + include_docker: + description: Include Docker image build in this reusable test run + required: false + type: boolean + default: true pull_request: branches: - main @@ -50,8 +55,9 @@ jobs: echo "has_engine_replay_risk=false" >> "$GITHUB_OUTPUT" fi echo "has_runtime_heavy_risk=true" >> "$GITHUB_OUTPUT" - # Keep full browser E2E outside the workflow-call / push path for now; - # merge_group runs cover the full browser suite before queued merges land. + # Keep deterministic full browser E2E in its own scheduled/manual + # workflow. The merge queue runs only smoke coverage for web-risk + # changes so queued merges remain fast. echo "has_web_risk=false" >> "$GITHUB_OUTPUT" echo "has_wasm_abi_risk=true" >> "$GITHUB_OUTPUT" echo "has_telegram_risk=true" >> "$GITHUB_OUTPUT" @@ -85,43 +91,43 @@ jobs: echo "docs_only=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(src/|crates/|channels-src/|tools-src/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|build\.rs$|\.github/workflows/(test|e2e|replay-gate|code_style|docker|release|release-plz|rebuild-release-image|coverage|live-canary|pr-label-classify|pr-label-scope|claude-review)\.yml$|\.github/actions/install-cargo-component/|\.github/(dependabot|labeler)\.yml$)'; then + if has_match '^(src/|crates/|channels-src/|tools-src/|tests/|migrations/|Cargo\.toml$|Cargo\.lock$|build\.rs$|\.github/workflows/(test|nightly-deep-ci|e2e|replay-gate|code_style|docker|release|release-plz|rebuild-release-image|coverage|live-canary|pr-label-classify|pr-label-scope|claude-review)\.yml$|\.github/actions/install-cargo-component/|\.github/(dependabot|labeler)\.yml$)'; then echo "has_core_code=true" >> "$GITHUB_OUTPUT" else echo "has_core_code=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(crates/ironclaw_(common|engine|skills)/|src/(agent|auth|bridge|context|extensions|llm|orchestrator|secrets|skills|tools|worker|workspace)/|tests/fixtures/llm_traces/|tests/snapshots/|tests/support/replay_outcome\.rs$|tests/e2e_engine_v2\.rs$|tests/e2e_live\.rs$|tests/e2e_recorded_trace\.rs$|tests/e2e_advanced_traces\.rs$|tests/e2e_trace_.*\.rs$|Cargo\.toml$|\.github/workflows/(replay-gate|test)\.yml$)'; then + if has_match '^(crates/ironclaw_(common|engine|skills)/|src/(agent|auth|bridge|context|extensions|llm|orchestrator|secrets|skills|tools|worker|workspace)/|tests/fixtures/llm_traces/|tests/snapshots/|tests/support/replay_outcome\.rs$|tests/e2e_engine_v2\.rs$|tests/e2e_live\.rs$|tests/e2e_recorded_trace\.rs$|tests/e2e_advanced_traces\.rs$|tests/e2e_trace_.*\.rs$|Cargo\.toml$|\.github/workflows/(replay-gate|test|nightly-deep-ci)\.yml$)'; then echo "has_engine_replay_risk=true" >> "$GITHUB_OUTPUT" else echo "has_engine_replay_risk=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(crates/ironclaw_(common|safety|skills)/|src/(agent|auth|bridge|config|context|db|extensions|gate|history|hooks|orchestrator|ownership|pairing|safety|sandbox|secrets|setup|skills|tools|webhooks|worker|workspace)/|src/channels/channel\.rs$|src/channels/manager\.rs$|src/channels/relay/|src/channels/wasm/|channels-src/telegram/|migrations/|src/app\.rs$|src/main\.rs$|src/lib\.rs$|tests/telegram_auth_integration\.rs$|tests/e2e_thread_scheduling\.rs$|\.github/workflows/test\.yml$)'; then + if has_match '^(crates/ironclaw_(common|safety|skills)/|src/(agent|auth|bridge|config|context|db|extensions|gate|history|hooks|orchestrator|ownership|pairing|safety|sandbox|secrets|setup|skills|tools|webhooks|worker|workspace)/|src/channels/channel\.rs$|src/channels/manager\.rs$|src/channels/relay/|src/channels/wasm/|channels-src/telegram/|migrations/|src/app\.rs$|src/main\.rs$|src/lib\.rs$|tests/telegram_auth_integration\.rs$|tests/e2e_thread_scheduling\.rs$|\.github/workflows/(test|nightly-deep-ci)\.yml$)'; then echo "has_runtime_heavy_risk=true" >> "$GITHUB_OUTPUT" else echo "has_runtime_heavy_risk=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(src/(auth|bridge|extensions|ownership|pairing|secrets|webhooks|workspace)/|src/channels/web/|crates/ironclaw_gateway/|src/channels/attachments\.rs$|tests/e2e/|tests/fixtures/gateway_traces/|tests/test-pages/|scripts/check_gateway_boundaries\.py$|\.github/workflows/(e2e|test)\.yml$)'; then + if has_match '^(src/(auth|bridge|extensions|ownership|pairing|secrets|webhooks|workspace)/|src/channels/web/|crates/ironclaw_gateway/|src/channels/attachments\.rs$|tests/e2e/|tests/fixtures/gateway_traces/|tests/test-pages/|scripts/check_gateway_boundaries\.py$|\.github/workflows/(e2e|test|nightly-deep-ci)\.yml$)'; then echo "has_web_risk=true" >> "$GITHUB_OUTPUT" else echo "has_web_risk=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(wit/|crates/ironclaw_common/|src/(channels/wasm|extensions|registry|tools/mcp|tools/wasm)/|channels-src/|tools-src/|registry/|scripts/build-wasm-extensions\.sh$|scripts/check-version-bumps\.sh$|tests/wit_compat\.rs$|tests/wasm_channel_integration\.rs$|tests/e2e_wasm_.*|Cargo\.toml$|Cargo\.lock$|\.github/workflows/test\.yml$)'; then + if has_match '^(wit/|crates/ironclaw_common/|src/(channels/wasm|extensions|registry|tools/mcp|tools/wasm)/|channels-src/|tools-src/|registry/|scripts/build-wasm-extensions\.sh$|scripts/check-version-bumps\.sh$|tests/wit_compat\.rs$|tests/wasm_channel_integration\.rs$|tests/e2e_wasm_.*|Cargo\.toml$|Cargo\.lock$|\.github/workflows/(test|nightly-deep-ci)\.yml$)'; then echo "has_wasm_abi_risk=true" >> "$GITHUB_OUTPUT" else echo "has_wasm_abi_risk=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(channels-src/telegram/|src/channels/wasm/|tests/telegram_auth_integration\.rs$|tests/e2e_telegram_message_routing\.rs$|tests/e2e/scenarios/test_telegram_.*|\.github/workflows/test\.yml$)'; then + if has_match '^(channels-src/telegram/|src/channels/wasm/|tests/telegram_auth_integration\.rs$|tests/e2e_telegram_message_routing\.rs$|tests/e2e/scenarios/test_telegram_.*|\.github/workflows/(test|nightly-deep-ci)\.yml$)'; then echo "has_telegram_risk=true" >> "$GITHUB_OUTPUT" else echo "has_telegram_risk=false" >> "$GITHUB_OUTPUT" fi - if has_match '^(channels-src/slack/|src/channels/wasm/|tests/slack_auth_integration\.rs$|\.github/workflows/test\.yml$)'; then + if has_match '^(channels-src/slack/|src/channels/wasm/|tests/slack_auth_integration\.rs$|\.github/workflows/(test|nightly-deep-ci)\.yml$)'; then echo "has_slack_risk=true" >> "$GITHUB_OUTPUT" else echo "has_slack_risk=false" >> "$GITHUB_OUTPUT" @@ -139,7 +145,7 @@ jobs: FULL='[{"name":"all-features","flags":"--no-default-features --features postgres,libsql,html-to-markdown,bedrock,import"},{"name":"default","flags":""},{"name":"libsql-only","flags":"--no-default-features --features libsql"}]' SLIM='[{"name":"all-features","flags":"--no-default-features --features postgres,libsql,html-to-markdown,bedrock,import"}]' - if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "merge_group" ]; then echo "test_matrix=${SLIM}" >> "$GITHUB_OUTPUT" echo "windows_matrix=${SLIM}" >> "$GITHUB_OUTPUT" else @@ -191,7 +197,9 @@ jobs: heavy-integration-tests: name: Heavy Integration Tests needs: changes - if: needs.changes.outputs.docs_only != 'true' && (github.event_name != 'pull_request' || needs.changes.outputs.has_runtime_heavy_risk == 'true') + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call' || needs.changes.outputs.has_runtime_heavy_risk == 'true') runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -218,7 +226,9 @@ jobs: telegram-integration-tests: name: Telegram Integration Tests (${{ matrix.partition }}) needs: changes - if: needs.changes.outputs.docs_only != 'true' && (github.event_name != 'pull_request' || needs.changes.outputs.has_runtime_heavy_risk == 'true') + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call' || needs.changes.outputs.has_runtime_heavy_risk == 'true') runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -276,18 +286,20 @@ jobs: mode: smoke web-e2e-full: - name: Web E2E Full + name: Web E2E Smoke (merge queue) needs: changes if: needs.changes.outputs.docs_only != 'true' && github.event_name == 'merge_group' && needs.changes.outputs.has_web_risk == 'true' uses: ./.github/workflows/e2e.yml with: ref: ${{ inputs.ref || github.sha }} - mode: full + mode: smoke telegram-tests: name: Telegram Channel Tests needs: changes - if: needs.changes.outputs.docs_only != 'true' && (github.event_name != 'pull_request' || needs.changes.outputs.has_telegram_risk == 'true') + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call' || needs.changes.outputs.has_telegram_risk == 'true') runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -309,7 +321,9 @@ jobs: slack-tests: name: Slack Channel Tests needs: changes - if: needs.changes.outputs.docs_only != 'true' && (github.event_name != 'pull_request' || needs.changes.outputs.has_slack_risk == 'true') + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call' || needs.changes.outputs.has_slack_risk == 'true') runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -336,7 +350,7 @@ jobs: windows-build: name: Windows Build (${{ matrix.name }}) needs: [changes, matrix-config] - if: needs.changes.outputs.docs_only != 'true' && github.event_name != 'pull_request' + if: needs.changes.outputs.docs_only != 'true' && (github.event_name == 'push' || github.event_name == 'workflow_call') runs-on: windows-latest strategy: fail-fast: false @@ -360,7 +374,9 @@ jobs: wasm-wit-compat: name: WASM WIT Compatibility needs: changes - if: needs.changes.outputs.docs_only != 'true' && (github.event_name != 'pull_request' || needs.changes.outputs.has_wasm_abi_risk == 'true') + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call' || needs.changes.outputs.has_wasm_abi_risk == 'true') runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -397,7 +413,10 @@ jobs: bench-compile: name: Benchmark Compilation needs: changes - if: needs.changes.outputs.docs_only != 'true' && github.event_name != 'pull_request' && needs.changes.outputs.has_core_code == 'true' + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || github.event_name == 'workflow_call') && + needs.changes.outputs.has_core_code == 'true' runs-on: ubuntu-latest steps: - name: Checkout repository @@ -417,7 +436,9 @@ jobs: docker-build: name: Docker Build needs: changes - if: needs.changes.outputs.docs_only != 'true' && github.event_name != 'pull_request' + if: > + needs.changes.outputs.docs_only != 'true' && + (github.event_name == 'push' || (github.event_name == 'workflow_call' && inputs.include_docker)) runs-on: ubuntu-latest steps: - name: Checkout repository @@ -549,7 +570,7 @@ jobs: require_success "tests" "${{ needs.tests.result }}" fi - if [[ "$event" != "pull_request" || "${{ needs.changes.outputs.has_runtime_heavy_risk }}" == "true" ]]; then + if [[ "$event" == "push" || "$event" == "workflow_call" || "${{ needs.changes.outputs.has_runtime_heavy_risk }}" == "true" ]]; then require_success "heavy-integration-tests" "${{ needs.heavy-integration-tests.result }}" require_success "telegram-integration-tests" "${{ needs.telegram-integration-tests.result }}" fi @@ -566,27 +587,27 @@ jobs: require_success "web-e2e-full" "${{ needs.web-e2e-full.result }}" fi - if [[ "$event" != "pull_request" || "${{ needs.changes.outputs.has_telegram_risk }}" == "true" ]]; then + if [[ "$event" == "push" || "$event" == "workflow_call" || "${{ needs.changes.outputs.has_telegram_risk }}" == "true" ]]; then require_success "telegram-tests" "${{ needs.telegram-tests.result }}" fi - if [[ "$event" != "pull_request" || "${{ needs.changes.outputs.has_slack_risk }}" == "true" ]]; then + if [[ "$event" == "push" || "$event" == "workflow_call" || "${{ needs.changes.outputs.has_slack_risk }}" == "true" ]]; then require_success "slack-tests" "${{ needs.slack-tests.result }}" fi - if [[ "$event" != "pull_request" || "${{ needs.changes.outputs.has_wasm_abi_risk }}" == "true" ]]; then + if [[ "$event" == "push" || "$event" == "workflow_call" || "${{ needs.changes.outputs.has_wasm_abi_risk }}" == "true" ]]; then require_success "wasm-wit-compat" "${{ needs.wasm-wit-compat.result }}" fi - if [[ "$event" != "pull_request" ]]; then + if [[ "$event" == "push" || ( "$event" == "workflow_call" && "${{ inputs.include_docker }}" == "true" ) ]]; then require_success "docker-build" "${{ needs.docker-build.result }}" fi - if [[ "$event" != "pull_request" ]]; then + if [[ "$event" == "push" || "$event" == "workflow_call" ]]; then require_success "windows-build" "${{ needs.windows-build.result }}" fi - if [[ "$event" != "pull_request" && "${{ needs.changes.outputs.has_core_code }}" == "true" ]]; then + if [[ ( "$event" == "push" || "$event" == "workflow_call" ) && "${{ needs.changes.outputs.has_core_code }}" == "true" ]]; then require_success "bench-compile" "${{ needs.bench-compile.result }}" fi diff --git a/AGENTS.md b/AGENTS.md index 90ad4bf5229..0a91e9f9286 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ Start with these deeper docs as needed: - `src/agent/CLAUDE.md` - `src/channels/web/CLAUDE.md` - `src/db/CLAUDE.md` -- `src/llm/CLAUDE.md` +- `crates/ironclaw_llm/CLAUDE.md` - `src/setup/README.md` - `src/tools/README.md` - `src/workspace/README.md` @@ -29,7 +29,7 @@ Start with these deeper docs as needed: - Web gateway/API/SSE/WebSocket: `src/channels/web/` - Persistence and DB abstractions: `src/db/` - Setup/onboarding/configuration flow: `src/setup/` -- LLM providers and routing: `src/llm/` +- LLM providers and routing: `crates/ironclaw_llm/` - Workspace, memory, embeddings, search: `src/workspace/` - Extensions, tools, channels, MCP, WASM: `src/extensions/`, `src/tools/`, `src/channels/` diff --git a/CHANGELOG.md b/CHANGELOG.md index f66e98dee1e..2eb2c47155e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,133 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.29.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.28.2...ironclaw-v0.29.0) - 2026-05-26 + +### Added + +- *(channels)* add WeCom channel ([#2394](https://github.com/nearai/ironclaw/pull/2394)) +- *(web)* support externally-provided tools in Responses API ([#3122](https://github.com/nearai/ironclaw/pull/3122)) +- *(gateway)* add logs download button ([#3588](https://github.com/nearai/ironclaw/pull/3588)) +- *(tui)* add Ctrl-S log download from the Logs tab ([#3658](https://github.com/nearai/ironclaw/pull/3658)) +- *(engine)* add `IRONCLAW_DISABLE_CODEACT` flag for disabling v2 CodeAct ([#3665](https://github.com/nearai/ironclaw/pull/3665)) + +### Fixed + +- *(markdown)* avoid converting emphasis inside generated Slack angle links ([#3532](https://github.com/nearai/ironclaw/pull/3532)) +- *(web)* restore NEAR AI API Key and Fetch Models in configure UI ([#3742](https://github.com/nearai/ironclaw/pull/3742)) + +### Changed + +- *(embeddings)* extract embeddings into `ironclaw_embeddings` crate ([#3739](https://github.com/nearai/ironclaw/pull/3739)) +- *(deps)* bump dependencies to address security advisories ([#3719](https://github.com/nearai/ironclaw/pull/3719)) +- *(deps)* update Wasmtime to clear cargo-deny advisory ([#4028](https://github.com/nearai/ironclaw/pull/4028)) + +### CI / Release + +- *(canary)* improve live canary counts, chat-install probe, and strict xfails ([#3682](https://github.com/nearai/ironclaw/pull/3682)) +- *(ci)* add `/benchmark` slash-command dispatcher ([#3808](https://github.com/nearai/ironclaw/pull/3808)) +- *(ci)* grant `pull-requests: write` for `/benchmark` reactions endpoint ([#3835](https://github.com/nearai/ironclaw/pull/3835)) +- *(ci)* post benchmark "started" comment with dispatcher run link ([#3836](https://github.com/nearai/ironclaw/pull/3836)) + +### Documentation + +- *(api)* document the Responses API end-to-end ([#3709](https://github.com/nearai/ironclaw/pull/3709)) + +## [0.28.2](https://github.com/nearai/ironclaw/compare/ironclaw-v0.28.1...ironclaw-v0.28.2) - 2026-05-14 + +### Fixed + +- *(extensions)* restore chat-driven `tool_install` + fix double-invoke + auto-approve footgun ([#3559](https://github.com/nearai/ironclaw/pull/3559)) + +### Changed + +- *(llm)* hide provider-specific auth, model fetch, and embeddings config behind facades ([#3416](https://github.com/nearai/ironclaw/pull/3416)) + +### Tests + +- *(e2e)* unxfail two auth-matrix tests now that contracts match ([#3589](https://github.com/nearai/ironclaw/pull/3589)) +- *(e2e)* make Skills lifecycle deterministic ([#3309](https://github.com/nearai/ironclaw/pull/3309)) + +## [0.28.1](https://github.com/nearai/ironclaw/compare/ironclaw-v0.28.0...ironclaw-v0.28.1) - 2026-05-11 + +### Added + +- *(channels)* add `pairing_approve` tool for Slack binding via chat ([#3396](https://github.com/nearai/ironclaw/pull/3396)) +- *(channels)* add WeChat registry artifact metadata ([#3386](https://github.com/nearai/ironclaw/pull/3386)) +- *(common)* describe paths and platform helpers in crate description ([#3498](https://github.com/nearai/ironclaw/pull/3498)) + +### Fixed + +- *(web)* bug bash — restart modal recovery, approval clarity, http defaults ([#3364](https://github.com/nearai/ironclaw/pull/3364)) +- *(bridge)* bypass agent-loop mpsc for inline-await Approval gates ([#3365](https://github.com/nearai/ironclaw/pull/3365)) +- *(missions)* auto-resume paused missions after gate resolution ([#3366](https://github.com/nearai/ironclaw/pull/3366)) +- *(workspace)* multi-tenant memory isolation ([#3374](https://github.com/nearai/ironclaw/pull/3374)) +- *(auth)* tighten Telegram pairing UX and OAuth-failure recovery ([#3381](https://github.com/nearai/ironclaw/pull/3381)) +- *(web)* isolate cross-tenant SSE/WS status events and thread access ([#3390](https://github.com/nearai/ironclaw/pull/3390)) +- *(channels)* activate WASM channels on headless servers ([#3233](https://github.com/nearai/ironclaw/pull/3233)) + +### Changed + +- *(llm)* extract multi-provider integration into ironclaw_llm crate ([#3387](https://github.com/nearai/ironclaw/pull/3387)) + +### CI / Release + +- *(canary)* seed github_token_scopes companion in auth-live-seeded ([#3384](https://github.com/nearai/ironclaw/pull/3384)) + +### Tests + +- *(e2e)* restore auth and approval coverage ([#3430](https://github.com/nearai/ironclaw/pull/3430)) +- *(e2e)* avoid REPL auth retry race ([#3437](https://github.com/nearai/ironclaw/pull/3437)) + +## [0.28.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.27.0...ironclaw-v0.28.0) - 2026-05-07 + +### Added + +- *(reborn)* land the reborn-integration substrate on `main`, introducing host foundation crates, capability host, runtime dispatcher, process lifecycle, filesystem, secrets, network, and extension manifest registry boundaries +- *(reborn)* add WIT-compatible WASM tool runtime ([#3097](https://github.com/nearai/ironclaw/pull/3097)) +- *(reborn)* add host runtime contract facade and services graph ([#3095](https://github.com/nearai/ironclaw/pull/3095), [#3126](https://github.com/nearai/ironclaw/pull/3126)) +- *(reborn)* add memory document storage boundary and search/plugin seams ([#3078](https://github.com/nearai/ironclaw/pull/3078), [#3079](https://github.com/nearai/ironclaw/pull/3079)) +- *(reborn)* add prompt write safety policy ([#3167](https://github.com/nearai/ironclaw/pull/3167)) +- *(reborn)* route WASM and MCP HTTP through shared egress ([#3123](https://github.com/nearai/ironclaw/pull/3123), [#3142](https://github.com/nearai/ironclaw/pull/3142)) +- *(reborn)* add host-controlled trust-class policy engine ([#3043](https://github.com/nearai/ironclaw/pull/3043)) +- *(channels)* add WeChat channel ([#1666](https://github.com/nearai/ironclaw/pull/1666)) +- *(channels)* add multi-tenant relay channel with per-user identity resolution ([#3253](https://github.com/nearai/ironclaw/pull/3253)) +- *(llm)* enable thinking for Ollama via default additional params ([#2372](https://github.com/nearai/ironclaw/pull/2372)) + +### Fixed + +- *(llm)* route DeepSeek, Gemini, and OpenRouter through dedicated rig-core clients ([#3326](https://github.com/nearai/ironclaw/pull/3326)) +- *(config)* keep startup LLM fallback in-memory only ([#3324](https://github.com/nearai/ironclaw/pull/3324)) +- *(engine)* inline gate await for Tier 0 and Tier 1 Approval gates ([#3157](https://github.com/nearai/ironclaw/pull/3157)) +- *(engine,web)* suppress restart-recovery noise on Projects tab; retry empty hydration on SSE open ([#3328](https://github.com/nearai/ironclaw/pull/3328)) +- *(bridge)* coerce engine action params per schema ([#3197](https://github.com/nearai/ironclaw/pull/3197)) +- *(bridge)* `mission_*` tools accept name; resolves #2583 ([#3155](https://github.com/nearai/ironclaw/pull/3155)) +- *(libsql)* parse scientific notation cost aggregates ([#3297](https://github.com/nearai/ironclaw/pull/3297)) +- *(reborn)* harden capability approval lifecycle ([#3111](https://github.com/nearai/ironclaw/pull/3111)) +- *(reborn)* harden edge-case contracts and runtime network policy handoff ([#3165](https://github.com/nearai/ironclaw/pull/3165)) + +### Changed + +- *(common)* clarify crate-level doc wording and align package description ([#3370](https://github.com/nearai/ironclaw/pull/3370), [#3372](https://github.com/nearai/ironclaw/pull/3372)) + +### CI / Release + +- cut over workflows for main merge queue ([#3104](https://github.com/nearai/ironclaw/pull/3104)) +- build ironclaw docker image with staging tag from main branch ([#3301](https://github.com/nearai/ironclaw/pull/3301)) +- *(release)* bump cargo-dist to 0.31.0 to fix installer ([#3172](https://github.com/nearai/ironclaw/pull/3172)) +- add deterministic nightly deep checks and full browser suite nightly ([#3261](https://github.com/nearai/ironclaw/pull/3261), [#3262](https://github.com/nearai/ironclaw/pull/3262)) +- add nightly failure issue alerts ([#3293](https://github.com/nearai/ironclaw/pull/3293)) + +### Docs + +- refresh feature parity against OpenClaw 2026.3.11–2026.4.30 ([#3310](https://github.com/nearai/ironclaw/pull/3310)) +- promote database and configuration pages from drafts to live; fix wrong defaults; expand variable reference + +### Tests + +- *(reborn)* add phase 1 integration coverage, host runtime vertical gates, and CapabilityHost integration coverage +- *(e2e)* add dedicated reborn e2e gate and stabilize coverage suite + ## [0.27.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.26.0...ironclaw-v0.27.0) - 2026-04-29 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index f3257cc19da..db929b29ee0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,13 +78,14 @@ All I/O is async with tokio. Use `Arc` for shared state, `RwLock` for concurr ## Extracted Crates -Safety logic lives in `crates/ironclaw_safety/`, skills in `crates/ironclaw_skills/`. **Import directly from the extracted crate** (e.g. `use ironclaw_safety::SafetyLayer`, `use ironclaw_skills::SkillRegistry`). Do not use `crate::safety::` or `crate::skills::` for types that originate in extracted crates — `src/safety/mod.rs` and `src/skills/mod.rs` no longer glob-re-export. Local items defined in those modules (e.g. `crate::skills::attenuate_tools`) are fine. +Safety logic lives in `crates/ironclaw_safety/`, skills in `crates/ironclaw_skills/`, multi-provider LLM integration in `crates/ironclaw_llm/`. **Import directly from the extracted crate** (e.g. `use ironclaw_safety::SafetyLayer`, `use ironclaw_skills::SkillRegistry`, `use ironclaw_llm::{LlmProvider, LlmError}`). Do not use `crate::safety::`, `crate::skills::`, or `crate::llm::` for types that originate in extracted crates — `src/llm/` was deleted in the LLM extraction, and `src/safety/mod.rs` / `src/skills/mod.rs` no longer glob-re-export. Local items defined in those modules (e.g. `crate::skills::attenuate_tools`) are fine. The `crate::error::LlmError` alias and `crate::config::*Config` re-exports are kept as a thin convenience: they forward to `ironclaw_llm::*` so existing call sites compile, but new code should import from the extracted crate. ## Project Structure ``` crates/ -└── ironclaw_safety/ # Extracted: prompt injection, validation, leak detection, policy +├── ironclaw_safety/ # Extracted: prompt injection, validation, leak detection, policy +└── ironclaw_llm/ # Extracted: multi-provider LLM integration (rig-core, OpenAI, Anthropic, NEAR AI, Bedrock, …) src/ ├── lib.rs # Library root, module declarations @@ -156,7 +157,7 @@ src/ │ ├── safety/ # Re-export shim for crates/ironclaw_safety (see Extracted Crates) │ -├── llm/ # Multi-provider LLM integration — see src/llm/CLAUDE.md +├── (llm/ was extracted to crates/ironclaw_llm/ — see Extracted Crates) │ ├── tools/ # Extensible tool system │ ├── tool.rs # Tool trait, ToolOutput, ToolError @@ -230,7 +231,8 @@ When modifying a module with a spec, read the spec first. Code follows spec; spe | `src/agent/` | `src/agent/CLAUDE.md` | | `src/channels/web/` | `src/channels/web/CLAUDE.md` | | `src/db/` | `src/db/CLAUDE.md` | -| `src/llm/` | `src/llm/CLAUDE.md` | +| `crates/ironclaw_llm/` | `crates/ironclaw_llm/CLAUDE.md` | +| `crates/ironclaw_embeddings/` | `crates/ironclaw_embeddings/AGENTS.md` | | `src/setup/` | `src/setup/README.md` | | `src/tools/` | `src/tools/README.md` | | `src/workspace/` | `src/workspace/README.md` | @@ -256,7 +258,7 @@ SKILL.md files extend the agent's prompt with domain-specific instructions. See ## Configuration -See `.env.example` for all environment variables. LLM backends (`nearai`, `openai`, `anthropic`, `ollama`, `openai_compatible`, `tinfoil`, `bedrock`) documented in `src/llm/CLAUDE.md`. +See `.env.example` for all environment variables. LLM backends (`nearai`, `openai`, `anthropic`, `ollama`, `openai_compatible`, `tinfoil`, `bedrock`) documented in `crates/ironclaw_llm/CLAUDE.md`. ## Adding a New Channel diff --git a/Cargo.lock b/Cargo.lock index a916ca547c2..22b12fa586a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -127,7 +127,7 @@ dependencies = [ "once_cell", "serde", "version_check", - "zerocopy 0.8.48", + "zerocopy 0.8.50", ] [[package]] @@ -202,7 +202,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -213,7 +213,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -271,6 +271,15 @@ dependencies = [ "x11rb", ] +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "archery" version = "1.2.2" @@ -345,9 +354,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -446,9 +455,9 @@ dependencies = [ [[package]] name = "async-signal" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ "async-io", "async-lock", @@ -548,15 +557,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.15" +version = "1.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +checksum = "517aa062d8bd9015ee23d6daa5e1c1372328412fdae4e6c4c1be9b69c6ad37a2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -568,12 +577,13 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http 1.4.0", + "http 1.4.1", "sha1", "time", "tokio", @@ -596,9 +606,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -606,9 +616,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -618,9 +628,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.2" +version = "1.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -634,7 +644,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "percent-encoding", "pin-project-lite", @@ -644,10 +654,11 @@ dependencies = [ [[package]] name = "aws-sdk-bedrockruntime" -version = "1.128.0" +version = "1.132.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949d34a5c329ed83e7146d2fc1ffc06473fdc9bcbc5fa3d3534abeb950569c5" +checksum = "41a2940faeb61f4f579a434bc3a546e9ab49a89596e94527d329281ef55fd44d" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -663,7 +674,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body-util", "regex-lite", "tracing", @@ -671,9 +682,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.97.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +checksum = "bee2719d4a5e5e147bb9e9b77490df6ece750df1094968aa857b09b618a1881a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -688,17 +699,18 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.99.0" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +checksum = "b30d254992d56ef19f430396e5765b11e0f5bd21a7a557cb12fca1c8c18b9636" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -712,16 +724,16 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.101.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +checksum = "59f4f8065fe615dbed9096458ba98dda6d641553ffd5aedd27e37e65211aca9f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -737,16 +749,16 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.4.2" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +checksum = "b7083fb918b38474ac65ffbf8a69fc8792d36879f4ac5f1667b43aec61efe9a5" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -756,11 +768,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] @@ -800,7 +812,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "percent-encoding", @@ -811,27 +823,21 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" +checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", - "h2 0.4.13", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.8.1", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.7", + "h2 0.4.14", + "http 1.4.1", + "hyper 1.10.1", + "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -841,10 +847,12 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.5" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] @@ -869,20 +877,21 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.3" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -894,33 +903,56 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.6" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "pin-project-lite", "tokio", "tracing", "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.1", +] + [[package]] name = "aws-smithy-types" -version = "1.4.7" +version = "1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +checksum = "53f93074121a1be41317b9aa607143ae17900631f7f59a99f2b905d519d6783b" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -946,13 +978,14 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.14" +version = "1.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "rustc_version", "tracing", @@ -988,19 +1021,19 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core 0.5.6", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.10.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1015,7 +1048,7 @@ dependencies = [ "sha1", "sync_wrapper 1.0.2", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite 0.29.0", "tower 0.5.3", "tower-layer", "tower-service", @@ -1047,7 +1080,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "mime", @@ -1112,7 +1145,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cexpr", "clang-sys", "lazy_static", @@ -1124,7 +1157,7 @@ dependencies = [ "quote", "regex", "rustc-hash 1.1.0", - "shlex", + "shlex 1.3.0", "syn 2.0.117", "which", ] @@ -1152,22 +1185,13 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "bitmaps" version = "3.2.1" @@ -1188,16 +1212,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", ] [[package]] @@ -1209,6 +1233,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1244,17 +1277,17 @@ dependencies = [ "futures-util", "hex", "home", - "http 1.4.0", + "http 1.4.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.10.1", "hyper-named-pipe", - "hyper-rustls 0.27.7", + "hyper-rustls 0.27.9", "hyper-util", "hyperlocal", "log", "pin-project-lite", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-pemfile", "rustls-pki-types", "serde", @@ -1333,9 +1366,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" dependencies = [ "allocator-api2", ] @@ -1474,7 +1507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8144c22e24bbcf26ade86cb6501a0916c46b7e4787abdb0045a467eb1645a1d" dependencies = [ "ambient-authority", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1535,22 +1568,16 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.58" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cexpr" version = "0.6.0" @@ -1664,7 +1691,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", "zeroize", ] @@ -1682,9 +1709,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1704,18 +1731,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.0" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c9f1dde76b736e3681f28cec9d5a61299cbaae0fce80a68e43724ad56031eb" +checksum = "e0a7a9bfdb35811f9e59832f0f05975114d2251b415fb534108e6f34060fd772" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1747,6 +1774,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -1780,9 +1813,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" dependencies = [ "castaway", "cfg-if", @@ -1794,9 +1827,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" dependencies = [ "castaway", "cfg-if", @@ -1808,9 +1841,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", @@ -1819,9 +1852,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1849,6 +1882,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1967,27 +2006,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046d4b584c3bb9b5eb500c8f29549bec36be11000f1ba2a927cef3d1a9875691" +checksum = "008f1a8d1da5074ad858f398775a6d1989031892e46927df5ed18d3be1ed8717" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b194a7870becb1490366fc0ae392ccd188065ff35f8391e77ac659db6fb977" +checksum = "9fd76237df1f4e26edb5ad7971d20280ed1e193331fd257f1b4e4dfefd88dda2" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6a4ab44c6b371e661846b97dab687387a60ac4e2f864e2d4257284aad9e889" +checksum = "380f0bc43e535df6855bbee649efb00bde39c3f33434c47c8e10ac836d21bf47" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -1995,9 +2034,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a44150c2f471a94023482bda1902710746e4bed9f9973d60c5a94319b06d" +checksum = "4811e3e4502de04257e90c0a93225b56d9b85e0f9ad10b81446b415511009610" dependencies = [ "serde", "serde_derive", @@ -2006,9 +2045,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b06598133b1dd76758b8b95f8d6747c124124aade50cea96a3d88b962da9fa" +checksum = "82ffadb34d497f3e76fb3b4baf764c24ba8a51512976a1b77f78bdbf8f4aa687" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -2034,9 +2073,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6190e2e7bcf0a678da2f715363d34ed530fedf7a2f0ab75edaefef72a70465ff" +checksum = "be4f6992eb6faf086ddc7deaaa5f279abfe7f5fd5ae5709bd38253450fc7b945" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -2047,24 +2086,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f583cf203d1aa8b79560e3b01f929bdacf9070b015eec4ea9c46e22a3f83e4a0" +checksum = "70e1b2aad7d055925a4ea9cdbfa9d1d987f9dfc8ad6b708be28f901ac620a298" [[package]] name = "cranelift-control" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803159df35cc398ae54473c150b16d6c77e92ab2948be638488de126a3328fbc" +checksum = "89a355348325e0a63b65c00def3871597b9fcc79d25456397010d16d872b3772" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3109e417257082d88087f5bcce677525bdaa8322b88dd7f175ed1a1fd41d546c" +checksum = "43f4847d93ce2c80d2bff929aa1004dfb3ce2cf5d881f6ced54b8d654d967ba3" dependencies = [ "cranelift-bitset", "serde", @@ -2074,9 +2113,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14db6b0e0e4994c581092df78d837be2072578f7cb2528f96a6cf895e56dee63" +checksum = "ba24e5fe5242cc445e7892ef0a51a4351cf716e3a04ac7a3a05820d056c39818" dependencies = [ "cranelift-codegen", "log", @@ -2086,15 +2125,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec66ea5025c7317383699778282ac98741d68444f956e3b1d7b62f12b7216e67" +checksum = "89bc2035de85c4f04ba7bd57eb5bd3a8b775235bf28852dbf87105115cb8919a" [[package]] name = "cranelift-native" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373ade56438e6232619d85678477d0a88a31b3581936e0503e61e96b546b0800" +checksum = "5ea6630c16921ab087792750f239d0c0173411e80179ca7c0ce0710ce9e7646a" dependencies = [ "cranelift-codegen", "libc", @@ -2103,9 +2142,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.130.1" +version = "0.131.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53619d3cd5c78fd998c6d9420547af26b72e6456f94c2a8a2334cb76b42baa" +checksum = "faa4bbad54fc28cc0da1f9a5d7f7f826ec8cafda3d503b401b2daaaa93c63ef0" [[package]] name = "crc" @@ -2118,9 +2157,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" @@ -2272,7 +2311,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", "mio", "parking_lot", @@ -2288,7 +2327,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", "derive_more 2.1.1", "document-features", @@ -2326,6 +2365,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cssparser" version = "0.36.0" @@ -2358,6 +2406,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -2367,7 +2424,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "serde", @@ -2422,9 +2479,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "date_header" @@ -2531,7 +2588,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "zeroize", @@ -2646,11 +2703,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories-next" version = "2.0.0" @@ -2679,7 +2748,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2699,15 +2768,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -2716,11 +2785,11 @@ dependencies = [ [[package]] name = "docker_credential" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", "serde_json", ] @@ -2788,7 +2857,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -2801,9 +2870,9 @@ checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "email_address" @@ -2887,7 +2956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3017,29 +3086,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fax" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" -dependencies = [ - "fax_derive", -] - -[[package]] -name = "fax_derive" -version = "0.2.0" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" [[package]] name = "fd-lock" @@ -3069,13 +3124,12 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ "cfg-if", "libc", - "libredox", ] [[package]] @@ -3146,9 +3200,9 @@ dependencies = [ [[package]] name = "fraction" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" dependencies = [ "lazy_static", "num", @@ -3283,9 +3337,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -3310,7 +3364,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "debugid", "rustc-hash 2.1.2", "serde", @@ -3345,7 +3399,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" dependencies = [ - "compact_str 0.9.0", + "compact_str 0.9.1", "get-size-derive2", "hashbrown 0.16.1", "ordermap", @@ -3432,7 +3486,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ "fnv", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.14.0", "stable_deref_trait", ] @@ -3491,7 +3545,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.13.0", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3500,17 +3554,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", - "indexmap 2.13.0", + "http 1.4.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3525,7 +3579,7 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", - "zerocopy 0.8.48", + "zerocopy 0.8.50", ] [[package]] @@ -3582,9 +3636,12 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -3642,7 +3699,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -3651,7 +3708,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -3683,7 +3749,7 @@ dependencies = [ "base64 0.22.1", "html-escape", "html5ever 0.39.0", - "lru 0.16.3", + "lru 0.16.4", "once_cell", "regex", "serde", @@ -3701,16 +3767,6 @@ dependencies = [ "markup5ever 0.36.1", ] -[[package]] -name = "html5ever" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" -dependencies = [ - "log", - "markup5ever 0.38.0", -] - [[package]] name = "html5ever" version = "0.39.0" @@ -3734,9 +3790,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -3760,7 +3816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http 1.4.1", ] [[package]] @@ -3771,7 +3827,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "pin-project-lite", ] @@ -3794,6 +3850,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -3820,22 +3885,21 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.14", + "http 1.4.1", "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -3848,7 +3912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.8.1", + "hyper 1.10.1", "hyper-util", "pin-project-lite", "tokio", @@ -3856,21 +3920,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.25.0" @@ -3891,20 +3940,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.8.1", + "http 1.4.1", + "hyper 1.10.1", "hyper-util", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", - "rustls-pki-types", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -3929,14 +3977,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", - "hyper 1.8.1", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -3952,7 +4000,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.8.1", + "hyper 1.10.1", "hyper-util", "pin-project-lite", "tokio", @@ -3985,12 +4033,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -3998,9 +4047,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -4011,9 +4060,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -4025,15 +4074,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -4045,15 +4094,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -4089,28 +4138,14 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", ] -[[package]] -name = "im-rc" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" -dependencies = [ - "bitmaps 2.1.0", - "rand_core 0.6.4", - "rand_xoshiro 0.6.0", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "image" version = "0.25.10" @@ -4132,10 +4167,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fade8ae6828627ad1fa094a891eccfb25150b383047190a3648d66d06186501" dependencies = [ "archery", - "bitmaps 3.2.1", + "bitmaps", "imbl-sized-chunks", "rand_core 0.9.5", - "rand_xoshiro 0.7.0", + "rand_xoshiro", "serde", "version_check", ] @@ -4146,7 +4181,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f4241005618a62f8d57b2febd02510fb96e0137304728543dfc5fd6f052c22d" dependencies = [ - "bitmaps 3.2.1", + "bitmaps", ] [[package]] @@ -4192,12 +4227,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -4223,9 +4258,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.1" +version = "1.47.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99322078b2c076829a1db959d49da554fabc4342257fc0ba5a070a1eb3a01cd8" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" dependencies = [ "console", "once_cell", @@ -4275,29 +4310,17 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "ironclaw" -version = "0.25.0" +version = "0.29.0" dependencies = [ + "aes", "aes-gcm", "agent-client-protocol", "aho-corasick", "anyhow", "async-trait", - "aws-config", - "aws-sdk-bedrockruntime", - "aws-smithy-types", - "axum 0.8.8", + "axum 0.8.9", "base64 0.22.1", "blake3", "bollard", @@ -4321,16 +4344,19 @@ dependencies = [ "glob", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "html-to-markdown-rs", "http-body-util", - "hyper 1.8.1", + "hyper 1.10.1", "hyper-util", "iana-time-zone", "insta", "ironclaw_common", + "ironclaw_embeddings", "ironclaw_engine", "ironclaw_gateway", + "ironclaw_llm", + "ironclaw_oauth", "ironclaw_safety", "ironclaw_skills", "ironclaw_tui", @@ -4338,8 +4364,9 @@ dependencies = [ "jsonschema", "jsonwebtoken", "libsql", - "lru 0.16.3", + "lru 0.16.4", "matrix-sdk", + "md-5 0.10.6", "mime_guess", "open", "pdf-extract", @@ -4347,7 +4374,7 @@ dependencies = [ "postgres-types", "pretty_assertions", "pty-process", - "rand 0.8.5", + "rand 0.8.6", "readabilityrs", "refinery", "regex", @@ -4355,8 +4382,8 @@ dependencies = [ "rig-core", "rust_decimal", "rust_decimal_macros", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustyline", "secrecy", "secret-service", @@ -4365,7 +4392,7 @@ dependencies = [ "serde", "serde_json", "serde_yml", - "sha2", + "sha2 0.10.9", "subtle", "tar", "tempfile", @@ -4381,14 +4408,14 @@ dependencies = [ "tokio-util", "toml 0.8.23", "tower 0.5.3", - "tower-http 0.6.8", + "tower-http 0.6.11", "tracing", "tracing-subscriber", "tracing-test", "url", "urlencoding", "uuid", - "wasmparser 0.245.1", + "wasmparser 0.246.2", "wasmtime", "wasmtime-wasi", "webpki-roots 0.26.11", @@ -4397,73 +4424,442 @@ dependencies = [ ] [[package]] -name = "ironclaw_common" -version = "0.2.0" +name = "ironclaw_approvals" +version = "0.1.0" dependencies = [ - "chrono-tz", - "serde", + "async-trait", + "ironclaw_authorization", + "ironclaw_events", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_run_state", "serde_json", + "tempfile", "thiserror 2.0.18", - "tracing", + "tokio", ] [[package]] -name = "ironclaw_engine" +name = "ironclaw_architecture" +version = "0.1.0" +dependencies = [ + "serde_json", +] + +[[package]] +name = "ironclaw_authorization" version = "0.1.0" dependencies = [ "async-trait", "chrono", - "cron", - "ironclaw_common", - "ironclaw_skills", - "monty", - "pretty_assertions", - "regex", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_trust", "serde", "serde_json", - "sha2", "tempfile", "thiserror 2.0.18", "tokio", - "tracing", - "uuid", ] [[package]] -name = "ironclaw_gateway" +name = "ironclaw_capabilities" version = "0.1.0" dependencies = [ - "serde", + "async-trait", + "chrono", + "ironclaw_approvals", + "ironclaw_authorization", + "ironclaw_dispatcher", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_processes", + "ironclaw_resources", + "ironclaw_run_state", + "ironclaw_trust", "serde_json", "thiserror 2.0.18", + "tokio", "tracing", ] [[package]] -name = "ironclaw_safety" -version = "0.2.1" +name = "ironclaw_common" +version = "0.4.2" dependencies = [ - "aho-corasick", - "criterion", - "regex", + "chrono-tz", + "dirs", + "serde", "serde_json", "thiserror 2.0.18", "tracing", - "url", ] [[package]] -name = "ironclaw_skills" +name = "ironclaw_dispatcher" version = "0.1.0" dependencies = [ - "chrono", - "futures", + "async-trait", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_resources", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-test", +] + +[[package]] +name = "ironclaw_embeddings" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-bedrockruntime", + "aws-smithy-types", + "ironclaw_llm", + "lru 0.16.4", + "reqwest 0.12.28", + "secrecy", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "ironclaw_engine" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "cron", + "ironclaw_common", + "ironclaw_skills", + "monty", + "pretty_assertions", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "ironclaw_events" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "ironclaw_host_api", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "uuid", +] + +[[package]] +name = "ironclaw_extensions" +version = "0.1.0" +dependencies = [ + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_trust", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml 0.8.23", + "url", +] + +[[package]] +name = "ironclaw_filesystem" +version = "0.1.0" +dependencies = [ + "async-trait", + "deadpool-postgres", + "ironclaw_host_api", + "libsql", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "ironclaw_gateway" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "ironclaw_host_api" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "rust_decimal", + "rust_decimal_macros", + "serde", + "serde_json", + "sha2 0.10.9", + "static_assertions", + "thiserror 2.0.18", + "uuid", +] + +[[package]] +name = "ironclaw_host_runtime" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "ironclaw_approvals", + "ironclaw_authorization", + "ironclaw_capabilities", + "ironclaw_dispatcher", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_mcp", + "ironclaw_network", + "ironclaw_processes", + "ironclaw_resources", + "ironclaw_run_state", + "ironclaw_safety", + "ironclaw_scripts", + "ironclaw_secrets", + "ironclaw_trust", + "ironclaw_wasm", + "rust_decimal", + "secrecy", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "wat", + "wit-component 0.245.1", + "wit-parser 0.245.1", +] + +[[package]] +name = "ironclaw_llm" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-bedrockruntime", + "aws-smithy-types", + "base64 0.22.1", + "bytes", + "chrono", + "dirs", + "eventsource-stream", + "futures", + "insta", + "ironclaw_common", + "ironclaw_safety", + "open", + "rand 0.8.6", + "regex", + "reqwest 0.12.28", + "rig-core", + "rust_decimal", + "rust_decimal_macros", + "secrecy", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-test", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "ironclaw_mcp" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_dispatcher", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_resources", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "ironclaw_memory" +version = "0.1.0" +dependencies = [ + "async-trait", + "deadpool-postgres", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_safety", + "jsonschema", + "libsql", + "pgvector", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tokio-postgres", + "tracing", + "uuid", +] + +[[package]] +name = "ironclaw_network" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_host_api", + "reqwest 0.12.28", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "ironclaw_oauth" +version = "0.1.0" +dependencies = [ + "ironclaw_common", + "thiserror 2.0.18", + "tokio", + "urlencoding", +] + +[[package]] +name = "ironclaw_processes" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures", + "ironclaw_events", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_resources", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "ironclaw_resources" +version = "0.1.0" +dependencies = [ + "ironclaw_host_api", + "rust_decimal", + "rust_decimal_macros", + "thiserror 2.0.18", +] + +[[package]] +name = "ironclaw_run_state" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_filesystem", + "ironclaw_host_api", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "ironclaw_safety" +version = "0.2.2" +dependencies = [ + "aho-corasick", + "criterion", + "regex", + "serde_json", + "thiserror 2.0.18", + "tracing", + "url", +] + +[[package]] +name = "ironclaw_scripts" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_dispatcher", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_resources", + "rust_decimal_macros", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "ironclaw_secrets" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_host_api", + "secrecy", + "thiserror 2.0.18", + "tokio", + "uuid", +] + +[[package]] +name = "ironclaw_skills" +version = "0.3.0" +dependencies = [ + "chrono", + "futures", "regex", "reqwest 0.12.28", "serde", "serde_json", "serde_yml", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 2.0.18", "tokio", @@ -4471,6 +4867,18 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "ironclaw_trust" +version = "0.1.0" +dependencies = [ + "chrono", + "ironclaw_host_api", + "serde", + "serde_json", + "static_assertions", + "thiserror 2.0.18", +] + [[package]] name = "ironclaw_tui" version = "0.1.0" @@ -4483,6 +4891,7 @@ dependencies = [ "ratatui", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", @@ -4490,6 +4899,29 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "ironclaw_wasm" +version = "0.1.0" +dependencies = [ + "async-trait", + "ironclaw_dispatcher", + "ironclaw_events", + "ironclaw_extensions", + "ironclaw_filesystem", + "ironclaw_host_api", + "ironclaw_resources", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "wasmtime", + "wasmtime-wasi", + "wat", + "wit-component 0.245.1", + "wit-parser 0.245.1", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -4519,7 +4951,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4617,27 +5049,32 @@ dependencies = [ [[package]] name = "jni" -version = "0.21.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cesu8", "cfg-if", "combine", - "jni-sys 0.3.1", + "jni-macros", + "jni-sys", "log", - "thiserror 1.0.69", + "simd_cesu8", + "thiserror 2.0.18", "walkdir", - "windows-sys 0.45.0", + "windows-link", ] [[package]] -name = "jni-sys" -version = "0.3.1" +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ - "jni-sys 0.4.1", + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", ] [[package]] @@ -4671,9 +5108,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.92" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -4764,17 +5201,17 @@ dependencies = [ [[package]] name = "kuchikikiki" -version = "0.9.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73885c6a3cefdf7a1db0327cefbe4b9b72cac94cae4b19ede4fa492d8af02a0" +checksum = "14683223e533503d404478bfd32826a4cba1b4906034ff74135404372390e87b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crc", "cssparser", - "html5ever 0.38.0", - "indexmap 2.13.0", + "html5ever 0.36.1", + "indexmap 2.14.0", "precomputed-hash", - "selectors 0.35.0", + "selectors", ] [[package]] @@ -4820,9 +5257,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "leb128" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "leb128fmt" @@ -4857,9 +5294,9 @@ checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -4879,14 +5316,11 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ - "bitflags 2.11.0", "libc", - "plain", - "redox_syscall 0.7.3", ] [[package]] @@ -4900,7 +5334,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "bincode", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "fallible-iterator 0.3.0", "futures", @@ -4955,7 +5389,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae65c66088dcd309abbd5617ae046abac2a2ee0a7fdada5127353bd68e0a27ea" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "fallible-iterator 0.2.0", "fallible-streaming-iterator", "hashlink 0.8.4", @@ -4969,10 +5403,10 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15a90128c708356af8f7d767c9ac2946692c9112b4f74f07b99a01a60680e413" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cc", "fallible-iterator 0.3.0", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", "memchr", "phf 0.11.3", @@ -5055,9 +5489,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -5076,9 +5510,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" [[package]] name = "lopdf" @@ -5088,10 +5522,10 @@ checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" dependencies = [ "encoding_rs", "flate2", - "indexmap 2.13.0", + "indexmap 2.14.0", "itoa", "log", - "md-5", + "md-5 0.10.6", "nom", "rangemap", "time", @@ -5109,9 +5543,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", ] @@ -5225,17 +5659,6 @@ dependencies = [ "web_atoms", ] -[[package]] -name = "markup5ever" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" -dependencies = [ - "log", - "tendril 0.5.0", - "web_atoms", -] - [[package]] name = "markup5ever" version = "0.39.0" @@ -5313,9 +5736,9 @@ dependencies = [ "futures-core", "futures-util", "gloo-timers", - "http 1.4.0", + "http 1.4.1", "imbl", - "indexmap 2.13.0", + "indexmap 2.14.0", "itertools 0.14.0", "js_int", "language-tags", @@ -5329,15 +5752,15 @@ dependencies = [ "oauth2-reqwest", "percent-encoding", "pin-project-lite", - "reqwest 0.13.2", + "reqwest 0.13.4", "ruma", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", "serde_html_form", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 2.0.18", "tokio", @@ -5347,7 +5770,7 @@ dependencies = [ "url", "urlencoding", "vodozemac", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", "zeroize", ] @@ -5358,7 +5781,7 @@ source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=211c9a7e05dc108d dependencies = [ "as_variant", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "decancer", "eyeball", "eyeball-im", @@ -5417,7 +5840,7 @@ dependencies = [ "futures-core", "futures-util", "hkdf", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "js_option", "matrix-sdk-common", @@ -5427,7 +5850,7 @@ dependencies = [ "ruma", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "subtle", "thiserror 2.0.18", "time", @@ -5461,7 +5884,7 @@ dependencies = [ "serde", "serde-wasm-bindgen", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tracing", @@ -5507,13 +5930,13 @@ dependencies = [ "blake3", "chacha20poly1305", "getrandom 0.4.2", - "hmac", + "hmac 0.12.1", "pbkdf2", "rand 0.10.1", "rmp-serde", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "zeroize", ] @@ -5570,14 +5993,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "memfd" @@ -5646,9 +6079,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "log", @@ -5666,7 +6099,7 @@ dependencies = [ "chrono", "fancy-regex", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.14.0", "itertools 0.14.0", "jiter", "libm", @@ -5701,7 +6134,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" dependencies = [ - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -5725,7 +6158,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -5738,7 +6171,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -5760,7 +6193,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5805,9 +6238,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -5868,12 +6301,12 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.0", - "rand 0.8.5", + "http 1.4.1", + "rand 0.8.6", "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -5885,7 +6318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234fb5c965bbce983ee5de636a7a51d6a3223da8067ea02f9ab2d2d78ac08be2" dependencies = [ "oauth2", - "reqwest 0.13.2", + "reqwest 0.13.4", ] [[package]] @@ -5903,7 +6336,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-graphics", "objc2-foundation", @@ -5915,7 +6348,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "dispatch2", "objc2", ] @@ -5926,7 +6359,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "dispatch2", "objc2", "objc2-core-foundation", @@ -5945,7 +6378,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -5956,7 +6389,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -5972,13 +6405,13 @@ dependencies = [ [[package]] name = "object" -version = "0.38.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271638cd5fa9cca89c4c304675ca658efc4e64a66c716b7cfe1afb4b9611dbbc" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.16.1", - "indexmap 2.13.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", "memchr", ] @@ -6008,9 +6441,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open" -version = "5.3.3" +version = "5.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb73a7fa3799b198970490a51174027ba0d4ec504b03cd08caf513d40024bc" +checksum = "2fbaa89d2ddc8473c78a3adf69eea8cffa28c483b8e02a971ef31527cd0fc92c" dependencies = [ "is-wsl", "libc", @@ -6056,11 +6489,11 @@ dependencies = [ [[package]] name = "ordermap" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfa78c92071bbd3628c22b1a964f7e0eb201dc1456555db072beb1662ecd6715" +checksum = "7f7476a5b122ff1fce7208e7ee9dccd0a516e835f5b8b19b8f3c98a34cf757c1" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", ] [[package]] @@ -6141,7 +6574,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -6221,7 +6654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6231,14 +6664,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.13.0", + "indexmap 2.14.0", ] [[package]] name = "pgvector" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b" +checksum = "3673cba5b9a124916096a423b806a9f29620972c6c97b08db5f2053e9428b481" dependencies = [ "bytes", "postgres-types", @@ -6300,7 +6733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -6356,18 +6789,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -6409,15 +6842,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "plain" -version = "0.2.3" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plotters" @@ -6453,7 +6880,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crc32fast", "fdeflate", "flate2", @@ -6524,27 +6951,27 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" +checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", - "md-5", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.2", - "sha2", + "rand 0.10.1", + "sha2 0.11.0", "stringprep", ] [[package]] name = "postgres-types" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" +checksum = "8dc729a129e682e8d24170cd30ae1aa01b336b096cbb56df6d534ffec133d186" dependencies = [ "bytes", "chrono", @@ -6563,9 +6990,9 @@ checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -6582,7 +7009,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.48", + "zerocopy 0.8.50", ] [[package]] @@ -6617,7 +7044,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.8+spec-1.1.0", + "toml_edit 0.25.12+spec-1.1.0", ] [[package]] @@ -6701,7 +7128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6743,16 +7170,16 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "memchr", "unicase", ] [[package]] name = "pulley-interpreter" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "010dec3755eb61b2f1051ecb3611b718460b7a74c131e474de2af20a845938af" +checksum = "dff0ead8b4616f81b3d3efd41ce41bcf9ea364a5d8df8be8a8a1f98b50104349" dependencies = [ "cranelift-bitset", "log", @@ -6762,9 +7189,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad360c32e85ca4b083ac0e2b6856e8f11c3d5060dafa7d5dc57b370857fa3018" +checksum = "f4389e5820b1b39810ac12a27aa665320cab3caa51913a79637c06f284cfe223" dependencies = [ "proc-macro2", "quote", @@ -6855,8 +7282,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.2", - "rustls 0.23.38", - "socket2 0.6.3", + "rustls 0.23.40", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -6873,10 +7300,10 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash 2.1.2", - "rustls 0.23.38", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -6894,7 +7321,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] @@ -6960,9 +7387,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -6971,9 +7398,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7034,15 +7461,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -7064,9 +7482,9 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cassowary", - "compact_str 0.8.1", + "compact_str 0.8.2", "crossterm 0.28.1", "indoc", "instability", @@ -7081,9 +7499,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -7101,11 +7519,11 @@ dependencies = [ [[package]] name = "readabilityrs" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb174b0af6c181a87d68b42800806657bfbdf88b566f819aaadb9d2a7b7699d" +checksum = "d90c6e1dad698d9f3c80a8d91bc0efc8c2397cb5ca4bbffb9c0fb88a9a66e6b8" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "kuchikikiki", "once_cell", "regex", @@ -7147,16 +7565,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", -] - -[[package]] -name = "redox_syscall" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" -dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7269,7 +7678,7 @@ checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "log", "rustc-hash 2.1.2", "smallvec", @@ -7328,14 +7737,15 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.14", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.10.1", + "hyper-rustls 0.27.9", "hyper-util", "js-sys", "log", @@ -7344,8 +7754,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", "serde_json", @@ -7355,39 +7765,39 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", - "tower-http 0.6.8", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.14", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.10.1", + "hyper-rustls 0.27.9", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.38", + "rustls 0.23.40", "rustls-pki-types", "rustls-platform-verifier", "sync_wrapper 1.0.2", @@ -7395,7 +7805,7 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", - "tower-http 0.6.8", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -7419,7 +7829,7 @@ dependencies = [ "futures", "futures-timer", "glob", - "http 1.4.0", + "http 1.4.1", "mime", "mime_guess", "nanoid", @@ -7504,8 +7914,8 @@ version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?rev=6ded4bed1651e30b34dd04cdaa50c763036abb0d#6ded4bed1651e30b34dd04cdaa50c763036abb0d" dependencies = [ "aho-corasick", - "bitflags 2.11.0", - "compact_str 0.9.0", + "bitflags 2.11.1", + "compact_str 0.9.1", "get-size2", "is-macro", "memchr", @@ -7521,9 +7931,9 @@ name = "ruff_python_parser" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?rev=6ded4bed1651e30b34dd04cdaa50c763036abb0d#6ded4bed1651e30b34dd04cdaa50c763036abb0d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "bstr", - "compact_str 0.9.0", + "compact_str 0.9.1", "get-size2", "memchr", "ruff_python_ast", @@ -7541,7 +7951,7 @@ name = "ruff_python_stdlib" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?rev=6ded4bed1651e30b34dd04cdaa50c763036abb0d#6ded4bed1651e30b34dd04cdaa50c763036abb0d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "unicode-ident", ] @@ -7596,7 +8006,7 @@ dependencies = [ "as_variant", "assign", "bytes", - "http 1.4.0", + "http 1.4.1", "js_int", "js_option", "maplit", @@ -7621,12 +8031,12 @@ dependencies = [ "date_header", "form_urlencoded", "getrandom 0.2.17", - "http 1.4.0", - "indexmap 2.13.0", + "http 1.4.1", + "indexmap 2.14.0", "js_int", "konst", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "regex", "ruma-identifiers-validation", "ruma-macros", @@ -7649,7 +8059,7 @@ version = "0.32.1" source = "git+https://github.com/ruma/ruma?rev=7680eebd9586669e1a4e5b1fd1c2c691221369d4#7680eebd9586669e1a4e5b1fd1c2c691221369d4" dependencies = [ "as_variant", - "indexmap 2.13.0", + "indexmap 2.14.0", "js_int", "js_option", "ruma-common", @@ -7705,7 +8115,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink 0.10.0", @@ -7715,16 +8125,16 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" dependencies = [ "arrayvec", "borsh", "bytes", "num-traits", "postgres-types", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", @@ -7774,7 +8184,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -7787,11 +8197,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7804,18 +8214,6 @@ dependencies = [ "rustix 1.1.4", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.22.4" @@ -7832,16 +8230,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.10", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] @@ -7861,9 +8259,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe 0.2.1", "rustls-pki-types", @@ -7882,9 +8280,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -7892,41 +8290,31 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", "once_cell", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-platform-verifier-android", - "rustls-webpki 0.103.10", + "rustls-webpki 0.103.13", "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", + "windows-sys 0.61.2", ] +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -7940,9 +8328,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -7962,7 +8350,7 @@ version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "clipboard-win", "fd-lock", @@ -8068,20 +8456,10 @@ dependencies = [ "getopts", "html5ever 0.36.1", "precomputed-hash", - "selectors 0.33.0", + "selectors", "tendril 0.4.3", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "seahash" version = "4.1.0" @@ -8122,9 +8500,9 @@ dependencies = [ "hkdf", "num", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", - "sha2", + "sha2 0.10.9", "zbus", ] @@ -8134,7 +8512,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -8147,7 +8525,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -8170,26 +8548,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7" dependencies = [ - "bitflags 2.11.0", - "cssparser", - "derive_more 2.1.1", - "log", - "new_debug_unreachable", - "phf 0.13.1", - "phf_codegen 0.13.1", - "precomputed-hash", - "rustc-hash 2.1.2", - "servo_arc", - "smallvec", -] - -[[package]] -name = "selectors" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2" -dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cssparser", "derive_more 2.1.1", "log", @@ -8204,9 +8563,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -8281,7 +8640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2f2d7ff8a2140333718bb329f5c40fc5f0865b84c426183ce14c97d2ab8154f" dependencies = [ "form_urlencoded", - "indexmap 2.13.0", + "indexmap 2.14.0", "itoa", "ryu", "serde_core", @@ -8289,9 +8648,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -8354,15 +8713,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.18.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -8373,9 +8733,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.18.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" dependencies = [ "darling", "proc-macro2", @@ -8383,26 +8743,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.13.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "serde_yml" version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", "itoa", "libyml", "memchr", @@ -8428,7 +8775,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -8445,7 +8792,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -8463,6 +8821,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook" version = "0.3.18" @@ -8509,6 +8873,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -8535,19 +8909,9 @@ dependencies = [ [[package]] name = "siphasher" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" - -[[package]] -name = "sized-chunks" -version = "0.6.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps 2.1.0", - "typenum", -] +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" @@ -8576,12 +8940,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8820,7 +9184,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -8841,7 +9205,7 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cap-fs-ext", "cap-std", "fd-lock", @@ -8859,9 +9223,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -8884,7 +9248,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9076,9 +9440,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -9132,9 +9496,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -9142,7 +9506,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2 0.6.4", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -9160,9 +9524,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -9171,9 +9535,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42" +checksum = "4dd8df5ef180f6364759a6f00f7aadda4fbbac86cdee37480826a6ff9f3574ce" dependencies = [ "async-trait", "byteorder", @@ -9188,8 +9552,8 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", - "socket2 0.6.3", + "rand 0.10.1", + "socket2 0.6.4", "tokio", "tokio-util", "whoami", @@ -9201,25 +9565,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144" dependencies = [ - "const-oid", + "const-oid 0.9.6", "ring", - "rustls 0.23.38", + "rustls 0.23.40", "tokio", "tokio-postgres", "tokio-rustls 0.26.4", "x509-cert", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.25.0" @@ -9237,7 +9591,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.38", + "rustls 0.23.40", "tokio", ] @@ -9287,8 +9641,8 @@ checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", - "rustls 0.23.38", - "rustls-native-certs 0.8.3", + "rustls 0.23.40", + "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -9297,14 +9651,14 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.28.0", + "tungstenite 0.29.0", ] [[package]] @@ -9339,7 +9693,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", "serde_core", "serde_spanned 1.1.1", "toml_datetime 0.7.5+spec-1.1.0", @@ -9358,7 +9712,7 @@ dependencies = [ "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.0", + "winnow 1.0.3", ] [[package]] @@ -9394,7 +9748,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -9404,14 +9758,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.8+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.0", + "winnow 1.0.3", ] [[package]] @@ -9420,7 +9774,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.0", + "winnow 1.0.3", ] [[package]] @@ -9493,7 +9847,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -9524,7 +9878,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", @@ -9540,19 +9894,18 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "iri-string", "pin-project-lite", "tokio", "tokio-util", @@ -9560,6 +9913,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -9707,11 +10061,11 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.1", "httparse", "log", - "rand 0.9.2", - "rustls 0.23.38", + "rand 0.9.4", + "rustls 0.23.40", "rustls-pki-types", "sha1", "thiserror 2.0.18", @@ -9720,19 +10074,18 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.1", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "sha1", "thiserror 2.0.18", - "utf-8", ] [[package]] @@ -9746,15 +10099,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typewit" -version = "1.15.0" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06fee3a8df48c50c55ad646a4e03b00a370da6fe1850ebf467a8d0165dfcafae" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" [[package]] name = "ucd-trie" @@ -9770,7 +10123,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9779,7 +10132,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", "web-time", ] @@ -9833,9 +10186,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-truncate" @@ -9885,7 +10238,7 @@ dependencies = [ "getopts", "log", "phf_codegen 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -9894,16 +10247,10 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -9955,9 +10302,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -10016,14 +10363,14 @@ dependencies = [ "ed25519-dalek", "getrandom 0.2.17", "hkdf", - "hmac", + "hmac 0.12.1", "matrix-pickle", "prost 0.14.3", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_bytes", "serde_json", - "sha2", + "sha2 0.10.9", "subtle", "thiserror 2.0.18", "x25519-dalek", @@ -10072,11 +10419,11 @@ dependencies = [ [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -10085,7 +10432,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] @@ -10099,9 +10446,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.115" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -10113,9 +10460,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.65" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -10123,9 +10470,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.115" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10133,9 +10480,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.115" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -10146,31 +10493,27 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.115" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-compose" -version = "0.245.1" +version = "0.246.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd23d12cc95c451c1306db5bc63075fbebb612bb70c53b4237b1ce5bc178343" +checksum = "f05a2b3bad87cc1ce45b63425ec09a854cc4cb369231c9fed1fee31538103efb" dependencies = [ "anyhow", "heck", - "im-rc", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", "petgraph", - "serde", - "serde_derive", - "serde_yaml", "smallvec", - "wasm-encoder 0.245.1", - "wasmparser 0.245.1", + "wasm-encoder 0.246.2", + "wasmparser 0.246.2", "wat", ] @@ -10194,6 +10537,26 @@ dependencies = [ "wasmparser 0.245.1", ] +[[package]] +name = "wasm-encoder" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +dependencies = [ + "leb128fmt", + "wasmparser 0.246.2", +] + +[[package]] +name = "wasm-encoder" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a879a421bd17c528b74721b2abf4c62e8f1d1889c2ba8c3c50d02deaf2ce395" +dependencies = [ + "leb128fmt", + "wasmparser 0.251.0", +] + [[package]] name = "wasm-metadata" version = "0.244.0" @@ -10201,11 +10564,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.14.0", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] +[[package]] +name = "wasm-metadata" +version = "0.245.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da55e60097e8b37b475a0fa35c3420dd71d9eb7bd66109978ab55faf56a57efb" +dependencies = [ + "anyhow", + "indexmap 2.14.0", + "wasm-encoder 0.245.1", + "wasmparser 0.245.1", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -10256,9 +10631,9 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.14.0", "semver", ] @@ -10268,33 +10643,56 @@ version = "0.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f08c9adee0428b7bddf3890fc27e015ac4b761cc608c822667102b8bfd6995e" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", + "hashbrown 0.16.1", + "indexmap 2.14.0", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +dependencies = [ + "bitflags 2.11.1", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.14.0", "semver", "serde", ] +[[package]] +name = "wasmparser" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "437970b35b1a85cfde9c74b2398352d8d653f3bd8e3a3db0c063ea8f5b4b36ff" +dependencies = [ + "bitflags 2.11.1", + "indexmap 2.14.0", + "semver", +] + [[package]] name = "wasmprinter" -version = "0.245.1" +version = "0.246.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41517a3716fbb8ccf46daa9c1325f760fcbff5168e75c7392288e410b91ac8" +checksum = "6e41f7493ba994b8a779430a4c25ff550fd5a40d291693af43a6ef48688f00e3" dependencies = [ "anyhow", "termcolor", - "wasmparser 0.245.1", + "wasmparser 0.246.2", ] [[package]] name = "wasmtime" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce205cd643d661b5ba5ba4717e13730262e8cdbc8f2eacbc7b906d45c1a74026" +checksum = "af4eccc0728f061979efa8ff4c962cff7041fead4baadb74973f01b9c47158a4" dependencies = [ "addr2line", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bumpalo", "cc", "cfg-if", @@ -10321,8 +10719,8 @@ dependencies = [ "target-lexicon", "tempfile", "wasm-compose", - "wasm-encoder 0.245.1", - "wasmparser 0.245.1", + "wasm-encoder 0.246.2", + "wasmparser 0.246.2", "wasmtime-environ", "wasmtime-internal-cache", "wasmtime-internal-component-macro", @@ -10341,9 +10739,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8b78abf3677d4a0a5db82e5015b4d085ff3a1b8b472cbb8c70d4b769f019ce" +checksum = "7e84dbe3208c1336a41546beb75927b3b37e2e4fce06653d214b407136fbe295" dependencies = [ "anyhow", "cpp_demangle", @@ -10352,7 +10750,7 @@ dependencies = [ "cranelift-entity", "gimli", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", "object", "postcard", @@ -10360,11 +10758,11 @@ dependencies = [ "semver", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "smallvec", "target-lexicon", - "wasm-encoder 0.245.1", - "wasmparser 0.245.1", + "wasm-encoder 0.246.2", + "wasmparser 0.246.2", "wasmprinter", "wasmtime-internal-component-util", "wasmtime-internal-core", @@ -10372,9 +10770,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cache" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4fd4103ba413c0da2e636f73490c6c8e446d708cbde7573703941bc3d6a448" +checksum = "910b8dcadc0888344b2dea5a087c836b58156d4f455c52b6dac0bdc776a9d029" dependencies = [ "base64 0.22.1", "directories-next", @@ -10383,7 +10781,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "toml 0.9.12+spec-1.1.0", "wasmtime-environ", "windows-sys 0.61.2", @@ -10392,9 +10790,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d6914f34be2f9d78d8ee9f422e834dfc204e71ccce697205fae95fed87892" +checksum = "c223bd503db76df8d74d1fcca39e734d25f7a0c1dcaf1509b67f3855d1b0f803" dependencies = [ "anyhow", "proc-macro2", @@ -10402,20 +10800,20 @@ dependencies = [ "syn 2.0.117", "wasmtime-internal-component-util", "wasmtime-internal-wit-bindgen", - "wit-parser 0.245.1", + "wit-parser 0.246.2", ] [[package]] name = "wasmtime-internal-component-util" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3751b0616b914fdd87fe1bf804694a078f321b000338e6476bc48a4d6e454f21" +checksum = "ab123ad511483a1b918399789d0cc7dea7c5c6476743df73949007b5b225fc74" [[package]] name = "wasmtime-internal-core" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22632b187e1b0716f1b9ac57ad29013bed33175fcb19e10bb6896126f82fac67" +checksum = "4364d345719bba7fc4c435992ea1cb0c118f1e90a88c6e6f22a7a4fc507700c6" dependencies = [ "anyhow", "hashbrown 0.16.1", @@ -10425,9 +10823,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3ca07b3e0bb3429674b173b5800577719d600774dd81bff58f775c0aaa64ee" +checksum = "c5a3bc28a172037c7864128bb208017a02bba659a59c27acacc048c09e25c1fc" dependencies = [ "cfg-if", "cranelift-codegen", @@ -10443,7 +10841,7 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror 2.0.18", - "wasmparser 0.245.1", + "wasmparser 0.246.2", "wasmtime-environ", "wasmtime-internal-core", "wasmtime-internal-unwinder", @@ -10452,9 +10850,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c8b2c9704eb1f33ead025ec16038277ccb63d0a14c31e99d5b765d7c36da55" +checksum = "3c90a899a47d3da6e384e7b4cad61fdcb27535a395742b32440bdf9980ea83fa" dependencies = [ "cc", "cfg-if", @@ -10467,9 +10865,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d950310d07391d34369f62c48336ebb14eacbd4d6f772bb5f349c24e838e0664" +checksum = "84f364747aa74c686b18925918e5cfd615a73c9613c7a31fc1cd86f42df12fbe" dependencies = [ "cc", "object", @@ -10479,9 +10877,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3606662c156962d096be3127b8b8ae8ee2f8be3f896dad29259ff01ddb64abfd" +checksum = "c3ba98c1492f530833e0d3cc17dbb0c3c57c9f1bb3b078ae44bb55a233e43eba" dependencies = [ "cfg-if", "libc", @@ -10491,9 +10889,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75eef0747e52dc545b075f64fd0e0cc237ae738e641266b1970e07e2d744bc32" +checksum = "94b8f8a89e8f3660646f820c7d8310a67094156bb866e9d56f1b00892e011206" dependencies = [ "cfg-if", "cranelift-codegen", @@ -10504,9 +10902,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b0a5dab02a8fb527f547855ecc0e05f9fdc3d5bd57b8b080349408f9a6cece" +checksum = "7a12754f1ffc4a3300d56d324c418b8b32cf029606618da22c7d076213882a3f" dependencies = [ "proc-macro2", "quote", @@ -10515,16 +10913,16 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8007342bd12ff400293a817973f7ecd6f1d9a8549a53369a9c1af357166f1f1e" +checksum = "4b06e4ed07adc579645e5c55c67b3138c49da2e468fad52d3db7b7a098ecc733" dependencies = [ "cranelift-codegen", "gimli", "log", "object", "target-lexicon", - "wasmparser 0.245.1", + "wasmparser 0.246.2", "wasmtime-environ", "wasmtime-internal-cranelift", "winch-codegen", @@ -10532,25 +10930,25 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7900c3e3c1d6e475bc225d73b02d6d5484815f260022e6964dca9558e50dd01a" +checksum = "0f08787948e3c983799d616ef7dd57463253e9ca8bab6607eef8134f12353f70" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "heck", - "indexmap 2.13.0", - "wit-parser 0.245.1", + "indexmap 2.14.0", + "wit-parser 0.246.2", ] [[package]] name = "wasmtime-wasi" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3e3ddcfad69e9eb025bd19bff70dad45bafe1d6eacd134c0ffdfc4c161d045" +checksum = "1b2f19834bc6edbc31ac95fdcfd5ddcd7643759265a1d545dec36ac6cc788ca8" dependencies = [ "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "cap-fs-ext", "cap-net-ext", @@ -10575,9 +10973,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ca5dd3b9f04a851c422d05f333366722742da46bff9369ae0191f32cf83565a" +checksum = "c3e0c6efdbaf90906016be9ed9ff17b7b58f393876287beebe5bd7fa1de54dbb" dependencies = [ "async-trait", "bytes", @@ -10597,31 +10995,31 @@ dependencies = [ [[package]] name = "wast" -version = "245.0.1" +version = "251.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cf1149285569120b8ce39db8b465e8a2b55c34cbb586bd977e43e2bc7300bf" +checksum = "5cc7467dda0a96142eb2c980329dfb62480b1e1d3622fdeb1a44e2bca6ceed74" dependencies = [ "bumpalo", "leb128fmt", "memchr", "unicode-width 0.2.0", - "wasm-encoder 0.245.1", + "wasm-encoder 0.251.0", ] [[package]] name = "wat" -version = "1.245.1" +version = "1.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd48d1679b6858988cb96b154dda0ec5bbb09275b71db46057be37332d5477be" +checksum = "81b1086c9e85b95bd6a229a928bc6c6d0662e42af0250c88d067b418831ea4d4" dependencies = [ - "wast 245.0.1", + "wast 251.0.0", ] [[package]] name = "web-sys" -version = "0.3.92" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -10640,9 +11038,9 @@ dependencies = [ [[package]] name = "web_atoms" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" +checksum = "d7cff6eef815df1834fd250e3a2ff436044d82a9f1bc1980ca1dbdf07effc538" dependencies = [ "phf 0.13.1", "phf_codegen 0.13.1", @@ -10665,14 +11063,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -10697,9 +11095,9 @@ dependencies = [ [[package]] name = "whoami" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" dependencies = [ "libc", "libredox", @@ -10710,11 +11108,11 @@ dependencies = [ [[package]] name = "wiggle" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1b1135efc8e5a008971897bea8d41ca56d8d501d4efb807842ae0a1c78f639" +checksum = "17b644ab90da80bbca28973192978ac452cbd876955bb209e6ff2cd1955e43a7" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "thiserror 2.0.18", "tracing", "wasmtime", @@ -10724,9 +11122,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7bc2b0d50ec8773b44fbfe1da6cb5cc44a92deaf8483233dcf0831e6db33172" +checksum = "521f9d558365357274d960340eb9eb4f4d768fafdc79f381fd2e13a85b925ebc" dependencies = [ "heck", "proc-macro2", @@ -10738,9 +11136,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6c7d44ea552e1fbfdcd7a2cd83f5c2d1e803d5b1a11e3462c06888b77f455f" +checksum = "8a386e86021363c9f0abd1e189e8f8a729d9b5aab2bb7172a3e40f2ab647a936" dependencies = [ "proc-macro2", "quote", @@ -10776,7 +11174,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10787,9 +11185,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "43.0.1" +version = "44.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f45f7172a2628c8317766e427babc0a400f9d10b1c0f0b0617c5ed5b79de6" +checksum = "f16496e92d2b232f9d195ae74f71a674aabae7b7fa722d39068836723d3b653c" dependencies = [ "cranelift-assembler-x64", "cranelift-codegen", @@ -10798,7 +11196,7 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror 2.0.18", - "wasmparser 0.245.1", + "wasmparser 0.246.2", "wasmtime-environ", "wasmtime-internal-core", "wasmtime-internal-cranelift", @@ -10874,15 +11272,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -10928,21 +11317,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -10991,12 +11365,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -11015,12 +11383,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -11039,12 +11401,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -11075,12 +11431,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -11099,12 +11449,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -11123,12 +11467,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -11147,12 +11485,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -11182,9 +11514,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -11195,7 +11527,7 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "windows-sys 0.59.0", ] @@ -11208,6 +11540,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -11227,12 +11565,12 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", - "wasm-metadata", + "wasm-metadata 0.244.0", "wit-bindgen-core", - "wit-component", + "wit-component 0.244.0", ] [[package]] @@ -11257,18 +11595,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.0", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", "serde_json", "wasm-encoder 0.244.0", - "wasm-metadata", + "wasm-metadata 0.244.0", "wasmparser 0.244.0", "wit-parser 0.244.0", ] +[[package]] +name = "wit-component" +version = "0.245.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4894f10d2d5cbc17c77e91f86a1e48e191a788da4425293b55c98b44ba3fcac9" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.245.1", + "wasm-metadata 0.245.1", + "wasmparser 0.245.1", + "wit-parser 0.245.1", +] + [[package]] name = "wit-parser" version = "0.244.0" @@ -11277,7 +11634,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", "semver", "serde", @@ -11296,7 +11653,7 @@ dependencies = [ "anyhow", "hashbrown 0.16.1", "id-arena", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", "semver", "serde", @@ -11306,6 +11663,25 @@ dependencies = [ "wasmparser 0.245.1", ] +[[package]] +name = "wit-parser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd979042b5ff288607ccf3b314145435453f20fc67173195f91062d2289b204d" +dependencies = [ + "anyhow", + "hashbrown 0.16.1", + "id-arena", + "indexmap 2.14.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.246.2", +] + [[package]] name = "witx" version = "0.9.1" @@ -11320,9 +11696,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -11368,7 +11744,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "tls_codec", @@ -11414,9 +11790,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -11425,9 +11801,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -11459,7 +11835,7 @@ dependencies = [ "hex", "nix 0.29.0", "ordered-stream", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_repr", "sha1", @@ -11510,11 +11886,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ - "zerocopy-derive 0.8.48", + "zerocopy-derive 0.8.50", ] [[package]] @@ -11530,9 +11906,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", @@ -11541,18 +11917,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -11582,9 +11958,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -11593,9 +11969,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -11604,9 +11980,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -11624,7 +12000,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap 2.13.0", + "indexmap 2.14.0", "memchr", "thiserror 2.0.18", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index 731d997f89c..1c92832b1c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,13 @@ [workspace] -members = [".", "crates/ironclaw_common", "crates/ironclaw_host_api", "crates/ironclaw_filesystem", "crates/ironclaw_memory", "crates/ironclaw_events", "crates/ironclaw_extensions", "crates/ironclaw_processes", "crates/ironclaw_dispatcher", "crates/ironclaw_scripts", "crates/ironclaw_mcp", "crates/ironclaw_wasm", "crates/ironclaw_capabilities", "crates/ironclaw_secrets", "crates/ironclaw_network", "crates/ironclaw_host_runtime", "crates/ironclaw_authorization", "crates/ironclaw_run_state", "crates/ironclaw_approvals", "crates/ironclaw_resources", "crates/ironclaw_trust", "crates/ironclaw_architecture", "crates/ironclaw_safety", "crates/ironclaw_skills", "crates/ironclaw_engine", "crates/ironclaw_gateway", "crates/ironclaw_tui"] +members = [".", "crates/ironclaw_common", "crates/ironclaw_host_api", "crates/ironclaw_filesystem", "crates/ironclaw_memory", "crates/ironclaw_events", "crates/ironclaw_extensions", "crates/ironclaw_processes", "crates/ironclaw_dispatcher", "crates/ironclaw_scripts", "crates/ironclaw_mcp", "crates/ironclaw_wasm", "crates/ironclaw_capabilities", "crates/ironclaw_secrets", "crates/ironclaw_network", "crates/ironclaw_host_runtime", "crates/ironclaw_authorization", "crates/ironclaw_run_state", "crates/ironclaw_approvals", "crates/ironclaw_resources", "crates/ironclaw_trust", "crates/ironclaw_architecture", "crates/ironclaw_safety", "crates/ironclaw_skills", "crates/ironclaw_oauth", "crates/ironclaw_llm", "crates/ironclaw_embeddings", "crates/ironclaw_engine", "crates/ironclaw_gateway", "crates/ironclaw_tui"] exclude = [ "channels-src/discord", + "channels-src/feishu", "channels-src/telegram", "channels-src/slack", + "channels-src/wechat", "channels-src/whatsapp", + "crates/ironclaw_silk_decoder", "tools-src/composio", "tools-src/github", "tools-src/gmail", @@ -21,7 +24,7 @@ exclude = [ [package] name = "ironclaw" -version = "0.27.0" +version = "0.29.0" edition = "2024" rust-version = "1.92" description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly" @@ -113,11 +116,14 @@ cron = "0.13" jsonschema = { version = "0.45", default-features = false } # Shared types -ironclaw_common = { path = "crates/ironclaw_common", version = "0.4.0" } +ironclaw_common = { path = "crates/ironclaw_common", version = "0.4.2" } # Safety/sanitization ironclaw_engine = { path = "crates/ironclaw_engine", version = "0.1.0" } ironclaw_gateway = { path = "crates/ironclaw_gateway", version = "0.1.0" } +ironclaw_llm = { path = "crates/ironclaw_llm", version = "0.1.0" } +ironclaw_embeddings = { path = "crates/ironclaw_embeddings", version = "0.1.0" } +ironclaw_oauth = { path = "crates/ironclaw_oauth", version = "0.1.0" } ironclaw_safety = { path = "crates/ironclaw_safety", version = "0.2.2" } ironclaw_skills = { path = "crates/ironclaw_skills", version = "0.3.0" } ironclaw_tui = { path = "crates/ironclaw_tui", optional = true, version = "0.1.0" } @@ -148,7 +154,7 @@ urlencoding = "2" # # NOTE: This pilot tracks a pinned matrix-rust-sdk commit until the released SDK line includes # the vodozemac contributory-behavior checks and the recursion-limit fix needed for this feature set. -matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk", rev = "211c9a7e05dc108d235a6c8301f503f4f77fa028", optional = true, default-features = false, features = [ +matrix-sdk = { version = "0.16.0", git = "https://github.com/matrix-org/matrix-rust-sdk", rev = "211c9a7e05dc108d235a6c8301f503f4f77fa028", optional = true, default-features = false, features = [ "sqlite", "e2e-encryption", ] } @@ -161,14 +167,16 @@ open = "5" pgvector = { version = "0.4", features = ["postgres"], optional = true } # WASM sandbox for untrusted tool execution -wasmtime = { version = "43.0.2", features = ["component-model"] } -wasmtime-wasi = "43.0.2" # WASI support for component model -wasmparser = "0.245.1" # WASM binary parsing for validation +wasmtime = { version = "44.0.2", features = ["component-model"] } +wasmtime-wasi = "44.0.2" # WASI support for component model +wasmparser = "0.246.2" # WASM binary parsing for validation # Cryptography for secrets management +aes = "0.8" aes-gcm = "0.10" hkdf = "0.12" hmac = "0.12" +md-5 = "0.10" sha2 = "0.10" blake3 = "1" rand = "0.8" @@ -177,10 +185,9 @@ subtle = "2" # Constant-time comparisons for token validation # Multi-provider LLM support rig-core = { version = "0.30", default-features = false, features = ["reqwest-rustls"] } -# AWS Bedrock (native Converse API, opt-in via --features bedrock) -aws-config = { version = "1", features = ["behavior-version-latest"], optional = true } -aws-sdk-bedrockruntime = { version = "1", optional = true } -aws-smithy-types = { version = "1", optional = true } +# AWS Bedrock: deps now live in `ironclaw_llm` (LLM provider) and +# `ironclaw_embeddings` (Titan embeddings); the top-level `bedrock` feature +# (below) re-exposes both via crate-feature forwarding. # Docker sandbox bollard = "0.18" @@ -238,6 +245,11 @@ testcontainers-modules = { version = "0.11", features = ["postgres"] } pretty_assertions = "1" tempfile = "3" insta = { version = "1.46.3", features = ["yaml"] } +# Pull the LLM crate's `testing` feature in only for tests so `StubLlm` and +# fault-injection helpers don't ship in release binaries. +ironclaw_llm = { path = "crates/ironclaw_llm", version = "0.1.0", features = ["testing"] } +# Same idea for embeddings: `MockEmbeddings` is only reachable in dev builds. +ironclaw_embeddings = { path = "crates/ironclaw_embeddings", version = "0.1.0", features = ["testing"] } [features] default = ["postgres", "libsql", "html-to-markdown", "tui", "matrix-e2ee"] @@ -262,7 +274,7 @@ integration = [] # test rig can spin up its embedded database. replay = ["libsql"] html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"] -bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"] +bedrock = ["ironclaw_llm/bedrock", "ironclaw_embeddings/bedrock"] tui = ["dep:ironclaw_tui"] import = ["dep:json5", "libsql"] # matrix-sdk-channel: use the matrix-sdk event loop for the native Matrix diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index c7250d1c02b..d0fd05c20a5 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -11,7 +11,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - 🚫 Out of scope (intentionally skipped) - ➖ N/A (not applicable to Rust implementation) -**Last reviewed against OpenClaw PRs:** 2026-03-10 (merged 2026-02-24 through 2026-03-10) +**Last reviewed against OpenClaw PRs:** 2026-05-02 (merged 2026-03-11 through 2026-04-30, OpenClaw releases 2026.3.11 → 2026.4.30) --- @@ -51,10 +51,18 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Agent event broadcast | ✅ | 🚧 | SSE broadcast manager exists (SseManager) but tool/job-state events not fully wired | | Channel health monitor | ✅ | ❌ | Auto-restart with configurable interval | | Presence system | ✅ | ❌ | Beacons on connect, system presence for agents | -| Trusted-proxy auth mode | ✅ | ❌ | Header-based auth for reverse proxies | -| APNs push pipeline | ✅ | ❌ | Wake disconnected iOS nodes via push | +| Trusted-proxy auth mode | ✅ | ❌ | Header-based auth for reverse proxies; `trustedProxy.allowLoopback` for same-host reverse proxies | +| APNs push pipeline | ✅ | ❌ | Wake disconnected iOS nodes via push; iOS push relay with App Attest verification | | Oversized payload guard | ✅ | 🚧 | HTTP webhook has 64KB body limit + Content-Length check; no chat.history cap | | Pre-prompt context diagnostics | ✅ | 🚧 | Token breakdown logged before LLM call (conversational dispatcher path); other LLM entry points not yet covered | +| OpenAI-compat `/v1/models`, `/v1/embeddings` | ✅ | ❌ | Discovery + embeddings on top of `/v1/chat/completions` | +| Outbound proxy routing | ✅ | ❌ | `proxy.enabled` + `proxy.proxyUrl`/`OPENCLAW_PROXY_URL` with strict http forward-proxy validation, loopback bypass; `openclaw proxy validate` | +| Diagnostics export bundle | ✅ | ❌ | Sanitized logs/status/health/config/stability snapshots for bug reports | +| Startup diagnostics timeline | ✅ | ❌ | Opt-in lifecycle/plugin-load phase tracing | +| Event-loop readiness in `/readyz` | ✅ | ❌ | Event-loop delay (p99/max), utilization, CPU ratio, `degraded` flag | +| OpenTelemetry exporter pipeline | ✅ | ❌ | Bundled `diagnostics-otel` plugin: model-call, tool, exec, outbound, context-assembly, memory pressure, harness lifecycle spans/metrics; W3C traceparent propagation; signal-specific OTLP endpoints | +| Prometheus exporter | ✅ | ❌ | Bundled `diagnostics-prometheus` plugin with protected scrape route | +| Stability snapshots / payload-free liveness | ✅ | ❌ | Default-on stability recording, event-loop delay/CPU snapshots in stability bundles | ### Owner: _Unassigned_ @@ -76,28 +84,38 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | iMessage | ✅ | ❌ | P3 | BlueBubbles or Linq recommended | | Linq | ✅ | ❌ | P3 | Real iMessage via API, no Mac required | | Feishu/Lark | ✅ | 🚧 | P3 | WASM channel with Event Subscription v2.0; Bitable/Docx tools planned | +| WeCom | ✅ | 🚧 | P2 | Standalone WASM channel focused on WeCom intelligent bot WebSocket inbound/outbound, pairing, group sessions, inbound media hydration, and direct Bot media upload/send; self-built app callback + Agent API deferred | | LINE | ✅ | ❌ | P3 | | +| WeChat (iLink bot) | ✅ | 🚧 | P2 | Extension-first channel (`channels-src/wechat`), single-account DM flow with QR login, typing, image send/receive, inbound file/voice/video handling, outbound image/video/file media, and SILK-to-WAV voice fallback; multi-account remains deferred | | WebChat | ✅ | ✅ | - | Web gateway chat | | Matrix | ✅ | 🚧 | P3 | Native channel (SDK + poll fallback); E2EE via `matrix-e2ee` feature gate; WASM channel also available | | Mattermost | ✅ | ❌ | P3 | Emoji reactions, interactive buttons, model picker | | Google Chat | ✅ | ❌ | P3 | | | MS Teams | ✅ | ❌ | P3 | | | Twitch | ✅ | ❌ | P3 | | -| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx, stale call reaper, pre-cached greeting | +| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx/Plivo, stale call reaper, `voicecall setup`/`smoke`, `openclaw_agent_consult` realtime tool, agent-scoped voice agents, dedicated STT/TTS providers (Deepgram, ElevenLabs, Mistral, OpenAI/xAI realtime) | +| Google Meet | ✅ | ❌ | P3 | Bundled participant plugin: Google OAuth, explicit URL joins, Chrome+Twilio realtime transports, paired chrome-node support, attendance/artifact exports, calendar-backed exports, `googlemeet doctor` | +| Yuanbao (Tencent) | ✅ | ❌ | P3 | External plugin (`openclaw-plugin-yuanbao`) for WebSocket bot DMs and group chats | +| WeCom | ✅ | ❌ | P3 | Official external plugin pinned to npm release | | Nostr | ✅ | ❌ | P3 | | ### Telegram-Specific Features (since Feb 2025) | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Forum topic creation | ✅ | ❌ | Create topics in forum groups | +| Forum topic creation | ✅ | ❌ | Create topics in forum groups; `message thread create` CLI; learns human topic names from service messages | | channel_post support | ✅ | ❌ | Bot-to-bot communication | | User message reactions | ✅ | ❌ | Surface inbound reactions | | sendPoll | ✅ | ❌ | Poll creation via agent | -| Cron/heartbeat topic targeting | ✅ | ❌ | Messages land in correct topic | +| Cron/heartbeat topic targeting | ✅ | ❌ | Messages land in correct topic; cron `--thread-id`, explicit `:topic:` precedence | | DM topics support | ✅ | ❌ | Agent/topic bindings in DMs and agent-scoped SessionKeys | | Persistent ACP topic binding | ✅ | ❌ | ACP harness sessions can pin to Telegram forum or DM topics | | sendVoice (voice note replies) | ✅ | ✅ | audio/ogg attachments sent as voice notes; prerequisite for TTS (#90) | +| Native quote replies + retry | ✅ | ❌ | `reply_parameters.quote` with fallback when `QUOTE_TEXT_INVALID` | +| Polling stall watchdog + liveness | ✅ | ❌ | Configurable `pollingStallThresholdMs`, status/doctor warnings, dedicated `getUpdates` confirmation | +| HTML mode + chunking | ✅ | ❌ | Long HTML messages chunked, plain-text fallback | +| Photo dimension preflight | ✅ | ❌ | Falls back to document send when photo dims invalid | +| Webhook-mode setWebhook recovery | ✅ | ❌ | Retries `setWebhook` after recoverable network failures | ### Discord-Specific Features (since Feb 2025) @@ -106,6 +124,11 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Forwarded attachment downloads | ✅ | ❌ | Fetch media from forwarded messages | | Faster reaction state machine | ✅ | ❌ | Watchdog + debounce | | Thread parent binding inheritance | ✅ | ❌ | Threads inherit parent routing | +| Persistent components/forms across restarts | ✅ | ❌ | Active buttons/selects/forms keep working across Gateway restarts until expiry | +| `autoArchiveDuration` per-channel | ✅ | ❌ | 1h/1d/3d/1w archive duration for auto-created threads | +| Auto thread name generation | ✅ | ❌ | LLM-generated concise titles (`autoThreadName: "generated"`) | +| Voice channel responses | ✅ | ❌ | `channels.discord.voice.model` LLM override; voice mode auto-rejoin after RESUMED | +| CJK reply chunking | ✅ | ❌ | Splits long CJK replies at punctuation/code-point-safe boundaries | ### Slack-Specific Features (since Feb 2025) @@ -115,13 +138,22 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Configurable stream modes | ✅ | ❌ | Per-channel stream behavior | | Thread ownership | ✅ | 🚧 | Reply participation memory is restart-stable and TTL-bounded; once the bot joins a thread, follow-ups inherit channel visibility. Full thread-level ownership tracking is still missing | | Download-file action | ✅ | ❌ | On-demand attachment downloads via message actions | +| App Home tab views | ✅ | ❌ | Default Home view on `app_home_opened`, included in setup manifests | +| Persistent thread participation | ✅ | ❌ | Bot-participated threads tracked across restarts | +| Block Kit limit hardening | ✅ | ❌ | Auto-truncates buttons/selects/values, drops oversized link URLs while preserving valid blocks | +| Socket Mode pong tuning | ✅ | ❌ | `clientPingTimeout`, `serverPingTimeout`, `pingPongLoggingEnabled` | +| Native model picker (`/models`) | ✅ | ❌ | Provider/model chooser via interactive components | ### Mattermost-Specific Features (since Mar 2026) | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Interactive buttons | ✅ | ❌ | Clickable message buttons with signed callback flow | +| Interactive buttons | ✅ | ❌ | Clickable message buttons with signed callback flow; slash callback validation hardened | | Interactive model picker | ✅ | ❌ | In-channel provider/model chooser | +| `replyToMode` thread reply control | ✅ | ❌ | Top-level posts can start thread-scoped sessions; `all`/`first`/never modes | +| Streaming draft preview | ✅ | ❌ | Thinking, tool activity, partial reply text streamed into a single draft post | +| WebSocket ping/pong keepalives | ✅ | ❌ | Stale TCP drops reconnect instead of leaving monitoring idle | +| DM-vs-channel routing fixes | ✅ | ❌ | DM replies stay top-level; channel/group reply roots preserved | ### Feishu/Lark-Specific Features (since Mar 2026) @@ -129,6 +161,32 @@ This document tracks feature parity between IronClaw (Rust implementation) and O |---------|----------|----------|-------| | Doc/table actions | ✅ | ❌ | `feishu_doc` supports tables, positional insert, color_text, image upload, and file upload | | Rich-text embedded media extraction | ✅ | ❌ | Pull video/media attachments from post messages | +| Native interactive cards | ✅ | ❌ | Outgoing replies sent as native cards with clickable buttons | +| Schema 2.0 card action callbacks | ✅ | ❌ | Accept new `context.open_chat_id` shape | +| Streaming cards | ✅ | ❌ | Single live card per turn with throttled edits, topic-thread streaming | +| WebSocket retry/backoff | ✅ | ❌ | Monitor-owned reconnects after SDK retry exhaustion | +| Voice-note transcription | ✅ | ❌ | Inbound voice via shared media audio path | +| Bitable placeholder cleanup | ✅ | ❌ | Remove default-valued rows in create-app cleanup | + +### QQBot-Specific Features (since Mar 2026) + +| Feature | OpenClaw | IronClaw | Notes | +|---------|----------|----------|-------| +| Engine architecture rewrite | ✅ | ❌ | Self-contained engine with QR onboarding, native `/bot-approve`, per-account resource stacks, credential backup/restore | +| Group chat full support | ✅ | ❌ | History tracking, @-mention gating, activation modes, per-group config, FIFO queue | +| C2C `stream_messages` | ✅ | ❌ | StreamingController lifecycle manager | +| Chunked media upload | ✅ | ❌ | Unified `sendMedia` for large files | + +### BlueBubbles-Specific Features (since Mar 2026) + +| Feature | OpenClaw | IronClaw | Notes | +|---------|----------|----------|-------| +| Persistent inbound GUID dedupe | ✅ | ❌ | File-backed cache survives restart, 7-12x cron-duplicate fix | +| Catchup replay | ✅ | ❌ | Per-account cursor + `/api/v1/message/query?after=` pass on restart | +| Reply-context API fallback | ✅ | ❌ | Opt-in fetch for reply-context cache misses | +| TTS opus-in-CAF voice memos | ✅ | ❌ | Pre-transcoded native voice-memo bubbles via `tts.voice.preferAudioFileFormat` | +| Per-group `systemPrompt` injection | ✅ | ❌ | Group-specific behavioral instructions with `*` wildcard | +| Per-message catchup retry ceiling | ✅ | ❌ | `catchup.maxFailureRetries` to skip persistently failing messages | ### Channel Features @@ -145,6 +203,11 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions/scopes | | Group session priming | ✅ | ❌ | Member roster injected for context | | Sender_id in trusted metadata | ✅ | ❌ | Exposed in system metadata | +| Per-group `systemPrompt` injection | ✅ | ❌ | Per-group/per-direct system prompts injected via `GroupSystemPrompt` (Telegram, Discord, WhatsApp, BlueBubbles) | +| Visible reply enforcement | ✅ | ❌ | `messages.visibleReplies` requires output via `message(action=send)`; group-scope override available | +| Active-run steering queue | ✅ | ❌ | `messages.queue` `steer` mode (default) drains queued messages at next model boundary; `queue` legacy one-at-a-time | +| Tool-progress streaming into previews | ✅ | ❌ | Tool progress shown in live preview edits (Discord/Slack/Telegram/Mattermost/Matrix) | +| `dmPolicy="open"` semantics | ✅ | 🚧 | Public open-DM only with effective wildcard; pairing-store senders no longer count for DM audits (OpenClaw fixed across all channels) | ### Owner: _Unassigned_ @@ -179,10 +242,29 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | `sandbox` | ✅ | ✅ | - | WASM sandbox | | `doctor` | ✅ | 🚧 | P2 | 16 subsystem checks | | `logs` | ✅ | 🚧 | P3 | `logs` (gateway.log tail), `--follow` (SSE live stream), `--level` (get/set). No DB-persisted log history. | -| `update` | ✅ | ❌ | P3 | Self-update | +| `update` | ✅ | ❌ | P3 | Self-update; `OPENCLAW_NO_AUTO_UPDATE=1` kill-switch | | `completion` | ✅ | ✅ | - | Shell completion | +| `migrate` | ✅ | ❌ | P3 | Bundled importers for Claude Code, Claude Desktop, Hermes (config, MCP servers, skills, command prompts, model providers, credentials) | +| `proxy validate` | ✅ | ❌ | P3 | Verify effective proxy config, reachability, allow/deny destinations | +| `plugins registry` | ✅ | ❌ | P3 | Inspect persisted plugin registry; `--refresh` repair | +| `plugins deps` | ✅ | ❌ | P3 | Inspect/repair bundled plugin runtime dependencies | +| `infer model run --gateway` | ✅ | ❌ | P3 | Raw model probes via Gateway; image `--file` + `--prompt` + `--timeout-ms` overrides | +| `infer image describe`/`describe-many` | ✅ | ❌ | P3 | Custom vision prompts/timeouts | +| `qa` (suite/telegram/credentials) | ✅ | ❌ | P3 | QA Lab CI runner with `--allow-failures` opt-out | +| `voicecall setup`/`smoke`/`continue` | ✅ | ❌ | P3 | Voice call provider readiness, dry-run smoke, gateway-delegated continue | +| `googlemeet doctor`/`recover-tab` | ✅ | ❌ | P3 | Meet OAuth/browser-state diagnostics, tab recovery | +| `matrix verify`/`encryption setup` | ✅ | ❌ | P3 | E2EE setup, recovery key rotation, cross-signing trust | +| `nodes remove` | ✅ | ❌ | P3 | Remove stale gateway-owned node pairing records | +| `nodes list` (paired view) | ✅ | ❌ | P3 | Default paired-node view with pending fallback | +| `cron run` / `cron edit --thread-id` | ✅ | 🚧 | P2 | Already partial; OpenClaw added cron stagger, finished-run webhook, `--failure-alert-include-skipped` | +| `sessions export-trajectory` | ✅ | ❌ | P3 | Per-run trajectory bundles with redacted transcripts/runtime events/prompts | | `/subagents spawn` | ✅ | ❌ | P3 | Spawn subagents from chat | | `/export-session` | ✅ | ❌ | P3 | Export current session transcript | +| `/export-trajectory` (chat) | ✅ | ❌ | P3 | Per-run exec-approved trajectory bundle, owner-only delivery | +| `/diagnostics` (owner-only) | ✅ | ❌ | P3 | Owner-only diagnostics export with sensitive-data preamble | +| `/codex computer-use status/install` | ✅ | ❌ | P3 | Codex desktop control setup with marketplace discovery | +| `/dock-*` route switches | ✅ | ❌ | P3 | Switch active session reply route through `session.identityLinks` | +| `--container` / `OPENCLAW_CONTAINER` | ✅ | ❌ | P3 | Run CLI commands inside running Docker/Podman container | ### Owner: _Unassigned_ @@ -197,35 +279,59 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Multi-provider failover | ✅ | ✅ | `FailoverProvider` tries providers sequentially on retryable errors | | Per-sender sessions | ✅ | ✅ | | | Global sessions | ✅ | ❌ | Optional shared context | -| Session pruning | ✅ | ❌ | Auto cleanup old sessions | +| Session pruning | ✅ | ❌ | Auto cleanup old sessions; oversized `sessions.json` rotation removed; entry/age caps enforced at load | | Context compaction | ✅ | ✅ | Auto summarization | -| Compaction model override | ✅ | ❌ | Use a dedicated provider/model for summarization only | +| Compaction model override | ✅ | ❌ | Use a dedicated provider/model for summarization only; `agents.defaults.compaction.memoryFlush.model` exact override | +| Compaction mid-turn precheck | ✅ | ❌ | `agents.defaults.compaction.midTurnPrecheck` triggers before next tool call instead of end-of-turn | | Post-compaction read audit | ✅ | ❌ | Layer 3: workspace rules appended to summaries | | Post-compaction context injection | ✅ | ❌ | Workspace context as system event | +| Compaction start/end notices | ✅ | ❌ | Opt-in lifecycle notices during compaction | | Custom system prompts | ✅ | ✅ | Template variables, safety guardrails | | Skills (modular capabilities) | ✅ | ✅ | Prompt-based skills with trust gating, attenuation, activation criteria, catalog, selector | +| Skill Workshop plugin | ✅ | ❌ | Captures reusable workflow corrections as pending or auto-applied workspace skills, threshold-based reviewer | +| Grouped skill directories | ✅ | ✅ | `skills///SKILL.md` discovery | +| Skill installer metadata | ✅ | ❌ | One-click install recipes (npm/pip), API key entry, source metadata | | Skill routing blocks | ✅ | 🚧 | ActivationCriteria (keywords, patterns, tags) but no "Use when / Don't use when" blocks | | Skill path compaction | ✅ | ❌ | ~ prefix to reduce prompt tokens | -| Thinking modes (off/minimal/low/medium/high/xhigh/adaptive) | ✅ | 🚧 | thinkingConfig for Gemini models (thinkingBudget/thinkingLevel); no per-level control yet | -| Per-model thinkingDefault override | ✅ | ❌ | Override thinking level per model; Anthropic Claude 4.6 defaults to adaptive | +| Thinking modes (off/minimal/low/medium/high/xhigh/adaptive/max) | ✅ | 🚧 | thinkingConfig for Gemini models; no per-level control yet; Anthropic Opus 4.7 `xhigh`+`adaptive`+`max`; DeepSeek V4 `xhigh`/`max` | +| Per-model thinkingDefault override | ✅ | ❌ | Override thinking level per model; Anthropic Claude 4.6/4.7 defaults to adaptive | +| Adaptive→provider thinking maps | ✅ | ❌ | `/think adaptive` maps to Gemini dynamic thinking, Anthropic adaptive, OpenAI flex | +| Native Codex app-server runtime | ✅ | ➖ | New embedded Codex harness with PreToolUse/PostToolUse/PermissionRequest relay; replaces ACP for `codex/*` models | +| Codex Computer Use | ✅ | ❌ | Desktop control setup with marketplace discovery, fail-closed MCP checks | +| Codex hooks bridge | ✅ | ❌ | Codex-native tool hooks → OpenClaw plugin hooks/approvals | +| Codex sub-agent metadata | ✅ | ❌ | Native Codex sub-agent session metadata without nested gateway patch | +| Codex context-engine integration | ✅ | ❌ | Bootstrap, assembly, post-turn maintenance, engine-owned compaction in Codex sessions | +| Active Memory plugin | ✅ | ❌ | Dedicated memory sub-agent right before main reply; configurable message/recent/full context modes; partial-recall on timeout; per-conversation `allowedChatIds`/`deniedChatIds` filters | +| Inferred follow-up commitments | ✅ | ❌ | Opt-in hidden batched extraction with per-agent/per-channel scoping, heartbeat delivery, CLI management; `commitments.enabled`/`maxPerDay` | +| `sessions_yield` | ✅ | ❌ | Orchestrators end current turn immediately, skip queued tool work, carry hidden follow-up payload to next turn | +| Subagent forked context | ✅ | ❌ | Optional inherit-requester-transcript for native `sessions_spawn` | +| `agents.defaults.contextInjection: "never"` | ✅ | ❌ | Disable workspace bootstrap injection per-agent | +| `agents.defaults.experimental.localModelLean` | ✅ | ❌ | Drop heavyweight default tools for weaker local models | +| `agents.files.get/set` workspace tools | ✅ | ❌ | Realpath-via-fd safety, symlink alias rejection | +| Trajectory export | ✅ | ❌ | Default-on local trajectory capture; `/export-trajectory` bundles with redacted transcripts/events/artifacts | | Block-level streaming | ✅ | ❌ | | | Tool-level streaming | ✅ | ❌ | | | Z.AI tool_stream | ✅ | ❌ | Real-time tool call streaming | | Plugin tools | ✅ | ✅ | WASM tools | | Tool policies (allow/deny) | ✅ | ✅ | | | Exec approvals (`/approve`) | ✅ | ✅ | TUI approval overlay | +| Tool inventory cache | ✅ | ❌ | Coalesced effective-tool inventory cache with channel-registry invalidation | +| Pending exec approval `errorMessage` cleanup | ✅ | ❌ | Failed restart-interrupted approval-pending sessions instead of replaying stale ids | | Elevated mode | ✅ | ❌ | Privileged execution | -| Subagent support | ✅ | ✅ | Task framework | +| Subagent support | ✅ | ✅ | Task framework; spawn-by-account-aware bindings, model overrides preserved | | `/subagents spawn` command | ✅ | ❌ | Spawn from chat | -| Auth profiles | ✅ | ❌ | Multiple auth strategies | +| Auth profiles | ✅ | ❌ | Multiple auth strategies; replaceDefaultModels migration semantics | | Generic API key rotation | ✅ | ❌ | Rotate keys across providers | -| Stuck loop detection | ✅ | ❌ | Exponential backoff on stuck agent loops | +| Stuck loop detection | ✅ | ❌ | Exponential backoff on stuck agent loops; unknown-tool guard default-on | | llms.txt discovery | ✅ | ❌ | Auto-discover site metadata | | Multiple images per tool call | ✅ | ❌ | Single tool call, multiple images | | URL allowlist (web_search/fetch) | ✅ | ❌ | Restrict web tool targets | | suppressToolErrors config | ✅ | ❌ | Hide tool errors from user | | Intent-first tool display | ✅ | ❌ | Details and exec summaries | | Transcript file size in status | ✅ | ❌ | Show size in session status | +| Stuck-session recovery | ✅ | ❌ | Conservative recovery releases stale lanes while preserving active embedded runs/replies | +| `Runner:` in `/status` | ✅ | ❌ | Reports embedded Pi/CLI-backed/ACP harness in session status | +| Voice Wake routing | ✅ | ❌ | Wake phrases can target named agent or session via gateway routing APIs | ### Owner: _Unassigned_ @@ -236,37 +342,94 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Provider | OpenClaw | IronClaw | Priority | Notes | |----------|----------|----------|----------|-------| | NEAR AI | ✅ | ✅ | - | Primary provider | -| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy; Opus 4.5, Sonnet 4, Sonnet 4.6, adaptive thinking default | -| OpenAI | ✅ | 🚧 | - | Via NEAR AI proxy; GPT-5.4 + Codex OAuth | -| AWS Bedrock | ✅ | ✅ | - | Native Converse API via aws-sdk-bedrockruntime (requires `--features bedrock`) | -| Google Gemini | ✅ | ✅ | - | OAuth (PKCE + S256), function calling, thinkingConfig, generationConfig | +| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy; Opus 4.7 (default, adaptive+xhigh+max), Opus 4.6, Sonnet 4.6 | +| OpenAI | ✅ | 🚧 | - | Via NEAR AI proxy; GPT-5.5 default, GPT-5.4-pro forward-compat, Codex OAuth, Responses API; image generation (`gpt-image-2`) via Codex OAuth | +| OpenAI Codex (native app-server) | ✅ | ➖ | - | App-server >=0.125.0 with native MCP hooks, dynamic tools, approval relay | +| AWS Bedrock | ✅ | ✅ | - | Native Converse API; Claude Opus 4.7 thinking profile (xhigh/adaptive/max); IAM bearer token refresh for Mantle | +| Google Gemini | ✅ | ✅ | - | OAuth (PKCE + S256), function calling, thinkingConfig, generationConfig; TTS (`gemini-embedding-2-preview`); image gen native API; ADC-backed Vertex | +| Google Gemini Live (realtime) | ✅ | ❌ | - | Realtime voice provider for Voice Call/Google Meet, bidirectional audio + function calls | | io.net | ✅ | ✅ | P3 | Via `ionet` adapter | -| Mistral | ✅ | ✅ | P3 | Via `mistral` adapter | +| Mistral | ✅ | ✅ | P3 | Via `mistral` adapter; Voice Call streaming STT | | Yandex AI Studio | ✅ | ✅ | P3 | Via `yandex` adapter | | Cloudflare Workers AI | ✅ | ✅ | P3 | Via `cloudflare` adapter | -| NVIDIA API | ✅ | ✅ | P3 | Via `nvidia` adapter and `providers.json` | -| OpenRouter | ✅ | ✅ | - | Via OpenAI-compatible provider (RigAdapter) | +| NVIDIA API | ✅ | ✅ | P3 | Via `nvidia` adapter; OpenClaw added bundled provider with API-key onboarding, static catalog, literal model-ref picker, NIM string-content compat | +| OpenRouter | ✅ | ✅ | - | Via OpenAI-compatible provider; OpenClaw added native video generation, `openrouter:auto`/`openrouter:free` aliases, Hunter/Healer Alpha, free-model fallback for `models scan` | | Tinfoil | ❌ | ✅ | - | Private inference provider (IronClaw-only) | | OpenAI-compatible | ❌ | ✅ | - | Generic OpenAI-compatible endpoint (RigAdapter); OpenAI-style image inputs default missing `image_url.detail` to `auto` | -| GitHub Copilot | ✅ | ✅ | - | Dedicated provider with OAuth token exchange (`GithubCopilotProvider`) | -| Ollama (local) | ✅ | ✅ | - | via `rig::providers::ollama` (full support) | +| GitHub Copilot | ✅ | ✅ | - | Dedicated provider with OAuth token exchange; default Opus model is `claude-opus-4.7`; GUI/RPC wizard device-code auth; `gpt-5.4` xhigh thinking | +| Ollama (local) | ✅ | ✅ | - | OpenClaw added Cloud + Local + cloud-only modes, browser sign-in, signed `/api/experimental/web_search`, `params.num_ctx`/`params.think`/`params.keep_alive`, `/api/show` capability detection | | Perplexity | ✅ | ❌ | P3 | Freshness parameter for web_search | -| MiniMax | ✅ | ❌ | P3 | Regional endpoint selection | +| MiniMax | ✅ | ❌ | P3 | Regional endpoint selection; portal OAuth + Token Plan + `MINIMAX_API_KEY`; image-01, music-2.6, video; `MiniMax-VL-01` for vision | | GLM-5 | ✅ | ✅ | P3 | Via Z.AI provider (`zai`) using OpenAI-compatible chat completions | -| node-llama-cpp | ✅ | ➖ | - | N/A for Rust | +| Tencent Cloud (TokenHub) | ✅ | ❌ | P3 | Bundled provider; Hy3 catalog with tiered pricing | +| DeepInfra | ✅ | ❌ | P3 | Bundled provider with `DEEPINFRA_API_KEY`, dynamic OpenAI-compatible discovery, image gen/edit, image/audio understanding, TTS, text-to-video, embeddings | +| Cerebras | ✅ | ❌ | P3 | Bundled plugin with onboarding, static catalog, manifest endpoint metadata | +| Z.AI / GLM-5 | ✅ | ✅ | - | OpenClaw added bundled GLM catalog/auth in plugin manifest, `params.preserveThinking` for `reasoning_content` replay | +| Qwen / Model Studio | ✅ | ❌ | P3 | Standard DashScope endpoints (CN + global) + Coding Plan; vLLM Qwen thinking controls | +| DeepSeek | ✅ | ❌ | P3 | V4 Pro/V4 Flash bundled, V4 Flash onboarding default, native `xhigh`/`max` thinking levels, `reasoning_content` replay support | +| Moonshot / Kimi | ✅ | ❌ | P3 | Kimi K2.6 default; native Anthropic-format tool calls; CN API endpoint support; `kimi-coding` web search via `KIMI_API_KEY` | +| xAI | ✅ | ❌ | P3 | Image gen (`grok-imagine-image`/`pro`), reference-image edits, six TTS voices (MP3/WAV/PCM/G.711), `grok-stt` audio transcription, realtime STT for Voice Call | +| Tencent Yuanbao | ✅ | ❌ | P3 | External plugin (`openclaw-plugin-yuanbao`) for chat | +| Vercel AI Gateway | ✅ | ❌ | P3 | Provider-owned thinking levels for trusted upstream refs | +| Codex/OpenAI image generation | ✅ | ❌ | P2 | `gpt-image-2`/`gpt-image-1.5` via Codex OAuth or API key; multipart reference-image edits; Azure deployment-scoped image URLs | +| OpenRouter image/video generation | ✅ | ❌ | P3 | Image gen + reference edits; native video generation through `video_generate` | +| MiniMax music/video | ✅ | ❌ | P3 | `music-2.6`, `video_generate`, `MiniMax-portal` registration | +| Google Veo (video gen) | ✅ | ❌ | P3 | Direct MLDev `video.uri` downloads; REST `predictLongRunning` fallback | +| fal Seedance 2.0 | ✅ | ❌ | P3 | Reference-to-video with multi-image/video/audio input | +| Comfy (image/video/music) | ✅ | ❌ | P3 | `plugins.entries.comfy.config` workflow + cloud auth | +| node-llama-cpp | ✅ | ➖ | - | OpenClaw made it optional (no longer auto-installed); local embeddings now opt-in | | llama.cpp (native) | ❌ | 🔮 | P3 | Rust bindings | ### Model Features | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Auto-discovery | ✅ | ❌ | | +| Auto-discovery | ✅ | ❌ | Manifest-backed `modelCatalog` with aliases/suppressions; cold installed-index fast path | | Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` | | Cooldown management | ✅ | ✅ | Lock-free per-provider cooldown in `FailoverProvider` | | Per-session model override | ✅ | ✅ | Model selector in TUI | -| Model selection UI | ✅ | ✅ | TUI keyboard shortcut | +| Model selection UI | ✅ | ✅ | TUI keyboard shortcut; OpenClaw added Quick Settings, mobile-aware picker | | Per-model thinkingDefault | ✅ | ❌ | Override thinking level per model in config | -| 1M context support | ✅ | ❌ | Anthropic extended context beta + OpenAI Codex GPT-5.4 1M context | +| 1M context support | ✅ | ❌ | Anthropic extended context beta + OpenAI Codex GPT-5.4 1M context; Claude Opus 4.7 + claude-cli normalized to 1M | +| Fast mode (`/fast`) | ✅ | ❌ | Anthropic `service_tier` + OpenAI `gpt-5.4-fast`; `/fast` toggle, TUI/Control UI/ACP, per-model defaults | +| Tiered model pricing | ✅ | ❌ | Pricing tiers from cached catalogs (Moonshot Kimi K2.6/K2.5, Hy3) for usage reports | +| `models scan` (free-model fallback) | ✅ | ❌ | Public OpenRouter free-model metadata when no `OPENROUTER_API_KEY` | +| Model catalog stale cache fallback | ✅ | ❌ | Serve last successful catalog while stale reloads refresh in background | +| `models.pricing.enabled` | ✅ | ❌ | Skip startup OpenRouter/LiteLLM pricing-catalog fetches for offline installs | +| Auth status card | ✅ | ❌ | OAuth token health + provider rate-limit pressure with `models.authStatus` RPC | +| Model fallback metadata | ✅ | ❌ | `model.fallback_step` trajectory events with from/to + chain position + final outcome | +| `prompt_cache_key` opt-in | ✅ | ❌ | `compat.supportsPromptCacheKey` per-provider opt-in | +| Replay normalization | ✅ | ❌ | Repair displaced/missing tool results, Anthropic/Bedrock thinking signature stripping, OpenAI Responses orphaned reasoning, Codex aborted-output replay | + +### TTS / STT / Realtime Voice + +| Feature | OpenClaw | IronClaw | Priority | Notes | +|---------|----------|----------|----------|-------| +| TTS (Microsoft / Edge) | ✅ | ❌ | P3 | Auto-enabled bundled provider; legacy `messages.tts.providers.edge` voices | +| TTS (OpenAI) | ✅ | ❌ | P3 | OpenAI-compatible `/audio/speech` | +| TTS (ElevenLabs v3) | ✅ | ❌ | P3 | `eleven_v3` model surfaced; PCM telephony | +| TTS (Google Gemini) | ✅ | ❌ | P3 | `audioProfile` + `speakerName` prompt control; PCM-to-Opus voice notes | +| TTS (Azure Speech) | ✅ | ❌ | P3 | Bundled provider, Speech-resource auth, SSML, native Ogg/Opus | +| TTS (Inworld) | ✅ | ❌ | P3 | Streaming synthesis, voice-note + PCM telephony | +| TTS (Volcengine/BytePlus Seed Speech) | ✅ | ❌ | P3 | Bundled provider, Ogg/Opus voice notes, MP3 file output | +| TTS (Xiaomi MiMo) | ✅ | ❌ | P3 | MP3/WAV + voice-note Opus transcoding | +| TTS (Local CLI) | ✅ | ❌ | P3 | Bundled local command speech provider with file/stdout/Opus/PCM | +| TTS (Gradium) | ✅ | ❌ | P3 | Bundled TTS provider with voice-note + telephony output | +| TTS (OpenRouter) | ✅ | ❌ | P3 | OpenAI-compatible `/audio/speech` via `OPENROUTER_API_KEY` | +| TTS (xAI) | ✅ | ❌ | P3 | Six grok voices, MP3/WAV/PCM/G.711 | +| TTS (DeepInfra) | ✅ | ❌ | P3 | Bundled provider | +| TTS (MiniMax) | ✅ | ❌ | P3 | Portal OAuth + Token Plan; HD model ids | +| TTS (Tinfoil/local MLX) | ✅ | ❌ | P3 | macOS Talk experimental MLX provider | +| TTS personas | ✅ | ❌ | P3 | Provider-aware personas with deterministic provider binding, `/tts persona`, Gemini `audio-profile-v1`, OpenAI instructions | +| Auto-TTS controls | ✅ | ❌ | P3 | `/tts latest`, `/tts chat on\|off\|default`; per-account/per-agent overrides | +| Talk Mode (browser realtime) | ✅ | ❌ | P3 | OpenAI Realtime + Google Live WebRTC/WS; ephemeral client secrets; `openclaw_agent_consult` handoff | +| STT (OpenAI Realtime) | ✅ | ❌ | P3 | Voice Call streaming transcription | +| STT (xAI realtime) | ✅ | ❌ | P3 | Voice Call streaming via `grok-stt` | +| STT (Deepgram) | ✅ | ❌ | P3 | Voice Call streaming | +| STT (ElevenLabs Scribe v2) | ✅ | ❌ | P3 | Batch + streaming inbound transcription | +| STT (Mistral) | ✅ | ❌ | P3 | Voice Call streaming | +| STT (SenseAudio) | ✅ | ❌ | P3 | Bundled batch audio transcription via `tools.media.audio` | +| STT (local Whisper CLI) | ✅ | ❌ | P3 | Configured/key-backed STT preferred over auto-detected local Whisper | ### Owner: _Unassigned_ @@ -279,17 +442,21 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Image processing (Sharp) | ✅ | ❌ | P2 | Resize, format convert | | Configurable image resize dims | ✅ | ❌ | P2 | Per-agent dimension config | | Multiple images per tool call | ✅ | ❌ | P2 | Single tool invocation, multiple images | -| Audio transcription | ✅ | ❌ | P2 | | -| Video support | ✅ | ❌ | P3 | | -| PDF analysis tool | ✅ | ❌ | P2 | Native Anthropic/Gemini path with text/image extraction fallback | +| Audio transcription | ✅ | ❌ | P2 | Multiple providers (see TTS/STT subsection in Section 6) | +| Video support | ✅ | ❌ | P3 | OpenRouter native video gen, MiniMax video, Google Veo, fal Seedance, OpenAI Sora | +| PDF analysis tool | ✅ | ❌ | P2 | Native Anthropic/Gemini path with text/image extraction fallback; bundled `document-extract` plugin owns `pdfjs-dist` | | PDF parsing | ✅ | 🚧 | P2 | Uploaded document attachments parse via `pdf-extract`; no `pdfjs-dist` fallback path | -| MIME detection | ✅ | ❌ | P2 | | +| MIME detection | ✅ | ❌ | P2 | Bounded MIME sniff + ZIP archive preflight | | Media caching | ✅ | ❌ | P3 | | -| Vision model integration | ✅ | ❌ | P2 | Image understanding | -| TTS (Edge TTS) | ✅ | ❌ | P3 | Text-to-speech | -| TTS (OpenAI) | ✅ | ❌ | P3 | | +| Vision model integration | ✅ | ❌ | P2 | Image understanding; `agents.defaults.imageModel`, Codex app-server image turns, configured-provider exact match | +| Image generation | ✅ | ❌ | P2 | OpenAI `gpt-image-2` / `gpt-image-1.5`, OpenRouter, Gemini, MiniMax `image-01`; quality + format + background hints | +| Music generation | ✅ | ❌ | P3 | MiniMax `music-2.6`, fal, video-to-music workflows | +| Multimodal memory indexing | ✅ | ❌ | P3 | Image + audio indexing for `memorySearch.extraPaths` via Gemini `gemini-embedding-2-preview` | +| Audio-as-voice routing | ✅ | ❌ | P2 | `[[audio_as_voice]]` directives on text tool-result `MEDIA:` payloads | +| TTS providers | ✅ | ❌ | P2 | See TTS/STT subsection in Section 6 | | Incremental TTS playback | ✅ | ❌ | P3 | iOS progressive playback | | Sticker-to-image | ✅ | ❌ | P3 | Telegram stickers | +| Per-channel media limits | ✅ | 🚧 | P2 | `mediaMaxMb` enforcement (already in Section 3); Signal `getAttachment` honors `mediaMaxMb` with base64 headroom | ### Owner: _Unassigned_ @@ -300,21 +467,37 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| | Dynamic loading | ✅ | ✅ | WASM modules | -| Manifest validation | ✅ | ✅ | WASM metadata | +| Manifest validation | ✅ | ✅ | WASM metadata; `modelCatalog`, `channelConfigs`, `setup.providers`, `setup.requiresRuntime`, `activation.onStartup` contracts | | HTTP path registration | ✅ | ❌ | Plugin routes | | Workspace-relative install | ✅ | ✅ | ~/.ironclaw/tools/ | | Channel plugins | ✅ | ✅ | WASM channels | | Auth plugins | ✅ | ❌ | | | Memory plugins | ✅ | ❌ | Custom backends + selectable memory slot | -| Context-engine plugins | ✅ | ❌ | Custom context management + subagent/context hooks | +| Context-engine plugins | ✅ | ❌ | Custom context management + subagent/context hooks; `info.id` slot match enforced | | Tool plugins | ✅ | ✅ | WASM tools | | Hook plugins | ✅ | ✅ | Declarative hooks from extension capabilities | -| Provider plugins | ✅ | ❌ | | +| Provider plugins | ✅ | ❌ | Manifest-backed catalogs/aliases/suppressions; setup auth metadata | | Plugin CLI (`install`, `list`) | ✅ | ✅ | `tool` subcommand | -| ClawHub registry | ✅ | ❌ | Discovery | +| ClawHub registry | ✅ | ❌ | Discovery; install scope `--profile`, `npm:` install prefix to skip ClawHub lookup, `clawhub:` install records | +| `git:` plugin installs | ✅ | ❌ | First-class `git:` install with ref checkout, commit metadata, `plugins update` for git sources | | `before_agent_start` hook | ✅ | ❌ | modelOverride/providerOverride support | +| `before_agent_finalize` hook | ✅ | ❌ | New finalize hook with run/message/sender/session/trace correlation | | `before_message_write` hook | ✅ | ❌ | Pre-write message interception | -| `llm_input`/`llm_output` hooks | ✅ | ❌ | LLM payload inspection | +| `before_dispatch` hook | ✅ | ❌ | Canonical inbound metadata; route handled replies through normal final delivery | +| `before_compaction`/`after_compaction` hooks | ✅ | ❌ | Codex-native compaction lifecycle | +| `llm_input`/`llm_output` hooks | ✅ | ❌ | LLM payload inspection (Codex app-server included) | +| `model_call_started`/`ended` hooks | ✅ | ❌ | Metadata-only, no prompts/responses/headers/raw provider request IDs | +| `cron_changed` hook | ✅ | ❌ | Typed cron lifecycle observer | +| `gateway_start` hook context | ✅ | ❌ | Startup config, workspace dir, live cron getter | +| `agent_end` observation hooks | ✅ | ❌ | 30s timeout for non-settling hooks | +| Plugin SDK state store | ✅ | ❌ | SQLite-backed `api.runtime.state.openKeyedStore` for restart-safe keyed registries with TTL/eviction | +| Plugin SDK Codex extensions | ✅ | ❌ | Async `tool_result` middleware, `after_tool_call` for Codex tool runs | +| Persisted plugin registry | ✅ | ❌ | Cold registry index, `openclaw plugins registry` inspection, `--refresh` repair | +| `plugins deps --repair` | ✅ | ❌ | Bundled runtime-deps inspect + repair without rerunning plugin runtime | +| Plugin install conflict-aware writes | ✅ | ❌ | Install/uninstall config writes are conflict-aware; managed plugin files removed only after config commit | +| Plugin compatibility registry | ✅ | ❌ | Central deprecation registry with dated owners + replacements + 3-month removal targets | +| Layered runtime-deps roots | ✅ | ❌ | `OPENCLAW_PLUGIN_STAGE_DIR` resolves read-only preinstalled deps before installing missing deps | +| Bundled provider catalogs in manifest | ✅ | ❌ | DeepInfra, Cerebras, Mistral, Moonshot, DeepSeek, Tencent, StepFun, Venice, Fireworks, Together, Groq, Qianfan, Xiaomi, BytePlus, Volcano Engine, NVIDIA | ### Owner: _Unassigned_ @@ -328,12 +511,20 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | JSON5 support | ✅ | ❌ | Comments, trailing commas | | YAML alternative | ✅ | ❌ | | | Environment variable interpolation | ✅ | ✅ | `${VAR}` | -| Config validation/schema | ✅ | ✅ | Type-safe Config struct + `openclaw config validate` | -| Hot-reload | ✅ | ❌ | | -| Legacy migration | ✅ | ➖ | | +| Config validation/schema | ✅ | ✅ | Type-safe Config struct + `openclaw config validate`; OpenClaw added top-3 issue surface for `config.set/patch/apply` | +| Hot-reload | ✅ | ❌ | Many plugins now re-read live runtime config (memory-lancedb, active-memory, github-copilot, ollama, openai, amazon-bedrock, codex, skill-workshop, diffs, gateway-tool); `OPENCLAW_NO_AUTO_UPDATE=1` kill-switch | +| Legacy migration | ✅ | ➖ | OpenClaw dropped automatic migrations older than two months | | State directory | ✅ `~/.openclaw-state/` | ✅ `~/.ironclaw/` | | | Credentials directory | ✅ | ✅ | Session files | | Full model compat fields in schema | ✅ | ❌ | pi-ai model compat exposed in config | +| `models.pricing.enabled` | ✅ | ❌ | Skip OpenRouter/LiteLLM pricing fetches for offline installs | +| `agents.list[].contextTokens` | ✅ | ❌ | Per-agent context window override | +| `gateway.handshakeTimeoutMs` | ✅ | ❌ | Tunable WebSocket pre-auth handshake budget | +| `--profile ` | ✅ | ❌ | Plugin install destinations resolve from active profile state dir | +| Config recovery on clobber | ✅ | ❌ | Restore last-known-good config on critical clobber signatures (missing metadata, missing `gateway.mode`, sharp size drops); foreground/service notices include rejected paths | +| Modular `$include` files | ✅ | ❌ | Single-file top-level includes for isolated mutations; `plugins install`/`update` updates `plugins.json5` instead of flattening | +| `config set --merge`/`--replace` | ✅ | ❌ | Additive vs intentional clobber for provider model maps | +| Wrapper-based service install | ✅ | ❌ | `--wrapper`/`OPENCLAW_WRAPPER` validated executable LaunchAgent/systemd wrappers | ### Owner: _Unassigned_ @@ -345,25 +536,35 @@ This document tracks feature parity between IronClaw (Rust implementation) and O |---------|----------|----------|-------| | Vector memory | ✅ | ✅ | pgvector | | Session-based memory | ✅ | ✅ | | -| Hybrid search (BM25 + vector) | ✅ | ✅ | RRF algorithm | +| Hybrid search (BM25 + vector) | ✅ | ✅ | RRF algorithm; vectorScore + textScore exposed alongside combined score | | Temporal decay (hybrid search) | ✅ | ❌ | Opt-in time-based scoring factor | | MMR re-ranking | ✅ | ❌ | Maximal marginal relevance for result diversity | | LLM-based query expansion | ✅ | ❌ | Expand FTS queries via LLM | | OpenAI embeddings | ✅ | ✅ | | | Bedrock embeddings | ❌ | ✅ | Reuses Bedrock region/profile auth for Titan Text Embeddings V2 | -| Gemini embeddings | ✅ | ❌ | | -| Local embeddings | ✅ | ❌ | | -| SQLite-vec backend | ✅ | ❌ | IronClaw uses PostgreSQL | -| LanceDB backend | ✅ | ❌ | Configurable auto-capture max length | -| QMD backend | ✅ | ❌ | | +| Gemini embeddings | ✅ | ❌ | `gemini-embedding-2-preview` with configurable output dimensions, automatic reindex on dim change | +| GitHub Copilot embeddings | ✅ | ❌ | Provider with token refresh, payload validation, remote overrides | +| Ollama embeddings | ✅ | ✅ | OpenClaw moved to `/api/embed` with batched `input`; per-host cache keys; non-batch concurrency knob | +| Local embeddings | ✅ | ❌ | `node-llama-cpp` now optional install | +| Asymmetric embedding endpoints | ✅ | ❌ | `inputType`/`queryInputType`/`documentInputType` for retrieval prefixes (Ollama: `nomic-embed-text`, `qwen3-embedding`, `mxbai-embed-large`) | +| SQLite-vec backend | ✅ | ❌ | IronClaw uses PostgreSQL; bundled-plugin runtime-deps mirror sqlite-vec | +| LanceDB backend | ✅ | ❌ | Configurable auto-capture max length; cloud storage support; OpenAI-compatible float embeddings, ZhiPu/DashScope normalization | +| QMD backend | ✅ | ❌ | Multi-collection `-c` filters, `--mask` collection patterns, opt-in `memory.qmd.update.startup` | +| Active Memory plugin | ✅ | ❌ | Memory sub-agent before main reply; partial recall on timeout; `allowedChatIds`/`deniedChatIds`; visible status fields | +| Memory wiki (people-aware) | ✅ | ❌ | Canonical aliases, person cards, relationship graphs, privacy/provenance reports, search modes (find-person/route-question/source-evidence/raw-claim) | +| Dreaming (REM cycles) | ✅ | ❌ | `## Light Sleep`/`## REM Sleep` phase blocks; `dreaming.storage.mode = "separate"` default; `dreaming.model` override | +| `recallMaxChars` cap | ✅ | ❌ | Bound recall embedding queries for small Ollama embedding models | +| `corpus=sessions` ranking | ✅ | ❌ | Session transcript hits with visibility/agent-to-agent policy | | Atomic reindexing | ✅ | ✅ | | | Embeddings batching | ✅ | ✅ | `embed_batch` on EmbeddingProvider trait | | Citation support | ✅ | ❌ | | | Memory CLI commands | ✅ | ✅ | `memory search/read/write/tree/status` CLI subcommands | +| `openclaw ltm list` | ✅ | ❌ | Real LanceDB LTM rows with `--limit`/createdAt ordering | | Flexible path structure | ✅ | ✅ | Filesystem-like API | | Identity files (AGENTS.md, etc.) | ✅ | ✅ | | | Daily logs | ✅ | ✅ | | | Heartbeat checklist | ✅ | ✅ | HEARTBEAT.md | +| Hybrid post-compaction reindex | ✅ | ❌ | `agents.defaults.compaction.postIndexSync`; `memorySearch.sync.sessions.postCompactionForce` | ### Owner: _Unassigned_ @@ -415,17 +616,27 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| -| Control UI Dashboard | ✅ | ✅ | - | Web gateway with chat, memory, jobs, logs, extensions | +| Control UI Dashboard | ✅ | ✅ | - | Web gateway with chat, memory, jobs, logs, extensions; modular Overview/Chat/Config/Agent/Session views, command palette, mobile bottom tabs | | Channel status view | ✅ | 🚧 | P2 | Gateway status widget, full channel view pending | -| Agent management | ✅ | ❌ | P3 | | +| Agent management | ✅ | ❌ | P3 | Agent Tool Access panel with compact live-tool chips, collapsible groups, per-tool toggles | | Model selection | ✅ | ✅ | - | TUI only | -| Config editing | ✅ | ❌ | P3 | | +| Config editing | ✅ | ❌ | P3 | Raw config pending-changes diff panel with redacted reveal | | Debug/logs viewer | ✅ | ✅ | - | Real-time log streaming with level/target filters | | WebChat interface | ✅ | ✅ | - | Web gateway chat with SSE/WebSocket | -| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI, improved asset resolution | -| Control UI i18n | ✅ | ❌ | P3 | English, Chinese, Portuguese | +| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI, improved asset resolution; macOS canvas hosts pushed A2UI without auto-reload | +| Control UI i18n | ✅ | ❌ | P3 | English, Chinese, Portuguese; expanded with Persian (fa), Dutch (nl), Vietnamese (vi), Italian (it), Arabic (ar), Thai (th), Traditional Chinese (zh-TW) | | WebChat theme sync | ✅ | ❌ | P3 | Sync with system dark/light mode | | Partial output on abort | ✅ | ❌ | P2 | Preserve partial output when aborting | +| PWA + Web Push | ✅ | ❌ | P3 | PWA install + Web Push notifications for Gateway chat | +| Talk Mode (browser realtime voice) | ✅ | ❌ | P3 | OpenAI Realtime + Google Live WebSocket; Gateway-minted ephemeral secrets; backend realtime relay | +| Steer queued messages | ✅ | ❌ | P3 | Steer action on queued messages injects follow-up into active run without retyping | +| Quick Settings dashboard | ✅ | ❌ | P3 | Refreshed grid + presets + quick-create flows + assistant avatar overrides | +| Markdown preview dialog | ✅ | ❌ | P3 | Lazy markdown preview + `@create-markdown/preview` v2 system theme | +| Cron job dashboard | ✅ | ❌ | P3 | Cron prompts/run summaries as sanitized markdown | +| Personal identity (operator) | ✅ | ❌ | P3 | Browser-local operator name + avatar through shared chat/avatar path | +| Trajectory export UI | ✅ | ❌ | P3 | Owner-private export approval flow | +| Restart-impacting Dreaming confirm | ✅ | ❌ | P3 | Restart warning before applying Dreaming mode changes | +| Mobile chat settings sheet | ✅ | ❌ | P3 | Persists mobile state through Lit-managed view-state | ### Owner: _Unassigned_ @@ -435,31 +646,43 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| -| Cron jobs | ✅ | ✅ | - | Routines with cron trigger | +| Cron jobs | ✅ | ✅ | - | Routines with cron trigger; runtime state split into `jobs-state.json`; `sessionTarget: "current"`/`session:` bindings | | Per-job model fallback override | ✅ | ❌ | P2 | `payload.fallbacks` overrides agent-level fallbacks | | Cron stagger controls | ✅ | ❌ | P3 | Default stagger for scheduled jobs | | Cron finished-run webhook | ✅ | ❌ | P3 | Webhook on job completion | -| Timezone support | ✅ | ✅ | - | Via cron expressions | +| `--thread-id` cron CLI | ✅ | 🚧 | P2 | Telegram forum topic delivery for scheduled announcements | +| `failureAlert.includeSkipped` | ✅ | ❌ | P3 | Persistently skipped jobs alert without counting skips as exec errors | +| `delivery.threadId` (gateway cron schemas) | ✅ | ❌ | P2 | Telegram forum topics + threaded channel destinations | +| Cron `nested` lane | ✅ | ❌ | P3 | `cron.maxConcurrentRuns` applies to dedicated `cron-nested` lane; non-cron flows keep their own lane | +| Cron stuck-session timeout | ✅ | ❌ | P3 | Aborts/cleans timed-out isolated turns before recording timeout | +| Timezone support | ✅ | ✅ | - | Via cron expressions; `--at` honors local wall-clock time across DST | | One-shot/recurring jobs | ✅ | ✅ | - | Manual + cron triggers | | Channel health monitor | ✅ | ❌ | P2 | Auto-restart with configurable interval | | `beforeInbound` hook | ✅ | ✅ | P2 | | | `beforeOutbound` hook | ✅ | ✅ | P2 | | | `beforeToolCall` hook | ✅ | ✅ | P2 | | | `before_agent_start` hook | ✅ | ❌ | P2 | Model/provider override | +| `before_agent_finalize` hook | ✅ | ❌ | P2 | Run/message/sender/session/trace correlation | | `before_message_write` hook | ✅ | ❌ | P2 | Pre-write interception | +| `before_dispatch` hook | ✅ | ❌ | P2 | Canonical inbound metadata; idempotency-key dedupe for hook agent deliveries | +| `before_compaction`/`after_compaction` | ✅ | ❌ | P3 | Codex-native compaction lifecycle | | `onMessage` hook | ✅ | ✅ | - | Routines with event trigger | | Structured system-event routines | ✅ | ✅ | P2 | `system_event` trigger + `event_emit` tool for event-driven automation | | `onSessionStart` hook | ✅ | ✅ | P2 | | | `onSessionEnd` hook | ✅ | ✅ | P2 | | | `transcribeAudio` hook | ✅ | ❌ | P3 | | | `transformResponse` hook | ✅ | ✅ | P2 | | -| `llm_input`/`llm_output` hooks | ✅ | ❌ | P3 | LLM payload inspection | +| `llm_input`/`llm_output` hooks | ✅ | ❌ | P3 | LLM payload inspection (Codex app-server included) | +| `model_call_started`/`ended` hooks | ✅ | ❌ | P3 | Metadata-only model/provider call telemetry | +| `cron_changed` hook | ✅ | ❌ | P3 | Typed gateway-owned cron lifecycle observer | +| Cron `jobId` hook context | ✅ | ❌ | P3 | Hook context carries originating job id | | Bundled hooks | ✅ | ✅ | P2 | Audit + declarative rule/webhook hooks | | Plugin hooks | ✅ | ✅ | P3 | Registered from WASM `capabilities.json` | -| Workspace hooks | ✅ | ✅ | P2 | `hooks/hooks.json` and `hooks/*.hook.json` | +| Workspace hooks | ✅ | ✅ | P2 | `hooks/hooks.json` and `hooks/*.hook.json`; realpath-fail-closed | | Outbound webhooks | ✅ | ✅ | P2 | Fire-and-forget lifecycle event delivery | -| Heartbeat system | ✅ | ✅ | - | Periodic execution | +| Heartbeat system | ✅ | ✅ | - | Periodic execution; `heartbeat.skipWhenBusy` for nested lane pressure; deferred under cron load | | Gmail pub/sub | ✅ | ❌ | P3 | | +| Inferred follow-up commitments | ✅ | ❌ | P3 | Heartbeat-delivered reminders; opt-in batched extraction | ### Owner: _Unassigned_ @@ -469,37 +692,50 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Gateway token auth | ✅ | ✅ | Bearer token auth on web gateway | -| Device pairing | ✅ | ❌ | | -| Tailscale identity | ✅ | ❌ | | -| Trusted-proxy auth | ✅ | ❌ | Header-based reverse proxy auth | -| OAuth flows | ✅ | 🚧 | NEAR AI OAuth + Gemini OAuth (PKCE, S256) + hosted extension/MCP OAuth broker; external auth-proxy rollout still pending | +| Gateway token auth | ✅ | ✅ | Bearer token auth on web gateway; per-request resolution for `secrets.reload`; method-specific least-privilege scopes for CLI Gateway calls | +| Device pairing | ✅ | ❌ | Single-use bootstrap setup codes; metadata-upgrade auto-approval for shared-secret loopback; scope/role/metadata pairing approval flows | +| Tailscale identity | ✅ | ❌ | Tailscale-authenticated Control UI bypass for browser device identity | +| Trusted-proxy auth | ✅ | ❌ | Header-based reverse proxy auth; `trustedProxy.allowLoopback` | +| OAuth flows | ✅ | 🚧 | NEAR AI OAuth + Gemini OAuth (PKCE, S256) + hosted extension/MCP OAuth broker; external auth-proxy rollout still pending; OpenClaw added bootstrap-token redemption scope allowlist | | DM pairing verification | ✅ | ✅ | ironclaw pairing approve, host APIs | -| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store | -| Per-group tool policies | ✅ | ❌ | | -| Exec approvals | ✅ | ✅ | TUI overlay | +| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store; canonical `dmPolicy="open"` only with effective wildcard across all channels | +| Per-group tool policies | ✅ | ❌ | Group-id validation against session/spawned context before applying group-scoped tool policies | +| Exec approvals | ✅ | ✅ | TUI overlay; `allow-once` idempotent grace; PATH-resolved basenames; secret redaction in approval prompts; Unicode normalization + zero-width stripping | +| Owner allowlists | ✅ | ❌ | `commands.ownerAllowFrom` bootstrapped from first approved DM pairing; channel-prefixed entries scoped to matching providers | | TLS 1.3 minimum | ✅ | ✅ | reqwest rustls | -| SSRF protection | ✅ | ✅ | WASM allowlist | +| SSRF protection | ✅ | ✅ | WASM allowlist; OpenClaw extended SSRF guard to BlueBubbles, Synology Chat, LINE, QQBot direct-upload, Tlon uploads, browser tabs/snapshots, voice-call Twilio webhooks, web fetch (incl. `fc00::/7` opt-in) | | SSRF IPv6 transition bypass block | ✅ | ❌ | Block IPv4-mapped IPv6 bypasses | | Cron webhook SSRF guard | ✅ | ❌ | SSRF checks on webhook delivery | | Loopback-first | ✅ | 🚧 | HTTP binds 0.0.0.0 | -| Docker sandbox | ✅ | ✅ | Orchestrator/worker containers | -| Podman support | ✅ | ❌ | Alternative to Docker | +| Docker sandbox | ✅ | ✅ | Orchestrator/worker containers; opt-in `sandbox.docker.gpus` passthrough | +| Podman support | ✅ | ❌ | `--container` accepts both Docker + Podman | | WASM sandbox | ❌ | ✅ | IronClaw innovation | | Sandbox env sanitization | ✅ | 🚧 | Shell tool scrubs env vars (secret detection); docker container env sanitization partial | +| `OPENCLAW_*` env block | ✅ | ❌ | Untrusted workspace `.env` cannot inject OpenClaw runtime-control vars | +| Workspace `.env` injection blocks | ✅ | ❌ | Block `CLOUDSDK_PYTHON`, ambient Homebrew, Windows system PATH vars, `MINIMAX_API_HOST`, `npm_execpath` | | Tool policies | ✅ | ✅ | | | Elevated mode | ✅ | ❌ | | -| Safe bins allowlist | ✅ | ❌ | Hardened path trust | -| LD*/DYLD* validation | ✅ | ❌ | | -| Path traversal prevention | ✅ | ✅ | Including config includes (OC-06) + workspace-only tool mounts | +| Safe bins allowlist | ✅ | ❌ | Hardened path trust; non-user-writable absolute helpers for CLI/ffmpeg/OpenSSL | +| LD*/DYLD* validation | ✅ | ❌ | Block Mercurial/Rust/Make env redirects in host exec sanitization | +| Path traversal prevention | ✅ | ✅ | Including config includes (OC-06) + workspace-only tool mounts; `realpath`-via-fd safety on agents.files.get/set | | Credential theft via env injection | ✅ | 🚧 | Shell env scrubbing + command injection detection; no full OC-09 defense | | Session file permissions (0o600) | ✅ | ✅ | Session token file set to 0o600 in llm/session.rs | | Skill download path restriction | ✅ | ❌ | Validated download roots prevent arbitrary write targets | -| Webhook signature verification | ✅ | ✅ | | -| Media URL validation | ✅ | ❌ | | -| Prompt injection defense | ✅ | ✅ | Pattern detection, sanitization | +| Skill installer metadata validation | ✅ | ❌ | Strict per-PM regex allowlists; URL protocol allowlist; sanitize metadata for terminal output | +| Webhook signature verification | ✅ | ✅ | Padded timing-safe compare even on wrong-length signatures (Nextcloud Talk, Feishu, LINE, Zalo) | +| Media URL validation | ✅ | ❌ | Reject non-HTTP(S) inbound attachment URLs; reject remote-host `file://` URLs in webchat embedding path | +| Prompt injection defense | ✅ | ✅ | Pattern detection, sanitization; OpenClaw added chat-template special-token stripping (Qwen/ChatML, Llama, Gemma, Mistral, Phi, GPT-OSS) | +| Internal scaffolding stripping | ✅ | ❌ | ``/`` stripped at final delivery boundary | | Leak detection | ✅ | ✅ | Secret exfiltration | | Dangerous tool re-enable warning | ✅ | ❌ | Warn when gateway.tools.allow re-enables HTTP tools | +| OpenGrep static analysis | ✅ | ❌ | Bundled rulepack + source-rule compiler + provenance check; PR/full scan workflows + SARIF upload to GitHub Code Scanning | +| Logging redaction expansion | ✅ | ❌ | Tencent/Alibaba/HuggingFace/Replicate API keys; payment credential field names; `sk-*`/Bearer/Authorization tokens at console + file sinks | +| Trace context propagation | ✅ | ❌ | W3C `traceparent` from trusted model-call context; replaces caller-supplied values | +| Forwarded-header IP detection | ✅ | ❌ | Treat any `Forwarded`/`X-Forwarded-*`/`X-Real-IP` as proxied before pairing locality checks | +| Trusted-content sanitization | ✅ | ❌ | Group/channel names rendered through fenced untrusted-metadata JSON; vCard/contact/location free-text neutralization | +| Per-tool MCP loopback policy | ✅ | ❌ | Owner-only tool visibility derived from authenticated owner-vs-non-owner bearers; no caller-controlled owner header | +| Mobile pairing TLS requirement | ✅ | ❌ | Plaintext `ws://` only on loopback; `OPENCLAW_ALLOW_INSECURE_PRIVATE_WS` for trusted private nets | +| Webhook auth rate-limit | ✅ | ❌ | Pre-auth `429` for bad webhook secrets (Zalo, etc.) | ### Owner: _Unassigned_ @@ -578,15 +814,21 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - ❌ Discord channel - 🚧 Matrix channel (native SDK + poll fallback; E2EE enabled via feature gate; media and rich threading pending) -- ❌ Other messaging platforms -- ❌ TTS/audio features -- ❌ Video support +- ❌ Other messaging platforms (Yuanbao, WeCom, Google Meet, Voice Call) +- ❌ TTS/audio features (12+ providers added in OpenClaw; see Section 6 TTS/STT subsection) +- ❌ Video support (OpenRouter/MiniMax/Veo/fal/Sora) - 🚧 Skills routing blocks (activation criteria exist, but no "Use when / Don't use when") -- ❌ Plugin registry +- ❌ Plugin registry / persisted plugin index / `git:` installs - ❌ Streaming (block/tool/Z.AI tool_stream) -- ❌ Memory: temporal decay, MMR re-ranking, query expansion -- ❌ Control UI i18n +- ❌ Memory: temporal decay, MMR re-ranking, query expansion, multimodal indexing, people-aware wiki +- ❌ Control UI i18n (now 12+ locales upstream) - ❌ Stuck loop detection +- ❌ Codex native app-server runtime + Computer Use +- ❌ Talk Mode / realtime voice (browser + backend) +- ❌ OpenTelemetry diagnostics + Prometheus exporter +- ❌ Active Memory + Skill Workshop + Trajectory export +- ❌ Outbound proxy routing + `proxy validate` +- ❌ `migrate` (Claude/Hermes import) --- diff --git a/README.md b/README.md index fc026f60fdb..331f944f0b5 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ IronClaw is the AI assistant you can actually trust with your personal and profe - Rust 1.92+ - PostgreSQL 15+ with [pgvector](https://github.com/pgvector/pgvector) extension - NEAR AI account (authentication handled via setup wizard) +- `libclang` and a working C toolchain if you build the WeChat voice/SILK path from source ## Download or Build @@ -144,6 +145,14 @@ cargo test For **full release** (after modifying channel sources), run `./scripts/build-all.sh` to rebuild channels first. +> **Optional:** WeChat voice notes (`audio/silk`) require the standalone +> `ironclaw-silk-decoder` helper to be transcribable. It's excluded from the +> default workspace build because `silk-codec` pulls in `bindgen`/`libclang`. +> Build it separately with `./crates/ironclaw_silk_decoder/build.sh` (needs +> libclang + a C toolchain) and put the resulting binary on `$PATH`, beside +> the `ironclaw` binary, or pointed at by `IRONCLAW_SILK_DECODER`. Without +> it, voice messages are still delivered — just as raw `audio/silk` blobs. + ### Database Setup diff --git a/README.zh-CN.md b/README.zh-CN.md index f41cfccfda7..11696854926 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -84,6 +84,7 @@ IronClaw 是一个你真正可以信赖的 AI 助手,无论是个人生活还 - Rust 1.85+ - PostgreSQL 15+,需安装 [pgvector](https://github.com/pgvector/pgvector) 扩展 - NEAR AI 账户(通过设置向导进行身份验证) +- 如果从源码构建微信语音 / SILK 路径,还需要安装 `libclang` 和可用的 C 编译工具链 ## 下载或编译 @@ -141,6 +142,14 @@ cargo test 如需进行**完整发布构建**(修改了渠道源码后),先运行 `./scripts/build-all.sh` 重新编译渠道。 +> **可选:** 微信语音消息(`audio/silk`)需要独立的 `ironclaw-silk-decoder` +> 辅助二进制才能被转写。该工具因为依赖 `silk-codec`(会引入 `bindgen` / +> `libclang`),已被排除在默认工作区之外。请使用 +> `./crates/ironclaw_silk_decoder/build.sh` 单独编译(需要 libclang 与 C +> 编译器),再把生成的二进制放到 `$PATH`、`ironclaw` 同级目录,或通过 +> `IRONCLAW_SILK_DECODER` 环境变量指定路径。即使没有它,语音消息仍会送达, +> 只是会以原始的 `audio/silk` 数据块形式呈现。 + ### 数据库设置 diff --git a/channels-src/slack/src/lib.rs b/channels-src/slack/src/lib.rs index ec54f506fb1..fe4b659a3fb 100644 --- a/channels-src/slack/src/lib.rs +++ b/channels-src/slack/src/lib.rs @@ -974,6 +974,203 @@ fn send_pairing_reply(channel_id: &str, code: &str) -> Result<(), String> { } } +// Private-use Unicode chars used as internal sentinels by `markdown_to_mrkdwn` +// to bracket references into the `protected` arena. User input is filtered of +// these before processing, so any remaining occurrence after step 1 was +// written by our own code and refers to a valid arena index. +const MRKDWN_PROTECT_START: char = '\u{E000}'; +const MRKDWN_PROTECT_END: char = '\u{E001}'; + +/// Single-pass expansion of `` sentinel +/// references in `s` against the `protected` arena. Unrecognized references +/// (bad index, non-numeric body) are emitted verbatim. Used both for the +/// final restore and for pre-expanding URL/text before pushing a generated +/// link span (so the span we push contains no further references). +fn expand_protected_spans(s: &str, protected: &[String]) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + loop { + let Some(start) = rest.find(MRKDWN_PROTECT_START) else { + out.push_str(rest); + break; + }; + out.push_str(&rest[..start]); + let after_start = &rest[start + MRKDWN_PROTECT_START.len_utf8()..]; + let Some(end) = after_start.find(MRKDWN_PROTECT_END) else { + out.push_str(&rest[start..]); + break; + }; + let idx_str = &after_start[..end]; + match idx_str.parse::().ok().and_then(|i| protected.get(i)) { + Some(span) => out.push_str(span), + None => { + out.push(MRKDWN_PROTECT_START); + out.push_str(idx_str); + out.push(MRKDWN_PROTECT_END); + } + } + rest = &after_start[end + MRKDWN_PROTECT_END.len_utf8()..]; + } + out +} + +/// Escape characters that would break Slack's `` parser when they +/// appear in the visible label. Slack uses `<` / `>` for the literal +/// `<` / `>` characters; `&` is left alone because escaping it as `&` +/// would double-escape input the agent already encoded. +fn escape_mrkdwn_label(text: &str) -> String { + text.replace('<', "<").replace('>', ">") +} + +fn markdown_to_mrkdwn(input: &str) -> String { + if input.is_empty() { + return String::new(); + } + + // Strip our internal sentinel chars from untrusted input so a message + // containing literal `\u{E000}N\u{E001}` cannot interfere with the + // protect/restore mechanism below. + let sanitized: String = input + .chars() + .filter(|c| *c != MRKDWN_PROTECT_START && *c != MRKDWN_PROTECT_END) + .collect(); + let input = sanitized.as_str(); + + let mut protected: Vec = Vec::new(); + let mut tmp = String::with_capacity(input.len()); + + // Protect Slack-native <...> constructs so we don't rewrite inside them. + // + // NOTE: We must not index `input` by byte offsets that are not UTF-8 + // character boundaries. Slack's special constructs are ASCII-only, but + // messages can contain arbitrary Unicode elsewhere. + let mut i = 0; + while i < input.len() { + let ch = input[i..].chars().next().unwrap(); + if ch == '<' { + let start = i; + i += ch.len_utf8(); + + // Scan forward to the next '>' (ASCII) without assuming anything + // about intervening UTF-8. + let mut j = i; + let mut found_end = None; + while j < input.len() { + let c = input[j..].chars().next().unwrap(); + if c == '>' { + found_end = Some(j + c.len_utf8()); + break; + } + j += c.len_utf8(); + } + + if let Some(end) = found_end { + let span = &input[start..end]; + let idx = protected.len(); + protected.push(span.to_string()); + tmp.push(MRKDWN_PROTECT_START); + tmp.push_str(&idx.to_string()); + tmp.push(MRKDWN_PROTECT_END); + i = end; + continue; + } + + // Unmatched '<' — treat as a literal. + tmp.push('<'); + continue; + } + + tmp.push(ch); + i += ch.len_utf8(); + } + + // Convert headings per-line. + let mut out = String::with_capacity(tmp.len()); + for (line_idx, line) in tmp.split('\n').enumerate() { + if line_idx > 0 { + out.push('\n'); + } + + if let Some(rest) = line.strip_prefix("# ") { + out.push('*'); + out.push_str(rest); + out.push('*'); + } else { + out.push_str(line); + } + } + + // Convert [text](url) -> . + let mut link_out = String::with_capacity(out.len()); + let mut s = out.as_str(); + while let Some(open_bracket) = s.find('[') { + link_out.push_str(&s[..open_bracket]); + s = &s[open_bracket..]; + + let Some(close_bracket) = s.find(']') else { + link_out.push_str(s); + s = ""; + break; + }; + let text = &s[1..close_bracket]; + + let after_bracket = &s[close_bracket + 1..]; + if !after_bracket.starts_with('(') { + link_out.push_str(&s[..close_bracket + 1]); + s = after_bracket; + continue; + } + + let Some(close_paren) = after_bracket.find(')') else { + link_out.push_str(&s[..close_bracket + 1]); + s = after_bracket; + continue; + }; + let url = &after_bracket[1..close_paren]; + + // Expand any sentinel refs embedded in url/text by step 1 so the + // link span we push into the arena contains no further references — + // the final restore pass does not re-scan content it has already + // emitted, so a buried sentinel would otherwise leak into the output + // as raw U+E000/U+E001 characters. + let url_expanded = expand_protected_spans(url, &protected); + let text_expanded = expand_protected_spans(text, &protected); + + // If the URL contains characters that would break Slack's + // `` parser, fall back to leaving the original markdown + // form intact. RFC 3986 disallows these in URLs anyway. + if url_expanded.contains(['<', '>', '|']) { + link_out.push('['); + link_out.push_str(text); + link_out.push_str("]("); + link_out.push_str(url); + link_out.push(')'); + s = &after_bracket[close_paren + 1..]; + continue; + } + + // Push the generated `` into the protected arena so the + // global `**`/`~~` replacement below can't rewrite anything inside + // it. Escape `<` / `>` in the visible label so they render literally + // rather than opening/closing a Slack span. + let span = format!("<{}|{}>", url_expanded, escape_mrkdwn_label(&text_expanded)); + let idx = protected.len(); + protected.push(span); + link_out.push(MRKDWN_PROTECT_START); + link_out.push_str(&idx.to_string()); + link_out.push(MRKDWN_PROTECT_END); + + s = &after_bracket[close_paren + 1..]; + } + link_out.push_str(s); + + // Convert **bold** -> *bold* and ~~strike~~ -> ~strike~. + // This is intentionally minimal and does not attempt full Markdown parsing. + let out = link_out.replace("~~", "~").replace("**", "*"); + + expand_protected_spans(&out, &protected) +} + /// Interpret a Slack `chat.postMessage` response body (returned with HTTP 200) /// as either success or a scoped failure. Extracted so the parsing logic can /// be unit-tested without the `channel_host` extern — see #1839. @@ -1006,7 +1203,8 @@ fn post_slack_message( text: &str, thread_ts: Option<&str>, ) -> Result, String> { - let payload = build_broadcast_payload(channel, text, thread_ts); + let converted = markdown_to_mrkdwn(text); + let payload = build_broadcast_payload(channel, &converted, thread_ts); let payload_bytes = serde_json::to_vec(&payload) .map_err(|e| format!("Failed to serialize payload: {}", e))?; @@ -1597,4 +1795,86 @@ mod tests { assert!(looks_like_slack_id(target)); assert_eq!(target, "C0123ABC"); } + + #[test] + fn test_markdown_to_mrkdwn_bold() { + assert_eq!(markdown_to_mrkdwn("a **b** c"), "a *b* c"); + } + + #[test] + fn test_markdown_to_mrkdwn_strike() { + assert_eq!(markdown_to_mrkdwn("~~x~~"), "~x~"); + } + + #[test] + fn test_markdown_to_mrkdwn_heading_multiline() { + assert_eq!(markdown_to_mrkdwn("# Title\nbody"), "*Title*\nbody"); + } + + #[test] + fn test_markdown_to_mrkdwn_link() { + assert_eq!( + markdown_to_mrkdwn("[near](https://example.com)"), + "" + ); + } + + #[test] + fn test_markdown_to_mrkdwn_preserves_slack_native_formatting() { + let input = "<@U123> <#C123|chan>"; + assert_eq!(markdown_to_mrkdwn(input), input); + } + + #[test] + fn test_markdown_to_mrkdwn_preserves_emphasis_inside_generated_link() { + // After `[text](url)` becomes ``, the global `**`/`~~` + // rewrite must not reach inside the generated span. Bold around the + // link is still converted; bold inside the link text stays literal. + assert_eq!( + markdown_to_mrkdwn("see [**bold**](https://e.com) **after**"), + "see *after*", + ); + } + + #[test] + fn test_markdown_to_mrkdwn_strips_internal_sentinel_chars() { + // Untrusted input containing the private-use sentinels must not be + // able to forge a protected-span reference or split surrounding + // formatting markers. The chars are stripped at the boundary. + let input = "a\u{E000}0\u{E001}b **c**"; + assert_eq!(markdown_to_mrkdwn(input), "a0b *c*"); + } + + #[test] + fn test_markdown_to_mrkdwn_escapes_brackets_in_link_label() { + // `<` and `>` in the visible label would otherwise open/close a + // Slack span and break the link. They must render as literal text. + assert_eq!( + markdown_to_mrkdwn("[ac](https://e.com)"), + "", + ); + } + + #[test] + fn test_markdown_to_mrkdwn_falls_back_when_url_has_pipe_or_gt() { + // A `|` or `>` inside the URL would corrupt ``. The + // converter leaves the original markdown form intact instead. + assert_eq!( + markdown_to_mrkdwn("[x](https://e.com/a|b)"), + "[x](https://e.com/a|b)", + ); + } + + #[test] + fn test_markdown_to_mrkdwn_link_with_slack_native_span_inside_label() { + // A Slack-native `<...>` span inside the markdown link label is + // protected by step 1, then expanded again before the link span is + // pushed — guaranteeing the final output contains no leftover + // sentinel characters and the bracket chars are escaped for the + // label. + assert_eq!( + markdown_to_mrkdwn("[<@U1> there](https://e.com)"), + "", + ); + } } diff --git a/channels-src/telegram/src/lib.rs b/channels-src/telegram/src/lib.rs index c82431ffdcc..214255688e5 100644 --- a/channels-src/telegram/src/lib.rs +++ b/channels-src/telegram/src/lib.rs @@ -1826,11 +1826,25 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<() /// Send a pairing code message to a chat. Used when an unknown user DMs the bot. fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> { + // The reply must name the IronClaw surface explicitly. An earlier wording + // ("Enter this code in IronClaw…") was ambiguous: users naturally pasted + // the code into their TUI/CLI chat, where there was no handler for it. + // The agent now also accepts `approve telegram ` typed in any chat + // surface, so we surface that path alongside the web/CLI options. + // + // Telegram itself is *not* listed as a chat surface here: the recipient + // is by definition unpaired, so their DMs are intercepted by the + // allowlist gate in `handle_message` *before* the agent parser ever + // sees `approve telegram `. They'd just get another pairing + // reply. Pairing approval requires an already-authenticated IronClaw + // surface (web / TUI / CLI). Reference: PR review on #3381. send_message( chat_id, &format!( - "Enter this code in IronClaw to pair your telegram account: `{}`. CLI fallback: `ironclaw pairing approve telegram {}`", - code, code + "Pair this Telegram account with IronClaw using one of:\n\ + • Web: Settings → Channels → Telegram → paste `{code}`\n\ + • Any signed-in IronClaw chat (TUI / web): type `approve telegram {code}`\n\ + • Terminal: `ironclaw pairing approve telegram {code}`", ), None, Some("Markdown"), diff --git a/channels-src/wechat/.gitignore b/channels-src/wechat/.gitignore new file mode 100644 index 00000000000..fa74b8eb077 --- /dev/null +++ b/channels-src/wechat/.gitignore @@ -0,0 +1,2 @@ +/target +/*.wasm diff --git a/channels-src/wechat/Cargo.lock b/channels-src/wechat/Cargo.lock new file mode 100644 index 00000000000..655c86f528f --- /dev/null +++ b/channels-src/wechat/Cargo.lock @@ -0,0 +1,568 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "spdx" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-encoder" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1" +dependencies = [ + "leb128", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap", + "semver", +] + +[[package]] +name = "wechat-channel" +version = "0.1.0" +dependencies = [ + "aes", + "base64", + "cipher", + "md-5", + "rand", + "serde", + "serde_json", + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3" +dependencies = [ + "wit-bindgen-rt", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/channels-src/wechat/Cargo.toml b/channels-src/wechat/Cargo.toml new file mode 100644 index 00000000000..e4384e2b338 --- /dev/null +++ b/channels-src/wechat/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "wechat-channel" +version = "0.1.0" +edition = "2021" +description = "WeChat iLink Bot channel for IronClaw" +license = "MIT OR Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.36" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +base64 = "0.22" +aes = "0.8" +cipher = "0.4" +md-5 = "0.10" +rand = "0.8" + +[profile.release] +opt-level = "s" +lto = true +strip = true +codegen-units = 1 + +[workspace] diff --git a/channels-src/wechat/build.sh b/channels-src/wechat/build.sh new file mode 100755 index 00000000000..c31881035c1 --- /dev/null +++ b/channels-src/wechat/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" + +echo "Building WeChat channel WASM component..." + +cargo build --release --target wasm32-wasip2 + +WASM_PATH="target/wasm32-wasip2/release/wechat_channel.wasm" + +if [ -f "$WASM_PATH" ]; then + if command -v wasm-tools >/dev/null 2>&1; then + wasm-tools component new "$WASM_PATH" -o wechat.wasm 2>/dev/null || cp "$WASM_PATH" wechat.wasm + wasm-tools strip wechat.wasm -o wechat.wasm + else + cp "$WASM_PATH" wechat.wasm + echo "wasm-tools not found; copied raw wasm output without component conversion/strip" + fi + + echo "Built: wechat.wasm ($(du -h wechat.wasm | cut -f1))" + echo "" + echo "To install:" + echo " mkdir -p ~/.ironclaw/channels" + echo " cp wechat.wasm wechat.capabilities.json ~/.ironclaw/channels/" +else + echo "Error: WASM output not found at $WASM_PATH" + exit 1 +fi diff --git a/channels-src/wechat/src/api.rs b/channels-src/wechat/src/api.rs new file mode 100644 index 00000000000..6e2adab9ea5 --- /dev/null +++ b/channels-src/wechat/src/api.rs @@ -0,0 +1,289 @@ +use base64::Engine as _; +use rand::RngCore; + +use crate::near::agent::channel_host; +use crate::types::{ + BaseInfo, GetConfigRequest, GetConfigResponse, GetUpdatesRequest, GetUpdatesResponse, + GetUploadUrlRequest, GetUploadUrlResponse, MessageItem, OutboundWechatMessage, + SendMessageRequest, SendTypingRequest, SendTypingResponse, TextItem, WechatConfig, + MESSAGE_ITEM_TEXT, MESSAGE_STATE_FINISH, MESSAGE_TYPE_BOT, +}; + +pub fn base_info() -> BaseInfo { + BaseInfo { + channel_version: env!("CARGO_PKG_VERSION").to_string(), + } +} + +fn ensure_trailing_slash(base_url: &str) -> String { + if base_url.ends_with('/') { + base_url.to_string() + } else { + format!("{base_url}/") + } +} + +fn random_wechat_uin() -> String { + let mut bytes = [0_u8; 12]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes) +} + +fn request_headers(body: &[u8]) -> String { + serde_json::json!({ + "Content-Type": "application/json", + "AuthorizationType": "ilink_bot_token", + "Authorization": "Bearer {WECHAT_BOT_TOKEN}", + "Content-Length": body.len().to_string(), + "X-WECHAT-UIN": random_wechat_uin(), + }) + .to_string() +} + +pub fn get_updates( + config: &WechatConfig, + get_updates_buf: &str, +) -> Result { + get_updates_with_timeout(config, get_updates_buf, config.long_poll_timeout_ms) +} + +pub fn get_updates_with_timeout( + config: &WechatConfig, + get_updates_buf: &str, + timeout_ms: u32, +) -> Result { + let body = serde_json::to_vec(&GetUpdatesRequest { + get_updates_buf: get_updates_buf.to_string(), + base_info: base_info(), + }) + .map_err(|e| format!("Failed to encode getUpdates request: {e}"))?; + let headers = request_headers(&body); + let url = format!( + "{}ilink/bot/getupdates", + ensure_trailing_slash(&config.base_url) + ); + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "WeChat getUpdates request: cursor_len={} timeout_ms={}", + get_updates_buf.len(), + timeout_ms + ), + ); + let response = + channel_host::http_request("POST", &url, &headers, Some(&body), Some(timeout_ms)) + .map_err(|e| format!("getUpdates request failed: {e}"))?; + + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "WeChat getUpdates response: status={} bytes={}", + response.status, + response.body.len() + ), + ); + + if response.status != 200 { + let body = String::from_utf8_lossy(&response.body); + return Err(format!("getUpdates returned {}: {}", response.status, body)); + } + + let parsed: GetUpdatesResponse = serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getUpdates response: {e}"))?; + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "WeChat getUpdates parsed: ret={:?} errcode={:?} msg_count={} next_cursor_len={}", + parsed.ret, + parsed.errcode, + parsed.msgs.len(), + parsed.get_updates_buf.as_deref().unwrap_or_default().len() + ), + ); + Ok(parsed) +} + +pub fn send_text_message( + config: &WechatConfig, + to_user_id: &str, + text: &str, + context_token: Option<&str>, +) -> Result<(), String> { + let message = SendMessageRequest { + msg: OutboundWechatMessage { + from_user_id: String::new(), + to_user_id: to_user_id.to_string(), + client_id: format!("wechat-{}", channel_host::now_millis()), + message_type: MESSAGE_TYPE_BOT, + message_state: MESSAGE_STATE_FINISH, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_TEXT), + text_item: Some(TextItem { + text: text.to_string(), + }), + image_item: None, + voice_item: None, + file_item: None, + video_item: None, + }], + context_token: context_token.map(str::to_string), + }, + base_info: base_info(), + }; + + send_message_request(config, &message) +} + +pub fn send_message_request( + config: &WechatConfig, + message: &SendMessageRequest, +) -> Result<(), String> { + let body = serde_json::to_vec(message) + .map_err(|e| format!("Failed to encode sendMessage request: {e}"))?; + let headers = request_headers(&body); + let url = format!( + "{}ilink/bot/sendmessage", + ensure_trailing_slash(&config.base_url) + ); + + let response = channel_host::http_request("POST", &url, &headers, Some(&body), Some(15_000)) + .map_err(|e| format!("sendMessage request failed: {e}"))?; + + if response.status != 200 { + let body = String::from_utf8_lossy(&response.body); + return Err(format!( + "sendMessage returned {}: {}", + response.status, body + )); + } + + Ok(()) +} + +pub fn get_upload_url( + config: &WechatConfig, + request: &GetUploadUrlRequest, +) -> Result { + let body = serde_json::to_vec(request) + .map_err(|e| format!("Failed to encode getUploadUrl request: {e}"))?; + let headers = request_headers(&body); + let url = format!( + "{}ilink/bot/getuploadurl", + ensure_trailing_slash(&config.base_url) + ); + + let response = channel_host::http_request("POST", &url, &headers, Some(&body), Some(15_000)) + .map_err(|e| format!("getUploadUrl request failed: {e}"))?; + + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "WeChat getUploadUrl response: status={} bytes={} media_type={} has_thumb_fields={}", + response.status, + response.body.len(), + request.media_type, + request.thumb_rawsize.is_some() + ), + ); + + if response.status != 200 { + let body = String::from_utf8_lossy(&response.body); + return Err(format!( + "getUploadUrl returned {}: {}", + response.status, body + )); + } + + let parsed: GetUploadUrlResponse = serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getUploadUrl response: {e}"))?; + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "WeChat getUploadUrl parsed: has_upload_param={} has_thumb_upload_param={}", + parsed + .upload_param + .as_deref() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false), + parsed + .thumb_upload_param + .as_deref() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false) + ), + ); + Ok(parsed) +} + +pub fn get_config( + config: &WechatConfig, + ilink_user_id: &str, + context_token: Option<&str>, +) -> Result { + let body = serde_json::to_vec(&GetConfigRequest { + ilink_user_id: ilink_user_id.to_string(), + context_token: context_token.map(str::to_string), + base_info: base_info(), + }) + .map_err(|e| format!("Failed to encode getConfig request: {e}"))?; + let headers = request_headers(&body); + let url = format!( + "{}ilink/bot/getconfig", + ensure_trailing_slash(&config.base_url) + ); + + let response = channel_host::http_request("POST", &url, &headers, Some(&body), Some(10_000)) + .map_err(|e| format!("getConfig request failed: {e}"))?; + + if response.status != 200 { + let body = String::from_utf8_lossy(&response.body); + return Err(format!("getConfig returned {}: {}", response.status, body)); + } + + serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getConfig response: {e}")) +} + +pub fn send_typing( + config: &WechatConfig, + ilink_user_id: &str, + typing_ticket: &str, + status: i32, +) -> Result<(), String> { + let body = serde_json::to_vec(&SendTypingRequest { + ilink_user_id: ilink_user_id.to_string(), + typing_ticket: typing_ticket.to_string(), + status, + base_info: base_info(), + }) + .map_err(|e| format!("Failed to encode sendTyping request: {e}"))?; + let headers = request_headers(&body); + let url = format!( + "{}ilink/bot/sendtyping", + ensure_trailing_slash(&config.base_url) + ); + + let response = channel_host::http_request("POST", &url, &headers, Some(&body), Some(10_000)) + .map_err(|e| format!("sendTyping request failed: {e}"))?; + + if response.status != 200 { + let body = String::from_utf8_lossy(&response.body); + return Err(format!("sendTyping returned {}: {}", response.status, body)); + } + + let parsed: SendTypingResponse = serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse sendTyping response: {e}"))?; + + if !matches!(parsed.ret, Some(0)) { + let errmsg = parsed + .errmsg + .as_deref() + .unwrap_or("unknown WeChat sendTyping error"); + return Err(format!( + "sendTyping returned ret={} errmsg={errmsg}", + parsed.ret.unwrap_or(-1) + )); + } + + Ok(()) +} diff --git a/channels-src/wechat/src/auth.rs b/channels-src/wechat/src/auth.rs new file mode 100644 index 00000000000..f1b131d37ce --- /dev/null +++ b/channels-src/wechat/src/auth.rs @@ -0,0 +1,7 @@ +pub const TOKEN_SECRET_NAME: &str = "wechat_bot_token"; +pub const CONFIG_PATH: &str = "config.json"; +pub const GET_UPDATES_BUF_PATH: &str = "state/get_updates_buf.json"; +pub const CONTEXT_TOKENS_PATH: &str = "state/context_tokens.json"; +pub const TYPING_TICKETS_PATH: &str = "state/typing_tickets.json"; +pub const PENDING_INBOUND_PATH: &str = "state/pending_inbound.json"; +pub const PROCESSED_MESSAGE_IDS_PATH: &str = "state/processed_message_ids.json"; diff --git a/channels-src/wechat/src/lib.rs b/channels-src/wechat/src/lib.rs new file mode 100644 index 00000000000..280ef291958 --- /dev/null +++ b/channels-src/wechat/src/lib.rs @@ -0,0 +1,1269 @@ +wit_bindgen::generate!({ + world: "sandboxed-channel", + path: "../../wit/channel.wit", +}); + +mod api; +mod auth; +mod media; +mod state; +mod types; + +use exports::near::agent::channel::{ + AgentResponse, Attachment, ChannelConfig, Guest, PollConfig, StatusType, StatusUpdate, +}; +use near::agent::channel_host::{self, EmittedMessage}; +use serde_json::json; + +use crate::auth::TOKEN_SECRET_NAME; +use crate::state::{ + has_processed_message_id, load_config, load_context_tokens, load_get_updates_buf, + load_pending_inbound_bundles, load_processed_message_ids, load_typing_tickets, persist_config, + persist_context_tokens, persist_get_updates_buf, persist_pending_inbound_bundles, + persist_processed_message_ids, persist_typing_tickets, remember_processed_message_id, + PendingInboundBundle, StoredInboundAttachment, TypingTicketEntry, +}; +use crate::types::{ + OutboundMetadata, WechatConfig, WechatMessage, MESSAGE_ITEM_TEXT, MESSAGE_TYPE_USER, + TYPING_STATUS_CANCEL, TYPING_STATUS_TYPING, +}; + +const TYPING_TICKET_TTL_MS: u64 = 24 * 60 * 60 * 1000; +const MAX_PROCESSED_MESSAGE_IDS: usize = 512; +const ATTACHMENT_DELIVERY_FAILED_FALLBACK: &str = + "I finished the request, but WeChat couldn't deliver the attachment."; + +#[cfg(not(test))] +pub(crate) fn debug_log(message: &str) { + channel_host::log(channel_host::LogLevel::Debug, message); +} + +#[cfg(test)] +pub(crate) fn debug_log(_message: &str) {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WechatStatusAction { + Typing, + Cancel, +} + +struct WechatChannel; + +struct FollowUpState<'a> { + current_cursor: &'a mut String, + context_tokens: &'a mut std::collections::HashMap, + context_tokens_changed: &'a mut bool, + pending_inbound: &'a mut std::collections::HashMap, + pending_inbound_changed: &'a mut bool, + processed_message_ids: &'a mut Vec, + processed_message_ids_changed: &'a mut bool, +} + +fn get_updates_error_message(response: &crate::types::GetUpdatesResponse) -> Option { + let errmsg = response + .errmsg + .as_deref() + .unwrap_or("unknown WeChat polling error"); + + if let Some(ret) = response.ret { + if ret != 0 { + return Some(format!("ret={ret} errmsg={errmsg}")); + } + } + + if let Some(errcode) = response.errcode { + if errcode != 0 { + return Some(format!("errcode={errcode} errmsg={errmsg}")); + } + } + + None +} + +impl Guest for WechatChannel { + fn on_start(config_json: String) -> Result { + let config = serde_json::from_str::(&config_json) + .map_err(|e| format!("Failed to parse WeChat config: {e}"))?; + persist_config(&config)?; + + Ok(ChannelConfig { + display_name: "WeChat".to_string(), + http_endpoints: Vec::new(), + poll: Some(PollConfig { + interval_ms: config.poll_interval_ms.max(30_000), + enabled: true, + }), + }) + } + + fn on_http_request( + _req: exports::near::agent::channel::IncomingHttpRequest, + ) -> exports::near::agent::channel::OutgoingHttpResponse { + exports::near::agent::channel::OutgoingHttpResponse { + status: 404, + headers_json: "{}".to_string(), + body: b"{\"error\":\"wechat channel does not expose webhooks\"}".to_vec(), + } + } + + fn on_poll() { + if !channel_host::secret_exists(TOKEN_SECRET_NAME) { + channel_host::log( + channel_host::LogLevel::Warn, + "WeChat bot token is missing; skipping poll", + ); + return; + } + + let config = load_config(); + let cursor = load_get_updates_buf(); + let mut current_cursor = cursor.clone(); + let mut context_tokens = load_context_tokens(); + let mut pending_inbound = match load_pending_inbound_bundles() { + Ok(bundles) => bundles, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to load WeChat pending inbound bundles: {error}"), + ); + return; + } + }; + let mut processed_message_ids = match load_processed_message_ids() { + Ok(message_ids) => message_ids, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to load WeChat processed message ids: {error}"), + ); + return; + } + }; + let mut pending_inbound_changed = false; + let mut processed_message_ids_changed = false; + + for bundle in take_due_pending_bundles(&mut pending_inbound, channel_host::now_millis()) { + pending_inbound_changed = true; + emit_buffered_bundle(bundle); + } + + match api::get_updates(&config, ¤t_cursor) { + Ok(response) => { + if response.errcode == Some(-14) { + channel_host::log( + channel_host::LogLevel::Error, + "WeChat getUpdates returned errcode=-14; reconnect the channel", + ); + return; + } + + if let Some(error) = get_updates_error_message(&response) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("WeChat getUpdates returned {error}"), + ); + } + + if let Some(next_cursor) = response.get_updates_buf.as_deref() { + if next_cursor != current_cursor { + current_cursor = next_cursor.to_string(); + if let Err(error) = persist_get_updates_buf(next_cursor) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat polling cursor: {error}"), + ); + } + } + } + + let mut context_tokens_changed = false; + for message in response.msgs { + let message_id = message.message_id; + if let Some(message_id) = message_id { + if has_processed_message_id(&processed_message_ids, message_id) { + continue; + } + } + if let Some(from_user_id) = message.from_user_id.as_deref() { + if let Some(context_token) = message.context_token.as_deref() { + let changed = context_tokens + .insert(from_user_id.to_string(), context_token.to_string()) + .as_deref() + != Some(context_token); + context_tokens_changed |= changed; + } + } + match incoming_bundle_from_message(&config, message) { + Ok(Some(bundle)) => { + let bundle_message_id = bundle.message_id; + let emitted = process_incoming_bundle( + &mut pending_inbound, + bundle, + &mut pending_inbound_changed, + channel_host::now_millis(), + u64::from(config.inbound_merge_window_ms), + ); + for emitted_bundle in emitted { + emit_buffered_bundle(emitted_bundle); + } + if let Some(message_id) = bundle_message_id { + processed_message_ids_changed |= remember_processed_message_id( + &mut processed_message_ids, + message_id, + MAX_PROCESSED_MESSAGE_IDS, + ); + } + } + Ok(None) => { + if let Some(message_id) = message_id { + processed_message_ids_changed |= remember_processed_message_id( + &mut processed_message_ids, + message_id, + MAX_PROCESSED_MESSAGE_IDS, + ); + } + } + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to map WeChat inbound message: {error}"), + ); + } + } + } + + collect_follow_up_bundles( + &config, + FollowUpState { + current_cursor: &mut current_cursor, + context_tokens: &mut context_tokens, + context_tokens_changed: &mut context_tokens_changed, + pending_inbound: &mut pending_inbound, + pending_inbound_changed: &mut pending_inbound_changed, + processed_message_ids: &mut processed_message_ids, + processed_message_ids_changed: &mut processed_message_ids_changed, + }, + ); + + for bundle in + take_due_pending_bundles(&mut pending_inbound, channel_host::now_millis()) + { + pending_inbound_changed = true; + emit_buffered_bundle(bundle); + } + + if context_tokens_changed { + if let Err(error) = persist_context_tokens(&context_tokens) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat context tokens: {error}"), + ); + } + } + + if pending_inbound_changed { + if let Err(error) = persist_pending_inbound_bundles(&pending_inbound) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat pending inbound bundles: {error}"), + ); + } + } + + if processed_message_ids_changed { + if let Err(error) = persist_processed_message_ids(&processed_message_ids) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat processed message ids: {error}"), + ); + } + } + } + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("WeChat polling failed: {error}"), + ); + } + } + } + + fn on_respond(response: AgentResponse) -> Result<(), String> { + let metadata = serde_json::from_str::(&response.metadata_json) + .map_err(|e| format!("Invalid WeChat response metadata: {e}"))?; + let config = load_config(); + let context_tokens = load_context_tokens(); + let context_token = metadata + .context_token + .clone() + .or_else(|| context_tokens.get(&metadata.from_user_id).cloned()); + if let Err(error) = send_typing_indicator( + &config, + &metadata, + context_token.as_deref(), + TYPING_STATUS_CANCEL, + false, + ) { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Failed to cancel WeChat typing indicator before reply: {error}"), + ); + } + + debug_log(&format!( + "WeChat on_respond: text_len={} attachments={}", + response.content.len(), + response.attachments.len() + )); + + send_response(&config, &metadata, &response, context_token.as_deref()) + } + + fn on_status(update: StatusUpdate) { + let Some(action) = classify_status_update(&update) else { + return; + }; + let metadata = match serde_json::from_str::(&update.metadata_json) { + Ok(metadata) => metadata, + Err(_) => { + channel_host::log( + channel_host::LogLevel::Debug, + "on_status: no valid WeChat metadata, skipping typing update", + ); + return; + } + }; + let config = load_config(); + let context_tokens = load_context_tokens(); + let context_token = resolve_context_token(&metadata, &context_tokens); + + let (typing_status, allow_ticket_fetch) = match action { + WechatStatusAction::Typing => (TYPING_STATUS_TYPING, true), + WechatStatusAction::Cancel => (TYPING_STATUS_CANCEL, false), + }; + + if let Err(error) = send_typing_indicator( + &config, + &metadata, + context_token.as_deref(), + typing_status, + allow_ticket_fetch, + ) { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("WeChat typing update failed: {error}"), + ); + } + } + + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Ok(()) + } + + fn on_shutdown() {} +} + +fn incoming_bundle_from_message( + config: &WechatConfig, + message: WechatMessage, +) -> Result, String> { + if message.message_type != Some(MESSAGE_TYPE_USER) { + return Ok(None); + } + + let from_user_id = match message.from_user_id.as_deref() { + Some(user_id) => user_id, + None => return Ok(None), + }; + + let text = extract_text(&message); + let attachments = media::extract_inbound_attachments(config, &message)? + .into_iter() + .map(StoredInboundAttachment::from) + .collect::>(); + if text.trim().is_empty() && attachments.is_empty() { + return Ok(None); + } + + Ok(Some(PendingInboundBundle { + from_user_id: from_user_id.to_string(), + to_user_id: message.to_user_id, + session_id: message.session_id, + context_token: message.context_token, + message_id: message.message_id, + flush_at_ms: 0, + text, + attachments, + })) +} + +fn process_incoming_bundle( + pending_inbound: &mut std::collections::HashMap, + mut bundle: PendingInboundBundle, + pending_inbound_changed: &mut bool, + now_ms: u64, + inbound_merge_window_ms: u64, +) -> Vec { + let key = bundle.from_user_id.clone(); + let bundle_has_text = !bundle.text.trim().is_empty(); + let bundle_has_attachments = !bundle.attachments.is_empty(); + + if let Some(mut pending) = pending_inbound.remove(&key) { + *pending_inbound_changed = true; + + if bundle_has_text { + let incoming_metadata = bundle.clone(); + pending.text = merge_text(&pending.text, &bundle.text); + pending.attachments.extend(bundle.attachments); + merge_bundle_metadata(&mut pending, &incoming_metadata); + return vec![pending]; + } + + let incoming_metadata = bundle.clone(); + pending.attachments.extend(bundle.attachments); + merge_bundle_metadata(&mut pending, &incoming_metadata); + pending.flush_at_ms = next_flush_deadline(now_ms, inbound_merge_window_ms); + pending_inbound.insert(key, pending); + return Vec::new(); + } + + if bundle_has_attachments && !bundle_has_text { + *pending_inbound_changed = true; + bundle.flush_at_ms = next_flush_deadline(now_ms, inbound_merge_window_ms); + pending_inbound.insert(key, bundle); + Vec::new() + } else { + vec![bundle] + } +} + +fn collect_follow_up_bundles(config: &WechatConfig, state: FollowUpState<'_>) { + while !state.pending_inbound.is_empty() { + let now_ms = channel_host::now_millis(); + let Some(timeout_ms) = next_follow_up_timeout_ms(state.pending_inbound, now_ms) else { + break; + }; + if timeout_ms == 0 { + break; + } + + let timeout_ms_u32 = timeout_ms.min(u64::from(u32::MAX)) as u32; + let response = + match api::get_updates_with_timeout(config, state.current_cursor, timeout_ms_u32) { + Ok(response) => response, + Err(_) => break, + }; + + if response.errcode == Some(-14) { + channel_host::log( + channel_host::LogLevel::Error, + "WeChat getUpdates returned errcode=-14 during follow-up merge window; reconnect the channel", + ); + break; + } + + if let Some(error) = get_updates_error_message(&response) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("WeChat getUpdates returned {error} during follow-up merge window"), + ); + } + + if let Some(next_cursor) = response.get_updates_buf.as_deref() { + if next_cursor != state.current_cursor { + *state.current_cursor = next_cursor.to_string(); + if let Err(error) = persist_get_updates_buf(next_cursor) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat polling cursor: {error}"), + ); + } + } + } + + let mut saw_relevant_message = false; + for message in response.msgs { + let message_id = message.message_id; + if let Some(message_id) = message_id { + if has_processed_message_id(state.processed_message_ids, message_id) { + continue; + } + } + if let Some(from_user_id) = message.from_user_id.as_deref() { + if let Some(context_token) = message.context_token.as_deref() { + let changed = state + .context_tokens + .insert(from_user_id.to_string(), context_token.to_string()) + .as_deref() + != Some(context_token); + *state.context_tokens_changed |= changed; + } + } + match incoming_bundle_from_message(config, message) { + Ok(Some(bundle)) => { + let bundle_message_id = bundle.message_id; + let emitted = process_incoming_bundle( + state.pending_inbound, + bundle, + state.pending_inbound_changed, + channel_host::now_millis(), + u64::from(config.inbound_merge_window_ms), + ); + if let Some(message_id) = bundle_message_id { + *state.processed_message_ids_changed |= remember_processed_message_id( + state.processed_message_ids, + message_id, + MAX_PROCESSED_MESSAGE_IDS, + ); + } + for emitted_bundle in emitted { + saw_relevant_message = true; + emit_buffered_bundle(emitted_bundle); + } + } + Ok(None) => { + if let Some(message_id) = message_id { + *state.processed_message_ids_changed |= remember_processed_message_id( + state.processed_message_ids, + message_id, + MAX_PROCESSED_MESSAGE_IDS, + ); + } + } + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to map WeChat inbound message: {error}"), + ); + } + } + } + + if !saw_relevant_message && state.pending_inbound.is_empty() { + break; + } + } +} + +fn next_flush_deadline(now_ms: u64, inbound_merge_window_ms: u64) -> u64 { + now_ms.saturating_add(inbound_merge_window_ms) +} + +fn next_follow_up_timeout_ms( + pending_inbound: &std::collections::HashMap, + now_ms: u64, +) -> Option { + pending_inbound + .values() + .map(|bundle| bundle.flush_at_ms.saturating_sub(now_ms)) + .min() +} + +fn take_due_pending_bundles( + pending_inbound: &mut std::collections::HashMap, + now_ms: u64, +) -> Vec { + let due_keys = pending_inbound + .iter() + .filter_map(|(key, bundle)| (bundle.flush_at_ms <= now_ms).then_some(key.clone())) + .collect::>(); + + due_keys + .into_iter() + .filter_map(|key| pending_inbound.remove(&key)) + .collect() +} + +fn emit_buffered_bundle(bundle: PendingInboundBundle) { + let metadata = json!({ + "from_user_id": bundle.from_user_id, + "to_user_id": bundle.to_user_id, + "message_id": bundle.message_id, + "session_id": bundle.session_id, + "context_token": bundle.context_token, + }); + + channel_host::emit_message(&EmittedMessage { + user_id: bundle.from_user_id.clone(), + user_name: None, + content: bundle.text, + thread_id: Some(format!("wechat:{}", bundle.from_user_id)), + metadata_json: metadata.to_string(), + attachments: bundle.attachments.into_iter().map(Into::into).collect(), + }); +} + +fn merge_bundle_metadata(target: &mut PendingInboundBundle, incoming: &PendingInboundBundle) { + if incoming.to_user_id.is_some() { + target.to_user_id = incoming.to_user_id.clone(); + } + if incoming.session_id.is_some() { + target.session_id = incoming.session_id.clone(); + } + if incoming.context_token.is_some() { + target.context_token = incoming.context_token.clone(); + } + if incoming.message_id.is_some() { + target.message_id = incoming.message_id; + } +} + +fn merge_text(existing: &str, incoming: &str) -> String { + let existing = existing.trim(); + let incoming = incoming.trim(); + match (existing.is_empty(), incoming.is_empty()) { + (true, true) => String::new(), + (true, false) => incoming.to_string(), + (false, true) => existing.to_string(), + (false, false) => format!("{existing}\n\n{incoming}"), + } +} + +fn send_response( + config: &WechatConfig, + metadata: &OutboundMetadata, + response: &AgentResponse, + context_token: Option<&str>, +) -> Result<(), String> { + send_response_with_handlers( + response, + |text| api::send_text_message(config, &metadata.from_user_id, text, context_token), + |attachment| match media::classify_outbound_media_kind(&attachment.mime_type) { + media::OutboundMediaKind::Image => media::send_image_attachment( + config, + &metadata.from_user_id, + attachment, + context_token, + ), + media::OutboundMediaKind::Video => media::send_video_attachment( + config, + &metadata.from_user_id, + attachment, + context_token, + ), + media::OutboundMediaKind::File => media::send_file_attachment( + config, + &metadata.from_user_id, + attachment, + context_token, + ), + }, + |message| channel_host::log(channel_host::LogLevel::Warn, &message), + ) +} + +fn send_response_with_handlers( + response: &AgentResponse, + mut send_text: FText, + mut send_attachment: FAttachment, + mut warn: FWarn, +) -> Result<(), String> +where + FText: FnMut(&str) -> Result<(), String>, + FAttachment: FnMut(&Attachment) -> Result<(), String>, + FWarn: FnMut(String), +{ + let remaining_text = response.content.trim().to_string(); + let mut sent_attachment = false; + let mut attachment_failures = 0usize; + + for attachment in &response.attachments { + debug_log(&format!( + "WeChat send_response: sending attachment filename='{}' mime='{}' bytes={}", + attachment.filename, + attachment.mime_type, + attachment.data.len() + )); + match send_attachment(attachment) { + Ok(()) => { + sent_attachment = true; + debug_log(&format!( + "WeChat send_response: attachment sent filename='{}'", + attachment.filename + )); + } + Err(error) => { + attachment_failures += 1; + let filename = if attachment.filename.trim().is_empty() { + "" + } else { + attachment.filename.as_str() + }; + warn(format!( + "Failed to send WeChat attachment '{}' ({}): {}", + filename, attachment.mime_type, error + )); + } + } + } + + let should_send_text = !remaining_text.is_empty() || !sent_attachment; + if should_send_text { + let fallback_text = if !remaining_text.is_empty() { + remaining_text.as_str() + } else if attachment_failures > 0 { + ATTACHMENT_DELIVERY_FAILED_FALLBACK + } else { + remaining_text.as_str() + }; + + debug_log(&format!( + "WeChat send_response: sending final text len={}", + fallback_text.len() + )); + send_text(fallback_text)?; + } + + Ok(()) +} + +fn extract_text(message: &WechatMessage) -> String { + message + .item_list + .iter() + .find_map(|item| { + if item.r#type == Some(MESSAGE_ITEM_TEXT) { + item.text_item.as_ref().map(|item| item.text.clone()) + } else if item.r#type == Some(crate::types::MESSAGE_ITEM_VOICE) { + item.voice_item + .as_ref() + .and_then(|item| item.text.as_ref()) + .cloned() + } else { + None + } + }) + .unwrap_or_default() +} + +fn is_terminal_text_status(message: &str) -> bool { + let trimmed = message.trim(); + trimmed.eq_ignore_ascii_case("done") + || trimmed.eq_ignore_ascii_case("interrupted") + || trimmed.eq_ignore_ascii_case("awaiting approval") + || trimmed.eq_ignore_ascii_case("rejected") +} + +fn classify_status_update(update: &StatusUpdate) -> Option { + match update.status { + StatusType::Thinking => Some(WechatStatusAction::Typing), + StatusType::Done + | StatusType::Interrupted + | StatusType::ApprovalNeeded + | StatusType::AuthRequired => Some(WechatStatusAction::Cancel), + StatusType::Status if is_terminal_text_status(&update.message) => { + Some(WechatStatusAction::Cancel) + } + StatusType::ToolStarted + | StatusType::ToolCompleted + | StatusType::ToolResult + | StatusType::Status + | StatusType::JobStarted + | StatusType::AuthCompleted => None, + } +} + +fn resolve_context_token( + metadata: &OutboundMetadata, + context_tokens: &std::collections::HashMap, +) -> Option { + metadata + .context_token + .clone() + .or_else(|| context_tokens.get(&metadata.from_user_id).cloned()) +} + +fn cached_typing_ticket(user_id: &str) -> Option { + let tickets = load_typing_tickets(); + let ticket = tickets.get(user_id)?; + let trimmed = ticket.ticket.trim(); + if trimmed.is_empty() { + return None; + } + + let age_ms = channel_host::now_millis().saturating_sub(ticket.fetched_at_ms); + if age_ms >= TYPING_TICKET_TTL_MS { + return None; + } + + Some(trimmed.to_string()) +} + +fn persist_typing_ticket(user_id: &str, ticket: &str) -> Result<(), String> { + let mut tickets = load_typing_tickets(); + tickets.insert( + user_id.to_string(), + TypingTicketEntry { + ticket: ticket.to_string(), + fetched_at_ms: channel_host::now_millis(), + }, + ); + persist_typing_tickets(&tickets) +} + +fn clear_typing_ticket(user_id: &str) -> Result<(), String> { + let mut tickets = load_typing_tickets(); + if tickets.remove(user_id).is_some() { + persist_typing_tickets(&tickets)?; + } + Ok(()) +} + +fn resolve_typing_ticket( + config: &WechatConfig, + user_id: &str, + context_token: Option<&str>, +) -> Result, String> { + if let Some(ticket) = cached_typing_ticket(user_id) { + return Ok(Some(ticket)); + } + + let response = api::get_config(config, user_id, context_token)?; + if !matches!(response.ret, Some(0)) { + let errmsg = response + .errmsg + .as_deref() + .unwrap_or("unknown WeChat getConfig error"); + return Err(format!( + "WeChat getConfig returned ret={} errmsg={errmsg}", + response.ret.unwrap_or(-1) + )); + } + + let Some(ticket) = response + .typing_ticket + .as_deref() + .map(str::trim) + .filter(|ticket| !ticket.is_empty()) + else { + return Ok(None); + }; + + if let Err(error) = persist_typing_ticket(user_id, ticket) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeChat typing ticket: {error}"), + ); + } + + Ok(Some(ticket.to_string())) +} + +fn send_typing_indicator( + config: &WechatConfig, + metadata: &OutboundMetadata, + context_token: Option<&str>, + status: i32, + allow_ticket_fetch: bool, +) -> Result<(), String> { + let ticket = if allow_ticket_fetch { + resolve_typing_ticket(config, &metadata.from_user_id, context_token)? + } else { + cached_typing_ticket(&metadata.from_user_id) + }; + + let Some(ticket) = ticket else { + return Ok(()); + }; + + if let Err(error) = api::send_typing(config, &metadata.from_user_id, &ticket, status) { + let _ = clear_typing_ticket(&metadata.from_user_id); + return Err(error); + } + + Ok(()) +} + +export!(WechatChannel); + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + + use super::{ + classify_status_update, extract_text, get_updates_error_message, merge_text, + process_incoming_bundle, send_response_with_handlers, take_due_pending_bundles, + PendingInboundBundle, StoredInboundAttachment, WechatStatusAction, + ATTACHMENT_DELIVERY_FAILED_FALLBACK, + }; + use crate::exports::near::agent::channel::{ + AgentResponse, Attachment, StatusType, StatusUpdate, + }; + use crate::types::{ + GetUpdatesResponse, MessageItem, VoiceItem, WechatMessage, MESSAGE_ITEM_VOICE, + }; + + fn make_bundle(user_id: &str, text: &str, image_count: usize) -> PendingInboundBundle { + PendingInboundBundle { + from_user_id: user_id.to_string(), + to_user_id: Some("bot".to_string()), + session_id: Some("session-1".to_string()), + context_token: Some("ctx-1".to_string()), + message_id: Some(1), + flush_at_ms: 0, + text: text.to_string(), + attachments: (0..image_count) + .map(|index| StoredInboundAttachment { + id: format!("att-{index}"), + mime_type: "image/jpeg".to_string(), + filename: Some(format!("photo-{index}.jpg")), + size_bytes: Some(128), + source_url: Some("https://example.com/image.jpg".to_string()), + storage_key: None, + extracted_text: None, + extras_json: "{}".to_string(), + }) + .collect(), + } + } + + fn make_updates_response(ret: Option, errcode: Option) -> GetUpdatesResponse { + GetUpdatesResponse { + ret, + errcode, + errmsg: None, + msgs: Vec::new(), + get_updates_buf: None, + } + } + + #[test] + fn test_get_updates_error_message_allows_empty_poll_without_ret() { + let response = make_updates_response(None, None); + assert_eq!(get_updates_error_message(&response), None); + } + + #[test] + fn test_get_updates_error_message_reports_nonzero_ret() { + let mut response = make_updates_response(Some(42), None); + response.errmsg = Some("bad cursor".to_string()); + assert_eq!( + get_updates_error_message(&response), + Some("ret=42 errmsg=bad cursor".to_string()) + ); + } + + #[test] + fn test_get_updates_error_message_reports_nonzero_errcode() { + let response = make_updates_response(None, Some(-14)); + assert_eq!( + get_updates_error_message(&response), + Some("errcode=-14 errmsg=unknown WeChat polling error".to_string()) + ); + } + + #[test] + fn test_classify_status_update_thinking_starts_typing() { + let update = StatusUpdate { + status: StatusType::Thinking, + message: "Thinking...".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(WechatStatusAction::Typing) + ); + } + + #[test] + fn test_classify_status_update_done_cancels_typing() { + let update = StatusUpdate { + status: StatusType::Done, + message: "Done".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(WechatStatusAction::Cancel) + ); + } + + #[test] + fn test_classify_status_update_approval_needed_cancels_typing() { + let update = StatusUpdate { + status: StatusType::ApprovalNeeded, + message: "Approval needed".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(WechatStatusAction::Cancel) + ); + } + + #[test] + fn test_classify_status_update_tool_started_is_ignored() { + let update = StatusUpdate { + status: StatusType::ToolStarted, + message: "Tool started".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_terminal_text_status_cancels_typing() { + let update = StatusUpdate { + status: StatusType::Status, + message: "Awaiting approval".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(WechatStatusAction::Cancel) + ); + } + + #[test] + fn test_classify_status_update_progress_status_is_ignored() { + let update = StatusUpdate { + status: StatusType::Status, + message: "Context compaction started".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_merge_text_joins_non_empty_segments() { + assert_eq!(merge_text("", "hello"), "hello"); + assert_eq!(merge_text("look", "what is this"), "look\n\nwhat is this"); + assert_eq!(merge_text("look", ""), "look"); + } + + #[test] + fn test_extract_text_uses_voice_transcript_when_present() { + let message = WechatMessage { + message_id: Some(1), + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_VOICE), + text_item: None, + image_item: None, + voice_item: Some(VoiceItem { + media: None, + encode_type: Some(6), + playtime: Some(1500), + text: Some("voice transcript".to_string()), + }), + file_item: None, + video_item: None, + }], + }; + + assert_eq!(extract_text(&message), "voice transcript"); + } + + #[test] + fn test_process_incoming_bundle_merges_buffered_image_with_follow_up_text() { + let mut pending = HashMap::new(); + let mut changed = false; + + let emitted = process_incoming_bundle( + &mut pending, + make_bundle("u1", "", 1), + &mut changed, + 100, + 5_000, + ); + assert!(emitted.is_empty()); + assert!(changed); + assert_eq!(pending.len(), 1); + assert_eq!(pending["u1"].flush_at_ms, 5100); + + changed = false; + let emitted = process_incoming_bundle( + &mut pending, + make_bundle("u1", "What is in this image?", 0), + &mut changed, + 200, + 5_000, + ); + assert!(changed); + assert!(pending.is_empty()); + assert_eq!(emitted.len(), 1); + assert_eq!(emitted[0].text, "What is in this image?"); + assert_eq!(emitted[0].attachments.len(), 1); + } + + #[test] + fn test_process_incoming_bundle_extends_window_for_attachment_only_follow_up() { + let mut pending = HashMap::new(); + let mut changed = false; + + let emitted = process_incoming_bundle( + &mut pending, + make_bundle("u1", "", 1), + &mut changed, + 100, + 5_000, + ); + assert!(emitted.is_empty()); + assert!(changed); + assert_eq!(pending["u1"].flush_at_ms, 5_100); + + changed = false; + let emitted = process_incoming_bundle( + &mut pending, + make_bundle("u1", "", 1), + &mut changed, + 700, + 5_000, + ); + assert!(emitted.is_empty()); + assert!(changed); + assert_eq!(pending["u1"].attachments.len(), 2); + assert_eq!(pending["u1"].flush_at_ms, 5_700); + } + + #[test] + fn test_process_incoming_bundle_emits_text_and_images_together_without_buffering() { + let mut pending = HashMap::new(); + let mut changed = false; + + let emitted = process_incoming_bundle( + &mut pending, + make_bundle("u1", "Look at this image", 1), + &mut changed, + 100, + 5_000, + ); + assert!(!changed); + assert!(pending.is_empty()); + assert_eq!(emitted.len(), 1); + assert_eq!(emitted[0].text, "Look at this image"); + assert_eq!(emitted[0].attachments.len(), 1); + } + + #[test] + fn test_take_due_pending_bundles_emits_only_expired_entries() { + let mut pending = HashMap::new(); + let mut expired = make_bundle("u1", "", 1); + expired.flush_at_ms = 100; + let mut fresh = make_bundle("u2", "", 1); + fresh.flush_at_ms = 300; + pending.insert(expired.from_user_id.clone(), expired); + pending.insert(fresh.from_user_id.clone(), fresh); + + let due = take_due_pending_bundles(&mut pending, 200); + assert_eq!(due.len(), 1); + assert_eq!(due[0].from_user_id, "u1"); + assert_eq!(pending.len(), 1); + assert!(pending.contains_key("u2")); + } + + #[test] + fn test_send_response_sends_attachments_before_text() { + let response = AgentResponse { + message_id: "msg-1".to_string(), + content: "Here is the image you asked for.".to_string(), + thread_id: None, + metadata_json: "{}".to_string(), + attachments: vec![Attachment { + filename: "cat.jpg".to_string(), + mime_type: "image/jpeg".to_string(), + data: vec![1, 2, 3], + }], + }; + let sent_events = RefCell::new(Vec::new()); + + let result = send_response_with_handlers( + &response, + |text| { + sent_events.borrow_mut().push(format!("text:{text}")); + Ok(()) + }, + |attachment| { + sent_events + .borrow_mut() + .push(format!("attachment:{}", attachment.filename)); + Ok(()) + }, + |_message| {}, + ); + + assert!(result.is_ok()); + assert_eq!( + sent_events.into_inner(), + vec![ + "attachment:cat.jpg".to_string(), + "text:Here is the image you asked for.".to_string() + ] + ); + } + + #[test] + fn test_send_response_falls_back_to_text_when_attachment_send_fails() { + let response = AgentResponse { + message_id: "msg-1".to_string(), + content: "Here is the image you asked for.".to_string(), + thread_id: None, + metadata_json: "{}".to_string(), + attachments: vec![Attachment { + filename: "cat.jpg".to_string(), + mime_type: "image/jpeg".to_string(), + data: vec![1, 2, 3], + }], + }; + let mut sent_texts = Vec::new(); + let mut warnings = Vec::new(); + + let result = send_response_with_handlers( + &response, + |text| { + sent_texts.push(text.to_string()); + Ok(()) + }, + |_attachment| Err("upload failed".to_string()), + |message| warnings.push(message), + ); + + assert!(result.is_ok()); + assert_eq!(sent_texts, vec!["Here is the image you asked for."]); + assert_eq!(warnings.len(), 1); + assert!(warnings[0].contains("upload failed")); + } + + #[test] + fn test_send_response_sends_generic_text_when_attachment_only_reply_fails() { + let response = AgentResponse { + message_id: "msg-1".to_string(), + content: String::new(), + thread_id: None, + metadata_json: "{}".to_string(), + attachments: vec![Attachment { + filename: "cat.jpg".to_string(), + mime_type: "image/jpeg".to_string(), + data: vec![1, 2, 3], + }], + }; + let mut sent_texts = Vec::new(); + + let result = send_response_with_handlers( + &response, + |text| { + sent_texts.push(text.to_string()); + Ok(()) + }, + |_attachment| Err("upload failed".to_string()), + |_message| {}, + ); + + assert!(result.is_ok()); + assert_eq!( + sent_texts, + vec![ATTACHMENT_DELIVERY_FAILED_FALLBACK.to_string()] + ); + } +} diff --git a/channels-src/wechat/src/media.rs b/channels-src/wechat/src/media.rs new file mode 100644 index 00000000000..b9c07575bb5 --- /dev/null +++ b/channels-src/wechat/src/media.rs @@ -0,0 +1,1050 @@ +use base64::Engine as _; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::exports::near::agent::channel::Attachment; +use crate::near::agent::channel_host::{self, InboundAttachment}; +use crate::types::{ + CdnMedia, FileItem, ImageItem, MessageItem, SendMessageRequest, VideoItem, WechatConfig, + MESSAGE_ITEM_FILE, MESSAGE_ITEM_IMAGE, MESSAGE_ITEM_VIDEO, MESSAGE_ITEM_VOICE, + MESSAGE_STATE_FINISH, MESSAGE_TYPE_BOT, UPLOAD_MEDIA_TYPE_FILE, UPLOAD_MEDIA_TYPE_IMAGE, + UPLOAD_MEDIA_TYPE_VIDEO, +}; + +const AES_BLOCK_SIZE: usize = 16; +const WECHAT_OUTBOUND_ENVELOPE_MAGIC: &[u8] = b"ICWXENC1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutboundMediaKind { + Image, + Video, + File, +} + +#[derive(Debug, Clone)] +pub struct UploadedMedia { + pub download_encrypted_query_param: String, + pub cdn_aes_key_base64: String, + pub file_size_ciphertext: u64, + pub plaintext_size: u64, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct PreparedWechatUpload { + raw_size: u64, + raw_md5: String, + ciphertext_size: u64, + filekey: String, + aes_key_hex: String, +} + +pub fn extract_inbound_attachments( + config: &WechatConfig, + message: &crate::types::WechatMessage, +) -> Result, String> { + message + .item_list + .iter() + .enumerate() + .filter_map(|(index, item)| { + map_inbound_attachment(config, message, item, index).transpose() + }) + .collect() +} + +pub fn send_image_attachment( + config: &WechatConfig, + to_user_id: &str, + attachment: &Attachment, + context_token: Option<&str>, +) -> Result<(), String> { + if attachment.data.is_empty() { + return Err(format!( + "WeChat image attachment '{}' has no data", + attachment.filename + )); + } + + let upload = upload_media_attachment(config, to_user_id, attachment, UPLOAD_MEDIA_TYPE_IMAGE)?; + let request = SendMessageRequest { + msg: crate::types::OutboundWechatMessage { + from_user_id: String::new(), + to_user_id: to_user_id.to_string(), + client_id: format!("wechat-{}", channel_host::now_millis()), + message_type: MESSAGE_TYPE_BOT, + message_state: MESSAGE_STATE_FINISH, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_IMAGE), + text_item: None, + image_item: Some(ImageItem { + media: Some(CdnMedia { + encrypt_query_param: Some(upload.download_encrypted_query_param.clone()), + aes_key: Some(upload.cdn_aes_key_base64.clone()), + encrypt_type: Some(1), + }), + thumb_media: None, + aeskey: None, + mid_size: Some(upload.file_size_ciphertext), + thumb_size: None, + }), + voice_item: None, + file_item: None, + video_item: None, + }], + context_token: context_token.map(str::to_string), + }, + base_info: crate::api::base_info(), + }; + + crate::api::send_message_request(config, &request) +} + +pub fn send_video_attachment( + config: &WechatConfig, + to_user_id: &str, + attachment: &Attachment, + context_token: Option<&str>, +) -> Result<(), String> { + let upload = upload_media_attachment(config, to_user_id, attachment, UPLOAD_MEDIA_TYPE_VIDEO)?; + let request = SendMessageRequest { + msg: crate::types::OutboundWechatMessage { + from_user_id: String::new(), + to_user_id: to_user_id.to_string(), + client_id: format!("wechat-{}", channel_host::now_millis()), + message_type: MESSAGE_TYPE_BOT, + message_state: MESSAGE_STATE_FINISH, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_VIDEO), + text_item: None, + image_item: None, + voice_item: None, + file_item: None, + video_item: Some(VideoItem { + media: Some(CdnMedia { + encrypt_query_param: Some(upload.download_encrypted_query_param.clone()), + aes_key: Some(upload.cdn_aes_key_base64.clone()), + encrypt_type: Some(1), + }), + thumb_media: None, + video_size: Some(upload.file_size_ciphertext), + thumb_size: None, + play_length: None, + }), + }], + context_token: context_token.map(str::to_string), + }, + base_info: crate::api::base_info(), + }; + + crate::api::send_message_request(config, &request) +} + +pub fn send_file_attachment( + config: &WechatConfig, + to_user_id: &str, + attachment: &Attachment, + context_token: Option<&str>, +) -> Result<(), String> { + let upload = upload_media_attachment(config, to_user_id, attachment, UPLOAD_MEDIA_TYPE_FILE)?; + let request = SendMessageRequest { + msg: crate::types::OutboundWechatMessage { + from_user_id: String::new(), + to_user_id: to_user_id.to_string(), + client_id: format!("wechat-{}", channel_host::now_millis()), + message_type: MESSAGE_TYPE_BOT, + message_state: MESSAGE_STATE_FINISH, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_FILE), + text_item: None, + image_item: None, + voice_item: None, + file_item: Some(FileItem { + media: Some(CdnMedia { + encrypt_query_param: Some(upload.download_encrypted_query_param.clone()), + aes_key: Some(upload.cdn_aes_key_base64.clone()), + encrypt_type: Some(1), + }), + file_name: Some(normalize_outbound_file_name(attachment)), + len: Some(upload.plaintext_size.to_string()), + }), + video_item: None, + }], + context_token: context_token.map(str::to_string), + }, + base_info: crate::api::base_info(), + }; + + crate::api::send_message_request(config, &request) +} + +pub fn classify_outbound_media_kind(mime_type: &str) -> OutboundMediaKind { + if mime_type.starts_with("image/") { + OutboundMediaKind::Image + } else if mime_type.starts_with("video/") { + OutboundMediaKind::Video + } else { + OutboundMediaKind::File + } +} + +fn map_inbound_attachment( + config: &WechatConfig, + message: &crate::types::WechatMessage, + item: &MessageItem, + index: usize, +) -> Result, String> { + if item.r#type == Some(MESSAGE_ITEM_IMAGE) { + return map_image_attachment(config, message, item, index); + } + if item.r#type == Some(MESSAGE_ITEM_VOICE) { + return map_voice_attachment(config, message, item, index); + } + if item.r#type == Some(MESSAGE_ITEM_FILE) { + return map_file_attachment(config, message, item, index); + } + if item.r#type == Some(MESSAGE_ITEM_VIDEO) { + return map_video_attachment(config, message, item, index); + } + Ok(None) +} + +fn map_image_attachment( + config: &WechatConfig, + message: &crate::types::WechatMessage, + item: &MessageItem, + index: usize, +) -> Result, String> { + if item.r#type != Some(MESSAGE_ITEM_IMAGE) { + return Ok(None); + } + + let image = item.image_item.as_ref().ok_or_else(|| { + format!( + "WeChat image message {:?} is missing image_item payload", + message.message_id + ) + })?; + let media = image.media.as_ref().ok_or_else(|| { + format!( + "WeChat image message {:?} is missing media payload", + message.message_id + ) + })?; + let encrypt_query_param = media.encrypt_query_param.as_deref().ok_or_else(|| { + format!( + "WeChat image message {:?} is missing encrypt_query_param", + message.message_id + ) + })?; + let message_id = message + .message_id + .ok_or_else(|| "WeChat image message is missing message_id".to_string())?; + let aes_key = preferred_image_aes_key(image, media).map(str::to_string); + + Ok(Some(InboundAttachment { + id: format!("wechat-image-{}-{}", message_id, index), + mime_type: "image/jpeg".to_string(), + filename: Some(format!("wechat-image-{}-{}.jpg", message_id, index)), + size_bytes: image.mid_size, + source_url: Some(build_cdn_download_url( + &config.cdn_base_url, + encrypt_query_param, + )), + storage_key: None, + extracted_text: None, + extras_json: json!({ "wechat_aes_key": aes_key }).to_string(), + })) +} + +fn map_file_attachment( + config: &WechatConfig, + message: &crate::types::WechatMessage, + item: &MessageItem, + index: usize, +) -> Result, String> { + if item.r#type != Some(MESSAGE_ITEM_FILE) { + return Ok(None); + } + + let file = item.file_item.as_ref().ok_or_else(|| { + format!( + "WeChat file message {:?} is missing file_item payload", + message.message_id + ) + })?; + let media = file.media.as_ref().ok_or_else(|| { + format!( + "WeChat file message {:?} is missing media payload", + message.message_id + ) + })?; + let encrypt_query_param = media.encrypt_query_param.as_deref().ok_or_else(|| { + format!( + "WeChat file message {:?} is missing encrypt_query_param", + message.message_id + ) + })?; + let aes_key = media + .aes_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + format!( + "WeChat file message {:?} is missing aes_key", + message.message_id + ) + })?; + let message_id = message + .message_id + .ok_or_else(|| "WeChat file message is missing message_id".to_string())?; + let filename = inbound_file_name(file, message_id, index); + let size_bytes = file.len.as_deref().and_then(parse_file_size); + + Ok(Some(InboundAttachment { + id: format!("wechat-file-{}-{}", message_id, index), + mime_type: infer_file_mime_type(&filename), + filename: Some(filename), + size_bytes, + source_url: Some(build_cdn_download_url( + &config.cdn_base_url, + encrypt_query_param, + )), + storage_key: None, + extracted_text: None, + extras_json: json!({ "wechat_aes_key": aes_key }).to_string(), + })) +} + +fn map_voice_attachment( + config: &WechatConfig, + message: &crate::types::WechatMessage, + item: &MessageItem, + index: usize, +) -> Result, String> { + if item.r#type != Some(MESSAGE_ITEM_VOICE) { + return Ok(None); + } + + let voice = item.voice_item.as_ref().ok_or_else(|| { + format!( + "WeChat voice message {:?} is missing voice_item payload", + message.message_id + ) + })?; + let media = voice.media.as_ref().ok_or_else(|| { + format!( + "WeChat voice message {:?} is missing media payload", + message.message_id + ) + })?; + let encrypt_query_param = media.encrypt_query_param.as_deref().ok_or_else(|| { + format!( + "WeChat voice message {:?} is missing encrypt_query_param", + message.message_id + ) + })?; + let aes_key = media + .aes_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + format!( + "WeChat voice message {:?} is missing aes_key", + message.message_id + ) + })?; + let message_id = message + .message_id + .ok_or_else(|| "WeChat voice message is missing message_id".to_string())?; + let (mime_type, extension) = infer_voice_media_type(voice.encode_type); + let duration_secs = voice.playtime.map(|millis| (millis / 1000) as u32); + + Ok(Some(InboundAttachment { + id: format!("wechat-voice-{}-{}", message_id, index), + mime_type: mime_type.to_string(), + filename: Some(format!( + "wechat-voice-{}-{}.{}", + message_id, index, extension + )), + size_bytes: None, + source_url: Some(build_cdn_download_url( + &config.cdn_base_url, + encrypt_query_param, + )), + storage_key: None, + extracted_text: voice + .text + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string), + extras_json: build_voice_extras_json(aes_key, duration_secs), + })) +} + +fn map_video_attachment( + config: &WechatConfig, + message: &crate::types::WechatMessage, + item: &MessageItem, + index: usize, +) -> Result, String> { + if item.r#type != Some(MESSAGE_ITEM_VIDEO) { + return Ok(None); + } + + let video = item.video_item.as_ref().ok_or_else(|| { + format!( + "WeChat video message {:?} is missing video_item payload", + message.message_id + ) + })?; + let media = video.media.as_ref().ok_or_else(|| { + format!( + "WeChat video message {:?} is missing media payload", + message.message_id + ) + })?; + let encrypt_query_param = media.encrypt_query_param.as_deref().ok_or_else(|| { + format!( + "WeChat video message {:?} is missing encrypt_query_param", + message.message_id + ) + })?; + let aes_key = media + .aes_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + format!( + "WeChat video message {:?} is missing aes_key", + message.message_id + ) + })?; + let message_id = message + .message_id + .ok_or_else(|| "WeChat video message is missing message_id".to_string())?; + + Ok(Some(InboundAttachment { + id: format!("wechat-video-{}-{}", message_id, index), + mime_type: "video/mp4".to_string(), + filename: Some(format!("wechat-video-{}-{}.mp4", message_id, index)), + size_bytes: video.video_size, + source_url: Some(build_cdn_download_url( + &config.cdn_base_url, + encrypt_query_param, + )), + storage_key: None, + extracted_text: None, + extras_json: json!({ "wechat_aes_key": aes_key }).to_string(), + })) +} + +fn preferred_image_aes_key<'a>(image: &'a ImageItem, media: &'a CdnMedia) -> Option<&'a str> { + image + .aeskey + .as_deref() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + media + .aes_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + }) +} + +fn inbound_file_name(file: &FileItem, message_id: i64, index: usize) -> String { + file.file_name + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| format!("wechat-file-{}-{}.bin", message_id, index)) +} + +fn parse_file_size(raw: &str) -> Option { + raw.trim().parse::().ok() +} + +fn infer_voice_media_type(encode_type: Option) -> (&'static str, &'static str) { + match encode_type { + Some(7) => ("audio/mpeg", "mp3"), + Some(8) => ("audio/ogg", "ogg"), + Some(5) => ("audio/amr", "amr"), + Some(6) => ("audio/silk", "silk"), + _ => ("audio/silk", "silk"), + } +} + +fn build_voice_extras_json(aes_key: &str, duration_secs: Option) -> String { + let mut extras = serde_json::Map::new(); + extras.insert("wechat_aes_key".to_string(), json!(aes_key)); + if let Some(duration_secs) = duration_secs { + extras.insert("duration_secs".to_string(), json!(duration_secs)); + } + serde_json::Value::Object(extras).to_string() +} + +fn infer_file_mime_type(filename: &str) -> String { + let extension = filename + .rsplit_once('.') + .map(|(_, ext)| ext.trim().to_ascii_lowercase()); + + match extension.as_deref() { + Some("pdf") => "application/pdf", + Some("doc") => "application/msword", + Some("docx") => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Some("xls") => "application/vnd.ms-excel", + Some("xlsx") => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + Some("ppt") => "application/vnd.ms-powerpoint", + Some("pptx") => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + Some("txt") => "text/plain", + Some("csv") => "text/csv", + Some("json") => "application/json", + Some("xml") => "application/xml", + Some("md") => "text/markdown", + Some("zip") => "application/zip", + Some("tar") => "application/x-tar", + Some("gz") => "application/gzip", + Some("mp3") => "audio/mpeg", + Some("ogg") => "audio/ogg", + Some("wav") => "audio/wav", + Some("mp4") => "video/mp4", + Some("mov") => "video/quicktime", + Some("webm") => "video/webm", + Some("mkv") => "video/x-matroska", + Some("avi") => "video/x-msvideo", + Some("png") => "image/png", + Some("jpg") | Some("jpeg") => "image/jpeg", + Some("gif") => "image/gif", + Some("webp") => "image/webp", + Some("bmp") => "image/bmp", + _ => "application/octet-stream", + } + .to_string() +} + +fn upload_media_attachment( + config: &WechatConfig, + to_user_id: &str, + attachment: &Attachment, + media_type: i32, +) -> Result { + if attachment.data.is_empty() { + return Err(format!( + "WeChat attachment '{}' has no data", + attachment.filename + )); + } + + let (prepared, ciphertext) = unpack_prepared_wechat_upload(&attachment.data)?; + let raw_size = prepared.raw_size; + let raw_md5 = prepared.raw_md5; + let file_size_ciphertext = prepared.ciphertext_size; + let filekey = prepared.filekey; + let aes_key_hex = prepared.aes_key_hex; + + let upload_request = build_upload_url_request( + media_type, + to_user_id, + &filekey, + raw_size, + &raw_md5, + file_size_ciphertext, + &aes_key_hex, + ); + let upload_url = crate::api::get_upload_url(config, &upload_request)?; + + let upload_param = upload_url + .upload_param + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "WeChat getUploadUrl returned no upload_param".to_string())?; + + let download_encrypted_query_param = + upload_cdn_payload(config, &filekey, upload_param, &ciphertext)?; + + let cdn_aes_key_base64 = + base64::engine::general_purpose::STANDARD.encode(aes_key_hex.as_bytes()); + + Ok(UploadedMedia { + download_encrypted_query_param, + cdn_aes_key_base64, + file_size_ciphertext, + plaintext_size: raw_size, + }) +} + +fn upload_cdn_payload( + config: &WechatConfig, + filekey: &str, + upload_param: &str, + ciphertext: &[u8], +) -> Result { + let cdn_upload_url = build_cdn_upload_url(&config.cdn_base_url, upload_param, filekey); + let upload_response = channel_host::http_request( + "POST", + &cdn_upload_url, + r#"{"Content-Type":"application/octet-stream"}"#, + Some(ciphertext), + Some(15_000), + ) + .map_err(|e| format!("WeChat CDN upload failed: {e}"))?; + + if upload_response.status != 200 { + let body = String::from_utf8_lossy(&upload_response.body); + return Err(format!( + "WeChat CDN upload returned {}: {}", + upload_response.status, body + )); + } + + let headers: std::collections::HashMap = + serde_json::from_str(&upload_response.headers_json) + .map_err(|e| format!("Failed to parse WeChat CDN upload headers: {e}"))?; + headers + .iter() + .find_map(|(key, value)| { + if key.eq_ignore_ascii_case("x-encrypted-param") { + Some(value.clone()) + } else { + None + } + }) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "WeChat CDN upload response missing x-encrypted-param".to_string()) +} + +fn build_upload_url_request( + media_type: i32, + to_user_id: &str, + filekey: &str, + raw_size: u64, + raw_md5: &str, + file_size_ciphertext: u64, + aes_key_hex: &str, +) -> crate::types::GetUploadUrlRequest { + crate::types::GetUploadUrlRequest { + filekey: filekey.to_string(), + media_type, + to_user_id: to_user_id.to_string(), + rawsize: raw_size, + rawfilemd5: raw_md5.to_string(), + filesize: file_size_ciphertext, + thumb_rawsize: None, + thumb_rawfilemd5: None, + thumb_filesize: None, + no_need_thumb: true, + aeskey: aes_key_hex.to_string(), + base_info: crate::api::base_info(), + } +} + +fn unpack_prepared_wechat_upload(data: &[u8]) -> Result<(PreparedWechatUpload, Vec), String> { + if !data.starts_with(WECHAT_OUTBOUND_ENVELOPE_MAGIC) { + return Err( + "WeChat outbound attachment is missing host-prepared encryption envelope".to_string(), + ); + } + + let header_len = WECHAT_OUTBOUND_ENVELOPE_MAGIC.len(); + if data.len() < header_len + 4 { + return Err("WeChat outbound attachment envelope is truncated".to_string()); + } + + let metadata_len = u32::from_le_bytes( + data[header_len..header_len + 4] + .try_into() + .map_err(|_| "Failed to decode WeChat outbound metadata length".to_string())?, + ) as usize; + let metadata_start = header_len + 4; + let metadata_end = metadata_start.saturating_add(metadata_len); + if metadata_end > data.len() { + return Err("WeChat outbound attachment metadata is truncated".to_string()); + } + + let metadata = + serde_json::from_slice::(&data[metadata_start..metadata_end]) + .map_err(|e| format!("Failed to parse WeChat outbound attachment metadata: {e}"))?; + let ciphertext = data[metadata_end..].to_vec(); + if metadata.ciphertext_size != padded_size(metadata.raw_size) { + return Err(format!( + "WeChat outbound attachment ciphertext size does not match padded raw size: raw_size={} ciphertext_size={}", + metadata.raw_size, metadata.ciphertext_size + )); + } + if metadata.ciphertext_size != ciphertext.len() as u64 { + return Err(format!( + "WeChat outbound attachment ciphertext size mismatch: metadata={} actual={}", + metadata.ciphertext_size, + ciphertext.len() + )); + } + + Ok((metadata, ciphertext)) +} + +fn normalize_outbound_file_name(attachment: &Attachment) -> String { + let trimmed = attachment.filename.trim(); + if trimmed.is_empty() { + "attachment.bin".to_string() + } else { + trimmed.to_string() + } +} + +fn build_cdn_download_url(cdn_base_url: &str, encrypted_query_param: &str) -> String { + format!( + "{}/download?encrypted_query_param={}", + cdn_base_url.trim_end_matches('/'), + percent_encode(encrypted_query_param) + ) +} + +fn build_cdn_upload_url(cdn_base_url: &str, upload_param: &str, filekey: &str) -> String { + format!( + "{}/upload?encrypted_query_param={}&filekey={}", + cdn_base_url.trim_end_matches('/'), + percent_encode(upload_param), + percent_encode(filekey) + ) +} + +fn percent_encode(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(byte as char); + } else { + encoded.push('%'); + encoded.push(nibble_to_hex(byte >> 4)); + encoded.push(nibble_to_hex(byte & 0x0F)); + } + } + encoded +} + +fn nibble_to_hex(nibble: u8) -> char { + match nibble { + 0..=9 => (b'0' + nibble) as char, + 10..=15 => (b'A' + (nibble - 10)) as char, + _ => '0', + } +} + +fn padded_size(raw_size: u64) -> u64 { + ((raw_size / AES_BLOCK_SIZE as u64) + 1) * AES_BLOCK_SIZE as u64 +} + +#[cfg(test)] +mod tests { + use super::{ + build_upload_url_request, build_voice_extras_json, classify_outbound_media_kind, + infer_file_mime_type, infer_voice_media_type, map_file_attachment, map_image_attachment, + map_video_attachment, map_voice_attachment, unpack_prepared_wechat_upload, + OutboundMediaKind, PreparedWechatUpload, WECHAT_OUTBOUND_ENVELOPE_MAGIC, + }; + use crate::types::{ + CdnMedia, FileItem, ImageItem, MessageItem, VideoItem, VoiceItem, WechatConfig, + WechatMessage, MESSAGE_ITEM_FILE, MESSAGE_ITEM_IMAGE, MESSAGE_ITEM_VIDEO, + MESSAGE_ITEM_VOICE, UPLOAD_MEDIA_TYPE_FILE, UPLOAD_MEDIA_TYPE_IMAGE, + }; + + #[test] + fn test_unpack_prepared_wechat_upload_reads_host_envelope() { + let metadata = PreparedWechatUpload { + raw_size: 18, + raw_md5: "0123456789abcdef0123456789abcdef".to_string(), + ciphertext_size: 32, + filekey: "abcd".to_string(), + aes_key_hex: "6162636465666768696a6b6c6d6e6f70".to_string(), + }; + let metadata_json = serde_json::to_vec(&metadata).unwrap(); + let ciphertext = vec![9u8; 32]; + let mut packed = Vec::new(); + packed.extend_from_slice(WECHAT_OUTBOUND_ENVELOPE_MAGIC); + packed.extend_from_slice(&(metadata_json.len() as u32).to_le_bytes()); + packed.extend_from_slice(&metadata_json); + packed.extend_from_slice(&ciphertext); + + let unpacked = unpack_prepared_wechat_upload(&packed).expect("parse envelope"); + assert_eq!(unpacked.0.raw_size, metadata.raw_size); + assert_eq!(unpacked.0.raw_md5, metadata.raw_md5); + assert_eq!(unpacked.1, ciphertext); + } + + #[test] + fn test_unpack_prepared_wechat_upload_requires_host_envelope() { + let error = + unpack_prepared_wechat_upload(b"plain-image-bytes").expect_err("raw bytes should fail"); + assert!(error.contains("missing host-prepared encryption envelope")); + } + + #[test] + fn test_build_upload_url_request_uses_no_need_thumb_for_images() { + let request = build_upload_url_request( + UPLOAD_MEDIA_TYPE_IMAGE, + "user-1", + "filekey-1", + 123, + "abc123", + 128, + "deadbeef", + ); + + assert_eq!(request.thumb_rawsize, None); + assert_eq!(request.thumb_rawfilemd5, None); + assert_eq!(request.thumb_filesize, None); + assert!(request.no_need_thumb); + } + + #[test] + fn test_build_upload_url_request_omits_thumbnail_fields_for_files() { + let request = build_upload_url_request( + UPLOAD_MEDIA_TYPE_FILE, + "user-1", + "filekey-1", + 123, + "abc123", + 128, + "deadbeef", + ); + + assert_eq!(request.thumb_rawsize, None); + assert_eq!(request.thumb_rawfilemd5, None); + assert_eq!(request.thumb_filesize, None); + assert!(request.no_need_thumb); + } + + #[test] + fn test_map_image_attachment_errors_when_message_id_missing() { + let config = WechatConfig::default(); + let message = WechatMessage { + message_id: None, + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_IMAGE), + text_item: None, + image_item: Some(ImageItem { + media: Some(CdnMedia { + encrypt_query_param: Some("enc".to_string()), + aes_key: Some("aes".to_string()), + encrypt_type: Some(1), + }), + thumb_media: None, + aeskey: None, + mid_size: Some(128), + thumb_size: None, + }), + voice_item: None, + file_item: None, + video_item: None, + }], + }; + + let error = map_image_attachment(&config, &message, &message.item_list[0], 0) + .expect_err("missing message_id should error"); + assert!(error.contains("missing message_id")); + } + + #[test] + fn test_map_file_attachment_uses_filename_and_size_metadata() { + let config = WechatConfig::default(); + let message = WechatMessage { + message_id: Some(42), + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_FILE), + text_item: None, + image_item: None, + voice_item: None, + file_item: Some(FileItem { + media: Some(CdnMedia { + encrypt_query_param: Some("enc".to_string()), + aes_key: Some("YWJjZGVmZ2hpamtsbW5vcA==".to_string()), + encrypt_type: Some(1), + }), + file_name: Some("report.PDF".to_string()), + len: Some("256".to_string()), + }), + video_item: None, + }], + }; + + let attachment = map_file_attachment(&config, &message, &message.item_list[0], 0) + .expect("file attachment should map") + .expect("file attachment should be present"); + assert_eq!(attachment.id, "wechat-file-42-0"); + assert_eq!(attachment.mime_type, "application/pdf"); + assert_eq!(attachment.filename.as_deref(), Some("report.PDF")); + assert_eq!(attachment.size_bytes, Some(256)); + assert!(attachment.extras_json.contains("wechat_aes_key")); + } + + #[test] + fn test_map_file_attachment_errors_when_message_id_missing() { + let config = WechatConfig::default(); + let message = WechatMessage { + message_id: None, + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_FILE), + text_item: None, + image_item: None, + voice_item: None, + file_item: Some(FileItem { + media: Some(CdnMedia { + encrypt_query_param: Some("enc".to_string()), + aes_key: Some("aes".to_string()), + encrypt_type: Some(1), + }), + file_name: Some("report.pdf".to_string()), + len: Some("256".to_string()), + }), + video_item: None, + }], + }; + + let error = map_file_attachment(&config, &message, &message.item_list[0], 0) + .expect_err("missing message_id should error"); + assert!(error.contains("missing message_id")); + } + + #[test] + fn test_infer_file_mime_type_defaults_to_octet_stream() { + assert_eq!( + infer_file_mime_type("archive.unknown"), + "application/octet-stream" + ); + assert_eq!(infer_file_mime_type("README"), "application/octet-stream"); + } + + #[test] + fn test_infer_voice_media_type_defaults_to_silk() { + assert_eq!(infer_voice_media_type(Some(6)), ("audio/silk", "silk")); + assert_eq!(infer_voice_media_type(Some(8)), ("audio/ogg", "ogg")); + assert_eq!(infer_voice_media_type(None), ("audio/silk", "silk")); + } + + #[test] + fn test_build_voice_extras_json_includes_duration() { + let extras = build_voice_extras_json("aes-key", Some(9)); + assert!(extras.contains("wechat_aes_key")); + assert!(extras.contains("duration_secs")); + } + + #[test] + fn test_map_voice_attachment_sets_audio_metadata() { + let config = WechatConfig::default(); + let message = WechatMessage { + message_id: Some(77), + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_VOICE), + text_item: None, + image_item: None, + voice_item: Some(VoiceItem { + media: Some(CdnMedia { + encrypt_query_param: Some("enc".to_string()), + aes_key: Some("YWJjZGVmZ2hpamtsbW5vcA==".to_string()), + encrypt_type: Some(1), + }), + encode_type: Some(8), + playtime: Some(4200), + text: Some("hello from voice".to_string()), + }), + file_item: None, + video_item: None, + }], + }; + + let attachment = map_voice_attachment(&config, &message, &message.item_list[0], 0) + .expect("voice attachment should map") + .expect("voice attachment should be present"); + assert_eq!(attachment.id, "wechat-voice-77-0"); + assert_eq!(attachment.mime_type, "audio/ogg"); + assert_eq!( + attachment.filename.as_deref(), + Some("wechat-voice-77-0.ogg") + ); + assert_eq!( + attachment.extracted_text.as_deref(), + Some("hello from voice") + ); + assert!(attachment.extras_json.contains("duration_secs")); + } + + #[test] + fn test_map_video_attachment_sets_video_metadata() { + let config = WechatConfig::default(); + let message = WechatMessage { + message_id: Some(88), + from_user_id: Some("user-1".to_string()), + to_user_id: Some("bot-1".to_string()), + session_id: None, + message_type: None, + context_token: None, + item_list: vec![MessageItem { + r#type: Some(MESSAGE_ITEM_VIDEO), + text_item: None, + image_item: None, + voice_item: None, + file_item: None, + video_item: Some(VideoItem { + media: Some(CdnMedia { + encrypt_query_param: Some("enc".to_string()), + aes_key: Some("YWJjZGVmZ2hpamtsbW5vcA==".to_string()), + encrypt_type: Some(1), + }), + thumb_media: None, + video_size: Some(2048), + thumb_size: None, + play_length: Some(6_000), + }), + }], + }; + + let attachment = map_video_attachment(&config, &message, &message.item_list[0], 0) + .expect("video attachment should map") + .expect("video attachment should be present"); + assert_eq!(attachment.id, "wechat-video-88-0"); + assert_eq!(attachment.mime_type, "video/mp4"); + assert_eq!( + attachment.filename.as_deref(), + Some("wechat-video-88-0.mp4") + ); + assert_eq!(attachment.size_bytes, Some(2048)); + assert!(attachment.extras_json.contains("wechat_aes_key")); + } + + #[test] + fn test_classify_outbound_media_kind_routes_supported_media_types() { + assert_eq!( + classify_outbound_media_kind("image/png"), + OutboundMediaKind::Image + ); + assert_eq!( + classify_outbound_media_kind("video/mp4"), + OutboundMediaKind::Video + ); + assert_eq!( + classify_outbound_media_kind("application/pdf"), + OutboundMediaKind::File + ); + } +} diff --git a/channels-src/wechat/src/state.rs b/channels-src/wechat/src/state.rs new file mode 100644 index 00000000000..4c1e671c5f0 --- /dev/null +++ b/channels-src/wechat/src/state.rs @@ -0,0 +1,213 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::auth::{ + CONFIG_PATH, CONTEXT_TOKENS_PATH, GET_UPDATES_BUF_PATH, PENDING_INBOUND_PATH, + PROCESSED_MESSAGE_IDS_PATH, TYPING_TICKETS_PATH, +}; +use crate::near::agent::channel_host; +use crate::types::WechatConfig; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TypingTicketEntry { + pub ticket: String, + pub fetched_at_ms: u64, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub struct StoredInboundAttachment { + pub id: String, + pub mime_type: String, + pub filename: Option, + pub size_bytes: Option, + pub source_url: Option, + pub storage_key: Option, + pub extracted_text: Option, + pub extras_json: String, +} + +impl From for StoredInboundAttachment { + fn from(value: channel_host::InboundAttachment) -> Self { + Self { + id: value.id, + mime_type: value.mime_type, + filename: value.filename, + size_bytes: value.size_bytes, + source_url: value.source_url, + storage_key: value.storage_key, + extracted_text: value.extracted_text, + extras_json: value.extras_json, + } + } +} + +impl From for channel_host::InboundAttachment { + fn from(value: StoredInboundAttachment) -> Self { + Self { + id: value.id, + mime_type: value.mime_type, + filename: value.filename, + size_bytes: value.size_bytes, + source_url: value.source_url, + storage_key: value.storage_key, + extracted_text: value.extracted_text, + extras_json: value.extras_json, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub struct PendingInboundBundle { + pub from_user_id: String, + pub to_user_id: Option, + pub session_id: Option, + pub context_token: Option, + pub message_id: Option, + pub flush_at_ms: u64, + pub text: String, + pub attachments: Vec, +} + +pub fn load_config() -> WechatConfig { + channel_host::workspace_read(CONFIG_PATH) + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .unwrap_or_default() +} + +pub fn persist_config(config: &WechatConfig) -> Result<(), String> { + let serialized = + serde_json::to_string(config).map_err(|e| format!("Failed to serialize config: {e}"))?; + channel_host::workspace_write(CONFIG_PATH, &serialized).map_err(|e| e.to_string()) +} + +pub fn load_get_updates_buf() -> String { + channel_host::workspace_read(GET_UPDATES_BUF_PATH) + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .unwrap_or_default() +} + +pub fn persist_get_updates_buf(value: &str) -> Result<(), String> { + let serialized = + serde_json::to_string(value).map_err(|e| format!("Failed to serialize cursor: {e}"))?; + channel_host::workspace_write(GET_UPDATES_BUF_PATH, &serialized).map_err(|e| e.to_string()) +} + +pub fn load_context_tokens() -> HashMap { + channel_host::workspace_read(CONTEXT_TOKENS_PATH) + .and_then(|raw| serde_json::from_str::>(&raw).ok()) + .unwrap_or_default() +} + +pub fn persist_context_tokens(tokens: &HashMap) -> Result<(), String> { + let serialized = + serde_json::to_string(tokens).map_err(|e| format!("Failed to serialize tokens: {e}"))?; + channel_host::workspace_write(CONTEXT_TOKENS_PATH, &serialized).map_err(|e| e.to_string()) +} + +pub fn load_typing_tickets() -> HashMap { + channel_host::workspace_read(TYPING_TICKETS_PATH) + .and_then(|raw| serde_json::from_str::>(&raw).ok()) + .unwrap_or_default() +} + +pub fn persist_typing_tickets(tickets: &HashMap) -> Result<(), String> { + let serialized = + serde_json::to_string(tickets).map_err(|e| format!("Failed to serialize tickets: {e}"))?; + channel_host::workspace_write(TYPING_TICKETS_PATH, &serialized).map_err(|e| e.to_string()) +} + +pub fn load_pending_inbound_bundles() -> Result, String> { + parse_pending_inbound_bundles(channel_host::workspace_read(PENDING_INBOUND_PATH).as_deref()) +} + +pub fn persist_pending_inbound_bundles( + bundles: &HashMap, +) -> Result<(), String> { + let serialized = + serde_json::to_string(bundles).map_err(|e| format!("Failed to serialize bundles: {e}"))?; + channel_host::workspace_write(PENDING_INBOUND_PATH, &serialized).map_err(|e| e.to_string()) +} + +fn parse_pending_inbound_bundles( + raw: Option<&str>, +) -> Result, String> { + match raw { + None => Ok(HashMap::new()), + Some(raw) => serde_json::from_str(raw) + .map_err(|e| format!("Failed to parse pending inbound bundles: {e}")), + } +} + +pub fn load_processed_message_ids() -> Result, String> { + parse_processed_message_ids(channel_host::workspace_read(PROCESSED_MESSAGE_IDS_PATH).as_deref()) +} + +pub fn persist_processed_message_ids(message_ids: &[i64]) -> Result<(), String> { + let serialized = serde_json::to_string(message_ids) + .map_err(|e| format!("Failed to serialize processed message ids: {e}"))?; + channel_host::workspace_write(PROCESSED_MESSAGE_IDS_PATH, &serialized) + .map_err(|e| e.to_string()) +} + +pub fn has_processed_message_id(processed_message_ids: &[i64], message_id: i64) -> bool { + processed_message_ids.contains(&message_id) +} + +pub fn remember_processed_message_id( + processed_message_ids: &mut Vec, + message_id: i64, + max_entries: usize, +) -> bool { + if processed_message_ids.contains(&message_id) { + return false; + } + processed_message_ids.push(message_id); + if processed_message_ids.len() > max_entries { + let excess = processed_message_ids.len() - max_entries; + processed_message_ids.drain(0..excess); + } + true +} + +fn parse_processed_message_ids(raw: Option<&str>) -> Result, String> { + match raw { + None => Ok(Vec::new()), + Some(raw) => serde_json::from_str(raw) + .map_err(|e| format!("Failed to parse processed message ids: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_pending_inbound_bundles_missing_file_returns_empty_map() { + let bundles = parse_pending_inbound_bundles(None).expect("missing state should be empty"); + assert!(bundles.is_empty()); + } + + #[test] + fn test_parse_pending_inbound_bundles_invalid_json_returns_error() { + let error = + parse_pending_inbound_bundles(Some("{not json")).expect_err("invalid json should err"); + assert!(error.contains("Failed to parse pending inbound bundles")); + } + + #[test] + fn test_parse_processed_message_ids_missing_file_returns_empty_vec() { + let ids = parse_processed_message_ids(None).expect("missing state should be empty"); + assert!(ids.is_empty()); + } + + #[test] + fn test_remember_processed_message_id_dedups_and_trims() { + let mut ids = vec![1, 2]; + assert!(!remember_processed_message_id(&mut ids, 2, 3)); + assert_eq!(ids, vec![1, 2]); + + assert!(remember_processed_message_id(&mut ids, 3, 2)); + assert_eq!(ids, vec![2, 3]); + } +} diff --git a/channels-src/wechat/src/types.rs b/channels-src/wechat/src/types.rs new file mode 100644 index 00000000000..0a87924c490 --- /dev/null +++ b/channels-src/wechat/src/types.rs @@ -0,0 +1,237 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WechatConfig { + #[serde(default = "default_base_url")] + pub base_url: String, + #[serde(default = "default_cdn_base_url")] + pub cdn_base_url: String, + #[serde(default = "default_bot_type")] + pub bot_type: String, + #[serde(default = "default_poll_interval_ms")] + pub poll_interval_ms: u32, + #[serde(default = "default_long_poll_timeout_ms")] + pub long_poll_timeout_ms: u32, + #[serde(default = "default_inbound_merge_window_ms")] + pub inbound_merge_window_ms: u32, +} + +fn default_base_url() -> String { + "https://ilinkai.weixin.qq.com".to_string() +} + +fn default_cdn_base_url() -> String { + "https://novac2c.cdn.weixin.qq.com/c2c".to_string() +} + +fn default_bot_type() -> String { + "3".to_string() +} + +fn default_poll_interval_ms() -> u32 { + 30_000 +} + +fn default_long_poll_timeout_ms() -> u32 { + 35_000 +} + +fn default_inbound_merge_window_ms() -> u32 { + 5_000 +} + +impl Default for WechatConfig { + fn default() -> Self { + Self { + base_url: default_base_url(), + cdn_base_url: default_cdn_base_url(), + bot_type: default_bot_type(), + poll_interval_ms: default_poll_interval_ms(), + long_poll_timeout_ms: default_long_poll_timeout_ms(), + inbound_merge_window_ms: default_inbound_merge_window_ms(), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct BaseInfo { + pub channel_version: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct GetUploadUrlRequest { + pub filekey: String, + pub media_type: i32, + pub to_user_id: String, + pub rawsize: u64, + pub rawfilemd5: String, + pub filesize: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub thumb_rawsize: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub thumb_rawfilemd5: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub thumb_filesize: Option, + pub no_need_thumb: bool, + pub aeskey: String, + pub base_info: BaseInfo, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct GetUpdatesRequest { + pub get_updates_buf: String, + pub base_info: BaseInfo, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct GetConfigRequest { + pub ilink_user_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub context_token: Option, + pub base_info: BaseInfo, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GetUpdatesResponse { + pub ret: Option, + pub errcode: Option, + pub errmsg: Option, + #[serde(default)] + pub msgs: Vec, + pub get_updates_buf: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GetUploadUrlResponse { + pub upload_param: Option, + pub thumb_upload_param: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SendMessageRequest { + pub msg: OutboundWechatMessage, + pub base_info: BaseInfo, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SendTypingRequest { + pub ilink_user_id: String, + pub typing_ticket: String, + pub status: i32, + pub base_info: BaseInfo, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OutboundWechatMessage { + pub from_user_id: String, + pub to_user_id: String, + pub client_id: String, + pub message_type: i32, + pub message_state: i32, + pub item_list: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub context_token: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct WechatMessage { + pub message_id: Option, + pub from_user_id: Option, + pub to_user_id: Option, + pub session_id: Option, + pub message_type: Option, + pub context_token: Option, + #[serde(default)] + pub item_list: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GetConfigResponse { + pub ret: Option, + pub errmsg: Option, + pub typing_ticket: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SendTypingResponse { + pub ret: Option, + pub errmsg: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct MessageItem { + pub r#type: Option, + pub text_item: Option, + pub image_item: Option, + pub voice_item: Option, + pub file_item: Option, + pub video_item: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TextItem { + pub text: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CdnMedia { + pub encrypt_query_param: Option, + pub aes_key: Option, + pub encrypt_type: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ImageItem { + pub media: Option, + pub thumb_media: Option, + pub aeskey: Option, + pub mid_size: Option, + pub thumb_size: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct VoiceItem { + pub media: Option, + pub encode_type: Option, + pub playtime: Option, + pub text: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FileItem { + pub media: Option, + pub file_name: Option, + pub len: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct VideoItem { + pub media: Option, + pub thumb_media: Option, + pub video_size: Option, + pub thumb_size: Option, + pub play_length: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OutboundMetadata { + pub from_user_id: String, + pub to_user_id: Option, + pub message_id: Option, + pub session_id: Option, + pub context_token: Option, +} + +pub const MESSAGE_TYPE_USER: i32 = 1; +pub const MESSAGE_TYPE_BOT: i32 = 2; +pub const MESSAGE_STATE_FINISH: i32 = 2; +pub const MESSAGE_ITEM_TEXT: i32 = 1; +pub const MESSAGE_ITEM_IMAGE: i32 = 2; +pub const MESSAGE_ITEM_VOICE: i32 = 3; +pub const MESSAGE_ITEM_FILE: i32 = 4; +pub const MESSAGE_ITEM_VIDEO: i32 = 5; +pub const TYPING_STATUS_TYPING: i32 = 1; +pub const TYPING_STATUS_CANCEL: i32 = 2; +pub const UPLOAD_MEDIA_TYPE_IMAGE: i32 = 1; +pub const UPLOAD_MEDIA_TYPE_VIDEO: i32 = 2; +pub const UPLOAD_MEDIA_TYPE_FILE: i32 = 3; diff --git a/channels-src/wechat/wechat.capabilities.json b/channels-src/wechat/wechat.capabilities.json new file mode 100644 index 00000000000..05f3329b00a --- /dev/null +++ b/channels-src/wechat/wechat.capabilities.json @@ -0,0 +1,51 @@ +{ + "version": "0.1.0", + "wit_version": "0.3.0", + "type": "channel", + "name": "wechat", + "description": "WeChat iLink Bot channel for direct-message chat via long polling", + "setup": { + "required_secrets": [ + { + "name": "wechat_bot_token", + "prompt": "Connect this channel from the WeChat setup flow. IronClaw stores the bot token after QR login succeeds.", + "optional": false + } + ], + "setup_url": "https://ilinkai.weixin.qq.com" + }, + "capabilities": { + "http": { + "allowlist": [ + { "host": "ilinkai.weixin.qq.com", "path_prefix": "/" }, + { "host": "novac2c.cdn.weixin.qq.com", "path_prefix": "/c2c/" } + ], + "rate_limit": { + "requests_per_minute": 60, + "requests_per_hour": 1200 + } + }, + "secrets": { + "allowed_names": ["wechat_*"] + }, + "channel": { + "allowed_paths": [], + "allow_polling": true, + "min_poll_interval_ms": 30000, + "workspace_prefix": "channels/wechat/", + "callback_timeout_secs": 45, + "emit_rate_limit": { + "messages_per_minute": 100, + "messages_per_hour": 5000 + } + } + }, + "config": { + "base_url": "https://ilinkai.weixin.qq.com", + "cdn_base_url": "https://novac2c.cdn.weixin.qq.com/c2c", + "bot_type": "3", + "poll_interval_ms": 30000, + "long_poll_timeout_ms": 35000, + "inbound_merge_window_ms": 5000 + } +} diff --git a/channels-src/wecom/Cargo.lock b/channels-src/wecom/Cargo.lock new file mode 100644 index 00000000000..cbf5347dcd3 --- /dev/null +++ b/channels-src/wecom/Cargo.lock @@ -0,0 +1,542 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "spdx" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-encoder" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1" +dependencies = [ + "leb128", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap", + "semver", +] + +[[package]] +name = "wecom-channel" +version = "0.1.0" +dependencies = [ + "aes", + "base64", + "cbc", + "md-5", + "serde", + "serde_json", + "sha1", + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3" +dependencies = [ + "wit-bindgen-rt", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/channels-src/wecom/Cargo.toml b/channels-src/wecom/Cargo.toml new file mode 100644 index 00000000000..ad6bf1e6ca6 --- /dev/null +++ b/channels-src/wecom/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "wecom-channel" +version = "0.1.0" +edition = "2021" +description = "WeCom bot-first channel for IronClaw" +license = "MIT OR Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.36" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +base64 = "0.22" +sha1 = "0.10" +md-5 = "0.10" +aes = "0.8" +cbc = "0.1" + +[profile.release] +opt-level = "s" +lto = true +strip = true +codegen-units = 1 + +[workspace] diff --git a/channels-src/wecom/build.sh b/channels-src/wecom/build.sh new file mode 100755 index 00000000000..58a0e7de928 --- /dev/null +++ b/channels-src/wecom/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Build the WeCom channel WASM component + +set -euo pipefail + +cd "$(dirname "$0")" + +echo "Building WeCom channel WASM component..." + +cargo build --release --target wasm32-wasip2 + +WASM_PATH="target/wasm32-wasip2/release/wecom_channel.wasm" + +if [ -f "$WASM_PATH" ]; then + if command -v wasm-tools >/dev/null 2>&1; then + wasm-tools component new "$WASM_PATH" -o wecom.wasm 2>/dev/null || cp "$WASM_PATH" wecom.wasm + wasm-tools strip wecom.wasm -o wecom.wasm + else + cp "$WASM_PATH" wecom.wasm + echo "Note: wasm-tools not found; wrote raw wasm artifact without component conversion/strip" + fi + + echo "Built: wecom.wasm ($(du -h wecom.wasm | cut -f1))" + echo "" + echo "To install:" + echo " mkdir -p ~/.ironclaw/channels" + echo " cp wecom.wasm wecom.capabilities.json ~/.ironclaw/channels/" +else + echo "Error: WASM output not found at $WASM_PATH" + exit 1 +fi diff --git a/channels-src/wecom/src/lib.rs b/channels-src/wecom/src/lib.rs new file mode 100644 index 00000000000..e1a5df8bd7e --- /dev/null +++ b/channels-src/wecom/src/lib.rs @@ -0,0 +1,3800 @@ +//! WeCom channel for IronClaw. +//! +//! Current shape: +//! - bot websocket is the session path for inbound text/events and replies +//! - bot upload/send commands handle generated media without app credentials +//! - HTTP callback and proactive REST send support are intentionally out of scope + +wit_bindgen::generate!({ + world: "sandboxed-channel", + path: "../../wit/channel.wit", +}); + +use aes::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit}; +use aes::Aes256; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::Engine as _; +use cbc::Decryptor; +use md5::Md5; +use serde::{Deserialize, Serialize}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use sha1::{Digest, Sha1}; +#[cfg(test)] +use std::cell::RefCell; +use std::collections::HashMap; + +use exports::near::agent::channel::{ + AgentResponse, Attachment, ChannelConfig, Guest, IncomingHttpRequest, OutgoingHttpResponse, + PollConfig, StatusType, StatusUpdate, +}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; + +const CHANNEL_NAME: &str = "wecom"; +const OWNER_ID_PATH: &str = "owner_id"; +const DM_POLICY_PATH: &str = "dm_policy"; +const ALLOW_FROM_PATH: &str = "allow_from"; +const RECENT_MSG_IDS_PATH: &str = "recent_msg_ids"; +const WEBSOCKET_EVENT_QUEUE_PATH: &str = "state/gateway_event_queue_processing"; +const WEBSOCKET_MEDIA_STATE_PATH: &str = "state/websocket_media_sends"; +const WEBSOCKET_MEDIA_CHUNK_BLOBS_PREFIX: &str = "state/websocket_media_chunks"; +const PENDING_INBOUND_PATH: &str = "state/pending_inbound_bundles"; +const INBOUND_MERGE_WINDOW_MS_PATH: &str = "state/inbound_merge_window_ms"; +const PENDING_ATTACHMENT_BLOBS_PREFIX: &str = "state/pending_attachment_blobs"; + +const WECOM_WS_REPLY_CMD: &str = "aibot_respond_msg"; +const WECOM_WS_WELCOME_CMD: &str = "aibot_respond_welcome_msg"; +const WECOM_WS_SEND_MSG_CMD: &str = "aibot_send_msg"; +const WECOM_WS_UPLOAD_MEDIA_INIT_CMD: &str = "aibot_upload_media_init"; +const WECOM_WS_UPLOAD_MEDIA_CHUNK_CMD: &str = "aibot_upload_media_chunk"; +const WECOM_WS_UPLOAD_MEDIA_FINISH_CMD: &str = "aibot_upload_media_finish"; + +const STREAM_CHUNK_LIMIT_BYTES: usize = 20_000; +const STATUS_MESSAGE_MAX_CHARS: usize = 1200; +const WEBSOCKET_MEDIA_CHUNK_SIZE: usize = 512 * 1024; +const MAX_WEBSOCKET_MEDIA_CHUNKS: usize = 100; +const MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE: usize = 4; +const MAX_WEBSOCKET_MEDIA_TOTAL_BYTES_PER_RESPONSE: usize = 20 * 1024 * 1024; +const MAX_WEBSOCKET_MEDIA_BATCH_ERRORS: usize = 10; +const MAX_ATTACHMENT_BYTES: usize = 20 * 1024 * 1024; +const MAX_WEBSOCKET_IMAGE_BYTES: usize = 10 * 1024 * 1024; +const MAX_WEBSOCKET_VOICE_BYTES: usize = 2 * 1024 * 1024; +const MAX_WEBSOCKET_VIDEO_BYTES: usize = 10 * 1024 * 1024; +const MAX_RECENT_MSG_IDS: usize = 256; +const MAX_RECENT_MSG_ID_AGE_MS: u64 = 24 * 60 * 60 * 1000; +const WEBSOCKET_MEDIA_SEND_TTL_MS: u64 = 10 * 60 * 1000; +const DEFAULT_INBOUND_MERGE_WINDOW_MS: u64 = 5_000; +const MAX_INBOUND_MERGE_WINDOW_MS: u64 = 60_000; +const WECOM_POLL_INTERVAL_MS: u32 = 1_000; + +type Aes256CbcDec = Decryptor; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum DmPolicy { + #[default] + Pairing, + Open, + Allowlist, +} + +impl DmPolicy { + fn as_str(self) -> &'static str { + match self { + Self::Pairing => "pairing", + Self::Open => "open", + Self::Allowlist => "allowlist", + } + } + + fn parse(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "open" => Self::Open, + "allowlist" => Self::Allowlist, + _ => Self::Pairing, + } + } +} + +#[derive(Debug, Deserialize)] +struct WecomConfig { + owner_id: Option, + dm_policy: Option, + allow_from: Option>, + inbound_merge_window_ms: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct WecomMessageMetadata { + to_user: String, + target: Option, + chat_id: Option, + chat_type: Option, + source_msg_id: Option, + ws_req_id: Option, + ws_chat_id: Option, + ws_chat_type: Option, + ws_reply_cmd: Option, +} + +#[derive(Debug, Clone)] +struct PairingReplyRoute { + req_id: String, + reply_cmd: String, + chat_type: Option, +} + +#[derive(Debug, Deserialize)] +struct WecomWsFrame { + headers: WecomWsHeaders, + body: T, +} + +#[derive(Debug, Deserialize)] +struct WecomWsHeaders { + req_id: String, +} + +#[derive(Debug, Deserialize)] +struct WecomWsAckFrame { + headers: WecomWsHeaders, + errcode: i64, + #[serde(default)] + errmsg: String, + #[serde(default)] + body: JsonValue, +} + +#[derive(Debug, Deserialize)] +struct WecomWsSender { + userid: String, +} + +#[derive(Debug, Deserialize)] +struct WecomWsTextContent { + content: String, +} + +#[derive(Debug, Deserialize)] +struct WecomWsBinaryContent { + url: String, + #[serde(default)] + #[cfg_attr(test, allow(dead_code))] + aeskey: Option, +} + +#[derive(Debug, Deserialize)] +struct WecomWsMixedItem { + #[serde(default, alias = "msgtype", alias = "type", alias = "itemtype")] + item_type: Option, + #[serde(default)] + text: Option, + #[serde(default)] + image: Option, + #[serde(default)] + file: Option, + #[serde(default)] + video: Option, +} + +#[derive(Debug, Deserialize)] +struct WecomWsMixedContent { + #[serde(default, alias = "msgItem", alias = "items")] + msg_item: Vec, +} + +#[derive(Debug, Deserialize)] +struct WecomWsQuoteContent { + #[serde(default, alias = "msgtype", alias = "type")] + msg_type: Option, + #[serde(default)] + text: Option, + #[serde(default)] + voice: Option, + #[serde(default)] + content: Option, +} + +#[derive(Debug, Deserialize)] +struct WecomWsMessageBody { + msgid: String, + #[serde(default)] + chatid: Option, + #[serde(default)] + chattype: Option, + from: WecomWsSender, + msgtype: String, + #[serde(default)] + text: Option, + #[serde(default)] + voice: Option, + #[serde(default)] + image: Option, + #[serde(default)] + file: Option, + #[serde(default)] + video: Option, + #[serde(default)] + mixed: Option, + #[serde(default)] + quote: Option, +} + +#[derive(Debug, Deserialize)] +struct WecomWsEventBody { + msgid: String, + #[serde(default)] + chatid: Option, + #[serde(default)] + chattype: Option, + from: WecomWsSender, + event: WecomWsEvent, +} + +#[derive(Debug, Deserialize)] +struct WecomWsEvent { + eventtype: String, + #[serde(flatten)] + extra: JsonMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OutboundMediaKind { + Image, + Voice, + Video, + File, +} + +impl OutboundMediaKind { + fn as_api_type(self) -> &'static str { + match self { + Self::Image => "image", + Self::Voice => "voice", + Self::Video => "video", + Self::File => "file", + } + } + + fn websocket_max_bytes(self) -> usize { + match self { + Self::Image => MAX_WEBSOCKET_IMAGE_BYTES, + Self::Voice => MAX_WEBSOCKET_VOICE_BYTES, + Self::Video => MAX_WEBSOCKET_VIDEO_BYTES, + Self::File => MAX_ATTACHMENT_BYTES, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Default)] +struct PendingWebsocketMediaState { + #[serde(default)] + sends: Vec, + #[serde(default)] + batches: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct PendingWebsocketMediaBatch { + id: String, + chat_id: String, + #[serde(default)] + created_at_ms: u64, + #[serde(default)] + response_req_id: String, + #[serde(default)] + response_cmd: String, + final_text: String, + remaining_media: usize, + sent_media: usize, + failed_media: usize, + #[serde(default)] + errors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PendingWebsocketMediaSend { + id: String, + batch_id: String, + chat_id: String, + #[serde(default)] + created_at_ms: u64, + media_type: String, + filename: String, + #[serde(default)] + md5_hex: String, + #[serde(default)] + chunk_blob_paths: Vec, + total_size: usize, + total_chunks: usize, + next_chunk_index: usize, + init_req_id: String, + #[serde(default)] + chunk_req_id: Option, + #[serde(default)] + finish_req_id: Option, + #[serde(default)] + send_req_id: Option, + #[serde(default)] + upload_id: Option, + #[serde(default)] + media_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct RecentMessageIdEntry { + id: String, + seen_at_ms: u64, +} + +struct PendingWebsocketMediaOutbound { + send_id: String, + payload: String, +} + +enum PendingWebsocketMediaAdvance { + Send(PendingWebsocketMediaOutbound, PendingWebsocketMediaSend), + Complete, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +struct StoredInboundAttachment { + id: String, + mime_type: String, + filename: Option, + size_bytes: Option, + extracted_text: Option, +} + +impl From for StoredInboundAttachment { + fn from(value: InboundAttachment) -> Self { + Self { + id: value.id, + mime_type: value.mime_type, + filename: value.filename, + size_bytes: value.size_bytes, + extracted_text: value.extracted_text, + } + } +} + +impl From for InboundAttachment { + fn from(value: StoredInboundAttachment) -> Self { + Self { + id: value.id, + mime_type: value.mime_type, + filename: value.filename, + size_bytes: value.size_bytes, + source_url: None, + storage_key: None, + extracted_text: value.extracted_text, + extras_json: "{}".to_string(), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +struct PendingInboundBundle { + user_id: String, + user_name: Option, + thread_id: String, + metadata_json: String, + content: String, + attachments: Vec, + flush_at_ms: u64, +} + +struct WebsocketMediaStartResult { + started: usize, + errors: Vec, +} + +fn inbound_merge_window_ms() -> u64 { + channel_host::workspace_read(INBOUND_MERGE_WINDOW_MS_PATH) + .and_then(|raw| raw.trim().parse::().ok()) + .unwrap_or(DEFAULT_INBOUND_MERGE_WINDOW_MS) + .min(MAX_INBOUND_MERGE_WINDOW_MS) +} + +fn merge_inbound_text(existing: &str, incoming: &str) -> String { + let existing = existing.trim(); + let incoming = incoming.trim(); + match (existing.is_empty(), incoming.is_empty()) { + (true, true) => String::new(), + (true, false) => incoming.to_string(), + (false, true) => existing.to_string(), + (false, false) => format!("{existing}\n\n{incoming}"), + } +} + +fn next_inbound_flush_deadline(now_ms: u64, merge_window_ms: u64) -> u64 { + now_ms.saturating_add(merge_window_ms) +} + +#[cfg(test)] +thread_local! { + static TEST_WORKSPACE: RefCell> = RefCell::new(HashMap::new()); + static TEST_WEBSOCKET_OUTBOUND: RefCell> = const { RefCell::new(Vec::new()) }; + static TEST_WEBSOCKET_SEND_ERROR: RefCell> = const { RefCell::new(None) }; +} + +#[cfg(test)] +fn test_reset_websocket_state() { + TEST_WORKSPACE.with(|workspace| workspace.borrow_mut().clear()); + TEST_WEBSOCKET_OUTBOUND.with(|outbound| outbound.borrow_mut().clear()); + TEST_WEBSOCKET_SEND_ERROR.with(|error| *error.borrow_mut() = None); +} + +fn read_wecom_workspace(path: &str) -> Option { + #[cfg(test)] + { + TEST_WORKSPACE.with(|workspace| workspace.borrow().get(path).cloned()) + } + #[cfg(not(test))] + { + channel_host::workspace_read(path) + } +} + +fn write_wecom_workspace(path: &str, content: &str) -> Result<(), String> { + #[cfg(test)] + { + TEST_WORKSPACE.with(|workspace| { + if content.is_empty() { + workspace.borrow_mut().remove(path); + } else { + workspace + .borrow_mut() + .insert(path.to_string(), content.to_string()); + } + }); + Ok(()) + } + #[cfg(not(test))] + { + channel_host::workspace_write(path, content) + } +} + +fn send_websocket_text(payload: &str) -> Result<(), String> { + #[cfg(test)] + { + if let Some(error) = TEST_WEBSOCKET_SEND_ERROR.with(|error| error.borrow_mut().take()) { + return Err(error); + } + TEST_WEBSOCKET_OUTBOUND.with(|outbound| outbound.borrow_mut().push(payload.to_string())); + Ok(()) + } + #[cfg(not(test))] + { + channel_host::websocket_send_text(payload) + } +} + +fn log_wecom(level: channel_host::LogLevel, message: &str) { + #[cfg(test)] + { + let _ = (level, message); + } + #[cfg(not(test))] + { + channel_host::log(level, message); + } +} + +fn pending_attachment_blob_path(attachment_id: &str) -> String { + let mut hasher = Md5::new(); + hasher.update(attachment_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("{PENDING_ATTACHMENT_BLOBS_PREFIX}/{digest}.b64") +} + +fn persist_pending_attachment_blob(attachment_id: &str, data: &[u8]) -> Result<(), String> { + if data.is_empty() { + return Ok(()); + } + let encoded = BASE64_STANDARD.encode(data); + channel_host::workspace_write(&pending_attachment_blob_path(attachment_id), &encoded) + .map_err(|e| format!("Failed to persist pending WeCom attachment blob: {e}")) +} + +fn load_pending_attachment_blob(attachment_id: &str) -> Result>, String> { + let Some(raw) = channel_host::workspace_read(&pending_attachment_blob_path(attachment_id)) + else { + return Ok(None); + }; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Ok(None); + } + BASE64_STANDARD + .decode(trimmed) + .map(Some) + .map_err(|e| format!("Failed to decode pending WeCom attachment blob: {e}")) +} + +fn clear_pending_attachment_blob(attachment_id: &str) { + let _ = channel_host::workspace_write(&pending_attachment_blob_path(attachment_id), ""); +} + +fn load_pending_inbound_bundles() -> HashMap { + let Some(raw) = channel_host::workspace_read(PENDING_INBOUND_PATH) else { + return HashMap::new(); + }; + let raw = raw.trim(); + if raw.is_empty() { + return HashMap::new(); + } + match serde_json::from_str(raw) { + Ok(value) => value, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to parse WeCom pending inbound bundles: {error}"), + ); + HashMap::new() + } + } +} + +fn persist_pending_inbound_bundles( + pending: &HashMap, +) -> Result<(), String> { + let serialized = serde_json::to_string(pending) + .map_err(|e| format!("Failed to serialize WeCom pending inbound bundles: {e}"))?; + channel_host::workspace_write(PENDING_INBOUND_PATH, &serialized) + .map_err(|e| format!("Failed to persist WeCom pending inbound bundles: {e}")) +} + +fn take_due_pending_inbound_bundles( + pending: &mut HashMap, + now_ms: u64, +) -> Vec { + let due_keys: Vec = pending + .iter() + .filter_map(|(key, bundle)| (bundle.flush_at_ms <= now_ms).then_some(key.clone())) + .collect(); + due_keys + .into_iter() + .filter_map(|key| pending.remove(&key)) + .collect() +} + +fn process_pending_inbound_bundle( + pending: &mut HashMap, + key: &str, + mut bundle: PendingInboundBundle, + now_ms: u64, + merge_window_ms: u64, +) -> Vec { + let bundle_has_text = !bundle.content.trim().is_empty(); + let bundle_has_attachments = !bundle.attachments.is_empty(); + + if let Some(mut existing) = pending.remove(key) { + existing.content = merge_inbound_text(&existing.content, &bundle.content); + existing.attachments.extend(bundle.attachments); + existing.user_id = bundle.user_id; + existing.user_name = bundle.user_name; + existing.thread_id = bundle.thread_id; + existing.metadata_json = bundle.metadata_json; + + let has_text = !existing.content.trim().is_empty(); + let has_attachments = !existing.attachments.is_empty(); + if has_text || merge_window_ms == 0 { + return vec![existing]; + } + if has_attachments { + existing.flush_at_ms = next_inbound_flush_deadline(now_ms, merge_window_ms); + pending.insert(key.to_string(), existing); + } + return Vec::new(); + } + + if bundle_has_attachments && !bundle_has_text && merge_window_ms > 0 { + bundle.flush_at_ms = next_inbound_flush_deadline(now_ms, merge_window_ms); + pending.insert(key.to_string(), bundle); + Vec::new() + } else { + vec![bundle] + } +} + +fn emit_pending_inbound_bundle(bundle: PendingInboundBundle) { + let mut attachments: Vec = + bundle.attachments.into_iter().map(Into::into).collect(); + let attachment_ids: Vec = attachments.iter().map(|att| att.id.clone()).collect(); + let mut hydrated = Vec::with_capacity(attachments.len()); + for mut attachment in attachments { + if let Err(error) = rehydrate_pending_inbound_attachment_data(&mut attachment) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!( + "Dropping WeCom attachment '{}' because inline data is unavailable: {}", + attachment.id, error + ), + ); + continue; + } + hydrated.push(attachment); + } + attachments = hydrated; + + if bundle.content.trim().is_empty() && attachments.is_empty() { + channel_host::log( + channel_host::LogLevel::Warn, + "Skipping buffered WeCom message because both content and attachments are empty after rehydrate", + ); + for attachment_id in attachment_ids { + clear_pending_attachment_blob(&attachment_id); + } + return; + } + + channel_host::emit_message(&EmittedMessage { + user_id: bundle.user_id, + user_name: bundle.user_name, + content: bundle.content, + thread_id: Some(bundle.thread_id), + metadata_json: bundle.metadata_json, + attachments, + }); + for attachment_id in attachment_ids { + clear_pending_attachment_blob(&attachment_id); + } +} + +fn emit_or_buffer_incoming_user_message( + user_id: String, + user_name: Option, + content: String, + thread_id: String, + metadata_json: String, + attachments: Vec, +) { + let now_ms = channel_host::now_millis(); + let mut pending = load_pending_inbound_bundles(); + let mut emitted = take_due_pending_inbound_bundles(&mut pending, now_ms); + let key = thread_id.clone(); + emitted.extend(process_pending_inbound_bundle( + &mut pending, + &key, + PendingInboundBundle { + user_id, + user_name, + content, + thread_id, + metadata_json, + attachments: attachments.into_iter().map(Into::into).collect(), + flush_at_ms: 0, + }, + now_ms, + inbound_merge_window_ms(), + )); + + if let Err(error) = persist_pending_inbound_bundles(&pending) { + channel_host::log(channel_host::LogLevel::Warn, &error); + } + + for bundle in emitted { + emit_pending_inbound_bundle(bundle); + } +} + +fn flush_due_pending_inbound_bundles() { + let now_ms = channel_host::now_millis(); + let mut pending = load_pending_inbound_bundles(); + let due = take_due_pending_inbound_bundles(&mut pending, now_ms); + if due.is_empty() { + return; + } + + if let Err(error) = persist_pending_inbound_bundles(&pending) { + channel_host::log(channel_host::LogLevel::Warn, &error); + } + + for bundle in due { + emit_pending_inbound_bundle(bundle); + } +} + +fn text_response(status: u16, body: &str) -> OutgoingHttpResponse { + OutgoingHttpResponse { + status, + headers_json: serde_json::json!({ + "Content-Type": "text/plain; charset=utf-8", + }) + .to_string(), + body: body.as_bytes().to_vec(), + } +} + +fn load_allow_from() -> Vec { + let mut allowed = Vec::new(); + // silent-ok: missing static allowlist means "no statically allowed senders"; + // pairing_read_allow_from below still provides DB-backed approvals. + if let Some(raw) = channel_host::workspace_read(ALLOW_FROM_PATH) { + match serde_json::from_str::>(&raw) { + Ok(values) => allowed = values, + Err(error) => channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to parse WeCom allow_from list: {error}"), + ), + } + } + if let Ok(stored) = channel_host::pairing_read_allow_from(CHANNEL_NAME) { + allowed.extend(stored); + } + allowed +} + +fn load_dm_policy() -> DmPolicy { + channel_host::workspace_read(DM_POLICY_PATH) + .map(|value| DmPolicy::parse(&value)) + .unwrap_or_default() +} + +struct SenderAccessSnapshot { + dm_policy: DmPolicy, + allow_from: Vec, +} + +impl SenderAccessSnapshot { + fn load() -> Self { + Self { + dm_policy: load_dm_policy(), + allow_from: load_allow_from(), + } + } +} + +fn send_pairing_reply(route: &PairingReplyRoute, code: &str) -> Result<(), String> { + let content = pairing_reply_text(route, code); + send_websocket_stream_reply(&route.req_id, &route.reply_cmd, &content) +} + +fn pairing_route_is_group(route: &PairingReplyRoute) -> bool { + chat_type_is_group(route.chat_type.as_deref()) +} + +fn chat_type_is_group(chat_type: Option<&str>) -> bool { + normalize_chat_type(chat_type).as_deref() == Some("group") +} + +fn chat_type_is_private(chat_type: Option<&str>) -> bool { + normalize_chat_type(chat_type).as_deref() == Some("private") +} + +fn dm_policy_allows_sender_without_allowlist(dm_policy: DmPolicy, chat_type: Option<&str>) -> bool { + dm_policy == DmPolicy::Open && chat_type_is_private(chat_type) +} + +fn should_send_pairing_reply(route: &PairingReplyRoute, created: bool) -> bool { + if pairing_route_is_group(route) { + // Group chats only receive the generic "please DM" notice once per + // pairing request to avoid leaking or spamming operational details. + created + } else { + // In private chats, always send the code so users can still get it + // even if the request was originally created in a group context. + true + } +} + +fn pairing_reply_text(route: &PairingReplyRoute, code: &str) -> String { + if pairing_route_is_group(route) { + "This WeCom channel requires approval before chatting. For security, please DM the bot to get your pairing code.".to_string() + } else { + format!( + "This WeCom channel requires approval before chatting. Pairing code: {}", + code + ) + } +} + +fn is_sender_allowed( + sender_id: &str, + chat_type: Option<&str>, + access: &SenderAccessSnapshot, + pairing_reply: Option, +) -> Result { + let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty()); + if owner_id.as_deref() == Some(sender_id) { + return Ok(true); + } + + if dm_policy_allows_sender_without_allowlist(access.dm_policy, chat_type) { + return Ok(true); + } + + if access + .allow_from + .iter() + .any(|entry| entry == "*" || entry == sender_id) + { + return Ok(true); + } + + if access.dm_policy == DmPolicy::Pairing { + let meta = serde_json::json!({ "user_id": sender_id }).to_string(); + let result = channel_host::pairing_upsert_request(CHANNEL_NAME, sender_id, &meta)?; + if let Some(reply_route) = pairing_reply { + if should_send_pairing_reply(&reply_route, result.created) { + let _ = send_pairing_reply(&reply_route, &result.code); + } + } + } + + Ok(false) +} + +fn base_mime_type(mime_type: &str) -> &str { + mime_type.split(';').next().unwrap_or("").trim() +} + +fn lowercase_filename_extension(filename: &str) -> Option { + let (_, ext) = filename.rsplit_once('.')?; + let ext = ext.trim(); + if ext.is_empty() { + None + } else { + Some(ext.to_ascii_lowercase()) + } +} + +fn preferred_outbound_media_kind(att: &Attachment) -> OutboundMediaKind { + let mime = base_mime_type(&att.mime_type).to_ascii_lowercase(); + let ext = lowercase_filename_extension(&att.filename); + + if matches!(mime.as_str(), "image/jpeg" | "image/png") + || matches!(ext.as_deref(), Some("jpg" | "jpeg" | "png")) + { + OutboundMediaKind::Image + } else if matches!(mime.as_str(), "audio/amr" | "audio/x-amr") || ext.as_deref() == Some("amr") + { + OutboundMediaKind::Voice + } else if mime == "video/mp4" || ext.as_deref() == Some("mp4") { + OutboundMediaKind::Video + } else { + OutboundMediaKind::File + } +} + +fn rehydrate_pending_inbound_attachment_data( + attachment: &mut InboundAttachment, +) -> Result<(), String> { + if let Some(data) = load_pending_attachment_blob(&attachment.id)? { + channel_host::store_attachment_data(&attachment.id, &data) + .map_err(|e| format!("Failed to restore pending WeCom attachment data: {e}"))?; + if attachment.size_bytes.is_none() { + attachment.size_bytes = Some(data.len() as u64); + } + return Ok(()); + } + + Err("Pending WeCom websocket attachment blob is missing; cannot restore inline data".to_string()) +} + +fn extract_filename_from_content_disposition(header: &str) -> Option { + let lower = header.to_ascii_lowercase(); + let idx = lower.find("filename=")?; + let raw = header[idx + "filename=".len()..].trim(); + Some( + raw.trim_matches('"') + .trim_matches('\'') + .split(';') + .next() + .unwrap_or(raw) + .to_string(), + ) +} + +fn websocket_default_mime_and_extension(msg_type: &str) -> (&'static str, &'static str) { + match msg_type { + "image" => ("image/jpeg", "jpg"), + "video" => ("video/mp4", "mp4"), + _ => ("application/octet-stream", "bin"), + } +} + +#[cfg_attr(test, allow(dead_code))] +fn header_value_case_insensitive<'a>( + headers: &'a JsonMap, + name: &str, +) -> Option<&'a str> { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .and_then(|(_, value)| value.as_str()) +} + +fn decode_base64_with_padding(value: &str) -> Result, String> { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Err("Base64 value is empty".to_string()); + } + + let mut padded = trimmed.to_string(); + let missing_padding = padded.len() % 4; + if missing_padding != 0 { + padded.push_str(&"=".repeat(4 - missing_padding)); + } + + BASE64_STANDARD + .decode(padded) + .map_err(|e| format!("Failed to decode base64 value: {e}")) +} + +fn remove_wecom_pkcs7_padding(data: &[u8]) -> Result<&[u8], String> { + let Some(last) = data.last().copied() else { + return Err("Decrypted payload is empty".to_string()); + }; + let pad_len = last as usize; + if pad_len == 0 || pad_len > 32 || pad_len > data.len() { + return Err(format!("Invalid WeCom PKCS#7 padding length: {pad_len}")); + } + if !data[data.len() - pad_len..] + .iter() + .all(|byte| *byte as usize == pad_len) + { + return Err("Invalid WeCom PKCS#7 padding bytes".to_string()); + } + Ok(&data[..data.len() - pad_len]) +} + +fn decrypt_websocket_media_payload(ciphertext: &[u8], aes_key: &str) -> Result, String> { + if ciphertext.is_empty() { + return Err("Encrypted websocket media payload is empty".to_string()); + } + if !ciphertext.len().is_multiple_of(16) { + return Err(format!( + "Encrypted websocket media length {} is not a multiple of AES block size", + ciphertext.len() + )); + } + + let key = decode_base64_with_padding(aes_key)?; + if key.len() != 32 { + return Err(format!( + "Unexpected websocket media AES key length: {}", + key.len() + )); + } + + let iv = &key[..16]; + let mut buf = ciphertext.to_vec(); + let decrypted = Aes256CbcDec::new_from_slices(&key, iv) + .map_err(|e| format!("Failed to initialize websocket media decryptor: {e}"))? + .decrypt_padded_mut::(&mut buf) + .map_err(|e| format!("Failed to decrypt websocket media payload: {e}"))?; + let unpadded = remove_wecom_pkcs7_padding(decrypted)?; + Ok(unpadded.to_vec()) +} + +#[cfg_attr(test, allow(dead_code))] +fn hydrate_websocket_binary_attachment_data( + attachment: &mut InboundAttachment, + msg_type: &str, + aes_key: Option<&str>, +) -> Result<(), String> { + let source_url = attachment + .source_url + .as_deref() + .ok_or_else(|| "Websocket attachment source_url is missing".to_string())?; + + let response = channel_host::http_request("GET", source_url, "{}", None, Some(30_000)) + .map_err(|e| format!("Failed to download websocket attachment: {e}"))?; + if response.status != 200 { + return Err(format!( + "Websocket attachment download returned {}: {}", + response.status, + String::from_utf8_lossy(&response.body) + )); + } + if response.body.len() > MAX_ATTACHMENT_BYTES { + return Err(format!( + "Websocket attachment exceeds {} bytes", + MAX_ATTACHMENT_BYTES + )); + } + + let headers: JsonMap = + serde_json::from_str(&response.headers_json).unwrap_or_default(); + let body = response.body; + let data = match aes_key { + Some(raw_key) if raw_key.trim().is_empty() => { + return Err("Websocket attachment aeskey is empty".to_string()); + } + Some(raw_key) => { + decrypt_websocket_media_payload(&body, raw_key.trim()).map_err(|decrypt_error| { + format!("Failed to decrypt websocket attachment payload: {decrypt_error}") + })? + } + None => body, + }; + if data.len() > MAX_ATTACHMENT_BYTES { + return Err(format!( + "Decrypted websocket attachment exceeds {} bytes", + MAX_ATTACHMENT_BYTES + )); + } + + channel_host::store_attachment_data(&attachment.id, &data) + .map_err(|e| format!("Failed to store websocket attachment data: {e}"))?; + persist_pending_attachment_blob(&attachment.id, &data)?; + + let (fallback_mime, _) = websocket_default_mime_and_extension(msg_type); + let mut mime_type = header_value_case_insensitive(&headers, "content-type") + .map(base_mime_type) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| fallback_mime.to_string()); + let mime_lower = mime_type.to_ascii_lowercase(); + if msg_type == "image" && !mime_lower.starts_with("image/") { + mime_type = "image/jpeg".to_string(); + } else if msg_type == "video" && !mime_lower.starts_with("video/") { + mime_type = "video/mp4".to_string(); + } + + if let Some(filename) = header_value_case_insensitive(&headers, "content-disposition") + .and_then(extract_filename_from_content_disposition) + .filter(|value| !value.trim().is_empty()) + { + attachment.filename = Some(filename); + } + attachment.mime_type = mime_type; + attachment.size_bytes = Some(data.len() as u64); + + Ok(()) +} + +fn chunk_text(text: &str, limit_bytes: usize) -> Vec { + let trimmed = text.trim(); + if trimmed.is_empty() { + return Vec::new(); + } + + let mut chunks = Vec::new(); + let mut current = String::new(); + for ch in trimmed.chars() { + let ch_len = ch.len_utf8(); + if !current.is_empty() && current.len() + ch_len > limit_bytes { + chunks.push(current); + current = String::new(); + } + current.push(ch); + } + if !current.is_empty() { + chunks.push(current); + } + chunks +} + +fn websocket_stream_id(req_id: &str) -> String { + format!("stream-{req_id}") +} + +fn build_websocket_stream_reply_payload( + req_id: &str, + reply_cmd: &str, + content: &str, + finish: bool, +) -> Result { + let payload = serde_json::json!({ + "cmd": reply_cmd, + "headers": { + "req_id": req_id, + }, + "body": { + "msgtype": "stream", + "stream": { + "id": websocket_stream_id(req_id), + "content": content, + "finish": finish, + }, + } + }); + serde_json::to_string(&payload) + .map_err(|e| format!("Failed to serialize WeCom websocket reply: {e}")) +} + +fn build_websocket_text_stream_reply_payloads( + req_id: &str, + reply_cmd: &str, + content: &str, + finish_last_chunk: bool, +) -> Result, String> { + let mut payloads = Vec::new(); + let chunks = chunk_text(content, STREAM_CHUNK_LIMIT_BYTES); + for (index, chunk) in chunks.iter().enumerate() { + let finish = finish_last_chunk && index + 1 == chunks.len(); + let payload = build_websocket_stream_reply_payload(req_id, reply_cmd, chunk, finish)?; + payloads.push(payload); + } + Ok(payloads) +} + +fn websocket_media_md5_hex(data: &[u8]) -> String { + let mut hasher = Md5::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +fn send_websocket_response(req_id: &str, reply_cmd: &str, content: &str) -> Result<(), String> { + for payload in build_websocket_text_stream_reply_payloads(req_id, reply_cmd, content, true)? { + send_websocket_text(&payload) + .map_err(|e| format!("Failed to send WeCom websocket reply: {e}"))?; + } + Ok(()) +} + +fn send_websocket_stream_reply(req_id: &str, reply_cmd: &str, content: &str) -> Result<(), String> { + send_websocket_response(req_id, reply_cmd, content) +} + +fn truncate_status_message(message: &str, max_chars: usize) -> String { + let mut chars = message.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + if chars.clone().count() < max_chars { + return message.to_string(); + } + + let keep = max_chars.saturating_sub(3).max(1); + let mut out = String::new(); + out.push(first); + out.extend(chars.take(keep.saturating_sub(1))); + out.push_str("..."); + out +} + +fn status_message_for_user(message: &str) -> Option { + let message = message.trim(); + if message.is_empty() { + None + } else { + Some(truncate_status_message(message, STATUS_MESSAGE_MAX_CHARS)) + } +} + +fn classify_status_update(update: &StatusUpdate) -> Option { + match update.status { + StatusType::Thinking => None, + StatusType::Done => None, + StatusType::Interrupted => status_message_for_user(&update.message) + .or_else(|| Some("Request interrupted. Please try again.".to_string())), + // Tool-level telemetry is too noisy in chat UX. + StatusType::ToolStarted | StatusType::ToolCompleted | StatusType::ToolResult => None, + StatusType::Status => { + let message = update.message.trim(); + if message.eq_ignore_ascii_case("Done") + || message.eq_ignore_ascii_case("Interrupted") + || message.eq_ignore_ascii_case("Awaiting approval") + || message.eq_ignore_ascii_case("Rejected") + { + None + } else { + status_message_for_user(message) + } + } + StatusType::ApprovalNeeded + | StatusType::JobStarted + | StatusType::AuthRequired + | StatusType::AuthCompleted => status_message_for_user(&update.message), + } +} + +fn send_status_notification(metadata: &WecomMessageMetadata, content: &str) -> Result<(), String> { + if let Some(req_id) = metadata.ws_req_id.as_deref() { + let reply_cmd = metadata + .ws_reply_cmd + .as_deref() + .unwrap_or(WECOM_WS_REPLY_CMD); + return send_websocket_stream_reply(req_id, reply_cmd, content); + } + + Err("WeCom Bot status update missing websocket route metadata".to_string()) +} + +fn status_update_is_sensitive(update: &StatusUpdate) -> bool { + matches!( + &update.status, + StatusType::ApprovalNeeded | StatusType::AuthRequired | StatusType::AuthCompleted + ) +} + +fn metadata_is_group_chat(metadata: &WecomMessageMetadata) -> bool { + chat_type_is_group( + metadata + .ws_chat_type + .as_deref() + .or(metadata.chat_type.as_deref()), + ) +} + +fn safe_group_status_content(update: &StatusUpdate, content: String) -> String { + if status_update_is_sensitive(update) { + "This action needs a private authorization step. Please DM the bot to continue." + .to_string() + } else { + content + } +} + +fn build_websocket_command_payload( + cmd: &str, + req_id: &str, + body: JsonValue, +) -> Result { + serde_json::to_string(&serde_json::json!({ + "cmd": cmd, + "headers": { + "req_id": req_id, + }, + "body": body, + })) + .map_err(|e| format!("Failed to serialize WeCom websocket command: {e}")) +} + +fn websocket_req_id_now_millis() -> u64 { + #[cfg(test)] + { + 1 + } + #[cfg(not(test))] + { + channel_host::now_millis() + } +} + +fn websocket_control_req_id(cmd: &str, seed: &str) -> String { + let now = websocket_req_id_now_millis(); + let mut hasher = Sha1::new(); + hasher.update(cmd.as_bytes()); + hasher.update(seed.as_bytes()); + hasher.update(now.to_string().as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("{cmd}_{now}_{}", &digest[..8]) +} + +fn websocket_media_batch_id(metadata: &WecomMessageMetadata) -> String { + let seed = metadata + .source_msg_id + .as_deref() + .or(metadata.ws_req_id.as_deref()) + .unwrap_or("response"); + websocket_control_req_id("ironclaw_wecom_media_batch", seed) +} + +fn websocket_media_send_id(batch_id: &str, index: usize, attachment: &Attachment) -> String { + let seed = format!( + "{batch_id}:{index}:{}:{}:{}", + attachment.filename, + attachment.mime_type, + attachment.data.len() + ); + websocket_control_req_id("ironclaw_wecom_media", &seed) +} + +fn websocket_media_chunk_blob_path(send_id: &str, chunk_index: usize) -> String { + let mut hasher = Sha1::new(); + hasher.update(send_id.as_bytes()); + hasher.update(b":"); + hasher.update(chunk_index.to_string().as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("{WEBSOCKET_MEDIA_CHUNK_BLOBS_PREFIX}/{digest}.b64") +} + +fn persist_websocket_media_chunks(send_id: &str, data: &[u8]) -> Result, String> { + let mut paths = Vec::new(); + for (index, chunk) in data.chunks(WEBSOCKET_MEDIA_CHUNK_SIZE).enumerate() { + let path = websocket_media_chunk_blob_path(send_id, index); + if let Err(error) = write_wecom_workspace(&path, &BASE64_STANDARD.encode(chunk)) { + cleanup_websocket_media_chunk_paths(&paths); + return Err(format!( + "Failed to persist WeCom websocket media chunk: {error}" + )); + } + paths.push(path); + } + Ok(paths) +} + +fn cleanup_websocket_media_chunk_paths(paths: &[String]) { + for path in paths { + if let Err(error) = write_wecom_workspace(path, "") { + log_wecom( + channel_host::LogLevel::Warn, + &format!("Failed to cleanup WeCom websocket media chunk blob: {error}"), + ); + } + } +} + +fn cleanup_websocket_media_chunks(send: &PendingWebsocketMediaSend) { + cleanup_websocket_media_chunk_paths(&send.chunk_blob_paths); +} + +fn read_websocket_media_chunk_base64( + send: &PendingWebsocketMediaSend, + chunk_index: usize, +) -> Result { + let path = send + .chunk_blob_paths + .get(chunk_index) + .ok_or_else(|| format!("WeCom websocket media chunk {chunk_index} path is missing"))?; + let raw = read_wecom_workspace(path) + .ok_or_else(|| format!("WeCom websocket media chunk {chunk_index} blob is missing"))?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(format!("WeCom websocket media chunk {chunk_index} blob is empty")); + } + Ok(trimmed.to_string()) +} + +fn prune_stale_pending_websocket_media_state( + state: &mut PendingWebsocketMediaState, + now_ms: u64, +) -> bool { + let mut removed = false; + let mut retained_sends = Vec::with_capacity(state.sends.len()); + for send in state.sends.drain(..) { + let age_ms = now_ms.saturating_sub(send.created_at_ms); + if send.created_at_ms == 0 || age_ms > WEBSOCKET_MEDIA_SEND_TTL_MS { + cleanup_websocket_media_chunks(&send); + removed = true; + } else { + retained_sends.push(send); + } + } + state.sends = retained_sends; + + let active_batch_ids: std::collections::HashSet<&str> = + state.sends.iter().map(|send| send.batch_id.as_str()).collect(); + let before_batches = state.batches.len(); + state.batches.retain(|batch| { + let age_ms = now_ms.saturating_sub(batch.created_at_ms); + active_batch_ids.contains(batch.id.as_str()) + || (batch.created_at_ms > 0 && age_ms <= WEBSOCKET_MEDIA_SEND_TTL_MS) + }); + removed || before_batches != state.batches.len() +} + +fn load_pending_websocket_media_state() -> PendingWebsocketMediaState { + let Some(raw) = read_wecom_workspace(WEBSOCKET_MEDIA_STATE_PATH) else { + return PendingWebsocketMediaState::default(); + }; + if raw.trim().is_empty() { + return PendingWebsocketMediaState::default(); + } + match serde_json::from_str(&raw) { + Ok(mut state) => { + if prune_stale_pending_websocket_media_state(&mut state, websocket_req_id_now_millis()) + { + if let Err(error) = persist_pending_websocket_media_state(&state) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist pruned WeCom websocket media state: {error}"), + ); + } + } + state + } + Err(error) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to parse pending WeCom websocket media state: {error}"), + ); + PendingWebsocketMediaState::default() + } + } +} + +fn persist_pending_websocket_media_state(state: &PendingWebsocketMediaState) -> Result<(), String> { + let json = serde_json::to_string(state) + .map_err(|e| format!("Failed to serialize pending WeCom websocket media state: {e}"))?; + write_wecom_workspace(WEBSOCKET_MEDIA_STATE_PATH, &json) +} + +fn websocket_media_target(metadata: &WecomMessageMetadata) -> Option { + metadata + .ws_chat_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .or_else(|| { + let to_user = metadata.to_user.trim(); + (!to_user.is_empty()).then_some(to_user) + }) + .map(str::to_string) +} + +fn classify_websocket_media(att: &Attachment) -> OutboundMediaKind { + let preferred = preferred_outbound_media_kind(att); + if att.data.len() > preferred.websocket_max_bytes() { + OutboundMediaKind::File + } else { + preferred + } +} + +fn validate_websocket_media_size( + media_kind: OutboundMediaKind, + size_bytes: usize, +) -> Result<(), String> { + if size_bytes > media_kind.websocket_max_bytes() { + return Err(format!( + "WeCom websocket {} attachment exceeds {} bytes", + media_kind.as_api_type(), + media_kind.websocket_max_bytes() + )); + } + let total_chunks = size_bytes.div_ceil(WEBSOCKET_MEDIA_CHUNK_SIZE).max(1); + if total_chunks > MAX_WEBSOCKET_MEDIA_CHUNKS { + return Err(format!( + "WeCom websocket attachment requires {total_chunks} chunks; maximum is {MAX_WEBSOCKET_MEDIA_CHUNKS}" + )); + } + Ok(()) +} + +fn build_websocket_media_init_payload(send: &PendingWebsocketMediaSend) -> Result { + if send.md5_hex.trim().is_empty() { + return Err("WeCom websocket media md5 is missing".to_string()); + } + build_websocket_command_payload( + WECOM_WS_UPLOAD_MEDIA_INIT_CMD, + &send.init_req_id, + serde_json::json!({ + "type": send.media_type, + "filename": send.filename, + "total_size": send.total_size, + "total_chunks": send.total_chunks, + "md5": send.md5_hex, + }), + ) +} + +fn build_pending_websocket_media_send( + batch_id: &str, + chat_id: &str, + attachment: &Attachment, + index: usize, +) -> Result<(PendingWebsocketMediaSend, String), String> { + if attachment.data.is_empty() { + return Err(format!( + "WeCom websocket attachment '{}' has no data", + attachment.filename + )); + } + + let media_kind = classify_websocket_media(attachment); + validate_websocket_media_size(media_kind, attachment.data.len())?; + let total_chunks = attachment + .data + .len() + .div_ceil(WEBSOCKET_MEDIA_CHUNK_SIZE) + .max(1); + let id = websocket_media_send_id(batch_id, index, attachment); + let chunk_blob_paths = persist_websocket_media_chunks(&id, &attachment.data)?; + let init_req_id = websocket_control_req_id(WECOM_WS_UPLOAD_MEDIA_INIT_CMD, &id); + let send = PendingWebsocketMediaSend { + id, + batch_id: batch_id.to_string(), + chat_id: chat_id.to_string(), + created_at_ms: websocket_req_id_now_millis(), + media_type: media_kind.as_api_type().to_string(), + filename: if attachment.filename.trim().is_empty() { + "attachment.bin".to_string() + } else { + attachment.filename.clone() + }, + md5_hex: websocket_media_md5_hex(&attachment.data), + chunk_blob_paths, + total_size: attachment.data.len(), + total_chunks, + next_chunk_index: 0, + init_req_id, + chunk_req_id: None, + finish_req_id: None, + send_req_id: None, + upload_id: None, + media_id: None, + }; + let payload = build_websocket_media_init_payload(&send)?; + Ok((send, payload)) +} + +fn build_websocket_media_chunk_payload(send: &PendingWebsocketMediaSend) -> Result { + let upload_id = send + .upload_id + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "WeCom websocket media upload_id missing before chunk".to_string())?; + let chunk_base64 = read_websocket_media_chunk_base64(send, send.next_chunk_index)?; + let req_id = send + .chunk_req_id + .as_deref() + .ok_or_else(|| "WeCom websocket media chunk req_id missing".to_string())?; + build_websocket_command_payload( + WECOM_WS_UPLOAD_MEDIA_CHUNK_CMD, + req_id, + serde_json::json!({ + "upload_id": upload_id, + "chunk_index": send.next_chunk_index, + "base64_data": chunk_base64, + }), + ) +} + +fn build_websocket_media_finish_payload( + send: &PendingWebsocketMediaSend, +) -> Result { + let upload_id = send + .upload_id + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "WeCom websocket media upload_id missing before finish".to_string())?; + let req_id = send + .finish_req_id + .as_deref() + .ok_or_else(|| "WeCom websocket media finish req_id missing".to_string())?; + build_websocket_command_payload( + WECOM_WS_UPLOAD_MEDIA_FINISH_CMD, + req_id, + serde_json::json!({ "upload_id": upload_id }), + ) +} + +fn build_websocket_active_media_payload( + send: &PendingWebsocketMediaSend, +) -> Result { + let media_id = send + .media_id + .as_deref() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "WeCom websocket media_id missing before send".to_string())?; + let req_id = send + .send_req_id + .as_deref() + .ok_or_else(|| "WeCom websocket media send req_id missing".to_string())?; + let media_type = send.media_type.as_str(); + build_websocket_command_payload( + WECOM_WS_SEND_MSG_CMD, + req_id, + serde_json::json!({ + "chatid": send.chat_id, + "msgtype": media_type, + media_type: { "media_id": media_id }, + }), + ) +} + +fn build_websocket_active_markdown_payload(chat_id: &str, content: &str) -> Result { + let req_id = websocket_control_req_id(WECOM_WS_SEND_MSG_CMD, content); + build_websocket_command_payload( + WECOM_WS_SEND_MSG_CMD, + &req_id, + serde_json::json!({ + "chatid": chat_id, + "msgtype": "markdown", + "markdown": { + "content": content, + }, + }), + ) +} + +fn prepare_next_websocket_media_chunk(send: &mut PendingWebsocketMediaSend) -> Result { + let req_id = websocket_control_req_id( + WECOM_WS_UPLOAD_MEDIA_CHUNK_CMD, + &format!("{}:{}", send.id, send.next_chunk_index), + ); + send.chunk_req_id = Some(req_id); + build_websocket_media_chunk_payload(send) +} + +fn prepare_websocket_media_finish(send: &mut PendingWebsocketMediaSend) -> Result { + let req_id = websocket_control_req_id(WECOM_WS_UPLOAD_MEDIA_FINISH_CMD, &send.id); + send.finish_req_id = Some(req_id); + build_websocket_media_finish_payload(send) +} + +fn prepare_websocket_active_media(send: &mut PendingWebsocketMediaSend) -> Result { + let req_id = websocket_control_req_id(WECOM_WS_SEND_MSG_CMD, &send.id); + send.send_req_id = Some(req_id); + build_websocket_active_media_payload(send) +} + +fn send_websocket_active_markdown(chat_id: &str, content: &str) -> Result<(), String> { + let content = content.trim(); + if content.is_empty() { + return Ok(()); + } + let payload = build_websocket_active_markdown_payload(chat_id, content)?; + send_websocket_text(&payload) + .map_err(|e| format!("Failed to send WeCom websocket markdown message: {e}")) +} + +fn start_websocket_media_batch( + metadata: &WecomMessageMetadata, + response_req_id: &str, + response_cmd: &str, + content: &str, + attachments: &[Attachment], +) -> WebsocketMediaStartResult { + let Some(chat_id) = websocket_media_target(metadata) else { + return WebsocketMediaStartResult { + started: 0, + errors: vec!["WeCom websocket media send requires chat_id or user_id".to_string()], + }; + }; + + let batch_id = websocket_media_batch_id(metadata); + let mut sends = Vec::new(); + let mut outbound_payloads = Vec::new(); + let mut errors = Vec::new(); + let mut selected_media_count = 0usize; + let mut selected_media_bytes = 0usize; + for (index, attachment) in attachments.iter().enumerate() { + if selected_media_count >= MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE { + push_websocket_media_error( + &mut errors, + format!( + "WeCom websocket media response exceeds {MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE} attachment(s); remaining attachments were skipped" + ), + ); + continue; + } + if selected_media_bytes.saturating_add(attachment.data.len()) + > MAX_WEBSOCKET_MEDIA_TOTAL_BYTES_PER_RESPONSE + { + push_websocket_media_error( + &mut errors, + format!( + "WeCom websocket media response exceeds {MAX_WEBSOCKET_MEDIA_TOTAL_BYTES_PER_RESPONSE} total bytes; attachment '{}' was skipped", + attachment.filename + ), + ); + continue; + } + match build_pending_websocket_media_send(&batch_id, &chat_id, attachment, index) { + Ok((send, payload)) => { + selected_media_count += 1; + selected_media_bytes += attachment.data.len(); + outbound_payloads.push(PendingWebsocketMediaOutbound { + send_id: send.id.clone(), + payload, + }); + sends.push(send); + } + Err(error) => push_websocket_media_error(&mut errors, error), + } + } + + let started = sends.len(); + if started > 0 { + let mut state = load_pending_websocket_media_state(); + state.batches.push(PendingWebsocketMediaBatch { + id: batch_id, + chat_id, + created_at_ms: websocket_req_id_now_millis(), + response_req_id: response_req_id.to_string(), + response_cmd: response_cmd.to_string(), + final_text: content.trim().to_string(), + remaining_media: started, + sent_media: 0, + failed_media: errors.len(), + errors: capped_websocket_media_errors(errors.clone()), + }); + state.sends.extend(sends.iter().cloned()); + if let Err(error) = persist_pending_websocket_media_state(&state) { + for send in &sends { + cleanup_websocket_media_chunks(send); + } + push_websocket_media_error( + &mut errors, + format!("Failed to persist WeCom websocket media state: {error}"), + ); + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeCom websocket media state: {error}"), + ); + return WebsocketMediaStartResult { started: 0, errors }; + } + + for outbound in outbound_payloads { + if let Err(error) = send_websocket_text(&outbound.payload) { + let error = format!("Failed to send WeCom websocket media init: {error}"); + fail_persisted_websocket_media_send(&outbound.send_id, error.clone()); + errors.push(error); + } + } + } + + WebsocketMediaStartResult { started, errors } +} + +fn append_websocket_media_errors_to_text(content: &str, errors: &[String]) -> String { + let first_error = errors + .first() + .map(String::as_str) + .unwrap_or("unknown error"); + let trimmed = content.trim(); + if trimmed.is_empty() { + format!("附件已生成,但发送失败:{first_error}") + } else { + format!("{trimmed}\n\n附件已生成,但发送失败:{first_error}") + } +} + +fn pending_websocket_media_matches_req(send: &PendingWebsocketMediaSend, req_id: &str) -> bool { + send.init_req_id == req_id + || send.chunk_req_id.as_deref() == Some(req_id) + || send.finish_req_id.as_deref() == Some(req_id) + || send.send_req_id.as_deref() == Some(req_id) +} + +fn json_body_string(body: &JsonValue, key: &str) -> Option { + body.get(key) + .and_then(JsonValue::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +fn fail_pending_websocket_media( + state: &mut PendingWebsocketMediaState, + send: &PendingWebsocketMediaSend, + error: String, +) { + log_wecom( + channel_host::LogLevel::Warn, + &format!("WeCom websocket media send failed: {error}"), + ); + cleanup_websocket_media_chunks(send); + complete_websocket_media_batch(state, &send.batch_id, Err(error)); +} + +fn fail_persisted_websocket_media_send(send_id: &str, error: String) { + let mut state = load_pending_websocket_media_state(); + let Some(pos) = state.sends.iter().position(|send| send.id == send_id) else { + log_wecom( + channel_host::LogLevel::Warn, + &format!("WeCom websocket media send failed after state was already cleared: {error}"), + ); + return; + }; + + let send = state.sends.remove(pos); + fail_pending_websocket_media(&mut state, &send, error); + if let Err(error) = persist_pending_websocket_media_state(&state) { + log_wecom( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeCom websocket media failure state: {error}"), + ); + } +} + +fn capped_websocket_media_errors(errors: Vec) -> Vec { + if errors.len() <= MAX_WEBSOCKET_MEDIA_BATCH_ERRORS { + return errors; + } + let omitted = errors.len() - MAX_WEBSOCKET_MEDIA_BATCH_ERRORS; + let mut capped: Vec = errors + .into_iter() + .take(MAX_WEBSOCKET_MEDIA_BATCH_ERRORS) + .collect(); + capped.push(format!( + "{omitted} additional WeCom websocket media error(s) omitted" + )); + capped +} + +fn push_websocket_media_error(errors: &mut Vec, error: String) { + if errors.len() < MAX_WEBSOCKET_MEDIA_BATCH_ERRORS { + errors.push(error); + } else if errors.len() == MAX_WEBSOCKET_MEDIA_BATCH_ERRORS { + errors.push(format!( + "Additional WeCom websocket media errors omitted after first {MAX_WEBSOCKET_MEDIA_BATCH_ERRORS}" + )); + } +} + +fn complete_websocket_media_batch( + state: &mut PendingWebsocketMediaState, + batch_id: &str, + result: Result<(), String>, +) { + let Some(pos) = state.batches.iter().position(|batch| batch.id == batch_id) else { + return; + }; + + let batch = &mut state.batches[pos]; + if batch.remaining_media > 0 { + batch.remaining_media -= 1; + } + match result { + Ok(()) => batch.sent_media += 1, + Err(error) => { + batch.failed_media += 1; + push_websocket_media_error(&mut batch.errors, error); + } + } + + if batch.remaining_media > 0 { + return; + } + + let batch = state.batches.remove(pos); + let final_text = if batch.failed_media > 0 { + append_websocket_media_errors_to_text(&batch.final_text, &batch.errors) + } else { + batch.final_text + }; + + let final_text = final_text.trim(); + if !final_text.is_empty() { + let send_result = if batch.response_req_id.trim().is_empty() { + send_websocket_active_markdown(&batch.chat_id, final_text) + } else { + let response_cmd = if batch.response_cmd.trim().is_empty() { + WECOM_WS_REPLY_CMD + } else { + batch.response_cmd.as_str() + }; + send_websocket_stream_reply(&batch.response_req_id, response_cmd, final_text) + }; + + if let Err(error) = send_result { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to send WeCom websocket final text after media: {error}"), + ); + } + } +} + +fn advance_pending_websocket_media( + mut send: PendingWebsocketMediaSend, + ack: &WecomWsAckFrame, +) -> Result { + let req_id = ack.headers.req_id.as_str(); + + if send.init_req_id == req_id { + let upload_id = json_body_string(&ack.body, "upload_id") + .ok_or_else(|| "WeCom websocket upload init ack missing upload_id".to_string())?; + send.upload_id = Some(upload_id); + send.next_chunk_index = 0; + let payload = prepare_next_websocket_media_chunk(&mut send)?; + let outbound = PendingWebsocketMediaOutbound { + send_id: send.id.clone(), + payload, + }; + return Ok(PendingWebsocketMediaAdvance::Send(outbound, send)); + } + + if send.chunk_req_id.as_deref() == Some(req_id) { + send.next_chunk_index += 1; + let payload = if send.next_chunk_index < send.total_chunks { + prepare_next_websocket_media_chunk(&mut send)? + } else { + prepare_websocket_media_finish(&mut send)? + }; + let outbound = PendingWebsocketMediaOutbound { + send_id: send.id.clone(), + payload, + }; + return Ok(PendingWebsocketMediaAdvance::Send(outbound, send)); + } + + if send.finish_req_id.as_deref() == Some(req_id) { + let media_id = json_body_string(&ack.body, "media_id") + .ok_or_else(|| "WeCom websocket upload finish ack missing media_id".to_string())?; + send.media_id = Some(media_id); + let payload = prepare_websocket_active_media(&mut send)?; + let outbound = PendingWebsocketMediaOutbound { + send_id: send.id.clone(), + payload, + }; + return Ok(PendingWebsocketMediaAdvance::Send(outbound, send)); + } + + if send.send_req_id.as_deref() == Some(req_id) { + return Ok(PendingWebsocketMediaAdvance::Complete); + } + + Err("WeCom websocket ack matched send but no media phase advanced".to_string()) +} + +fn parse_websocket_ack_frame(frame: &str) -> Option { + let value: JsonValue = serde_json::from_str(frame).ok()?; + value.get("errcode")?.as_i64()?; + serde_json::from_value(value).ok() +} + +enum WebsocketAckApplyResult { + Unknown, + Applied { + outbound: Option, + }, +} + +fn apply_websocket_ack_to_media_state( + state: &mut PendingWebsocketMediaState, + ack: &WecomWsAckFrame, +) -> WebsocketAckApplyResult { + let Some(pos) = state + .sends + .iter() + .position(|send| pending_websocket_media_matches_req(send, &ack.headers.req_id)) + else { + return WebsocketAckApplyResult::Unknown; + }; + + let send = state.sends.remove(pos); + let batch_id = send.batch_id.clone(); + let mut outbound = None; + if ack.errcode != 0 { + let error = format!( + "req_id={} errcode={} errmsg={}", + ack.headers.req_id, ack.errcode, ack.errmsg + ); + fail_pending_websocket_media(state, &send, error); + } else { + let send_for_failure = send.clone(); + match advance_pending_websocket_media(send, ack) { + Ok(PendingWebsocketMediaAdvance::Send(next_outbound, next)) => { + outbound = Some(next_outbound); + state.sends.push(next); + } + Ok(PendingWebsocketMediaAdvance::Complete) => { + cleanup_websocket_media_chunks(&send_for_failure); + complete_websocket_media_batch(state, &batch_id, Ok(())); + } + Err(error) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to advance WeCom websocket media send: {error}"), + ); + cleanup_websocket_media_chunks(&send_for_failure); + complete_websocket_media_batch(state, &batch_id, Err(error)); + } + } + } + + WebsocketAckApplyResult::Applied { outbound } +} + +fn handle_websocket_ack_frame(ack: WecomWsAckFrame) { + let mut state = load_pending_websocket_media_state(); + let outbound = match apply_websocket_ack_to_media_state(&mut state, &ack) { + WebsocketAckApplyResult::Unknown => return, + WebsocketAckApplyResult::Applied { outbound } => outbound, + }; + + if let Err(error) = persist_pending_websocket_media_state(&state) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeCom websocket media state: {error}"), + ); + return; + } + + if let Some(outbound) = outbound { + if let Err(error) = send_websocket_text(&outbound.payload) { + fail_persisted_websocket_media_send( + &outbound.send_id, + format!("Failed to send WeCom websocket media command: {error}"), + ); + } + } +} + +fn websocket_fallback_target(sender_id: &str, chat_type: Option<&str>) -> String { + if chat_type == Some("group") { + String::new() + } else { + sender_id.to_string() + } +} + +fn normalize_chat_type(chat_type: Option<&str>) -> Option { + let kind = chat_type.map(str::trim).filter(|value| !value.is_empty())?; + Some(match kind { + "single" => "private".to_string(), + other => other.to_string(), + }) +} + +fn wecom_conversation_scope( + sender_id: &str, + chat_id: Option<&str>, + chat_type: Option<&str>, +) -> String { + let normalized_chat_type = normalize_chat_type(chat_type); + if normalized_chat_type.as_deref() == Some("group") { + if let Some(group_chat_id) = chat_id.map(str::trim).filter(|value| !value.is_empty()) { + return format!("wecom:group:{group_chat_id}"); + } + } + format!("wecom:dm:{sender_id}") +} + +fn websocket_metadata_json( + sender_id: &str, + msg_id: &str, + req_id: &str, + chat_id: Option<&str>, + chat_type: Option<&str>, + reply_cmd: &str, +) -> Result { + let normalized_chat_type = normalize_chat_type(chat_type); + let normalized_chat_id = chat_id + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string); + let to_user = websocket_fallback_target(sender_id, chat_type); + let target = normalized_chat_id + .clone() + .or_else(|| (!to_user.is_empty()).then_some(to_user.clone())); + + serde_json::to_string(&WecomMessageMetadata { + to_user, + target, + chat_id: normalized_chat_id, + chat_type: normalized_chat_type, + source_msg_id: Some(msg_id.to_string()), + ws_req_id: Some(req_id.to_string()), + ws_chat_id: chat_id.map(str::to_string), + ws_chat_type: chat_type.map(str::to_string), + ws_reply_cmd: Some(reply_cmd.to_string()), + }) + .map_err(|e| format!("Failed to serialize WeCom websocket metadata: {e}")) +} + +fn websocket_attachment_from_binary( + msg_id: &str, + msg_type: &str, + content: &WecomWsBinaryContent, +) -> InboundAttachment { + let (mime_type, extension) = websocket_default_mime_and_extension(msg_type); + let attachment = InboundAttachment { + id: format!("{msg_id}:{msg_type}"), + mime_type: mime_type.to_string(), + filename: Some(format!("{msg_id}.{extension}")), + size_bytes: None, + source_url: Some(content.url.clone()), + storage_key: None, + extracted_text: None, + extras_json: serde_json::json!({ "wecom_ws_msgtype": msg_type }).to_string(), + }; + + #[cfg(not(test))] + let attachment = { + let mut hydrated = attachment; + if let Err(error) = hydrate_websocket_binary_attachment_data( + &mut hydrated, + msg_type, + content.aeskey.as_deref(), + ) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!( + "Failed to hydrate WeCom websocket {} attachment '{}': {}", + msg_type, hydrated.id, error + ), + ); + } + hydrated + }; + + attachment +} + +fn websocket_quote_context(quote: Option<&WecomWsQuoteContent>) -> Option { + let quote = quote?; + let quoted_text = quote + .text + .as_ref() + .map(|text| text.content.trim()) + .filter(|text| !text.is_empty()) + .or_else(|| { + quote + .voice + .as_ref() + .map(|voice| voice.content.trim()) + .filter(|text| !text.is_empty()) + }) + .or_else(|| { + quote + .content + .as_deref() + .map(str::trim) + .filter(|text| !text.is_empty()) + })?; + + let quote_kind = quote + .msg_type + .as_deref() + .map(str::trim) + .filter(|kind| !kind.is_empty()) + .unwrap_or("message"); + Some(format!("Quoted {quote_kind}: {quoted_text}")) +} + +fn with_websocket_quote_context(content: String, quote: Option<&WecomWsQuoteContent>) -> String { + match websocket_quote_context(quote) { + Some(quoted) if content.trim().is_empty() => quoted, + Some(quoted) => format!("{quoted}\n\n{content}"), + None => content, + } +} + +fn websocket_event_summary(event: &WecomWsEvent) -> Option { + match event.eventtype.as_str() { + "enter_chat" => Some("User entered the WeCom bot chat.".to_string()), + "template_card_event" => { + let event_key = event + .extra + .get("event_key") + .or_else(|| event.extra.get("eventKey")) + .and_then(JsonValue::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()); + Some(match event_key { + Some(key) => format!("User clicked a WeCom template card action: {key}"), + None => "User clicked a WeCom template card action.".to_string(), + }) + } + "feedback_event" => { + let score = event + .extra + .get("score") + .or_else(|| event.extra.get("rating")) + .and_then(JsonValue::as_i64); + Some(match score { + Some(score) => format!("User submitted WeCom feedback with score {score}."), + None => "User submitted WeCom feedback.".to_string(), + }) + } + _ => None, + } +} + +fn infer_mixed_item_type(item: &WecomWsMixedItem) -> &str { + item.item_type + .as_deref() + .filter(|kind| !kind.trim().is_empty()) + .unwrap_or_else(|| { + if item.text.is_some() { + "text" + } else if item.image.is_some() { + "image" + } else if item.file.is_some() { + "file" + } else if item.video.is_some() { + "video" + } else { + "unknown" + } + }) +} + +fn websocket_mixed_content_parts( + msg_id: &str, + mixed: &WecomWsMixedContent, +) -> (String, Vec) { + let mut text_parts = Vec::new(); + let mut attachments = Vec::new(); + + for (index, item) in mixed.msg_item.iter().enumerate() { + match infer_mixed_item_type(item) { + "text" => { + if let Some(text) = item + .text + .as_ref() + .map(|text| text.content.trim()) + .filter(|text| !text.is_empty()) + { + text_parts.push(text.to_string()); + } + } + "image" => { + if let Some(image) = item.image.as_ref() { + attachments.push(websocket_attachment_from_binary( + &format!("{msg_id}:{index}"), + "image", + image, + )); + } + } + "file" => { + if let Some(file) = item.file.as_ref() { + attachments.push(websocket_attachment_from_binary( + &format!("{msg_id}:{index}"), + "file", + file, + )); + } + } + "video" => { + if let Some(video) = item.video.as_ref() { + attachments.push(websocket_attachment_from_binary( + &format!("{msg_id}:{index}"), + "video", + video, + )); + } + } + _ => {} + } + } + + (text_parts.join("\n"), attachments) +} + +fn handle_websocket_message_frame( + frame: WecomWsFrame, + access: &SenderAccessSnapshot, +) { + let body = frame.body; + if !should_process_message_id(&body.msgid) { + return; + } + + let sender_id = body.from.userid; + match is_sender_allowed( + &sender_id, + body.chattype.as_deref(), + access, + Some(PairingReplyRoute { + req_id: frame.headers.req_id.clone(), + reply_cmd: WECOM_WS_REPLY_CMD.to_string(), + chat_type: body.chattype.clone(), + }), + ) { + Ok(true) => {} + Ok(false) => return, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("WeCom websocket sender authorization failed: {error}"), + ); + return; + } + } + + let mut attachments = Vec::new(); + let content = match body.msgtype.as_str() { + "text" => body.text.map(|text| text.content).unwrap_or_default(), + "voice" => body.voice.map(|voice| voice.content).unwrap_or_default(), + "markdown" => body.text.map(|text| text.content).unwrap_or_default(), + "image" => { + if let Some(image) = body.image.as_ref() { + attachments.push(websocket_attachment_from_binary( + &body.msgid, + "image", + image, + )); + } + String::new() + } + "file" => { + if let Some(file) = body.file.as_ref() { + attachments.push(websocket_attachment_from_binary(&body.msgid, "file", file)); + } + String::new() + } + "video" => { + if let Some(video) = body.video.as_ref() { + attachments.push(websocket_attachment_from_binary( + &body.msgid, + "video", + video, + )); + } + String::new() + } + "mixed" => { + if let Some(mixed) = body.mixed.as_ref() { + let (mixed_text, mixed_attachments) = + websocket_mixed_content_parts(&body.msgid, mixed); + attachments.extend(mixed_attachments); + mixed_text + } else { + String::new() + } + } + other => { + channel_host::log( + channel_host::LogLevel::Info, + &format!("Ignoring unsupported WeCom websocket message type: {other}"), + ); + return; + } + }; + let content = with_websocket_quote_context(content, body.quote.as_ref()); + + let metadata_json = match websocket_metadata_json( + &sender_id, + &body.msgid, + &frame.headers.req_id, + body.chatid.as_deref(), + body.chattype.as_deref(), + WECOM_WS_REPLY_CMD, + ) { + Ok(json) => json, + Err(error) => { + channel_host::log(channel_host::LogLevel::Error, &error); + return; + } + }; + let conversation_scope = + wecom_conversation_scope(&sender_id, body.chatid.as_deref(), body.chattype.as_deref()); + + emit_or_buffer_incoming_user_message( + sender_id, + None, + content, + conversation_scope, + metadata_json, + attachments, + ); +} + +fn handle_websocket_event_frame( + frame: WecomWsFrame, + access: &SenderAccessSnapshot, +) { + let body = frame.body; + if !should_process_message_id(&body.msgid) { + return; + } + + let Some(content) = websocket_event_summary(&body.event) else { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Ignoring WeCom websocket event type: {}", + body.event.eventtype + ), + ); + return; + }; + + let reply_cmd = if body.event.eventtype == "enter_chat" { + WECOM_WS_WELCOME_CMD + } else { + WECOM_WS_REPLY_CMD + }; + + let sender_id = body.from.userid; + match is_sender_allowed( + &sender_id, + body.chattype.as_deref(), + access, + Some(PairingReplyRoute { + req_id: frame.headers.req_id.clone(), + reply_cmd: reply_cmd.to_string(), + chat_type: body.chattype.clone(), + }), + ) { + Ok(true) => {} + Ok(false) => return, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("WeCom websocket sender authorization failed: {error}"), + ); + return; + } + } + + let metadata_json = match websocket_metadata_json( + &sender_id, + &body.msgid, + &frame.headers.req_id, + body.chatid.as_deref(), + body.chattype.as_deref(), + reply_cmd, + ) { + Ok(json) => json, + Err(error) => { + channel_host::log(channel_host::LogLevel::Error, &error); + return; + } + }; + let conversation_scope = + wecom_conversation_scope(&sender_id, body.chatid.as_deref(), body.chattype.as_deref()); + + channel_host::emit_message(&EmittedMessage { + user_id: sender_id, + user_name: None, + content, + thread_id: Some(conversation_scope), + metadata_json, + attachments: Vec::new(), + }); +} + +fn process_websocket_event_queue() { + let queue_json = channel_host::workspace_read(WEBSOCKET_EVENT_QUEUE_PATH).unwrap_or_default(); + if queue_json.trim().is_empty() || queue_json.trim() == "[]" { + return; + } + + let frames: Vec = match serde_json::from_str(&queue_json) { + Ok(value) => value, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to deserialize WeCom websocket queue: {error}"), + ); + let _ = channel_host::workspace_write(WEBSOCKET_EVENT_QUEUE_PATH, "[]"); + return; + } + }; + + if let Err(error) = channel_host::workspace_write(WEBSOCKET_EVENT_QUEUE_PATH, "[]") { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to clear WeCom websocket queue: {error}"), + ); + } + + let access = SenderAccessSnapshot::load(); + + for frame in frames { + let cmd = serde_json::from_str::(&frame) + .ok() + .and_then(|value| { + value + .get("cmd") + .and_then(serde_json::Value::as_str) + .map(str::to_string) + }); + + match cmd.as_deref() { + Some("aibot_msg_callback") => { + match serde_json::from_str::>(&frame) { + Ok(parsed) => handle_websocket_message_frame(parsed, &access), + Err(error) => channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to parse WeCom websocket message frame: {error}"), + ), + } + } + Some("aibot_event_callback") => { + match serde_json::from_str::>(&frame) { + Ok(parsed) => handle_websocket_event_frame(parsed, &access), + Err(error) => channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to parse WeCom websocket event frame: {error}"), + ), + } + } + Some(other) => { + if let Some(ack) = parse_websocket_ack_frame(&frame) { + handle_websocket_ack_frame(ack); + } else { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Ignoring WeCom websocket control frame: {other}"), + ); + } + } + None => { + if let Some(ack) = parse_websocket_ack_frame(&frame) { + handle_websocket_ack_frame(ack); + } + } + } + } +} + +fn update_recent_message_ids( + existing_json: Option<&str>, + msg_id: &str, + max_ids: usize, + now_ms: u64, + ttl_ms: u64, +) -> Result<(bool, String), String> { + let mut ids: Vec = match existing_json.filter(|s| !s.trim().is_empty()) { + Some(raw) => serde_json::from_str::>(raw).or_else(|_| { + serde_json::from_str::>(raw).map(|legacy| { + legacy + .into_iter() + .map(|id| RecentMessageIdEntry { + id, + seen_at_ms: now_ms, + }) + .collect() + }) + }) + .map_err(|e| format!("Failed to parse recent WeCom message ids: {e}"))?, + None => Vec::new(), + }; + + ids.retain(|entry| now_ms.saturating_sub(entry.seen_at_ms) <= ttl_ms); + + if ids.iter().any(|existing| existing.id == msg_id) { + let json = serde_json::to_string(&ids) + .map_err(|e| format!("Failed to serialize recent WeCom message ids: {e}"))?; + return Ok((false, json)); + } + + ids.push(RecentMessageIdEntry { + id: msg_id.to_string(), + seen_at_ms: now_ms, + }); + if ids.len() > max_ids { + let to_drop = ids.len() - max_ids; + ids.drain(0..to_drop); + } + + let json = serde_json::to_string(&ids) + .map_err(|e| format!("Failed to serialize recent WeCom message ids: {e}"))?; + Ok((true, json)) +} + +fn should_process_message_id(msg_id: &str) -> bool { + match update_recent_message_ids( + channel_host::workspace_read(RECENT_MSG_IDS_PATH).as_deref(), + msg_id, + MAX_RECENT_MSG_IDS, + channel_host::now_millis(), + MAX_RECENT_MSG_ID_AGE_MS, + ) { + Ok((true, json)) => { + if let Err(error) = channel_host::workspace_write(RECENT_MSG_IDS_PATH, &json) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to persist WeCom dedupe state: {error}"), + ); + } + true + } + Ok((false, _)) => false, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to update WeCom dedupe state: {error}"), + ); + true + } + } +} + +struct WecomChannel; + +export!(WecomChannel); + +impl Guest for WecomChannel { + fn on_start(config_json: String) -> Result { + let config: WecomConfig = serde_json::from_str(&config_json) + .map_err(|e| format!("Failed to parse WeCom config: {e}"))?; + + let _ = + channel_host::workspace_write(OWNER_ID_PATH, config.owner_id.as_deref().unwrap_or("")); + let dm_policy = config.dm_policy.unwrap_or_default(); + let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy.as_str()); + let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default()) + .unwrap_or_else(|_| "[]".to_string()); + let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json); + let inbound_merge_window_ms = config + .inbound_merge_window_ms + .map(u64::from) + .unwrap_or(DEFAULT_INBOUND_MERGE_WINDOW_MS) + .min(MAX_INBOUND_MERGE_WINDOW_MS); + let _ = channel_host::workspace_write( + INBOUND_MERGE_WINDOW_MS_PATH, + &inbound_merge_window_ms.to_string(), + ); + + Ok(ChannelConfig { + display_name: "WeCom".to_string(), + http_endpoints: Vec::new(), + poll: Some(PollConfig { + interval_ms: WECOM_POLL_INTERVAL_MS, + enabled: true, + }), + }) + } + + fn on_http_request(_req: IncomingHttpRequest) -> OutgoingHttpResponse { + text_response(404, "WeCom Bot channel does not expose an HTTP callback endpoint") + } + + fn on_poll() { + process_websocket_event_queue(); + flush_due_pending_inbound_bundles(); + } + + fn on_respond(response: AgentResponse) -> Result<(), String> { + let metadata: WecomMessageMetadata = serde_json::from_str(&response.metadata_json) + .map_err(|e| format!("Failed to parse WeCom response metadata: {e}"))?; + + if let Some(req_id) = metadata.ws_req_id.as_deref() { + let reply_cmd = metadata + .ws_reply_cmd + .as_deref() + .unwrap_or(WECOM_WS_REPLY_CMD); + + if !response.attachments.is_empty() { + let media = start_websocket_media_batch( + &metadata, + req_id, + reply_cmd, + &response.content, + &response.attachments, + ); + if media.started > 0 { + return Ok(()); + } + + let content = + append_websocket_media_errors_to_text(&response.content, &media.errors); + send_websocket_stream_reply(req_id, reply_cmd, &content)?; + return Ok(()); + } + + if !response.content.trim().is_empty() { + send_websocket_stream_reply(req_id, reply_cmd, &response.content)?; + } + return Ok(()); + } + + Err("WeCom Bot outbound requires websocket route metadata".to_string()) + } + + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("WeCom Bot broadcast is not supported without an active websocket chat route".to_string()) + } + + fn on_status(update: StatusUpdate) { + let Some(content) = classify_status_update(&update) else { + return; + }; + let metadata: WecomMessageMetadata = match serde_json::from_str(&update.metadata_json) { + Ok(metadata) => metadata, + Err(error) => { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Failed to parse WeCom status metadata: {error}"), + ); + return; + } + }; + let content = if metadata_is_group_chat(&metadata) { + safe_group_status_content(&update, content) + } else { + content + }; + if let Err(error) = send_status_notification(&metadata, &content) { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Failed to send WeCom status notification: {error}"), + ); + } + } + + fn on_shutdown() { + channel_host::log(channel_host::LogLevel::Info, "WeCom channel shutting down"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aes::cipher::block_padding::NoPadding; + use aes::cipher::BlockEncryptMut; + use cbc::Encryptor; + + const WECOM_CAPABILITIES_JSON: &str = include_str!("../wecom.capabilities.json"); + + type Aes256CbcEnc = Encryptor; + + fn encrypt_websocket_media_for_test(key_bytes: &[u8; 32], plaintext: &[u8]) -> Vec { + let block_size = 32usize; + let pad_len = block_size - (plaintext.len() % block_size); + let mut padded = plaintext.to_vec(); + padded.extend(std::iter::repeat_n(pad_len as u8, pad_len)); + + let iv = &key_bytes[..16]; + let msg_len = padded.len(); + Aes256CbcEnc::new_from_slices(key_bytes, iv) + .expect("encryptor") + .encrypt_padded_mut::(&mut padded, msg_len) + .expect("encrypt") + .to_vec() + } + + fn websocket_ack_for_test(req_id: &str, body: JsonValue) -> WecomWsAckFrame { + WecomWsAckFrame { + headers: WecomWsHeaders { + req_id: req_id.to_string(), + }, + errcode: 0, + errmsg: String::new(), + body, + } + } + + #[test] + fn capabilities_are_bot_only() { + let caps: serde_json::Value = + serde_json::from_str(WECOM_CAPABILITIES_JSON).expect("capabilities parse"); + let required = caps["setup"]["required_secrets"] + .as_array() + .expect("required secrets array"); + assert!(required + .iter() + .any(|entry| entry["name"] == "wecom_bot_id" && entry["optional"] == false)); + assert!(required + .iter() + .any(|entry| entry["name"] == "wecom_bot_secret" && entry["optional"] == false)); + assert!(required + .iter() + .all(|entry| !entry["name"].as_str().unwrap_or("").contains("corp"))); + assert!(caps["setup"]["validation_endpoint"].is_null()); + assert!(caps["capabilities"]["channel"]["webhook"].is_null()); + } + + #[test] + fn websocket_stream_reply_payloads_chunk_content_and_reuse_req_id() { + let content = "a".repeat(STREAM_CHUNK_LIMIT_BYTES + 17); + let payloads = build_websocket_text_stream_reply_payloads( + "req-123", + WECOM_WS_REPLY_CMD, + &content, + true, + ) + .expect("payloads"); + + assert_eq!(payloads.len(), 2); + let first: serde_json::Value = serde_json::from_str(&payloads[0]).expect("first payload"); + let second: serde_json::Value = serde_json::from_str(&payloads[1]).expect("second payload"); + + assert_eq!(first["cmd"], serde_json::json!(WECOM_WS_REPLY_CMD)); + assert_eq!(first["headers"]["req_id"], serde_json::json!("req-123")); + assert_eq!( + first["body"]["stream"]["id"], + serde_json::json!(websocket_stream_id("req-123")) + ); + assert_eq!( + first["body"]["stream"]["content"] + .as_str() + .expect("first chunk") + .len(), + STREAM_CHUNK_LIMIT_BYTES + ); + assert_eq!(first["body"]["stream"]["finish"], serde_json::json!(false)); + assert_eq!( + second["body"]["stream"]["content"] + .as_str() + .expect("second chunk") + .len(), + 17 + ); + assert_eq!(second["body"]["stream"]["finish"], serde_json::json!(true)); + } + + #[test] + fn websocket_media_upload_payloads_match_aibot_sdk_shape() { + test_reset_websocket_state(); + let chunk_blob_paths = + persist_websocket_media_chunks("send-1", b"abc").expect("persist chunk"); + let mut send = PendingWebsocketMediaSend { + id: "send-1".to_string(), + batch_id: "batch-1".to_string(), + chat_id: "ZhangSan".to_string(), + created_at_ms: 1, + media_type: "image".to_string(), + filename: "cat.jpg".to_string(), + md5_hex: websocket_media_md5_hex(b"abc"), + chunk_blob_paths, + total_size: 3, + total_chunks: 1, + next_chunk_index: 0, + init_req_id: "init-1".to_string(), + chunk_req_id: Some("chunk-1".to_string()), + finish_req_id: Some("finish-1".to_string()), + send_req_id: Some("send-req-1".to_string()), + upload_id: Some("upload-1".to_string()), + media_id: Some("media-1".to_string()), + }; + + let init: serde_json::Value = + serde_json::from_str(&build_websocket_media_init_payload(&send).expect("init")) + .expect("init json"); + assert_eq!( + init["cmd"], + serde_json::json!(WECOM_WS_UPLOAD_MEDIA_INIT_CMD) + ); + assert_eq!(init["body"]["type"], serde_json::json!("image")); + assert_eq!(init["body"]["filename"], serde_json::json!("cat.jpg")); + assert_eq!(init["body"]["total_size"], serde_json::json!(3)); + assert_eq!( + init["body"]["md5"], + serde_json::json!("900150983cd24fb0d6963f7d28e17f72") + ); + + let chunk: serde_json::Value = + serde_json::from_str(&build_websocket_media_chunk_payload(&send).expect("chunk")) + .expect("chunk json"); + assert_eq!( + chunk["cmd"], + serde_json::json!(WECOM_WS_UPLOAD_MEDIA_CHUNK_CMD) + ); + assert_eq!(chunk["body"]["upload_id"], serde_json::json!("upload-1")); + assert_eq!(chunk["body"]["chunk_index"], serde_json::json!(0)); + assert_eq!(chunk["body"]["base64_data"], serde_json::json!("YWJj")); + + let finish: serde_json::Value = + serde_json::from_str(&build_websocket_media_finish_payload(&send).expect("finish")) + .expect("finish json"); + assert_eq!( + finish["cmd"], + serde_json::json!(WECOM_WS_UPLOAD_MEDIA_FINISH_CMD) + ); + assert_eq!(finish["body"]["upload_id"], serde_json::json!("upload-1")); + + send.media_id = Some("media-2".to_string()); + let media: serde_json::Value = + serde_json::from_str(&build_websocket_active_media_payload(&send).expect("send media")) + .expect("media json"); + assert_eq!(media["cmd"], serde_json::json!(WECOM_WS_SEND_MSG_CMD)); + assert_eq!(media["body"]["chatid"], serde_json::json!("ZhangSan")); + assert_eq!(media["body"]["msgtype"], serde_json::json!("image")); + assert_eq!( + media["body"]["image"]["media_id"], + serde_json::json!("media-2") + ); + } + + #[test] + fn websocket_media_ack_updates_state_before_returning_next_payload() { + test_reset_websocket_state(); + let attachment = make_outbound_attachment( + "cat.jpg", + "image/jpeg", + WEBSOCKET_MEDIA_CHUNK_SIZE + 1, + ); + let (send, _init_payload) = + build_pending_websocket_media_send("batch-1", "chat-1", &attachment, 0) + .expect("pending send"); + let init_req_id = send.init_req_id.clone(); + let mut state = PendingWebsocketMediaState { + sends: vec![send], + batches: vec![PendingWebsocketMediaBatch { + id: "batch-1".to_string(), + chat_id: "chat-1".to_string(), + created_at_ms: 1, + response_req_id: "reply-1".to_string(), + response_cmd: WECOM_WS_REPLY_CMD.to_string(), + final_text: "done".to_string(), + remaining_media: 1, + sent_media: 0, + failed_media: 0, + errors: Vec::new(), + }], + }; + + let first = apply_websocket_ack_to_media_state( + &mut state, + &websocket_ack_for_test(&init_req_id, serde_json::json!({"upload_id": "upload-1"})), + ); + let first_payload = match first { + WebsocketAckApplyResult::Applied { + outbound: Some(outbound), + } => outbound.payload, + _ => panic!("init ack should prepare first chunk"), + }; + + let first_chunk_req_id = state.sends[0] + .chunk_req_id + .clone() + .expect("chunk req id must be persisted before send"); + assert!( + first_payload.contains(&first_chunk_req_id), + "payload should use the req_id already present in state" + ); + + let second = apply_websocket_ack_to_media_state( + &mut state, + &websocket_ack_for_test(&first_chunk_req_id, serde_json::json!({})), + ); + let second_payload = match second { + WebsocketAckApplyResult::Applied { + outbound: Some(outbound), + } => outbound.payload, + _ => panic!("first chunk ack should prepare second chunk"), + }; + + let second_chunk_req_id = state.sends[0] + .chunk_req_id + .as_deref() + .expect("next chunk req id must be persisted before send"); + assert_ne!(first_chunk_req_id, second_chunk_req_id); + assert!( + second_payload.contains(second_chunk_req_id), + "next payload should use the req_id already present in state" + ); + } + + #[test] + fn handle_websocket_ack_frame_persists_state_before_sending_next_command() { + test_reset_websocket_state(); + let attachment = make_outbound_attachment("cat.jpg", "image/jpeg", 3); + let (send, _init_payload) = + build_pending_websocket_media_send("batch-1", "chat-1", &attachment, 0) + .expect("pending send"); + let init_req_id = send.init_req_id.clone(); + let state = PendingWebsocketMediaState { + sends: vec![send], + batches: vec![PendingWebsocketMediaBatch { + id: "batch-1".to_string(), + chat_id: "chat-1".to_string(), + created_at_ms: 1, + response_req_id: "reply-1".to_string(), + response_cmd: WECOM_WS_REPLY_CMD.to_string(), + final_text: "done".to_string(), + remaining_media: 1, + sent_media: 0, + failed_media: 0, + errors: Vec::new(), + }], + }; + persist_pending_websocket_media_state(&state).expect("persist state"); + + handle_websocket_ack_frame(websocket_ack_for_test( + &init_req_id, + serde_json::json!({"upload_id": "upload-1"}), + )); + + let outbound = TEST_WEBSOCKET_OUTBOUND.with(|outbound| outbound.borrow().clone()); + assert_eq!(outbound.len(), 1); + let persisted = load_pending_websocket_media_state(); + let chunk_req_id = persisted.sends[0] + .chunk_req_id + .as_deref() + .expect("chunk req_id persisted"); + assert!(outbound[0].contains(chunk_req_id)); + } + + #[test] + fn handle_websocket_ack_frame_records_media_failure_when_next_command_send_fails() { + test_reset_websocket_state(); + let attachment = make_outbound_attachment("cat.jpg", "image/jpeg", 3); + let (send, _init_payload) = + build_pending_websocket_media_send("batch-2", "chat-2", &attachment, 0) + .expect("pending send"); + let chunk_paths = send.chunk_blob_paths.clone(); + let init_req_id = send.init_req_id.clone(); + let state = PendingWebsocketMediaState { + sends: vec![send], + batches: vec![PendingWebsocketMediaBatch { + id: "batch-2".to_string(), + chat_id: "chat-2".to_string(), + created_at_ms: 1, + response_req_id: "reply-2".to_string(), + response_cmd: WECOM_WS_REPLY_CMD.to_string(), + final_text: "done".to_string(), + remaining_media: 1, + sent_media: 0, + failed_media: 0, + errors: Vec::new(), + }], + }; + persist_pending_websocket_media_state(&state).expect("persist state"); + TEST_WEBSOCKET_SEND_ERROR.with(|error| { + *error.borrow_mut() = Some("send failed".to_string()); + }); + + handle_websocket_ack_frame(websocket_ack_for_test( + &init_req_id, + serde_json::json!({"upload_id": "upload-2"}), + )); + + let persisted = load_pending_websocket_media_state(); + assert!(persisted.sends.is_empty()); + assert!(persisted.batches.is_empty()); + for path in chunk_paths { + assert!( + read_wecom_workspace(&path).is_none(), + "failed media chunk blob should be cleaned up" + ); + } + let outbound = TEST_WEBSOCKET_OUTBOUND.with(|outbound| outbound.borrow().clone()); + assert!( + outbound + .iter() + .any(|payload| payload.contains("附件已生成,但发送失败")), + "failure should still surface a visible text response" + ); + } + + #[test] + fn stale_websocket_media_state_prunes_chunk_blobs() { + test_reset_websocket_state(); + let attachment = make_outbound_attachment("old.jpg", "image/jpeg", 3); + let (mut send, _init_payload) = + build_pending_websocket_media_send("batch-old", "chat-old", &attachment, 0) + .expect("pending send"); + let chunk_paths = send.chunk_blob_paths.clone(); + send.created_at_ms = 1; + let mut state = PendingWebsocketMediaState { + sends: vec![send], + batches: vec![PendingWebsocketMediaBatch { + id: "batch-old".to_string(), + chat_id: "chat-old".to_string(), + created_at_ms: 1, + response_req_id: "reply-old".to_string(), + response_cmd: WECOM_WS_REPLY_CMD.to_string(), + final_text: "done".to_string(), + remaining_media: 1, + sent_media: 0, + failed_media: 0, + errors: Vec::new(), + }], + }; + + assert!(prune_stale_pending_websocket_media_state( + &mut state, + WEBSOCKET_MEDIA_SEND_TTL_MS + 2 + )); + + assert!(state.sends.is_empty()); + assert!(state.batches.is_empty()); + for path in chunk_paths { + assert!(read_wecom_workspace(&path).is_none()); + } + } + + #[test] + fn websocket_media_errors_are_capped() { + let errors = (0..(MAX_WEBSOCKET_MEDIA_BATCH_ERRORS + 5)) + .map(|idx| format!("error-{idx}")) + .collect::>(); + + let capped = capped_websocket_media_errors(errors); + + assert_eq!(capped.len(), MAX_WEBSOCKET_MEDIA_BATCH_ERRORS + 1); + assert!(capped.last().unwrap().contains("omitted")); + } + + #[test] + fn websocket_media_batch_caps_startup_validation_errors() { + test_reset_websocket_state(); + let metadata = WecomMessageMetadata { + to_user: "ZhangSan".to_string(), + target: Some("chat-1".to_string()), + chat_id: Some("chat-1".to_string()), + chat_type: Some("private".to_string()), + source_msg_id: Some("msg-1".to_string()), + ws_req_id: Some("req-1".to_string()), + ws_chat_id: Some("chat-1".to_string()), + ws_chat_type: Some("private".to_string()), + ws_reply_cmd: Some(WECOM_WS_REPLY_CMD.to_string()), + }; + let attachments: Vec = (0..(MAX_WEBSOCKET_MEDIA_BATCH_ERRORS + 5)) + .map(|idx| make_outbound_attachment(&format!("empty-{idx}.png"), "image/png", 0)) + .collect(); + + let result = start_websocket_media_batch( + &metadata, + "req-1", + WECOM_WS_REPLY_CMD, + "caption", + &attachments, + ); + + assert_eq!(result.started, 0); + assert_eq!(result.errors.len(), MAX_WEBSOCKET_MEDIA_BATCH_ERRORS + 1); + assert!(result.errors.last().unwrap().contains("omitted")); + assert!(load_pending_websocket_media_state().sends.is_empty()); + } + + #[test] + fn websocket_media_ack_accepts_numeric_created_at() { + let ack = parse_websocket_ack_frame( + r#"{"headers":{"req_id":"finish-1"},"errcode":0,"errmsg":"ok","body":{"type":"image","media_id":"media-1","created_at":1776832851}}"#, + ) + .expect("ack"); + + assert_eq!(ack.headers.req_id, "finish-1"); + assert_eq!(ack.errcode, 0); + assert_eq!( + json_body_string(&ack.body, "media_id").as_deref(), + Some("media-1") + ); + } + + #[test] + fn websocket_media_target_prefers_chat_id_over_user_id() { + let metadata = WecomMessageMetadata { + to_user: "ZhangSan".to_string(), + target: Some("wr-chat".to_string()), + chat_id: Some("wr-chat".to_string()), + chat_type: Some("group".to_string()), + source_msg_id: Some("msg-1".to_string()), + ws_req_id: Some("req-1".to_string()), + ws_chat_id: Some("wr-chat".to_string()), + ws_chat_type: Some("group".to_string()), + ws_reply_cmd: Some(WECOM_WS_REPLY_CMD.to_string()), + }; + + assert_eq!( + websocket_media_target(&metadata).as_deref(), + Some("wr-chat") + ); + } + + #[test] + fn websocket_media_batch_enforces_per_response_limits_before_persisting() { + test_reset_websocket_state(); + let metadata = WecomMessageMetadata { + to_user: "ZhangSan".to_string(), + target: Some("chat-1".to_string()), + chat_id: Some("chat-1".to_string()), + chat_type: Some("private".to_string()), + source_msg_id: Some("msg-1".to_string()), + ws_req_id: Some("req-1".to_string()), + ws_chat_id: Some("chat-1".to_string()), + ws_chat_type: Some("private".to_string()), + ws_reply_cmd: Some(WECOM_WS_REPLY_CMD.to_string()), + }; + let attachments: Vec = (0..(MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE + 2)) + .map(|idx| make_outbound_attachment(&format!("image-{idx}.png"), "image/png", 1)) + .collect(); + + let result = start_websocket_media_batch( + &metadata, + "req-1", + WECOM_WS_REPLY_CMD, + "caption", + &attachments, + ); + + assert_eq!(result.started, MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE); + assert!( + result + .errors + .iter() + .any(|error| error.contains("attachment(s)")) + ); + let persisted = load_pending_websocket_media_state(); + assert_eq!( + persisted.sends.len(), + MAX_WEBSOCKET_MEDIA_ATTACHMENTS_PER_RESPONSE + ); + } + + #[test] + fn websocket_metadata_json_marks_group_chats_as_agent_api_ineligible() { + let json = websocket_metadata_json( + "zhangsan", + "msg-1", + "req-1", + Some("chat-1"), + Some("group"), + WECOM_WS_REPLY_CMD, + ) + .expect("metadata json"); + + let metadata: WecomMessageMetadata = serde_json::from_str(&json).expect("metadata"); + assert_eq!(metadata.to_user, ""); + assert_eq!(metadata.target.as_deref(), Some("chat-1")); + assert_eq!(metadata.chat_id.as_deref(), Some("chat-1")); + assert_eq!(metadata.chat_type.as_deref(), Some("group")); + assert_eq!(metadata.ws_req_id.as_deref(), Some("req-1")); + assert_eq!(metadata.ws_chat_id.as_deref(), Some("chat-1")); + assert_eq!(metadata.ws_chat_type.as_deref(), Some("group")); + assert_eq!(metadata.ws_reply_cmd.as_deref(), Some(WECOM_WS_REPLY_CMD)); + } + + #[test] + fn classify_status_update_surfaces_failures_and_ignores_noise() { + let thinking = StatusUpdate { + status: StatusType::Thinking, + message: "Thinking".to_string(), + metadata_json: "{}".to_string(), + }; + assert!(classify_status_update(&thinking).is_none()); + + let tool_started = StatusUpdate { + status: StatusType::ToolStarted, + message: "Tool started".to_string(), + metadata_json: "{}".to_string(), + }; + assert!(classify_status_update(&tool_started).is_none()); + + let status_done = StatusUpdate { + status: StatusType::Status, + message: "Done".to_string(), + metadata_json: "{}".to_string(), + }; + assert!(classify_status_update(&status_done).is_none()); + + let provider_error = StatusUpdate { + status: StatusType::Status, + message: "Provider error: upstream 502".to_string(), + metadata_json: "{}".to_string(), + }; + assert_eq!( + classify_status_update(&provider_error).as_deref(), + Some("Provider error: upstream 502") + ); + + let interrupted = StatusUpdate { + status: StatusType::Interrupted, + message: "".to_string(), + metadata_json: "{}".to_string(), + }; + assert_eq!( + classify_status_update(&interrupted).as_deref(), + Some("Request interrupted. Please try again.") + ); + } + + #[test] + fn dm_policy_open_only_bypasses_allowlist_for_private_chats() { + assert!(dm_policy_allows_sender_without_allowlist( + DmPolicy::Open, + Some("single") + )); + assert!(dm_policy_allows_sender_without_allowlist( + DmPolicy::Open, + Some("private") + )); + assert!(!dm_policy_allows_sender_without_allowlist( + DmPolicy::Open, + Some("group") + )); + assert!(!dm_policy_allows_sender_without_allowlist( + DmPolicy::Open, + None + )); + assert!(!dm_policy_allows_sender_without_allowlist( + DmPolicy::Pairing, + Some("single") + )); + } + + #[test] + fn sensitive_status_uses_generic_group_chat_message() { + let approval = StatusUpdate { + status: StatusType::ApprovalNeeded, + message: "Approve tool call with secret context".to_string(), + metadata_json: "{}".to_string(), + }; + + let safe = safe_group_status_content( + &approval, + classify_status_update(&approval).expect("approval status content"), + ); + + assert!(safe.contains("private authorization")); + assert!(!safe.contains("secret context")); + } + + #[test] + fn parse_realistic_websocket_group_mixed_payload() { + let raw = r#"{ + "cmd":"aibot_msg_callback", + "headers":{"req_id":"req-group-1"}, + "body":{ + "msgid":"msg-group-1", + "chatid":"wr7NnM9z0", + "chattype":"group", + "from":{"userid":"ZhangSan"}, + "msgtype":"mixed", + "mixed":{ + "msgItem":[ + {"itemtype":"text","text":{"content":"请看这张图"}}, + {"itemtype":"image","image":{"url":"https://openws.work.weixin.qq.com/image/1","aeskey":"dGVzdGtleQ=="}} + ] + }, + "quote":{ + "msgtype":"text", + "text":{"content":"上一条消息"} + } + } + }"#; + + let frame: WecomWsFrame = + serde_json::from_str(raw).expect("frame parse"); + assert_eq!(frame.headers.req_id, "req-group-1"); + assert_eq!(frame.body.msgtype, "mixed"); + assert_eq!(frame.body.chatid.as_deref(), Some("wr7NnM9z0")); + assert_eq!(frame.body.chattype.as_deref(), Some("group")); + assert_eq!(frame.body.from.userid, "ZhangSan"); + + let (content, attachments) = websocket_mixed_content_parts( + &frame.body.msgid, + frame.body.mixed.as_ref().expect("mixed"), + ); + assert_eq!(content, "请看这张图"); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "msg-group-1:1:image"); + assert_eq!( + attachments[0].source_url.as_deref(), + Some("https://openws.work.weixin.qq.com/image/1") + ); + assert_eq!( + with_websocket_quote_context(content, frame.body.quote.as_ref()), + "Quoted text: 上一条消息\n\n请看这张图" + ); + assert_eq!( + wecom_conversation_scope( + &frame.body.from.userid, + frame.body.chatid.as_deref(), + frame.body.chattype.as_deref() + ), + "wecom:group:wr7NnM9z0" + ); + } + + #[test] + fn parse_realistic_websocket_event_payload_with_event_key_alias() { + let raw = r#"{ + "cmd":"aibot_event_callback", + "headers":{"req_id":"req-event-2"}, + "body":{ + "msgid":"evt-group-1", + "chatid":"wr7NnM9z0", + "chattype":"group", + "from":{"userid":"LiSi"}, + "event":{ + "eventtype":"template_card_event", + "eventKey":"approve_order" + } + } + }"#; + + let frame: WecomWsFrame = serde_json::from_str(raw).expect("event parse"); + assert_eq!(frame.headers.req_id, "req-event-2"); + assert_eq!(frame.body.event.eventtype, "template_card_event"); + assert_eq!( + websocket_event_summary(&frame.body.event).as_deref(), + Some("User clicked a WeCom template card action: approve_order") + ); + assert_eq!( + wecom_conversation_scope( + &frame.body.from.userid, + frame.body.chatid.as_deref(), + frame.body.chattype.as_deref() + ), + "wecom:group:wr7NnM9z0" + ); + } + + #[test] + fn wecom_conversation_scope_splits_group_and_dm() { + assert_eq!( + wecom_conversation_scope("zhangsan", Some("chat-1"), Some("group")), + "wecom:group:chat-1" + ); + assert_eq!( + wecom_conversation_scope("zhangsan", Some("chat-1"), Some("single")), + "wecom:dm:zhangsan" + ); + assert_eq!( + wecom_conversation_scope("zhangsan", None, Some("private")), + "wecom:dm:zhangsan" + ); + } + + fn make_pending_bundle( + thread_id: &str, + content: &str, + attachment_count: usize, + ) -> PendingInboundBundle { + PendingInboundBundle { + user_id: "zhangsan".to_string(), + user_name: None, + thread_id: thread_id.to_string(), + metadata_json: r#"{"to_user":"zhangsan"}"#.to_string(), + content: content.to_string(), + attachments: (0..attachment_count) + .map(|idx| StoredInboundAttachment { + id: format!("att-{idx}"), + mime_type: "image/jpeg".to_string(), + filename: Some(format!("att-{idx}.jpg")), + size_bytes: Some(3), + extracted_text: None, + }) + .collect(), + flush_at_ms: 0, + } + } + + #[test] + fn stored_inbound_attachment_strips_transport_sensitive_fields() { + let inbound = InboundAttachment { + id: "msg-1:image".to_string(), + mime_type: "image/jpeg".to_string(), + filename: Some("pic.jpg".to_string()), + size_bytes: Some(3), + source_url: Some("https://openws.work.weixin.qq.com/file".to_string()), + storage_key: Some("tmp-key".to_string()), + extracted_text: None, + extras_json: r#"{"aeskey":"secret","wecom_ws_msgtype":"image"}"#.to_string(), + }; + + let stored = StoredInboundAttachment::from(inbound); + let stored_json = serde_json::to_string(&stored).expect("stored json"); + assert!(!stored_json.contains("source_url")); + assert!(!stored_json.contains("storage_key")); + assert!(!stored_json.contains("aeskey")); + + let restored: InboundAttachment = stored.into(); + assert_eq!(restored.source_url, None); + assert_eq!(restored.storage_key, None); + assert_eq!(restored.extras_json, "{}"); + } + + #[test] + fn pending_attachment_blob_path_uses_hashed_identifier() { + let path = pending_attachment_blob_path("msg-1:image"); + assert!(path.starts_with("state/pending_attachment_blobs/")); + assert!(path.ends_with(".b64")); + assert!(!path.contains("msg-1:image")); + } + + #[test] + fn process_pending_inbound_bundle_buffers_attachment_only_message() { + let mut pending = HashMap::new(); + let emitted = process_pending_inbound_bundle( + &mut pending, + "wecom:dm:zhangsan", + make_pending_bundle("wecom:dm:zhangsan", "", 1), + 100, + 5_000, + ); + + assert!(emitted.is_empty()); + let stored = pending + .get("wecom:dm:zhangsan") + .expect("attachment-only message should be buffered"); + assert_eq!(stored.attachments.len(), 1); + assert_eq!(stored.flush_at_ms, 5_100); + } + + #[test] + fn process_pending_inbound_bundle_merges_buffered_attachment_with_follow_up_text() { + let mut pending = HashMap::new(); + let _ = process_pending_inbound_bundle( + &mut pending, + "wecom:dm:zhangsan", + make_pending_bundle("wecom:dm:zhangsan", "", 1), + 100, + 5_000, + ); + + let emitted = process_pending_inbound_bundle( + &mut pending, + "wecom:dm:zhangsan", + make_pending_bundle("wecom:dm:zhangsan", "请看这张图", 0), + 150, + 5_000, + ); + assert_eq!(emitted.len(), 1); + assert!(pending.is_empty()); + assert_eq!(emitted[0].content, "请看这张图"); + assert_eq!(emitted[0].attachments.len(), 1); + } + + #[test] + fn take_due_pending_inbound_bundles_only_returns_expired_entries() { + let mut pending = HashMap::from([ + ( + "wecom:dm:zhangsan".to_string(), + PendingInboundBundle { + flush_at_ms: 200, + ..make_pending_bundle("wecom:dm:zhangsan", "", 1) + }, + ), + ( + "wecom:dm:lisi".to_string(), + PendingInboundBundle { + flush_at_ms: 400, + ..make_pending_bundle("wecom:dm:lisi", "", 1) + }, + ), + ]); + + let due = take_due_pending_inbound_bundles(&mut pending, 250); + assert_eq!(due.len(), 1); + assert_eq!(due[0].thread_id, "wecom:dm:zhangsan"); + assert!(pending.contains_key("wecom:dm:lisi")); + } + + #[test] + fn pairing_reply_hides_code_in_group_chat() { + let route = PairingReplyRoute { + req_id: "req-1".to_string(), + reply_cmd: WECOM_WS_REPLY_CMD.to_string(), + chat_type: Some("group".to_string()), + }; + + assert!(should_send_pairing_reply(&route, true)); + assert!(!should_send_pairing_reply(&route, false)); + + let content = pairing_reply_text(&route, "ABCD1234"); + assert!(content.contains("please DM")); + assert!(!content.contains("ABCD1234")); + } + + #[test] + fn pairing_reply_always_sends_code_in_private_chat() { + let route = PairingReplyRoute { + req_id: "req-2".to_string(), + reply_cmd: WECOM_WS_REPLY_CMD.to_string(), + chat_type: Some("single".to_string()), + }; + + assert!(should_send_pairing_reply(&route, true)); + assert!(should_send_pairing_reply(&route, false)); + + let content = pairing_reply_text(&route, "EFGH5678"); + assert!(content.contains("EFGH5678")); + } + + #[test] + fn websocket_quote_context_prefixes_user_content() { + let quote = WecomWsQuoteContent { + msg_type: Some("text".to_string()), + text: Some(WecomWsTextContent { + content: "Earlier message".to_string(), + }), + voice: None, + content: None, + }; + + let content = with_websocket_quote_context("Current message".to_string(), Some("e)); + assert_eq!(content, "Quoted text: Earlier message\n\nCurrent message"); + } + + #[test] + fn websocket_mixed_content_parts_extract_text_and_images() { + let mixed = WecomWsMixedContent { + msg_item: vec![ + WecomWsMixedItem { + item_type: Some("text".to_string()), + text: Some(WecomWsTextContent { + content: "First line".to_string(), + }), + image: None, + file: None, + video: None, + }, + WecomWsMixedItem { + item_type: Some("image".to_string()), + text: None, + image: Some(WecomWsBinaryContent { + url: "https://example.com/image".to_string(), + aeskey: Some("aes".to_string()), + }), + file: None, + video: None, + }, + WecomWsMixedItem { + item_type: None, + text: Some(WecomWsTextContent { + content: "Second line".to_string(), + }), + image: None, + file: None, + video: None, + }, + ], + }; + + let (content, attachments) = websocket_mixed_content_parts("msg-1", &mixed); + assert_eq!(content, "First line\nSecond line"); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "msg-1:1:image"); + assert_eq!( + attachments[0].source_url.as_deref(), + Some("https://example.com/image") + ); + assert_eq!( + serde_json::from_str::(&attachments[0].extras_json) + .expect("extras json")["wecom_ws_msgtype"], + serde_json::json!("image") + ); + assert!(!attachments[0].extras_json.contains("aes")); + } + + #[test] + fn websocket_event_summary_formats_interactive_events() { + let template_card_event = WecomWsEvent { + eventtype: "template_card_event".to_string(), + extra: serde_json::from_value(serde_json::json!({ + "event_key": "approve" + })) + .expect("template card extras"), + }; + assert_eq!( + websocket_event_summary(&template_card_event).as_deref(), + Some("User clicked a WeCom template card action: approve") + ); + + let feedback_event = WecomWsEvent { + eventtype: "feedback_event".to_string(), + extra: serde_json::from_value(serde_json::json!({ + "score": 5 + })) + .expect("feedback extras"), + }; + assert_eq!( + websocket_event_summary(&feedback_event).as_deref(), + Some("User submitted WeCom feedback with score 5.") + ); + } + + #[test] + fn update_recent_message_ids_rejects_duplicates() { + let existing = r#"["msg-1","msg-2"]"#; + let (is_new, json) = update_recent_message_ids(Some(existing), "msg-2", 8, 100, 1_000) + .expect("dedupe update"); + + assert!(!is_new); + let ids: Vec = serde_json::from_str(&json).expect("ids parse"); + assert_eq!( + ids.iter().map(|entry| entry.id.as_str()).collect::>(), + vec!["msg-1", "msg-2"] + ); + } + + #[test] + fn update_recent_message_ids_trims_oldest_entries() { + let existing = r#"["msg-1","msg-2","msg-3"]"#; + let (is_new, json) = update_recent_message_ids(Some(existing), "msg-4", 3, 100, 1_000) + .expect("dedupe update"); + + assert!(is_new); + let ids: Vec = serde_json::from_str(&json).expect("ids parse"); + assert_eq!( + ids.iter().map(|entry| entry.id.as_str()).collect::>(), + vec!["msg-2", "msg-3", "msg-4"] + ); + } + + #[test] + fn update_recent_message_ids_prunes_expired_entries() { + let existing = serde_json::to_string(&vec![ + RecentMessageIdEntry { + id: "old".to_string(), + seen_at_ms: 10, + }, + RecentMessageIdEntry { + id: "fresh".to_string(), + seen_at_ms: 95, + }, + ]) + .expect("serialize dedupe entries"); + + let (is_new, json) = update_recent_message_ids(Some(&existing), "new", 8, 100, 20) + .expect("dedupe update"); + + assert!(is_new); + let ids: Vec = serde_json::from_str(&json).expect("ids parse"); + assert_eq!( + ids.iter().map(|entry| entry.id.as_str()).collect::>(), + vec!["fresh", "new"] + ); + } + + #[test] + fn decrypt_websocket_media_payload_round_trips_with_trimmed_key_padding() { + let key_bytes = [11u8; 32]; + let key_base64 = BASE64_STANDARD.encode(key_bytes); + let trimmed_key = key_base64.trim_end_matches('='); + let plaintext = b"wecom websocket media payload"; + + let ciphertext = encrypt_websocket_media_for_test(&key_bytes, plaintext); + let decrypted = + decrypt_websocket_media_payload(&ciphertext, trimmed_key).expect("decrypt media"); + + assert_eq!(decrypted, plaintext); + } + + #[test] + fn remove_wecom_pkcs7_padding_rejects_inconsistent_padding() { + let data = b"hello\x04\x04\x04\x03"; + let error = remove_wecom_pkcs7_padding(data).expect_err("padding mismatch"); + assert!(error.contains("padding")); + } + + fn make_outbound_attachment(filename: &str, mime_type: &str, size_bytes: usize) -> Attachment { + Attachment { + filename: filename.to_string(), + mime_type: mime_type.to_string(), + data: vec![0; size_bytes], + } + } + + #[test] + fn base_mime_type_strips_parameters() { + assert_eq!(base_mime_type("audio/amr; codecs=amr"), "audio/amr"); + assert_eq!(base_mime_type("image/png"), "image/png"); + assert_eq!(base_mime_type(""), ""); + } + + #[test] + fn classify_websocket_media_maps_supported_wecom_types() { + assert_eq!( + classify_websocket_media(&make_outbound_attachment("photo.png", "image/png", 128)), + OutboundMediaKind::Image + ); + assert_eq!( + classify_websocket_media(&make_outbound_attachment("voice.amr", "audio/amr", 128)), + OutboundMediaKind::Voice + ); + assert_eq!( + classify_websocket_media(&make_outbound_attachment("clip.mp4", "video/mp4", 128)), + OutboundMediaKind::Video + ); + assert_eq!( + classify_websocket_media(&make_outbound_attachment( + "report.pdf", + "application/pdf", + 128 + )), + OutboundMediaKind::File + ); + } + + #[test] + fn classify_websocket_media_uses_filename_extension_when_mime_is_generic() { + assert_eq!( + classify_websocket_media(&make_outbound_attachment( + "screenshot.jpeg", + "application/octet-stream", + 128 + )), + OutboundMediaKind::Image + ); + assert_eq!( + classify_websocket_media(&make_outbound_attachment( + "recording.amr", + "application/octet-stream", + 128 + )), + OutboundMediaKind::Voice + ); + } + + #[test] + fn classify_websocket_media_falls_back_to_file_when_specific_media_is_too_large() { + assert_eq!( + classify_websocket_media(&make_outbound_attachment( + "photo.png", + "image/png", + MAX_WEBSOCKET_IMAGE_BYTES + 1 + )), + OutboundMediaKind::File + ); + assert_eq!( + classify_websocket_media(&make_outbound_attachment( + "clip.mp4", + "video/mp4", + MAX_WEBSOCKET_VIDEO_BYTES + 1 + )), + OutboundMediaKind::File + ); + } + + #[test] + fn validate_websocket_media_size_rejects_oversized_files() { + assert!( + validate_websocket_media_size(OutboundMediaKind::File, MAX_ATTACHMENT_BYTES).is_ok() + ); + assert!( + validate_websocket_media_size(OutboundMediaKind::File, MAX_ATTACHMENT_BYTES + 1) + .is_err() + ); + } +} diff --git a/channels-src/wecom/wecom.capabilities.json b/channels-src/wecom/wecom.capabilities.json new file mode 100644 index 00000000000..17411ff78af --- /dev/null +++ b/channels-src/wecom/wecom.capabilities.json @@ -0,0 +1,73 @@ +{ + "version": "0.1.0", + "wit_version": "0.3.1", + "type": "channel", + "name": "wecom", + "description": "WeCom intelligent bot channel with WebSocket inbound, pairing, group-aware sessions, and direct Bot media sends", + "auth": { + "secret_name": "wecom_bot_id", + "display_name": "WeCom", + "instructions": "Create a WeCom intelligent bot and collect its Bot ID and Bot Secret.", + "setup_url": "https://open.work.weixin.qq.com/", + "token_hint": "Bot ID and Bot Secret come from the WeCom intelligent bot setup page.", + "env_var": "WECOM_BOT_ID" + }, + "setup": { + "required_secrets": [ + { + "name": "wecom_bot_id", + "prompt": "Enter the BotId from your WeCom intelligent bot settings.", + "validation": "^[A-Za-z0-9_-]{3,128}$", + "optional": false + }, + { + "name": "wecom_bot_secret", + "prompt": "Enter the Secret from your WeCom intelligent bot settings.", + "validation": "^[A-Za-z0-9_-]{16,128}$", + "optional": false + } + ], + "setup_url": "https://open.work.weixin.qq.com/" + }, + "capabilities": { + "http": { + "allowlist": [ + { "host": "openws.work.weixin.qq.com", "path_prefix": "/", "methods": ["GET"] }, + { "host": "*.myqcloud.com", "path_prefix": "/", "methods": ["GET"] } + ], + "max_response_bytes": 20971520, + "rate_limit": { + "requests_per_minute": 60, + "requests_per_hour": 2000 + } + }, + "websocket": { + "url": "wss://openws.work.weixin.qq.com", + "connect_on_start": true, + "protocol": "wecom_aibot", + "bot_id_secret_name": "wecom_bot_id", + "bot_secret_name": "wecom_bot_secret", + "heartbeat_interval_ms": 30000, + "max_missed_heartbeat_acks": 2 + }, + "secrets": { + "allowed_names": ["wecom_bot_id", "wecom_bot_secret"] + }, + "channel": { + "allowed_paths": [], + "allow_polling": true, + "workspace_prefix": "channels/wecom/", + "callback_timeout_secs": 45, + "emit_rate_limit": { + "messages_per_minute": 100, + "messages_per_hour": 5000 + } + } + }, + "config": { + "inbound_merge_window_ms": 5000, + "owner_id": null, + "dm_policy": "pairing", + "allow_from": [] + } +} diff --git a/crates/ironclaw_common/CHANGELOG.md b/crates/ironclaw_common/CHANGELOG.md index f9db2fa6d64..ae8d5669e56 100644 --- a/crates/ironclaw_common/CHANGELOG.md +++ b/crates/ironclaw_common/CHANGELOG.md @@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.2](https://github.com/nearai/ironclaw/compare/ironclaw_common-v0.4.1...ironclaw_common-v0.4.2) - 2026-05-11 + +### Added + +- *(common)* describe paths and platform helpers in crate description ([#3498](https://github.com/nearai/ironclaw/pull/3498)) + +### Other + +- *(llm)* extract multi-provider integration into ironclaw_llm crate ([#3387](https://github.com/nearai/ironclaw/pull/3387)) + +## [0.4.1](https://github.com/nearai/ironclaw/compare/ironclaw_common-v0.4.0...ironclaw_common-v0.4.1) - 2026-05-07 + +### Added + +- *(common)* align crate description with lib.rs doc wording ([#3372](https://github.com/nearai/ironclaw/pull/3372)) + +### Fixed + +- *(common)* clarify crate-level doc wording ([#3370](https://github.com/nearai/ironclaw/pull/3370)) + ## [0.4.0](https://github.com/nearai/ironclaw/compare/ironclaw_common-v0.3.0...ironclaw_common-v0.4.0) - 2026-04-29 ### Added diff --git a/crates/ironclaw_common/Cargo.toml b/crates/ironclaw_common/Cargo.toml index c9bfa4761ad..9f9bb0ca900 100644 --- a/crates/ironclaw_common/Cargo.toml +++ b/crates/ironclaw_common/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "ironclaw_common" -version = "0.4.0" +version = "0.4.2" edition = "2024" rust-version = "1.92" -description = "Shared types and utilities for the IronClaw workspace" +description = "Shared types, paths, and platform helpers used across the IronClaw workspace" authors = ["NEAR AI "] license = "MIT OR Apache-2.0" homepage = "https://github.com/nearai/ironclaw" @@ -14,6 +14,7 @@ dist = false [dependencies] chrono-tz = "0.10" +dirs = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/crates/ironclaw_common/src/attachment.rs b/crates/ironclaw_common/src/attachment.rs new file mode 100644 index 00000000000..871020e184f --- /dev/null +++ b/crates/ironclaw_common/src/attachment.rs @@ -0,0 +1,60 @@ +//! Channel-agnostic incoming attachment types. +//! +//! `IncomingAttachment` carries a single file/media item attached to a +//! message received from any channel (Telegram, web, REPL, WASM, …). It is +//! pure data with no transport-trait coupling; the channel layer wraps it +//! into an `IncomingMessage`, while `ironclaw_llm::transcription` operates +//! directly on `&mut [IncomingAttachment]` to fill `extracted_text` for +//! audio inputs. + +/// Kind of attachment carried on an incoming message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttachmentKind { + /// Audio content (voice notes, audio files). + Audio, + /// Image content (photos, screenshots). + Image, + /// Document content (PDFs, files). + Document, +} + +impl AttachmentKind { + /// Infer attachment kind from a MIME type string. + pub fn from_mime_type(mime: &str) -> Self { + let base = mime.split(';').next().unwrap_or(mime).trim(); + if base.starts_with("audio/") { + Self::Audio + } else if base.starts_with("image/") { + Self::Image + } else { + Self::Document + } + } +} + +/// A file or media attachment on an incoming message. +#[derive(Debug, Clone)] +pub struct IncomingAttachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + pub id: String, + /// What kind of content this is. + pub kind: AttachmentKind, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + pub mime_type: String, + /// Original filename, if known. + pub filename: Option, + /// File size in bytes, if known. + pub size_bytes: Option, + /// URL to download the file from the channel's API. + pub source_url: Option, + /// Opaque key for host-side storage (e.g., after download/caching). + pub storage_key: Option, + /// Relative path to a project-local copy saved on disk, if persisted. + pub local_path: Option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + pub extracted_text: Option, + /// Raw file bytes (for small files downloaded by the channel). + pub data: Vec, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} diff --git a/crates/ironclaw_common/src/env_helpers.rs b/crates/ironclaw_common/src/env_helpers.rs new file mode 100644 index 00000000000..74f3f1c66cb --- /dev/null +++ b/crates/ironclaw_common/src/env_helpers.rs @@ -0,0 +1,105 @@ +//! Thread-safe runtime env-var overlay shared across the workspace. +//! +//! Replaces `std::env::set_var` (which is UB in multi-threaded programs on +//! Rust 1.82+) with an in-process `Mutex` that callers consult via +//! [`env_or_override`]. The main crate layers an additional secrets overlay +//! on top of this; `ironclaw_llm` and other workspace crates use this module +//! directly when they need the runtime override semantics without pulling in +//! the rest of the binary. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +/// Crate-wide mutex for tests that mutate the process environment. +/// +/// Acquire this before any `unsafe { std::env::set_var / remove_var }` call +/// so concurrent tests don't race. Recovers from poison since one panicked +/// test shouldn't cascade. +pub static ENV_MUTEX: Mutex<()> = Mutex::new(()); + +/// Acquire the env-var mutex, recovering from poison. +pub fn lock_env() -> std::sync::MutexGuard<'static, ()> { + ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()) +} + +static RUNTIME_ENV_OVERRIDES: OnceLock>> = OnceLock::new(); + +fn runtime_overrides() -> &'static Mutex> { + RUNTIME_ENV_OVERRIDES.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Optional secondary env lookup registered by the main crate at startup. +/// +/// `ironclaw` keeps a separate `INJECTED_VARS` overlay populated from the +/// encrypted secrets store (so API keys can be read without `set_var`). +/// `ironclaw_llm` does not have direct access to that overlay, so the main +/// crate registers a closure here that consults it. Callers of +/// [`env_or_override`] then see the union of: real env, runtime overrides, +/// and the registered fallback. +type EnvFallback = Box Option + Send + Sync>; +static SECONDARY_FALLBACK: OnceLock = OnceLock::new(); + +/// Install a secondary env lookup. Idempotent: subsequent calls are ignored. +pub fn register_secondary_fallback(f: impl Fn(&str) -> Option + Send + Sync + 'static) { + let _ = SECONDARY_FALLBACK.set(Box::new(f)); +} + +/// Set a runtime env override (thread-safe alternative to `std::env::set_var`). +pub fn set_runtime_env(key: &str, value: &str) { + runtime_overrides() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(key.to_string(), value.to_string()); +} + +/// Read an env var, checking real env first, then runtime overrides, then any +/// secondary fallback registered by the embedding application. +/// +/// Empty values are treated as unset at every layer. +pub fn env_or_override(key: &str) -> Option { + if let Ok(val) = std::env::var(key) + && !val.is_empty() + { + return Some(val); + } + + if let Some(val) = runtime_overrides() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(key) + .filter(|v| !v.is_empty()) + .cloned() + { + return Some(val); + } + + if let Some(fallback) = SECONDARY_FALLBACK.get() + && let Some(val) = fallback(key).filter(|v| !v.is_empty()) + { + return Some(val); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runtime_override_round_trip() { + let _guard = lock_env(); + set_runtime_env("IRONCLAW_TEST_RUNTIME_OVERRIDE", "1"); + assert_eq!( + env_or_override("IRONCLAW_TEST_RUNTIME_OVERRIDE"), + Some("1".to_string()) + ); + } + + #[test] + fn empty_runtime_override_treated_as_unset() { + let _guard = lock_env(); + set_runtime_env("IRONCLAW_TEST_EMPTY", ""); + assert_eq!(env_or_override("IRONCLAW_TEST_EMPTY"), None); + } +} diff --git a/crates/ironclaw_common/src/event.rs b/crates/ironclaw_common/src/event.rs index 061963b23e7..3bf5283da5d 100644 --- a/crates/ironclaw_common/src/event.rs +++ b/crates/ironclaw_common/src/event.rs @@ -312,6 +312,27 @@ pub enum AppEvent { #[serde(skip_serializing_if = "Option::is_none")] thread_id: Option, }, + /// Caller-provided external tool was emitted by the LLM and the + /// thread is paused until the caller POSTs back a + /// `function_call_output`. Used by the Responses API + /// (`/v1/responses`) to surface a `function_call` + /// `ResponseOutputItem` in lieu of the approval-card UX that + /// `GateRequired` carries. + /// + /// `request_id` is the engine pending-gate id (used by the resume + /// path to find the gate). `call_id` is the LLM-emitted tool call + /// identifier echoed back in `function_call_output.call_id`. + /// `arguments` is the JSON-stringified tool parameters per the + /// OpenAI Responses wire shape. + #[serde(rename = "external_tool_call")] + ExternalToolCall { + request_id: String, + call_id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "error")] Error { /// Sanitized, channel-agnostic message shown to users. Never @@ -708,6 +729,7 @@ impl AppEvent { Self::OnboardingState { .. } => "onboarding_state", Self::GateRequired { .. } => "gate_required", Self::GateResolved { .. } => "gate_resolved", + Self::ExternalToolCall { .. } => "external_tool_call", Self::Error { .. } => "error", Self::Heartbeat => "heartbeat", Self::JobMessage { .. } => "job_message", @@ -833,6 +855,13 @@ mod tests { resume_kind: serde_json::Value::Null, thread_id: None, }, + AppEvent::ExternalToolCall { + request_id: String::new(), + call_id: String::new(), + name: String::new(), + arguments: String::new(), + thread_id: None, + }, AppEvent::Error { message: String::new(), thread_id: None, diff --git a/crates/ironclaw_common/src/lib.rs b/crates/ironclaw_common/src/lib.rs index 52777fdfaf9..88259b825bb 100644 --- a/crates/ironclaw_common/src/lib.rs +++ b/crates/ironclaw_common/src/lib.rs @@ -1,10 +1,15 @@ -//! Shared types and utilities for the IronClaw workspace. +//! Shared types, paths, and platform helpers used across the IronClaw workspace. +pub mod attachment; +pub mod env_helpers; mod event; mod identity; +pub mod paths; +pub mod platform; mod timezone; mod util; +pub use attachment::{AttachmentKind, IncomingAttachment}; pub use event::{ AppEvent, CodeExecutionFailureCategory, JobResultStatus, JobResultStatusParseError, OnboardingStateDto, PlanStepDto, SelfImprovementPhase, ToolDecisionDto, @@ -14,8 +19,10 @@ pub use identity::{ MAX_EXTERNAL_THREAD_ID_LEN, MAX_MCP_SERVER_NAME_LEN, MAX_NAME_LEN, McpServerName, McpServerNameError, }; +pub use paths::{compute_ironclaw_base_dir, ironclaw_base_dir}; +pub use platform::PlatformInfo; pub use timezone::{ValidTimezone, deserialize_option_lenient}; -pub use util::truncate_preview; +pub use util::{truncate_for_preview, truncate_preview}; /// Maximum worker agent loop iterations. Used by the orchestrator (server-side /// clamp in `create_job_inner`) and the worker runtime (`worker/job.rs`). diff --git a/crates/ironclaw_common/src/paths.rs b/crates/ironclaw_common/src/paths.rs new file mode 100644 index 00000000000..81764742a45 --- /dev/null +++ b/crates/ironclaw_common/src/paths.rs @@ -0,0 +1,57 @@ +//! Shared filesystem path helpers. +//! +//! `ironclaw_base_dir()` resolves the IronClaw base directory used for env +//! files, session tokens, the libsql database, and other per-instance state. +//! Override with the `IRONCLAW_BASE_DIR` environment variable; defaults to +//! `~/.ironclaw`. + +use std::path::PathBuf; +use std::sync::LazyLock; + +const IRONCLAW_BASE_DIR_ENV: &str = "IRONCLAW_BASE_DIR"; + +static IRONCLAW_BASE_DIR: LazyLock = LazyLock::new(compute_ironclaw_base_dir); + +/// Compute the IronClaw base directory from the environment. +/// +/// Bypasses the `LazyLock` cache. Use this in tests that mutate +/// `IRONCLAW_BASE_DIR`; production callers should use [`ironclaw_base_dir`]. +pub fn compute_ironclaw_base_dir() -> PathBuf { + std::env::var(IRONCLAW_BASE_DIR_ENV) + .map(PathBuf::from) + .map(|path| { + if path.as_os_str().is_empty() { + default_base_dir() + } else if !path.is_absolute() { + eprintln!( + "Warning: IRONCLAW_BASE_DIR is a relative path '{}', resolved against current directory", + path.display() + ); + path + } else { + path + } + }) + .unwrap_or_else(|_| default_base_dir()) +} + +fn default_base_dir() -> PathBuf { + if let Some(home) = dirs::home_dir() { + home.join(".ironclaw") + } else { + eprintln!("Warning: Could not determine home directory, using current directory"); + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from("/tmp")) + .join(".ironclaw") + } +} + +/// Get the IronClaw base directory. +/// +/// Override with `IRONCLAW_BASE_DIR`. Defaults to `~/.ironclaw` (or +/// `./.ironclaw` if the home directory cannot be determined). +/// +/// Thread-safe: the value is computed once and cached in a `LazyLock`. +pub fn ironclaw_base_dir() -> PathBuf { + IRONCLAW_BASE_DIR.clone() +} diff --git a/crates/ironclaw_common/src/platform.rs b/crates/ironclaw_common/src/platform.rs new file mode 100644 index 00000000000..dd4940be4ba --- /dev/null +++ b/crates/ironclaw_common/src/platform.rs @@ -0,0 +1,59 @@ +//! Runtime platform metadata injected into system prompts for self-awareness. +//! +//! Provides the agent with knowledge about its own identity and environment +//! so it can answer questions about itself, its capabilities, and its +//! configuration without relying on training data. + +/// Runtime platform metadata. +#[derive(Debug, Clone, Default)] +pub struct PlatformInfo { + /// Software version (from `CARGO_PKG_VERSION`). + pub version: Option, + /// LLM backend name (e.g. "nearai", "openai", "anthropic"). + pub llm_backend: Option, + /// Active model name. + pub model_name: Option, + /// Database backend (e.g. "libsql", "postgres"). + pub database_backend: Option, + /// Active channel names (e.g. ["telegram", "cli"]). + pub active_channels: Vec, + /// Owner identifier. + pub owner_id: Option, + /// Project repository URL. + pub repo_url: Option, +} + +impl PlatformInfo { + /// Format as a prompt section. Returns just the identity line if no other + /// info is set. + pub fn to_prompt_section(&self) -> String { + let mut lines = Vec::new(); + + lines.push("You are **IronClaw**, a secure autonomous AI assistant platform.".into()); + if let Some(ref v) = self.version { + lines.push(format!("- Version: {v}")); + } + if let Some(ref repo) = self.repo_url { + lines.push(format!("- Repository: {repo}")); + } + if let Some(ref owner) = self.owner_id { + lines.push(format!("- Owner: {owner}")); + } + if let Some(ref backend) = self.llm_backend { + let model = self.model_name.as_deref().unwrap_or("default"); + lines.push(format!("- LLM: {backend} ({model})")); + } + if let Some(ref db) = self.database_backend { + lines.push(format!("- Database: {db}")); + } + if !self.active_channels.is_empty() { + lines.push(format!("- Channels: {}", self.active_channels.join(", "))); + } + + if lines.len() <= 1 { + return format!("\n\n## Platform\n\n{}\n", lines[0]); + } + + format!("\n\n## Platform\n\n{}\n", lines.join("\n")) + } +} diff --git a/crates/ironclaw_common/src/util.rs b/crates/ironclaw_common/src/util.rs index 4f054671d08..a552d875895 100644 --- a/crates/ironclaw_common/src/util.rs +++ b/crates/ironclaw_common/src/util.rs @@ -1,5 +1,31 @@ //! Shared utility functions. +/// Collapse a multi-line string into a single line and truncate to `max_chars` chars. +/// +/// Unlike `truncate_preview` (which works in bytes and preserves newlines for +/// XML payloads), this normalises whitespace and works in chars — suitable for +/// log lines that should fit on a single screen row. +pub fn truncate_for_preview(output: &str, max_chars: usize) -> String { + let collapsed: String = output + .chars() + .take(max_chars + 50) + .map(|c| if c == '\n' { ' ' } else { c }) + .collect::() + .split_whitespace() + .collect::>() + .join(" "); + if collapsed.chars().count() > max_chars { + let byte_offset = collapsed + .char_indices() + .nth(max_chars) + .map(|(i, _)| i) + .unwrap_or(collapsed.len()); + format!("{}...", &collapsed[..byte_offset]) + } else { + collapsed + } +} + /// Truncate a string to at most `max_bytes` bytes at a char boundary, appending "...". /// /// If the input is wrapped in `...` and truncation diff --git a/crates/ironclaw_embeddings/AGENTS.md b/crates/ironclaw_embeddings/AGENTS.md new file mode 100644 index 00000000000..5ed0c838cc1 --- /dev/null +++ b/crates/ironclaw_embeddings/AGENTS.md @@ -0,0 +1,51 @@ +# ironclaw_embeddings + +Owns the shared embedding-provider trait, concrete provider impls, the LRU caching decorator, and the async factory used by everything in the workspace that needs vector embeddings. + +## Responsibilities + +- Define `EmbeddingProvider`, `EmbeddingError`, and the pure-data `EmbeddingsConfig` shape that callers fill from their own settings layer. +- House concrete provider impls: OpenAI / OpenAI-compatible, NEAR AI, Ollama, and AWS Bedrock (gated behind the `bedrock` cargo feature). +- Provide the `create_provider(config, deps)` async factory — the only supported way to construct a provider. +- Wrap an `Arc` in `CachedEmbeddingProvider` (LRU) via `EmbeddingCacheConfig` for callers that want hot-path caching. +- Run a baseline defense-in-depth URL check (`url_check::check_base_url`) inside the factory: reject non-http(s) schemes, unparseable URLs, and the AlwaysBlocked IP class (cloud-metadata `169.254.169.254`, link-local, multicast, `0.0.0.0`/`::`). +- Expose `default_dimension_for_model` so the binary can pick a dimension without hard-coding the table. + +## Non-responsibilities + +- Do not read `Settings`, env vars, or DB rows. The binary-side resolver at `src/config/embeddings.rs::resolve_embeddings_config` owns that — it produces an `EmbeddingsConfig` and hands it to the factory. +- Do not implement the full operator-tunable SSRF policy. `validate_operator_base_url` in `src/config/helpers.rs` is the policy-aware layer with allow/deny lists and DNS resolution; the crate's `url_check::check_base_url` is only the AlwaysBlocked-class floor. +- Do not decide cache size, whether to cache, or how to wire the provider into a workspace. Callers choose via `Workspace::with_embeddings_cached` / `with_embeddings_uncached`. +- Do not expose concrete provider constructors. `OpenAiEmbeddings`, `NearAiEmbeddings`, `OllamaEmbeddings`, `BedrockEmbeddings`, and `MockEmbeddings` stay crate-private; downstream code holds `Arc` only. +- Do not perform background work, retries, or circuit-breaking. Providers are thin clients; resilience belongs to the caller. + +## Public surface + +| Symbol | Use | +|--------|-----| +| `EmbeddingProvider` trait, `EmbeddingError` | Trait object + error returned by every provider | +| `create_provider(config, deps) -> Option>` | The factory. Returns `None` when embeddings are disabled or the resolved config can't yield a working provider | +| `ProviderDeps { session, bedrock_setup }` | Runtime wiring the factory needs that doesn't live in `EmbeddingsConfig` | +| `EmbeddingsConfig`, `DEFAULT_EMBEDDING_CACHE_SIZE`, `default_dimension_for_model` | Pure-data config + helpers consumed by the binary's resolver | +| `CachedEmbeddingProvider`, `EmbeddingCacheConfig` | LRU caching decorator | +| `BedrockEmbeddingSetup` | Compiled unconditionally so callers can build one without the `bedrock` feature; the underlying impl is `#[cfg(feature = "bedrock")]` | +| `MockEmbeddings` | Deterministic test double, gated behind the `testing` cargo feature | + +## Safety rules + +- Concrete provider constructors stay crate-private. New providers are reached only through `create_provider`; this is the single security boundary all callers must traverse. +- Any provider that takes a base URL MUST call `url_check::check_base_url` in its factory match arm before its constructor runs. The crate-level check is the AlwaysBlocked floor; do not omit it on the assumption that a downstream resolver will catch it — a caller constructing `EmbeddingsConfig` directly skips the resolver entirely. +- `max_input_length` measures bytes (matches `str::len()`), not characters. Keep the trait docs, each impl's inline comment, and any `text.len()` truncation logic consistent. +- `embed_batch` overrides on each provider must validate every input against `max_input_length()` before issuing the request — the caller-level `embed()` length check does not run for batches. +- `EmbeddingsConfig` is plain data with no construction-time validation of base URLs. The factory is the only enforcement point; do not move URL validation into `EmbeddingsConfig::new` (there isn't one) or onto `Deserialize`, or two construction paths will drift. +- The `testing` and `bedrock` features must be additive only — the default build must continue to compile and run without either. + +## Where the binary plugs in + +The wiring lives outside this crate: + +- `src/config/embeddings.rs::resolve_embeddings_config` reads `Settings` + env vars, runs the operator-tunable SSRF policy, and returns an `EmbeddingsConfig`. +- `src/app.rs` (and `src/cli/mod.rs`) call `ironclaw_embeddings::create_provider(&cfg, ProviderDeps { session, bedrock_setup })` and attach the result to `Workspace` via `with_embeddings_cached`. +- `src/cli/doctor.rs::check_embeddings` reports configuration status and credential presence per provider. + +When changing this crate's public surface, grep those three call sites first. diff --git a/crates/ironclaw_embeddings/CLAUDE.md b/crates/ironclaw_embeddings/CLAUDE.md new file mode 100644 index 00000000000..fded5dcf773 --- /dev/null +++ b/crates/ironclaw_embeddings/CLAUDE.md @@ -0,0 +1 @@ +See [AGENTS.md](./AGENTS.md). diff --git a/crates/ironclaw_embeddings/Cargo.toml b/crates/ironclaw_embeddings/Cargo.toml new file mode 100644 index 00000000000..af2099720d1 --- /dev/null +++ b/crates/ironclaw_embeddings/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "ironclaw_embeddings" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +description = "Embedding-provider trait and implementations (OpenAI, NearAI, Ollama, AWS Bedrock) with LRU caching" +authors = ["NEAR AI "] +license = "MIT OR Apache-2.0" +homepage = "https://github.com/nearai/ironclaw" +repository = "https://github.com/nearai/ironclaw" +publish = false + +[package.metadata.dist] +dist = false + +[features] +default = [] +bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"] +# Exposes `MockEmbeddings` for downstream test harnesses. Off in release +# builds so the deterministic-hash mock can't be reached from production +# code paths. +testing = [] + +[dependencies] +ironclaw_llm = { path = "../ironclaw_llm", version = "0.1.0" } + +async-trait = "0.1" +lru = "0.16.3" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } +secrecy = { version = "0.10", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" +tracing = "0.1" + +aws-config = { version = "1", features = ["behavior-version-latest"], optional = true } +aws-sdk-bedrockruntime = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio", "behavior-version-latest"], optional = true } +aws-smithy-types = { version = "1", optional = true } + +[dev-dependencies] +tokio = { version = "1", features = ["full", "test-util"] } diff --git a/crates/ironclaw_embeddings/src/bedrock.rs b/crates/ironclaw_embeddings/src/bedrock.rs new file mode 100644 index 00000000000..9c8f7d5e20f --- /dev/null +++ b/crates/ironclaw_embeddings/src/bedrock.rs @@ -0,0 +1,168 @@ +//! AWS Bedrock embedding provider (Titan Text Embeddings V2). +//! +//! [`BedrockEmbeddingSetup`] (region + profile) is compiled +//! unconditionally so callers can construct it without depending on the +//! `bedrock` feature flag — when the feature is off the factory just +//! ignores it. Only the provider `impl` itself (the `imp` submodule +//! below) is gated on `#[cfg(feature = "bedrock")]`, because it pulls +//! in the heavy AWS SDK dependencies. + +/// AWS Bedrock parameters needed by the embedding provider. +/// +/// Defined here rather than re-using `ironclaw_llm::BedrockConfig` so the +/// embeddings layer does not couple to LLM-side config types. Callers +/// (which already hold an `LlmConfig`) translate at the boundary. +#[derive(Debug, Clone)] +pub struct BedrockEmbeddingSetup { + pub region: String, + pub profile: Option, +} + +#[cfg(feature = "bedrock")] +mod imp { + use async_trait::async_trait; + use serde::{Deserialize, Serialize}; + + use crate::provider::{EmbeddingError, EmbeddingProvider}; + + use super::BedrockEmbeddingSetup; + + /// AWS Bedrock embedding provider using Titan Text Embeddings V2. + pub(crate) struct BedrockEmbeddings { + client: aws_sdk_bedrockruntime::Client, + model: String, + dimension: usize, + } + + impl BedrockEmbeddings { + /// Create a new Bedrock embedding provider. + pub(crate) async fn new( + setup: &BedrockEmbeddingSetup, + model: impl Into, + dimension: usize, + ) -> Result { + let mut builder = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(aws_config::Region::new(setup.region.clone())); + if let Some(ref profile) = setup.profile { + builder = builder.profile_name(profile); + } + + let sdk_config = builder.load().await; + Ok(Self { + client: aws_sdk_bedrockruntime::Client::new(&sdk_config), + model: model.into(), + dimension, + }) + } + } + + #[derive(Debug, Serialize)] + struct BedrockTitanEmbeddingRequest<'a> { + #[serde(rename = "inputText")] + input_text: &'a str, + dimensions: usize, + normalize: bool, + } + + #[derive(Debug, Deserialize)] + struct BedrockTitanEmbeddingResponse { + embedding: Vec, + } + + fn map_bedrock_invoke_model_error( + error: &aws_sdk_bedrockruntime::error::SdkError< + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError, + R, + >, + ) -> EmbeddingError { + use aws_sdk_bedrockruntime::error::SdkError; + use aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError; + + match error { + SdkError::ServiceError(service_err) => match service_err.err() { + InvokeModelError::ThrottlingException(_) => { + EmbeddingError::RateLimited { retry_after: None } + } + InvokeModelError::AccessDeniedException(_) => EmbeddingError::AuthFailed, + InvokeModelError::ValidationException(e) => { + EmbeddingError::InvalidResponse(format!( + "Bedrock validation error: {}", + e.message().unwrap_or("unknown") + )) + } + InvokeModelError::ModelNotReadyException(e) => EmbeddingError::HttpError(format!( + "Bedrock model not ready: {}", + e.message().unwrap_or("unknown") + )), + other => EmbeddingError::HttpError(format!("Bedrock service error: {other:?}")), + }, + SdkError::TimeoutError(_) => { + EmbeddingError::HttpError("Bedrock request timed out".to_string()) + } + other => EmbeddingError::HttpError(format!("Bedrock request failed: {other:?}")), + } + } + + #[async_trait] + impl EmbeddingProvider for BedrockEmbeddings { + fn dimension(&self) -> usize { + self.dimension + } + + fn model_name(&self) -> &str { + &self.model + } + + fn max_input_length(&self) -> usize { + 32_000 + } + + async fn embed(&self, text: &str) -> Result, EmbeddingError> { + if text.len() > self.max_input_length() { + return Err(EmbeddingError::TextTooLong { + length: text.len(), + max: self.max_input_length(), + }); + } + + let request = BedrockTitanEmbeddingRequest { + input_text: text, + dimensions: self.dimension, + normalize: true, + }; + + let body = serde_json::to_vec(&request).map_err(|e| { + EmbeddingError::InvalidResponse(format!("Failed to serialize request: {}", e)) + })?; + + let response = self + .client + .invoke_model() + .model_id(&self.model) + .content_type("application/json") + .accept("application/json") + .body(aws_smithy_types::Blob::new(body)) + .send() + .await + .map_err(|e| map_bedrock_invoke_model_error(&e))?; + + let result: BedrockTitanEmbeddingResponse = + serde_json::from_slice(response.body.as_ref()).map_err(|e| { + EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) + })?; + + if result.embedding.len() != self.dimension { + return Err(EmbeddingError::InvalidResponse(format!( + "Bedrock returned embedding of dimension {}, expected {}", + result.embedding.len(), + self.dimension, + ))); + } + + Ok(result.embedding) + } + } +} + +#[cfg(feature = "bedrock")] +pub(crate) use imp::BedrockEmbeddings; diff --git a/src/workspace/embedding_cache.rs b/crates/ironclaw_embeddings/src/cache.rs similarity index 99% rename from src/workspace/embedding_cache.rs rename to crates/ironclaw_embeddings/src/cache.rs index 60c2eb08656..4f22fc352cf 100644 --- a/src/workspace/embedding_cache.rs +++ b/crates/ironclaw_embeddings/src/cache.rs @@ -12,7 +12,8 @@ use async_trait::async_trait; use lru::LruCache; use sha2::{Digest, Sha256}; -use crate::workspace::embeddings::{EmbeddingError, EmbeddingProvider}; +use crate::config::DEFAULT_EMBEDDING_CACHE_SIZE; +use crate::provider::{EmbeddingError, EmbeddingProvider}; /// Configuration for the embedding cache. #[derive(Debug, Clone)] @@ -28,7 +29,7 @@ pub struct EmbeddingCacheConfig { impl Default for EmbeddingCacheConfig { fn default() -> Self { Self { - max_entries: crate::config::DEFAULT_EMBEDDING_CACHE_SIZE, + max_entries: DEFAULT_EMBEDDING_CACHE_SIZE, } } } diff --git a/crates/ironclaw_embeddings/src/config.rs b/crates/ironclaw_embeddings/src/config.rs new file mode 100644 index 00000000000..5894ac8139a --- /dev/null +++ b/crates/ironclaw_embeddings/src/config.rs @@ -0,0 +1,83 @@ +//! Embeddings provider configuration (resolved from binary-side `Settings`). +//! +//! The resolver that reads `Settings` lives in the binary +//! (`src/config/embeddings.rs::resolve_embeddings_config`); this crate only +//! owns the resolved data shape and helpers that depend on nothing but the +//! shape itself. + +use secrecy::{ExposeSecret, SecretString}; + +/// Default maximum number of cached embeddings. +pub const DEFAULT_EMBEDDING_CACHE_SIZE: usize = 10_000; + +/// Embeddings provider configuration. +#[derive(Debug, Clone)] +pub struct EmbeddingsConfig { + /// Whether embeddings are enabled. + pub enabled: bool, + /// Provider to use: "openai", "nearai", "ollama", or "bedrock" + pub provider: String, + /// OpenAI API key (for OpenAI provider). + pub openai_api_key: Option, + /// Model to use for embeddings. + pub model: String, + /// Ollama base URL (for Ollama provider). Defaults to http://localhost:11434. + pub ollama_base_url: String, + /// Embedding vector dimension. Inferred from the model name when not set explicitly. + pub dimension: usize, + /// Custom base URL for OpenAI-compatible embedding providers. + /// When set, overrides the default `https://api.openai.com`. + pub openai_base_url: Option, + /// Base URL for the NEAR AI embeddings endpoint. + /// + /// Copied from `LlmConfig::nearai::base_url` by the resolver so embeddings + /// share the LLM's NEAR AI endpoint. Only consulted when `provider == "nearai"`. + pub nearai_base_url: String, + /// Maximum entries in the embedding LRU cache (default 10,000). + /// + /// Approximate raw embedding payload: `cache_size × dimension × 4 bytes`. + /// 10,000 × 1536 floats ≈ 58 MB (payload only; actual memory is higher + /// due to HashMap buckets, per-entry Vec/timestamp overhead). + pub cache_size: usize, +} + +impl Default for EmbeddingsConfig { + fn default() -> Self { + let model = "text-embedding-3-small".to_string(); + let dimension = default_dimension_for_model(&model); + Self { + enabled: false, + provider: "openai".to_string(), + openai_api_key: None, + model, + ollama_base_url: "http://localhost:11434".to_string(), + dimension, + openai_base_url: None, + nearai_base_url: "https://api.near.ai".to_string(), + cache_size: DEFAULT_EMBEDDING_CACHE_SIZE, + } + } +} + +impl EmbeddingsConfig { + /// Get the OpenAI API key if configured. + pub fn openai_api_key(&self) -> Option<&str> { + self.openai_api_key.as_ref().map(|s| s.expose_secret()) + } +} + +/// Infer the embedding dimension from a well-known model name. +/// +/// Falls back to 1536 (OpenAI text-embedding-3-small default) for unknown models. +pub fn default_dimension_for_model(model: &str) -> usize { + match model { + "text-embedding-3-small" => 1536, + "text-embedding-3-large" => 3072, + "text-embedding-ada-002" => 1536, + "amazon.titan-embed-text-v2:0" => 1024, + "nomic-embed-text" => 768, + "mxbai-embed-large" => 1024, + "all-minilm" => 384, + _ => 1536, + } +} diff --git a/crates/ironclaw_embeddings/src/factory.rs b/crates/ironclaw_embeddings/src/factory.rs new file mode 100644 index 00000000000..c1660cb373f --- /dev/null +++ b/crates/ironclaw_embeddings/src/factory.rs @@ -0,0 +1,229 @@ +//! Async factory that builds the configured [`EmbeddingProvider`]. + +use std::sync::Arc; + +use ironclaw_llm::SessionManager; + +use crate::bedrock::BedrockEmbeddingSetup; +use crate::config::EmbeddingsConfig; +use crate::nearai::NearAiEmbeddings; +use crate::ollama::OllamaEmbeddings; +use crate::openai::OpenAiEmbeddings; +use crate::provider::EmbeddingProvider; +use crate::url_check::check_base_url; + +/// Runtime wiring the factory needs that doesn't fit in [`EmbeddingsConfig`]. +/// +/// `EmbeddingsConfig` is pure data (Debug/Clone, populated from `Settings`). +/// These are shared runtime objects supplied by the host and consulted only +/// by the matching provider — `session` for `nearai`, `bedrock_setup` for +/// `bedrock`. Construct once at startup and pass into [`create_provider`]. +#[derive(Clone)] +pub struct ProviderDeps { + pub session: Arc, + pub bedrock_setup: Option, +} + +/// Build the configured embedding provider. +/// +/// Returns `None` if embeddings are disabled or required credentials are +/// missing. +pub async fn create_provider( + config: &EmbeddingsConfig, + deps: ProviderDeps, +) -> Option> { + if !config.enabled { + tracing::debug!("Embeddings disabled (set EMBEDDING_ENABLED=true to enable)"); + return None; + } + + match config.provider.as_str() { + "nearai" => { + if let Err(e) = check_base_url(&config.nearai_base_url, "nearai_base_url") { + tracing::warn!("Refusing to build NEAR AI embeddings: {e}"); + return None; + } + tracing::debug!( + "Embeddings enabled via NEAR AI (model: {}, dim: {})", + config.model, + config.dimension, + ); + Some(Arc::new( + NearAiEmbeddings::new(&config.nearai_base_url, deps.session) + .with_model(&config.model, config.dimension), + ) as Arc) + } + "bedrock" => { + #[cfg(feature = "bedrock")] + { + let Some(bedrock) = deps.bedrock_setup.as_ref() else { + tracing::warn!( + "Embeddings configured for Bedrock but no Bedrock setup is available" + ); + return None; + }; + tracing::debug!( + "Embeddings enabled via Bedrock (model: {}, region: {}, dim: {})", + config.model, + bedrock.region, + config.dimension, + ); + match crate::bedrock::BedrockEmbeddings::new( + bedrock, + &config.model, + config.dimension, + ) + .await + { + Ok(provider) => Some(Arc::new(provider) as Arc), + Err(e) => { + tracing::warn!("Failed to initialize Bedrock embeddings provider: {e}"); + None + } + } + } + #[cfg(not(feature = "bedrock"))] + { + let _ = deps.bedrock_setup; + tracing::warn!( + "Embeddings configured for Bedrock but the `bedrock` feature is disabled" + ); + None + } + } + "ollama" => { + if let Err(e) = check_base_url(&config.ollama_base_url, "ollama_base_url") { + tracing::warn!("Refusing to build Ollama embeddings: {e}"); + return None; + } + tracing::debug!( + "Embeddings enabled via Ollama (model: {}, url: {}, dim: {})", + config.model, + config.ollama_base_url, + config.dimension, + ); + Some(Arc::new( + OllamaEmbeddings::new(&config.ollama_base_url) + .with_model(&config.model, config.dimension), + ) as Arc) + } + _ => { + if let Some(api_key) = config.openai_api_key() { + let mut provider = + OpenAiEmbeddings::with_model(api_key, &config.model, config.dimension); + if let Some(ref base_url) = config.openai_base_url { + if let Err(e) = check_base_url(base_url, "openai_base_url") { + tracing::warn!("Refusing to build OpenAI embeddings: {e}"); + return None; + } + tracing::debug!( + "Embeddings enabled via OpenAI (model: {}, base_url: {}, dim: {})", + config.model, + base_url, + config.dimension, + ); + provider = provider.with_base_url(base_url); + } else { + tracing::debug!( + "Embeddings enabled via OpenAI (model: {}, dim: {})", + config.model, + config.dimension, + ); + } + Some(Arc::new(provider) as Arc) + } else { + tracing::warn!("Embeddings configured but OPENAI_API_KEY not set"); + None + } + } + } +} + +#[cfg(test)] +mod tests { + //! Regression coverage for the public factory + config surface: anyone + //! that constructs `EmbeddingsConfig` directly and calls + //! `create_provider` must hit the baseline URL check before any HTTP + //! work happens. See PR #3739 review (P1). + use super::*; + use crate::config::EmbeddingsConfig; + use ironclaw_llm::{SessionConfig, SessionManager}; + use secrecy::SecretString; + + fn stub_deps() -> ProviderDeps { + ProviderDeps { + session: Arc::new(SessionManager::new(SessionConfig::default())), + bedrock_setup: None, + } + } + + fn config_with_provider(provider: &str) -> EmbeddingsConfig { + EmbeddingsConfig { + enabled: true, + provider: provider.to_string(), + ..EmbeddingsConfig::default() + } + } + + #[tokio::test] + async fn rejects_blocked_ollama_base_url() { + let cfg = EmbeddingsConfig { + ollama_base_url: "https://169.254.169.254".to_string(), + ..config_with_provider("ollama") + }; + let provider = create_provider(&cfg, stub_deps()).await; + assert!( + provider.is_none(), + "Ollama provider must not be built with cloud-metadata IP" + ); + } + + #[tokio::test] + async fn rejects_blocked_nearai_base_url() { + let cfg = EmbeddingsConfig { + nearai_base_url: "https://169.254.169.254".to_string(), + ..config_with_provider("nearai") + }; + let provider = create_provider(&cfg, stub_deps()).await; + assert!( + provider.is_none(), + "NEAR AI provider must not be built with cloud-metadata IP" + ); + } + + #[tokio::test] + async fn rejects_blocked_openai_base_url() { + let cfg = EmbeddingsConfig { + openai_api_key: Some(SecretString::from("sk-stub".to_string())), + openai_base_url: Some("https://169.254.169.254".to_string()), + ..config_with_provider("openai") + }; + let provider = create_provider(&cfg, stub_deps()).await; + assert!( + provider.is_none(), + "OpenAI-compatible provider must not be built with cloud-metadata IP" + ); + } + + #[tokio::test] + async fn rejects_non_http_scheme() { + let cfg = EmbeddingsConfig { + ollama_base_url: "file:///etc/passwd".to_string(), + ..config_with_provider("ollama") + }; + assert!(create_provider(&cfg, stub_deps()).await.is_none()); + } + + #[tokio::test] + async fn accepts_localhost_ollama() { + let cfg = EmbeddingsConfig { + ollama_base_url: "http://localhost:11434".to_string(), + ..config_with_provider("ollama") + }; + let provider = create_provider(&cfg, stub_deps()).await; + assert!( + provider.is_some(), + "loopback Ollama is a legitimate operator endpoint" + ); + } +} diff --git a/crates/ironclaw_embeddings/src/lib.rs b/crates/ironclaw_embeddings/src/lib.rs new file mode 100644 index 00000000000..5265a830849 --- /dev/null +++ b/crates/ironclaw_embeddings/src/lib.rs @@ -0,0 +1,30 @@ +//! Embedding-provider trait + caching decorator. +//! +//! Concrete provider implementations (OpenAI, NEAR AI, Ollama, AWS Bedrock) +//! are crate-internal — construct one through [`create_provider`] using +//! [`EmbeddingsConfig`] + [`ProviderDeps`]. Callers should only ever hold +//! `Arc`. +//! +//! The resolver that reads the binary-side `Settings` lives in +//! `src/config/embeddings.rs::resolve_embeddings_config`; everything else +//! (trait, error, config shape, cache, factory, providers) lives here. + +mod bedrock; +mod cache; +mod config; +mod factory; +#[cfg(any(test, feature = "testing"))] +mod mock; +mod nearai; +mod ollama; +mod openai; +mod provider; +mod url_check; + +pub use bedrock::BedrockEmbeddingSetup; +pub use cache::{CachedEmbeddingProvider, EmbeddingCacheConfig}; +pub use config::{DEFAULT_EMBEDDING_CACHE_SIZE, EmbeddingsConfig, default_dimension_for_model}; +pub use factory::{ProviderDeps, create_provider}; +#[cfg(any(test, feature = "testing"))] +pub use mock::MockEmbeddings; +pub use provider::{EmbeddingError, EmbeddingProvider}; diff --git a/crates/ironclaw_embeddings/src/mock.rs b/crates/ironclaw_embeddings/src/mock.rs new file mode 100644 index 00000000000..1c00fb5121d --- /dev/null +++ b/crates/ironclaw_embeddings/src/mock.rs @@ -0,0 +1,105 @@ +//! Deterministic mock embedding provider for tests. + +use async_trait::async_trait; + +use crate::provider::{EmbeddingError, EmbeddingProvider}; + +/// A mock embedding provider for testing. +/// +/// Generates deterministic embeddings based on text hash. +/// Useful for unit and integration tests. +pub struct MockEmbeddings { + dimension: usize, +} + +impl MockEmbeddings { + /// Create a new mock embeddings provider with the given dimension. + pub fn new(dimension: usize) -> Self { + Self { dimension } + } +} + +#[async_trait] +impl EmbeddingProvider for MockEmbeddings { + fn dimension(&self) -> usize { + self.dimension + } + + fn model_name(&self) -> &str { + "mock-embedding" + } + + fn max_input_length(&self) -> usize { + 10_000 + } + + async fn embed(&self, text: &str) -> Result, EmbeddingError> { + // Generate a deterministic embedding based on text hash + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut hasher); + let hash = hasher.finish(); + + let mut embedding = Vec::with_capacity(self.dimension); + let mut seed = hash; + for _ in 0..self.dimension { + // Simple LCG for deterministic random values + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + let value = (seed as f32 / u64::MAX as f32) * 2.0 - 1.0; + embedding.push(value); + } + + // Normalize to unit length + let magnitude: f32 = embedding.iter().map(|x| x * x).sum::().sqrt(); + if magnitude > 0.0 { + for x in &mut embedding { + *x /= magnitude; + } + } + + Ok(embedding) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_mock_embeddings() { + let provider = MockEmbeddings::new(128); + + let embedding = provider.embed("hello world").await.unwrap(); + assert_eq!(embedding.len(), 128); + + // Check normalization (should be unit vector) + let magnitude: f32 = embedding.iter().map(|x| x * x).sum::().sqrt(); + assert!((magnitude - 1.0).abs() < 0.001); + } + + #[tokio::test] + async fn test_mock_embeddings_deterministic() { + let provider = MockEmbeddings::new(64); + + let emb1 = provider.embed("test").await.unwrap(); + let emb2 = provider.embed("test").await.unwrap(); + + // Same input should produce same embedding + assert_eq!(emb1, emb2); + } + + #[tokio::test] + async fn test_mock_embeddings_batch() { + let provider = MockEmbeddings::new(64); + + let texts = vec!["hello".to_string(), "world".to_string()]; + let embeddings = provider.embed_batch(&texts).await.unwrap(); + + assert_eq!(embeddings.len(), 2); + assert_eq!(embeddings[0].len(), 64); + assert_eq!(embeddings[1].len(), 64); + + // Different texts should produce different embeddings + assert_ne!(embeddings[0], embeddings[1]); + } +} diff --git a/crates/ironclaw_embeddings/src/nearai.rs b/crates/ironclaw_embeddings/src/nearai.rs new file mode 100644 index 00000000000..e0044ac27d7 --- /dev/null +++ b/crates/ironclaw_embeddings/src/nearai.rs @@ -0,0 +1,144 @@ +//! NEAR AI embedding provider. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::provider::{EmbeddingError, EmbeddingProvider}; + +/// NEAR AI embedding provider using the NEAR AI API. +/// +/// Uses the same session-based auth as the LLM provider. +pub(crate) struct NearAiEmbeddings { + client: reqwest::Client, + base_url: String, + session: std::sync::Arc, + model: String, + dimension: usize, +} + +impl NearAiEmbeddings { + /// Create a new NEAR AI embedding provider. + /// + /// Uses the same session manager as the LLM provider for auth. + pub(crate) fn new( + base_url: impl Into, + session: std::sync::Arc, + ) -> Self { + Self { + client: reqwest::Client::new(), + base_url: base_url.into(), + session, + model: "text-embedding-3-small".to_string(), + dimension: 1536, + } + } + + /// Use a specific model. + pub(crate) fn with_model(mut self, model: impl Into, dimension: usize) -> Self { + self.model = model.into(); + self.dimension = dimension; + self + } +} + +#[derive(Debug, Serialize)] +struct NearAiEmbeddingRequest<'a> { + model: &'a str, + input: &'a [String], +} + +#[derive(Debug, Deserialize)] +struct NearAiEmbeddingResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct NearAiEmbeddingData { + embedding: Vec, +} + +#[async_trait] +impl EmbeddingProvider for NearAiEmbeddings { + fn dimension(&self) -> usize { + self.dimension + } + + fn model_name(&self) -> &str { + &self.model + } + + fn max_input_length(&self) -> usize { + 32_000 + } + + async fn embed(&self, text: &str) -> Result, EmbeddingError> { + if text.len() > self.max_input_length() { + return Err(EmbeddingError::TextTooLong { + length: text.len(), + max: self.max_input_length(), + }); + } + + let embeddings = self.embed_batch(&[text.to_string()]).await?; + embeddings + .into_iter() + .next() + .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) + } + + async fn embed_batch(&self, texts: &[String]) -> Result>, EmbeddingError> { + use secrecy::ExposeSecret; + + if texts.is_empty() { + return Ok(Vec::new()); + } + + let request = NearAiEmbeddingRequest { + model: &self.model, + input: texts, + }; + + let token = self + .session + .get_token() + .await + .map_err(|_| EmbeddingError::AuthFailed)?; + + let url = format!("{}/v1/embeddings", self.base_url); + + let response = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", token.expose_secret())) + .json(&request) + .send() + .await?; + + let status = response.status(); + + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(EmbeddingError::AuthFailed); + } + + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + let retry_after = Some(ironclaw_llm::retry::parse_retry_after( + response.headers().get("retry-after"), + )); + return Err(EmbeddingError::RateLimited { retry_after }); + } + + if !status.is_success() { + let error_text = response.text().await.unwrap_or_default(); + return Err(EmbeddingError::HttpError(format!( + "Status {}: {}", + status, error_text + ))); + } + + let result: NearAiEmbeddingResponse = response.json().await.map_err(|e| { + EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) + })?; + + Ok(result.data.into_iter().map(|d| d.embedding).collect()) + } +} diff --git a/crates/ironclaw_embeddings/src/ollama.rs b/crates/ironclaw_embeddings/src/ollama.rs new file mode 100644 index 00000000000..fe523a0385a --- /dev/null +++ b/crates/ironclaw_embeddings/src/ollama.rs @@ -0,0 +1,125 @@ +//! Ollama embedding provider (local Ollama HTTP API). + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::provider::{EmbeddingError, EmbeddingProvider}; + +/// Ollama embedding provider using a local Ollama instance. +/// +/// Ollama serves embedding models (e.g. `nomic-embed-text`, `mxbai-embed-large`) +/// via a REST API, typically at `http://localhost:11434`. +pub(crate) struct OllamaEmbeddings { + client: reqwest::Client, + base_url: String, + model: String, + dimension: usize, +} + +impl OllamaEmbeddings { + /// Create a new Ollama embedding provider. + /// + /// Defaults to `nomic-embed-text` (768 dimensions). + pub(crate) fn new(base_url: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + base_url: base_url.into(), + model: "nomic-embed-text".to_string(), + dimension: 768, + } + } + + /// Use a specific model with a given dimension. + pub(crate) fn with_model(mut self, model: impl Into, dimension: usize) -> Self { + self.model = model.into(); + self.dimension = dimension; + self + } +} + +#[derive(Debug, Serialize)] +struct OllamaEmbedRequest<'a> { + model: &'a str, + input: &'a [String], +} + +#[derive(Debug, Deserialize)] +struct OllamaEmbedResponse { + embeddings: Vec>, +} + +#[async_trait] +impl EmbeddingProvider for OllamaEmbeddings { + fn dimension(&self) -> usize { + self.dimension + } + + fn model_name(&self) -> &str { + &self.model + } + + fn max_input_length(&self) -> usize { + // Most Ollama embedding models support ~8192 tokens, budgeted + // here as ~32_000 UTF-8 bytes (matches `str::len()` semantics — + // see `EmbeddingProvider::max_input_length` doc). + 32_000 + } + + async fn embed(&self, text: &str) -> Result, EmbeddingError> { + if text.len() > self.max_input_length() { + return Err(EmbeddingError::TextTooLong { + length: text.len(), + max: self.max_input_length(), + }); + } + + let embeddings = self.embed_batch(&[text.to_string()]).await?; + embeddings + .into_iter() + .next() + .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) + } + + async fn embed_batch(&self, texts: &[String]) -> Result>, EmbeddingError> { + if texts.is_empty() { + return Ok(Vec::new()); + } + + let request = OllamaEmbedRequest { + model: &self.model, + input: texts, + }; + + let url = format!("{}/api/embed", self.base_url); + + let response = self.client.post(&url).json(&request).send().await?; + + let status = response.status(); + + if !status.is_success() { + let error_text = response.text().await.unwrap_or_default(); + return Err(EmbeddingError::HttpError(format!( + "Ollama returned HTTP {}: {}", + status, error_text + ))); + } + + let result: OllamaEmbedResponse = response.json().await.map_err(|e| { + EmbeddingError::InvalidResponse(format!("Failed to parse Ollama response: {}", e)) + })?; + + // Validate that returned embeddings match the configured dimension. + for (i, emb) in result.embeddings.iter().enumerate() { + if emb.len() != self.dimension { + return Err(EmbeddingError::InvalidResponse(format!( + "Ollama returned embedding of dimension {}, expected {} at index {}", + emb.len(), + self.dimension, + i + ))); + } + } + + Ok(result.embeddings) + } +} diff --git a/crates/ironclaw_embeddings/src/openai.rs b/crates/ironclaw_embeddings/src/openai.rs new file mode 100644 index 00000000000..a3e68a54ece --- /dev/null +++ b/crates/ironclaw_embeddings/src/openai.rs @@ -0,0 +1,242 @@ +//! OpenAI embedding provider (also used for any OpenAI-compatible endpoint). + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::provider::{EmbeddingError, EmbeddingProvider}; + +/// Default base URL for the OpenAI API. +const OPENAI_API_BASE_URL: &str = "https://api.openai.com"; + +/// OpenAI embedding provider using text-embedding-ada-002 or text-embedding-3-small. +/// +/// Supports any OpenAI-compatible embedding endpoint via [`with_base_url`](Self::with_base_url). +pub(crate) struct OpenAiEmbeddings { + client: reqwest::Client, + api_key: String, + model: String, + dimension: usize, + base_url: String, +} + +impl OpenAiEmbeddings { + /// Create a new OpenAI embedding provider with the default model. + /// + /// Uses text-embedding-3-small which has 1536 dimensions. + #[allow(dead_code)] + pub(crate) fn new(api_key: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + api_key: api_key.into(), + model: "text-embedding-3-small".to_string(), + dimension: 1536, + base_url: OPENAI_API_BASE_URL.to_string(), + } + } + + /// Use text-embedding-ada-002 model. + #[allow(dead_code)] + pub(crate) fn ada_002(api_key: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + api_key: api_key.into(), + model: "text-embedding-ada-002".to_string(), + dimension: 1536, + base_url: OPENAI_API_BASE_URL.to_string(), + } + } + + /// Use text-embedding-3-large model. + #[allow(dead_code)] + pub(crate) fn large(api_key: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + api_key: api_key.into(), + model: "text-embedding-3-large".to_string(), + dimension: 3072, + base_url: OPENAI_API_BASE_URL.to_string(), + } + } + + /// Use a custom model with specified dimension. + pub(crate) fn with_model( + api_key: impl Into, + model: impl Into, + dimension: usize, + ) -> Self { + Self { + client: reqwest::Client::new(), + api_key: api_key.into(), + model: model.into(), + dimension, + base_url: OPENAI_API_BASE_URL.to_string(), + } + } + + /// Set a custom base URL for OpenAI-compatible embedding providers. + /// + /// The URL must use `http://` or `https://` scheme. If no scheme is present, + /// `https://` is prepended automatically. Trailing slashes are stripped. + pub(crate) fn with_base_url(mut self, base_url: &str) -> Self { + let url = base_url.trim(); + + // Auto-prepend https:// if no scheme is present. + let mut url = if !url.starts_with("http://") && !url.starts_with("https://") { + tracing::debug!( + "No scheme in embedding base URL '{}', prepending https://", + url + ); + format!("https://{url}") + } else { + url.to_string() + }; + + while url.ends_with('/') { + url.pop(); + } + + self.base_url = url; + self + } +} + +#[derive(Debug, Serialize)] +struct OpenAiEmbeddingRequest<'a> { + model: &'a str, + input: &'a [String], +} + +#[derive(Debug, Deserialize)] +struct OpenAiEmbeddingResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct OpenAiEmbeddingData { + embedding: Vec, +} + +#[async_trait] +impl EmbeddingProvider for OpenAiEmbeddings { + fn dimension(&self) -> usize { + self.dimension + } + + fn model_name(&self) -> &str { + &self.model + } + + fn max_input_length(&self) -> usize { + // text-embedding-3-small/large + ada-002: ~8191 tokens, budgeted + // here as ~32_000 UTF-8 bytes (matches `str::len()` semantics — + // see `EmbeddingProvider::max_input_length` doc). + 32_000 + } + + async fn embed(&self, text: &str) -> Result, EmbeddingError> { + if text.len() > self.max_input_length() { + return Err(EmbeddingError::TextTooLong { + length: text.len(), + max: self.max_input_length(), + }); + } + + let embeddings = self.embed_batch(&[text.to_string()]).await?; + embeddings + .into_iter() + .next() + .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) + } + + async fn embed_batch(&self, texts: &[String]) -> Result>, EmbeddingError> { + if texts.is_empty() { + return Ok(Vec::new()); + } + + let request = OpenAiEmbeddingRequest { + model: &self.model, + input: texts, + }; + + let url = format!("{}/v1/embeddings", self.base_url); + + let response = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .json(&request) + .send() + .await?; + + let status = response.status(); + + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(EmbeddingError::AuthFailed); + } + + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + let retry_after = Some(ironclaw_llm::retry::parse_retry_after( + response.headers().get("retry-after"), + )); + return Err(EmbeddingError::RateLimited { retry_after }); + } + + if !status.is_success() { + let error_text = response.text().await.unwrap_or_default(); + return Err(EmbeddingError::HttpError(format!( + "Status {}: {}", + status, error_text + ))); + } + + let result: OpenAiEmbeddingResponse = response.json().await.map_err(|e| { + EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) + })?; + + Ok(result.data.into_iter().map(|d| d.embedding).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_openai_embeddings_config() { + let provider = OpenAiEmbeddings::new("test-key"); + assert_eq!(provider.dimension(), 1536); + assert_eq!(provider.model_name(), "text-embedding-3-small"); + assert_eq!(provider.base_url, OPENAI_API_BASE_URL); + + let provider = OpenAiEmbeddings::large("test-key"); + assert_eq!(provider.dimension(), 3072); + assert_eq!(provider.model_name(), "text-embedding-3-large"); + assert_eq!(provider.base_url, OPENAI_API_BASE_URL); + } + + #[test] + fn test_openai_with_base_url_valid() { + let provider = + OpenAiEmbeddings::new("test-key").with_base_url("https://custom.example.com"); + assert_eq!(provider.base_url, "https://custom.example.com"); + } + + #[test] + fn test_openai_with_base_url_strips_trailing_slashes() { + let provider = + OpenAiEmbeddings::new("test-key").with_base_url("https://custom.example.com///"); + assert_eq!(provider.base_url, "https://custom.example.com"); + } + + #[test] + fn test_openai_with_base_url_http_scheme() { + let provider = OpenAiEmbeddings::new("test-key").with_base_url("http://localhost:8080"); + assert_eq!(provider.base_url, "http://localhost:8080"); + } + + #[test] + fn test_openai_with_base_url_schemeless_prepends_https() { + let provider = OpenAiEmbeddings::new("test-key").with_base_url("custom.example.com/v1"); + assert_eq!(provider.base_url, "https://custom.example.com/v1"); + } +} diff --git a/crates/ironclaw_embeddings/src/provider.rs b/crates/ironclaw_embeddings/src/provider.rs new file mode 100644 index 00000000000..c4bcb2bec44 --- /dev/null +++ b/crates/ironclaw_embeddings/src/provider.rs @@ -0,0 +1,66 @@ +//! [`EmbeddingProvider`] trait and shared [`EmbeddingError`] type. + +use async_trait::async_trait; + +/// Error type for embedding operations. +#[derive(Debug, thiserror::Error)] +pub enum EmbeddingError { + #[error("HTTP request failed: {0}")] + HttpError(String), + + #[error("Invalid response: {0}")] + InvalidResponse(String), + + #[error("Rate limited, retry after {retry_after:?}")] + RateLimited { + retry_after: Option, + }, + + #[error("Authentication failed")] + AuthFailed, + + #[error("Text too long: {length} > {max}")] + TextTooLong { length: usize, max: usize }, + + #[error("Invalid provider URL '{url}': {reason}")] + InvalidUrl { url: String, reason: String }, +} + +impl From for EmbeddingError { + fn from(e: reqwest::Error) -> Self { + EmbeddingError::HttpError(e.to_string()) + } +} + +/// Trait for embedding providers. +#[async_trait] +pub trait EmbeddingProvider: Send + Sync { + /// Get the embedding dimension. + fn dimension(&self) -> usize; + + /// Get the model name. + fn model_name(&self) -> &str; + + /// Maximum input length in **bytes** (matches `str::len()` semantics). + /// + /// Provider implementations enforce this against `text.len()`, which + /// counts UTF-8 bytes, not Unicode characters. Implementations document + /// the byte budget for their underlying model (typically derived from a + /// token limit; e.g. 8191 tokens ≈ 32_000 bytes for the OpenAI + /// embedding family). + fn max_input_length(&self) -> usize; + + /// Generate an embedding for a single text. + async fn embed(&self, text: &str) -> Result, EmbeddingError>; + + /// Generate embeddings for multiple texts (batched). + /// + /// Default implementation calls embed() for each text. + async fn embed_batch(&self, texts: &[String]) -> Result>, EmbeddingError> { + let mut embeddings = Vec::with_capacity(texts.len()); + for text in texts { + embeddings.push(self.embed(text).await?); + } + Ok(embeddings) + } +} diff --git a/crates/ironclaw_embeddings/src/url_check.rs b/crates/ironclaw_embeddings/src/url_check.rs new file mode 100644 index 00000000000..1d2522caf2c --- /dev/null +++ b/crates/ironclaw_embeddings/src/url_check.rs @@ -0,0 +1,134 @@ +//! Baseline base-URL validation for the embedding-provider factory. +//! +//! This is a **defense-in-depth** check, not the full SSRF policy. The binary +//! continues to apply a richer, operator-tunable policy (`validate_operator_base_url` +//! in `src/config/helpers.rs`) at config-resolve time; this module covers the +//! case where a downstream caller constructs `EmbeddingsConfig` directly and +//! reaches `create_provider` without going through that path. +//! +//! What this enforces: +//! - URL parses +//! - Scheme is `http` or `https` +//! - Host (when it is a literal IP) is not in the `AlwaysBlocked` class: +//! cloud-metadata (`169.254.169.254`), link-local, multicast, the +//! unspecified `0.0.0.0`/`::`. These are *never* legitimate operator +//! endpoints, regardless of policy. +//! +//! What this does NOT do: +//! - DNS-resolve hostnames (the binary's policy does that; doing it here +//! would couple the crate to a runtime and to a DNS-availability heuristic). +//! - Reject private/loopback IPs — those are legitimate for self-hosted +//! Ollama and similar setups; the operator-tunable policy in the binary +//! makes that call. + +use std::net::{IpAddr, Ipv4Addr}; + +use crate::provider::EmbeddingError; + +/// Validate a base URL configured for an embedding provider. +/// +/// Returns `Err(EmbeddingError::InvalidUrl { .. })` on parse failure, +/// non-http(s) scheme, missing host, or an `AlwaysBlocked` literal IP host. +pub(crate) fn check_base_url(url: &str, field_name: &str) -> Result<(), EmbeddingError> { + let parsed = reqwest::Url::parse(url).map_err(|e| EmbeddingError::InvalidUrl { + url: url.to_string(), + reason: format!("{field_name}: {e}"), + })?; + + let scheme = parsed.scheme(); + if scheme != "http" && scheme != "https" { + return Err(EmbeddingError::InvalidUrl { + url: url.to_string(), + reason: format!("{field_name}: only http/https are allowed (got '{scheme}')"), + }); + } + + let host = parsed + .host_str() + .ok_or_else(|| EmbeddingError::InvalidUrl { + url: url.to_string(), + reason: format!("{field_name}: missing host"), + })?; + + let normalized_host = host.trim_start_matches('[').trim_end_matches(']'); + if let Ok(ip) = normalized_host.parse::() + && is_always_blocked(&ip) + { + return Err(EmbeddingError::InvalidUrl { + url: url.to_string(), + reason: format!("{field_name}: host '{host}' is not a permitted endpoint"), + }); + } + + Ok(()) +} + +fn is_always_blocked(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + v4.is_unspecified() + || v4.is_multicast() + || v4.is_link_local() + || *v4 == Ipv4Addr::new(169, 254, 169, 254) + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_always_blocked(&IpAddr::V4(v4)); + } + v6.is_unspecified() || v6.octets()[0] == 0xff || (v6.segments()[0] & 0xffc0) == 0xfe80 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_normal_endpoints() { + check_base_url("https://api.openai.com", "F").unwrap(); + check_base_url("http://localhost:11434", "F").unwrap(); + check_base_url("https://api.near.ai", "F").unwrap(); + check_base_url("http://192.168.1.50:8000", "F").unwrap(); // private — allowed at this layer + check_base_url("http://127.0.0.1:11434", "F").unwrap(); + } + + #[test] + fn rejects_aws_metadata_ip() { + let err = check_base_url("https://169.254.169.254", "OLLAMA_BASE_URL") + .expect_err("metadata IP must be rejected"); + assert!(matches!(err, EmbeddingError::InvalidUrl { .. })); + let msg = err.to_string(); + assert!( + msg.contains("OLLAMA_BASE_URL"), + "field name in message: {msg}" + ); + assert!(msg.contains("169.254.169.254"), "host in message: {msg}"); + } + + #[test] + fn rejects_link_local_ipv6() { + check_base_url("https://[fe80::1]", "F").expect_err("link-local IPv6 rejected"); + } + + #[test] + fn rejects_multicast() { + check_base_url("http://224.0.0.1", "F").expect_err("multicast rejected"); + } + + #[test] + fn rejects_unspecified() { + check_base_url("http://0.0.0.0", "F").expect_err("0.0.0.0 rejected"); + } + + #[test] + fn rejects_non_http_scheme() { + let err = check_base_url("file:///etc/passwd", "F").expect_err("file:// rejected"); + assert!(err.to_string().contains("http/https")); + } + + #[test] + fn rejects_unparseable() { + check_base_url("not a url", "F").expect_err("garbage rejected"); + } +} diff --git a/crates/ironclaw_engine/CLAUDE.md b/crates/ironclaw_engine/CLAUDE.md index 35329f8bcfc..ff5168f61b8 100644 --- a/crates/ironclaw_engine/CLAUDE.md +++ b/crates/ironclaw_engine/CLAUDE.md @@ -123,11 +123,16 @@ The engine defines three traits that the host crate implements: Python execution via Monty interpreter (`executor/scripting.rs`). Follows the RLM (Recursive Language Model) pattern. -For engine v2 prompt surfacing, blocked managed integrations are described in -`Activatable Integrations` and the model is expected to use -`tool_activate(name=...)` first. Newly enabled tools become visible on the -next top-level orchestrator turn; CodeAct does not hot-refresh callable tools -mid-step. +For engine v2 prompt surfacing, installed-but-unauthed provider tools (e.g. +`gmail` without an OAuth token) are direct-callable: the engine's auth +preflight raises an `Authentication` gate at execute time, the inline-await +machinery parks the VM, and the OAuth callback delivers the resolved +credential to retry the action. Integrations that need user-driven setup +(`NeedsSetup`, `Inactive`, `AvailableNotInstalled`) are listed under +`Activatable Integrations` and the model installs them by calling +`tool_install(name="")` directly (issue #3533 / PR #3559 — the +hidden gate on `tool_install` from #2868 was removed; the tool's +`requires_approval = UnlessAutoApproved` mediates user consent). **Context as variables** (not attention input): - Thread messages injected as `context` Python variable diff --git a/crates/ironclaw_engine/Cargo.toml b/crates/ironclaw_engine/Cargo.toml index ae82bfe4def..748d49f0b33 100644 --- a/crates/ironclaw_engine/Cargo.toml +++ b/crates/ironclaw_engine/Cargo.toml @@ -16,7 +16,7 @@ dist = false [dependencies] async-trait = "0.1" cron = "0.13" -ironclaw_common = { path = "../ironclaw_common", version = "0.4.0" } +ironclaw_common = { path = "../ironclaw_common", version = "0.4.2" } ironclaw_skills = { path = "../ironclaw_skills", version = "0.3.0", default-features = false } chrono = { version = "0.4", features = ["serde"] } monty = { git = "https://github.com/pydantic/monty.git", tag = "v0.0.16" } diff --git a/crates/ironclaw_engine/src/executor/context.rs b/crates/ironclaw_engine/src/executor/context.rs index 0f5ca364b85..602a894e7a6 100644 --- a/crates/ironclaw_engine/src/executor/context.rs +++ b/crates/ironclaw_engine/src/executor/context.rs @@ -184,6 +184,10 @@ mod tests { thread_goal: Some("search the web".into()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }, ) .await @@ -224,6 +228,10 @@ mod tests { thread_goal: Some("hello".into()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }, ) .await @@ -260,6 +268,10 @@ mod tests { thread_goal: Some("hello".into()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }, ) .await diff --git a/crates/ironclaw_engine/src/executor/loop_engine.rs b/crates/ironclaw_engine/src/executor/loop_engine.rs index d8db5127303..e175aa2fea1 100644 --- a/crates/ironclaw_engine/src/executor/loop_engine.rs +++ b/crates/ironclaw_engine/src/executor/loop_engine.rs @@ -121,9 +121,15 @@ pub struct ExecutionLoop { store: Option>, /// Runtime platform metadata for self-awareness in system prompts. platform_info: Option, + /// Host gate controller, attached to every `ThreadExecutionContext` + /// this loop builds so executors can pause in place on `Approval` + /// gates. Required: callers without an inline-await surface use + /// [`crate::gate::CancellingGateController::arc()`]. + gate_controller: Arc, } impl ExecutionLoop { + #[allow(clippy::too_many_arguments)] pub fn new( thread: Thread, llm: Arc, @@ -132,6 +138,7 @@ impl ExecutionLoop { policy: Arc, signal_rx: SignalReceiver, user_id: String, + gate_controller: Arc, ) -> Self { Self { thread, @@ -146,6 +153,7 @@ impl ExecutionLoop { retrieval: None, store: None, platform_info: None, + gate_controller, } } @@ -250,6 +258,7 @@ impl ExecutionLoop { &self.thread, StepId::new(), None, + self.gate_controller.clone(), ); let capabilities_result = self .effects @@ -429,6 +438,7 @@ impl ExecutionLoop { self.retrieval.as_ref(), self.store.as_ref(), self.platform_info.as_ref(), + &self.gate_controller, &checkpoint.persisted_state, ) .await; @@ -762,7 +772,16 @@ mod tests { let (tx, rx) = crate::runtime::messaging::signal_channel(16); - let exec = ExecutionLoop::new(thread, llm, effects, leases, policy, rx, "test-user".into()); + let exec = ExecutionLoop::new( + thread, + llm, + effects, + leases, + policy, + rx, + "test-user".into(), + crate::gate::CancellingGateController::arc(), + ); (exec, tx) } @@ -882,6 +901,7 @@ mod tests { policy, rx, "test-user".into(), + crate::gate::CancellingGateController::arc(), ); let outcome = exec.run().await.unwrap(); @@ -956,7 +976,11 @@ mod tests { name: "slack".into(), display_name: Some("Slack".into()), kind: CapabilitySummaryKind::Provider, - status: CapabilityStatus::NeedsAuth, + // NeedsSetup keeps slack visible in the + // Activatable Integrations prompt section. + // NeedsAuth would render in the regular action + // inventory (post-#3133 direct-callable path). + status: CapabilityStatus::NeedsSetup, description: Some("Slack workspace integration".into()), action_preview: vec!["slack_send".into()], routing_hint: None, @@ -1020,6 +1044,7 @@ mod tests { policy, rx, "test-user".into(), + crate::gate::CancellingGateController::arc(), ); let outcome = exec.run().await.unwrap(); @@ -1030,7 +1055,7 @@ mod tests { assert!(!system_prompt.contains("## Available tools (call as Python functions)")); assert!(system_prompt.contains("`slack` [provider]")); assert!(system_prompt.contains("## Activatable Integrations")); - assert!(system_prompt.contains("tool_activate(name=\"\")")); + assert!(system_prompt.contains("need user setup before their tools become callable")); assert!(!system_prompt.contains("`telegram` [channel]")); assert!(system_prompt.contains("## Prior Knowledge (from completed threads)")); assert!(system_prompt.contains("GitHub API Skill")); @@ -1117,7 +1142,11 @@ mod tests { name: "slack".into(), display_name: Some("Slack".into()), kind: CapabilitySummaryKind::Provider, - status: CapabilityStatus::NeedsAuth, + // NeedsSetup keeps slack visible in the + // Activatable Integrations prompt section. + // NeedsAuth would render in the regular action + // inventory (post-#3133 direct-callable path). + status: CapabilityStatus::NeedsSetup, description: Some("Slack workspace integration".into()), action_preview: vec!["slack_send".into()], routing_hint: None, @@ -1174,6 +1203,7 @@ mod tests { policy, rx, "test-user".into(), + crate::gate::CancellingGateController::arc(), ); let outcome = exec.run().await.unwrap(); @@ -1266,8 +1296,16 @@ mod tests { .unwrap(); let (_tx, rx) = crate::runtime::messaging::signal_channel(16); - let mut exec = - ExecutionLoop::new(thread, llm, effects, leases, policy, rx, "test-user".into()); + let mut exec = ExecutionLoop::new( + thread, + llm, + effects, + leases, + policy, + rx, + "test-user".into(), + crate::gate::CancellingGateController::arc(), + ); exec.refresh_system_prompt(&[], true, &mut checkpoint).await; assert_eq!(exec.thread.messages[0].content, old_prompt); @@ -1317,8 +1355,16 @@ mod tests { .unwrap(); let (_tx, rx) = crate::runtime::messaging::signal_channel(16); - let mut exec = - ExecutionLoop::new(thread, llm, effects, leases, policy, rx, "test-user".into()); + let mut exec = ExecutionLoop::new( + thread, + llm, + effects, + leases, + policy, + rx, + "test-user".into(), + crate::gate::CancellingGateController::arc(), + ); exec.refresh_system_prompt(&[], false, &mut checkpoint) .await; @@ -1975,8 +2021,16 @@ mod tests { .unwrap(); let (_tx, rx) = crate::runtime::messaging::signal_channel(16); - let mut exec = - ExecutionLoop::new(thread, llm, effects, leases, policy, rx, "test-user".into()); + let mut exec = ExecutionLoop::new( + thread, + llm, + effects, + leases, + policy, + rx, + "test-user".into(), + crate::gate::CancellingGateController::arc(), + ); exec.run().await.unwrap(); diff --git a/crates/ironclaw_engine/src/executor/orchestrator.rs b/crates/ironclaw_engine/src/executor/orchestrator.rs index ed591e551c6..d88e67e0cd8 100644 --- a/crates/ironclaw_engine/src/executor/orchestrator.rs +++ b/crates/ironclaw_engine/src/executor/orchestrator.rs @@ -445,6 +445,7 @@ pub async fn execute_orchestrator( retrieval: Option<&RetrievalEngine>, store: Option<&Arc>, platform_info: Option<&crate::executor::prompt::PlatformInfo>, + gate_controller: &Arc, persisted_state: &serde_json::Value, ) -> Result { let mut total_tokens = TokenUsage::default(); @@ -560,7 +561,15 @@ pub async fn execute_orchestrator( // __execute_code_step__(code, state) "__execute_code_step__" => { handle_execute_code_step( - args, kwargs, thread, llm, effects, leases, policy, event_tx, + args, + kwargs, + thread, + llm, + effects, + leases, + policy, + event_tx, + gate_controller, ) .await } @@ -568,7 +577,14 @@ pub async fn execute_orchestrator( // __execute_action__(name, params, call_id=...) "__execute_action__" => { handle_execute_action( - args, kwargs, thread, effects, leases, policy, event_tx, + args, + kwargs, + thread, + effects, + leases, + policy, + event_tx, + gate_controller, ) .await } @@ -576,7 +592,13 @@ pub async fn execute_orchestrator( // __execute_actions_parallel__(calls) "__execute_actions_parallel__" => { handle_execute_actions_parallel( - args, thread, effects, leases, policy, event_tx, + args, + thread, + effects, + leases, + policy, + event_tx, + gate_controller, ) .await } @@ -734,7 +756,14 @@ async fn handle_llm_complete( } let active_leases = deps.leases.active_for_thread(thread.id).await; - let actions_context = thread_execution_context(thread, StepId::new(), None); + // Read-only path: `available_actions` and the message refresh below + // don't pause; inert controller is correct. + let actions_context = thread_execution_context( + thread, + StepId::new(), + None, + crate::gate::CancellingGateController::arc(), + ); let actions = deps .effects .available_actions(&active_leases, &actions_context) @@ -876,6 +905,7 @@ async fn handle_execute_code_step( leases: &Arc, policy: &Arc, event_tx: Option<&tokio::sync::broadcast::Sender>, + gate_controller: &Arc, ) -> ExtFunctionResult { let code = match args.first() { Some(obj) => monty_to_string(obj), @@ -892,7 +922,7 @@ async fn handle_execute_code_step( .map(monty_to_json) .unwrap_or(serde_json::json!({})); - let exec_ctx = thread_execution_context(thread, StepId::new(), None); + let exec_ctx = thread_execution_context(thread, StepId::new(), None, gate_controller.clone()); // Run user code in a nested Monty VM (same pattern as rlm_query) let code_start = std::time::Instant::now(); @@ -1065,6 +1095,7 @@ async fn handle_execute_code_step( /// /// Python owns the working transcript and decides how tool outputs are /// represented in internal message history. +#[allow(clippy::too_many_arguments)] async fn handle_execute_action( args: &[MontyObject], kwargs: &[(MontyObject, MontyObject)], @@ -1073,6 +1104,7 @@ async fn handle_execute_action( leases: &Arc, policy: &Arc, event_tx: Option<&tokio::sync::broadcast::Sender>, + gate_controller: &Arc, ) -> ExtFunctionResult { let name = match extract_string_arg(args, kwargs, "name", 0) { Some(n) => n, @@ -1091,7 +1123,12 @@ async fn handle_execute_action( let call_id = extract_string_kwarg(kwargs, "call_id").unwrap_or_default(); - let mut exec_ctx = thread_execution_context(thread, StepId::new(), Some(call_id.clone())); + let mut exec_ctx = thread_execution_context( + thread, + StepId::new(), + Some(call_id.clone()), + gate_controller.clone(), + ); let active_leases = leases.active_for_thread(thread.id).await; let inventory = match effects .available_action_inventory(&active_leases, &exec_ctx) @@ -1211,7 +1248,13 @@ async fn handle_execute_action( return ExtFunctionResult::Return(json_to_monty(&result)); } crate::capability::policy::PolicyDecision::RequireApproval { .. } => { - let output = serde_json::json!({"status": "gate_paused", "gate_name": "approval"}); + // Inline gate-await on policy-raised approval. Mirrors + // `structured.rs::execute_action_batch_with_results`: emit + // the request, pause the executor in place, and either + // fall through to lease consume + execute on approval, or + // emit ActionFailed and surface a deny-style result on + // denial. No more `gate_paused` sentinel + thread re-entry + // for this code path. emit_and_record( thread, event_tx, @@ -1220,26 +1263,59 @@ async fn handle_execute_action( call_id: call_id.clone(), parameters: Some(params.clone()), description: None, - allow_always: None, - gate_name: None, + allow_always: Some(true), + gate_name: Some("approval".into()), params_summary: summarize_params(&name, ¶ms), }, &call_id, &name, - &output, + &serde_json::json!({}), ); - let result = serde_json::json!({ - "gate_paused": true, - "gate_name": "approval", - "action_name": name, - "call_id": call_id, - "parameters": params, - "resume_kind": serde_json::to_value(crate::gate::ResumeKind::Approval { - allow_always: true, + + let resume_kind = crate::gate::ResumeKind::Approval { allow_always: true }; + let resolution = gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id: thread.id, + user_id: thread.user_id.clone(), + gate_name: "approval".into(), + action_name: name.clone(), + call_id: call_id.clone(), + parameters: params.clone(), + resume_kind, + conversation_id: exec_ctx.conversation_id, }) - .unwrap_or_default(), - }); - return ExtFunctionResult::Return(json_to_monty(&result)); + .await; + + if let Some(outcome) = + crate::executor::scripting::denial_outcome_for_resolution(&resolution) + { + let error = outcome.event_error(); + let output = serde_json::json!({"error": &error}); + emit_and_record( + thread, + event_tx, + EventKind::ActionFailed { + step_id: exec_ctx.step_id, + action_name: name.clone(), + call_id: call_id.clone(), + error: error.clone(), + duration_ms: 0, + params_summary: summarize_params(&name, ¶ms), + }, + &call_id, + &name, + &output, + ); + let result = serde_json::json!({ + "output": output, + "is_error": true, + }); + return ExtFunctionResult::Return(json_to_monty(&result)); + } + // Approved — fall through to lease consume + execute. + // The adapter's per-call ApprovalRequirement gate (if + // any) is independent of the policy gate and will be + // handled inline by the wrapper below if it fires. } crate::capability::policy::PolicyDecision::Allow => {} } @@ -1280,135 +1356,38 @@ async fn handle_execute_action( } }; - // 4. Execute + // 4. Execute via the inline-await wrapper. Tool-raised + // `Err(GatePaused)` from `effects.execute_action` is converted to a + // `gate_paused` JSON sentinel by the adapter shim and then handled + // inline by `execute_single_action_with_inline_retry`: pause the + // user, retry on approval (bounded), surface deny-style results + // on denial. No more `gate_paused` sentinel returned to Python + // from this path. let ps = summarize_params(&canonical_name, ¶ms); - let execution_start = std::time::Instant::now(); - match effects - .execute_action(&canonical_name, params, &lease, &exec_ctx) - .await - { - Ok(r) => { - // Effect adapters wrap tool errors as `Ok(ActionResult { is_error: true })` - // — surface them as `ActionFailed` so traces and observers see the - // failure. See `resolve_tool_future` in `scripting.rs` for the same - // pattern on the structured-tool path. - if r.is_error { - let error_msg = r - .output - .get("error") - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| r.output.to_string()); - let duration_ms = r.duration.as_millis() as u64; - emit_and_record( - thread, - event_tx, - EventKind::ActionFailed { - step_id: exec_ctx.step_id, - action_name: name.clone(), - call_id: call_id.clone(), - error: error_msg, - duration_ms: if duration_ms > 0 { - duration_ms - } else { - execution_start.elapsed().as_millis() as u64 - }, - params_summary: ps.clone(), - }, - &call_id, - &name, - &r.output, - ); - } else { - emit_and_record( - thread, - event_tx, - EventKind::ActionExecuted { - step_id: exec_ctx.step_id, - action_name: name.clone(), - call_id: call_id.clone(), - duration_ms: r.duration.as_millis() as u64, - params_summary: ps.clone(), - }, - &call_id, - &name, - &r.output, - ); - } - let result = serde_json::json!({ - "action_name": r.action_name, - "output": r.output, - "is_error": r.is_error, - "duration_ms": r.duration.as_millis(), - }); - ExtFunctionResult::Return(json_to_monty(&result)) - } - Err(EngineError::GatePaused { - gate_name, - action_name: _, - call_id: _, - parameters, - resume_kind, - resume_output, - paused_lease, - }) => { - let _ = leases.refund_use(lease.id).await; - let output = serde_json::json!({"status": "gate_paused", "gate_name": gate_name}); - emit_and_record( - thread, - event_tx, - EventKind::ApprovalRequested { - action_name: name.clone(), - call_id: call_id.clone(), - parameters: Some((*parameters).clone()), - description: None, - allow_always: match resume_kind.as_ref() { - crate::gate::ResumeKind::Approval { allow_always } => Some(*allow_always), - _ => None, - }, - gate_name: Some(gate_name.clone()), - params_summary: summarize_params(&name, ¶meters), - }, - &call_id, - &name, - &output, - ); - let result = serde_json::json!({ - "gate_paused": true, - "gate_name": gate_name, - "action_name": name, - "call_id": call_id, - "parameters": parameters, - "resume_kind": serde_json::to_value(&*resume_kind).unwrap_or_default(), - "resume_output": resume_output, - "paused_lease": paused_lease.as_deref().cloned(), - }); - ExtFunctionResult::Return(json_to_monty(&result)) - } - Err(e) => { - let output = serde_json::json!({"error": e.to_string()}); - emit_and_record( - thread, - event_tx, - EventKind::ActionFailed { - step_id: exec_ctx.step_id, - action_name: name.clone(), - call_id: call_id.clone(), - error: e.to_string(), - duration_ms: execution_start.elapsed().as_millis() as u64, - params_summary: ps, - }, - &call_id, - &name, - &output, - ); - let result = serde_json::json!({ - "output": output, - "is_error": true, - }); - ExtFunctionResult::Return(json_to_monty(&result)) - } + let (result_json, events, _output, _final_lease_id) = execute_single_action_with_inline_retry( + effects, + leases, + &canonical_name, + params, + &call_id, + lease, + &exec_ctx, + ps, + thread.id, + &thread.user_id, + ) + .await; + for event in events { + emit_and_record( + thread, + event_tx, + event, + &call_id, + &name, + &serde_json::json!({}), + ); } + ExtFunctionResult::Return(json_to_monty(&result_json)) } /// Handle `__execute_actions_parallel__(calls)`. @@ -1420,6 +1399,7 @@ async fn handle_execute_action( /// same shape as `__execute_action__` output, plus an optional gate pause payload. /// /// Events are emitted in original call order after all parallel executions complete. +#[allow(clippy::too_many_arguments)] async fn handle_execute_actions_parallel( args: &[MontyObject], thread: &mut Thread, @@ -1427,6 +1407,7 @@ async fn handle_execute_actions_parallel( leases: &Arc, policy: &Arc, event_tx: Option<&tokio::sync::broadcast::Sender>, + gate_controller: &Arc, ) -> ExtFunctionResult { // Parse the calls list from the first argument (list of dicts) let calls_json = args @@ -1475,7 +1456,7 @@ async fn handle_execute_actions_parallel( } let step_id = StepId::new(); - let actions_context = thread_execution_context(thread, step_id, None); + let actions_context = thread_execution_context(thread, step_id, None, gate_controller.clone()); let active_leases = leases.active_for_thread(thread.id).await; let inventory = match effects .available_action_inventory(&active_leases, &actions_context) @@ -1514,7 +1495,12 @@ async fn handle_execute_actions_parallel( for pc in &parsed { // Find the action definition from the callable inventory. - let mut exec_ctx = thread_execution_context(thread, step_id, Some(pc.call_id.clone())); + let mut exec_ctx = thread_execution_context( + thread, + step_id, + Some(pc.call_id.clone()), + gate_controller.clone(), + ); if let Some(ref inventory) = inventory { exec_ctx.available_actions_snapshot = Some(Arc::clone(&available_actions)); exec_ctx.available_action_inventory_snapshot = Some(Arc::clone(inventory)); @@ -1606,66 +1592,74 @@ async fn handle_execute_actions_parallel( continue; } crate::capability::policy::PolicyDecision::RequireApproval { .. } => { - // Emit events for earlier errors, then interrupt - let mut results_json = Vec::with_capacity(preflight.len() + 1); - for pf in preflight { - match pf { - Some(PfOutcome::Error { - result_json, - event, - output: _, - }) => { - let ev = ThreadEvent::new(thread.id, event); - if let Some(tx) = event_tx { - let _ = tx.send(ev.clone()); - } - thread.events.push(ev); - results_json.push(result_json); - } - Some(PfOutcome::Runnable { .. }) | None => { - results_json.push(serde_json::json!(null)); - } - } - } - // Add the approval entry - let ev = ThreadEvent::new( + // Inline gate-await: pause this preflight call in place + // until the user resolves the gate. On approval, fall + // through to lease consumption + queue for execution. + // On denial, push an ActionFailed result and continue + // preflight so the rest of the batch still runs — + // mirrors `structured.rs::execute_action_batch_with_results`. + // + // The bridge controller serializes concurrent inline + // gates per (user, thread), so two preflight calls that + // both gate get prompted sequentially rather than the + // second silently cancelling. + let approval_ev = ThreadEvent::new( thread.id, EventKind::ApprovalRequested { action_name: pc.name.clone(), call_id: pc.call_id.clone(), parameters: Some(pc.params.clone()), description: None, - allow_always: None, - gate_name: None, + allow_always: Some(true), + gate_name: Some("approval".into()), params_summary: summarize_params(&pc.name, &pc.params), }, ); if let Some(tx) = event_tx { - let _ = tx.send(ev.clone()); + let _ = tx.send(approval_ev.clone()); } - thread.events.push(ev); + thread.events.push(approval_ev); thread.updated_at = chrono::Utc::now(); - results_json.push(serde_json::json!({ - "gate_paused": true, - "gate_name": "approval", - "action_name": &pc.name, - "call_id": &pc.call_id, - "parameters": &pc.params, - "resume_kind": serde_json::to_value(crate::gate::ResumeKind::Approval { - allow_always: true, + let resume_kind = crate::gate::ResumeKind::Approval { allow_always: true }; + let resolution = gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id: thread.id, + user_id: thread.user_id.clone(), + gate_name: "approval".into(), + action_name: pc.name.clone(), + call_id: pc.call_id.clone(), + parameters: pc.params.clone(), + resume_kind, + conversation_id: exec_ctx.conversation_id, }) - .unwrap_or_default(), - })); - // Pad with nulls for calls that weren't reached so the - // Python-side loop can emit ActionResult placeholders for - // every tool call in the assistant message. - while results_json.len() < parsed.len() { - results_json.push(serde_json::json!(null)); + .await; + + if let Some(outcome) = + crate::executor::scripting::denial_outcome_for_resolution(&resolution) + { + let error = outcome.event_error(); + let output = serde_json::json!({"error": &error}); + let result_json = serde_json::json!({ + "output": &output, + "is_error": true, + }); + let event = EventKind::ActionFailed { + step_id, + action_name: action_name.clone(), + call_id: pc.call_id.clone(), + error, + duration_ms: 0, + params_summary: summarize_params(&pc.name, &pc.params), + }; + preflight.push(Some(PfOutcome::Error { + result_json, + event, + output, + })); + continue; } - return ExtFunctionResult::Return(json_to_monty(&serde_json::json!( - results_json - ))); + // Approved — fall through to lease consume + runnable. } crate::capability::policy::PolicyDecision::Allow => {} } @@ -1707,9 +1701,11 @@ async fn handle_execute_actions_parallel( // ── Phase 2: Execute in parallel ──────────────────────────── - // Slot array: index → execution result + // Slot array: index → execution result. `slot_events` is + // `Vec` per slot so the inline-retry path can record + // multiple events (ApprovalRequested + post-retry outcome). let mut slot_results: Vec> = vec![None; parsed.len()]; - let mut slot_events: Vec> = vec![None; parsed.len()]; + let mut slot_events: Vec>> = vec![None; parsed.len()]; let mut slot_outputs: Vec> = vec![None; parsed.len()]; // Separate runnable from errors let mut runnable: Vec<(usize, crate::types::capability::CapabilityLease)> = Vec::new(); @@ -1721,7 +1717,7 @@ async fn handle_execute_actions_parallel( output, }) => { slot_results[idx] = Some(result_json); - slot_events[idx] = Some(event); + slot_events[idx] = Some(vec![event]); slot_outputs[idx] = Some(output); } Some(PfOutcome::Runnable { lease }) => { @@ -1732,7 +1728,7 @@ async fn handle_execute_actions_parallel( } if runnable.len() == 1 { - // Single call: execute directly + // Single call: execute directly with inline gate-await retry. let (idx, lease) = runnable.into_iter().next().unwrap(); // safety: len()==1 checked above let pc = &parsed[idx]; let action_name = available_actions @@ -1740,34 +1736,50 @@ async fn handle_execute_actions_parallel( .find(|action| action.matches_name(&pc.name)) .map(|action| action.name.clone()) .unwrap_or_else(|| pc.name.clone()); - let mut exec_ctx = thread_execution_context(thread, step_id, Some(pc.call_id.clone())); + let mut exec_ctx = thread_execution_context( + thread, + step_id, + Some(pc.call_id.clone()), + gate_controller.clone(), + ); if let Some(ref inventory) = inventory { exec_ctx.available_actions_snapshot = Some(Arc::clone(&available_actions)); exec_ctx.available_action_inventory_snapshot = Some(Arc::clone(inventory)); } let ps = summarize_params(&action_name, &pc.params); - let (result_json, event, output) = execute_single_action( - effects, - &action_name, - pc.params.clone(), - &pc.call_id, - &lease, - &exec_ctx, - ps, - ) - .await; - if interrupted_result_needs_refund(&result_json) { - let _ = leases.refund_use(lease.id).await; - } + let (result_json, events, output, _final_lease_id) = + execute_single_action_with_inline_retry( + effects, + leases, + &action_name, + pc.params.clone(), + &pc.call_id, + lease, + &exec_ctx, + ps, + thread.id, + &thread.user_id, + ) + .await; slot_results[idx] = Some(result_json); - slot_events[idx] = Some(event); + slot_events[idx] = Some(events); slot_outputs[idx] = Some(output); } else if runnable.len() > 1 { - // Multiple calls: execute in parallel via JoinSet + // Multiple calls: execute in parallel via JoinSet. Each task + // carries its own inline retry loop so one tool's gate doesn't + // block the rest of the batch — and the legacy "double-execute + // on resume" bug never fires for parallel batches either. let mut join_set = tokio::task::JoinSet::new(); let effects = effects.clone(); - // Capture once outside the loop — the thread's metadata is stable - // for the duration of the parallel batch. + let leases_arc = Arc::clone(leases); + // Build the base execution context once from the live thread. + // Per-task contexts clone this and overwrite `current_call_id` + // and the action snapshots — far cheaper than cloning the full + // `Thread` (which carries message/event transcripts) per task. + let base_exec_ctx = + thread_execution_context(thread, step_id, None, gate_controller.clone()); + let thread_id = thread.id; + let user_id = thread.user_id.clone(); for (idx, lease) in runnable { let pc_name = available_actions .iter() @@ -1777,8 +1789,11 @@ async fn handle_execute_actions_parallel( let pc_params = parsed[idx].params.clone(); let pc_call_id = parsed[idx].call_id.clone(); let effects = effects.clone(); + let leases = Arc::clone(&leases_arc); + let user_id = user_id.clone(); let lease = lease.clone(); - let mut exec_ctx = thread_execution_context(thread, step_id, Some(pc_call_id.clone())); + let mut exec_ctx = base_exec_ctx.clone(); + exec_ctx.current_call_id = Some(pc_call_id.clone()); if let Some(ref inventory) = inventory { exec_ctx.available_actions_snapshot = Some(Arc::clone(&available_actions)); exec_ctx.available_action_inventory_snapshot = Some(Arc::clone(inventory)); @@ -1786,28 +1801,32 @@ async fn handle_execute_actions_parallel( let ps = summarize_params(&pc_name, &pc_params); join_set.spawn(async move { - let (result_json, event, output) = execute_single_action( - &effects, - &pc_name, - pc_params, - &pc_call_id, - &lease, - &exec_ctx, - ps, - ) - .await; - (idx, lease.id, result_json, event, output) + let (result_json, events, output, final_lease_id) = + execute_single_action_with_inline_retry( + &effects, + &leases, + &pc_name, + pc_params, + &pc_call_id, + lease, + &exec_ctx, + ps, + thread_id, + &user_id, + ) + .await; + (idx, final_lease_id, result_json, events, output) }); } while let Some(join_result) = join_set.join_next().await { match join_result { - Ok((idx, lease_id, result_json, event, output)) => { - if interrupted_result_needs_refund(&result_json) { - let _ = leases.refund_use(lease_id).await; - } + Ok((idx, _lease_id, result_json, events, output)) => { + // The inline-retry helper already refunded any + // leases consumed during gate-await. No + // additional bookkeeping needed here. slot_results[idx] = Some(result_json); - slot_events[idx] = Some(event); + slot_events[idx] = Some(events); slot_outputs[idx] = Some(output); } Err(e) => { @@ -1828,12 +1847,14 @@ async fn handle_execute_actions_parallel( .take() .unwrap_or(serde_json::json!({"error": "no output"})); - if let Some(event) = slot_events[idx].take() { - let ev = ThreadEvent::new(thread.id, event); - if let Some(tx) = event_tx { - let _ = tx.send(ev.clone()); + if let Some(events) = slot_events[idx].take() { + for event in events { + let ev = ThreadEvent::new(thread.id, event); + if let Some(tx) = event_tx { + let _ = tx.send(ev.clone()); + } + thread.events.push(ev); } - thread.events.push(ev); } results_json.push(result_json.clone()); @@ -1953,6 +1974,267 @@ fn interrupted_result_needs_refund(result: &serde_json::Value) -> bool { result.get("gate_paused").and_then(|v| v.as_bool()) == Some(true) } +/// Like [`execute_single_action`] but pauses inline on +/// `Approval`-kind gate paused results and retries. Bounded by +/// [`crate::executor::scripting::MAX_INLINE_GATE_RETRIES`] so a +/// misbehaving tool can't spin a CPU. +/// +/// Used by `__execute_actions_parallel__` for both the single-runnable +/// and multi-runnable branches. Without this wrapper the multi-runnable +/// branch falls through to the legacy `gate_paused` sentinel + thread +/// re-entry, which double-executes earlier non-idempotent calls in the +/// same batch — exactly the bug this PR exists to prevent. +#[allow(clippy::too_many_arguments)] +async fn execute_single_action_with_inline_retry( + effects: &Arc, + leases: &Arc, + name: &str, + params: serde_json::Value, + call_id: &str, + initial_lease: crate::types::capability::CapabilityLease, + exec_ctx: &ThreadExecutionContext, + params_summary: Option, + thread_id: crate::types::thread::ThreadId, + user_id: &str, +) -> ( + serde_json::Value, + Vec, + serde_json::Value, + crate::types::capability::LeaseId, +) { + let mut current_lease = initial_lease; + let mut call_ctx = exec_ctx.clone(); + // `accumulated_events` carries every event the inline-retry loop + // observes — `ApprovalRequested` from each gate-paused iteration, + // plus the final `ActionExecuted` / `ActionFailed`. The caller + // appends them all to the thread event log so observers see the + // full sequence. + let mut accumulated_events: Vec = Vec::new(); + for _ in 0..crate::executor::scripting::MAX_INLINE_GATE_RETRIES { + let (result_json, event, output) = execute_single_action( + effects, + name, + params.clone(), + call_id, + ¤t_lease, + &call_ctx, + params_summary.clone(), + ) + .await; + // Reset the one-shot approval flag — only the call immediately + // following an approval should carry it. + call_ctx.call_approval_granted = false; + + if !interrupted_result_needs_refund(&result_json) { + // Not a gate pause — terminal event; record it and return. + accumulated_events.push(event); + return (result_json, accumulated_events, output, current_lease.id); + } + + // Gate paused. Approval and Authentication get the inline-await + // treatment (#3133 / #3166): host controller resolves them in + // place, the suspended call retries, and the orchestrator + // continues without unwinding. External keeps the legacy + // `gate_paused` sentinel + re-entry path because its resolution + // payload (callback body) can't be handed back to a suspended + // call. + let resume_kind: crate::gate::ResumeKind = result_json + .get("resume_kind") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or(crate::gate::ResumeKind::Approval { + allow_always: false, + }); + if !matches!( + resume_kind, + crate::gate::ResumeKind::Approval { .. } + | crate::gate::ResumeKind::Authentication { .. } + ) { + accumulated_events.push(event); + return (result_json, accumulated_events, output, current_lease.id); + } + + // Approval gate fired — record the request before pausing the + // controller so observers see the prompt regardless of how the + // resolution lands. + accumulated_events.push(event); + + // Refund the lease use this attempt consumed; we'll re-consume + // on retry if the user approves. EXCEPTION: when the gate carries + // cached `resume_output`, the action has already executed (post- + // execution Authentication gate) and the cached-output branch + // below will return without re-consuming. Refunding now would + // let a successful side-effecting action consume zero lease + // uses. See matching guards in `scripting::resolve_tool_future` + // and `structured::execute_with_inline_gate_retry`. Tracked by + // the #3559 security review. + let gate_carries_resume_output = result_json + .get("resume_output") + .is_some_and(|v| !v.is_null()); + if !gate_carries_resume_output { + let _ = leases.refund_use(current_lease.id).await; + } + + // Use the gate-provided parameters from the GatePaused payload, + // not the original caller `params`: the safety layer may have + // transformed/redacted them, and the prompt the user sees must + // match what the tool actually wanted to run with. Mirrors the + // contract in `structured::execute_with_inline_gate_retry`. + let gate_parameters = result_json + .get("parameters") + .cloned() + .unwrap_or_else(|| params.clone()); + let resolution = exec_ctx + .gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id, + user_id: user_id.to_string(), + gate_name: result_json + .get("gate_name") + .and_then(|v| v.as_str()) + .unwrap_or("approval") + .to_string(), + action_name: name.to_string(), + call_id: call_id.to_string(), + parameters: gate_parameters, + resume_kind: resume_kind.clone(), + conversation_id: exec_ctx.conversation_id, + }) + .await; + + if let Some(outcome) = + crate::executor::scripting::denial_outcome_for_resolution(&resolution) + { + // Cancelled+Authentication → fall through to legacy + // `gate_paused` sentinel so missions / non-inline-aware + // controllers can still surface a Paused state. See the + // matching branch in + // `structured::execute_with_inline_gate_retry`. The + // already-accumulated `ApprovalRequested` event was + // pushed before the pause; we re-emit it on the new + // result_json carrying the original gate metadata. + if matches!(resolution, crate::gate::GateResolution::Cancelled) + && matches!(resume_kind, crate::gate::ResumeKind::Authentication { .. }) + { + return (result_json, accumulated_events, output, current_lease.id); + } + let error_msg = outcome.event_error(); + let denial = serde_json::json!({"error": &error_msg}); + let denial_event = EventKind::ActionFailed { + step_id: exec_ctx.step_id, + action_name: name.to_string(), + call_id: call_id.to_string(), + error: error_msg, + duration_ms: 0, + params_summary: params_summary.clone(), + }; + accumulated_events.push(denial_event); + let result_json = serde_json::json!({ + "action_name": name, + "output": &denial, + "is_error": true, + "duration_ms": 0, + }); + return (result_json, accumulated_events, denial, current_lease.id); + } + + // Approved. If the bridge cached the action's output before raising + // this gate (post-execution Authentication gate path — see + // `effect_adapter::auth_gate_from_extension_result` and the + // `check_tool_readiness` path), the action has already run and we + // just needed user-side resolution. Return the cached output + // instead of re-executing. Without this shortcut, retrying + // `tool_install` re-downloads the WASM and runs through the + // `effect_adapter::enforce_tool_permission` approval check a + // second time, raising a fresh gate the user has no way to + // resolve. Tracked by #3533. + if let Some(cached_output) = result_json.get("resume_output").cloned() + && !cached_output.is_null() + { + let event = EventKind::ActionExecuted { + step_id: exec_ctx.step_id, + action_name: name.to_string(), + call_id: call_id.to_string(), + duration_ms: 0, + params_summary: params_summary.clone(), + }; + accumulated_events.push(event); + let result_json = serde_json::json!({ + "action_name": name, + "output": cached_output.clone(), + "is_error": false, + "duration_ms": 0, + }); + return ( + result_json, + accumulated_events, + cached_output, + current_lease.id, + ); + } + + // Re-consume a lease use and mark the next call as pre-approved. + match leases.find_and_consume(thread_id, name).await { + Ok(new_lease) => { + current_lease = new_lease; + call_ctx.call_approval_granted = true; + continue; + } + Err(e) => { + let err = + serde_json::json!({"error": format!("lease exhausted after approval: {e}")}); + let lease_event = EventKind::ActionFailed { + step_id: exec_ctx.step_id, + action_name: name.to_string(), + call_id: call_id.to_string(), + error: format!("lease exhausted after approval: {e}"), + duration_ms: 0, + params_summary: params_summary.clone(), + }; + accumulated_events.push(lease_event); + let result_json = serde_json::json!({ + "action_name": name, + "output": &err, + "is_error": true, + "duration_ms": 0, + }); + return (result_json, accumulated_events, err, current_lease.id); + } + } + } + + // Retry budget exhausted — tool kept gating after every approval. + // The last loop iteration ended with a successful `find_and_consume` + // whose lease was never used; refund it before returning so a + // misbehaving tool can't slowly drain `max_uses` across approvals. + // Best-effort; if the lease was already revoked/expired the refund + // is a no-op. + let _ = leases.refund_use(current_lease.id).await; + let err = serde_json::json!({ + "error": format!( + "tool '{name}' still requires approval after {} retries", + crate::executor::scripting::MAX_INLINE_GATE_RETRIES + ), + }); + accumulated_events.push(EventKind::ActionFailed { + step_id: exec_ctx.step_id, + action_name: name.to_string(), + call_id: call_id.to_string(), + error: format!( + "tool kept gating after {} approvals", + crate::executor::scripting::MAX_INLINE_GATE_RETRIES + ), + duration_ms: 0, + params_summary, + }); + let result_json = serde_json::json!({ + "action_name": name, + "output": &err, + "is_error": true, + "duration_ms": 0, + }); + (result_json, accumulated_events, err, current_lease.id) +} + /// Handle `__check_signals__()`. fn handle_check_signals(signal_rx: &mut SignalReceiver, thread: &mut Thread) -> ExtFunctionResult { match signal_rx.try_recv() { @@ -2213,7 +2495,15 @@ async fn handle_get_actions( } let active_leases = leases.active_for_thread(thread.id).await; - let actions_context = thread_execution_context(thread, StepId::new(), None); + // Read-only path: `available_actions` doesn't pause, so an inert + // controller is correct. Plumbing the live one here would buy + // nothing. + let actions_context = thread_execution_context( + thread, + StepId::new(), + None, + crate::gate::CancellingGateController::arc(), + ); match effects .available_actions(&active_leases, &actions_context) .await @@ -3172,7 +3462,7 @@ mod tests { // // Extracts the helper functions from the default orchestrator and // evaluates `signals_tool_intent(text)` directly, mirroring the V1 - // Rust unit test suite in src/llm/reasoning.rs. + // Rust unit test suite in crates/ironclaw_llm/src/reasoning.rs. /// Run a Python expression that returns a bool by prepending the /// orchestrator helper definitions and wrapping in `FINAL(expr)`. @@ -4323,6 +4613,7 @@ mod tests { .await .expect("grant lease"); + let controller = crate::gate::CancellingGateController::arc(); let result = handle_execute_action( &[ MontyObject::String("echo".into()), @@ -4337,6 +4628,7 @@ mod tests { &leases, &policy, None, + &controller, ) .await; @@ -5158,6 +5450,7 @@ FINAL(batch_error_count) ]; let (tx, _rx) = tokio::sync::broadcast::channel(16); + let controller = crate::gate::CancellingGateController::arc(); let _result = handle_execute_code_step( args, &[], @@ -5167,6 +5460,7 @@ FINAL(batch_error_count) &leases, &policy, Some(&tx), + &controller, ) .await; diff --git a/crates/ironclaw_engine/src/executor/prompt.rs b/crates/ironclaw_engine/src/executor/prompt.rs index 37fdb18752b..95a62876d28 100644 --- a/crates/ironclaw_engine/src/executor/prompt.rs +++ b/crates/ironclaw_engine/src/executor/prompt.rs @@ -22,63 +22,9 @@ use crate::types::capability::{ use crate::types::message::{MessageRole, ThreadMessage}; use crate::types::project::ProjectId; -/// Runtime platform metadata injected into system prompts for self-awareness. -/// -/// Provides the agent with knowledge about its own identity and environment -/// so it can answer questions about itself, its capabilities, and its -/// configuration without relying on training data. -#[derive(Debug, Clone, Default)] -pub struct PlatformInfo { - /// Software version (from CARGO_PKG_VERSION). - pub version: Option, - /// LLM backend name (e.g. "nearai", "openai", "anthropic"). - pub llm_backend: Option, - /// Active model name. - pub model_name: Option, - /// Database backend (e.g. "libsql", "postgres"). - pub database_backend: Option, - /// Active channel names (e.g. ["telegram", "cli"]). - pub active_channels: Vec, - /// Owner identifier. - pub owner_id: Option, - /// Project repository URL. - pub repo_url: Option, -} - -impl PlatformInfo { - /// Format as a prompt section. Returns empty string if no info is set. - pub fn to_prompt_section(&self) -> String { - let mut lines = Vec::new(); - - lines.push("You are **IronClaw**, a secure autonomous AI assistant platform.".into()); - if let Some(ref v) = self.version { - lines.push(format!("- Version: {v}")); - } - if let Some(ref repo) = self.repo_url { - lines.push(format!("- Repository: {repo}")); - } - if let Some(ref owner) = self.owner_id { - lines.push(format!("- Owner: {owner}")); - } - if let Some(ref backend) = self.llm_backend { - let model = self.model_name.as_deref().unwrap_or("default"); - lines.push(format!("- LLM: {backend} ({model})")); - } - if let Some(ref db) = self.database_backend { - lines.push(format!("- Database: {db}")); - } - if !self.active_channels.is_empty() { - lines.push(format!("- Channels: {}", self.active_channels.join(", "))); - } - - if lines.len() <= 1 { - // Only the identity line, no runtime details — still include it - return format!("\n\n## Platform\n\n{}\n", lines[0]); - } - - format!("\n\n## Platform\n\n{}\n", lines.join("\n")) - } -} +// Runtime platform metadata lives in `ironclaw_common::platform`. Re-exported +// from this module's path for back-compat with prior call sites. +pub use ironclaw_common::platform::PlatformInfo; /// The main instruction block (before tool listing). const CODEACT_PREAMBLE: &str = include_str!("../../prompts/codeact_preamble.md"); @@ -86,6 +32,36 @@ const CODEACT_PREAMBLE: &str = include_str!("../../prompts/codeact_preamble.md") /// The strategy/closing block appended after the dynamic metadata sections. const CODEACT_POSTAMBLE: &str = include_str!("../../prompts/codeact_postamble.md"); +/// Structured-tools-only preamble used when `IRONCLAW_DISABLE_CODEACT` is set. +const STRUCTURED_TOOL_PREAMBLE: &str = r#"You are IronClaw, a personal AI assistant. + +## Execution mode + +Use the provider's structured tool_calls interface for every action. +Do not emit Python, repl, py, or other executable fenced code blocks. +Do not call tools as Python functions. +Do not write tool invocations in assistant text. Never output `[[call_tool ...]]`, ``, ``, JSON tool-call blobs, or function-style calls such as `tool_name(...)`. +Only the provider-level `tool_calls` field invokes tools. If you need a tool, return a structured tool call instead of describing or printing the call. +When no action is needed, answer in plain text. +"#; + +/// Structured-tools-only postamble used when `IRONCLAW_DISABLE_CODEACT` is set. +const STRUCTURED_TOOL_POSTAMBLE: &str = r#" +## Strategy + +Use structured tool calls when you need data, persistence, external effects, or system state. +After tool results are available, continue with another structured tool call or return the final plain-text answer. +Some integrations use literal UI blocks such as `[[choice_set]]...[[/choice_set]]` in final user-facing text. These are UI markup only; do not invent other bracketed control blocks, especially `[[call_tool ...]]`. +"#; + +/// Whether CodeAct (Tier 1 Python execution) is disabled by env var. +pub fn codeact_disabled() -> bool { + matches!( + std::env::var("IRONCLAW_DISABLE_CODEACT").as_deref(), + Ok("true" | "1") + ) +} + /// Marker for the engine-owned CodeAct system prompt. const CODEACT_SYSTEM_PROMPT_MARKER: &str = "\n"; const CODEACT_LEGACY_OPENING: &str = "You are an AI assistant with a Python REPL environment."; @@ -133,7 +109,13 @@ pub async fn build_codeact_system_prompt( } else { None }; - build_codeact_system_prompt_inner(capabilities, compact_actions, overlay.as_deref(), platform) + build_codeact_system_prompt_inner( + codeact_disabled(), + capabilities, + compact_actions, + overlay.as_deref(), + platform, + ) } /// Build the system prompt using pre-fetched memory docs. @@ -148,18 +130,36 @@ pub fn build_codeact_system_prompt_with_docs( platform: Option<&PlatformInfo>, ) -> String { let overlay = extract_prompt_overlay(system_docs); - build_codeact_system_prompt_inner(capabilities, compact_actions, overlay.as_deref(), platform) + build_codeact_system_prompt_inner( + codeact_disabled(), + capabilities, + compact_actions, + overlay.as_deref(), + platform, + ) } /// Shared prompt builder used by both the async and pre-fetched-docs variants. -fn build_codeact_system_prompt_inner( +/// +/// `disable_codeact` is threaded as an explicit parameter (rather than read +/// from the env directly) so tests can exercise both branches without +/// process-global env mutation. +pub(crate) fn build_codeact_system_prompt_inner( + disable_codeact: bool, capabilities: &[CapabilitySummary], compact_actions: &[ActionDef], overlay: Option<&str>, platform: Option<&PlatformInfo>, ) -> String { + tracing::debug!(codeact_disabled = disable_codeact, "engine v2 prompt mode"); + let (preamble, postamble) = if disable_codeact { + (STRUCTURED_TOOL_PREAMBLE, STRUCTURED_TOOL_POSTAMBLE) + } else { + (CODEACT_PREAMBLE, CODEACT_POSTAMBLE) + }; + let mut prompt = String::from(CODEACT_SYSTEM_PROMPT_MARKER); - prompt.push_str(CODEACT_PREAMBLE); + prompt.push_str(preamble); // Inject platform identity and runtime metadata if let Some(info) = platform { @@ -184,19 +184,28 @@ fn build_codeact_system_prompt_inner( } } - let compact_actions: Vec<_> = compact_actions - .iter() - .filter(|action| matches!(action.model_tool_surface, ModelToolSurface::CompactToolInfo)) - .collect(); - - if !compact_actions.is_empty() { - prompt.push_str(CODEACT_ENABLED_TOOLS_HEADING); - prompt.push('\n'); - prompt.push_str( - "These enabled tools are shown in compact form. Before calling one, always check its schema with `tool_info(name=\"\", detail=\"schema\")`.\n\n", - ); - for action in compact_actions { - prompt.push_str(&render_enabled_tool(action)); + // In disabled-CodeAct mode the "Enabled Tools" listing is omitted: + // compact actions are emitted into the provider tool list (see + // `LlmBridgeAdapter::complete`) with their full schemas, so the prompt + // would only duplicate that surface and the `tool_info` schema-lookup + // instruction wouldn't apply. Without this guard, compact tools used to + // appear in the prompt as "available" but never made it into + // `tool_calls`, leaving them effectively unreachable (PR #3665 review). + if !disable_codeact { + let compact_actions: Vec<_> = compact_actions + .iter() + .filter(|action| matches!(action.model_tool_surface, ModelToolSurface::CompactToolInfo)) + .collect(); + + if !compact_actions.is_empty() { + prompt.push_str(CODEACT_ENABLED_TOOLS_HEADING); + prompt.push('\n'); + prompt.push_str( + "These enabled tools are shown in compact form. Before calling one, always check its schema with `tool_info(name=\"\", detail=\"schema\")`.\n\n", + ); + for action in compact_actions { + prompt.push_str(&render_enabled_tool(action)); + } } } @@ -204,14 +213,20 @@ fn build_codeact_system_prompt_inner( prompt.push_str(CODEACT_ACTIVATABLE_INTEGRATIONS_HEADING); prompt.push('\n'); prompt.push_str( - "If you need one of these integrations, call `tool_activate(name=\"\")` first. After it succeeds, its tools will be available on the next turn. If you need parameter details before enabling one, call `tool_info(name=\"\", detail=\"summary\")` on one of the previewed tools.\n\n", + "These integrations need user setup before their tools become callable. \ + When the user asks to connect/install/enable one of them, call \ + `tool_install(name=\"\")` directly — don't enumerate alternatives or \ + describe manual UI steps. If credentials are missing the engine raises an \ + auth gate at execute time and the user is prompted in chat. \ + For parameter details before installing, call \ + `tool_info(name=\"\", detail=\"summary\")` on a preview tool.\n\n", ); for capability in activatable_integrations { prompt.push_str(&render_activatable_integration(capability)); } } - prompt.push_str(CODEACT_POSTAMBLE); + prompt.push_str(postamble); prompt } @@ -310,13 +325,16 @@ const fn capability_kind_label(kind: CapabilitySummaryKind) -> &'static str { } fn is_activatable_integration(capability: &CapabilitySummary) -> bool { + // NeedsAuth is intentionally NOT here: post-#3133, installed-but-unauthed + // provider tools are direct-callable (the engine's auth preflight raises + // an Authentication gate at execute time) so they live in the regular + // action inventory, not in the separate setup-required section. matches!( capability.kind, CapabilitySummaryKind::Provider | CapabilitySummaryKind::Channel ) && matches!( capability.status, - CapabilityStatus::NeedsAuth - | CapabilityStatus::NeedsSetup + CapabilityStatus::NeedsSetup | CapabilityStatus::Inactive | CapabilityStatus::Latent | CapabilityStatus::AvailableNotInstalled @@ -576,7 +594,11 @@ mod tests { name: "slack".into(), display_name: None, kind: crate::types::capability::CapabilitySummaryKind::Provider, - status: CapabilityStatus::NeedsAuth, + // NeedsSetup (not NeedsAuth) lands in "Activatable + // Integrations". NeedsAuth tools are direct-callable + // post-#3133, so they live in the regular action + // inventory rather than the setup-required section. + status: CapabilityStatus::NeedsSetup, description: Some("Slack workspace integration".into()), action_preview: vec!["slack_send".into(), "slack_history".into()], routing_hint: None, @@ -593,9 +615,14 @@ mod tests { assert!(prompt.contains("Usable through message")); assert!(prompt.contains("## Activatable Integrations")); assert!(prompt.contains("`slack` [provider]")); - assert!(prompt.contains("tool_activate(name=\"\")")); + assert!(prompt.contains("need user setup before their tools become callable")); assert!(prompt.contains("tool_info(name=\"\", detail=\"summary\")")); assert!(prompt.contains("Unlocks: `slack_send`, `slack_history`")); + // Regression for #3533: the prompt must direct the model to call + // tool_install for activatable integrations instead of narrating + // manual UI steps or enumerating alternatives. + assert!(prompt.contains("tool_install(name=\"\")")); + assert!(prompt.contains("don't enumerate alternatives")); } #[test] @@ -605,7 +632,10 @@ mod tests { name: "gmail".into(), display_name: Some("Gmail".into()), kind: CapabilitySummaryKind::Provider, - status: CapabilityStatus::NeedsAuth, + // NeedsSetup keeps gmail in Activatable Integrations. + // NeedsAuth gmail would render in the regular action + // inventory instead (post-#3133 direct-callable path). + status: CapabilityStatus::NeedsSetup, description: Some("Gmail integration".into()), action_preview: vec!["gmail_send".into()], routing_hint: None, @@ -646,6 +676,29 @@ mod tests { assert_eq!(prompt.matches("`gmail` [provider]").count(), 1); } + #[test] + fn needs_auth_capability_is_not_activatable_integration() { + // Post-#3133: gmail with NeedsAuth status (installed but missing + // OAuth) is direct-callable. The auth gate raises at execute + // time, so the capability does NOT belong in the Activatable + // Integrations section. + let prompt = build_codeact_system_prompt_with_docs( + &[CapabilitySummary { + name: "gmail".into(), + display_name: Some("Gmail".into()), + kind: CapabilitySummaryKind::Provider, + status: CapabilityStatus::NeedsAuth, + description: Some("Gmail integration".into()), + action_preview: vec!["gmail_send".into()], + routing_hint: None, + }], + &[], + &[], + None, + ); + assert!(!prompt.contains("## Activatable Integrations")); + } + #[test] fn prompt_no_longer_duplicates_callable_tool_inventory() { let prompt = build_codeact_system_prompt_with_docs(&[], &[], &[], None); @@ -765,4 +818,75 @@ mod tests { assert!(refreshed.contains("GitHub API Skill")); assert!(refreshed.contains("/missing")); } + + /// PR #3665 review (serrrfirat). With CodeAct disabled the structured-tool + /// prompt previously listed compact actions under "## Enabled Tools" with + /// a `tool_info` schema-lookup instruction — but the LLM adapter only + /// emitted FullSchema actions to the provider tool list. The result was + /// that compact tools (mission_create, gmail_send, notion_search, ...) + /// appeared in the prompt as "available" but could not be called via + /// `tool_calls`. Fix: skip the "Enabled Tools" section in disabled mode + /// and emit every action into the provider tool list instead (the + /// adapter-side half of this fix lives in `src/bridge/llm_adapter.rs`). + #[test] + fn disabled_codeact_omits_enabled_tools_section_and_keeps_activatable() { + let actions = vec![ + ActionDef { + name: "mission_create".into(), + description: "Create scheduled or event-driven missions.".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: Vec::new(), + requires_approval: false, + model_tool_surface: ModelToolSurface::CompactToolInfo, + discovery: None, + }, + ActionDef { + name: "http".into(), + description: "Make HTTP requests.".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: Vec::new(), + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }, + ]; + let capabilities = vec![CapabilitySummary { + name: "gmail".into(), + display_name: Some("Gmail".into()), + kind: CapabilitySummaryKind::Provider, + status: CapabilityStatus::NeedsSetup, + description: Some("Gmail integration".into()), + action_preview: vec!["gmail_send".into()], + routing_hint: None, + }]; + + // Enabled (control): the existing section renders. + let enabled = build_codeact_system_prompt_inner(false, &capabilities, &actions, None, None); + assert!(enabled.contains("## Enabled Tools")); + assert!(enabled.contains("- `mission_create`")); + assert!(enabled.contains("## Activatable Integrations")); + + // Disabled: section is gone, but Activatable Integrations stays + // (the model still needs to know what `tool_install` can target), + // and `mission_create` does NOT appear in the prompt — it's only + // reachable via the provider tool list now. + let disabled = build_codeact_system_prompt_inner(true, &capabilities, &actions, None, None); + assert!( + !disabled.contains("## Enabled Tools"), + "Enabled Tools section must be omitted in disabled-CodeAct mode" + ); + assert!( + !disabled.contains("mission_create"), + "compact action must not appear in prompt — it's in the provider tool list" + ); + assert!( + !disabled.contains("detail=\"schema\""), + "schema-lookup instruction is meaningless when provider sends full schemas \ + (the `detail=\"summary\"` reference in Activatable Integrations is fine)" + ); + assert!( + disabled.contains("## Activatable Integrations"), + "Activatable Integrations is still needed so the model can tool_install" + ); + } } diff --git a/crates/ironclaw_engine/src/executor/scripting.rs b/crates/ironclaw_engine/src/executor/scripting.rs index 67d46d6a16e..cc0129c9d05 100644 --- a/crates/ironclaw_engine/src/executor/scripting.rs +++ b/crates/ironclaw_engine/src/executor/scripting.rs @@ -16,10 +16,43 @@ //! - Output truncated to configurable limit with variable listing //! - `asyncio.gather()` for parallel tool execution (via ResolveFutures) +use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; +tokio::task_local! { + /// Side-channel between `drive_inline_gate`'s `Cancelled+Authentication` + /// fallback and `execute_code`'s exit. When the inline-await for an + /// Authentication gate cancels (e.g. because the controller has no + /// `PerExecutionContext` registered — the typical case for mission + /// child threads), the fallback writes the original `ThreadOutcome::GatePaused` + /// here before raising the legacy `RuntimeError("execution paused by + /// gate ...")`. `execute_code` reads it on the way out and surfaces + /// it as `CodeExecutionResult::need_approval`, which the orchestrator + /// then converts to `ThreadOutcome::GatePaused` so the mission flow + /// (#3133 half-1) transitions the mission to Paused. + /// + /// Without this, Tier 1 mission child threads would silently swallow + /// the gate, the mission would stay Active, and the cron would keep + /// re-firing — the original #3133 ghost-fire pattern. + static PENDING_GATE_STASH: + RefCell>; +} + +/// Drain the per-execution `PENDING_GATE_STASH`. Called at every +/// script-error exit in `execute_code_with_skills_inner` so that a +/// `Cancelled+Authentication` gate raised during a Monty `call.resume` +/// surfaces as `CodeExecutionResult::need_approval` regardless of which +/// resume path the script error propagates through. Returns `None` when +/// the task-local isn't in scope (only happens outside `execute_code`). +fn take_pending_gate_stash() -> Option { + PENDING_GATE_STASH + .try_with(|cell| cell.borrow_mut().take()) + .ok() + .flatten() +} + use monty::{ ExcType, ExtFunctionResult, LimitedTracker, MontyDate, MontyDateTime, MontyException, MontyObject, MontyRun, NameLookupResult, OsFunction, PrintWriter, ResourceLimits, RunProgress, @@ -121,6 +154,17 @@ fn build_date_today() -> MontyObject { } /// Default resource limits for Monty execution. +/// +/// `max_duration` is wall-clock from VM start and ticks during inline +/// gate-await pauses (we await user input *inside* the same Monty +/// execution). 30s is what catches runaway CPU-bound scripts that +/// don't allocate (`while True: x += 1`); raising it to "30 min so +/// human approvals fit" hangs those tests. Tradeoff: with 30s, an +/// approval that takes longer than 30s timeouts the script and the +/// user has to retry. Most approvals come back in seconds; longer +/// ones are a documented limitation. A proper "active CPU vs paused" +/// timer split is on the follow-up list (see +/// `docs/plans/2026-05-01-codeact-inline-gate-await.md`). fn default_limits() -> ResourceLimits { ResourceLimits::new() .max_duration(Duration::from_secs(30)) @@ -245,6 +289,101 @@ pub fn compact_output_metadata(stdout: &str, return_value: &serde_json::Value) - } } +// ── Gate resolution mapping ───────────────────────────────── + +/// Why a gate did not approve. Distinguishes user-driven denial from +/// "no live approval handler reached the user" so script-facing and +/// event-log messages don't mislabel a cancellation/expiry as a user +/// denial. +/// +/// Wrapping behavior used to be `format!("user denied tool 'X': {reason}")` +/// for every non-`Approved` resolution; that incorrectly read +/// "user denied tool 'X': cancelled" when the script ran under +/// [`crate::gate::CancellingGateController`] (no controller wired) or +/// when the bridge controller cancelled on expiry/shutdown. The user +/// never saw a prompt in those cases — they didn't deny anything. +/// +/// Helpers here produce the right wording per surface +/// (event-log error, script-facing exception, `EngineError::Effect` +/// reason) so all Tier 0 / Tier 1 call sites stay consistent. +#[derive(Debug, Clone)] +pub(crate) enum DenialOutcome { + /// User actively denied the gate (or the host's controller treats + /// "no input" as deny). Reason text typically comes from the user + /// or the controller's deny reason. + DeniedByUser { reason: String }, + /// No live approval handler reached the user — controller missing + /// (`CancellingGateController`), bridge controller cancelled on + /// expiry/shutdown, or the engine got back a resolution variant + /// the inline path doesn't support. + Unavailable { detail: String }, +} + +impl DenialOutcome { + /// Pre-formatted `error` string for `EventKind::ActionFailed`. + /// Surfaces in trace/audit/observer paths; a "denied:" prefix here + /// lined up with the policy-deny path before the gate controller + /// existed, so user-driven denials keep that prefix for continuity. + /// `Unavailable` uses a distinct prefix so an operator scanning + /// logs can tell apart "user said no" from "no prompt was shown". + pub(crate) fn event_error(&self) -> String { + match self { + Self::DeniedByUser { reason } => format!("denied: {reason}"), + Self::Unavailable { detail } => format!("approval unavailable: {detail}"), + } + } + + /// Pre-formatted `RuntimeError` message for CodeAct scripts. + /// Identifies the tool by name so scripts can branch on the + /// failure cause, and surfaces the distinction between + /// user-driven denial and no-handler/cancelled directly in the + /// message text — pre-fix the latter incorrectly read + /// "user denied tool 'X': cancelled". + pub(crate) fn script_message(&self, tool_name: &str) -> String { + match self { + Self::DeniedByUser { reason } => { + format!("user denied tool '{tool_name}': {reason}") + } + Self::Unavailable { detail } => { + format!("approval for tool '{tool_name}' unavailable: {detail}") + } + } + } + + /// Bare reason string for `EngineError::Effect` (Tier 0 structured + /// path, where the error gets bubbled up rather than rendered as + /// a Python exception). Same shape as `event_error`. + pub(crate) fn effect_reason(&self) -> String { + self.event_error() + } +} + +/// Single source of truth shared by Tier 0 (`structured.rs`) and Tier 1 +/// (sync preflight + async output paths in this module) so denial +/// messages can't drift between executors. +/// +/// Returns `None` for `Approved` (the only outcome that lets execution +/// continue). +pub(crate) fn denial_outcome_for_resolution( + resolution: &crate::gate::GateResolution, +) -> Option { + match resolution { + crate::gate::GateResolution::Approved { .. } => None, + crate::gate::GateResolution::Denied { reason } => Some(DenialOutcome::DeniedByUser { + reason: reason.clone().unwrap_or_else(|| "denied by user".into()), + }), + crate::gate::GateResolution::Cancelled => Some(DenialOutcome::Unavailable { + detail: "approval cancelled".into(), + }), + crate::gate::GateResolution::CredentialProvided { .. } + | crate::gate::GateResolution::ExternalCallback { .. } => { + Some(DenialOutcome::Unavailable { + detail: "unsupported gate resolution".into(), + }) + } + } +} + // ── Step 0 orientation preamble ───────────────────────────── /// Build the Step 0 orientation preamble that auto-executes before the @@ -418,6 +557,43 @@ pub async fn execute_code_with_skills( capability_policies: &[crate::types::capability::PolicyRule], persisted_state: &serde_json::Value, skill_snippet_names: &[String], +) -> Result { + // Scope the per-execution PENDING_GATE_STASH task-local so the + // inline-await Cancelled+Authentication fallback (deeper inside + // `drive_inline_gate`) has a side channel to surface the original + // gate as `need_approval` on the way out. See the static's + // doc-comment for why this exists. + PENDING_GATE_STASH + .scope(RefCell::new(None), async move { + execute_code_with_skills_inner( + code, + thread, + llm, + effects, + leases, + policy, + context, + capability_policies, + persisted_state, + skill_snippet_names, + ) + .await + }) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn execute_code_with_skills_inner( + code: &str, + thread: &Thread, + llm: &Arc, + effects: &Arc, + leases: &LeaseManager, + policy: &PolicyEngine, + context: &ThreadExecutionContext, + capability_policies: &[crate::types::capability::PolicyRule], + persisted_state: &serde_json::Value, + skill_snippet_names: &[String], ) -> Result { let mut stdout = String::new(); let mut action_results = Vec::new(); @@ -511,14 +687,22 @@ pub async fn execute_code_with_skills( let mut progress = match run_result { Ok(Ok(p)) => p, Ok(Err(e)) => { - // Runtime error flows back to LLM + // Runtime error flows back to LLM. Before classifying it, + // check the inline-await fallback stash: if a Cancelled+ + // Authentication gate fired and surfaced as a + // `RuntimeError("execution paused by gate ...")`, surface + // it as `need_approval` so the orchestrator can produce + // `ThreadOutcome::GatePaused` and mission flows transition + // to Paused. Without this, Tier 1 mission threads silently + // swallow the gate (the original #3133 ghost-fire shape). + let pending_gate = take_pending_gate_stash(); let category = classify_runtime_error(&e.to_string()); return Ok(CodeExecutionResult { return_value: serde_json::Value::Null, stdout: format!("{stdout}\nError: {e}"), action_results, events, - need_approval: None, + need_approval: pending_gate, recursive_tokens, final_answer: None, failure: Some(category), @@ -630,6 +814,7 @@ pub async fn execute_code_with_skills( leases, policy, &mut recursive_tokens, + &execution_context.gate_controller, ) .await, ), @@ -652,13 +837,16 @@ pub async fn execute_code_with_skills( })) { Ok(Ok(p)) => progress = p, Ok(Err(e)) => { + // Read the inline-await stash before the + // runtime-error exit. See `take_pending_gate_stash`. + let pending_gate = take_pending_gate_stash(); stdout.push_str(&format!("\nError: {e}")); return Ok(CodeExecutionResult { return_value: serde_json::Value::Null, stdout, action_results, events, - need_approval: None, + need_approval: pending_gate, recursive_tokens, final_answer, failure: Some(classify_runtime_error(&e.to_string())), @@ -690,13 +878,14 @@ pub async fn execute_code_with_skills( })) { Ok(Ok(p)) => progress = p, Ok(Err(e)) => { + let pending_gate = take_pending_gate_stash(); stdout.push_str(&format!("\nError: {e}")); return Ok(CodeExecutionResult { return_value: serde_json::Value::Null, stdout, action_results, events, - need_approval: None, + need_approval: pending_gate, recursive_tokens, final_answer, failure: Some(classify_runtime_error(&e.to_string())), @@ -764,7 +953,6 @@ pub async fn execute_code_with_skills( action_name, call_id: str_call_id, lease_id: lease.id, - parameters: params.clone(), params_summary: ps, }, ); @@ -839,16 +1027,250 @@ pub async fn execute_code_with_skills( } } PreflightResult::GatePaused(outcome) => { - return Ok(CodeExecutionResult { - return_value: serde_json::Value::Null, - stdout, - action_results, - events, - need_approval: Some(outcome), - recursive_tokens, - final_answer: None, - failure: None, - }); + // Inline gate-await: keep the Monty VM alive, + // pause for the user, and continue from the + // exact suspension point on resolution. The + // controller is required on the context — code + // paths that don't pause supply + // `CancellingGateController`, which surfaces + // gates as a typed denial here. + let crate::runtime::messaging::ThreadOutcome::GatePaused { + gate_name, + action_name: gate_action_name, + call_id: gate_call_id, + parameters: gate_parameters, + resume_kind, + .. + } = outcome + else { + // ThreadOutcome::GatePaused is the only variant + // PreflightResult::GatePaused builds; falling + // back here would indicate a programmer error. + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout, + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::ToolError), + }); + }; + + let resolution = execution_context + .gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id: thread.id, + user_id: thread.user_id.clone(), + gate_name: gate_name.clone(), + action_name: gate_action_name.clone(), + call_id: gate_call_id.clone(), + parameters: gate_parameters.clone(), + resume_kind: resume_kind.clone(), + conversation_id: execution_context.conversation_id, + }) + .await; + + let denial = denial_outcome_for_resolution(&resolution); + + if let Some(outcome) = denial { + // Record the denial in the thread event log + // before resuming Monty so observers / trace + // analysis see consistent ActionFailed output + // across all denial paths (this site + + // `drive_inline_gate` + `structured.rs`). + events.push(EventKind::ActionFailed { + step_id: execution_context.step_id, + action_name: gate_action_name.clone(), + call_id: gate_call_id.clone(), + error: outcome.event_error(), + duration_ms: 0, + params_summary: crate::types::event::summarize_params( + &gate_action_name, + &gate_parameters, + ), + }); + // Resume Monty with a typed exception. RuntimeError + // is what we emit; the message is explicit so users + // (and the LLM) can distinguish denial from other + // runtime errors. `script_message` distinguishes a + // user-driven denial ("user denied tool 'X': ...") + // from a no-handler / expired / cancelled gate + // ("approval for tool 'X' unavailable: ..."). + let ext_result = ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(outcome.script_message(&gate_action_name)), + )); + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + call.resume(ext_result, PrintWriter::CollectString(&mut stdout)) + })) { + Ok(Ok(p)) => progress = p, + Ok(Err(e)) => { + stdout.push_str(&format!("\nError: {e}")); + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout, + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::ToolError), + }); + } + Err(_) => { + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout: format!( + "{stdout}\nVmPanic: Monty VM panicked during resume" + ), + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::VmPanic), + }); + } + } + continue; + } + + // Approved. Re-do preflight — the bridge installed + // any auto-approve preference before delivering the + // resolution, so policy now returns Allow. + let retry_preflight = preflight_action( + &gate_action_name, + &gate_parameters, + thread, + leases, + policy, + &execution_context, + capability_policies, + &gate_call_id, + &mut events, + ) + .await; + match retry_preflight { + PreflightResult::Approved(lease) => { + let effects = effects.clone(); + let name = gate_action_name.clone(); + let params_clone = gate_parameters.clone(); + let lease_clone = lease.clone(); + let mut ctx = execution_context.clone(); + ctx.current_call_id = Some(gate_call_id.clone()); + // Carry the user's one-shot approval + // into the retry call so the host + // skips its per-call approval check. + ctx.call_approval_granted = true; + let ps = + crate::types::event::summarize_params(&name, &gate_parameters); + + let handle = tokio::spawn(async move { + let execution_start = Instant::now(); + let result = effects + .execute_action(&name, params_clone, &lease_clone, &ctx) + .await; + (result, execution_start.elapsed().as_millis() as u64) + }); + + pending_futures.insert( + monty_call_id, + PendingFuture::Tool { + handle, + action_name: gate_action_name, + call_id: gate_call_id, + lease_id: lease.id, + params_summary: ps, + }, + ); + + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + call.resume_pending(PrintWriter::CollectString(&mut stdout)) + })) { + Ok(Ok(p)) => progress = p, + Ok(Err(e)) => { + stdout.push_str(&format!("\nError: {e}")); + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout, + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::ToolError), + }); + } + Err(_) => { + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout: format!( + "{stdout}\nVmPanic: Monty VM panicked during resume_pending" + ), + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::VmPanic), + }); + } + } + } + PreflightResult::Denied(ext_result) => { + // Race: someone changed the lease/policy + // between approval and retry. Surface the + // error to Python so the script can handle + // it (or crash uncaught). + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + call.resume(ext_result, PrintWriter::CollectString(&mut stdout)) + })) { + Ok(Ok(p)) => progress = p, + _ => { + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout, + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::ToolError), + }); + } + } + } + PreflightResult::GatePaused(_) => { + // Policy still says approval needed even + // after user said yes. Treat as denial so + // we don't loop forever. + let ext_result = ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(format!( + "tool '{gate_action_name}' still requires approval after resolution" + )), + )); + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + call.resume(ext_result, PrintWriter::CollectString(&mut stdout)) + })) { + Ok(Ok(p)) => progress = p, + _ => { + return Ok(CodeExecutionResult { + return_value: serde_json::Value::Null, + stdout, + action_results, + events, + need_approval: None, + recursive_tokens, + final_answer, + failure: Some(CodeExecutionFailure::ToolError), + }); + } + } + } + } } } } @@ -872,7 +1294,6 @@ pub async fn execute_code_with_skills( action_name, call_id, lease_id, - parameters, params_summary, } => { resolve_tool_future( @@ -880,9 +1301,9 @@ pub async fn execute_code_with_skills( &action_name, &call_id, lease_id, - parameters, params_summary, leases, + effects, context, &mut action_results, &mut events, @@ -908,13 +1329,14 @@ pub async fn execute_code_with_skills( })) { Ok(Ok(p)) => progress = p, Ok(Err(e)) => { + let pending_gate = take_pending_gate_stash(); stdout.push_str(&format!("\nError: {e}")); return Ok(CodeExecutionResult { return_value: serde_json::Value::Null, stdout, action_results, events, - need_approval: None, + need_approval: pending_gate, recursive_tokens, final_answer, failure: Some(classify_runtime_error(&e.to_string())), @@ -1098,12 +1520,18 @@ pub fn code_hash(code: &str) -> String { /// via `ResolveFutures`. Can be a tool execution or an LLM call. enum PendingFuture { /// Tool action execution. + /// + /// We deliberately don't carry the call's `parameters` here: when + /// the tool returns `EngineError::GatePaused`, the gate's own + /// parameter snapshot (potentially safety-transformed) is the + /// source of truth for the user-facing prompt and the inline + /// retry. Caching the original would make it a misleading second + /// source. Tool { handle: tokio::task::JoinHandle<(Result, u64)>, action_name: String, call_id: String, lease_id: crate::types::capability::LeaseId, - parameters: serde_json::Value, params_summary: Option, }, /// LLM call (llm_query / llm_query_batched / rlm_query). @@ -1518,6 +1946,7 @@ async fn handle_rlm_query( leases: &LeaseManager, policy: &PolicyEngine, recursive_tokens: &mut TokenUsage, + gate_controller: &Arc, ) -> ExtFunctionResult { let prompt = extract_string_arg(args, kwargs, "prompt", 0); let prompt = match prompt { @@ -1618,6 +2047,7 @@ async fn handle_rlm_query( child_policy, rx, "rlm_child".to_string(), + gate_controller.clone(), ); debug!( @@ -1682,16 +2112,327 @@ async fn handle_llm_query_batched_standalone( // ── Future resolution helpers ─────────────────────────────── +/// Maximum number of inline gate-await iterations for a single tool +/// call. The first attempt comes from the caller; this cap covers +/// retries triggered by post-approval policy still demanding approval +/// (e.g. a second gate kicks in after auto-approve was installed). +/// Three is enough for any plausible chain — a tool that gates more +/// than that is misbehaving and we'd rather surface a clean error +/// than spin forever. +/// +/// Shared with Tier 0 (`structured::execute_with_inline_gate_retry`) +/// so both executors enforce the same upper bound. +pub(crate) const MAX_INLINE_GATE_RETRIES: usize = 3; + +/// Inputs needed to drive one inline gate await. +struct InlineGate { + gate_name: String, + action_name: String, + call_id: String, + parameters: serde_json::Value, + resume_kind: crate::gate::ResumeKind, + /// Pre-computed action output cached at gate-raise time. When the action + /// has *already executed* and only a follow-up resolution (e.g. OAuth) is + /// pending — `effect_adapter` raising an Authentication gate after a + /// successful `tool_install` is the canonical case — the bridge attaches + /// the install's output here. On resolution we return that cached output + /// instead of re-executing the action, which would otherwise re-download + /// the WASM bundle and re-raise a fresh approval gate (#3533 follow-up). + resume_output: Option, +} + +/// Drive an `Approval` gate to terminal resolution, retrying the +/// action up to [`MAX_INLINE_GATE_RETRIES`] times if the post-approval +/// retry itself returns `GatePaused`. +/// +/// Centralizes Tier 1's gate handling so the async output path and the +/// sync preflight path emit consistent events / error messages — and +/// so a misbehaving tool (gates repeatedly after approval) produces a +/// bounded `RuntimeError` instead of leaking the legacy +/// "execution paused by gate" message. +#[allow(clippy::too_many_arguments)] +async fn drive_inline_gate( + mut gate: InlineGate, + leases: &LeaseManager, + effects: &Arc, + context: &ThreadExecutionContext, + action_results: &mut Vec, + events: &mut Vec, + params_summary: Option, +) -> ExtFunctionResult { + for _ in 0..MAX_INLINE_GATE_RETRIES { + let resolution = context + .gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id: context.thread_id, + user_id: context.user_id.clone(), + gate_name: gate.gate_name.clone(), + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + parameters: gate.parameters.clone(), + resume_kind: gate.resume_kind.clone(), + conversation_id: context.conversation_id, + }) + .await; + + if let Some(outcome) = denial_outcome_for_resolution(&resolution) { + // Cancelled+Authentication → unwind via the legacy + // `RuntimeError("execution paused by gate ...")` so the + // outer orchestrator can produce `ThreadOutcome::GatePaused` + // and missions can transition to Paused. Cancelled here + // means the controller can't resolve the auth inline (no + // OAuth wiring) — the legacy unwind path is the right + // fallback. Denied / explicit user-cancel remain failures. + if matches!(resolution, crate::gate::GateResolution::Cancelled) + && matches!( + gate.resume_kind, + crate::gate::ResumeKind::Authentication { .. } + ) + { + // Stash the original gate so `execute_code`'s exit can + // surface it as `need_approval`. Without this, Tier 1 + // mission child threads silently swallow the gate and + // the cron keeps re-firing the mission (#3133 ghost + // fire). Best-effort — if the task-local isn't in + // scope (theoretically impossible since we always run + // inside `execute_code_with_skills`'s scope, but + // defensive), `try_with` no-ops. + let _ = PENDING_GATE_STASH.try_with(|cell| { + *cell.borrow_mut() = + Some(crate::runtime::messaging::ThreadOutcome::GatePaused { + gate_name: gate.gate_name.clone(), + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + parameters: gate.parameters.clone(), + resume_kind: gate.resume_kind.clone(), + resume_output: None, + paused_lease: None, + }); + }); + return ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(format!("execution paused by gate '{}'", gate.gate_name)), + )); + } + events.push(EventKind::ActionFailed { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + error: outcome.event_error(), + duration_ms: 0, + params_summary, + }); + return ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(outcome.script_message(&gate.action_name)), + )); + } + + // Approved. If the bridge cached the action's output before raising + // this gate (post-execution Authentication gate path — see + // `effect_adapter::auth_gate_from_extension_result` and the + // `check_tool_readiness` path), the action has already run and we + // just needed user-side resolution. Skip re-execution and return + // the cached output directly. Without this short-circuit, the + // retry re-runs `tool_install` (re-downloading the WASM) and the + // second pass through `effect_adapter::enforce_tool_permission` + // raises a brand-new approval gate that the user has no way to + // resolve. Tracked by #3533. + if let Some(cached_output) = gate.resume_output.take() { + events.push(EventKind::ActionExecuted { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + duration_ms: 0, + params_summary, + }); + let monty_val = json_to_monty(&cached_output); + action_results.push(ActionResult { + call_id: gate.call_id.clone(), + action_name: gate.action_name.clone(), + output: cached_output, + is_error: false, + duration: std::time::Duration::ZERO, + }); + return ExtFunctionResult::Return(monty_val); + } + + // Re-acquire a lease use and retry the action. The bridge installed + // any auto-approve preference before delivering the resolution, so + // policy now returns Allow. + // + // Note: `find_and_consume` may select a different lease than + // the originally-refunded one if multiple grants cover this + // action. That's fine for the use-counter contract; if leases + // ever carry per-grant identity (credential bindings) this + // assumption needs to be re-evaluated. + let lease = match leases + .find_and_consume(context.thread_id, &gate.action_name) + .await + { + Ok(l) => l, + Err(e) => { + return ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(format!("lease unavailable after approval: {e}")), + )); + } + }; + // Carry the user's one-shot approval into the retry call so + // the host's `EffectExecutor` skips the + // `ApprovalRequirement::Always` / AskEachTime gate that would + // otherwise fire again. Mirrors the legacy + // `execute_resolved_pending_action(approval_already_granted=true)` + // path. Scoped to this single call by `current_call_id`. + let mut retry_ctx = context.clone(); + retry_ctx.current_call_id = Some(gate.call_id.clone()); + retry_ctx.call_approval_granted = true; + let retry_start = Instant::now(); + let retry_result = effects + .execute_action( + &gate.action_name, + gate.parameters.clone(), + &lease, + &retry_ctx, + ) + .await; + let retry_duration_ms = retry_start.elapsed().as_millis() as u64; + + match retry_result { + Ok(result) => { + if result.is_error { + let error_msg = result + .output + .get("error") + .and_then(|v| v.as_str()) + .map(String::from) + .unwrap_or_else(|| result.output.to_string()); + let duration_ms = result.duration.as_millis() as u64; + events.push(EventKind::ActionFailed { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + error: error_msg, + duration_ms: if duration_ms > 0 { + duration_ms + } else { + retry_duration_ms + }, + params_summary, + }); + } else { + events.push(EventKind::ActionExecuted { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + duration_ms: result.duration.as_millis() as u64, + params_summary, + }); + } + let monty_val = json_to_monty(&result.output); + action_results.push(result); + return ExtFunctionResult::Return(monty_val); + } + Err(EngineError::GatePaused { + gate_name, + action_name, + call_id, + parameters, + resume_kind, + resume_output, + .. + }) if matches!( + *resume_kind, + crate::gate::ResumeKind::Approval { .. } + | crate::gate::ResumeKind::Authentication { .. } + ) => + { + // Refund the use we just consumed — the next loop + // iteration will pause and re-consume on resolution. + // EXCEPTION: when the retry's gate carries a cached + // `resume_output`, the next iteration will return that + // cached output without re-consuming; refunding here + // would zero out the lease use the retry already spent. + if resume_output.is_none() { + let _ = leases.refund_use(lease.id).await; + } + events.push(EventKind::ApprovalRequested { + action_name: action_name.clone(), + call_id: call_id.clone(), + parameters: Some((*parameters).clone()), + description: None, + allow_always: match *resume_kind { + crate::gate::ResumeKind::Approval { allow_always } => Some(allow_always), + _ => None, + }, + gate_name: Some(gate_name.clone()), + params_summary: params_summary.clone(), + }); + gate = InlineGate { + gate_name, + action_name, + call_id, + parameters: *parameters, + resume_kind: *resume_kind, + resume_output: resume_output.map(|b| *b), + }; + continue; + } + Err(e) => { + events.push(EventKind::ActionFailed { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + error: e.to_string(), + duration_ms: retry_duration_ms, + params_summary, + }); + return ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(e.to_string()), + )); + } + } + } + + // Retry budget exhausted — the tool kept gating after every + // approval. Surface as a typed error so the script can react; + // the user already approved this many times in a row, no point + // asking again. + events.push(EventKind::ActionFailed { + step_id: context.step_id, + action_name: gate.action_name.clone(), + call_id: gate.call_id.clone(), + error: format!("tool kept gating after {MAX_INLINE_GATE_RETRIES} approvals"), + duration_ms: 0, + params_summary, + }); + ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(format!( + "tool '{}' still requires approval after {MAX_INLINE_GATE_RETRIES} retries", + gate.action_name + )), + )) +} + /// Resolve a pending tool execution future. +/// +/// Deliberately does NOT take the original `parameters`: when a tool +/// returns `EngineError::GatePaused`, the gate carries its own +/// parameter snapshot (possibly transformed by the safety layer) and +/// that's what we surface to the user. Threading the original +/// parameters through here would make them a misleading second +/// source of truth. #[allow(clippy::too_many_arguments)] async fn resolve_tool_future( handle: tokio::task::JoinHandle<(Result, u64)>, action_name: &str, call_id: &str, lease_id: crate::types::capability::LeaseId, - parameters: serde_json::Value, params_summary: Option, leases: &LeaseManager, + effects: &Arc, context: &ThreadExecutionContext, action_results: &mut Vec, events: &mut Vec, @@ -1740,30 +2481,81 @@ async fn resolve_tool_future( Ok(( Err(EngineError::GatePaused { gate_name, - action_name, - call_id, + action_name: gate_action_name, + call_id: gate_call_id, + parameters: gate_parameters, resume_kind, + resume_output, .. }), _, )) => { - let _ = leases.refund_use(lease_id).await; + // Skip the refund when the gate carries cached `resume_output`: + // the action has already executed (post-execution Authentication + // gate), and `drive_inline_gate` will return the cached output + // on approval without re-consuming a lease. Refunding here would + // let a successful side-effecting action consume zero uses. + // Matching guards live in `structured::execute_with_inline_gate_retry` + // and `orchestrator::execute_action_with_inline_gate`. Tracked by + // the #3559 security review. + if resume_output.is_none() { + let _ = leases.refund_use(lease_id).await; + } events.push(EventKind::ApprovalRequested { - action_name, - call_id, - parameters: Some(parameters), + action_name: gate_action_name.clone(), + call_id: gate_call_id.clone(), + parameters: Some((*gate_parameters).clone()), description: None, allow_always: match *resume_kind { crate::gate::ResumeKind::Approval { allow_always } => Some(allow_always), _ => None, }, gate_name: Some(gate_name.clone()), - params_summary, + params_summary: params_summary.clone(), }); - ExtFunctionResult::Error(MontyException::new( - ExcType::RuntimeError, - Some(format!("execution paused by gate '{gate_name}'")), - )) + + // External resume kinds keep the legacy re-entry path — + // their resolution installs callback-payload state that + // can't be handed back to a suspended call. Approval and + // Authentication both go through `drive_inline_gate`: + // Approval resolves on user click, Authentication resolves + // when `bridge::resolve_inline_gates_for_credential` (the + // OAuth-callback hook from #3133 half-2) delivers + // `GateResolution::Approved` to the parked controller. + // (Mission-scoped resumes go through + // `bridge::resume_paused_missions_for_credential` — that's + // a separate path for background missions whose child + // threads were paused on the same gate.) In both + // inline-await cases the action retries inline and the + // script continues without unwinding. + if !matches!( + *resume_kind, + crate::gate::ResumeKind::Approval { .. } + | crate::gate::ResumeKind::Authentication { .. } + ) { + return ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(format!("execution paused by gate '{gate_name}'")), + )); + } + + drive_inline_gate( + InlineGate { + gate_name, + action_name: gate_action_name, + call_id: gate_call_id, + parameters: *gate_parameters, + resume_kind: *resume_kind, + resume_output: resume_output.map(|b| *b), + }, + leases, + effects, + context, + action_results, + events, + params_summary, + ) + .await } Ok((Err(e), execution_duration_ms)) => { events.push(EventKind::ActionFailed { @@ -2076,6 +2868,10 @@ mod tests { thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } @@ -3567,6 +4363,371 @@ result = await tool_info(name="mission-create", detail="schema") ); } + // ── Inline gate-await tests ───────────────────────────────── + + /// Effects stub: returns `Err(EngineError::GatePaused)` on first + /// call for the given action, then a success result. Mimics a tool + /// that gates mid-execution (e.g. policy escalation, leak detection). + struct GatingThenOkEffects { + action: String, + success_output: serde_json::Value, + call_count: Mutex, + /// Captures `context.call_approval_granted` for every call. + /// Lets tests assert the inline-retry path correctly forwards + /// the user's one-shot approval to the host's + /// `EffectExecutor` — the integration point where bugs in + /// approval propagation surface. + approval_flags_observed: Mutex>, + actions: Vec, + } + + impl GatingThenOkEffects { + fn new(action: &str, output: serde_json::Value) -> Self { + Self { + action: action.into(), + success_output: output, + call_count: Mutex::new(0), + approval_flags_observed: Mutex::new(Vec::new()), + actions: vec![test_action(action)], + } + } + fn calls(&self) -> u32 { + *self.call_count.lock().unwrap() + } + fn approval_flags(&self) -> Vec { + self.approval_flags_observed.lock().unwrap().clone() + } + } + + #[async_trait::async_trait] + impl EffectExecutor for GatingThenOkEffects { + async fn execute_action( + &self, + name: &str, + params: serde_json::Value, + _lease: &CapabilityLease, + ctx: &ThreadExecutionContext, + ) -> Result { + let mut count = self.call_count.lock().unwrap(); + *count += 1; + self.approval_flags_observed + .lock() + .unwrap() + .push(ctx.call_approval_granted); + if *count == 1 && name == self.action { + return Err(EngineError::GatePaused { + gate_name: "approval".into(), + action_name: name.into(), + call_id: "test_call".into(), + parameters: Box::new(params), + resume_kind: Box::new(crate::gate::ResumeKind::Approval { allow_always: true }), + resume_output: None, + paused_lease: None, + }); + } + Ok(ActionResult { + call_id: String::new(), + action_name: name.into(), + output: self.success_output.clone(), + is_error: false, + duration: Duration::from_millis(1), + }) + } + + async fn available_actions( + &self, + _: &[CapabilityLease], + _: &ThreadExecutionContext, + ) -> Result, EngineError> { + Ok(self.actions.clone()) + } + + async fn available_capabilities( + &self, + _: &[CapabilityLease], + _: &ThreadExecutionContext, + ) -> Result, EngineError> { + Ok(vec![]) + } + } + + /// Test gate controller. Returns the canned resolution and records + /// every pause request for assertion. + struct StubGateController { + resolution: Mutex>, + pauses: Mutex>, + } + + impl StubGateController { + fn approving() -> Arc { + Arc::new(Self { + resolution: Mutex::new(Some(crate::gate::GateResolution::Approved { + always: false, + })), + pauses: Mutex::new(Vec::new()), + }) + } + fn denying() -> Arc { + Arc::new(Self { + resolution: Mutex::new(Some(crate::gate::GateResolution::Denied { + reason: Some("not now".into()), + })), + pauses: Mutex::new(Vec::new()), + }) + } + fn pause_count(&self) -> usize { + self.pauses.lock().unwrap().len() + } + } + + #[async_trait::async_trait] + impl crate::gate::GateController for StubGateController { + async fn pause( + &self, + request: crate::gate::GatePauseRequest, + ) -> crate::gate::GateResolution { + self.pauses.lock().unwrap().push(request); + self.resolution + .lock() + .unwrap() + .clone() + .unwrap_or(crate::gate::GateResolution::Cancelled) + } + } + + /// Regression for the user-reported bug: a CodeAct script that + /// calls a tool which gates mid-execution should NOT abort with + /// `RuntimeError: execution paused by gate 'approval'`. With a + /// controller wired and an `Approved` resolution, the script runs + /// to completion and the tool's success result is delivered to + /// Python. + #[tokio::test] + async fn codeact_gate_inline_await_approved_delivers_result() { + let thread = make_test_thread(); + let effects = Arc::new(GatingThenOkEffects::new( + "github_search", + serde_json::json!({"items": [{"number": 1, "title": "ok"}]}), + )); + let effects_dyn: Arc = effects.clone(); + let controller = StubGateController::approving(); + let leases = LeaseManager::new(); + let policy = PolicyEngine::new(); + let mut ctx = make_exec_context(&thread); + ctx.gate_controller = controller.clone() as Arc; + + leases + .grant(thread.id, "tools", GrantedActions::All, None, None) + .await + .unwrap(); + + let result = execute_code( + r#" +result = await github_search(query="repo:foo") +print(result) +"#, + &thread, + &(Arc::new(StubLlm) as Arc), + &effects_dyn, + &leases, + &policy, + &ctx, + &[], + &serde_json::json!({}), + ) + .await + .expect("execute_code did not return Err"); + + assert!( + result.failure.is_none(), + "expected clean completion, got failure: {:?}, stdout={}", + result.failure, + result.stdout + ); + assert_eq!(controller.pause_count(), 1, "controller should pause once"); + assert!( + !result.stdout.contains("execution paused by gate"), + "stdout must not contain the pre-fix error string; got: {}", + result.stdout + ); + assert_eq!( + effects.calls(), + 2, + "expected one gating call + one retry after approval" + ); + // The retry call MUST observe `call_approval_granted=true` so + // the host's `EffectExecutor` skips its per-call approval + // check. This is the regression coverage for serrrfirat's + // review on PR #3157: without one-shot propagation, tools + // with `ApprovalRequirement::Always` would gate again on + // retry and the approval loop would never converge. + assert_eq!( + effects.approval_flags(), + vec![false, true], + "first call must be unapproved, retry must carry the user's approval" + ); + assert_eq!(result.action_results.len(), 1); + assert!(!result.action_results[0].is_error); + assert_eq!( + result.action_results[0].output["items"][0]["number"], + serde_json::json!(1) + ); + } + + /// Denial surfaces inside the script as a typed `RuntimeError` + /// with a clear message — catchable by user code. + #[tokio::test] + async fn codeact_gate_inline_await_denied_raises_in_script() { + let thread = make_test_thread(); + let effects = Arc::new(GatingThenOkEffects::new( + "github_create_issue", + serde_json::json!({"unused": true}), + )); + let effects_dyn: Arc = effects.clone(); + let controller = StubGateController::denying(); + let leases = LeaseManager::new(); + let policy = PolicyEngine::new(); + let mut ctx = make_exec_context(&thread); + ctx.gate_controller = controller.clone() as Arc; + + leases + .grant(thread.id, "tools", GrantedActions::All, None, None) + .await + .unwrap(); + + // Uncaught: the script raises; the failure is observable on + // CodeExecutionResult and the message identifies the tool + + // reason so the caller / LLM can decide what to do next. + let result = execute_code( + r#" +result = await github_create_issue(title="x") +print(result) +"#, + &thread, + &(Arc::new(StubLlm) as Arc), + &effects_dyn, + &leases, + &policy, + &ctx, + &[], + &serde_json::json!({}), + ) + .await + .expect("execute_code did not return Err"); + + assert_eq!(controller.pause_count(), 1); + // The retry never happens on denial — only the initial call. + assert_eq!(effects.calls(), 1, "denial must not retry the action"); + assert_eq!( + result.failure, + Some(CodeExecutionFailure::RuntimeError), + "denial must surface as RuntimeError on CodeExecutionResult" + ); + // The stdout message must identify the tool and the reason so + // the caller can debug / surface to the LLM. + assert!( + result + .stdout + .contains("user denied tool 'github_create_issue'"), + "denial message must identify the tool; got stdout: {}", + result.stdout + ); + assert!( + result.stdout.contains("not now"), + "denial message must include the user-supplied reason; got stdout: {}", + result.stdout + ); + // The pre-fix message must NOT appear — that was the user's + // reported bug ("Error: RuntimeError: execution paused by + // gate 'approval'"). + assert!( + !result.stdout.contains("execution paused by gate"), + "denial must not surface as the pre-fix bug message; got: {}", + result.stdout + ); + } + + /// With the default `CancellingGateController`, an `Approval` gate + /// raised mid-execution surfaces as a typed cancellation that the + /// script can catch as `RuntimeError`. The message must read + /// "approval … unavailable" rather than "user denied" — the user + /// never saw a prompt, so the deny framing would be misleading. + /// + /// Two regression checks in one test: + /// - the pre-fix bug message (`"execution paused by gate 'approval'"`) + /// must NEVER appear — removing the inline-await wiring would + /// otherwise silently reproduce the original bug. + /// - the message must NOT misattribute the cancellation to a user + /// denial; a future change that broadens `DenialOutcome::DeniedByUser` + /// to cover `Cancelled` would regress to the misleading wording. + #[tokio::test] + async fn codeact_default_controller_cancels_approval_gates() { + let thread = make_test_thread(); + let effects = Arc::new(GatingThenOkEffects::new( + "github_create_issue", + serde_json::json!({"unused": true}), + )); + let effects_dyn: Arc = effects.clone(); + let leases = LeaseManager::new(); + let policy = PolicyEngine::new(); + // make_exec_context defaults to `CancellingGateController` — + // this is the inert controller every test path uses, and + // matches what production paths that don't pause supply. + let ctx = make_exec_context(&thread); + + leases + .grant(thread.id, "tools", GrantedActions::All, None, None) + .await + .unwrap(); + + let result = execute_code( + r#" +try: + result = await github_create_issue(title="x") + outcome = "approved" +except RuntimeError as e: + outcome = str(e) +print(outcome) +"#, + &thread, + &(Arc::new(StubLlm) as Arc), + &effects_dyn, + &leases, + &policy, + &ctx, + &[], + &serde_json::json!({}), + ) + .await + .expect("execute_code did not return Err"); + + assert!( + !result.stdout.contains("execution paused by gate"), + "the legacy bug message must never surface; got: {}", + result.stdout + ); + // CancellingGateController returns `Cancelled`, which now maps + // to the `Unavailable` outcome — the script sees + // "approval for tool '…' unavailable: approval cancelled". + assert!( + result + .stdout + .contains("approval for tool 'github_create_issue' unavailable"), + "default controller must surface as approval-unavailable, not user-denied; got: {}", + result.stdout + ); + // Cancellation must NOT misattribute to a user denial — the + // user never saw a prompt. A regression to "user denied tool + // …: cancelled" would silently reintroduce the misleading + // wording this DenialOutcome split exists to fix. + assert!( + !result.stdout.contains("user denied tool"), + "cancellation must not surface as user-denied; got: {}", + result.stdout + ); + // Cancellation does not retry the action. + assert_eq!(effects.calls(), 1); + } + // ── Error classification tests ────────────────────────────── #[test] diff --git a/crates/ironclaw_engine/src/executor/structured.rs b/crates/ironclaw_engine/src/executor/structured.rs index e759c23fbb6..2b3f3be0fcf 100644 --- a/crates/ironclaw_engine/src/executor/structured.rs +++ b/crates/ironclaw_engine/src/executor/structured.rs @@ -56,14 +56,13 @@ pub async fn execute_action_calls( calls: &[ActionCall], thread: &Thread, effects: &Arc, - leases: &LeaseManager, + leases: &Arc, policy: &PolicyEngine, context: &ThreadExecutionContext, capability_policies: &[crate::types::capability::PolicyRule], ) -> Result { let mut preflight_results: Vec = Vec::with_capacity(calls.len()); let mut early_events = Vec::new(); - let mut early_results = Vec::new(); let active_leases = leases.active_for_thread(thread.id).await; let available_inventory = Arc::new( effects @@ -183,38 +182,75 @@ pub async fn execute_action_calls( continue; } PolicyDecision::RequireApproval { .. } => { - // Collect error results from earlier preflight failures - for pf in preflight_results { - if let PreflightOutcome::Error { result, event, .. } = pf { - early_results.push(result); - early_events.push(event); - } - } + // Inline gate-await: pause this preflight loop in place + // until the user resolves the gate. On approval, fall + // through to lease consumption and queue the call for + // execution. On denial, mark the call failed and continue + // preflight for the rest of the batch — same blast radius + // as a policy-Deny. + // + // The controller is required on the context. Code paths + // that don't pause supply `CancellingGateController`, + // which surfaces as a typed denial here. + // + // Policy doesn't carry the `allow_always` axis; default + // to the historical value (`true`) so the UI offers it. + let resume_kind = crate::gate::ResumeKind::Approval { allow_always: true }; early_events.push(EventKind::ApprovalRequested { action_name: call.action_name.clone(), call_id: call.id.clone(), parameters: Some(call.parameters.clone()), description: None, - allow_always: None, - gate_name: None, + allow_always: Some(true), + gate_name: Some("approval".into()), params_summary: crate::types::event::summarize_params( &call.action_name, &call.parameters, ), }); - return Ok(ActionBatchResult { - results: early_results, - events: early_events, - need_approval: Some(ThreadOutcome::GatePaused { + let resolution = context + .gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id: thread.id, + user_id: thread.user_id.clone(), gate_name: "approval".into(), action_name: call.action_name.clone(), call_id: call.id.clone(), parameters: call.parameters.clone(), - resume_kind: crate::gate::ResumeKind::Approval { allow_always: true }, - resume_output: None, - paused_lease: None, - }), - }); + resume_kind, + conversation_id: context.conversation_id, + }) + .await; + + let denial = crate::executor::scripting::denial_outcome_for_resolution(&resolution); + if let Some(outcome) = denial { + let error_msg = outcome.event_error(); + let error_result = ActionResult { + call_id: call.id.clone(), + action_name: call.action_name.clone(), + output: serde_json::json!({"error": &error_msg}), + is_error: true, + duration: std::time::Duration::ZERO, + }; + let event = EventKind::ActionFailed { + step_id: context.step_id, + action_name: call.action_name.clone(), + call_id: call.id.clone(), + error: error_msg, + duration_ms: 0, + params_summary: crate::types::event::summarize_params( + &call.action_name, + &call.parameters, + ), + }; + preflight_results.push(PreflightOutcome::Error { + index: idx, + result: error_result, + event, + }); + continue; + } + // Approved: fall through to lease-consume + runnable-queue. } PolicyDecision::Allow => {} } @@ -233,8 +269,14 @@ pub async fn execute_action_calls( // All approved calls run concurrently. Results are collected in a // HashMap keyed by original index, then merged in order. - // Separate runnable from preflight errors - let mut slot_results: Vec> = vec![None; calls.len()]; + // Separate runnable from preflight errors. Each slot carries the + // call's terminal `(ActionResult, EventKind)` plus any + // pre-terminal `ApprovalRequested` events emitted by the inline + // retry helper. Pre-terminal events are flushed before the + // terminal event in the merge phase so audit observers see + // "approval asked → action executed/failed" in order. + let mut slot_results: Vec)>> = + vec![None; calls.len()]; let mut runnable_indices = Vec::new(); for pf in preflight_results { @@ -245,7 +287,7 @@ pub async fn execute_action_calls( event, .. } => { - slot_results[index] = Some((result, event)); + slot_results[index] = Some((result, event, Vec::new())); } PreflightOutcome::Runnable { index, lease } => { runnable_indices.push((index, lease)); @@ -260,28 +302,44 @@ pub async fn execute_action_calls( let exec_ctx = stamp_execution_context(context, &call.id, &available_actions, &available_inventory); let execution_start = Instant::now(); - let exec_result = effects - .execute_action( - &call.action_name, - call.parameters.clone(), - &lease, - &exec_ctx, - ) - .await; + let (exec_result, pre_events) = execute_with_inline_gate_retry( + effects, + leases, + &lease, + call, + &exec_ctx, + thread.id, + &thread.user_id, + ) + .await; if interrupted_call_needs_refund(&exec_result) { let _ = leases.refund_use(lease.id).await; } - slot_results[idx] = Some(classify_exec_result( + let (result, event) = classify_exec_result( exec_result, call, &exec_ctx, execution_start.elapsed().as_millis() as u64, - )); + ); + slot_results[idx] = Some((result, event, pre_events)); } else if runnable_indices.len() > 1 { - // Multiple calls: execute in parallel via JoinSet + // Multiple calls: execute in parallel via JoinSet. Each task + // wraps the call in `execute_with_inline_gate_retry` so a tool + // raising `Approval` mid-execution pauses inline through the + // shared bridge controller (which serializes concurrent gates + // per (user, thread)) and either retries on approval or + // surfaces a typed denial — same contract as the single-call + // fast path. Without this wrapper, parallel batches reverted + // to the legacy `gate_paused` sentinel + thread re-entry path, + // re-introducing the double-execution bug for any + // already-completed sibling calls in the same batch. let mut join_set = tokio::task::JoinSet::new(); - let effects = effects.clone(); + // Capture thread metadata once outside the spawn loop. Avoids + // cloning the full `Thread` (with message/event transcripts) + // per task — the helper only needs the id + user_id. + let thread_id = thread.id; + let user_id = thread.user_id.clone(); for (idx, lease) in runnable_indices { let call = calls[idx].clone(); let ctx = stamp_execution_context( @@ -290,18 +348,22 @@ pub async fn execute_action_calls( &available_actions, &available_inventory, ); - let effects = effects.clone(); + let effects = Arc::clone(effects); + let leases = Arc::clone(leases); let lease = lease.clone(); + let user_id = user_id.clone(); join_set.spawn(async move { let execution_start = Instant::now(); - let result = effects - .execute_action(&call.action_name, call.parameters.clone(), &lease, &ctx) - .await; + let (result, pre_events) = execute_with_inline_gate_retry( + &effects, &leases, &lease, &call, &ctx, thread_id, &user_id, + ) + .await; ( idx, lease.id, result, + pre_events, call, ctx, execution_start.elapsed().as_millis() as u64, @@ -311,16 +373,13 @@ pub async fn execute_action_calls( while let Some(join_result) = join_set.join_next().await { match join_result { - Ok((idx, lease_id, result, call, ctx, execution_duration_ms)) => { + Ok((idx, lease_id, result, pre_events, call, ctx, execution_duration_ms)) => { if interrupted_call_needs_refund(&result) { let _ = leases.refund_use(lease_id).await; } - slot_results[idx] = Some(classify_exec_result( - result, - &call, - &ctx, - execution_duration_ms, - )); + let (action_result, event) = + classify_exec_result(result, &call, &ctx, execution_duration_ms); + slot_results[idx] = Some((action_result, event, pre_events)); } Err(e) => { // Task panicked — should not happen, but handle gracefully @@ -333,11 +392,15 @@ pub async fn execute_action_calls( // ── Phase 3: Merge results in original call order ─────────── let mut results = Vec::with_capacity(calls.len()); - let mut events = Vec::new(); + // `early_events` carries ApprovalRequested events emitted during + // preflight; they're emitted *before* any per-call result event so + // observers see "approval asked → action failed" in the right + // order. + let mut events = std::mem::take(&mut early_events); let mut first_interrupt: Option = None; for (idx, slot) in slot_results.into_iter().enumerate() { - if let Some((result, event)) = slot { + if let Some((result, event, pre_events)) = slot { // Record the first gate pause as the batch interrupt but still // collect all other results. if first_interrupt.is_none() @@ -376,6 +439,12 @@ pub async fn execute_action_calls( .and_then(|value| serde_json::from_value(value).ok()), }); } + // Pre-terminal `ApprovalRequested` events from the inline + // retry helper come first so the audit log reads as + // "approval asked → action " in order. + for pe in pre_events { + events.push(pe); + } results.push(result); events.push(event); } @@ -522,6 +591,262 @@ fn interrupted_call_needs_refund(result: &Result) -> matches!(result, Err(EngineError::GatePaused { .. })) } +/// Run a single tool action with inline gate-await retry. +/// +/// If the executor returns `Err(EngineError::GatePaused { resume_kind: Approval, .. })`, +/// refund the lease, pause for the user via the context's controller, +/// and retry on approval. On denial / cancellation, surface as a +/// deny-style `EngineError::Effect` so the caller produces an +/// `ActionFailed` event rather than a "gate_paused" sentinel. +/// +/// Bounded by [`MAX_INLINE_GATE_RETRIES`]: a misbehaving tool that +/// keeps gating after each approval surfaces a clean error rather than +/// pinning a CPU. The bridge installs auto-approve before delivering +/// the resolution, so well-behaved chains converge in 1–2 iterations +/// and the cap is only ever hit on bugs. +/// +/// Authentication resume kinds also flow through this loop now — +/// `bridge::resolve_inline_gates_for_credential` (the OAuth-callback +/// hook from #3133 half-2) delivers `GateResolution::Approved` to the +/// parked controller as soon as the credential lands in the secrets +/// store, so the retry sees the credential and the action succeeds. +/// (`bridge::resume_paused_missions_for_credential` is the parallel +/// path for missions whose child threads were paused — separate from +/// the inline-await waiters this loop drives.) +/// External resume kinds still keep the legacy re-entry path: their +/// resolution installs callback-payload state that the suspended call +/// can't see without unwinding. +/// +/// Returns `(final_result, events)` where `events` carries the +/// `ApprovalRequested` audit events emitted across retry iterations +/// — one per gate-pause cycle, in the order they fired. Callers +/// MUST emit these events before the per-call outcome event so +/// replay/audit observers see "approval asked → action " +/// instead of just the final outcome. +async fn execute_with_inline_gate_retry( + effects: &Arc, + leases: &LeaseManager, + lease: &CapabilityLease, + call: &ActionCall, + exec_ctx: &ThreadExecutionContext, + thread_id: crate::types::thread::ThreadId, + user_id: &str, +) -> (Result, Vec) { + let mut current_lease = lease.clone(); + // `call_ctx` carries the one-shot approval flag across retries. + // First iteration: false (the gate hasn't fired yet). After each + // approval we set true; we reset to false immediately after the + // call so a re-gating tool doesn't get the flag handed back twice + // for one approval. + let mut call_ctx = exec_ctx.clone(); + let mut emitted_events: Vec = Vec::new(); + for _ in 0..crate::executor::scripting::MAX_INLINE_GATE_RETRIES { + let result = effects + .execute_action( + &call.action_name, + call.parameters.clone(), + ¤t_lease, + &call_ctx, + ) + .await; + call_ctx.call_approval_granted = false; + + // Snapshot the original gate (for re-emission on + // Cancelled+Authentication, see below). + let original_err = match &result { + Err(EngineError::GatePaused { + resume_kind, + paused_lease, + resume_output, + gate_name, + action_name, + call_id, + parameters, + }) if matches!( + **resume_kind, + crate::gate::ResumeKind::Approval { .. } + | crate::gate::ResumeKind::Authentication { .. } + ) => + { + Some(EngineError::GatePaused { + gate_name: gate_name.clone(), + action_name: action_name.clone(), + call_id: call_id.clone(), + parameters: parameters.clone(), + resume_kind: resume_kind.clone(), + paused_lease: paused_lease.clone(), + resume_output: resume_output.clone(), + }) + } + _ => None, + }; + let (gate_name, action_name, call_id, parameters, resume_kind, resume_output) = match result + { + Err(EngineError::GatePaused { + gate_name, + action_name, + call_id, + parameters, + resume_kind, + resume_output, + .. + }) if matches!( + *resume_kind, + crate::gate::ResumeKind::Approval { .. } + | crate::gate::ResumeKind::Authentication { .. } + ) => + { + ( + gate_name, + action_name, + call_id, + *parameters, + *resume_kind, + resume_output.map(|b| *b), + ) + } + other => return (other, emitted_events), + }; + + // Emit the audit event BEFORE awaiting the controller so + // observers see the request even if the user never resolves. + // Mirrors the orchestrator (Tier 1) path which records the + // event before calling `pause()`. + let allow_always = match resume_kind { + crate::gate::ResumeKind::Approval { allow_always } => Some(allow_always), + _ => None, + }; + emitted_events.push(EventKind::ApprovalRequested { + action_name: action_name.clone(), + call_id: call_id.clone(), + parameters: Some(parameters.clone()), + description: None, + allow_always, + gate_name: Some(gate_name.clone()), + params_summary: crate::types::event::summarize_params(&call.action_name, ¶meters), + }); + + // Refund the lease use this attempt consumed; we'll re-consume + // on retry if the user approves. EXCEPTION: when `resume_output` + // is set, the action has already executed successfully (the + // gate is a post-execution Authentication gate carrying cached + // output) — the cached-output branch below will return without + // re-consuming. Refunding now would net the successful action + // to zero lease uses, letting a side-effecting tool drain + // `max_uses=∞` for free. See `drive_inline_gate` in scripting.rs + // and `execute_action_with_inline_gate` in orchestrator.rs for + // the matching guards. Tracked by the #3559 security review. + if resume_output.is_none() { + let _ = leases.refund_use(current_lease.id).await; + } + + let resolution = exec_ctx + .gate_controller + .pause(crate::gate::GatePauseRequest { + thread_id, + user_id: user_id.to_string(), + gate_name: gate_name.clone(), + action_name: action_name.clone(), + call_id: call_id.clone(), + parameters: parameters.clone(), + resume_kind: resume_kind.clone(), + conversation_id: exec_ctx.conversation_id, + }) + .await; + + if let Some(outcome) = + crate::executor::scripting::denial_outcome_for_resolution(&resolution) + { + // Cancelled+Authentication → unwind to legacy + // `ThreadOutcome::GatePaused` so missions / non-inline-aware + // controllers can still surface a Paused state. Cancelled + // here means the controller can't resolve the auth inline + // (e.g. `CancellingGateController` in tests, or a + // BridgeGateController without OAuth wiring) — that's + // semantically "no inline path exists" and the legacy + // unwind is the right fallback. Denied / explicit + // Cancelled-by-user remain failures. + if matches!(resolution, crate::gate::GateResolution::Cancelled) + && matches!(resume_kind, crate::gate::ResumeKind::Authentication { .. }) + && let Some(err) = original_err + { + return (Err(err), emitted_events); + } + return ( + Err(EngineError::Effect { + reason: outcome.effect_reason(), + }), + emitted_events, + ); + } + + // Approved. If the bridge cached the action's output before raising + // this gate (post-execution Authentication gate path — see + // `effect_adapter::auth_gate_from_extension_result`), the action has + // already run and we just needed user-side resolution. Skip + // re-execution and synthesize a successful ActionResult from the + // cached output. Mirrors the Tier 1 shortcut in + // `scripting::drive_inline_gate`. Tracked by #3533. + // + // Do NOT emit `ActionExecuted` here. The caller wraps the + // `Ok(ActionResult)` we return in `classify_exec_result`, which + // emits the terminal `ActionExecuted` for the Ok branch. Emitting + // here would produce two `ActionExecuted` events for one action, + // confusing audit observers. (Tier 1's `scripting::drive_inline_gate` + // and Tier 1 alt's `orchestrator::execute_action_with_inline_gate` + // emit themselves because their callers don't run an Ok-branch + // classifier — see the #3559 review for why structured is the + // outlier here.) + if let Some(cached_output) = resume_output { + return ( + Ok(ActionResult { + call_id, + action_name, + output: cached_output, + is_error: false, + duration: std::time::Duration::ZERO, + }), + emitted_events, + ); + } + + // Re-consume a lease use and mark the next call as pre-approved so + // the host's `EffectExecutor` skips its approval check. + match leases.find_and_consume(thread_id, &call.action_name).await { + Ok(new_lease) => { + current_lease = new_lease; + call_ctx.call_approval_granted = true; + continue; + } + Err(e) => { + return ( + Err(EngineError::Effect { + reason: format!("lease exhausted after approval: {e}"), + }), + emitted_events, + ); + } + } + } + + // Retry budget exhausted. The last loop iteration ended with a + // successful `find_and_consume` whose lease was never used — + // refund it before returning so a misbehaving tool can't slowly + // drain `max_uses` across approvals. Best-effort; if the lease + // was already revoked/expired the refund is a no-op. + let _ = leases.refund_use(current_lease.id).await; + ( + Err(EngineError::Effect { + reason: format!( + "tool '{}' still requires approval after {} retries", + call.action_name, + crate::executor::scripting::MAX_INLINE_GATE_RETRIES + ), + }), + emitted_events, + ) +} + #[cfg(test)] mod tests { use super::*; @@ -615,6 +940,10 @@ mod tests { thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } @@ -738,7 +1067,14 @@ mod tests { "test-user", ThreadConfig::default(), ); - let effects: Arc = Arc::new(MockEffects::new(vec![], vec![])); + // Inventory contains `web_search` (so the callable-inventory + // gate passes), but no lease is granted — the lease lookup is + // the failure point this test exercises. Without `web_search` + // in the inventory, preflight short-circuits on + // "action is not callable in this execution context" before + // ever reaching the lease check. + let effects: Arc = + Arc::new(MockEffects::new(vec![test_action("web_search")], vec![])); let leases = Arc::new(LeaseManager::new()); let policy = Arc::new(PolicyEngine::new()); let ctx = make_exec_context(&thread); @@ -760,7 +1096,10 @@ mod tests { if let Some(EventKind::ActionFailed { call_id, error, .. }) = result.events.first() { assert_eq!(call_id, "call_no_lease_123"); - assert!(error.contains("no lease")); + assert!( + error.contains("no lease"), + "expected error message to mention 'no lease', got: {error}" + ); } else { panic!("expected ActionFailed event"); } @@ -927,24 +1266,26 @@ mod tests { .await .unwrap(); - match result.need_approval { - Some(ThreadOutcome::GatePaused { - gate_name, - action_name, - call_id, - .. - }) => { - assert_eq!(gate_name, "approval"); - assert_eq!(action_name, "create-issue"); - assert_eq!(call_id, "call_alias_policy"); - } - other => panic!("expected approval gate for aliased action, got {other:?}"), - } - + // The default `CancellingGateController` cancels the gate + // synchronously, so the batch surfaces a denied error result for + // the aliased call rather than the legacy + // `need_approval = Some(ThreadOutcome::GatePaused {...})`. + // What still must hold: the gate was *evaluated* (an + // ApprovalRequested event was emitted with the aliased name and + // the original call_id), the action did NOT execute, and the + // result is_error. + assert!( + result.need_approval.is_none(), + "controller-driven path must not bubble need_approval up; got {:?}", + result.need_approval + ); + assert_eq!(result.results.len(), 1); assert!( - result.results.is_empty(), - "policy preflight should pause before executing the aliased action" + result.results[0].is_error, + "denied gate must surface as an error result" ); + assert_eq!(result.results[0].call_id, "call_alias_policy"); + assert_eq!(result.results[0].action_name, "create-issue"); assert!( result.events.iter().any(|event| matches!( event, @@ -953,6 +1294,14 @@ mod tests { )), "approval event should use the aliased action name and original call id" ); + assert!( + result.events.iter().any(|event| matches!( + event, + EventKind::ActionFailed { action_name, call_id, .. } + if action_name == "create-issue" && call_id == "call_alias_policy" + )), + "denied gate should produce an ActionFailed event" + ); } // ── GatePaused(Authentication) tests ───────────────────── @@ -1130,6 +1479,158 @@ mod tests { } } + /// #3559 security review (finding 2): when a post-execution + /// Authentication gate carries cached `resume_output`, the inline + /// retry path must NOT refund the lease use the action already + /// consumed. The cached-output branch returns without re-consuming, + /// so refunding would net a successful side-effecting action to + /// zero lease uses — letting a `max_uses=N` budget execute N+ times + /// for free. + /// + /// Wires `MockEffects` to return `GatePaused { resume_output: Some(...) }` + /// on the first call, drives it through `execute_action_calls` with an + /// always-approve test gate controller, and asserts: + /// 1. The cached output is returned (Ok result, no re-execution). + /// 2. Exactly one `ActionExecuted` event is emitted. + /// 3. The lease's `uses_remaining` ends at `Some(0)` — the original + /// consumption stands; the refund was correctly skipped. + /// + /// Pre-fix (`refund_use` ran unconditionally), `uses_remaining` would + /// end at `Some(1)` and a subsequent call would still succeed, + /// breaking the `max_uses=1` contract. + #[tokio::test] + async fn resume_output_replay_consumes_exactly_one_lease_use() { + use crate::gate::{GateController, GatePauseRequest, GateResolution}; + + struct ApprovingGateController; + + #[async_trait::async_trait] + impl GateController for ApprovingGateController { + async fn pause(&self, _request: GatePauseRequest) -> GateResolution { + GateResolution::Approved { always: false } + } + } + + let thread = Thread::new( + "test", + ThreadType::Foreground, + ProjectId::new(), + "test-user", + ThreadConfig::default(), + ); + + let cached_output = serde_json::json!({"installed": "gmail", "ok": true}); + let effects: Arc = Arc::new(MockEffects::new( + vec![test_action("tool_install")], + vec![Err(EngineError::GatePaused { + gate_name: "authentication".into(), + action_name: "tool_install".into(), + call_id: "call_install_1".into(), + parameters: Box::new(serde_json::json!({"name": "gmail"})), + resume_kind: Box::new(crate::gate::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("google_oauth_token") + .unwrap(), + instructions: "Connect Google".into(), + auth_url: None, + }), + // The bridge cached the action's output before raising + // the post-execution Authentication gate — this is the + // exact shape `effect_adapter::auth_gate_from_extension_result` + // produces for a successful `tool_install` that needs + // user-side OAuth completion. + resume_output: Some(Box::new(cached_output.clone())), + paused_lease: None, + })], + )); + let leases = Arc::new(LeaseManager::new()); + let policy = Arc::new(PolicyEngine::new()); + + // Grant a lease with `max_uses: Some(1)` — the budget this test + // asserts the engine honors. + let lease = leases + .grant(thread.id, "tools", GrantedActions::All, None, Some(1)) + .await + .unwrap(); + assert_eq!( + lease.uses_remaining, + Some(1), + "freshly granted lease should start with full budget" + ); + + let mut ctx = make_exec_context(&thread); + ctx.gate_controller = Arc::new(ApprovingGateController); + + let calls = vec![ActionCall { + id: "call_install_1".into(), + action_name: "tool_install".into(), + parameters: serde_json::json!({"name": "gmail"}), + }]; + + let result = execute_action_calls(&calls, &thread, &effects, &leases, &policy, &ctx, &[]) + .await + .unwrap(); + + // 1. The cached output reached the caller via an Ok result. + assert_eq!(result.results.len(), 1); + assert!( + !result.results[0].is_error, + "cached-output replay must surface as a successful ActionResult" + ); + assert_eq!( + result.results[0].output, cached_output, + "ActionResult.output must be the gate's cached resume_output verbatim" + ); + assert!( + result.need_approval.is_none(), + "inline-approved gate must NOT propagate as a top-level need_approval" + ); + + // 2. Exactly one terminal `ActionExecuted` for the call. The + // pre-gate emission is `ApprovalRequested`, not + // `ActionExecuted`, so the count check catches a future + // regression that re-introduces a double-emit through the + // classifier path. + let action_executed_count = result + .events + .iter() + .filter(|e| { + matches!( + e, + EventKind::ActionExecuted { call_id, .. } if call_id == "call_install_1" + ) + }) + .count(); + assert_eq!( + action_executed_count, 1, + "expected exactly one ActionExecuted for the cached-output replay; got events={:?}", + result.events + ); + + // 3. The lease use the action consumed before pausing was NOT + // refunded — the original consumption is the correct + // accounting for the already-executed action. The + // `max_uses=1` budget is now exhausted, which surfaces as + // `Err(LeaseExpired)` from both `check` (exhausted leases + // fail `is_valid()`) and from a second `find_and_consume` + // attempt. Pre-fix the refund would have ran, leaving + // `uses_remaining: Some(1)` and both checks would succeed. + match leases.check(lease.id).await { + Err(EngineError::LeaseExpired { .. }) => {} + other => panic!( + "expected LeaseExpired after cached-output replay (budget should be \ + exhausted); got {other:?}" + ), + } + match leases.find_and_consume(thread.id, "tool_install").await { + Err(_) => {} + Ok(extra_lease) => panic!( + "max_uses=1 contract violated: a second `find_and_consume` succeeded \ + with uses_remaining={:?} — the refund-skip must keep the budget at zero", + extra_lease.uses_remaining + ), + } + } + /// Regular EngineError::Effect (not GatePaused) should NOT interrupt — /// it becomes a normal error result and execution continues. #[tokio::test] @@ -1812,4 +2313,221 @@ mod tests { other => panic!("expected ActionFailed event, got {other:?}"), } } + + // ── Inline-retry ApprovalRequested audit-event tests ──────── + + /// Local stub gate controller for the inline-retry tests below. + /// Approves once on first pause, returns the canned resolution + /// thereafter; records every pause request for assertion. + struct StubGateController { + resolution: Mutex>, + pauses: Mutex>, + } + + impl StubGateController { + fn approving_arc() -> std::sync::Arc { + std::sync::Arc::new(Self { + resolution: Mutex::new(Some(crate::gate::GateResolution::Approved { + always: false, + })), + pauses: Mutex::new(Vec::new()), + }) + } + + fn denying_arc() -> std::sync::Arc { + std::sync::Arc::new(Self { + resolution: Mutex::new(Some(crate::gate::GateResolution::Denied { + reason: Some("user declined".into()), + })), + pauses: Mutex::new(Vec::new()), + }) + } + + fn pause_count(&self) -> usize { + self.pauses.lock().unwrap().len() + } + } + + #[async_trait::async_trait] + impl crate::gate::GateController for StubGateController { + async fn pause( + &self, + request: crate::gate::GatePauseRequest, + ) -> crate::gate::GateResolution { + self.pauses.lock().unwrap().push(request); + self.resolution + .lock() + .unwrap() + .clone() + .unwrap_or(crate::gate::GateResolution::Cancelled) + } + } + + /// Mid-execution `GatePaused(Approval)` followed by user approval + /// must emit BOTH `ApprovalRequested` and the final + /// `ActionExecuted` event in order. Regression for the audit drop + /// noted by serrrfirat on the structured executor. + #[tokio::test] + async fn inline_retry_emits_approval_requested_event_before_outcome() { + let thread = Thread::new( + "audit-test", + ThreadType::Foreground, + ProjectId::new(), + "audit-user", + ThreadConfig::default(), + ); + + // Effects: + // call 1 → Err(GatePaused) — tool raises mid-execution gate + // call 2 → Ok(success) — after user approval, retry succeeds + let effects: Arc = Arc::new(MockEffects::new( + vec![test_action("write_file")], + vec![ + Err(EngineError::GatePaused { + gate_name: "approval".into(), + action_name: "write_file".into(), + call_id: "call_audit_1".into(), + parameters: Box::new(serde_json::json!({"path": "/tmp/x"})), + resume_kind: Box::new(crate::gate::ResumeKind::Approval { allow_always: true }), + resume_output: None, + paused_lease: None, + }), + Ok(ActionResult { + call_id: String::new(), + action_name: "write_file".into(), + output: serde_json::json!({"bytes_written": 12}), + is_error: false, + duration: Duration::from_millis(7), + }), + ], + )); + let leases = Arc::new(LeaseManager::new()); + let policy = Arc::new(PolicyEngine::new()); + let mut ctx = make_exec_context(&thread); + let controller = StubGateController::approving_arc(); + ctx.gate_controller = controller.clone(); + + leases + .grant(thread.id, "fs", GrantedActions::All, None, None) + .await + .unwrap(); + + let calls = vec![ActionCall { + id: "call_audit_1".into(), + action_name: "write_file".into(), + parameters: serde_json::json!({"path": "/tmp/x"}), + }]; + + let result = execute_action_calls(&calls, &thread, &effects, &leases, &policy, &ctx, &[]) + .await + .unwrap(); + + // Controller saw exactly one pause. + assert_eq!(controller.pause_count(), 1); + + // Events MUST contain ApprovalRequested followed by ActionExecuted. + let approval_idx = result + .events + .iter() + .position(|e| matches!(e, EventKind::ApprovalRequested { .. })) + .expect("ApprovalRequested must be emitted"); + let executed_idx = result + .events + .iter() + .position(|e| matches!(e, EventKind::ActionExecuted { .. })) + .expect("ActionExecuted must be emitted after approval"); + assert!( + approval_idx < executed_idx, + "ApprovalRequested must come before ActionExecuted; got events={:?}", + result.events + ); + + // ApprovalRequested carries the call's identifying metadata. + match &result.events[approval_idx] { + EventKind::ApprovalRequested { + action_name, + call_id, + gate_name, + allow_always, + .. + } => { + assert_eq!(action_name, "write_file"); + assert_eq!(call_id, "call_audit_1"); + assert_eq!(gate_name.as_deref(), Some("approval")); + assert_eq!(*allow_always, Some(true)); + } + other => panic!("expected ApprovalRequested, got {other:?}"), + } + + // The terminal action result is success (not gate_paused). + assert!(!result.results[0].is_error); + assert!(result.need_approval.is_none()); + } + + /// Same shape, but the user denies. The `ApprovalRequested` event + /// is still emitted before the `ActionFailed` event so audit logs + /// see the full lifecycle. + #[tokio::test] + async fn inline_retry_emits_approval_requested_event_before_denial() { + let thread = Thread::new( + "audit-test-denied", + ThreadType::Foreground, + ProjectId::new(), + "audit-user", + ThreadConfig::default(), + ); + + let effects: Arc = Arc::new(MockEffects::new( + vec![test_action("write_file")], + vec![Err(EngineError::GatePaused { + gate_name: "approval".into(), + action_name: "write_file".into(), + call_id: "call_audit_2".into(), + parameters: Box::new(serde_json::json!({"path": "/tmp/y"})), + resume_kind: Box::new(crate::gate::ResumeKind::Approval { + allow_always: false, + }), + resume_output: None, + paused_lease: None, + })], + )); + let leases = Arc::new(LeaseManager::new()); + let policy = Arc::new(PolicyEngine::new()); + let mut ctx = make_exec_context(&thread); + let controller = StubGateController::denying_arc(); + ctx.gate_controller = controller.clone(); + + leases + .grant(thread.id, "fs", GrantedActions::All, None, None) + .await + .unwrap(); + + let calls = vec![ActionCall { + id: "call_audit_2".into(), + action_name: "write_file".into(), + parameters: serde_json::json!({"path": "/tmp/y"}), + }]; + + let result = execute_action_calls(&calls, &thread, &effects, &leases, &policy, &ctx, &[]) + .await + .unwrap(); + + assert_eq!(controller.pause_count(), 1); + + let approval_idx = result + .events + .iter() + .position(|e| matches!(e, EventKind::ApprovalRequested { .. })) + .expect("ApprovalRequested must be emitted even on denial"); + let failed_idx = result + .events + .iter() + .position(|e| matches!(e, EventKind::ActionFailed { .. })) + .expect("ActionFailed must be emitted after denial"); + assert!( + approval_idx < failed_idx, + "ApprovalRequested must come before ActionFailed; got events={:?}", + result.events + ); + } } diff --git a/crates/ironclaw_engine/src/executor/thread_context.rs b/crates/ironclaw_engine/src/executor/thread_context.rs index d4db1111990..d6b599efc0b 100644 --- a/crates/ironclaw_engine/src/executor/thread_context.rs +++ b/crates/ironclaw_engine/src/executor/thread_context.rs @@ -1,13 +1,25 @@ +use std::str::FromStr; +use std::sync::Arc; + +use crate::gate::GateController; use crate::traits::effect::ThreadExecutionContext; +use crate::types::conversation::ConversationId; use crate::types::step::StepId; use crate::types::thread::Thread; use ironclaw_common::ValidTimezone; +use uuid::Uuid; /// Build an execution context from the current thread state. +/// +/// `gate_controller` is required: callers thread through the controller +/// they were constructed with so the executor can pause inline on +/// `Approval` gates. Code paths that don't pause supply +/// [`crate::gate::CancellingGateController::arc()`]. pub(crate) fn thread_execution_context( thread: &Thread, step_id: StepId, current_call_id: Option, + gate_controller: Arc, ) -> ThreadExecutionContext { ThreadExecutionContext { thread_id: thread.id, @@ -29,5 +41,18 @@ pub(crate) fn thread_execution_context( thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: thread + .metadata + .get("conversation_scope") + .and_then(|v| v.as_str()) + .and_then(|s| Uuid::parse_str(s).ok()), + gate_controller, + call_approval_granted: false, + conversation_id: thread + .metadata + .get("conversation_id") + .and_then(|v| v.as_str()) + .and_then(|s| Uuid::from_str(s).ok()) + .map(ConversationId), } } diff --git a/crates/ironclaw_engine/src/executor/trace.rs b/crates/ironclaw_engine/src/executor/trace.rs index 4f0a5004313..a552e19a02a 100644 --- a/crates/ironclaw_engine/src/executor/trace.rs +++ b/crates/ironclaw_engine/src/executor/trace.rs @@ -5,8 +5,8 @@ //! self-improvement mission and surfaced in debug logs. //! //! **There is no separate engine trace file.** Live trace recording for the -//! whole system is handled by `RecordingLlm` in the host crate -//! (`src/llm/recording.rs`), gated by `IRONCLAW_RECORD_TRACE`. Because the +//! whole system is handled by `RecordingLlm` in `ironclaw_llm` +//! (`crates/ironclaw_llm/src/recording.rs`), gated by `IRONCLAW_RECORD_TRACE`. Because the //! engine's `LlmBackend` is wired to the same provider chain, engine LLM //! interactions are captured by that single recorder — no engine-side env var //! and no second JSON file. diff --git a/crates/ironclaw_engine/src/gate/mod.rs b/crates/ironclaw_engine/src/gate/mod.rs index 1019fa2358a..b42d3a096b0 100644 --- a/crates/ironclaw_engine/src/gate/mod.rs +++ b/crates/ironclaw_engine/src/gate/mod.rs @@ -150,6 +150,126 @@ pub trait ExecutionGate: Send + Sync { async fn evaluate(&self, ctx: &GateContext<'_>) -> GateDecision; } +// ── Inline gate await ──────────────────────────────────────── + +/// What the executor needs to surface to the user when an `Approval` +/// gate fires inside a live execution. +/// +/// The host implementation of [`GateController`] is responsible for: +/// 1. Persisting whatever metadata the UI / channel layer needs to +/// render the approval prompt. +/// 2. Dispatching the prompt to the originating channel. +/// 3. Awaiting the user's response and returning it as a +/// [`GateResolution`] without re-entering the engine. +/// +/// Carries `thread_id` and `user_id` so a single shared controller +/// can route the request to the right host-side per-execution context +/// (conversation id, channel metadata, etc.) without the engine having +/// to thread bridge-internal types through. +#[derive(Debug, Clone)] +pub struct GatePauseRequest { + pub thread_id: crate::types::thread::ThreadId, + pub user_id: String, + pub gate_name: String, + pub action_name: String, + pub call_id: String, + pub parameters: serde_json::Value, + pub resume_kind: ResumeKind, + /// Originating conversation, if any. Lets the host route an inline + /// gate to the right UI surface when the same user has multiple + /// concurrent conversations (e.g. two browser tabs). `None` for + /// background mission threads. + pub conversation_id: Option, +} + +/// Host-supplied callback that pauses a live engine execution until +/// the user resolves an `Approval` gate. +/// +/// This is the mechanism that lets both Tier 0 (structured) and Tier 1 +/// (CodeAct/Monty) executions wait for user input *without* unwinding +/// the call stack. The executor stays inside its own loop awaiting +/// `pause()`, so all in-memory state (Monty VM frame, partially-executed +/// parallel batch, leases) is preserved across the wait. On resolution +/// the executor proceeds inline — no thread re-entry, no replay, no +/// double-execution of side effects from earlier tool calls in the +/// same step. +/// +/// Handles `ResumeKind::Approval` and `ResumeKind::Authentication`. +/// External resume kinds still keep the legacy re-entry-based flow: +/// their resolution installs callback-payload state that can't be +/// handed back to the suspended call without unwinding. +/// +/// For Authentication, the host-side controller is expected to resolve +/// the gate with `GateResolution::Approved` once the credential has +/// been written to the secrets store (the OAuth-callback path on the +/// gateway handles this — see +/// `bridge::resolve_inline_gates_for_credential`, which wakes parked +/// inline-await waiters). `bridge::resume_paused_missions_for_credential` +/// is the parallel path that resumes background missions whose +/// child threads were paused by the same gate. The paused tool call +/// retries inline and reads the credential the same way it would +/// have on a fresh execution. +#[async_trait] +pub trait GateController: Send + Sync { + /// Pause execution until the user resolves the gate. + /// + /// Implementations MUST eventually return some [`GateResolution`] + /// (returning `Cancelled` is acceptable on shutdown / timeout). + /// They MUST NOT block forever — callers rely on this future + /// completing so the surrounding execution can either continue or + /// terminate cleanly. + async fn pause(&self, request: GatePauseRequest) -> GateResolution; + + /// Wake any [`pause`] futures currently parked on `thread_id` with + /// [`GateResolution::Cancelled`] and discard their pending state. + /// + /// `ThreadManager::stop_thread()` calls this before sending + /// `ThreadSignal::Stop`. Without it, an engine task parked inside + /// `pause()` is not polling the thread signal channel and will + /// continue waiting for the user (or up to the host's gate-expiry + /// window) before observing the stop request — leaving the running + /// task and pending prompt orphaned. + /// + /// Default implementation is a no-op; overrides should be + /// idempotent and tolerant of concurrent calls. Implementations + /// that don't track per-thread waiters can ignore this call. + /// + /// [`pause`]: GateController::pause + async fn cancel_thread(&self, _thread_id: crate::types::thread::ThreadId) {} +} + +/// Default [`GateController`] that cancels every pause request. +/// +/// `ThreadExecutionContext::gate_controller` is non-optional: every +/// execution context must carry *some* controller. This impl is the +/// drop-in for code paths where pausing is meaningless or already +/// resolved upstream: +/// +/// - **Post-resolution replay** (`execute_pending_gate_action`) — the +/// gate has already been resolved before this call site runs. +/// - **Mission protected writes** — background paths with no +/// originating user channel to surface a prompt on. +/// - **Tests** that don't exercise the gate flow. +/// +/// Returning [`GateResolution::Cancelled`] surfaces as a typed denial +/// in both Tier 0 and Tier 1 — never as the original user-visible +/// "execution paused by gate" bug message. +pub struct CancellingGateController; + +impl CancellingGateController { + /// Construct as a `dyn`-trait Arc, the form most call sites need. + pub fn arc() -> std::sync::Arc { + std::sync::Arc::new(Self) + } +} + +#[async_trait] +impl GateController for CancellingGateController { + async fn pause(&self, _request: GatePauseRequest) -> GateResolution { + GateResolution::Cancelled + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ironclaw_engine/src/gate/tool_tier.rs b/crates/ironclaw_engine/src/gate/tool_tier.rs index 256f46879fa..c5021df8f52 100644 --- a/crates/ironclaw_engine/src/gate/tool_tier.rs +++ b/crates/ironclaw_engine/src/gate/tool_tier.rs @@ -23,7 +23,6 @@ pub const AUTONOMOUS_TOOL_DENYLIST: &[&str] = &[ "restart", "tool_install", "tool_auth", - "tool_activate", "tool_remove", "tool_upgrade", "skill_install", diff --git a/crates/ironclaw_engine/src/lib.rs b/crates/ironclaw_engine/src/lib.rs index 20498c4070e..0820516dc96 100644 --- a/crates/ironclaw_engine/src/lib.rs +++ b/crates/ironclaw_engine/src/lib.rs @@ -75,19 +75,25 @@ pub use gate::lease::LeaseGate; pub use gate::pipeline::GatePipeline; pub use gate::tool_tier::{ToolTier, classify_tool_tier}; pub use gate::{ - ExecutionGate, ExecutionMode, GateContext, GateDecision, GateResolution, ResumeKind, + CancellingGateController, ExecutionGate, ExecutionMode, GateContext, GateController, + GateDecision, GatePauseRequest, GateResolution, ResumeKind, }; // ── Re-exports: runtime ─────────────────────────────────────── pub use executor::prompt::PlatformInfo; pub use runtime::conversation::ConversationManager; -pub use runtime::manager::ThreadManager; +pub use runtime::manager::{ + ENGINE_RESTART_RECOVERY_METADATA_KEY, PENDING_APPROVAL_METADATA_KEY, + RUNTIME_CHECKPOINT_METADATA_KEY, ThreadManager, +}; pub use runtime::messaging::ThreadOutcome; pub use runtime::mission::{ - BudgetGate, FireRateLimit, MissionGateInfo, MissionManager, MissionNotification, MissionUpdate, + BudgetGate, FireRateLimit, GateResolutionOutcome, MissionManager, MissionNotification, + MissionUpdate, }; pub use runtime::tree::ThreadTree; +pub use types::mission::MissionGateInfo; pub use types::conversation::{ ConversationEntry, ConversationId, ConversationSurface, EntrySender, diff --git a/crates/ironclaw_engine/src/runtime/conversation.rs b/crates/ironclaw_engine/src/runtime/conversation.rs index 74b42e90be2..3f31ae12fd6 100644 --- a/crates/ironclaw_engine/src/runtime/conversation.rs +++ b/crates/ironclaw_engine/src/runtime/conversation.rs @@ -189,6 +189,27 @@ impl ConversationManager { /// The per-conversation `Mutex` is held for the entire operation — from /// the active-thread check through `save_conversation`. This eliminates /// the TOCTOU double-spawn window present in the old 5-phase split. + /// + /// **`extra_initial_metadata` is spawn-only.** It is merged into the + /// thread's `metadata` map ONLY when this call allocates a new + /// thread (the `None` active-foreground branch below). On the + /// `Running` (inject) and `Resumable` (resume) paths the caller- + /// supplied metadata is *ignored* — those threads already exist + /// with their own metadata. Callers that need per-request state + /// (e.g. the bridge's external tool catalog keyed by + /// `conversation_scope`) must therefore either re-establish the + /// state out-of-band per request (the responses_api handler + /// re-registers its catalog every request and the bridge + /// `transfer`s it onto the engine thread_id), or persist the + /// value with `ThreadManager::set_thread_metadata` before resume + /// so the engine reload picks it up. Extending this method to + /// also write metadata on inject/resume is a future change, not + /// the current contract. + // Bundling these into an options struct would just push the + // argument list around without making any caller easier to read — + // every caller passes literal None for the optional fields and + // every required field is already a typed newtype. + #[allow(clippy::too_many_arguments)] pub async fn handle_user_message( &self, conversation_id: ConversationId, @@ -197,6 +218,7 @@ impl ConversationManager { user_id: &str, thread_config: ThreadConfig, user_timezone: Option<&str>, + extra_initial_metadata: Option>, ) -> Result { let conv_arc = self.get_conversation_lock(conversation_id).await?; let mut conv = conv_arc.lock().await; @@ -303,6 +325,28 @@ impl ConversationManager { serde_json::Value::String(tz.to_string()), ); } + // Stash the originating conversation_id so + // `thread_execution_context` can surface it to the host + // via `ThreadExecutionContext.conversation_id`. This + // lets the bridge match an inline-await gate to the + // right UI surface when the same user has multiple + // concurrent conversations. + initial_metadata.insert( + "conversation_id".into(), + serde_json::Value::String(conversation_id.to_string()), + ); + // Merge caller-supplied metadata (e.g. `conversation_scope` + // from the responses_api bridge so the EffectExecutor can + // resolve per-conversation state without racing the + // executor task that starts immediately after spawn). + // Built-in keys above win on conflict so callers can't + // shadow `source_channel`, `user_timezone`, or + // `conversation_id`. + if let Some(extra) = extra_initial_metadata { + for (k, v) in extra { + initial_metadata.entry(k).or_insert(v); + } + } // Spawn new foreground thread with conversation history. // `goal` holds the full message (the orchestrator feeds it as @@ -860,6 +904,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await .unwrap(); @@ -928,6 +973,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await .unwrap(); @@ -1027,6 +1073,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await .unwrap(); @@ -1076,6 +1123,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await }); @@ -1087,6 +1135,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await }); @@ -1163,6 +1212,7 @@ mod tests { "user1", ThreadConfig::default(), None, + None, ) .await .unwrap(); diff --git a/crates/ironclaw_engine/src/runtime/lease_refresh.rs b/crates/ironclaw_engine/src/runtime/lease_refresh.rs index 324cf859830..58b9866afa0 100644 --- a/crates/ironclaw_engine/src/runtime/lease_refresh.rs +++ b/crates/ironclaw_engine/src/runtime/lease_refresh.rs @@ -20,8 +20,14 @@ pub(crate) async fn reconcile_dynamic_tool_lease( lease_planner: &LeasePlanner, ) -> Result<(), EngineError> { let active_leases = leases.active_for_thread(thread.id).await; - let context = - crate::executor::thread_context::thread_execution_context(thread, StepId::new(), None); + // Lease reconciliation only reads `available_actions`; no tool + // execution happens here, so an inert controller is sufficient. + let context = crate::executor::thread_context::thread_execution_context( + thread, + StepId::new(), + None, + crate::gate::CancellingGateController::arc(), + ); let actions = effects.available_actions(&active_leases, &context).await?; if actions.is_empty() { return Ok(()); diff --git a/crates/ironclaw_engine/src/runtime/manager.rs b/crates/ironclaw_engine/src/runtime/manager.rs index bedf50af590..b0d3d595a30 100644 --- a/crates/ironclaw_engine/src/runtime/manager.rs +++ b/crates/ironclaw_engine/src/runtime/manager.rs @@ -44,6 +44,15 @@ pub struct ThreadManager { completed: Arc>>, /// Broadcast channel for thread events (for live status updates). event_tx: tokio::sync::broadcast::Sender, + /// Host-supplied callback that turns `Approval` gates into inline + /// awaits instead of unwinding the call stack. The engine attaches + /// it to every `ThreadExecutionContext` so both Tier 0 and Tier 1 + /// executors can pause a live VM in place. + /// + /// Defaults to [`crate::gate::CancellingGateController`] — every + /// gate cancels with a typed denial. Hosts that want real inline + /// await call [`Self::set_gate_controller`] during bootstrap. + gate_controller: tokio::sync::RwLock>, } impl ThreadManager { @@ -68,9 +77,24 @@ impl ThreadManager { running: Arc::new(RwLock::new(HashMap::new())), completed: Arc::new(RwLock::new(HashMap::new())), event_tx, + gate_controller: tokio::sync::RwLock::new(crate::gate::CancellingGateController::arc()), } } + /// Install (or replace) the host-supplied gate controller. + /// + /// Called once during bridge bootstrap. Subsequent thread spawns + /// pick up the controller and propagate it into every + /// `ThreadExecutionContext` they construct. + pub async fn set_gate_controller(&self, controller: Arc) { + *self.gate_controller.write().await = controller; + } + + /// Snapshot the current gate controller. + pub async fn gate_controller(&self) -> Arc { + self.gate_controller.read().await.clone() + } + /// Subscribe to thread events for live status updates. pub fn subscribe_events( &self, @@ -335,11 +359,21 @@ impl ThreadManager { let store_for_retrieval = Arc::clone(&self.store); let retrieval = crate::memory::RetrievalEngine::new(store_for_retrieval); - let exec_loop = ExecutionLoop::new(thread, llm, effects, leases, policy, rx, user_id) - .with_capabilities(Arc::clone(&self.capabilities)) - .with_event_tx(self.event_tx.clone()) - .with_retrieval(retrieval) - .with_store(Arc::clone(&self.store)); + let gate_controller = self.gate_controller.read().await.clone(); + let exec_loop = ExecutionLoop::new( + thread, + llm, + effects, + leases, + policy, + rx, + user_id, + gate_controller, + ) + .with_capabilities(Arc::clone(&self.capabilities)) + .with_event_tx(self.event_tx.clone()) + .with_retrieval(retrieval) + .with_store(Arc::clone(&self.store)); // Spawn background task let store_for_task = Arc::clone(&self.store); @@ -416,6 +450,14 @@ impl ThreadManager { } /// Send a stop signal to a running thread. + /// + /// Wakes any [`crate::gate::GateController::pause`] futures that + /// are currently parked on this thread BEFORE sending + /// `ThreadSignal::Stop` so the engine task can observe the stop + /// promptly. Without the explicit cancel, a thread parked inside + /// `pause()` (inline approval await) is not polling the signal + /// channel and would continue waiting until the user resolves the + /// prompt or the gate expires. pub async fn stop_thread(&self, thread_id: ThreadId, user_id: &str) -> Result<(), EngineError> { // Validate ownership before allowing stop. if let Some(thread) = self.store.load_thread(thread_id).await? @@ -426,6 +468,14 @@ impl ThreadManager { entity: format!("thread {thread_id}"), }); } + + // Wake any inline gate await blocked on this thread first. The + // controller is shared across spawned engine tasks; a parked + // `pause()` future polled inside an executor doesn't see + // `ThreadSignal::Stop` directly. + let controller = self.gate_controller.read().await.clone(); + controller.cancel_thread(thread_id).await; + let running = self.running.read().await; if let Some(rt) = running.get(&thread_id) { let _ = rt.signal_tx.send(ThreadSignal::Stop).await; @@ -598,13 +648,14 @@ impl ThreadManager { /// Reconcile persisted non-terminal threads after process startup. /// /// The current engine does not support mid-thread replay/resume, so any - /// thread left in a non-terminal state is marked failed-safe. + /// thread left in a non-terminal state is marked failed-safe. Threads + /// transitioned to `Failed` here carry the + /// [`ENGINE_RESTART_RECOVERY_METADATA_KEY`] flag so callers can + /// distinguish them from real, user-actionable failures. pub async fn recover_project_threads( &self, project_id: ProjectId, ) -> Result, EngineError> { - const PENDING_APPROVAL_METADATA_KEY: &str = "pending_approval"; - const RUNTIME_CHECKPOINT_METADATA_KEY: &str = "runtime_checkpoint"; // System operation: recover all non-terminal threads regardless of user. let threads = self.store.list_all_threads(project_id).await?; let mut recovered = Vec::new(); @@ -638,6 +689,16 @@ impl ThreadManager { continue; } + // Tag the thread before transitioning so downstream consumers + // (projects "needs attention" feed, health rollup) can skip + // restart-recovery noise and only surface real failures. + if let Some(obj) = thread.metadata.as_object_mut() { + obj.insert( + ENGINE_RESTART_RECOVERY_METADATA_KEY.to_string(), + serde_json::Value::Bool(true), + ); + } + if thread .transition_to( ThreadState::Failed, @@ -655,6 +716,24 @@ impl ThreadManager { } } +/// Metadata key set on a thread that has an in-flight pending-approval +/// gate. Persisted threads carrying this key skip restart-recovery so the +/// gate survives a process restart. +pub const PENDING_APPROVAL_METADATA_KEY: &str = "pending_approval"; + +/// Metadata key set on a thread that has a serialized runtime checkpoint +/// (CodeAct VM state, nudge counters, compaction count). Threads carrying +/// this key are suspended on restart instead of failed. +pub const RUNTIME_CHECKPOINT_METADATA_KEY: &str = "runtime_checkpoint"; + +/// Metadata key set on threads that were forced into `Failed` by +/// [`ThreadManager::recover_project_threads`] because the process +/// restarted before they could complete. The thread did not fail for +/// user-visible reasons; the projects "needs attention" surface filters +/// these out so an upgrade does not cascade into a wall of phantom +/// failure warnings. +pub const ENGINE_RESTART_RECOVERY_METADATA_KEY: &str = "engine_restart_recovery"; + fn is_resolved_call_message(message: &ThreadMessage, call_id: &str) -> bool { if message.role == MessageRole::ActionResult && message.action_call_id.as_deref() == Some(call_id) @@ -782,7 +861,15 @@ mod tests { _: &[CapabilityLease], _: &crate::traits::effect::ThreadExecutionContext, ) -> Result, EngineError> { - Ok(vec![]) + Ok(vec![ActionDef { + name: "test_tool".into(), + description: "Test".into(), + parameters_schema: serde_json::json!({}), + effects: vec![EffectType::ReadLocal], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }]) } async fn available_capabilities( @@ -1428,15 +1515,25 @@ mod tests { .await .unwrap(); - // Give it a moment to start, then stop - tokio::time::sleep(Duration::from_millis(10)).await; - let _ = mgr.stop_thread(tid, "test-user").await; + mgr.stop_thread(tid, "user").await.unwrap(); let outcome = mgr.join_thread(tid).await.unwrap(); - assert!(matches!( - outcome, - ThreadOutcome::Stopped | ThreadOutcome::Completed { .. } | ThreadOutcome::MaxIterations - )); + // The test's primary assertion is that `join_thread` returns — + // `stop_thread` must not deadlock. Any terminal outcome + // satisfies that. Failed is included because the executor's + // consecutive-error guard (#2325) will trip on the + // unregistered `test_tool` calls if it lands before the stop + // signal does, which is timing-dependent. + assert!( + matches!( + outcome, + ThreadOutcome::Stopped + | ThreadOutcome::Completed { .. } + | ThreadOutcome::MaxIterations + | ThreadOutcome::Failed { .. } + ), + "join_thread returned a non-terminal outcome: {outcome:?}" + ); } #[tokio::test] @@ -1507,6 +1604,28 @@ mod tests { assert_eq!(saved.state, ThreadState::Failed); let events = store.load_events(running.id).await.unwrap(); assert!(!events.is_empty()); + + // Restart-recovery flag must be set so the projects "needs + // attention" feed can filter these out (#3274). + assert_eq!( + saved + .metadata + .get(ENGINE_RESTART_RECOVERY_METADATA_KEY) + .and_then(|v| v.as_bool()), + Some(true), + "recovered thread should carry the engine_restart_recovery flag" + ); + + // Threads that were already terminal before recovery must NOT + // gain the flag — they failed for real reasons. + let saved_completed = store.load_thread(completed.id).await.unwrap().unwrap(); + assert!( + saved_completed + .metadata + .get(ENGINE_RESTART_RECOVERY_METADATA_KEY) + .is_none(), + "pre-existing failed thread must not be flagged as restart-recovery" + ); } #[tokio::test] diff --git a/crates/ironclaw_engine/src/runtime/mission.rs b/crates/ironclaw_engine/src/runtime/mission.rs index feb9166aec7..e8406fd57f1 100644 --- a/crates/ironclaw_engine/src/runtime/mission.rs +++ b/crates/ironclaw_engine/src/runtime/mission.rs @@ -25,7 +25,8 @@ use crate::traits::workspace::WorkspaceReader; use crate::types::error::EngineError; use crate::types::memory::{DocId, DocType, MemoryDoc}; use crate::types::mission::{ - Mission, MissionCadence, MissionId, MissionStatus, next_cron_fire, next_cron_fire_required, + Mission, MissionCadence, MissionGateInfo, MissionId, MissionStatus, next_cron_fire, + next_cron_fire_required, }; use crate::types::project::ProjectId; use crate::types::shared_owner_id; @@ -114,47 +115,23 @@ pub struct MissionNotification { pub gate: Option, } -/// Gate metadata carried alongside a `MissionNotification` when the -/// mission's child thread paused on an unresolved gate. +/// Outcome of a gate resolution as far as the mission auto-resume path +/// cares. Approved → resume + maybe immediate fire. Denied or Cancelled +/// → transition to Failed so the user has to explicitly fix-and-resume. /// -/// The bridge translates this into a `StatusUpdate::AuthRequired` -/// (or `ApprovalNeeded`) so the user-facing auth tray (gateway UI) / -/// chat-channel auth prompt fires for the mission, the same way it -/// fires for foreground threads. The mission itself is moved to -/// `MissionStatus::Paused` so the cron scheduler stops re-firing -/// until the gate is resolved. -#[derive(Debug, Clone)] -pub struct MissionGateInfo { - /// Engine-side gate name (e.g., `auth_required`, `approval_required`). - pub gate_name: String, - /// Action that paused (e.g., `tool_activate`, `gmail`, `http`). - pub action_name: String, - /// Original action parameters at the time of the pause. The bridge - /// needs these to run the canonical extension-name resolver - /// (`AuthManager::resolve_extension_name_for_auth_flow`) — for - /// `tool_activate(name="gmail")` the resolver reads the explicit - /// `name` param, which has higher precedence than the credential - /// fallback. - pub parameters: serde_json::Value, - /// Engine-side LLM tool-call id (e.g. `call_657a9167...`). This is - /// the identifier the engine needs to inject the resolved action - /// result back onto a paused thread when half-2 of #3133 wires up - /// auto-resume. **Not** what the user-facing auth tray sees — that - /// is `gate_request_id` below. (Per Copilot review on #3155: the - /// gateway parses the channel-side `request_id` as a UUID, so - /// forwarding the LLM call_id there would 400 every resolve POST.) - pub call_id: String, - /// Freshly-generated UUID identifying this surfaced gate to the - /// user. Forwarded by the bridge into - /// `StatusUpdate::AuthRequired.request_id` / - /// `StatusUpdate::ApprovalNeeded.request_id` so the gateway UI - /// can render the auth-tray entry and the resolve POST handler - /// (which `Uuid::parse_str`'s its input) accepts the response. - /// Mapping back to the engine `call_id` for half-2 auto-resume is - /// out of scope for this PR (#3166). - pub gate_request_id: uuid::Uuid, - /// What kind of resolution unblocks this gate. - pub resume_kind: crate::gate::ResumeKind, +/// Kept as a small standalone enum (rather than reusing +/// [`crate::gate::GateResolution`]) because the auto-resume path doesn't +/// need the resolution payload (token, callback body, allow-always +/// flag) — only the success/deny disposition. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GateResolutionOutcome { + /// User approved the gate — resume the mission. + Approved, + /// User denied the gate — mark the mission Failed. + Denied, + /// Gate was cancelled (timeout, shutdown, explicit cancel) — same + /// terminal outcome as Denied. + Cancelled, } /// Optional updates to apply to a mission via [`MissionManager::update_mission`]. @@ -642,6 +619,11 @@ impl MissionManager { // `update_mission`/`fire_mission` could modify other fields that the // second save would then silently overwrite with the stale reload. mission.status = MissionStatus::Active; + // Clear the paused_gate marker; the gate is no longer pending. If the + // user resumed manually before the gate was actually resolved, the + // mission will simply re-pause on its next fire and write a fresh + // marker. + mission.paused_gate = None; if let MissionCadence::Cron { ref expression, ref timezone, @@ -1085,6 +1067,261 @@ impl MissionManager { Ok(missions.into_iter().find(|m| m.name == name)) } + /// Resume every paused mission belonging to `user_id` whose + /// `paused_gate` was waiting for the credential named `credential_name`. + /// + /// Half-2 of #3133. When the user completes OAuth or stores a token + /// under a known credential name, every paused mission whose child + /// thread tripped the matching `Authentication` gate is transitioned + /// `Paused → Active`, scheduled for an immediate fire if it is + /// cron-driven, and has its `paused_gate` cleared. + /// + /// Returns the ids of the missions that were resumed. + /// + /// Multiple paused missions can share a credential — for example two + /// different Gmail-based missions both stalled on `google_oauth_token` + /// — so the result is a `Vec`, not an `Option`. + pub async fn resume_paused_for_credential( + &self, + credential_name: &ironclaw_common::CredentialName, + user_id: &str, + ) -> Result, EngineError> { + let candidates = self.list_paused_missions_for_user(user_id).await?; + let target = credential_name.as_str(); + let mut resumed = Vec::new(); + for mission in candidates { + let matches = match mission.paused_gate.as_ref().map(|g| &g.resume_kind) { + Some(crate::gate::ResumeKind::Authentication { + credential_name: cred, + .. + }) => cred.as_str() == target, + _ => false, + }; + if !matches { + continue; + } + // Resume scoping: + // - `resume_mission` itself enforces ownership: shared + // missions accept a shared-owner caller, user-owned + // missions accept only their owner. Pass `mission.user_id` + // so the access check matches. + // - `fire_mission` runs the work — for shared missions it + // should run under the requesting user (who just wrote + // the credential that unblocked it), not under the + // `__shared__` placeholder, because the spawned child + // thread needs the requesting user's project / secret + // scope. + let resume_owner = mission.user_id.clone(); + let fire_owner = if mission.owner_id().is_shared() { + user_id.to_string() + } else { + resume_owner.clone() + }; + match self.resume_mission(mission.id, &resume_owner).await { + Ok(()) => { + debug!( + mission_id = %mission.id, + credential = %target, + "auto-resumed paused mission after credential write" + ); + resumed.push(mission.id); + // Cron-driven and event-driven missions get an + // immediate fire so the user sees follow-through. A + // mission paused mid-manual-fire still re-fires here: + // the user already initiated the action, completing + // OAuth was their continuation. Best-effort — a + // failure to spawn the immediate thread does not + // unwind the resume. + // + // Event-driven missions (`OnEvent`, `OnSystemEvent`, + // `Webhook`) auto-fire here with `trigger_payload = + // None`. The original triggering payload is gone by + // the time OAuth lands — best-effort continuation + // matches the legacy behavior. A future improvement + // is to preserve the trigger payload on + // `paused_gate` and replay it here; tracked as a + // follow-up. + if !matches!(mission.cadence, MissionCadence::Manual) + && let Err(e) = self.fire_mission(mission.id, &fire_owner, None).await + { + debug!( + mission_id = %mission.id, + error = %e, + "post-resume immediate fire failed; will retry on next tick" + ); + } + } + Err(e) => { + debug!( + mission_id = %mission.id, + error = %e, + "failed to auto-resume paused mission" + ); + } + } + } + Ok(resumed) + } + + /// Resume the paused mission whose `paused_gate.gate_request_id` + /// matches `gate_request_id`. Used by the gate-resolve path + /// (`/api/chat/gate/resolve`) for `Approval` and `External` gates. + /// + /// On `Approved`: transitions the mission `Paused → Active` and + /// kicks an immediate fire (same as the credential path). + /// On `Denied` / `Cancelled`: marks the mission `Failed` instead, so + /// the user has to explicitly resume after fixing the underlying + /// disagreement. + pub async fn resume_paused_for_request_id( + &self, + gate_request_id: uuid::Uuid, + resolution: GateResolutionOutcome, + user_id: &str, + ) -> Result, EngineError> { + let candidates = self.list_paused_missions_for_user(user_id).await?; + let Some(snapshot) = candidates.into_iter().find(|m| { + m.paused_gate + .as_ref() + .is_some_and(|g| g.gate_request_id == gate_request_id) + }) else { + return Ok(None); + }; + let mission_id = snapshot.id; + // Re-load the mission and re-check that the same + // `paused_gate.gate_request_id` is still pending in a single + // mutate-and-save round-trip. Without this, between the + // snapshot scan above and `resume_mission`, the mission could + // have been manually resumed (or paused again on a different + // gate) — `resume_mission` itself accepts both `Paused` and + // `Failed`, so it would transition a freshly re-paused mission + // back to Active and silently clear the new gate. + let mut mission = + self.store + .load_mission(mission_id) + .await? + .ok_or_else(|| EngineError::Store { + reason: format!("mission {mission_id} not found"), + })?; + // Ownership check (mirror of `resume_mission`). + let allowed = if mission.owner_id().is_shared() { + crate::types::is_shared_owner(user_id) + } else { + mission.is_owned_by(user_id) + }; + if !allowed { + return Err(EngineError::AccessDenied { + user_id: user_id.to_string(), + entity: format!("mission {mission_id}"), + }); + } + // Atomic gate-request-id match. If the live mission is no + // longer paused on this exact gate, return `None` instead of + // mutating — the caller can interpret that as "someone else + // already handled it". + let still_matches = mission.status == MissionStatus::Paused + && mission + .paused_gate + .as_ref() + .is_some_and(|g| g.gate_request_id == gate_request_id); + if !still_matches { + debug!( + mission_id = %mission_id, + %gate_request_id, + live_status = ?mission.status, + "mission no longer paused on this gate at resume time; skipping" + ); + return Ok(None); + } + let cadence_for_fire = mission.cadence.clone(); + // Resume scoping mirror of `resume_paused_for_credential`: + // ownership-check uses `mission.user_id`, fire uses the + // requesting user for shared missions so the spawned thread + // sees the requesting user's project / secret scope. + let fire_owner = if mission.owner_id().is_shared() { + user_id.to_string() + } else { + mission.user_id.clone() + }; + match resolution { + GateResolutionOutcome::Approved => { + mission.status = MissionStatus::Active; + mission.paused_gate = None; + if let MissionCadence::Cron { + ref expression, + ref timezone, + } = mission.cadence + { + mission.next_fire_at = + Some(next_cron_fire_required(expression, timezone.as_ref())?); + } + mission.updated_at = chrono::Utc::now(); + self.store.save_mission(&mission).await?; + { + let mut active = self.active.write().await; + if !active.contains(&mission_id) { + active.push(mission_id); + } + } + debug!( + mission_id = %mission_id, + %gate_request_id, + "auto-resumed paused mission after approval gate resolved" + ); + if !matches!(cadence_for_fire, MissionCadence::Manual) + && let Err(e) = self.fire_mission(mission_id, &fire_owner, None).await + { + debug!( + mission_id = %mission_id, + error = %e, + "post-resume immediate fire failed; will retry on next tick" + ); + } + Ok(Some(mission_id)) + } + GateResolutionOutcome::Denied | GateResolutionOutcome::Cancelled => { + mission.status = MissionStatus::Failed; + mission.paused_gate = None; + mission.approach_history.push(format!( + "FAILED: gate {gate_request_id} denied or cancelled" + )); + mission.updated_at = chrono::Utc::now(); + self.store.save_mission(&mission).await?; + self.active.write().await.retain(|mid| *mid != mission_id); + debug!( + mission_id = %mission_id, + %gate_request_id, + "marked paused mission Failed after gate denial/cancel" + ); + Ok(Some(mission_id)) + } + } + } + + /// Helper: list every paused mission visible to `user_id` across all + /// projects. Walks `Store::list_all_projects` + per-project + /// `list_missions_with_shared` because the engine has no "paused + /// mission index" today; the cardinality is small (paused missions + /// are rare) so the linear scan is fine. + async fn list_paused_missions_for_user( + &self, + user_id: &str, + ) -> Result, EngineError> { + let projects = self.store.list_all_projects().await?; + let mut paused = Vec::new(); + for project in projects { + let missions = self + .store + .list_missions_with_shared(project.id, user_id) + .await?; + paused.extend( + missions + .into_iter() + .filter(|m| m.status == MissionStatus::Paused && m.paused_gate.is_some()), + ); + } + Ok(paused) + } + /// Fire all active `OnSystemEvent` missions whose source and event_type match. /// /// The optional `payload` is forwarded as `trigger_payload` to each mission's @@ -2456,14 +2693,20 @@ async fn process_mission_outcome_and_notify( kind = resume_kind.kind_name() )); notify_response = Some(nudge); - gate_info = Some(MissionGateInfo { + let info = MissionGateInfo { gate_name: gate_name.clone(), action_name: action_name.clone(), parameters: parameters.clone(), call_id: call_id.clone(), gate_request_id: uuid::Uuid::new_v4(), resume_kind: resume_kind.clone(), - }); + }; + // Persist the gate metadata on the mission itself so the + // credential-write / gate-resolve auto-resume paths (#3166) + // can match a paused mission back to the gate it's waiting + // on without keeping a parallel in-memory map. + mission.paused_gate = Some(info.clone()); + gate_info = Some(info); } _ => {} } @@ -2754,6 +2997,13 @@ async fn dispatch_protected_write( thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + // Mission writes never gate (synthetic lease, internal path). + // If `memory_write` ever surfaces a gate it cancels cleanly via + // a typed denial rather than reproducing the legacy unwind bug. + gate_controller: crate::gate::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; effects @@ -3342,6 +3592,7 @@ mod tests { threads: tokio::sync::RwLock>, missions: tokio::sync::RwLock>, docs: tokio::sync::RwLock>, + projects: tokio::sync::RwLock>, /// Optional gate that blocks the next `save_mission` call until /// the test releases it. Used by `fire_mission_arms_cooldown_before_save` /// to deterministically observe the in-flight save state. @@ -3357,6 +3608,7 @@ mod tests { threads: tokio::sync::RwLock::new(HashMap::new()), missions: tokio::sync::RwLock::new(HashMap::new()), docs: tokio::sync::RwLock::new(Vec::new()), + projects: tokio::sync::RwLock::new(HashMap::new()), save_mission_gate: tokio::sync::Mutex::new(None), save_mission_started: tokio::sync::Notify::new(), } @@ -3438,12 +3690,19 @@ mod tests { Ok(vec![]) } - // ── Project (noop) ── - async fn save_project(&self, _: &Project) -> Result<(), EngineError> { + // ── Project ── + async fn save_project(&self, project: &Project) -> Result<(), EngineError> { + self.projects + .write() + .await + .insert(project.id, project.clone()); Ok(()) } - async fn load_project(&self, _: ProjectId) -> Result, EngineError> { - Ok(None) + async fn load_project(&self, id: ProjectId) -> Result, EngineError> { + Ok(self.projects.read().await.get(&id).cloned()) + } + async fn list_all_projects(&self) -> Result, EngineError> { + Ok(self.projects.read().await.values().cloned().collect()) } // ── MemoryDoc ── @@ -4164,8 +4423,8 @@ mod tests { } /// Regression for #3133. When a mission's child thread emits - /// `ThreadOutcome::GatePaused` (e.g. the inner `tool_activate(gmail)` - /// triggered an OAuth gate), the previous `_ => {}` arm in + /// `ThreadOutcome::GatePaused` (e.g. a direct `gmail` call triggered + /// the engine's auth preflight gate), the previous `_ => {}` arm in /// `process_mission_outcome_and_notify` swallowed it: the mission /// stayed `Active` and the cron scheduler kept re-firing the same /// broken mission every tick without surfacing anything to the user. @@ -4213,7 +4472,7 @@ mod tests { let outcome = ThreadOutcome::GatePaused { gate_name: "auth_required".into(), - action_name: "tool_activate".into(), + action_name: "gmail".into(), call_id: "call-gmail-1".into(), parameters: serde_json::json!({"name": "gmail"}), resume_kind: ResumeKind::Authentication { @@ -4254,7 +4513,7 @@ mod tests { .gate .as_ref() .expect("notification must carry MissionGateInfo for GatePaused"); - assert_eq!(gate.action_name, "tool_activate"); + assert_eq!(gate.action_name, "gmail"); // call_id round-trips the engine's LLM tool-call id verbatim — // the bridge keeps it for half-2 auto-resume but does NOT // surface it to the channel. @@ -4325,6 +4584,326 @@ mod tests { "cron tick filters by Active; pausing on a gate must take \ the mission off the active path" ); + + // (5) The mission carries persistent gate metadata so the + // credential-write / gate-resolve auto-resume paths (#3166) + // can find it without a parallel index. + let pg = mission + .paused_gate + .as_ref() + .expect("paused_gate must be persisted on the mission for auto-resume to work"); + assert_eq!(pg.gate_request_id, gate.gate_request_id); + assert_eq!(pg.call_id, gate.call_id); + } + + /// Half-2 of #3133. After a mission paused on an Authentication + /// gate and the user completes OAuth (writing the credential), + /// the auto-resume path must transition the mission Paused → Active + /// and clear `paused_gate`. + #[tokio::test] + async fn oauth_completion_resumes_paused_mission() { + use crate::gate::ResumeKind; + use ironclaw_common::CredentialName; + + let store = Arc::new(TestStore::new()); + let store_dyn = Arc::clone(&store) as Arc; + let mgr = make_mission_manager(Arc::clone(&store_dyn)); + let project = Project::new("test-user", "test-project", ""); + let project_id = project.id; + store.save_project(&project).await.unwrap(); + + let id = mgr + .create_mission( + project_id, + "test-user", + "Gmail Drafter", + "Create a Gmail draft on every tick", + MissionCadence::Cron { + expression: "*/3 * * * *".into(), + timezone: None, + }, + vec!["gateway".to_string()], + ) + .await + .unwrap(); + + // Drive the mission into a Paused state via a GatePaused outcome. + let (notification_tx, _rx) = tokio::sync::broadcast::channel::(8); + let outcome = ThreadOutcome::GatePaused { + gate_name: "auth_required".into(), + action_name: "gmail".into(), + call_id: "call-gmail-1".into(), + parameters: serde_json::json!({"name": "gmail"}), + resume_kind: ResumeKind::Authentication { + credential_name: CredentialName::new("google_oauth_token").unwrap(), + instructions: "Sign in with Google.".into(), + auth_url: None, + }, + resume_output: None, + paused_lease: None, + }; + process_mission_outcome_and_notify( + &store_dyn, + None, + id, + ThreadId::new(), + &outcome, + ¬ification_tx, + None, + ) + .await + .unwrap(); + assert_eq!( + mgr.get_mission(id).await.unwrap().unwrap().status, + MissionStatus::Paused + ); + + // OAuth completes — the mission should auto-resume. + let cred = CredentialName::new("google_oauth_token").unwrap(); + let resumed = mgr + .resume_paused_for_credential(&cred, "test-user") + .await + .unwrap(); + assert!( + resumed.contains(&id), + "auto-resume must include the matching paused mission; got {resumed:?}" + ); + + let mission = mgr.get_mission(id).await.unwrap().unwrap(); + assert_eq!( + mission.status, + MissionStatus::Active, + "credential write must transition Paused → Active" + ); + assert!( + mission.paused_gate.is_none(), + "paused_gate must be cleared after auto-resume" + ); + } + + /// Half-2 of #3133. A credential write under an unrelated name must + /// not resume a mission that was waiting on a different credential. + #[tokio::test] + async fn unrelated_credential_write_does_not_resume_paused_mission() { + use crate::gate::ResumeKind; + use ironclaw_common::CredentialName; + + let store = Arc::new(TestStore::new()); + let store_dyn = Arc::clone(&store) as Arc; + let mgr = make_mission_manager(Arc::clone(&store_dyn)); + let project = Project::new("test-user", "test-project", ""); + let project_id = project.id; + store.save_project(&project).await.unwrap(); + + let id = mgr + .create_mission( + project_id, + "test-user", + "Gmail Drafter", + "Create a Gmail draft on every tick", + MissionCadence::Cron { + expression: "*/3 * * * *".into(), + timezone: None, + }, + Vec::new(), + ) + .await + .unwrap(); + + let (notification_tx, _rx) = tokio::sync::broadcast::channel::(8); + let outcome = ThreadOutcome::GatePaused { + gate_name: "auth_required".into(), + action_name: "gmail".into(), + call_id: "call-gmail-2".into(), + parameters: serde_json::json!({"name": "gmail"}), + resume_kind: ResumeKind::Authentication { + credential_name: CredentialName::new("google_oauth_token").unwrap(), + instructions: "Sign in with Google.".into(), + auth_url: None, + }, + resume_output: None, + paused_lease: None, + }; + process_mission_outcome_and_notify( + &store_dyn, + None, + id, + ThreadId::new(), + &outcome, + ¬ification_tx, + None, + ) + .await + .unwrap(); + + // A *different* credential name lands — must NOT resume the Gmail mission. + let unrelated = CredentialName::new("notion_api_token").unwrap(); + let resumed = mgr + .resume_paused_for_credential(&unrelated, "test-user") + .await + .unwrap(); + assert!( + resumed.is_empty(), + "unrelated credential write must not resume; got {resumed:?}" + ); + + let mission = mgr.get_mission(id).await.unwrap().unwrap(); + assert_eq!( + mission.status, + MissionStatus::Paused, + "Paused mission must stay Paused when an unrelated credential is written" + ); + assert!( + mission.paused_gate.is_some(), + "paused_gate must remain so the matching credential write can still resume later" + ); + } + + /// Half-2 of #3133. An approval gate resolved via + /// `/api/chat/gate/resolve` with `Approved` resumes the matching + /// paused mission keyed by `gate_request_id`. + #[tokio::test] + async fn gate_resolution_approved_resumes_matching_paused_mission() { + use crate::gate::ResumeKind; + + let store = Arc::new(TestStore::new()); + let store_dyn = Arc::clone(&store) as Arc; + let mgr = make_mission_manager(Arc::clone(&store_dyn)); + let project = Project::new("test-user", "test-project", ""); + let project_id = project.id; + store.save_project(&project).await.unwrap(); + + let id = mgr + .create_mission( + project_id, + "test-user", + "Risky Sender", + "Send the daily summary email", + MissionCadence::Cron { + expression: "*/3 * * * *".into(), + timezone: None, + }, + vec!["gateway".to_string()], + ) + .await + .unwrap(); + + let (notification_tx, mut rx) = tokio::sync::broadcast::channel::(8); + let outcome = ThreadOutcome::GatePaused { + gate_name: "approval_required".into(), + action_name: "gmail".into(), + call_id: "call-send-1".into(), + parameters: serde_json::json!({"action": "send_message"}), + resume_kind: ResumeKind::Approval { allow_always: true }, + resume_output: None, + paused_lease: None, + }; + process_mission_outcome_and_notify( + &store_dyn, + None, + id, + ThreadId::new(), + &outcome, + ¬ification_tx, + None, + ) + .await + .unwrap(); + + let notif = rx.try_recv().unwrap(); + let request_id = notif.gate.unwrap().gate_request_id; + + let resumed = mgr + .resume_paused_for_request_id(request_id, GateResolutionOutcome::Approved, "test-user") + .await + .unwrap(); + assert_eq!(resumed, Some(id)); + + let mission = mgr.get_mission(id).await.unwrap().unwrap(); + assert_eq!(mission.status, MissionStatus::Active); + assert!(mission.paused_gate.is_none()); + } + + /// Half-2 of #3133. A `Denied` (or `Cancelled`) gate-resolve marks + /// the paused mission Failed — the user has to explicitly resume + /// after fixing whatever caused the denial. + #[tokio::test] + async fn gate_resolution_denied_marks_paused_mission_failed() { + use crate::gate::ResumeKind; + + let store = Arc::new(TestStore::new()); + let store_dyn = Arc::clone(&store) as Arc; + let mgr = make_mission_manager(Arc::clone(&store_dyn)); + let project = Project::new("test-user", "test-project", ""); + let project_id = project.id; + store.save_project(&project).await.unwrap(); + + let id = mgr + .create_mission( + project_id, + "test-user", + "Risky Sender", + "Send the daily summary email", + MissionCadence::Cron { + expression: "*/3 * * * *".into(), + timezone: None, + }, + vec!["gateway".to_string()], + ) + .await + .unwrap(); + + let (notification_tx, mut rx) = tokio::sync::broadcast::channel::(8); + let outcome = ThreadOutcome::GatePaused { + gate_name: "approval_required".into(), + action_name: "gmail".into(), + call_id: "call-send-2".into(), + parameters: serde_json::json!({}), + resume_kind: ResumeKind::Approval { + allow_always: false, + }, + resume_output: None, + paused_lease: None, + }; + process_mission_outcome_and_notify( + &store_dyn, + None, + id, + ThreadId::new(), + &outcome, + ¬ification_tx, + None, + ) + .await + .unwrap(); + + let notif = rx.try_recv().unwrap(); + let request_id = notif.gate.unwrap().gate_request_id; + + let resumed = mgr + .resume_paused_for_request_id(request_id, GateResolutionOutcome::Denied, "test-user") + .await + .unwrap(); + assert_eq!(resumed, Some(id)); + + let mission = mgr.get_mission(id).await.unwrap().unwrap(); + assert_eq!( + mission.status, + MissionStatus::Failed, + "Denied gate must transition the mission to Failed, not Active" + ); + assert!( + mission.paused_gate.is_none(), + "Failed mission must not retain stale paused_gate metadata" + ); + assert!( + mission + .approach_history + .iter() + .any(|e| e.contains("denied") || e.contains("cancelled")), + "approach_history should record the gate denial: {:?}", + mission.approach_history + ); } #[tokio::test] diff --git a/crates/ironclaw_engine/src/traits/effect.rs b/crates/ironclaw_engine/src/traits/effect.rs index d88ad4d607e..bcd6a72f857 100644 --- a/crates/ironclaw_engine/src/traits/effect.rs +++ b/crates/ironclaw_engine/src/traits/effect.rs @@ -6,7 +6,9 @@ use std::sync::Arc; +use crate::gate::GateController; use crate::types::capability::{ActionDef, ActionInventory, CapabilityLease, CapabilitySummary}; +use crate::types::conversation::ConversationId; use crate::types::error::EngineError; use crate::types::project::ProjectId; use crate::types::step::{ActionResult, StepId}; @@ -17,7 +19,7 @@ use ironclaw_common::ValidTimezone; /// /// Passed to the executor so it can make context-dependent decisions /// (e.g. different tool behavior in background vs foreground threads). -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ThreadExecutionContext { pub thread_id: ThreadId, pub thread_type: ThreadType, @@ -42,6 +44,72 @@ pub struct ThreadExecutionContext { pub available_actions_snapshot: Option>, /// Snapshot of the full action inventory visible to the current step. pub available_action_inventory_snapshot: Option>, + /// Originating conversation scope identifier supplied by the host + /// channel before the engine allocated `thread_id`. Lets the host's + /// effect executor look up per-conversation state (for example a + /// caller-supplied tool catalog) against the same key the host + /// registered under, without racing the engine task that started + /// running before the host could rebind the state onto the engine + /// `thread_id`. + pub conversation_scope: Option, + /// Host-supplied callback that lets the executor pause inline on + /// `Approval` gates instead of unwinding back to the orchestrator. + /// + /// Required. Code paths that don't pause (post-resolution replay, + /// background mission writes, tests) supply + /// [`crate::gate::CancellingGateController::arc()`], which surfaces + /// any unexpected gate as a typed denial rather than the historical + /// "execution paused by gate" RuntimeError leak. + pub gate_controller: Arc, + /// Set to `true` when the host has already collected user approval + /// for *this specific call* (matched by `current_call_id`) and the + /// executor is retrying it inline. The host's `EffectExecutor` impl + /// uses this to skip the `ApprovalRequirement::Always` / + /// `AskEachTime` gate that would otherwise re-fire on retry — + /// mirrors the legacy `execute_resolved_pending_action` path that + /// passes `approval_already_granted=true`. + /// + /// One-shot: scoped to a single retry call. Reset to `false` on + /// any context not owned by an inline retry. + pub call_approval_granted: bool, + /// The conversation that originated this thread, if any. Carried + /// into [`crate::gate::GatePauseRequest`] so the host can match a + /// gate to the originating UI surface even when the same user has + /// multiple concurrent conversations (e.g. two browser tabs). + /// `None` for background mission threads with no user-facing + /// conversation. + pub conversation_id: Option, +} + +// Manual Debug impl: `dyn GateController` is not Debug, but the rest of +// the struct is. The controller is opaque host-supplied state — it +// renders as a constant marker rather than its internals. +impl std::fmt::Debug for ThreadExecutionContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ThreadExecutionContext") + .field("thread_id", &self.thread_id) + .field("thread_type", &self.thread_type) + .field("project_id", &self.project_id) + .field("user_id", &self.user_id) + .field("step_id", &self.step_id) + .field("current_call_id", &self.current_call_id) + .field("source_channel", &self.source_channel) + .field("user_timezone", &self.user_timezone) + .field("thread_goal", &self.thread_goal) + .field( + "available_actions_snapshot", + &self.available_actions_snapshot.as_ref().map(|a| a.len()), + ) + .field( + "available_action_inventory_snapshot", + &self.available_action_inventory_snapshot.is_some(), + ) + .field("conversation_scope", &self.conversation_scope) + .field("gate_controller", &"") + .field("call_approval_granted", &self.call_approval_granted) + .field("conversation_id", &self.conversation_id) + .finish() + } } /// Abstraction over capability action execution. diff --git a/crates/ironclaw_engine/src/traits/store.rs b/crates/ironclaw_engine/src/traits/store.rs index a9b96d54353..7b6ecebaad3 100644 --- a/crates/ironclaw_engine/src/traits/store.rs +++ b/crates/ironclaw_engine/src/traits/store.rs @@ -215,6 +215,11 @@ pub trait Store: Send + Sync { } let mut missions = self.list_missions(project_id, user_id).await?; missions.extend(self.list_shared_missions(project_id).await?); + // Sort deterministically so callers (LLM tool dispatch, replay + // tests, UI lists) see a stable order independent of HashMap + // iteration. Sort key: name (stable, human-readable), then id + // (tiebreaker for any duplicate names). + missions.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.0.cmp(&b.id.0))); Ok(missions) } diff --git a/crates/ironclaw_engine/src/types/capability.rs b/crates/ironclaw_engine/src/types/capability.rs index 92f58aff19e..14f0b5115a0 100644 --- a/crates/ironclaw_engine/src/types/capability.rs +++ b/crates/ironclaw_engine/src/types/capability.rs @@ -297,8 +297,10 @@ pub enum CapabilitySummaryKind { /// Ready callable actions stay in `ActionInventory`. `CapabilitySummary` /// covers: /// - runtime/contextual information that should stay in background prompt/UI -/// - blocked managed integrations that are shown separately and enabled via -/// `tool_activate(name=...)` +/// - integrations that need user setup before becoming callable +/// (`NeedsSetup`, `Inactive`, `Latent`, `AvailableNotInstalled`); these +/// surface to the model under `Activatable Integrations` so it can tell +/// the user what's available but cannot be enabled by the model itself #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CapabilitySummary { /// Stable capability identifier (for example `telegram` or `slack`). diff --git a/crates/ironclaw_engine/src/types/mission.rs b/crates/ironclaw_engine/src/types/mission.rs index af6622d9666..52989baea5a 100644 --- a/crates/ironclaw_engine/src/types/mission.rs +++ b/crates/ironclaw_engine/src/types/mission.rs @@ -11,6 +11,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::gate::ResumeKind; use crate::types::error::EngineError; use crate::types::project::ProjectId; use crate::types::thread::ThreadId; @@ -100,6 +101,35 @@ pub enum MissionCadence { Manual, } +/// Persisted gate metadata for a mission whose child thread paused on +/// an unresolved gate (auth, approval, or external callback). +/// +/// Two identifiers are carried because they serve different consumers: +/// +/// - `call_id` — engine-side LLM tool-call id (e.g. `call_657a9167...`). +/// Internal to engine resume. +/// - `gate_request_id` — UUID surfaced to the user-facing auth tray and +/// `/api/chat/gate/resolve` handler. The handler `Uuid::parse_str`'s +/// the field, so it MUST be a UUID, not the engine call_id. +/// +/// Used by both half-1 (`MissionNotification.gate`) and half-2 +/// (`Mission.paused_gate`) of #3133. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MissionGateInfo { + /// Engine-side gate name (e.g., `auth_required`, `approval_required`). + pub gate_name: String, + /// Action that paused (e.g., `gmail`, `http`, `slack_send`). + pub action_name: String, + /// Original action parameters at the time of the pause. + pub parameters: serde_json::Value, + /// Engine-side LLM tool-call id (e.g. `call_657a9167...`). + pub call_id: String, + /// Freshly-generated UUID identifying this surfaced gate to the user. + pub gate_request_id: Uuid, + /// What kind of resolution unblocks this gate. + pub resume_kind: ResumeKind, +} + /// A mission — a long-running goal that spawns threads over time. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Mission { @@ -181,6 +211,16 @@ pub struct Mission { pub updated_at: DateTime, /// When the next thread should be spawned (for Cron cadence). pub next_fire_at: Option>, + + /// When the mission's child thread paused on an unresolved gate, this + /// records the gate metadata so the credential-write / gate-resolve + /// paths can match a paused mission back to its waiting gate and + /// auto-resume it (#3166 / half-2 of #3133). + /// + /// Cleared when the mission resumes (manually or automatically) or + /// fires successfully. + #[serde(default)] + pub paused_gate: Option, } impl Mission { @@ -245,6 +285,7 @@ impl Mission { created_at: now, updated_at: now, next_fire_at: None, + paused_gate: None, } } diff --git a/crates/ironclaw_gateway/src/assets.rs b/crates/ironclaw_gateway/src/assets.rs index 5850e5d9232..10708fc6c5e 100644 --- a/crates/ironclaw_gateway/src/assets.rs +++ b/crates/ironclaw_gateway/src/assets.rs @@ -172,3 +172,23 @@ pub const ADMIN_CSS: &str = include_str!("../static/admin/admin.css"); /// Admin panel JavaScript. pub const ADMIN_JS: &str = include_str!("../static/admin/admin.js"); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn logs_toolbar_exposes_download_button() { + assert!(INDEX_HTML.contains("id=\"logs-download-btn\"")); + assert!(INDEX_HTML.contains("data-i18n=\"logs.download\"")); + } + + #[test] + fn logs_surface_can_export_buffer_as_jsonl() { + assert!(APP_JS.contains("downloadLogsJsonl")); + assert!(APP_JS.contains("serializeLogEntriesAsJsonl")); + assert!(APP_JS.contains("ironclaw-logs-")); + assert!(APP_JS.contains("logs-download-btn').addEventListener('click'")); + assert!(APP_JS.contains("setTimeout(() => URL.revokeObjectURL(url)")); + } +} diff --git a/crates/ironclaw_gateway/static/i18n/en.js b/crates/ironclaw_gateway/static/i18n/en.js index 4c76a0d5fd5..714d4785a02 100644 --- a/crates/ironclaw_gateway/static/i18n/en.js +++ b/crates/ironclaw_gateway/static/i18n/en.js @@ -29,6 +29,9 @@ I18n.register('en', { 'restart.progressSubtitle': 'Please wait for the process to restart...', 'restart.checkLogs': 'Check the Logs tab for details after restart completes.', 'restart.closeTooltip': 'Close', + 'restart.timedOut': 'Restart is taking longer than expected. The server may not have come back online yet — refresh the page once the connection indicator turns green.', + 'restart.refreshPage': 'Refresh page', + 'restart.dismiss': 'Dismiss', // Theme 'theme.tooltipDark': 'Theme: Dark (click for Light)', @@ -105,7 +108,9 @@ I18n.register('en', { 'status.restartTooltip': 'Gracefully restart the process', 'status.active': 'Active', 'status.installed': 'Installed', + 'status.configured': 'Configured', 'status.awaitingPairing': 'Awaiting Pairing', + 'status.pairingShort': 'Pairing', // Dashboard 'dashboard.connections': 'Connections', @@ -212,6 +217,7 @@ I18n.register('en', { 'logs.clientLevel': 'Client Log Level', 'logs.pause': 'Pause', 'logs.resume': 'Resume', + 'logs.download': 'Download', 'logs.clear': 'Clear', 'logs.autoScroll': 'Auto-scroll', 'logs.filter': 'Filter logs...', @@ -322,16 +328,16 @@ I18n.register('en', { 'time.daysFromNow': 'in {n}d', // Tool Approval - 'approval.title': 'Tool requires approval', - 'approval.description': 'A tool is requesting permission to run.', + 'approval.title': 'Approve tool call', + 'approval.description': 'The agent wants to run this tool. Approve to continue, or deny to stop here.', 'approval.approve': 'Approve', 'approval.deny': 'Deny', - 'approval.always': 'Always', + 'approval.always': 'Always allow', 'approval.approved': 'Approved', 'approval.alwaysApproved': 'Always approved', 'approval.denied': 'Denied', - 'approval.showParams': 'Show parameters', - 'approval.hideParams': 'Hide parameters', + 'approval.showParams': 'Show full parameters', + 'approval.hideParams': 'Hide full parameters', // Authentication Required 'authRequired.title': 'Authentication required for {name}', @@ -499,13 +505,48 @@ I18n.register('en', { // Configure 'config.title': 'Configure {name}', + 'config.telegramOwnerHint': 'After saving, IronClaw will show a one-time code. Send `/start CODE` to your bot in Telegram and IronClaw will finish setup automatically.', + 'config.telegramChallengeTitle': 'Telegram owner verification', + 'config.telegramOwnerWaiting': 'Waiting for Telegram owner verification...', + 'config.telegramCommandLabel': 'Send this in Telegram:', + 'config.telegramStartOver': 'Start over', + 'config.telegramStartOverHint': 'Telegram verification did not complete. Click Start over to generate a new code and try again.', + 'config.telegramOpenBot': 'Open bot in Telegram', + 'config.wechatHint': 'Open the WeChat QR page in a new tab, then scan and confirm in WeChat.', + 'config.wechatConnect': 'Open QR Page', + 'config.wechatWaiting': 'Waiting for scan...', + 'config.wechatPreparingQr': 'Preparing WeChat QR page...', + 'config.wechatQrTitle': 'Open WeChat QR Page', + 'config.wechatQrOpen': 'Open QR Page', + 'config.wechatQrIntro': 'The QR flow opens in a separate tab.', + 'config.wechatQrReady': 'QR page is ready. Open it in a new tab, then scan and confirm in WeChat.', + 'config.wechatQrWaiting': 'Preparing the WeChat QR page...', + 'config.wechatQrScanned': 'QR scanned. Confirm the login in WeChat.', + 'config.wechatQrRefreshed': 'QR page refreshed.', + 'config.wechatQrRefreshedHint': 'The previous QR page expired. Open the new page and scan again.', + 'config.wechatConnected': 'WeChat connected.', + 'config.wechatQrFailed': 'WeChat connection failed.', + 'config.interactiveLoginStartFailed': 'Failed to start interactive login', + 'config.interactiveLoginFailed': 'Interactive login failed', + 'config.connectedSuccess': '{name} connected successfully', 'config.optional': ' (optional)', 'config.alreadySet': '(already set — leave empty to keep)', 'config.alreadyConfigured': 'Already configured', 'config.autoGenerate': 'Auto-generated if empty', + 'config.optionalSection': 'Optional settings', + 'config.optionalSectionSummary': 'Optional settings ({count})', + 'config.requiredFieldMissing': '{name} is required.', + 'config.invalidField': '{name} has an invalid format.', + 'config.controlCharsNotAllowed': '{name} cannot contain control characters.', + 'config.requiredPlaceholder': 'Required', + 'config.optionalPlaceholder': 'Optional', 'config.save': 'Save', 'config.cancel': 'Cancel', + // Setup fields + 'setup.secret.wecom_bot_id': 'Bot ID', + 'setup.secret.wecom_bot_secret': 'Bot Secret', + // Settings toolbar 'settings.export': 'Export', 'settings.import': 'Import', diff --git a/crates/ironclaw_gateway/static/i18n/index.js b/crates/ironclaw_gateway/static/i18n/index.js index 9fe9094cdcd..d1be48eba00 100644 --- a/crates/ironclaw_gateway/static/i18n/index.js +++ b/crates/ironclaw_gateway/static/i18n/index.js @@ -37,6 +37,9 @@ const I18n = { localStorage.setItem('ironclaw_language', lang); this.updateHtmlLang(); this.updatePageContent(); + window.dispatchEvent(new CustomEvent('ironclaw:language-changed', { + detail: { lang } + })); return true; } return false; diff --git a/crates/ironclaw_gateway/static/i18n/ko.js b/crates/ironclaw_gateway/static/i18n/ko.js index ac12c03ba50..ed5a1eaa128 100644 --- a/crates/ironclaw_gateway/static/i18n/ko.js +++ b/crates/ironclaw_gateway/static/i18n/ko.js @@ -29,6 +29,9 @@ I18n.register('ko', { 'restart.progressSubtitle': '프로세스가 재시작될 때까지 기다려 주세요...', 'restart.checkLogs': '재시작이 완료된 후 자세한 내용은 로그 탭을 확인하세요.', 'restart.closeTooltip': '닫기', + 'restart.timedOut': '재시작이 예상보다 오래 걸리고 있습니다. 서버가 아직 복구되지 않았을 수 있습니다 — 연결 표시등이 녹색으로 바뀐 뒤 페이지를 새로 고쳐 주세요.', + 'restart.refreshPage': '페이지 새로 고침', + 'restart.dismiss': '닫기', // 테마 'theme.tooltipDark': '테마: 다크 (클릭하여 라이트로 변경)', @@ -212,6 +215,7 @@ I18n.register('ko', { 'logs.clientLevel': '클라이언트 로그 레벨', 'logs.pause': '일시정지', 'logs.resume': '재개', + 'logs.download': '다운로드', 'logs.clear': '지우기', 'logs.autoScroll': '자동 스크롤', 'logs.filter': '로그 필터링...', @@ -322,16 +326,16 @@ I18n.register('ko', { 'time.daysFromNow': '{n}일 후', // 도구 승인 - 'approval.title': '도구 승인 필요', - 'approval.description': '도구가 실행 권한을 요청하고 있습니다.', + 'approval.title': '도구 호출 승인', + 'approval.description': '에이전트가 이 도구를 실행하려고 합니다. 계속하려면 승인, 여기서 중단하려면 거부를 선택하세요.', 'approval.approve': '승인', 'approval.deny': '거부', - 'approval.always': '항상', + 'approval.always': '항상 허용', 'approval.approved': '승인됨', 'approval.alwaysApproved': '항상 승인', 'approval.denied': '거부됨', - 'approval.showParams': '매개변수 표시', - 'approval.hideParams': '매개변수 숨기기', + 'approval.showParams': '전체 매개변수 표시', + 'approval.hideParams': '전체 매개변수 숨기기', // 인증 필요 'authRequired.title': '{name}에 인증 필요', @@ -502,9 +506,20 @@ I18n.register('ko', { 'config.alreadySet': '(이미 설정됨 — 비워두면 유지)', 'config.alreadyConfigured': '이미 구성됨', 'config.autoGenerate': '비어 있으면 자동 생성됨', + 'config.optionalSection': '선택 설정', + 'config.optionalSectionSummary': '선택 설정 ({count})', + 'config.requiredFieldMissing': '{name}은(는) 필수입니다.', + 'config.invalidField': '{name} 형식이 올바르지 않습니다.', + 'config.controlCharsNotAllowed': '{name}에는 제어 문자를 사용할 수 없습니다.', + 'config.requiredPlaceholder': '필수', + 'config.optionalPlaceholder': '선택', 'config.save': '저장', 'config.cancel': '취소', + // 설정 필드 + 'setup.secret.wecom_bot_id': 'Bot ID', + 'setup.secret.wecom_bot_secret': 'Bot Secret', + // 설정 도구 모음 'settings.export': '내보내기', 'settings.import': '가져오기', diff --git a/crates/ironclaw_gateway/static/i18n/zh-CN.js b/crates/ironclaw_gateway/static/i18n/zh-CN.js index e0fd4932527..7c933c7fa01 100644 --- a/crates/ironclaw_gateway/static/i18n/zh-CN.js +++ b/crates/ironclaw_gateway/static/i18n/zh-CN.js @@ -29,7 +29,10 @@ I18n.register('zh-CN', { 'restart.progressSubtitle': '请等待进程重启...', 'restart.checkLogs': '重启完成后,请查看日志标签页了解详情。', 'restart.closeTooltip': '关闭', - + 'restart.timedOut': '重启耗时比预期更长。服务可能尚未恢复——请待连接指示灯变绿后刷新页面。', + 'restart.refreshPage': '刷新页面', + 'restart.dismiss': '关闭', + // 主题 'theme.tooltipDark': '主题:深色(点击切换浅色)', 'theme.tooltipLight': '主题:浅色(点击切换跟随系统)', @@ -105,7 +108,9 @@ I18n.register('zh-CN', { 'status.restartTooltip': '优雅地重启进程', 'status.active': '已激活', 'status.installed': '已安装', + 'status.configured': '已配置', 'status.awaitingPairing': '等待配对', + 'status.pairingShort': '配对中', // 仪表盘 'dashboard.connections': '连接数', @@ -212,6 +217,7 @@ I18n.register('zh-CN', { 'logs.clientLevel': '客户端日志级别', 'logs.pause': '暂停', 'logs.resume': '继续', + 'logs.download': '下载', 'logs.clear': '清空', 'logs.autoScroll': '自动滚动', 'logs.filter': '筛选日志...', @@ -322,16 +328,16 @@ I18n.register('zh-CN', { 'time.daysFromNow': '{n}天后', // 工具审批 - 'approval.title': '工具需要审批', - 'approval.description': '一个工具请求运行权限。', + 'approval.title': '批准工具调用', + 'approval.description': '智能体想要运行此工具。批准以继续,或拒绝以在此停止。', 'approval.approve': '批准', 'approval.deny': '拒绝', 'approval.always': '始终允许', 'approval.approved': '已批准', 'approval.alwaysApproved': '始终批准', 'approval.denied': '已拒绝', - 'approval.showParams': '显示参数', - 'approval.hideParams': '隐藏参数', + 'approval.showParams': '显示完整参数', + 'approval.hideParams': '隐藏完整参数', // 认证 'authRequired.title': '{name} 需要认证', @@ -498,13 +504,48 @@ I18n.register('zh-CN', { // 配置 'config.title': '配置 {name}', + 'config.telegramOwnerHint': '保存后,IronClaw 会显示一次性验证码。将 `/start CODE` 发送给你的 Telegram 机器人,IronClaw 会自动完成设置。', + 'config.telegramChallengeTitle': 'Telegram 所有者验证', + 'config.telegramOwnerWaiting': '正在等待 Telegram 所有者验证...', + 'config.telegramCommandLabel': '请在 Telegram 中发送:', + 'config.telegramStartOver': '重新开始', + 'config.telegramStartOverHint': 'Telegram 验证未完成。点击“重新开始”以生成新的验证码并重试。', + 'config.telegramOpenBot': '在 Telegram 中打开机器人', + 'config.wechatHint': '在新标签页打开微信扫码页,然后在微信里扫码并确认。', + 'config.wechatConnect': '打开扫码页', + 'config.wechatWaiting': '等待扫码中...', + 'config.wechatPreparingQr': '正在准备微信扫码页...', + 'config.wechatQrTitle': '打开微信扫码页', + 'config.wechatQrOpen': '打开扫码页', + 'config.wechatQrIntro': '扫码流程会在新标签页中打开。', + 'config.wechatQrReady': '扫码页已就绪。请在新标签页打开后,用微信扫码并确认。', + 'config.wechatQrWaiting': '正在准备微信扫码页...', + 'config.wechatQrScanned': '已扫码,请在微信中确认登录。', + 'config.wechatQrRefreshed': '扫码页已刷新。', + 'config.wechatQrRefreshedHint': '之前的扫码页已过期,请打开新页面重新扫码。', + 'config.wechatConnected': '微信已连接。', + 'config.wechatQrFailed': '微信连接失败。', + 'config.interactiveLoginStartFailed': '启动交互式登录失败', + 'config.interactiveLoginFailed': '交互式登录失败', + 'config.connectedSuccess': '{name} 连接成功', 'config.optional': '(可选)', 'config.alreadySet': '(已设置 — 留空以保持不变)', 'config.alreadyConfigured': '已配置', 'config.autoGenerate': '如果为空则自动生成', + 'config.optionalSection': '可选设置', + 'config.optionalSectionSummary': '可选设置({count} 项)', + 'config.requiredFieldMissing': '请填写{name}。', + 'config.invalidField': '{name}格式不正确。', + 'config.controlCharsNotAllowed': '{name}不能包含控制字符。', + 'config.requiredPlaceholder': '必填', + 'config.optionalPlaceholder': '可选', 'config.save': '保存', 'config.cancel': '取消', + // 设置字段 + 'setup.secret.wecom_bot_id': '机器人 ID', + 'setup.secret.wecom_bot_secret': '机器人 Secret', + // 设置工具栏 'settings.export': '导出', 'settings.import': '导入', diff --git a/crates/ironclaw_gateway/static/index.html b/crates/ironclaw_gateway/static/index.html index acf4cea9d50..223717ce7cf 100644 --- a/crates/ironclaw_gateway/static/index.html +++ b/crates/ironclaw_gateway/static/index.html @@ -149,6 +149,15 @@

Restart IronClaw Instance

Check the Logs tab for details after the restart completes.

+ @@ -330,6 +339,7 @@

Restart IronClaw Instance

+
diff --git a/crates/ironclaw_gateway/static/js/core/history.js b/crates/ironclaw_gateway/static/js/core/history.js index a3b2c91d329..e6257bea9aa 100644 --- a/crates/ironclaw_gateway/static/js/core/history.js +++ b/crates/ironclaw_gateway/static/js/core/history.js @@ -239,8 +239,15 @@ function loadHistory(before) { hasMore = data.has_more || false; oldestTimestamp = data.oldest_timestamp || null; - }).catch(() => { - // No history or no active thread + }).catch((err) => { + // Surface the error in DevTools and flag for SSE-open retry (#3274). + // The previous silent swallow left the user staring at an empty chat + // when the very first request after auth raced engine initialization + // and only a manual refresh recovered. + console.error('[chat] loadHistory failed:', err); + if (window._initialHydrationPending) { + window._initialHydrationPending.history = true; + } }).finally(() => { loadingOlder = false; removeScrollSpinner(); @@ -520,7 +527,12 @@ function loadThreads() { enableChatInput(); } } - }).catch(() => {}); + }).catch((err) => { + console.error('[chat] loadThreads failed:', err); + if (window._initialHydrationPending) { + window._initialHydrationPending.threads = true; + } + }); } function disableChatInputReadOnly() { diff --git a/crates/ironclaw_gateway/static/js/core/init-auth.js b/crates/ironclaw_gateway/static/js/core/init-auth.js index 45085efc2df..c4b22bfa2c0 100644 --- a/crates/ironclaw_gateway/static/js/core/init-auth.js +++ b/crates/ironclaw_gateway/static/js/core/init-auth.js @@ -1,4 +1,35 @@ +// Tracks loaders that failed on the very first call after `initApp()`. The +// SSE `onopen` handler in `core/sse.js` retries each flagged loader exactly +// once — see `runInitialHydrationRetry` below. Defensive net for the upgrade +// race in #3274 where the first hydration request loses to in-flight engine +// state initialization or DB migration; a manual refresh used to be the only +// recovery path. See `.claude/rules/error-handling.md` (silent-failure rule). +function runInitialHydrationRetry() { + var pending = window._initialHydrationPending; + if (!pending || window._hydrationRetryDone) return; + window._hydrationRetryDone = true; + if (pending.threads && typeof loadThreads === 'function') { + console.info('[hydration] retrying loadThreads after SSE connect'); + loadThreads(); + } + if (pending.history && typeof loadHistory === 'function') { + console.info('[hydration] retrying loadHistory after SSE connect'); + loadHistory(); + } + if (pending.missions + && currentTab === 'missions' + && typeof loadMissions === 'function') { + console.info('[hydration] retrying loadMissions after SSE connect'); + loadMissions(); + } + window._initialHydrationPending = null; +} + function initApp() { + // Reset hydration tracker each time we (re-)initialize the app — token + // re-auth and OIDC auto-auth both flow through here. + window._initialHydrationPending = { threads: false, history: false, missions: false }; + window._hydrationRetryDone = false; var authScreen = document.getElementById('auth-screen'); var app = document.getElementById('app'); // Cross-fade: fade out auth screen, then show app @@ -328,6 +359,59 @@ function apiFetch(path, options) { let isRestarting = false; // Track if we're currently restarting let restartEnabled = false; // Track if restart is available in this deployment +let _restartWatchdogTimer = null; // Bug #3082: surface stuck restarts to the user. +const RESTART_WATCHDOG_MS = 45000; + +function clearRestartWatchdog() { + if (_restartWatchdogTimer) { + clearTimeout(_restartWatchdogTimer); + _restartWatchdogTimer = null; + } +} + +function showRestartLoaderError(message) { + const errEl = document.getElementById('restart-loader-error'); + const errText = document.getElementById('restart-loader-error-text'); + if (!errEl || !errText) return; + errText.textContent = message; + errEl.style.display = 'block'; + // Hide the indeterminate progress bar so the user understands the wait is over. + const bar = document.querySelector('#restart-loader .restart-progress-bar'); + if (bar) bar.style.display = 'none'; +} + +function dismissRestartLoader() { + clearRestartWatchdog(); + const loaderEl = document.getElementById('restart-loader'); + if (loaderEl) loaderEl.style.display = 'none'; + const errEl = document.getElementById('restart-loader-error'); + if (errEl) errEl.style.display = 'none'; + const bar = document.querySelector('#restart-loader .restart-progress-bar'); + if (bar) bar.style.display = ''; + isRestarting = false; + const restartBtn = document.getElementById('restart-btn'); + const restartIcon = document.getElementById('restart-icon'); + if (restartBtn) restartBtn.disabled = false; + if (restartIcon) restartIcon.classList.remove('spinning'); +} + +// Reset the loader to a fresh "spinning" state and arm the watchdog. +// Shared between user-initiated restarts (confirmRestart) and +// agent-initiated restarts surfaced via SSE (tryShowRestartModal) so +// every entry path gets the same #3082 recovery behavior. +function armRestartWatchdog() { + const errEl = document.getElementById('restart-loader-error'); + if (errEl) errEl.style.display = 'none'; + const bar = document.querySelector('#restart-loader .restart-progress-bar'); + if (bar) bar.style.display = ''; + + clearRestartWatchdog(); + _restartWatchdogTimer = setTimeout(() => { + _restartWatchdogTimer = null; + if (!isRestarting) return; + showRestartLoaderError(I18n.t('restart.timedOut')); + }, RESTART_WATCHDOG_MS); +} function triggerRestart() { if (!currentThreadId) { @@ -358,9 +442,10 @@ function confirmRestart() { restartBtn.disabled = true; if (restartIcon) restartIcon.classList.add('spinning'); - // Show progress modal + // Show progress modal and arm the watchdog (#3082). const loaderEl = document.getElementById('restart-loader'); loaderEl.style.display = 'flex'; + armRestartWatchdog(); // Send restart command via chat console.log('[confirmRestart] Sending /restart command to server'); @@ -378,10 +463,12 @@ function confirmRestart() { .catch((err) => { console.error('[confirmRestart] Restart request failed:', err); addMessage('system', I18n.t('error.restartFailed', { message: err.message })); + // Cancel the watchdog and clear the restarting flag so the 45 s + // timeout doesn't later replace this specific error with a generic + // "taking longer than expected" message (#3082). + clearRestartWatchdog(); isRestarting = false; - restartBtn.disabled = false; - if (restartIcon) restartIcon.classList.remove('spinning'); - loaderEl.style.display = 'none'; + showRestartLoaderError(I18n.t('error.restartFailed', { message: err.message })); }); } @@ -391,7 +478,8 @@ function cancelRestart() { } function tryShowRestartModal() { - // Defensive callback for when restart is detected in messages. + // Defensive callback for when restart is detected in messages — covers + // agent-initiated restarts that didn't go through confirmRestart(). if (!isRestarting) { isRestarting = true; const restartBtn = document.getElementById('restart-btn'); @@ -399,9 +487,11 @@ function tryShowRestartModal() { restartBtn.disabled = true; if (restartIcon) restartIcon.classList.add('spinning'); - // Show progress modal + // Show progress modal and arm the same watchdog as confirmRestart so + // this path also recovers if the server never comes back (#3082). const loaderEl = document.getElementById('restart-loader'); loaderEl.style.display = 'flex'; + armRestartWatchdog(); } } diff --git a/crates/ironclaw_gateway/static/js/core/onboarding.js b/crates/ironclaw_gateway/static/js/core/onboarding.js index 7c12e582bff..192d7c23628 100644 --- a/crates/ironclaw_gateway/static/js/core/onboarding.js +++ b/crates/ironclaw_gateway/static/js/core/onboarding.js @@ -1,3 +1,42 @@ +// Issue #2991: extract the most-load-bearing parameter as a one-line summary +// so the user can decide without opening the parameter blob. Returns null +// when no useful summary can be derived. Output is bounded to one line and +// ~120 chars so a long URL or multi-line shell script can't push the +// approval buttons off-screen. +const APPROVAL_SUMMARY_MAX_LEN = 120; +function truncateApprovalSummary(s) { + if (s.length <= APPROVAL_SUMMARY_MAX_LEN) return s; + return s.slice(0, APPROVAL_SUMMARY_MAX_LEN - 1) + '…'; +} +function summarizeApprovalParams(toolName, params) { + if (!params || typeof params !== 'object') return null; + const name = String(toolName || '').toLowerCase().replace(/-/g, '_'); + if (name === 'http' || name === 'http_request' || name === 'web_fetch') { + const method = String(params.method || 'GET').toUpperCase(); + const url = typeof params.url === 'string' ? params.url.trim() + : typeof params.endpoint === 'string' ? params.endpoint.trim() : ''; + if (url.length > 0) return truncateApprovalSummary(method + ' ' + url); + } + if (name === 'shell' || name === 'bash' || name === 'exec') { + const raw = params.command || params.cmd || params.script; + if (typeof raw === 'string') { + // Collapse newlines + runs of whitespace so multi-line scripts + // render on a single line. + const cmd = raw.replace(/\s+/g, ' ').trim(); + if (cmd.length > 0) return truncateApprovalSummary(cmd); + } + } + if (name === 'file_write' || name === 'write_file' || name === 'apply_patch' + || name === 'file_read' || name === 'read_file' || name === 'list_dir') { + const raw = params.path || params.target; + if (typeof raw === 'string') { + const path = raw.trim(); + if (path.length > 0) return truncateApprovalSummary(path); + } + } + return null; +} + function showApproval(data) { // Avoid duplicate cards on reconnect/history refresh. const existing = document.querySelector('.approval-card[data-request-id="' + CSS.escape(data.request_id) + '"]'); @@ -22,6 +61,25 @@ function showApproval(data) { toolName.textContent = humanizeToolName(data.tool_name); card.appendChild(toolName); + // Try to render an actionable one-line summary from the parameters + // (e.g. "GET https://api.example.com/foo") so the approval prompt is + // self-explanatory instead of "A tool is requesting permission" (#2991). + let parsedParams = null; + if (data.parameters) { + try { + parsedParams = JSON.parse(data.parameters); + } catch (_e) { + parsedParams = null; + } + } + const summary = summarizeApprovalParams(data.tool_name, parsedParams); + if (summary) { + const summaryEl = document.createElement('div'); + summaryEl.className = 'approval-summary'; + summaryEl.textContent = summary; + card.appendChild(summaryEl); + } + if (data.description) { const desc = document.createElement('div'); desc.className = 'approval-description'; diff --git a/crates/ironclaw_gateway/static/js/core/sse.js b/crates/ironclaw_gateway/static/js/core/sse.js index 4e26815ac9f..3960ba9cd07 100644 --- a/crates/ironclaw_gateway/static/js/core/sse.js +++ b/crates/ironclaw_gateway/static/js/core/sse.js @@ -61,15 +61,10 @@ function connectSSE(lastEventIdOverride) { setTimeout(() => { lostBanner.remove(); }, 2000); } - // If we were restarting, close the modal and reset button now that server is back + // If we were restarting, close the modal and reset button now that server is back. + // dismissRestartLoader() also clears the watchdog timer (#3082). if (isRestarting) { - const loaderEl = document.getElementById('restart-loader'); - if (loaderEl) loaderEl.style.display = 'none'; - const restartBtn = document.getElementById('restart-btn'); - const restartIcon = document.getElementById('restart-icon'); - if (restartBtn) restartBtn.disabled = false; - if (restartIcon) restartIcon.classList.remove('spinning'); - isRestarting = false; + dismissRestartLoader(); } if (sseHasConnectedBefore && currentThreadId) { @@ -88,6 +83,12 @@ function connectSSE(lastEventIdOverride) { // Refresh sidebar so stale spinners are removed immediately. processingThreads.clear(); debouncedLoadThreads(); + // Retry any first-load loader (chat history, threads, missions) that + // raced engine init and failed silently. SSE-accept implies the + // backend has stabilized — see init-auth.js for the rationale (#3274). + if (typeof runInitialHydrationRetry === 'function') { + runInitialHydrationRetry(); + } sseHasConnectedBefore = true; }; @@ -177,7 +178,25 @@ function connectSSE(lastEventIdOverride) { _doneWithoutResponseTimer = null; } finalizeActivityGroup(); - addMessage('assistant', data.content); + + const messages = document.querySelectorAll('#chat-messages .message'); + const lastMessage = messages.length > 0 ? messages[messages.length - 1] : null; + const lastAssistantAlreadyHasResponse = Boolean( + lastMessage + && lastMessage.classList.contains('assistant') + && (lastMessage.getAttribute('data-raw') || '') === data.content + ); + + // Streamed responses already accumulated `data.content` into the + // bubble we just finalized. Separately, a thread switch or history + // refresh can render the completed response from /history before the + // matching SSE response arrives. In both cases, adding another bubble + // would render the same final assistant message twice. Only create a + // new bubble when the final response is not already the latest chat + // message (the normal non-streaming path). + if (!streamingMsg && !lastAssistantAlreadyHasResponse) { + addMessage('assistant', data.content); + } pruneOldMessages(); enableChatInput(); // Refresh thread list so new titles appear after first message diff --git a/crates/ironclaw_gateway/static/js/core/ui-helpers.js b/crates/ironclaw_gateway/static/js/core/ui-helpers.js index 20294ed68b2..14eb0b8e9d5 100644 --- a/crates/ironclaw_gateway/static/js/core/ui-helpers.js +++ b/crates/ironclaw_gateway/static/js/core/ui-helpers.js @@ -249,6 +249,9 @@ document.getElementById('restart-close-btn').addEventListener('click', () => can document.getElementById('restart-cancel-btn').addEventListener('click', () => cancelRestart()); document.getElementById('restart-confirm-btn').addEventListener('click', () => confirmRestart()); document.getElementById('restart-btn').addEventListener('click', () => triggerRestart()); +// Bug #3082 recovery affordances on the progress modal. +document.getElementById('restart-refresh-btn').addEventListener('click', () => window.location.reload()); +document.getElementById('restart-dismiss-btn').addEventListener('click', () => dismissRestartLoader()); document.getElementById('thread-new-btn').addEventListener('click', () => createNewThread()); document.getElementById('thread-toggle-btn').addEventListener('click', () => toggleThreadSidebar()); document.getElementById('send-btn').addEventListener('click', () => sendMessage()); @@ -257,6 +260,7 @@ document.getElementById('memory-save-btn').addEventListener('click', () => saveM document.getElementById('memory-cancel-btn').addEventListener('click', () => cancelMemoryEdit()); document.getElementById('logs-server-level').addEventListener('change', (e) => setServerLogLevel(e.target.value)); document.getElementById('logs-pause-btn').addEventListener('click', () => toggleLogsPause()); +document.getElementById('logs-download-btn').addEventListener('click', () => downloadLogsJsonl()); document.getElementById('logs-clear-btn').addEventListener('click', () => clearLogs()); document.getElementById('wasm-install-btn').addEventListener('click', () => installWasmExtension()); document.getElementById('mcp-add-btn').addEventListener('click', () => addMcpServer()); diff --git a/crates/ironclaw_gateway/static/js/surfaces/config.js b/crates/ironclaw_gateway/static/js/surfaces/config.js index 704b7f18e11..1ed3ad9bafe 100644 --- a/crates/ironclaw_gateway/static/js/surfaces/config.js +++ b/crates/ironclaw_gateway/static/js/surfaces/config.js @@ -7,6 +7,12 @@ const ADAPTER_LABELS = { ollama: 'Ollama', bedrock: 'AWS Bedrock', nearai: 'NEAR AI', + openai_codex: 'OpenAI Codex', + gemini_oauth: 'Gemini CLI OAuth', + github_copilot: 'GitHub Copilot', + deep_seek: 'DeepSeek', + gemini: 'Google Gemini', + open_router: 'OpenRouter', }; let _builtinProviders = []; @@ -62,6 +68,24 @@ function scrollToProviders() { /** Check whether a provider has all required credentials (API key + base URL if required). */ function isProviderConfigured(provider) { + // ── Non-API-key credential gate ──────────────────────────────────────── + // Built-ins with `credential_kind` other than `api_key` / + // `open_ai_compatible` / `ollama` (NEAR AI session token, Gemini + // OAuth creds file, OpenAI Codex device-code session, AWS Bedrock + // creds) carry no `api_key_required` signal — the backend ships + // `has_credentials` as the authoritative gate so the UI can't show + // them as Use-ready on a fresh install and accidentally trigger an + // interactive OAuth from a settings request. + if (provider.builtin) { + const kind = provider.credential_kind; + const isApiKeyShaped = kind === 'api_key' + || kind === 'open_ai_compatible' + || kind === 'ollama' + || kind === undefined; + if (!isApiKeyShaped && provider.has_credentials !== true) { + return false; + } + } // ── API key check ────────────────────────────────────────────────────── // Built-in providers carry `api_key_required` from the backend registry. // Custom providers don't — derive the requirement from the adapter instead: @@ -112,6 +136,23 @@ function isProviderConfigured(provider) { * isProviderConfigured — keep the two in sync. */ function providerMissingReason(provider) { + // Non-api-key gate — matches isProviderConfigured above. + if (provider.builtin) { + const kind = provider.credential_kind; + const isApiKeyShaped = kind === 'api_key' + || kind === 'open_ai_compatible' + || kind === 'ollama' + || kind === undefined; + if (!isApiKeyShaped && provider.has_credentials !== true) { + // Surface the specific missing credential kind so the toast can + // point the user at the right setup flow. + if (kind === 'session_token') return 'session_token'; + if (kind === 'o_auth_device_code') return 'oauth_session'; + if (kind === 'file_based_credentials') return 'credentials_file'; + if (kind === 'aws_credentials') return 'aws_credentials'; + return 'credentials'; + } + } // API key check — matches isProviderConfigured above. const needsKey = provider.builtin ? provider.api_key_required !== false @@ -335,7 +376,17 @@ function configureBuiltinProvider(id) { baseUrlInput.readOnly = false; baseUrlInput.style.opacity = ''; baseUrlInput.placeholder = p.base_url || ''; - document.getElementById('provider-api-key-row').style.display = p.api_key_required !== false ? '' : 'none'; + // `accepts_api_key` tells us whether the provider supports API key + // auth at all (so the field should be visible) — distinct from + // `api_key_required`, which says whether API key is the *only* way to + // configure it. NEAR AI is dual-auth (session token or API key) and + // ships `api_key_required: false` + `accepts_api_key: true`. Fall back + // to `api_key_required !== false` for older payloads that haven't been + // upgraded yet. + const acceptsApiKey = p.accepts_api_key !== undefined + ? p.accepts_api_key !== false + : p.api_key_required !== false; + document.getElementById('provider-api-key-row').style.display = acceptsApiKey ? '' : 'none'; document.getElementById('fetch-models-btn').style.display = p.can_list_models ? '' : 'none'; const apiKeyInput = document.getElementById('provider-api-key'); const hasDbKey = override.api_key === API_KEY_UNCHANGED; diff --git a/crates/ironclaw_gateway/static/js/surfaces/extensions.js b/crates/ironclaw_gateway/static/js/surfaces/extensions.js index 82191e6b367..e866a868dcd 100644 --- a/crates/ironclaw_gateway/static/js/surfaces/extensions.js +++ b/crates/ironclaw_gateway/static/js/surfaces/extensions.js @@ -580,7 +580,9 @@ function showConfigureModal(name, options) { .then((setup) => { const secrets = Array.isArray(setup.secrets) ? setup.secrets : []; const setupFields = Array.isArray(setup.fields) ? setup.fields : []; - if (secrets.length === 0 && setupFields.length === 0) { + const interactiveLogin = setup.interactive_login || null; + const onboarding = setup.onboarding || null; + if (secrets.length === 0 && setupFields.length === 0 && !interactiveLogin) { if (options && options.authData) { showAuthCard(options.authData); } else { @@ -588,7 +590,7 @@ function showConfigureModal(name, options) { } return; } - renderConfigureModal(name, secrets, setupFields, setup.onboarding || null, options); + renderConfigureModal(name, secrets, setupFields, interactiveLogin, onboarding, options); }) .catch((err) => { showToast(I18n.t('extensions.setupLoadFailed', { message: err.message }), 'error'); @@ -598,7 +600,151 @@ function showConfigureModal(name, options) { }); } -function renderConfigureModal(name, secrets, setupFields, onboarding, options) { +function translateOrFallback(key, fallback) { + const value = I18n.t(key); + return value === key ? fallback : value; +} + +function compactSetupPrompt(prompt) { + return String(prompt || '') + .replace(/^enter\s+(your\s+)?/i, '') + .replace(/\s*\((optional|for example|numeric|[0-9]+\s*chars?|leave empty).*?\)\s*/gi, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function setupFieldLabel(item) { + const fallback = compactSetupPrompt(item.prompt || item.name); + return translateOrFallback('setup.secret.' + item.name, fallback || item.name); +} + +function updateConfigureModalI18n(root) { + const scope = root || document; + scope.querySelectorAll('[data-configure-label-name]').forEach(function(label) { + const name = label.getAttribute('data-configure-label-name'); + const prompt = label.getAttribute('data-configure-label-prompt') || name; + label.textContent = setupFieldLabel({ name: name, prompt: prompt }); + }); + scope.querySelectorAll('[data-configure-placeholder]').forEach(function(input) { + const key = input.getAttribute('data-configure-placeholder'); + input.placeholder = I18n.t(key); + }); + scope.querySelectorAll('[data-configure-title-name]').forEach(function(title) { + title.textContent = I18n.t('config.title', { + name: title.getAttribute('data-configure-title-name') || '' + }); + }); + scope.querySelectorAll('[data-configure-optional-summary]').forEach(function(summary) { + summary.textContent = I18n.t('config.optionalSectionSummary', { + count: summary.getAttribute('data-configure-optional-summary') || '0' + }); + }); +} + +function basicConfigureValidationError(field, value) { + const label = setupFieldLabel(field); + if (/[\x00-\x1F\x7F]/.test(value)) { + return I18n.t('config.controlCharsNotAllowed', { name: label }); + } + + const validators = { + wecom_bot_id: /^[A-Za-z0-9_-]{3,128}$/, + wecom_bot_secret: /^[A-Za-z0-9_-]{16,128}$/ + }; + const validator = validators[field.name]; + if (validator && !validator.test(value)) { + return I18n.t('config.invalidField', { name: label }); + } + return null; +} + +function createConfigureField(item, kind, optionalGroup) { + const field = document.createElement('div'); + field.className = 'configure-field'; + if (kind === 'secret') field.dataset.secretName = item.name; + + const label = document.createElement('label'); + const labelText = document.createElement('span'); + labelText.setAttribute('data-configure-label-name', item.name); + labelText.setAttribute('data-configure-label-prompt', item.prompt || item.name); + labelText.textContent = setupFieldLabel(item); + label.appendChild(labelText); + if (item.optional && !optionalGroup) { + const opt = document.createElement('span'); + opt.className = 'field-optional'; + opt.textContent = I18n.t('config.optional'); + label.appendChild(opt); + } + field.appendChild(label); + + const inputRow = document.createElement('div'); + inputRow.className = 'configure-input-row'; + + const input = document.createElement('input'); + input.type = kind === 'field' && item.input_type !== 'password' ? 'text' : 'password'; + input.name = item.name; + const placeholderKey = item.provided + ? 'config.alreadySet' + : item.optional + ? 'config.optionalPlaceholder' + : 'config.requiredPlaceholder'; + input.setAttribute('data-configure-placeholder', placeholderKey); + input.placeholder = I18n.t(placeholderKey); + // Do not copy extension-provided regexes into HTML pattern. Browser regex + // engines can backtrack catastrophically; server-side validation is the + // security boundary for manifest-provided secret.validation patterns. + inputRow.appendChild(input); + + if (item.provided) { + const badge = document.createElement('span'); + badge.className = 'field-provided'; + badge.textContent = '\u2713'; + badge.title = I18n.t('config.alreadyConfigured'); + badge.setAttribute('data-i18n-title', 'config.alreadyConfigured'); + inputRow.appendChild(badge); + } + if (item.auto_generate && !item.provided) { + const hint = document.createElement('span'); + hint.className = 'field-autogen'; + hint.textContent = I18n.t('config.autoGenerate'); + hint.setAttribute('data-i18n', 'config.autoGenerate'); + inputRow.appendChild(hint); + } + + field.appendChild(inputRow); + return { + node: field, + field: { + kind: kind, + name: item.name, + input: input, + optional: !!item.optional, + provided: !!item.provided, + autoGenerate: !!item.auto_generate, + prompt: item.prompt || item.name + } + }; +} + +function appendConfigureFieldGroup(form, fields, items, kind, optionalGroup) { + for (const item of items) { + const built = createConfigureField(item, kind, optionalGroup); + built.field.input.addEventListener('keydown', (e) => { + if (e.key === 'Enter') submitConfigureModal( + form.closest('.configure-overlay')?.getAttribute('data-extension-name') || '', + fields + ); + }); + form.appendChild(built.node); + fields.push(built.field); + } +} + +window.addEventListener('ironclaw:language-changed', function() { + updateConfigureModalI18n(document); +}); + +function renderConfigureModal(name, secrets, setupFields, interactiveLogin, onboarding, options) { // Cancel any existing auth-flow overlay before replacing it. // Remove directly (don't clear authFlowPending) since a new overlay is about to be appended. var existingOverlay = document.querySelector('.configure-overlay'); @@ -630,6 +776,7 @@ function renderConfigureModal(name, secrets, setupFields, onboarding, options) { const header = document.createElement('h3'); header.textContent = I18n.t('config.title', { name: name }); + header.setAttribute('data-configure-title-name', name); modal.appendChild(header); if (onboarding && onboarding.credential_instructions) { @@ -639,96 +786,50 @@ function renderConfigureModal(name, secrets, setupFields, onboarding, options) { modal.appendChild(hint); } + if (interactiveLogin) { + const hint = document.createElement('div'); + hint.className = 'configure-hint'; + hint.textContent = interactiveLoginHintText(name, interactiveLogin); + modal.appendChild(hint); + } + const form = document.createElement('div'); form.className = 'configure-form'; const fields = []; - for (const secret of secrets) { - const field = document.createElement('div'); - field.className = 'configure-field'; - field.dataset.secretName = secret.name; - - const label = document.createElement('label'); - label.textContent = secret.prompt; - if (secret.optional) { - const opt = document.createElement('span'); - opt.className = 'field-optional'; - opt.textContent = I18n.t('config.optional'); - label.appendChild(opt); - } - field.appendChild(label); - - const inputRow = document.createElement('div'); - inputRow.className = 'configure-input-row'; - - const input = document.createElement('input'); - input.type = 'password'; - input.name = secret.name; - input.placeholder = secret.provided ? I18n.t('config.alreadySet') : ''; - input.addEventListener('keydown', (e) => { - if (e.key === 'Enter') submitConfigureModal(name, fields); - }); - inputRow.appendChild(input); - - if (secret.provided) { - const badge = document.createElement('span'); - badge.className = 'field-provided'; - badge.textContent = '\u2713'; - badge.title = I18n.t('config.alreadyConfigured'); - inputRow.appendChild(badge); - } - if (secret.auto_generate && !secret.provided) { - const hint = document.createElement('span'); - hint.className = 'field-autogen'; - hint.textContent = I18n.t('config.autoGenerate'); - inputRow.appendChild(hint); - } - - field.appendChild(inputRow); - form.appendChild(field); - fields.push({ kind: 'secret', name: secret.name, input: input }); + const requiredSecrets = secrets.filter((secret) => !secret.optional); + const optionalSecrets = secrets.filter((secret) => secret.optional); + const requiredSetupFields = setupFields.filter((field) => !field.optional); + const optionalSetupFields = setupFields.filter((field) => field.optional); + + appendConfigureFieldGroup(form, fields, requiredSecrets, 'secret', false); + appendConfigureFieldGroup(form, fields, requiredSetupFields, 'field', false); + + const optionalCount = optionalSecrets.length + optionalSetupFields.length; + if (optionalCount > 0) { + const optionalDetails = document.createElement('details'); + optionalDetails.className = 'configure-optional-group'; + + const summary = document.createElement('summary'); + summary.setAttribute('data-configure-optional-summary', String(optionalCount)); + summary.textContent = I18n.t('config.optionalSectionSummary', { count: String(optionalCount) }); + optionalDetails.appendChild(summary); + + const optionalBody = document.createElement('div'); + optionalBody.className = 'configure-optional-fields'; + appendConfigureFieldGroup(optionalBody, fields, optionalSecrets, 'secret', true); + appendConfigureFieldGroup(optionalBody, fields, optionalSetupFields, 'field', true); + optionalDetails.appendChild(optionalBody); + form.appendChild(optionalDetails); } - for (const setupField of setupFields) { - const field = document.createElement('div'); - field.className = 'configure-field'; - - const label = document.createElement('label'); - label.textContent = setupField.prompt; - if (setupField.optional) { - const opt = document.createElement('span'); - opt.className = 'field-optional'; - opt.textContent = I18n.t('config.optional'); - label.appendChild(opt); - } - field.appendChild(label); - - const inputRow = document.createElement('div'); - inputRow.className = 'configure-input-row'; - - const input = document.createElement('input'); - input.type = setupField.input_type === 'password' ? 'password' : 'text'; - input.name = setupField.name; - input.placeholder = setupField.provided ? I18n.t('config.alreadySet') : ''; - input.addEventListener('keydown', (e) => { - if (e.key === 'Enter') submitConfigureModal(name, fields); - }); - inputRow.appendChild(input); - - if (setupField.provided) { - const badge = document.createElement('span'); - badge.className = 'field-provided'; - badge.textContent = '\u2713'; - badge.title = I18n.t('config.alreadyConfigured'); - inputRow.appendChild(badge); - } - - field.appendChild(inputRow); - form.appendChild(field); - fields.push({ kind: 'field', name: setupField.name, input: input }); + if (fields.length > 0) { + modal.appendChild(form); } - modal.appendChild(form); + if (interactiveLogin) { + modal.appendChild(renderInteractiveLoginPanel(name)); + } const error = document.createElement('div'); error.className = 'configure-inline-error'; @@ -738,11 +839,23 @@ function renderConfigureModal(name, secrets, setupFields, onboarding, options) { const actions = document.createElement('div'); actions.className = 'configure-actions'; - const submitBtn = document.createElement('button'); - submitBtn.className = 'btn-ext activate'; - submitBtn.textContent = I18n.t('config.save'); - submitBtn.addEventListener('click', () => submitConfigureModal(name, fields)); - actions.appendChild(submitBtn); + if (fields.length > 0) { + const submitBtn = document.createElement('button'); + submitBtn.className = 'btn-ext activate'; + submitBtn.textContent = I18n.t('config.save'); + submitBtn.addEventListener('click', () => submitConfigureModal(name, fields)); + actions.appendChild(submitBtn); + } + + if (interactiveLogin) { + const loginBtn = document.createElement('button'); + loginBtn.className = 'btn-ext activate'; + loginBtn.dataset.defaultLabel = interactiveLoginDefaultLabel(name, interactiveLogin); + loginBtn.textContent = loginBtn.dataset.defaultLabel; + loginBtn.dataset.interactiveLogin = 'true'; + loginBtn.addEventListener('click', () => startInteractiveLogin(name, overlay)); + actions.appendChild(loginBtn); + } const cancelBtn = document.createElement('button'); cancelBtn.className = 'btn-ext remove'; @@ -760,7 +873,222 @@ function renderConfigureModal(name, secrets, setupFields, onboarding, options) { overlay.appendChild(modal); document.body.appendChild(overlay); - if (fields.length > 0) fields[0].input.focus(); + if (fields.length > 0) { + fields[0].input.focus(); + } else { + const loginBtn = overlay.querySelector('.configure-actions button[data-interactive-login="true"]'); + if (loginBtn) loginBtn.focus(); + } +} + +function renderInteractiveLoginPanel(name) { + const panel = document.createElement('div'); + panel.className = 'configure-qr-login'; + panel.style.display = 'none'; + + const title = document.createElement('div'); + title.className = 'configure-verification-title'; + title.textContent = + name === 'wechat' ? I18n.t('config.wechatQrTitle') : I18n.t('auth.connect'); + panel.appendChild(title); + + const status = document.createElement('div'); + status.className = 'configure-verification-instructions'; + status.textContent = interactiveLoginStatusText(name, null); + status.dataset.qrStatus = 'true'; + panel.appendChild(status); + + const link = document.createElement('a'); + link.className = 'configure-verification-link'; + link.textContent = + name === 'wechat' ? I18n.t('config.wechatQrOpen') : I18n.t('auth.connect'); + link.target = '_blank'; + link.rel = 'noreferrer noopener'; + link.style.display = 'none'; + link.dataset.qrLink = 'true'; + panel.appendChild(link); + + return panel; +} + +function interactiveLoginHintText(name, interactiveLogin) { + if (name === 'wechat') return I18n.t('config.wechatHint'); + return (interactiveLogin && interactiveLogin.instructions) || ''; +} + +function interactiveLoginDefaultLabel(name, interactiveLogin) { + if (name === 'wechat') return I18n.t('config.wechatConnect'); + return (interactiveLogin && interactiveLogin.button_label) || I18n.t('auth.connect'); +} + +function interactiveLoginWaitingLabel(name) { + if (name === 'wechat') return I18n.t('config.wechatWaiting'); + return I18n.t('status.connecting'); +} + +function interactiveLoginStatusText(name, res) { + if (name !== 'wechat') return (res && res.message) || ''; + if (!res) return I18n.t('config.wechatQrIntro'); + + switch (res.status) { + case 'pending': + return res.qr_code_url ? I18n.t('config.wechatQrReady') : I18n.t('config.wechatQrWaiting'); + case 'scanned': + return I18n.t('config.wechatQrScanned'); + case 'refreshed': + return I18n.t('config.wechatQrRefreshed'); + case 'succeeded': + return I18n.t('config.wechatConnected'); + case 'failed': + return res.message || I18n.t('config.wechatQrFailed'); + default: + return res.message || I18n.t('config.wechatQrIntro'); + } +} + +function getInteractiveLoginButton(overlay) { + return overlay && overlay.querySelector('.configure-actions button[data-interactive-login="true"]'); +} + +function getInteractiveLoginPanel(overlay) { + return overlay && overlay.querySelector('.configure-qr-login'); +} + +function updateInteractiveLoginPanel(overlay, res) { + const panel = getInteractiveLoginPanel(overlay); + if (!panel) return; + const name = overlay && overlay.dataset ? overlay.dataset.extensionName : ''; + const status = panel.querySelector('[data-qr-status="true"]'); + const link = panel.querySelector('[data-qr-link="true"]'); + + panel.style.display = ''; + if (status) { + if (name === 'wechat' && res.status === 'refreshed') { + status.textContent = I18n.t('config.wechatQrRefreshedHint'); + } else { + status.textContent = interactiveLoginStatusText(name, res); + } + } + + if (link && res.qr_code_url) { + link.href = res.qr_code_url; + link.style.display = ''; + } +} + +function maybeOpenInteractiveLoginUrl(name, overlay, res) { + if (name !== 'wechat' || !overlay || !res || !res.qr_code_url) return; + const qrUrl = res.qr_code_url; + if (overlay.dataset.interactiveLoginLastOpenedUrl === qrUrl) return; + overlay.dataset.interactiveLoginLastOpenedUrl = qrUrl; + window.open(qrUrl, '_blank', 'noopener,noreferrer'); +} + +function setInteractiveLoginBusy(overlay, busy, label) { + const loginBtn = getInteractiveLoginButton(overlay); + if (!loginBtn) return; + loginBtn.disabled = !!busy; + loginBtn.textContent = label || loginBtn.dataset.defaultLabel || I18n.t('auth.connect'); +} + +function interactiveLoginPollDelayMs(status) { + switch (status) { + case 'refreshed': + return 2000; + case 'pending': + case 'scanned': + return 3000; + default: + return 3000; + } +} + +function startInteractiveLogin(name, overlay) { + if (!overlay || !document.body.contains(overlay)) return; + clearConfigureInlineError(overlay); + setConfigureInlineStatus( + overlay, + name === 'wechat' ? I18n.t('config.wechatPreparingQr') : I18n.t('status.connecting'), + ); + setInteractiveLoginBusy(overlay, true, interactiveLoginWaitingLabel(name)); + + apiFetch('/api/extensions/' + encodeURIComponent(name) + '/login/start', { + method: 'POST', + body: { force: true }, + }) + .then((res) => { + if (!overlay || !document.body.contains(overlay)) return; + if (!res.success || !res.session_id) { + setInteractiveLoginBusy(overlay, false); + setConfigureInlineError( + overlay, + res.message || I18n.t('config.interactiveLoginStartFailed'), + ); + setConfigureInlineStatus(overlay, ''); + return; + } + + overlay.dataset.interactiveLoginSessionId = res.session_id; + updateInteractiveLoginPanel(overlay, res); + maybeOpenInteractiveLoginUrl(name, overlay, res); + setConfigureInlineStatus(overlay, interactiveLoginStatusText(name, res)); + pollInteractiveLogin(name, overlay, res.session_id); + }) + .catch((err) => { + if (!overlay || !document.body.contains(overlay)) return; + setInteractiveLoginBusy(overlay, false); + setConfigureInlineError( + overlay, + err.message || I18n.t('config.interactiveLoginStartFailed'), + ); + setConfigureInlineStatus(overlay, ''); + }); +} + +function pollInteractiveLogin(name, overlay, sessionId) { + if (!overlay || !document.body.contains(overlay)) return; + if (overlay.dataset.interactiveLoginSessionId !== sessionId) return; + + apiFetch('/api/extensions/' + encodeURIComponent(name) + '/login/poll', { + method: 'POST', + body: { session_id: sessionId }, + }) + .then((res) => { + if (!overlay || !document.body.contains(overlay)) return; + if (overlay.dataset.interactiveLoginSessionId !== sessionId) return; + + updateInteractiveLoginPanel(overlay, res); + maybeOpenInteractiveLoginUrl(name, overlay, res); + setConfigureInlineStatus(overlay, interactiveLoginStatusText(name, res)); + + if (res.status === 'pending' || res.status === 'scanned' || res.status === 'refreshed') { + if (res.status === 'refreshed') { + setInteractiveLoginBusy(overlay, true, interactiveLoginWaitingLabel(name)); + } + window.setTimeout(function() { + pollInteractiveLogin(name, overlay, sessionId); + }, interactiveLoginPollDelayMs(res.status)); + return; + } + + if (res.success && res.activated) { + closeConfigureModal(name); + showToast(res.message || I18n.t('config.connectedSuccess', { name: name }), 'success'); + refreshCurrentSettingsTab(); + return; + } + + setInteractiveLoginBusy(overlay, false); + setConfigureInlineError(overlay, res.message || I18n.t('config.interactiveLoginFailed')); + setConfigureInlineStatus(overlay, ''); + }) + .catch((err) => { + if (!overlay || !document.body.contains(overlay)) return; + if (overlay.dataset.interactiveLoginSessionId !== sessionId) return; + setInteractiveLoginBusy(overlay, false); + setConfigureInlineError(overlay, err.message || I18n.t('config.interactiveLoginFailed')); + setConfigureInlineStatus(overlay, ''); + }); } function setConfigureInlineError(overlay, message) { @@ -770,6 +1098,15 @@ function setConfigureInlineError(overlay, message) { error.style.display = message ? 'block' : 'none'; } +function setConfigureInlineStatus(overlay, message) { + const panel = getInteractiveLoginPanel(overlay); + if (!panel) return; + const status = panel.querySelector('[data-qr-status="true"]'); + if (!status) return; + panel.style.display = message ? '' : 'none'; + status.textContent = message || ''; +} + function clearConfigureInlineError(overlay) { setConfigureInlineError(overlay, ''); } @@ -778,6 +1115,33 @@ function submitConfigureModal(name, fields, options) { options = options || {}; const secrets = {}; const setupFields = {}; + const overlay = getConfigureOverlay(name) || document.querySelector('.configure-overlay'); + clearConfigureInlineError(overlay); + + for (const f of fields) { + f.input.classList.remove('configure-input-invalid'); + const value = f.input.value.trim(); + const missingRequired = !f.optional && !f.provided && !f.autoGenerate && !value; + if (missingRequired) { + const message = I18n.t('config.requiredFieldMissing', { name: setupFieldLabel(f) }); + f.input.classList.add('configure-input-invalid'); + setConfigureInlineError(overlay, message); + showToast(message, 'error'); + f.input.focus(); + return; + } + if (value) { + const validationError = basicConfigureValidationError(f, value); + if (validationError) { + f.input.classList.add('configure-input-invalid'); + setConfigureInlineError(overlay, validationError); + showToast(validationError, 'error'); + f.input.focus(); + return; + } + } + } + for (const f of fields) { const value = f.input.value.trim(); if (!value) { @@ -790,10 +1154,8 @@ function submitConfigureModal(name, fields, options) { } } - const overlay = getConfigureOverlay(name) || document.querySelector('.configure-overlay'); const requestId = overlay ? overlay.getAttribute('data-request-id') : null; const threadId = overlay ? overlay.getAttribute('data-thread-id') : null; - clearConfigureInlineError(overlay); // Disable buttons to prevent double-submit var btns = overlay ? overlay.querySelectorAll('.configure-actions button') : []; diff --git a/crates/ironclaw_gateway/static/js/surfaces/logs.js b/crates/ironclaw_gateway/static/js/surfaces/logs.js index 84dcca4525c..2d8fbc0cafb 100644 --- a/crates/ironclaw_gateway/static/js/surfaces/logs.js +++ b/crates/ironclaw_gateway/static/js/surfaces/logs.js @@ -1,6 +1,7 @@ const LOG_MAX_ENTRIES = 2000; let logsPaused = false; let logBuffer = []; // buffer while paused +let downloadLogEntries = []; // entries available for JSONL download function connectLogSSE() { if (logEventSource) logEventSource.close(); @@ -12,6 +13,7 @@ function connectLogSSE() { logEventSource.addEventListener('log', (e) => { const entry = JSON.parse(e.data); + rememberLogEntryForDownload(entry); if (logsPaused) { logBuffer.push(entry); return; @@ -24,6 +26,41 @@ function connectLogSSE() { }; } +function rememberLogEntryForDownload(entry) { + downloadLogEntries.push(entry); + while (downloadLogEntries.length > LOG_MAX_ENTRIES) { + downloadLogEntries.shift(); + } +} + +function serializeLogEntriesAsJsonl(entries) { + return entries.map(entry => JSON.stringify(entry)).join('\n') + (entries.length ? '\n' : ''); +} + +function logsDownloadFilename() { + const stamp = new Date() + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d{3}Z$/, 'Z') + .replace('T', '-'); + return 'ironclaw-logs-' + stamp + '.jsonl'; +} + +function downloadLogsJsonl() { + const blob = new Blob([serializeLogEntriesAsJsonl(downloadLogEntries)], { + type: 'application/x-ndjson;charset=utf-8', + }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = logsDownloadFilename(); + link.style.display = 'none'; + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 0); +} + function prependLogEntry(entry) { const output = document.getElementById('logs-output'); @@ -96,6 +133,7 @@ function clearLogs() { if (!confirm(I18n.t('logs.confirmClear'))) return; document.getElementById('logs-output').innerHTML = ''; logBuffer = []; + downloadLogEntries = []; } // Re-apply filters when level or target changes diff --git a/crates/ironclaw_gateway/static/js/surfaces/projects.js b/crates/ironclaw_gateway/static/js/surfaces/projects.js index b6455f14e89..303ea1a607f 100644 --- a/crates/ironclaw_gateway/static/js/surfaces/projects.js +++ b/crates/ironclaw_gateway/static/js/surfaces/projects.js @@ -719,7 +719,15 @@ function loadMissions() { renderMissionsList(currentMissionList); renderMissionsActivity(threadData.threads || []); enrichMissionProgress(currentMissionList); - }).catch(function() {}); + }).catch(function(err) { + // See #3274: a silent catch here left the Missions tab blank when the + // first request raced engine init. Log + flag so the SSE-open retry + // in init-auth.js refetches once the engine is fully ready. + console.error('[missions] loadMissions failed:', err); + if (window._initialHydrationPending) { + window._initialHydrationPending.missions = true; + } + }); } function renderMissionsSummary(s) { diff --git a/crates/ironclaw_gateway/static/styles/components/topbar.css b/crates/ironclaw_gateway/static/styles/components/topbar.css index 8ff10a3052c..1fd52656d1c 100644 --- a/crates/ironclaw_gateway/static/styles/components/topbar.css +++ b/crates/ironclaw_gateway/static/styles/components/topbar.css @@ -449,6 +449,24 @@ margin-bottom: 0; } +.restart-loader-error { + margin-top: 1.25rem; + padding-top: 1rem; + border-top: 1px solid var(--border-soft); +} + +.restart-loader-error-text { + color: var(--text-primary); + font-size: 0.875rem; + margin: 0 0 0.75rem 0; +} + +.restart-loader-error-actions { + display: flex; + gap: 0.5rem; + justify-content: flex-end; +} + .restart-modal-info a { color: var(--accent-brand); text-decoration: none; diff --git a/crates/ironclaw_gateway/static/styles/surfaces/chat.css b/crates/ironclaw_gateway/static/styles/surfaces/chat.css index 9bd54932761..2eda92fe906 100644 --- a/crates/ironclaw_gateway/static/styles/surfaces/chat.css +++ b/crates/ironclaw_gateway/static/styles/surfaces/chat.css @@ -609,6 +609,17 @@ font-family: var(--font-mono); } +.approval-summary { + font-size: var(--text-sm); + color: var(--text); + font-family: var(--font-mono); + background: var(--code-bg); + padding: 6px 10px; + border-radius: var(--radius); + word-break: break-all; + white-space: pre-wrap; +} + .approval-description { font-size: var(--text-sm); color: var(--text-secondary); diff --git a/crates/ironclaw_gateway/static/styles/surfaces/extensions.css b/crates/ironclaw_gateway/static/styles/surfaces/extensions.css index f1cda5df6d1..1eb1aa1e4bc 100644 --- a/crates/ironclaw_gateway/static/styles/surfaces/extensions.css +++ b/crates/ironclaw_gateway/static/styles/surfaces/extensions.css @@ -191,15 +191,18 @@ /* WASM channel setup stepper */ .ext-stepper { display: flex; - align-items: center; + align-items: flex-start; gap: 0; margin: 8px 0 4px; + min-width: 0; } .stepper-step { display: flex; align-items: center; - gap: 4px; + gap: 6px; + min-width: 0; + flex: 1 1 0; } .stepper-circle { @@ -216,7 +219,10 @@ .stepper-label { font-size: var(--text-xs); - white-space: nowrap; + white-space: normal; + overflow-wrap: anywhere; + line-height: 1.25; + min-width: 0; } .stepper-step.completed .stepper-circle { @@ -269,11 +275,11 @@ } .stepper-connector { - width: 20px; height: 2px; background: var(--border); margin: 0 4px; - flex-shrink: 0; + flex: 0 0 20px; + align-self: center; } .stepper-connector.completed { @@ -546,6 +552,77 @@ line-height: 1.5; } +.configure-verification { + display: flex; + flex-direction: column; + gap: 10px; + margin: 16px 0 0 0; + padding: 12px; + border-radius: 8px; + background: var(--bg-secondary); + border: 1px solid var(--border); +} + +.configure-qr-login { + display: flex; + flex-direction: column; + gap: 12px; + margin: 16px 0 0 0; + padding: 12px; + border-radius: 8px; + background: var(--bg-secondary); + border: 1px solid var(--border); +} + +.configure-verification-title { + font-size: var(--text-sm); + font-weight: 600; + color: var(--text-primary); +} + +.configure-verification-instructions { + font-size: var(--text-sm); + line-height: 1.5; + color: var(--text-secondary); +} + +.configure-verification-code { + display: inline-block; + width: fit-content; + padding: 6px 10px; + border-radius: 6px; + background: rgba(255, 255, 255, 0.06); + border: 1px solid var(--border); + color: var(--text-primary); + font-size: var(--text-sm); +} + +.configure-verification-link { + display: inline-flex; + align-items: center; + justify-content: center; + width: fit-content; + padding: 10px 14px; + border-radius: 10px; + border: 1px solid var(--accent); + background: var(--accent-subtle); + color: var(--accent, var(--text-link, #4ea3ff)); + font-size: var(--text-sm); + font-weight: 600; + text-decoration: none; + transition: background var(--transition-fast), transform 150ms var(--ease-spring); +} + +.configure-verification-link:hover { + background: var(--badge-sandbox-bg); + transform: translateY(-1px); + text-decoration: none; +} + +.configure-verification-link:active { + transform: scale(0.98); +} + .configure-inline-error { margin: 16px 0 0 0; padding: 10px 12px; @@ -563,6 +640,29 @@ gap: var(--space-4); } +.configure-optional-group { + border-top: 1px solid var(--border); + padding-top: var(--space-3); +} + +.configure-optional-group summary { + cursor: pointer; + color: var(--text-secondary); + font-size: var(--text-sm); + font-weight: 600; + list-style-position: inside; +} + +.configure-optional-group[open] summary { + margin-bottom: var(--space-3); +} + +.configure-optional-fields { + display: flex; + flex-direction: column; + gap: var(--space-4); +} + .configure-field label { display: block; font-size: var(--text-sm); @@ -592,6 +692,11 @@ border-color: var(--accent); } +.configure-input-row input.configure-input-invalid { + border-color: var(--danger); + box-shadow: 0 0 0 3px rgba(220, 38, 38, 0.18); +} + .field-optional { color: var(--text-secondary); font-style: italic; diff --git a/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs b/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs new file mode 100644 index 00000000000..8d8c28e678a --- /dev/null +++ b/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs @@ -0,0 +1,735 @@ +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use chrono::Utc; +use ironclaw_approvals::LeaseApproval; +use ironclaw_authorization::{ + CapabilityLeaseStatus, CapabilityLeaseStore, GrantAuthorizer, InMemoryCapabilityLeaseStore, + TrustAwareCapabilityDispatchAuthorizer, +}; +use ironclaw_events::{ + DurableEventLog, EventStreamKey, InMemoryDurableEventLog, InMemoryEventSink, ReadScope, + RuntimeEventKind, +}; +use ironclaw_extensions::{ExtensionManifest, ExtensionPackage, ExtensionRegistry}; +use ironclaw_filesystem::LocalFilesystem; +use ironclaw_host_api::*; +use ironclaw_host_runtime::{ + CapabilitySurfaceVersion, HostHttpEgressService, HostRuntime, HostRuntimeServices, + NetworkObligationPolicyStore, RuntimeCapabilityOutcome, RuntimeCapabilityRequest, + RuntimeCapabilityResumeRequest, RuntimeFailureKind, RuntimeSecretInjectionStore, + RuntimeStatusRequest, SurfaceKind, +}; +use ironclaw_network::{ + NetworkHttpEgress, NetworkHttpError, NetworkHttpRequest, NetworkHttpResponse, NetworkUsage, +}; +use ironclaw_processes::{InMemoryProcessResultStore, InMemoryProcessStore, ProcessServices}; +use ironclaw_resources::{InMemoryResourceGovernor, ResourceAccount, ResourceTally}; +use ironclaw_run_state::{ + InMemoryApprovalRequestStore, InMemoryRunStateStore, RunStateStore, RunStatus, +}; +use ironclaw_scripts::{ + ScriptBackend, ScriptBackendOutput, ScriptBackendRequest, ScriptRuntime, ScriptRuntimeConfig, +}; +use ironclaw_secrets::{InMemorySecretStore, SecretMaterial}; +use ironclaw_trust::{ + AdminConfig, AdminEntry, AuthorityCeiling, EffectiveTrustClass, HostTrustAssignment, + HostTrustPolicy, TrustDecision, TrustProvenance, +}; +use serde_json::json; + +#[tokio::test] +async fn reborn_e2e_gate_invokes_script_through_host_runtime_with_status_events_and_resources() { + let governor = Arc::new(InMemoryResourceGovernor::new()); + let run_state = Arc::new(InMemoryRunStateStore::new()); + let event_log = Arc::new(InMemoryDurableEventLog::new()); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::clone(&governor), + Arc::new(GrantAuthorizer::new()), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_durable_event_log(Arc::clone(&event_log)); + let runtime = services.host_runtime(); + let context = execution_context_with_dispatch_grant(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + + let surface = runtime + .visible_capabilities(ironclaw_host_runtime::VisibleCapabilityRequest::new( + scope.clone(), + context.correlation_id, + SurfaceKind::new("gateway-smoke").unwrap(), + )) + .await + .unwrap(); + assert_eq!(surface.version.as_str(), "surface-v1"); + assert_eq!(surface.descriptors.len(), 1); + assert_eq!(surface.descriptors[0].id, script_capability_id()); + + let health = runtime.health().await.unwrap(); + assert!(health.ready); + assert!(health.missing_runtime_backends.is_empty()); + + let status_before = runtime + .runtime_status(RuntimeStatusRequest::new( + scope.clone(), + CorrelationId::new(), + )) + .await + .unwrap(); + assert!(status_before.active_work.is_empty()); + + let input = json!({ + "message": "reborn e2e happy path", + "secret_sentinel": "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK", + "host_path_sentinel": "/private/tmp/reborn-e2e-gate" + }); + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate { + output_bytes: Some(4096), + ..ResourceEstimate::default() + }, + input.clone(), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + + match outcome { + RuntimeCapabilityOutcome::Completed(completed) => { + assert_eq!(completed.capability_id, script_capability_id()); + assert_eq!(completed.output, input); + assert!(completed.usage.output_bytes > 0); + } + other => panic!("expected completed outcome, got {other:?}"), + } + + let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap(); + assert_eq!(run.status, RunStatus::Completed); + let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone()); + assert_eq!( + governor.reserved_for(&tenant_account), + ResourceTally::default() + ); + assert!(governor.usage_for(&tenant_account).output_bytes > 0); + + let status_after = runtime + .runtime_status(RuntimeStatusRequest::new( + scope.clone(), + CorrelationId::new(), + )) + .await + .unwrap(); + assert!(status_after.active_work.is_empty()); + + let replay = event_log + .read_after_cursor( + &EventStreamKey::from_scope(&scope), + &ReadScope::any(), + None, + 10, + ) + .await + .unwrap(); + let kinds = replay + .entries + .iter() + .map(|entry| entry.record.kind) + .collect::>(); + assert_eq!( + kinds, + vec![ + RuntimeEventKind::DispatchRequested, + RuntimeEventKind::RuntimeSelected, + RuntimeEventKind::DispatchSucceeded, + ] + ); + let serialized = serde_json::to_string(&replay).unwrap(); + for forbidden in [ + "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK", + "/private/tmp/reborn-e2e-gate", + ] { + assert!( + !serialized.contains(forbidden), + "durable Reborn E2E event replay leaked {forbidden}: {serialized}" + ); + } +} + +#[tokio::test] +async fn reborn_e2e_gate_blocks_for_approval_resumes_once_and_rejects_replay() { + let fixture = approval_resume_fixture(); + let runtime = fixture.services.host_runtime(); + let context = execution_context_without_grants(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + let input = json!({"message": "approval resume through Reborn E2E gate"}); + + let gate = block_for_approval(&runtime, context.clone(), input.clone()).await; + let blocked_run = fixture + .run_state + .get(&scope, invocation_id) + .await + .unwrap() + .unwrap(); + assert_eq!(blocked_run.status, RunStatus::BlockedApproval); + assert_eq!( + blocked_run.approval_request_id, + Some(gate.approval_request_id) + ); + + let lease = + approve_dispatch_for_services(&fixture.services, &scope, gate.approval_request_id).await; + + let resumed = runtime + .resume_capability(RuntimeCapabilityResumeRequest::new( + context.clone(), + gate.approval_request_id, + script_capability_id(), + ResourceEstimate::default(), + input.clone(), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + match resumed { + RuntimeCapabilityOutcome::Completed(completed) => { + assert_eq!(completed.capability_id, script_capability_id()); + assert_eq!(completed.output, input); + } + other => panic!("expected completed approval resume, got {other:?}"), + } + assert_eq!( + fixture + .capability_leases + .get(&scope, lease.grant.id) + .await + .unwrap() + .status, + CapabilityLeaseStatus::Consumed + ); + assert_event_kinds( + &fixture.events, + &[ + RuntimeEventKind::DispatchRequested, + RuntimeEventKind::RuntimeSelected, + RuntimeEventKind::DispatchSucceeded, + ], + ); + + let replay = runtime + .resume_capability(RuntimeCapabilityResumeRequest::new( + context, + gate.approval_request_id, + script_capability_id(), + ResourceEstimate::default(), + json!({"message": "approval resume through Reborn E2E gate"}), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + assert_failed_outcome(replay, RuntimeFailureKind::Authorization); + assert_eq!( + fixture.events.events().len(), + 3, + "replayed approval resume must fail before a second runtime dispatch" + ); +} + +#[tokio::test] +async fn reborn_e2e_gate_fails_unsupported_obligations_before_runtime_events_or_success() { + let run_state = Arc::new(InMemoryRunStateStore::new()); + let events = InMemoryEventSink::new(); + let governor = Arc::new(InMemoryResourceGovernor::new()); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::clone(&governor), + Arc::new(ObligatingAuthorizer), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_event_sink(Arc::new(events.clone())); + let runtime = services.host_runtime(); + let context = execution_context_with_dispatch_grant(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate::default(), + json!({"message": "unsupported obligation"}), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + + assert_failed_outcome(outcome, RuntimeFailureKind::Backend); + assert!(events.events().is_empty()); + let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap(); + assert_eq!(run.status, RunStatus::Failed); + assert_eq!(run.error_kind.as_deref(), Some("ObligationFailed")); + let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone()); + assert_eq!( + governor.reserved_for(&tenant_account), + ResourceTally::default() + ); + assert_eq!( + governor.usage_for(&tenant_account), + ResourceTally::default() + ); +} + +#[test] +fn reborn_e2e_gate_host_http_consumes_staged_policy_and_secret_once() { + let network = RecordingNetwork::ok(NetworkHttpResponse { + status: 200, + headers: vec![], + body: br#"{"ok":true}"#.to_vec(), + usage: NetworkUsage { + request_bytes: 5, + response_bytes: 11, + resolved_ip: None, + }, + }); + let network_recorder = network.requests.clone(); + let policy_store = Arc::new(NetworkObligationPolicyStore::new()); + let secret_injections = Arc::new(RuntimeSecretInjectionStore::new()); + let scope = sample_scope(InvocationId::new()); + let capability_id = script_capability_id(); + let handle = SecretHandle::new("api-token").unwrap(); + let staged_policy = sample_policy(); + policy_store.insert(&scope, &capability_id, staged_policy.clone()); + secret_injections + .insert( + &scope, + &capability_id, + &handle, + SecretMaterial::from("sk-reborn-e2e-staged-secret"), + ) + .unwrap(); + let service = HostHttpEgressService::new(network, InMemorySecretStore::new()) + .with_network_policy_store(Arc::clone(&policy_store)) + .with_secret_injection_store(Arc::clone(&secret_injections)); + + let request = RuntimeHttpEgressRequest { + runtime: RuntimeKind::Script, + scope: scope.clone(), + capability_id: capability_id.clone(), + method: NetworkMethod::Post, + url: "https://api.example.test/v1/run".to_string(), + headers: vec![], + body: b"hello".to_vec(), + network_policy: caller_supplied_policy(), + credential_injections: vec![RuntimeCredentialInjection { + handle: handle.clone(), + source: RuntimeCredentialSource::StagedObligation { + capability_id: capability_id.clone(), + }, + target: RuntimeCredentialTarget::Header { + name: "authorization".to_string(), + prefix: Some("Bearer ".to_string()), + }, + required: true, + }], + response_body_limit: Some(4096), + timeout_ms: None, + }; + + let response = service + .execute(request.clone()) + .expect("host HTTP egress should use staged Reborn policy and secret material"); + assert_eq!(response.status, 200); + let recorded = network_recorder.lock().unwrap(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].policy, staged_policy); + assert_eq!( + recorded[0] + .headers + .iter() + .find(|(name, _)| name == "authorization"), + Some(&( + "authorization".to_string(), + "Bearer sk-reborn-e2e-staged-secret".to_string() + )) + ); + drop(recorded); + assert!( + secret_injections + .take(&scope, &capability_id, &handle) + .unwrap() + .is_none(), + "staged secret material must be consumed exactly once" + ); + assert_eq!( + policy_store.get(&scope, &capability_id), + Some(staged_policy), + "host egress must leave staged network policy for invocation/process lifecycle cleanup" + ); + + let replay = service + .execute(request) + .expect_err("consumed staged secret must not be reusable"); + assert!(matches!(replay, RuntimeHttpEgressError::Credential { .. })); + assert_eq!( + network_recorder.lock().unwrap().len(), + 1, + "replay must fail before a second outbound transport attempt" + ); +} + +type InMemoryServices = HostRuntimeServices< + LocalFilesystem, + InMemoryResourceGovernor, + InMemoryProcessStore, + InMemoryProcessResultStore, +>; + +struct ApprovalFixture { + services: InMemoryServices, + run_state: Arc, + capability_leases: Arc, + events: InMemoryEventSink, +} + +fn approval_resume_fixture() -> ApprovalFixture { + let run_state = Arc::new(InMemoryRunStateStore::new()); + let approval_requests = Arc::new(InMemoryApprovalRequestStore::new()); + let capability_leases = Arc::new(InMemoryCapabilityLeaseStore::new()); + let events = InMemoryEventSink::new(); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::new(InMemoryResourceGovernor::new()), + Arc::new(ApprovalThenGrantAuthorizer), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_approval_requests(approval_requests) + .with_capability_leases(Arc::clone(&capability_leases)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_event_sink(Arc::new(events.clone())); + + ApprovalFixture { + services, + run_state, + capability_leases, + events, + } +} + +async fn block_for_approval( + runtime: &impl HostRuntime, + context: ExecutionContext, + input: serde_json::Value, +) -> ironclaw_host_runtime::RuntimeApprovalGate { + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate::default(), + input, + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + match outcome { + RuntimeCapabilityOutcome::ApprovalRequired(gate) => gate, + other => panic!("expected approval gate, got {other:?}"), + } +} + +async fn approve_dispatch_for_services( + services: &InMemoryServices, + scope: &ResourceScope, + approval_request_id: ApprovalRequestId, +) -> ironclaw_authorization::CapabilityLease { + services + .approval_resolver() + .expect("approval resolver should be configured") + .approve_dispatch( + scope, + approval_request_id, + LeaseApproval { + issued_by: Principal::HostRuntime, + allowed_effects: vec![EffectKind::DispatchCapability], + mounts: MountView::default(), + network: NetworkPolicy::default(), + secrets: Vec::new(), + resource_ceiling: None, + expires_at: None, + max_invocations: Some(1), + }, + ) + .await + .unwrap() +} + +struct ApprovalThenGrantAuthorizer; + +#[async_trait] +impl TrustAwareCapabilityDispatchAuthorizer for ApprovalThenGrantAuthorizer { + async fn authorize_dispatch_with_trust( + &self, + context: &ExecutionContext, + descriptor: &CapabilityDescriptor, + estimate: &ResourceEstimate, + trust_decision: &TrustDecision, + ) -> Decision { + if context.grants.grants.is_empty() { + Decision::RequireApproval { + request: ApprovalRequest { + id: ApprovalRequestId::new(), + correlation_id: context.correlation_id, + requested_by: Principal::Extension(context.extension_id.clone()), + action: Box::new(Action::Dispatch { + capability: descriptor.id.clone(), + estimated_resources: estimate.clone(), + }), + invocation_fingerprint: None, + reason: "approval required".to_string(), + reusable_scope: None, + }, + } + } else { + GrantAuthorizer::new() + .authorize_dispatch_with_trust(context, descriptor, estimate, trust_decision) + .await + } + } +} + +struct ObligatingAuthorizer; + +#[async_trait] +impl TrustAwareCapabilityDispatchAuthorizer for ObligatingAuthorizer { + async fn authorize_dispatch_with_trust( + &self, + _context: &ExecutionContext, + _descriptor: &CapabilityDescriptor, + _estimate: &ResourceEstimate, + _trust_decision: &TrustDecision, + ) -> Decision { + Decision::Allow { + obligations: Obligations::new(vec![Obligation::AuditBefore]).unwrap(), + } + } +} + +struct EchoScriptBackend; + +impl ScriptBackend for EchoScriptBackend { + fn execute(&self, request: ScriptBackendRequest) -> Result { + let value = serde_json::from_str(&request.stdin_json).map_err(|error| error.to_string())?; + Ok(ScriptBackendOutput::json(value)) + } +} + +#[derive(Clone)] +struct RecordingNetwork { + response: Result, + requests: Arc>>, +} + +impl RecordingNetwork { + fn ok(response: NetworkHttpResponse) -> Self { + Self { + response: Ok(response), + requests: Arc::new(Mutex::new(Vec::new())), + } + } +} + +impl NetworkHttpEgress for RecordingNetwork { + fn execute( + &self, + request: NetworkHttpRequest, + ) -> Result { + self.requests.lock().unwrap().push(request); + self.response.clone() + } +} + +fn registry_with_manifest(manifest: &str) -> ExtensionRegistry { + let mut registry = ExtensionRegistry::new(); + let manifest = ExtensionManifest::parse(manifest).unwrap(); + let package = ExtensionPackage::from_manifest( + manifest, + VirtualPath::new("/system/extensions/script").unwrap(), + ) + .unwrap(); + registry.insert(package).unwrap(); + registry +} + +fn execution_context_with_dispatch_grant() -> ExecutionContext { + let mut grants = CapabilitySet::default(); + grants.grants.push(dispatch_grant()); + ExecutionContext::local_default( + UserId::new("user").unwrap(), + ExtensionId::new("caller").unwrap(), + RuntimeKind::Script, + TrustClass::UserTrusted, + grants, + MountView::default(), + ) + .unwrap() +} + +fn execution_context_without_grants() -> ExecutionContext { + ExecutionContext::local_default( + UserId::new("user").unwrap(), + ExtensionId::new("caller").unwrap(), + RuntimeKind::Script, + TrustClass::UserTrusted, + CapabilitySet::default(), + MountView::default(), + ) + .unwrap() +} + +fn dispatch_grant() -> CapabilityGrant { + CapabilityGrant { + id: CapabilityGrantId::new(), + capability: script_capability_id(), + grantee: Principal::Extension(ExtensionId::new("caller").unwrap()), + issued_by: Principal::HostRuntime, + constraints: GrantConstraints { + allowed_effects: vec![EffectKind::DispatchCapability], + mounts: MountView::default(), + network: NetworkPolicy::default(), + secrets: Vec::new(), + resource_ceiling: None, + expires_at: None, + max_invocations: None, + }, + } +} + +fn local_manifest_trust_policy() -> HostTrustPolicy { + HostTrustPolicy::new(vec![Box::new(AdminConfig::with_entries(vec![ + AdminEntry::for_local_manifest( + PackageId::new("script").unwrap(), + "/system/extensions/script/manifest.toml".to_string(), + None, + HostTrustAssignment::user_trusted(), + vec![EffectKind::DispatchCapability], + None, + ), + ]))]) + .unwrap() +} + +fn trust_decision_with_dispatch_authority() -> TrustDecision { + TrustDecision { + effective_trust: EffectiveTrustClass::user_trusted(), + authority_ceiling: AuthorityCeiling { + allowed_effects: vec![EffectKind::DispatchCapability], + max_resource_ceiling: None, + }, + provenance: TrustProvenance::Default, + evaluated_at: Utc::now(), + } +} + +fn sample_scope(invocation_id: InvocationId) -> ResourceScope { + ResourceScope { + tenant_id: TenantId::new("tenant1").unwrap(), + user_id: UserId::new("user1").unwrap(), + agent_id: None, + project_id: None, + mission_id: None, + thread_id: None, + invocation_id, + } +} + +fn sample_policy() -> NetworkPolicy { + NetworkPolicy { + allowed_targets: vec![NetworkTargetPattern { + scheme: Some(NetworkScheme::Https), + host_pattern: "api.example.test".to_string(), + port: None, + }], + deny_private_ip_ranges: true, + max_egress_bytes: Some(4096), + } +} + +fn caller_supplied_policy() -> NetworkPolicy { + NetworkPolicy { + allowed_targets: vec![NetworkTargetPattern { + scheme: Some(NetworkScheme::Https), + host_pattern: "caller.example.test".to_string(), + port: None, + }], + deny_private_ip_ranges: false, + max_egress_bytes: Some(1), + } +} + +fn script_capability_id() -> CapabilityId { + CapabilityId::new("script.echo").unwrap() +} + +fn assert_event_kinds(events: &InMemoryEventSink, expected: &[RuntimeEventKind]) { + let actual = events + .events() + .into_iter() + .map(|event| event.kind) + .collect::>(); + assert_eq!(actual, expected); +} + +fn assert_failed_outcome(outcome: RuntimeCapabilityOutcome, expected: RuntimeFailureKind) { + match outcome { + RuntimeCapabilityOutcome::Failed(failure) => assert_eq!(failure.kind, expected), + other => panic!("expected failed outcome {expected:?}, got {other:?}"), + } +} + +const SCRIPT_MANIFEST: &str = r#" +id = "script" +name = "Script Echo" +version = "0.1.0" +description = "Script echo test extension" +trust = "third_party" + +[runtime] +kind = "script" +runner = "sandboxed_process" +command = "echo-script" +args = [] + +[[capabilities]] +id = "script.echo" +description = "Echo text through script runtime" +effects = ["dispatch_capability"] +default_permission = "allow" +parameters_schema = { type = "object" } +"#; diff --git a/src/llm/CLAUDE.md b/crates/ironclaw_llm/CLAUDE.md similarity index 85% rename from src/llm/CLAUDE.md rename to crates/ironclaw_llm/CLAUDE.md index 9c48d9bf61b..bb2a9c6f681 100644 --- a/src/llm/CLAUDE.md +++ b/crates/ironclaw_llm/CLAUDE.md @@ -6,7 +6,7 @@ Multi-provider LLM integration with circuit breaker, retry, failover, and respon | File | Role | |------|------| -| `mod.rs` | Provider factory (`create_llm_provider`, `build_provider_chain`); `LlmBackend` enum | +| `lib.rs` | Provider factory (`create_llm_provider`, `build_provider_chain`); registry-protocol dispatch | | `config.rs` | LLM config types (`LlmConfig`, `RegistryProviderConfig`, `NearAiConfig`, `BedrockConfig`) | | `error.rs` | `LlmError` enum used by all providers | | `provider.rs` | `LlmProvider` trait, `ChatMessage`, `ToolCall`, `CompletionRequest`, `sanitize_tool_messages` | @@ -27,6 +27,20 @@ Multi-provider LLM integration with circuit breaker, retry, failover, and respon | `smart_routing.rs` | `SmartRoutingProvider` — 13-dimension complexity scorer routes cheap vs primary model | | `recording.rs` | `RecordingLlm` — trace capture for E2E replay testing (`IRONCLAW_RECORD_TRACE`) | | `bedrock.rs` | AWS Bedrock provider via native Converse API (feature-gated: `--features bedrock`) | +| `anthropic_oauth.rs` | Anthropic OAuth provider (Claude.ai subscription / OAuth tokens, fallback when no API key) | +| `gemini_oauth.rs` | Gemini OAuth provider (Cloud OAuth credentials → `generativelanguage.googleapis.com`) | +| `github_copilot.rs` | GitHub Copilot Chat provider (uses dedicated reqwest client, not `RigAdapter`) | +| `github_copilot_auth.rs` | Copilot session-token exchange and refresh (`CopilotTokenManager`) | +| `host.rs` | Host-side trait surface: `SessionDb`, `SessionSecrets`, `SessionRenewer`, `SessionKeyPersistor` (binary supplies adapters in `src/llm_host.rs`) | +| `runtime.rs` | `SwappableLlmProvider` + `LlmReloadHandle` for hot-reloading the provider chain on settings change | +| `registry.rs` | Provider registry (`ProviderDefinition`, `ProviderProtocol`); resolves backend strings to clients | +| `tool_schema.rs` | Tool schema normalization policies (`FlattenOnly` for NearAI, strict OpenAI for `RigAdapter` / Codex) | +| `transcription/{mod,openai,chat_completions}.rs` | Audio transcription pipeline (Whisper / chat-completions back-ends) | +| `image_models.rs` | Image-generation model metadata table | +| `vision_models.rs` | Vision-capable model registry for attachment routing | +| `reasoning_models.rs` | Reasoning-capable model registry (Codex, R1, o-series, etc.) used for thinking-mode dispatch | +| `models.rs` | Top-level model-name catalog and helpers | +| `testing/` | `StubLlm`, `StubErrorKind`, `fault_injection` — gated behind the `testing` cargo feature for downstream test harnesses | ## Provider Selection @@ -161,10 +175,24 @@ Key notes: - `complete_with_tools()` is never cached (tool calls can have side effects) — `CachedProvider` always passes them through. To add a new provider: -1. Create `src/llm/myprovider.rs` implementing `LlmProvider` -2. Add variant to `LlmBackend` in `mod.rs` -3. Wire into the factory match in `mod.rs` -4. Add env vars to `config/llm.rs` and `.env.example` +1. Create `crates/ironclaw_llm/src/myprovider.rs` implementing `LlmProvider` +2. Add a `ProviderProtocol` variant in `registry.rs` (or wire a backend-string match in `lib.rs` for non-registry providers like `nearai`/`bedrock`/`openai_codex`) +3. Wire into the factory dispatch in `lib.rs` (`create_registry_provider` for registry-backed protocols, top-level `create_llm_provider` for backend-string-keyed providers) +4. Add env vars to `src/config/llm.rs` (main crate) and `.env.example` +5. If the provider needs persistent state (session tokens, refresh tokens, etc.), use the host traits in `host.rs` — never reach for `crate::db`, `crate::secrets`, or `crate::bootstrap`. The crate must stay independent of the binary; the binary supplies adapter impls in `src/llm_host.rs`. + +## Host Trait Surface + +`host.rs` defines four traits that decouple `ironclaw_llm` from the binary: + +| Trait | Purpose | Binary adapter | +|-------|---------|----------------| +| `SessionDb` | JSON settings persistence | `DatabaseSessionDb` in `src/llm_host.rs` | +| `SessionSecrets` | Encrypted secrets store | `SecretsStoreSessionSecrets` in `src/llm_host.rs` | +| `SessionRenewer` | Interactive NEAR-AI re-auth flow | CLI/wizard impl wired in `src/setup/` | +| `SessionKeyPersistor` | Runtime env overlay + `.env` upsert | `BootstrapKeyPersistor` in `src/llm_host.rs` | + +`NoopSessionRenewer` and `NoopKeyPersistor` are provided for headless / hosted contexts (return errors / no-ops). The binary plugs concrete impls into `SessionManager` at startup. ## Response Cache diff --git a/crates/ironclaw_llm/Cargo.toml b/crates/ironclaw_llm/Cargo.toml new file mode 100644 index 00000000000..c0f665d1be3 --- /dev/null +++ b/crates/ironclaw_llm/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "ironclaw_llm" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +description = "Multi-provider LLM integration with retry, failover, circuit breaker, and response caching" +authors = ["NEAR AI "] +license = "MIT OR Apache-2.0" +homepage = "https://github.com/nearai/ironclaw" +repository = "https://github.com/nearai/ironclaw" +publish = false + +[package.metadata.dist] +dist = false + +[features] +default = [] +bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"] +# Opt-in StubLlm + credentials helpers for downstream test code. +testing = [] + +[dependencies] +ironclaw_common = { path = "../ironclaw_common", version = "0.4.2" } +ironclaw_safety = { path = "../ironclaw_safety", version = "0.2.2" } + +async-trait = "0.1" +anyhow = "1" +base64 = "0.22.1" +bytes = "1" +chrono = { version = "0.4", features = ["serde"] } +dirs = "6" +eventsource-stream = "0.2" +futures = "0.3" +open = "5" +rand = "0.8" +urlencoding = "2" +regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] } +rig-core = { version = "0.30", default-features = false, features = ["reqwest-rustls"] } +rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] } +rust_decimal_macros = "1" +secrecy = { version = "0.10", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" +tokio = { version = "1", features = ["sync", "io-util", "io-std", "time", "macros", "rt", "fs", "process", "net"] } +tracing = "0.1" +url = "2" +uuid = { version = "1", features = ["v4", "serde"] } + +aws-config = { version = "1", features = ["behavior-version-latest"], optional = true } +aws-sdk-bedrockruntime = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio", "behavior-version-latest"], optional = true } +aws-smithy-types = { version = "1", optional = true } + +[dev-dependencies] +insta = { version = "1.46.3", features = ["yaml"] } +tempfile = "3" +tokio = { version = "1", features = ["full", "test-util"] } +tracing-test = "0.2" diff --git a/src/llm/anthropic_oauth.rs b/crates/ironclaw_llm/src/anthropic_oauth.rs similarity index 91% rename from src/llm/anthropic_oauth.rs rename to crates/ironclaw_llm/src/anthropic_oauth.rs index 97748602e56..ff1a5c02903 100644 --- a/src/llm/anthropic_oauth.rs +++ b/crates/ironclaw_llm/src/anthropic_oauth.rs @@ -14,14 +14,58 @@ use rust_decimal::Decimal; use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; -use crate::llm::config::RegistryProviderConfig; -use crate::llm::costs; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::config::RegistryProviderConfig; +use crate::costs; +use crate::error::LlmError; +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, strip_unsupported_completion_params, strip_unsupported_tool_params, }; + +/// Read a fresh `claude login` OAuth token from the OS credential store. +/// +/// Mirrors `ironclaw::config::ClaudeCodeConfig::extract_oauth_token` but is +/// inlined here so this crate doesn't depend on the main binary. Used to +/// retry once after a 401 if the user's OAuth token has been rotated by +/// Claude Code's background refresh. +fn refresh_claude_oauth_token() -> Option { + if cfg!(target_os = "macos") { + match std::process::Command::new("security") + .args([ + "find-generic-password", + "-s", + "Claude Code-credentials", + "-w", + ]) + .output() + { + Ok(output) if output.status.success() => { + if let Ok(json) = String::from_utf8(output.stdout) { + return parse_oauth_access_token(json.trim()); + } + } + _ => {} + } + } + if let Some(home) = dirs::home_dir() { + let creds_path = home.join(".claude").join(".credentials.json"); + if let Ok(json) = std::fs::read_to_string(&creds_path) { + return parse_oauth_access_token(&json); + } + } + None +} + +fn parse_oauth_access_token(json: &str) -> Option { + let creds: serde_json::Value = serde_json::from_str(json).ok()?; + let token = creds["claudeAiOauth"]["accessToken"].as_str()?; + if !token.starts_with("sk-ant-oat") { + tracing::debug!("Ignoring credential store token with unexpected prefix"); + return None; + } + Some(token.to_string()) +} const ANTHROPIC_API_URL: &str = "https://api.anthropic.com/v1/messages"; /// OAuth beta requires 2023-06-01; the 2024-10-22 version is not valid with the beta flag. const ANTHROPIC_API_VERSION: &str = "2023-06-01"; @@ -142,7 +186,7 @@ impl AnthropicOAuthProvider { if !status.is_success() { // Parse Retry-After header before consuming the body. - let retry_after = Some(crate::llm::retry::parse_retry_after( + let retry_after = Some(crate::retry::parse_retry_after( response.headers().get("retry-after"), )); @@ -160,7 +204,7 @@ impl AnthropicOAuthProvider { // Keychain refresh write (fixes race in #1136). tokio::time::sleep(std::time::Duration::from_millis(500)).await; - if let Some(fresh) = crate::config::ClaudeCodeConfig::extract_oauth_token() { + if let Some(fresh) = refresh_claude_oauth_token() { let fresh_token = SecretString::from(fresh); // Retry once with the refreshed token let retry = self @@ -188,7 +232,7 @@ impl AnthropicOAuthProvider { reason: format!("Failed to read response body: {}", e), })?; return serde_json::from_str(&text).map_err(|e| { - let truncated = crate::agent::truncate_for_preview(&text, 512); + let truncated = ironclaw_common::truncate_for_preview(&text, 512); LlmError::InvalidResponse { provider: "anthropic_oauth".to_string(), reason: format!("JSON parse error: {}. Raw: {}", e, truncated), @@ -210,7 +254,7 @@ impl AnthropicOAuthProvider { retry_after, }); } - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); return Err(LlmError::RequestFailed { provider: "anthropic_oauth".to_string(), reason: format!("HTTP {}: {}", status, truncated), @@ -229,7 +273,7 @@ impl AnthropicOAuthProvider { ); serde_json::from_str(&response_text).map_err(|e| { - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); LlmError::InvalidResponse { provider: "anthropic_oauth".to_string(), reason: format!("JSON parse error: {}. Raw: {}", e, truncated), @@ -350,6 +394,7 @@ impl LlmProvider for AnthropicOAuthProvider { input_tokens: response.usage.input_tokens, output_tokens: response.usage.output_tokens, cache_creation_input_tokens: response.usage.cache_creation_input_tokens, + reasoning: None, cache_read_input_tokens: response.usage.cache_read_input_tokens, }) } @@ -580,6 +625,7 @@ fn extract_response_content(response: &AnthropicResponse) -> (Option, Ve name: name.clone(), arguments: input.clone(), reasoning: None, + signature: None, }); } } @@ -629,6 +675,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"q": "test"}), reasoning: None, + signature: None, }]; let messages = vec![ ChatMessage::user("Search for test"), diff --git a/crates/ironclaw_llm/src/auth.rs b/crates/ironclaw_llm/src/auth.rs new file mode 100644 index 00000000000..febd16989de --- /dev/null +++ b/crates/ironclaw_llm/src/auth.rs @@ -0,0 +1,335 @@ +//! Public auth facade for `ironclaw_llm`. +//! +//! External callers (the setup wizard, the `ironclaw login` subcommand, and +//! the LLM config loader) interact with provider authentication through this +//! module only. The per-provider implementations (`github_copilot_auth`, +//! `gemini_oauth`, `openai_codex_session`, `codex_auth`) are crate-private — +//! callers must not import them directly. +//! +//! Verbs exposed: +//! - [`start_login`]: run an interactive login flow (device code, OAuth refresh, +//! etc.) for a [`LoginRequest`]. +//! - [`validate_token`]: confirm a manually-supplied token works against the +//! given backend (used by the wizard's paste-token branches). +//! - [`default_headers`]: provider-specific request headers (e.g. GitHub +//! Copilot's editor-identity headers) that the LLM config loader merges +//! into outbound requests. +//! - [`load_persisted_credentials`] / [`default_credentials_path`]: read +//! credentials another CLI tool (e.g. Codex CLI) has already persisted. + +use std::path::{Path, PathBuf}; + +use secrecy::SecretString; + +use crate::codex_auth; +use crate::config::OpenAiCodexConfig; +use crate::github_copilot_auth; +use crate::openai_codex_session::OpenAiCodexSessionManager; + +/// Identifies a backend for non-interactive auth queries +/// ([`validate_token`], [`default_headers`]). Mirrors [`LoginRequest`] +/// without the per-backend payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuthBackend { + GithubCopilot, + Gemini, + OpenAiCodex, +} + +/// Identifies a CLI-style credential file that another tool maintains and +/// `ironclaw_llm` can read directly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CredentialSource { + /// Codex CLI's `auth.json`. Loaded when the LLM config sees + /// `LLM_USE_CODEX_AUTH=true`. + CodexCli, +} + +/// Caller-supplied UX hooks for interactive login flows. +/// +/// Keeps display/browser-launch concerns out of `ironclaw_llm`: the wizard, +/// the CLI subcommand, and any future TUI/web setup all implement this trait +/// against their own UI layer. +pub trait AuthPrompt: Send + Sync { + /// Show a device-code verification URL plus the one-time code the user + /// must enter. The implementation is also responsible for opening the + /// browser if the host environment supports it. + fn show_device_code(&self, verification_uri: &str, user_code: &str); +} + +/// Per-backend payload for [`start_login`]. +/// +/// Each variant carries only the inputs the backend actually needs at login +/// time (file paths, endpoint overrides). Constants like client IDs live +/// inside the LLM crate. +#[derive(Debug, Clone)] +pub enum LoginRequest { + /// GitHub Copilot device-code login. The returned [`AuthOutcome`] + /// includes the OAuth token in `token_to_persist` so the caller can + /// store it in their secrets store. + GithubCopilot, + + /// Gemini Cloud Code OAuth: refresh existing credentials at the given + /// path or fail. Cloud Code project_id is reported in `display`. + Gemini { credentials_path: PathBuf }, + + /// OpenAI Codex (ChatGPT subscription) device-code login. Tokens are + /// persisted to disk by `ironclaw_llm`; the caller does not see them. + OpenAiCodex(OpenAiCodexLoginOptions), +} + +/// OpenAI Codex login options, mirroring the `OPENAI_CODEX_*` env vars. +/// +/// All fields are optional — `None` means "use the built-in default". +/// Construct from env via [`OpenAiCodexLoginOptions::from_env`]. +#[derive(Debug, Clone, Default)] +pub struct OpenAiCodexLoginOptions { + pub auth_endpoint: Option, + pub api_base_url: Option, + pub client_id: Option, + pub session_path: Option, +} + +impl OpenAiCodexLoginOptions { + /// Build options by reading `OPENAI_CODEX_AUTH_URL`, + /// `OPENAI_CODEX_API_URL`, `OPENAI_CODEX_CLIENT_ID`, and + /// `OPENAI_CODEX_SESSION_PATH` from the environment. + pub fn from_env() -> Self { + Self { + auth_endpoint: std::env::var("OPENAI_CODEX_AUTH_URL").ok(), + api_base_url: std::env::var("OPENAI_CODEX_API_URL").ok(), + client_id: std::env::var("OPENAI_CODEX_CLIENT_ID").ok(), + session_path: std::env::var("OPENAI_CODEX_SESSION_PATH") + .ok() + .map(PathBuf::from), + } + } + + /// Build options from a fully-resolved [`OpenAiCodexConfig`] — the + /// shape produced by the binary's `LlmConfig::resolve` pipeline, + /// which already layered TOML / env / DB precedence. + /// + /// Use this from `ironclaw login --openai-codex` so config-file + /// overrides for endpoints / client id / session path keep working, + /// not just env vars. Each field becomes `Some(_)` so it wins over + /// the built-in default in [`Self::into_codex_config`]. + pub fn from_resolved_config(cfg: &OpenAiCodexConfig) -> Self { + Self { + auth_endpoint: Some(cfg.auth_endpoint.clone()), + api_base_url: Some(cfg.api_base_url.clone()), + client_id: Some(cfg.client_id.clone()), + session_path: Some(cfg.session_path.clone()), + } + } + + fn into_codex_config(self) -> OpenAiCodexConfig { + let mut cfg = OpenAiCodexConfig::default(); + if let Some(v) = self.auth_endpoint { + cfg.auth_endpoint = v; + } + if let Some(v) = self.api_base_url { + cfg.api_base_url = v; + } + if let Some(v) = self.client_id { + cfg.client_id = v; + } + if let Some(v) = self.session_path { + cfg.session_path = v; + } + cfg + } +} + +/// Outcome of a successful interactive login. +/// +/// `token_to_persist` is `Some` when the caller is responsible for storing +/// the credential (e.g. GitHub Copilot — wizard saves to the secrets store). +/// It is `None` when `ironclaw_llm` already persisted credentials on disk +/// (e.g. OpenAI Codex session file). +#[derive(Debug, Default)] +pub struct AuthOutcome { + pub token_to_persist: Option, + /// Display-only (key, value) pairs for the caller's setup UX. For + /// Gemini this includes the Cloud Code project_id when available. + pub display: Vec<(String, String)>, +} + +/// Credentials loaded from a CLI-style auth file (currently Codex CLI's +/// `auth.json`). The caller decides whether to use them as-is or override. +#[derive(Debug)] +pub struct PersistedCredentials { + pub token: SecretString, + pub refresh_token: Option, + /// True when the credential is an OAuth subscription token (e.g. Codex + /// ChatGPT mode); false for raw API keys. Affects routing/base-URL + /// decisions on the caller side. + pub is_subscription: bool, + /// Provider-specific base URL to honour for this credential type. + pub base_url: String, + /// File the credentials were loaded from. Carried forward so the + /// provider can persist refreshed tokens back to the same place. + pub source_path: Option, +} + +/// Errors surfaced by the auth facade. Provider-specific error chains are +/// flattened into a single string for the caller — internal types do not +/// cross the boundary. +#[derive(Debug, thiserror::Error)] +pub enum AuthError { + #[error("{backend}: {reason}")] + LoginFailed { + backend: &'static str, + reason: String, + }, + + #[error("{backend}: token validation failed: {reason}")] + InvalidToken { + backend: &'static str, + reason: String, + }, + + /// The backend does not validate a single bearer token — its + /// credentials are managed end-to-end via [`start_login`] (OAuth + /// device-code, credential file, etc.). Callers asking to validate + /// a plain token against one of these backends should route through + /// `start_login` instead. + #[error("{backend:?}: token validation not supported; credentials are managed via start_login")] + TokenValidationNotSupported { backend: AuthBackend }, + + #[error("{0}")] + Other(String), +} + +impl AuthError { + fn login(backend: &'static str, reason: impl ToString) -> Self { + Self::LoginFailed { + backend, + reason: reason.to_string(), + } + } + + fn invalid(backend: &'static str, reason: impl ToString) -> Self { + Self::InvalidToken { + backend, + reason: reason.to_string(), + } + } +} + +/// Run an interactive login flow. +pub async fn start_login( + request: LoginRequest, + prompt: &dyn AuthPrompt, +) -> Result { + match request { + LoginRequest::GithubCopilot => start_github_copilot_login(prompt).await, + LoginRequest::Gemini { credentials_path } => start_gemini_login(&credentials_path).await, + LoginRequest::OpenAiCodex(opts) => start_openai_codex_login(opts).await, + } +} + +/// Validate a manually-supplied token against the backend's auth endpoint. +pub async fn validate_token(backend: AuthBackend, token: &str) -> Result<(), AuthError> { + match backend { + AuthBackend::GithubCopilot => { + let client = http_client()?; + github_copilot_auth::validate_token(&client, token) + .await + .map_err(|e| AuthError::invalid("github_copilot", e)) + } + AuthBackend::Gemini | AuthBackend::OpenAiCodex => { + Err(AuthError::TokenValidationNotSupported { backend }) + } + } +} + +/// Default request headers a backend wants on every API call. +pub fn default_headers(backend: AuthBackend) -> Vec<(String, String)> { + match backend { + AuthBackend::GithubCopilot => github_copilot_auth::default_headers(), + AuthBackend::Gemini | AuthBackend::OpenAiCodex => Vec::new(), + } +} + +/// Default file path for a credential source. +pub fn default_credentials_path(source: CredentialSource) -> PathBuf { + match source { + CredentialSource::CodexCli => codex_auth::default_codex_auth_path(), + } +} + +/// Load CLI-stored credentials. Returns `None` if the file is missing, +/// unreadable, or contains no usable credentials. +pub fn load_persisted_credentials( + source: CredentialSource, + override_path: Option<&Path>, +) -> Option { + match source { + CredentialSource::CodexCli => { + let path = override_path + .map(Path::to_path_buf) + .unwrap_or_else(codex_auth::default_codex_auth_path); + let creds = codex_auth::load_codex_credentials(&path)?; + Some(PersistedCredentials { + base_url: creds.base_url().to_string(), + token: creds.token, + refresh_token: creds.refresh_token, + is_subscription: creds.is_chatgpt_mode, + source_path: creds.auth_path, + }) + } + } +} + +// ── Per-backend dispatchers ────────────────────────────────────────────── + +fn http_client() -> Result { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(15)) + .build() + .map_err(|e| AuthError::Other(format!("http client build failed: {e}"))) +} + +async fn start_github_copilot_login(prompt: &dyn AuthPrompt) -> Result { + let client = http_client()?; + let device = github_copilot_auth::request_device_code(&client) + .await + .map_err(|e| AuthError::login("github_copilot", e))?; + prompt.show_device_code(&device.verification_uri, &device.user_code); + let token = github_copilot_auth::wait_for_device_login(&client, &device) + .await + .map_err(|e| AuthError::login("github_copilot", e))?; + github_copilot_auth::validate_token(&client, &token) + .await + .map_err(|e| AuthError::invalid("github_copilot", e))?; + Ok(AuthOutcome { + token_to_persist: Some(SecretString::from(token)), + display: Vec::new(), + }) +} + +async fn start_gemini_login(credentials_path: &Path) -> Result { + let manager = crate::gemini_oauth::CredentialManager::new(credentials_path) + .map_err(|e| AuthError::login("gemini", e))?; + let cred = manager + .get_valid_credential() + .await + .map_err(|e| AuthError::login("gemini", e))?; + let mut display = Vec::new(); + if let Some(project_id) = cred.project_id.clone() { + display.push(("project_id".to_string(), project_id)); + } + Ok(AuthOutcome { + token_to_persist: None, + display, + }) +} + +async fn start_openai_codex_login(opts: OpenAiCodexLoginOptions) -> Result { + let mgr = OpenAiCodexSessionManager::new(opts.into_codex_config()) + .map_err(|e| AuthError::login("openai_codex", e))?; + mgr.device_code_login() + .await + .map_err(|e| AuthError::login("openai_codex", e))?; + Ok(AuthOutcome::default()) +} diff --git a/src/llm/bedrock.rs b/crates/ironclaw_llm/src/bedrock.rs similarity index 96% rename from src/llm/bedrock.rs rename to crates/ironclaw_llm/src/bedrock.rs index 4326cabbb8e..809044cc732 100644 --- a/src/llm/bedrock.rs +++ b/crates/ironclaw_llm/src/bedrock.rs @@ -20,9 +20,9 @@ use aws_sdk_bedrockruntime::types::{ use aws_smithy_types::Document; use rust_decimal::Decimal; -use crate::llm::config::BedrockConfig; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::config::BedrockConfig; +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, }; @@ -96,7 +96,7 @@ impl LlmProvider for BedrockProvider { let model_id = self.current_model_id(); let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); // Bedrock requires toolConfig when messages contain ToolUse/ToolResult // blocks. Messages may carry tool history from prior agentic iterations, // but complete() has no tools to build a toolConfig — strip them. @@ -152,7 +152,7 @@ impl LlmProvider for BedrockProvider { let model_id = self.current_model_id(); let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let tool_config = build_tool_config(&request.tools, request.tool_choice.as_deref())?; @@ -206,6 +206,7 @@ impl LlmProvider for BedrockProvider { output_tokens, finish_reason: map_stop_reason(response.stop_reason()), cache_creation_input_tokens: 0, + reasoning: None, cache_read_input_tokens: 0, }) } @@ -293,8 +294,8 @@ fn build_inference_config( /// Note: this intentionally loses structured tool_call_id correlation — the text /// representation is sufficient for force_text mode where no further tool dispatch /// occurs. -fn strip_tool_blocks(messages: &mut [crate::llm::provider::ChatMessage]) { - use crate::llm::provider::Role; +fn strip_tool_blocks(messages: &mut [crate::provider::ChatMessage]) { + use crate::provider::Role; let mut stripped = 0u32; for msg in messages.iter_mut() { @@ -335,9 +336,9 @@ fn strip_tool_blocks(messages: &mut [crate::llm::provider::ChatMessage]) { /// 3. Consecutive tool results are merged into a single User message. /// 4. Bedrock requires strict user/assistant alternation. fn convert_messages( - messages: &[crate::llm::provider::ChatMessage], + messages: &[crate::provider::ChatMessage], ) -> Result<(Vec, Vec), LlmError> { - use crate::llm::provider::Role; + use crate::provider::Role; let mut system_blocks = Vec::new(); let mut bedrock_messages: Vec = Vec::new(); @@ -578,6 +579,7 @@ fn extract_content_blocks( name: tu.name().to_string(), arguments: document_to_json(tu.input()), reasoning: None, + signature: None, }); } // Ignore reasoning, citations, images, etc. @@ -748,7 +750,7 @@ pub(crate) fn document_to_json(doc: &Document) -> serde_json::Value { #[cfg(test)] mod tests { use super::*; - use crate::llm::provider::{ChatMessage, Role}; + use crate::provider::{ChatMessage, Role}; #[test] fn test_json_to_document_round_trip() { @@ -811,17 +813,19 @@ mod tests { #[test] fn test_convert_messages_tool_results_merge_into_user() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "echo".to_string(), arguments: serde_json::json!({"text": "hi"}), reasoning: None, + signature: None, }; - let tc2 = crate::llm::provider::ToolCall { + let tc2 = crate::provider::ToolCall { id: "call_2".to_string(), name: "time".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let messages = vec![ @@ -856,11 +860,12 @@ mod tests { #[test] fn test_convert_messages_assistant_with_tool_calls() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }; let messages = vec![ @@ -880,11 +885,12 @@ mod tests { #[test] fn test_convert_messages_empty_assistant_content_with_tool_calls() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let messages = vec![ @@ -978,6 +984,7 @@ mod tests { name: Some("echo".to_string()), tool_calls: None, content_parts: Vec::new(), + reasoning: None, }, ]; @@ -1045,17 +1052,19 @@ mod tests { fn test_full_tool_round_trip_conversation() { // Simulate a complete tool-use conversation: // system → user → assistant(tool_calls) → tool_results → user follow-up - let tc1 = crate::llm::provider::ToolCall { + let tc1 = crate::provider::ToolCall { id: "call_abc".to_string(), name: "get_weather".to_string(), arguments: serde_json::json!({"city": "NYC"}), reasoning: None, + signature: None, }; - let tc2 = crate::llm::provider::ToolCall { + let tc2 = crate::provider::ToolCall { id: "call_def".to_string(), name: "get_time".to_string(), arguments: serde_json::json!({"tz": "EST"}), reasoning: None, + signature: None, }; let messages = vec![ @@ -1213,11 +1222,12 @@ mod tests { #[test] fn test_strip_tool_blocks_removes_tool_content() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "echo".to_string(), arguments: serde_json::json!({"text": "hi"}), reasoning: None, + signature: None, }; let mut messages = vec![ @@ -1257,11 +1267,12 @@ mod tests { /// tool history to complete(), which has no toolConfig. #[test] fn test_strip_tool_blocks_then_convert_produces_no_tool_blocks() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_abc".to_string(), name: "get_weather".to_string(), arguments: serde_json::json!({"city": "NYC"}), reasoning: None, + signature: None, }; let mut messages = vec![ @@ -1273,7 +1284,7 @@ mod tests { ]; // Simulate the complete() pipeline - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); strip_tool_blocks(&mut messages); let (_, bedrock_msgs) = convert_messages(&messages).unwrap(); @@ -1294,11 +1305,12 @@ mod tests { #[test] fn test_complete_with_tools_empty_tools_strips_history() { - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "time".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let mut messages = vec![ @@ -1308,7 +1320,7 @@ mod tests { ]; // Simulate complete_with_tools() with empty tools - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let tool_config = build_tool_config(&[], None).unwrap(); assert!(tool_config.is_none()); @@ -1331,11 +1343,12 @@ mod tests { // Edge case: assistant message with ONLY tool_calls (no text) becomes // empty after stripping. convert_messages() should skip it, and the // subsequent tool-result-turned-user message should merge correctly. - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "search".to_string(), arguments: serde_json::json!({"q": "test"}), reasoning: None, + signature: None, }; let mut messages = vec![ @@ -1345,7 +1358,7 @@ mod tests { ChatMessage::user("Thanks"), ]; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); strip_tool_blocks(&mut messages); let (_, bedrock_msgs) = convert_messages(&messages).unwrap(); @@ -1386,11 +1399,12 @@ mod tests { parameters: serde_json::json!({"type": "object"}), }]; - let tc = crate::llm::provider::ToolCall { + let tc = crate::provider::ToolCall { id: "call_1".to_string(), name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let mut messages = vec![ @@ -1399,7 +1413,7 @@ mod tests { ChatMessage::tool_result("call_1", "echo", "done"), ]; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let tool_config = build_tool_config(&tools, Some("none")).unwrap(); assert!(tool_config.is_none()); diff --git a/src/llm/circuit_breaker.rs b/crates/ironclaw_llm/src/circuit_breaker.rs similarity index 99% rename from src/llm/circuit_breaker.rs rename to crates/ironclaw_llm/src/circuit_breaker.rs index 4740fe02f03..9c0ade4f0f6 100644 --- a/src/llm/circuit_breaker.rs +++ b/crates/ironclaw_llm/src/circuit_breaker.rs @@ -19,8 +19,8 @@ use async_trait::async_trait; use rust_decimal::Decimal; use tokio::sync::Mutex; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; @@ -324,11 +324,11 @@ mod tests { use crate::testing::StubLlm; fn make_request() -> CompletionRequest { - CompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")]) + CompletionRequest::new(vec![crate::ChatMessage::user("hello")]) } fn make_tool_request() -> ToolCompletionRequest { - ToolCompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")], vec![]) + ToolCompletionRequest::new(vec![crate::ChatMessage::user("hello")], vec![]) } fn fast_config(threshold: u32) -> CircuitBreakerConfig { diff --git a/src/llm/codex_auth.rs b/crates/ironclaw_llm/src/codex_auth.rs similarity index 100% rename from src/llm/codex_auth.rs rename to crates/ironclaw_llm/src/codex_auth.rs diff --git a/src/llm/codex_chatgpt.rs b/crates/ironclaw_llm/src/codex_chatgpt.rs similarity index 99% rename from src/llm/codex_chatgpt.rs rename to crates/ironclaw_llm/src/codex_chatgpt.rs index e7dcf40da0a..adc7755cd38 100644 --- a/src/llm/codex_chatgpt.rs +++ b/crates/ironclaw_llm/src/codex_chatgpt.rs @@ -733,6 +733,7 @@ impl LlmProvider for CodexChatGptProvider { name: tc.name, arguments: args, reasoning: None, + signature: None, } }) .collect(); @@ -755,6 +756,7 @@ impl LlmProvider for CodexChatGptProvider { finish_reason, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -827,6 +829,7 @@ mod tests { name: "search".to_string(), arguments: json!({"query": "rust"}), reasoning: None, + signature: None, }; let msg = ChatMessage::assistant_with_tool_calls(Some("thinking...".into()), vec![tc]); let items = CodexChatGptProvider::message_to_input_items(&msg); diff --git a/src/llm/codex_test_helpers.rs b/crates/ironclaw_llm/src/codex_test_helpers.rs similarity index 100% rename from src/llm/codex_test_helpers.rs rename to crates/ironclaw_llm/src/codex_test_helpers.rs diff --git a/src/llm/config.rs b/crates/ironclaw_llm/src/config.rs similarity index 67% rename from src/llm/config.rs rename to crates/ironclaw_llm/src/config.rs index e4d93a5384c..0dbaa9848cc 100644 --- a/src/llm/config.rs +++ b/crates/ironclaw_llm/src/config.rs @@ -9,9 +9,10 @@ use std::path::PathBuf; use secrecy::SecretString; -use crate::bootstrap::ironclaw_base_dir; -use crate::llm::registry::ProviderProtocol; -use crate::llm::session::SessionConfig; +use crate::error::LlmConfigError; +use crate::registry::ProviderProtocol; +use crate::session::SessionConfig; +use ironclaw_common::paths::ironclaw_base_dir; /// Sentinel value used as `api_key` when only an OAuth token is present. /// @@ -133,6 +134,32 @@ impl Default for OpenAiCodexConfig { } } +impl OpenAiCodexConfig { + /// Build a Codex config from already-resolved overrides, falling back to + /// crate defaults for any field the caller leaves as `None`. Callers + /// (the binary) own env / settings precedence and SSRF validation; this + /// helper centralises the default values inside the crate. + pub fn build( + model: Option, + auth_endpoint: Option, + api_base_url: Option, + client_id: Option, + session_path: Option, + token_refresh_margin_secs: Option, + ) -> Self { + let defaults = Self::default(); + Self { + model: model.unwrap_or(defaults.model), + auth_endpoint: auth_endpoint.unwrap_or(defaults.auth_endpoint), + api_base_url: api_base_url.unwrap_or(defaults.api_base_url), + client_id: client_id.unwrap_or(defaults.client_id), + session_path: session_path.unwrap_or(defaults.session_path), + token_refresh_margin_secs: token_refresh_margin_secs + .unwrap_or(defaults.token_refresh_margin_secs), + } + } +} + /// Configuration for AWS Bedrock (native Converse API). #[derive(Debug, Clone)] pub struct BedrockConfig { @@ -146,6 +173,52 @@ pub struct BedrockConfig { pub profile: Option, } +impl BedrockConfig { + /// Default region used when none is configured. + pub const DEFAULT_REGION: &'static str = "us-east-1"; + + /// Valid cross-region inference prefixes accepted by Bedrock. + pub const VALID_CROSS_REGION_PREFIXES: &'static [&'static str] = + &["us", "eu", "apac", "global"]; + + /// Build a Bedrock config from already-resolved overrides. + /// + /// - `region` falls back to [`Self::DEFAULT_REGION`] when `None`. + /// - `model` is required (returns [`LlmConfigError::MissingRequired`] when `None`). + /// - `cross_region`, when set, is validated against + /// [`Self::VALID_CROSS_REGION_PREFIXES`]. + pub fn build( + region: Option, + model: Option, + cross_region: Option, + profile: Option, + ) -> Result { + let region = region.unwrap_or_else(|| Self::DEFAULT_REGION.to_string()); + let model = model.ok_or_else(|| LlmConfigError::MissingRequired { + key: "BEDROCK_MODEL".to_string(), + hint: "Set BEDROCK_MODEL or selected_model when LLM_BACKEND=bedrock".to_string(), + })?; + if let Some(ref cr) = cross_region + && !Self::VALID_CROSS_REGION_PREFIXES.contains(&cr.as_str()) + { + return Err(LlmConfigError::InvalidValue { + key: "BEDROCK_CROSS_REGION".to_string(), + message: format!( + "'{}' is not valid, expected one of: {}", + cr, + Self::VALID_CROSS_REGION_PREFIXES.join(", ") + ), + }); + } + Ok(Self { + region, + model, + cross_region, + profile, + }) + } +} + /// LLM provider configuration. /// /// NearAI remains the default backend with its own config struct (session auth). @@ -289,7 +362,7 @@ impl NearAiConfig { /// appropriate base URL (cloud-api when API key is present, /// private.near.ai for session-token auth). pub(crate) fn for_model_discovery() -> Self { - let api_key = crate::config::helpers::env_or_override("NEARAI_API_KEY") + let api_key = ironclaw_common::env_helpers::env_or_override("NEARAI_API_KEY") .filter(|k| !k.is_empty()) .map(SecretString::from); @@ -298,7 +371,7 @@ impl NearAiConfig { } else { "https://private.near.ai" }; - let base_url = crate::config::helpers::env_or_override("NEARAI_BASE_URL") + let base_url = ironclaw_common::env_helpers::env_or_override("NEARAI_BASE_URL") .unwrap_or_else(|| default_base.to_string()); Self { @@ -343,10 +416,117 @@ pub struct GeminiOauthConfig { } impl GeminiOauthConfig { + /// Default model used when none is configured. + pub const DEFAULT_MODEL: &'static str = "gemini-2.5-flash"; + pub fn default_credentials_path() -> PathBuf { dirs::home_dir() .unwrap_or_else(|| PathBuf::from(".")) .join(".gemini") .join("oauth_creds.json") } + + /// Build a Gemini OAuth config from already-resolved overrides. + /// + /// Falls back to [`Self::DEFAULT_MODEL`] and + /// [`Self::default_credentials_path`] when their respective overrides + /// are absent. + pub fn build(model: Option, credentials_path: Option) -> Self { + Self { + model: model.unwrap_or_else(|| Self::DEFAULT_MODEL.to_string()), + credentials_path: credentials_path.unwrap_or_else(Self::default_credentials_path), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bedrock_build_applies_default_region() { + let cfg = BedrockConfig::build(None, Some("model-x".to_string()), None, None) + .expect("model is set"); + assert_eq!(cfg.region, BedrockConfig::DEFAULT_REGION); + assert_eq!(cfg.model, "model-x"); + assert!(cfg.cross_region.is_none()); + assert!(cfg.profile.is_none()); + } + + #[test] + fn bedrock_build_requires_model() { + let err = BedrockConfig::build(Some("us-west-2".into()), None, None, None) + .expect_err("model is required"); + assert!(matches!( + err, + LlmConfigError::MissingRequired { ref key, .. } if key == "BEDROCK_MODEL" + )); + } + + #[test] + fn bedrock_build_validates_cross_region() { + for ok in BedrockConfig::VALID_CROSS_REGION_PREFIXES { + let cfg = + BedrockConfig::build(None, Some("model".into()), Some((*ok).to_string()), None) + .expect("valid prefix"); + assert_eq!(cfg.cross_region.as_deref(), Some(*ok)); + } + + let err = BedrockConfig::build(None, Some("model".into()), Some("ap".to_string()), None) + .expect_err("'ap' is not a valid prefix"); + assert!(matches!( + err, + LlmConfigError::InvalidValue { ref key, .. } if key == "BEDROCK_CROSS_REGION" + )); + } + + #[test] + fn gemini_oauth_build_applies_defaults() { + let cfg = GeminiOauthConfig::build(None, None); + assert_eq!(cfg.model, GeminiOauthConfig::DEFAULT_MODEL); + assert_eq!( + cfg.credentials_path, + GeminiOauthConfig::default_credentials_path() + ); + + let cfg = GeminiOauthConfig::build( + Some("gemini-foo".into()), + Some(PathBuf::from("/tmp/creds.json")), + ); + assert_eq!(cfg.model, "gemini-foo"); + assert_eq!(cfg.credentials_path, PathBuf::from("/tmp/creds.json")); + } + + #[test] + fn openai_codex_build_applies_defaults() { + let cfg = OpenAiCodexConfig::build(None, None, None, None, None, None); + let defaults = OpenAiCodexConfig::default(); + assert_eq!(cfg.model, defaults.model); + assert_eq!(cfg.auth_endpoint, defaults.auth_endpoint); + assert_eq!(cfg.api_base_url, defaults.api_base_url); + assert_eq!(cfg.client_id, defaults.client_id); + assert_eq!(cfg.session_path, defaults.session_path); + assert_eq!( + cfg.token_refresh_margin_secs, + defaults.token_refresh_margin_secs + ); + } + + #[test] + fn openai_codex_build_overrides_take_precedence() { + let cfg = OpenAiCodexConfig::build( + Some("gpt-overridden".into()), + Some("https://auth.example".into()), + Some("https://api.example".into()), + Some("client-z".into()), + Some(PathBuf::from("/tmp/sess.json")), + Some(60), + ); + assert_eq!(cfg.model, "gpt-overridden"); + assert_eq!(cfg.auth_endpoint, "https://auth.example"); + assert_eq!(cfg.api_base_url, "https://api.example"); + assert_eq!(cfg.client_id, "client-z"); + assert_eq!(cfg.session_path, PathBuf::from("/tmp/sess.json")); + assert_eq!(cfg.token_refresh_margin_secs, 60); + } } diff --git a/src/llm/costs.rs b/crates/ironclaw_llm/src/costs.rs similarity index 100% rename from src/llm/costs.rs rename to crates/ironclaw_llm/src/costs.rs diff --git a/src/llm/error.rs b/crates/ironclaw_llm/src/error.rs similarity index 93% rename from src/llm/error.rs rename to crates/ironclaw_llm/src/error.rs index 29ede6f61d2..4afe4e7598e 100644 --- a/src/llm/error.rs +++ b/crates/ironclaw_llm/src/error.rs @@ -2,6 +2,21 @@ use std::time::Duration; +/// Errors that occur while assembling LLM configuration from settings/env. +/// +/// Distinct from [`LlmError`] (runtime / request errors): these fire before +/// any provider is constructed, when a per-backend config struct is being +/// built. The binary's `crate::error::ConfigError` carries a +/// `From` impl so callers can `?` through both layers. +#[derive(Debug, thiserror::Error)] +pub enum LlmConfigError { + #[error("Missing required configuration: {key}. {hint}")] + MissingRequired { key: String, hint: String }, + + #[error("Invalid configuration value for {key}: {message}")] + InvalidValue { key: String, message: String }, +} + /// LLM provider errors. #[derive(Debug, thiserror::Error)] pub enum LlmError { diff --git a/src/llm/failover.rs b/crates/ironclaw_llm/src/failover.rs similarity index 99% rename from src/llm/failover.rs rename to crates/ironclaw_llm/src/failover.rs index a23934d1e73..3d80ccc1565 100644 --- a/src/llm/failover.rs +++ b/crates/ironclaw_llm/src/failover.rs @@ -17,13 +17,13 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use rust_decimal::Decimal; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; -use crate::llm::retry::is_retryable; +use crate::retry::is_retryable; /// Configuration for per-provider cooldown behavior. /// @@ -388,7 +388,7 @@ mod tests { use std::sync::{Mutex, RwLock}; use std::time::Duration; - use crate::llm::provider::{CompletionResponse, FinishReason, ToolCompletionResponse}; + use crate::provider::{CompletionResponse, FinishReason, ToolCompletionResponse}; /// A mock LLM provider that returns a predetermined result. struct MockProvider { @@ -423,6 +423,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }))), } } @@ -537,11 +538,11 @@ mod tests { } fn make_request() -> CompletionRequest { - CompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")]) + CompletionRequest::new(vec![crate::ChatMessage::user("hello")]) } fn make_tool_request() -> ToolCompletionRequest { - ToolCompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")], vec![]) + ToolCompletionRequest::new(vec![crate::ChatMessage::user("hello")], vec![]) } // Test 1: Primary succeeds, no failover occurs. @@ -833,6 +834,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } diff --git a/src/llm/gemini_oauth.rs b/crates/ironclaw_llm/src/gemini_oauth.rs similarity index 99% rename from src/llm/gemini_oauth.rs rename to crates/ironclaw_llm/src/gemini_oauth.rs index 9e035be99c4..9af4afcca08 100644 --- a/src/llm/gemini_oauth.rs +++ b/crates/ironclaw_llm/src/gemini_oauth.rs @@ -15,7 +15,7 @@ use url::Url; use crate::config::GeminiOauthConfig; use crate::error::LlmError; -use crate::llm::provider::{ +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata, Role, ToolCall, ToolDefinition, }; @@ -193,6 +193,11 @@ pub struct GeminiCredits { } /// Extended response metadata parsed from Gemini API responses. +// Fields are unread now that the only consumer (`last_response_meta`) is +// unused after `gemini_oauth` was made `pub(crate)`. Kept to preserve the +// move's no-behavior-change guarantee; delete in a follow-up if no caller +// emerges. +#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct GeminiResponseMeta { /// Model version actually used (from response). @@ -433,6 +438,10 @@ impl CredentialManager { } } + // Unused after `pub mod gemini_oauth` → `pub(crate) mod gemini_oauth`. Kept + // in place because the boundary-cleanup move is meant to be behavior-preserving; + // delete in a follow-up if there's no future caller. + #[allow(dead_code)] pub async fn get_valid_access_token(&self) -> Result { let cred = self.get_valid_credential().await?; Ok(cred.access_token) @@ -952,6 +961,8 @@ impl GeminiOauthProvider { } /// Returns the latest response metadata from the last API call. + // Unused after module privatization; see `get_valid_access_token` above. + #[allow(dead_code)] pub fn last_response_meta(&self) -> GeminiResponseMeta { self.last_response_meta .lock() @@ -1070,6 +1081,8 @@ impl GeminiOauthProvider { } /// Count tokens for the given messages using the Gemini countTokens API. + // Unused after module privatization; see `get_valid_access_token` above. + #[allow(dead_code)] pub async fn count_tokens(&self, messages: &[ChatMessage]) -> Result { let sigs = self .thought_signatures @@ -1939,6 +1952,7 @@ impl GeminiOauthProvider { name, arguments: args, reasoning: None, + signature: None, }); } } @@ -2105,8 +2119,8 @@ impl LlmProvider for GeminiOauthProvider { async fn complete_with_tools( &self, - request: crate::llm::provider::ToolCompletionRequest, - ) -> Result { + request: crate::provider::ToolCompletionRequest, + ) -> Result { let tool_defs = if request.tools.is_empty() { None } else { @@ -2150,7 +2164,7 @@ impl LlmProvider for GeminiOauthProvider { sigs.retain(|id, _| live_ids.contains(id.as_str())); } - Ok(crate::llm::provider::ToolCompletionResponse { + Ok(crate::provider::ToolCompletionResponse { content: if response.content.is_empty() { None } else { @@ -2162,6 +2176,7 @@ impl LlmProvider for GeminiOauthProvider { tool_calls, cache_read_input_tokens: response.cache_read_input_tokens, cache_creation_input_tokens: response.cache_creation_input_tokens, + reasoning: None, }) } } @@ -2763,6 +2778,7 @@ mod tests { name: "read_file".to_string(), arguments: serde_json::json!({"path": "/tmp/x"}), reasoning: None, + signature: None, }], ), ChatMessage::tool_result("call_1", "read_file", r#"{"output":"hello"}"#), @@ -2802,6 +2818,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], ), ChatMessage::tool_result("call_1", "echo", r#"{"output":"ok"}"#), diff --git a/src/llm/github_copilot.rs b/crates/ironclaw_llm/src/github_copilot.rs similarity index 96% rename from src/llm/github_copilot.rs rename to crates/ironclaw_llm/src/github_copilot.rs index 88be16c9cf0..4088cd70f1c 100644 --- a/src/llm/github_copilot.rs +++ b/crates/ironclaw_llm/src/github_copilot.rs @@ -18,11 +18,11 @@ use rust_decimal::Decimal; use secrecy::ExposeSecret; use serde::{Deserialize, Serialize}; -use crate::llm::config::RegistryProviderConfig; -use crate::llm::costs; -use crate::llm::error::LlmError; -use crate::llm::github_copilot_auth::CopilotTokenManager; -use crate::llm::provider::{ +use crate::config::RegistryProviderConfig; +use crate::costs; +use crate::error::LlmError; +use crate::github_copilot_auth::CopilotTokenManager; +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, strip_unsupported_completion_params, strip_unsupported_tool_params, @@ -112,8 +112,8 @@ impl GithubCopilotProvider { let token = self.token_manager.get_token().await.map_err(|e| { tracing::warn!(error = %e, "Copilot: token exchange failed"); match &e { - crate::llm::github_copilot_auth::GithubCopilotAuthError::AccessDenied - | crate::llm::github_copilot_auth::GithubCopilotAuthError::Expired => { + crate::github_copilot_auth::GithubCopilotAuthError::AccessDenied + | crate::github_copilot_auth::GithubCopilotAuthError::Expired => { LlmError::AuthFailed { provider: "github_copilot".to_string(), } @@ -148,7 +148,7 @@ impl GithubCopilotProvider { if !status.is_success() { // Use shared retry-after parser (supports HTTP-date, default 60s) - let retry_after = Some(crate::llm::retry::parse_retry_after( + let retry_after = Some(crate::retry::parse_retry_after( response.headers().get(reqwest::header::RETRY_AFTER), )); @@ -159,7 +159,7 @@ impl GithubCopilotProvider { tracing::warn!( status = %status, - body = %crate::agent::truncate_for_preview(&response_text, 256), + body = %ironclaw_common::truncate_for_preview(&response_text, 256), "Copilot: API error response" ); @@ -181,7 +181,7 @@ impl GithubCopilotProvider { retry_after, }); } - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); return Err(LlmError::RequestFailed { provider: "github_copilot".to_string(), reason: format!("HTTP {status}: {truncated}"), @@ -194,7 +194,7 @@ impl GithubCopilotProvider { })?; serde_json::from_str(&response_text).map_err(|e| { - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); tracing::warn!( error = %e, body = %truncated, @@ -347,6 +347,7 @@ impl LlmProvider for GithubCopilotProvider { .map(|u| u.completion_tokens) .unwrap_or(0), cache_creation_input_tokens: 0, + reasoning: None, cache_read_input_tokens: 0, }) } @@ -604,6 +605,7 @@ fn extract_choice_content(choice: &OpenAiChoice) -> (Option, Vec String { if body.len() <= LIMIT { return body.to_string(); } - let end = crate::util::floor_char_boundary(body, LIMIT); + // Walk back to a UTF-8 char boundary. Equivalent to the (still-unstable) + // `str::floor_char_boundary` used by the main crate's `crate::util`. + let mut end = LIMIT; + while end > 0 && !body.is_char_boundary(end) { + end -= 1; + } format!("{}...", &body[..end]) } diff --git a/crates/ironclaw_llm/src/host.rs b/crates/ironclaw_llm/src/host.rs new file mode 100644 index 00000000000..8efdb95595d --- /dev/null +++ b/crates/ironclaw_llm/src/host.rs @@ -0,0 +1,118 @@ +//! Host-side abstractions used by `ironclaw_llm` to talk to the embedding +//! application without depending on its internals. +//! +//! `SessionManager` and the interactive NEAR-AI renewal flow used to call +//! into the main crate's `Database`, `SecretsStore`, `crate::config::helpers`, +//! and `crate::setup::*`. Keeping those calls direct meant `ironclaw_llm` +//! couldn't compile without the rest of the binary. The traits in this +//! module narrow that surface to exactly what the LLM crate needs; the main +//! crate provides adapter impls. + +use std::sync::Arc; + +use async_trait::async_trait; +use secrecy::SecretString; + +use crate::error::LlmError; + +/// Subset of a settings table used by `SessionManager` to persist NEAR-AI +/// session tokens (and read them back across restarts). +#[async_trait] +pub trait SessionDb: Send + Sync { + /// Persist a JSON value under `(user_id, key)`. + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), String>; + + /// Read a JSON value, returning `Ok(None)` if absent. + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result, String>; +} + +/// Subset of an encrypted secrets store used by `SessionManager` to persist +/// NEAR-AI session tokens at rest under user-keyed encryption. +#[async_trait] +pub trait SessionSecrets: Send + Sync { + /// Create or replace the secret named `name` for `user_id`. + async fn create( + &self, + user_id: &str, + name: &str, + value: String, + provider: Option<&str>, + ) -> Result<(), String>; + + /// Decrypt the secret named `name` for `user_id`. + async fn get_decrypted(&self, user_id: &str, name: &str) -> Result; +} + +/// Hooks the host installs to recover from a session failure. +/// +/// The default `NoopRenewer` returns `LlmError::SessionRenewalFailed` for +/// every renewal request, which is appropriate for headless / hosted +/// deployments where the session token is set via env var. CLI builds wire +/// in an interactive impl that drives the OAuth menu in `src/llm_session/`. +#[async_trait] +pub trait SessionRenewer: Send + Sync { + /// Drive an interactive (or automated) renewal flow. + /// + /// The implementation is expected to either: + /// - call `manager.set_token(...)` and `manager.save_session_for_renewer(...)` + /// with a fresh session token, returning `Ok(())`, or + /// - persist an API key through `SessionKeyPersistor` and return `Ok(())` + /// without setting a session token (the caller falls back to API-key + /// auth on the next request). + async fn renew(&self, manager: &super::session::SessionManager) -> Result<(), LlmError>; +} + +/// Persistence hooks for one-shot API-key entry: thread-safe runtime env +/// overlay plus best-effort write to the host's `.env` file. +pub trait SessionKeyPersistor: Send + Sync { + /// Make `value` visible to in-process env lookups for the rest of the + /// process (thread-safe overlay; no UB from `set_var`). + fn set_runtime_env(&self, key: &str, value: &str); + + /// Persist `key=value` to the host's bootstrap `.env` so it survives + /// across restarts. May fail; failure is non-fatal for the caller. + fn upsert_bootstrap_var(&self, key: &str, value: &str) -> std::io::Result<()>; +} + +// Default no-op implementations for headless contexts. + +/// A `SessionRenewer` that always fails. Headless deployments use this +/// (the LLM call returns `SessionExpired`, which surfaces to the user). +pub struct NoopSessionRenewer; + +#[async_trait] +impl SessionRenewer for NoopSessionRenewer { + async fn renew(&self, _manager: &super::session::SessionManager) -> Result<(), LlmError> { + Err(LlmError::SessionRenewalFailed { + provider: "nearai".to_string(), + reason: "interactive session renewal is unavailable in this build; \ + set NEARAI_SESSION_TOKEN or NEARAI_API_KEY env var instead" + .to_string(), + }) + } +} + +/// A `SessionKeyPersistor` that does nothing. +pub struct NoopKeyPersistor; + +impl SessionKeyPersistor for NoopKeyPersistor { + fn set_runtime_env(&self, _key: &str, _value: &str) {} + fn upsert_bootstrap_var(&self, _key: &str, _value: &str) -> std::io::Result<()> { + Ok(()) + } +} + +/// Convenience type aliases. +pub type SharedSessionDb = Arc; +pub type SharedSessionSecrets = Arc; +pub type SharedSessionRenewer = Arc; +pub type SharedSessionKeyPersistor = Arc; diff --git a/src/llm/image_models.rs b/crates/ironclaw_llm/src/image_models.rs similarity index 100% rename from src/llm/image_models.rs rename to crates/ironclaw_llm/src/image_models.rs diff --git a/src/llm/mod.rs b/crates/ironclaw_llm/src/lib.rs similarity index 74% rename from src/llm/mod.rs rename to crates/ironclaw_llm/src/lib.rs index b0fdbd00268..64cf26fba06 100644 --- a/src/llm/mod.rs +++ b/crates/ironclaw_llm/src/lib.rs @@ -9,6 +9,7 @@ //! - **AWS Bedrock**: Native Converse API via aws-sdk-bedrockruntime mod anthropic_oauth; +pub mod auth; #[cfg(feature = "bedrock")] mod bedrock; pub mod circuit_breaker; @@ -18,13 +19,13 @@ pub mod config; pub mod costs; pub mod error; pub mod failover; -pub mod gemini_oauth; +pub(crate) mod gemini_oauth; mod github_copilot; pub(crate) mod github_copilot_auth; -mod nearai_chat; -pub mod oauth_helpers; +pub mod host; +pub mod nearai_chat; pub mod openai_codex_provider; -pub mod openai_codex_session; +pub(crate) mod openai_codex_session; mod provider; mod reasoning; pub mod recording; @@ -36,9 +37,12 @@ pub mod runtime; pub mod session; pub mod smart_routing; mod token_refreshing; -pub(crate) mod tool_schema; +pub mod tool_schema; pub mod transcription; +#[cfg(any(test, feature = "testing"))] +pub mod testing; + #[cfg(test)] mod codex_test_helpers; @@ -49,16 +53,21 @@ pub mod vision_models; pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider}; pub use config::{ - BedrockConfig, CacheRetention, LlmConfig, NearAiConfig, OAUTH_PLACEHOLDER, OpenAiCodexConfig, - RegistryProviderConfig, + BedrockConfig, CacheRetention, GeminiOauthConfig, LlmConfig, NearAiConfig, OAUTH_PLACEHOLDER, + OpenAiCodexConfig, RegistryProviderConfig, }; -pub use error::LlmError; +pub use error::{LlmConfigError, LlmError}; pub use failover::{CooldownConfig, FailoverProvider}; -pub use gemini_oauth::GeminiOauthProvider; +pub(crate) use gemini_oauth::GeminiOauthProvider; +pub use host::{ + NoopKeyPersistor, NoopSessionRenewer, SessionDb, SessionKeyPersistor, SessionRenewer, + SessionSecrets, SharedSessionDb, SharedSessionKeyPersistor, SharedSessionRenewer, + SharedSessionSecrets, +}; pub use nearai_chat::{DEFAULT_MODEL, ModelInfo, NearAiChatProvider, default_models}; pub use openai_codex_provider::OpenAiCodexProvider; -pub use openai_codex_session::{OpenAiCodexSession, OpenAiCodexSessionManager}; -pub(crate) use provider::sanitize_tool_messages; +pub(crate) use openai_codex_session::OpenAiCodexSessionManager; +pub use provider::sanitize_tool_messages; pub use provider::{ ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, ImageUrl, LlmProvider, ModelMetadata, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, @@ -70,8 +79,8 @@ pub use reasoning::{ TokenUsage, ToolSelection, is_silent_reply, llm_signals_tool_intent, user_signals_execution_intent, }; -pub(crate) use reasoning::{clean_response, recover_tool_calls_from_content}; -pub use recording::RecordingLlm; +pub use reasoning::{clean_response, recover_tool_calls_from_content}; +pub use recording::{MemorySnapshotEntry, RecordingLlm}; pub use registry::{ProviderDefinition, ProviderProtocol, ProviderRegistry}; pub use response_cache::{CachedProvider, ResponseCacheConfig}; pub use retry::{RetryConfig, RetryProvider}; @@ -187,6 +196,9 @@ fn create_registry_provider( ProviderProtocol::OpenAiCompletions => create_openai_compat_from_registry(config), ProviderProtocol::Anthropic => create_anthropic_from_registry(config), ProviderProtocol::Ollama => create_ollama_from_registry(config), + ProviderProtocol::DeepSeek => create_deepseek_from_registry(config), + ProviderProtocol::Gemini => create_gemini_from_registry(config), + ProviderProtocol::OpenRouter => create_openrouter_from_registry(config), ProviderProtocol::GithubCopilot => { let provider = github_copilot::GithubCopilotProvider::new(config, request_timeout_secs)?; @@ -198,6 +210,22 @@ fn create_registry_provider( ); Ok(Arc::new(provider)) } + // Protocols with a dedicated config slot on `LlmConfig` are + // dispatched in `create_llm_provider` before this function is + // reached. They never carry a `RegistryProviderConfig`, so this + // arm is only reachable as an internal logic bug. + ProviderProtocol::Bedrock + | ProviderProtocol::OpenAiCodex + | ProviderProtocol::GeminiOauth + | ProviderProtocol::NearAi => Err(LlmError::RequestFailed { + provider: config.provider_id.clone(), + reason: format!( + "Provider '{}' uses a dedicated config slot on LlmConfig and \ + must be dispatched in create_llm_provider, not via \ + RegistryProviderConfig.", + config.provider_id + ), + }), } } @@ -260,14 +288,24 @@ fn create_openai_compat_from_registry( let name = match reqwest::header::HeaderName::from_bytes(key.as_bytes()) { Ok(n) => n, Err(e) => { - tracing::warn!(header = %key, error = %e, "Skipping extra header: invalid name"); + tracing::warn!( + provider = %config.provider_id, + header = %key, + error = %e, + "Skipping extra header: invalid name", + ); continue; } }; let val = match reqwest::header::HeaderValue::from_str(value) { Ok(v) => v, Err(e) => { - tracing::warn!(header = %key, error = %e, "Skipping extra header: invalid value"); + tracing::warn!( + provider = %config.provider_id, + header = %key, + error = %e, + "Skipping extra header: invalid value", + ); continue; } }; @@ -329,7 +367,7 @@ fn create_anthropic_from_registry( let api_key_is_placeholder = config .api_key .as_ref() - .is_some_and(|k| k.expose_secret() == crate::llm::config::OAUTH_PLACEHOLDER); + .is_some_and(|k| k.expose_secret() == crate::config::OAUTH_PLACEHOLDER); if config.oauth_token.is_some() && (config.api_key.is_none() || api_key_is_placeholder) { tracing::debug!( provider = %config.provider_id, @@ -341,7 +379,7 @@ fn create_anthropic_from_registry( return Ok(Arc::new(provider)); } - use crate::llm::config::CacheRetention; + use crate::config::CacheRetention; use rig::providers::anthropic; let api_key = config @@ -424,6 +462,222 @@ fn create_ollama_from_registry( Ok(Arc::new(adapter)) } +/// Build a DeepSeek provider via rig-core's dedicated DeepSeek client. +/// +/// Routing through this client (rather than the generic OpenAI-compat path) +/// is what makes thinking-mode tool calling work: rig-core's DeepSeek +/// implementation captures `reasoning_content` from each response and writes +/// it back onto the assistant message in the next request. Without that +/// round-trip the API rejects the second turn with HTTP 400 ("The +/// reasoning_content in the thinking mode must be passed back to the API"). +/// See #3201. +fn create_deepseek_from_registry( + config: &RegistryProviderConfig, +) -> Result, LlmError> { + use rig::providers::deepseek; + + let api_key = config + .api_key + .as_ref() + .map(|k| k.expose_secret().to_string()) + .ok_or_else(|| LlmError::AuthFailed { + provider: config.provider_id.clone(), + })?; + + let client: deepseek::Client = if config.base_url.is_empty() { + deepseek::Client::new(&api_key) + } else { + deepseek::Client::builder() + .api_key(&api_key) + .base_url(&config.base_url) + .build() + } + .map_err(|e| LlmError::RequestFailed { + provider: config.provider_id.clone(), + reason: format!("Failed to create DeepSeek client: {e}"), + })?; + + let model = client.completion_model(&config.model); + + tracing::debug!( + provider = %config.provider_id, + model = %config.model, + base_url = if config.base_url.is_empty() { "default" } else { &config.base_url }, + "Using DeepSeek provider (preserves reasoning_content across turns)" + ); + + Ok(Arc::new( + RigAdapter::new(model, &config.model) + .with_unsupported_params(config.unsupported_params.clone()), + )) +} + +/// Build an OpenRouter provider via rig-core's dedicated OpenRouter client. +/// +/// Routing through this client (rather than the generic OpenAI-compat path) +/// preserves OpenRouter's `reasoning`, `reasoning_details`, and per-tool-call +/// signatures across turns. The generic OpenAI client strips all of them, so +/// any thinking-mode model accessed via OpenRouter (Claude with thinking, +/// OpenAI o-series, DeepSeek-R1, Gemini 2.5+, Qwen QwQ, …) loses its +/// reasoning artifacts on the assistant message and the next request fails +/// the same way as #3201 / #3225. +fn create_openrouter_from_registry( + config: &RegistryProviderConfig, +) -> Result, LlmError> { + use rig::providers::openrouter; + + let api_key = config + .api_key + .as_ref() + .map(|k| k.expose_secret().to_string()) + .ok_or_else(|| LlmError::AuthFailed { + provider: config.provider_id.clone(), + })?; + + // OpenRouter attribution headers (`HTTP-Referer`, `X-Title`) and any other + // user-configured extras must follow the request through. The `http` crate + // normalizes header names to lowercase internally, so configuring + // `HTTP-Referer` or `X-Title` (canonical OpenRouter spelling) parses fine. + let mut extra_headers = reqwest::header::HeaderMap::new(); + for (key, value) in &config.extra_headers { + let name = match reqwest::header::HeaderName::from_bytes(key.as_bytes()) { + Ok(n) => n, + Err(e) => { + tracing::warn!( + provider = %config.provider_id, + header = %key, + error = %e, + "Skipping extra header: invalid name", + ); + continue; + } + }; + let val = match reqwest::header::HeaderValue::from_str(value) { + Ok(v) => v, + Err(e) => { + tracing::warn!( + provider = %config.provider_id, + header = %key, + error = %e, + "Skipping extra header: invalid value", + ); + continue; + } + }; + extra_headers.insert(name, val); + } + + let mut builder = openrouter::Client::builder().api_key(&api_key); + if !config.base_url.is_empty() { + builder = builder.base_url(&config.base_url); + } + if !extra_headers.is_empty() { + builder = builder.http_headers(extra_headers); + } + + let client: openrouter::Client = builder.build().map_err(|e| LlmError::RequestFailed { + provider: config.provider_id.clone(), + reason: format!("Failed to create OpenRouter client: {e}"), + })?; + + let model = client.completion_model(&config.model); + + tracing::debug!( + provider = %config.provider_id, + model = %config.model, + base_url = if config.base_url.is_empty() { "default" } else { &config.base_url }, + "Using OpenRouter provider (preserves reasoning + signatures across turns)" + ); + + Ok(Arc::new( + RigAdapter::new(model, &config.model) + .with_unsupported_params(config.unsupported_params.clone()), + )) +} + +/// Build a Gemini provider via rig-core's dedicated Gemini client. +/// +/// Routing through this client (rather than the generic OpenAI-compat path +/// at `/v1beta/openai`) is what makes Gemini thinking-mode tool calling +/// work: rig-core's Gemini implementation round-trips `thought_signature` +/// on each `functionCall`. Without that round-trip the API rejects the +/// next turn with HTTP 400 ("Function call is missing a thought_signature +/// in functionCall parts"). See #3225. +/// +/// This is API-key auth only (`GEMINI_API_KEY`). Users on Gemini OAuth go +/// through the separate `gemini_oauth` backend. +fn create_gemini_from_registry( + config: &RegistryProviderConfig, +) -> Result, LlmError> { + use rig::providers::gemini; + + let api_key = config + .api_key + .as_ref() + .map(|k| k.expose_secret().to_string()) + .ok_or_else(|| LlmError::AuthFailed { + provider: config.provider_id.clone(), + })?; + + // Pre-3201/3225 installs persisted the OpenAI-shim URL + // (`https://generativelanguage.googleapis.com/v1beta/openai`) under + // `llm_builtin_overrides[gemini].base_url`. Passing that to rig-core's + // native Gemini client would produce + // `…/v1beta/openai/v1beta/models/{model}:generateContent` and break every + // request. Discard any persisted shim URL and use the native default. + let base_url = sanitize_gemini_base_url(&config.base_url); + + let client: gemini::Client = if base_url.is_empty() { + gemini::Client::new(&api_key) + } else { + gemini::Client::builder() + .api_key(&api_key) + .base_url(&base_url) + .build() + } + .map_err(|e| LlmError::RequestFailed { + provider: config.provider_id.clone(), + reason: format!("Failed to create Gemini client: {e}"), + })?; + + let model = client.completion_model(&config.model); + + tracing::debug!( + provider = %config.provider_id, + model = %config.model, + base_url = if base_url.is_empty() { "default" } else { &base_url }, + "Using Gemini provider (preserves thought_signature across turns)" + ); + + Ok(Arc::new( + RigAdapter::new(model, &config.model) + .with_unsupported_params(config.unsupported_params.clone()), + )) +} + +/// Discard pre-3225 OpenAI-shim Gemini URLs (`…/v1beta/openai`). +/// +/// Returns the empty string to signal "use rig-core's native default" when the +/// configured base URL is the legacy shim. Other URLs (custom proxies, region +/// endpoints, etc.) pass through unchanged. +fn sanitize_gemini_base_url(base_url: &str) -> String { + let trimmed = base_url.trim().trim_end_matches('/'); + if trimmed.is_empty() { + return String::new(); + } + let lower = trimmed.to_ascii_lowercase(); + if lower.ends_with("/v1beta/openai") || lower.ends_with("/v1/openai") { + tracing::warn!( + stale_base_url = %base_url, + "Ignoring legacy OpenAI-shim base URL for native Gemini provider; \ + using rig-core default. Clear `llm_builtin_overrides[gemini].base_url` \ + in settings to silence this warning." + ); + return String::new(); + } + trimmed.to_string() +} + /// Create an OpenAI Codex provider with OAuth authentication. /// /// This is async because it needs to ensure authentication before @@ -791,7 +1045,7 @@ fn normalize_openai_base_url(url: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::llm::config::NearAiConfig; + use crate::config::NearAiConfig; fn test_nearai_config() -> NearAiConfig { NearAiConfig { @@ -1010,7 +1264,7 @@ mod tests { // Switch to a registry backend without a provider config — this is // a deterministic failure path in `create_llm_provider` (returns - // `AuthFailed`). See `src/llm/mod.rs::create_llm_provider`. + // `AuthFailed`). See `create_llm_provider` above. let mut broken = test_llm_config(); broken.backend = "openai".to_string(); broken.provider = None; @@ -1114,4 +1368,66 @@ mod tests { "https://api.example.com/custom" ); } + + /// Regression for #3225: pre-PR, the configure UI/setup default for + /// Gemini was the OpenAI shim URL ending in `/v1beta/openai`. Once + /// `ProviderProtocol::Gemini` switches to rig-core's native client + /// (which appends `/v1beta/models/{model}:generateContent`), passing + /// the persisted shim URL through would produce + /// `…/v1beta/openai/v1beta/models/...` and break every Gemini call. + /// `sanitize_gemini_base_url` must strip those legacy values. + #[test] + fn sanitize_gemini_base_url_strips_legacy_openai_shim() { + // The exact string the old configure UI persisted. + assert_eq!( + sanitize_gemini_base_url("https://generativelanguage.googleapis.com/v1beta/openai"), + "", + "legacy OpenAI-shim base URL must be discarded so rig-core's \ + native default takes over", + ); + // With trailing slash (also seen in saved overrides). + assert_eq!( + sanitize_gemini_base_url("https://generativelanguage.googleapis.com/v1beta/openai/"), + "", + ); + // Case-insensitive on the suffix match. + assert_eq!( + sanitize_gemini_base_url("https://Generativelanguage.googleapis.com/V1beta/OpenAI"), + "", + ); + // The alternate `/v1/openai` shape (some adapters used this). + assert_eq!( + sanitize_gemini_base_url("https://example.com/v1/openai"), + "", + ); + } + + /// Empty/whitespace-only input must still be treated as "use the default", + /// not get accidentally upgraded to a real URL. + #[test] + fn sanitize_gemini_base_url_passes_through_empty() { + assert_eq!(sanitize_gemini_base_url(""), ""); + assert_eq!(sanitize_gemini_base_url(" "), ""); + } + + /// Custom proxies / region endpoints / native Gemini bases must + /// pass through unchanged (modulo trailing-slash trimming). + #[test] + fn sanitize_gemini_base_url_preserves_custom_endpoints() { + // Native default base (rig-core would also use this). + assert_eq!( + sanitize_gemini_base_url("https://generativelanguage.googleapis.com"), + "https://generativelanguage.googleapis.com", + ); + // Custom proxy. + assert_eq!( + sanitize_gemini_base_url("https://gemini-proxy.internal.example.com"), + "https://gemini-proxy.internal.example.com", + ); + // Trailing slash gets trimmed. + assert_eq!( + sanitize_gemini_base_url("https://gemini-proxy.internal.example.com/"), + "https://gemini-proxy.internal.example.com", + ); + } } diff --git a/src/llm/models.rs b/crates/ironclaw_llm/src/models.rs similarity index 70% rename from src/llm/models.rs rename to crates/ironclaw_llm/src/models.rs index 3275c8d0257..73d57933ce8 100644 --- a/src/llm/models.rs +++ b/crates/ironclaw_llm/src/models.rs @@ -1,4 +1,58 @@ //! Model discovery and fetching for multiple LLM providers. +//! +//! External callers should use [`fetch_models_for`] — a single verb-based +//! facade that dispatches on a provider ID string. The per-provider +//! fetcher functions below are `pub(crate)` and not part of the public +//! surface of `ironclaw_llm`. + +/// Options for [`fetch_models_for`]. +#[derive(Debug, Default)] +pub struct ModelFetchOptions<'a> { + /// API key for backends that authenticate per request (anthropic, + /// openai, openai-compatible). Optional — fetchers fall back to env + /// vars and then to static defaults if no key is available. + pub api_key: Option<&'a str>, + /// Base URL for self-hosted or proxied backends (ollama, + /// openai-compatible). `None` uses the per-backend default + /// (e.g. `http://localhost:11434` for ollama). + pub base_url: Option<&'a str>, +} + +/// Fetch the model catalog for a given backend. +/// +/// Dispatches on `provider_id`: +/// - `"anthropic"` → Anthropic `/v1/models` +/// - `"openai"` → OpenAI `/v1/models` (filtered to chat-capable models) +/// - `"ollama"` → local Ollama `/api/tags` +/// - any other ID → generic OpenAI-compatible `/v1/models` against +/// `options.base_url`. Used by openrouter, deepseek, custom endpoints, +/// etc. +/// +/// For `anthropic` / `openai` / `ollama`, the per-backend fetcher falls +/// back to its own static default list on network or auth failure so +/// the setup wizard can still progress offline. +/// +/// The generic openai-compatible branch has **no static fallback** — it +/// returns an empty list if `options.base_url` is missing/empty or the +/// `/v1/models` call fails. Callers must handle the empty case (e.g. +/// fall back to the registry's default model). +pub async fn fetch_models_for( + provider_id: &str, + options: &ModelFetchOptions<'_>, +) -> Vec<(String, String)> { + match provider_id { + "anthropic" => fetch_anthropic_models(options.api_key).await, + "openai" => fetch_openai_models(options.api_key).await, + "ollama" => { + let base_url = options.base_url.unwrap_or("http://localhost:11434"); + fetch_ollama_models(base_url).await + } + _ => { + let base_url = options.base_url.unwrap_or(""); + fetch_openai_compatible_models(base_url, options.api_key).await + } + } +} /// Fetch models from the Anthropic API. /// @@ -22,9 +76,7 @@ pub(crate) async fn fetch_anthropic_models(cached_key: Option<&str>) -> Vec<(Str // Fall back to OAuth token if no API key let oauth_token = if api_key.is_none() { - crate::config::helpers::optional_env("ANTHROPIC_OAUTH_TOKEN") - .ok() - .flatten() + ironclaw_common::env_helpers::env_or_override("ANTHROPIC_OAUTH_TOKEN") .filter(|t| !t.is_empty()) } else { None @@ -331,15 +383,15 @@ pub(crate) async fn fetch_openai_compatible_models( /// /// Uses [`NearAiConfig::for_model_discovery()`] to construct a minimal NEAR AI /// config, then wraps it in an `LlmConfig` with session config for auth. -pub(crate) fn build_nearai_model_fetch_config() -> crate::config::LlmConfig { - let auth_base_url = crate::config::helpers::env_or_override("NEARAI_AUTH_URL") +pub fn build_nearai_model_fetch_config() -> crate::config::LlmConfig { + let auth_base_url = ironclaw_common::env_helpers::env_or_override("NEARAI_AUTH_URL") .unwrap_or_else(|| "https://private.near.ai".to_string()); crate::config::LlmConfig { backend: "nearai".to_string(), - session: crate::llm::session::SessionConfig { + session: crate::session::SessionConfig { auth_base_url, - session_path: crate::config::llm::default_session_path(), + session_path: ironclaw_common::paths::ironclaw_base_dir().join("session.json"), }, nearai: crate::config::NearAiConfig::for_model_discovery(), provider: None, @@ -357,3 +409,51 @@ pub(crate) fn build_nearai_model_fetch_config() -> crate::config::LlmConfig { response_cache_max_entries: 1000, } } + +#[cfg(test)] +mod classifier_tests { + use super::*; + + #[test] + fn is_openai_chat_model_includes_gpt5_and_filters_non_chat_variants() { + assert!(is_openai_chat_model("gpt-5")); + assert!(is_openai_chat_model("gpt-5-mini-2026-01-01")); + assert!(is_openai_chat_model("o3-2025-04-16")); + assert!(!is_openai_chat_model("chatgpt-image-latest")); + assert!(!is_openai_chat_model("gpt-4o-realtime-preview")); + assert!(!is_openai_chat_model("gpt-4o-mini-transcribe")); + assert!(!is_openai_chat_model("text-embedding-3-large")); + } + + #[test] + fn sort_openai_models_prioritizes_best_models_first() { + let mut models = vec![ + ("gpt-4o-mini".to_string(), "gpt-4o-mini".to_string()), + ("gpt-5-mini".to_string(), "gpt-5-mini".to_string()), + ("o3".to_string(), "o3".to_string()), + ("gpt-4.1".to_string(), "gpt-4.1".to_string()), + ("gpt-5".to_string(), "gpt-5".to_string()), + ]; + + sort_openai_models(&mut models); + + let ordered: Vec = models.into_iter().map(|(id, _)| id).collect(); + assert_eq!( + ordered, + vec![ + "gpt-5".to_string(), + "gpt-5-mini".to_string(), + "o3".to_string(), + "gpt-4.1".to_string(), + "gpt-4o-mini".to_string(), + ] + ); + } + + #[tokio::test] + async fn fetch_ollama_models_unreachable_fallback() { + // Point at a port nothing listens on. + let models = fetch_ollama_models("http://127.0.0.1:1").await; + assert!(!models.is_empty(), "should fall back to static defaults"); + } +} diff --git a/src/llm/nearai_chat.rs b/crates/ironclaw_llm/src/nearai_chat.rs similarity index 97% rename from src/llm/nearai_chat.rs rename to crates/ironclaw_llm/src/nearai_chat.rs index b9830164385..33774fdd7d2 100644 --- a/src/llm/nearai_chat.rs +++ b/crates/ironclaw_llm/src/nearai_chat.rs @@ -17,14 +17,14 @@ use rust_decimal::prelude::MathematicalOps; use secrecy::ExposeSecret; use serde::{Deserialize, Serialize}; -use crate::llm::config::NearAiConfig; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::config::NearAiConfig; +use crate::error::LlmError; +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, }; -use crate::llm::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; -use crate::llm::{costs, session::SessionManager}; +use crate::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; +use crate::{costs, session::SessionManager}; /// Information about an available model from NEAR AI API. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -269,7 +269,7 @@ impl NearAiChatProvider { let retry_after_header: Option = response .headers() .get("retry-after") - .map(crate::llm::retry::parse_retry_after_value); + .map(crate::retry::parse_retry_after_value); let response_text = response.text().await.map_err(|e| LlmError::RequestFailed { provider: "nearai_chat".to_string(), reason: format!("Failed to read response body: {}", e), @@ -316,7 +316,7 @@ impl NearAiChatProvider { // can trigger automatic compaction instead of crashing. if status_code == 413 { let lower = response_text.to_ascii_lowercase(); - let (used, limit) = crate::llm::rig_adapter::parse_token_counts(&lower); + let (used, limit) = crate::rig_adapter::parse_token_counts(&lower); return Err(LlmError::ContextLengthExceeded { used, limit }); } @@ -331,7 +331,7 @@ impl NearAiChatProvider { "payload too large", ]; if CONTEXT_PATTERNS.iter().any(|p| lower.contains(p)) { - let (used, limit) = crate::llm::rig_adapter::parse_token_counts(&lower); + let (used, limit) = crate::rig_adapter::parse_token_counts(&lower); return Err(LlmError::ContextLengthExceeded { used, limit }); } } @@ -349,7 +349,8 @@ impl NearAiChatProvider { tracing::debug!( provider = "nearai_chat", status = status_code, - body_preview = crate::agent::truncate_for_preview(&response_text, 512).as_str(), + body_preview = + ironclaw_common::truncate_for_preview(&response_text, 512).as_str(), "NEAR AI Chat upstream 5xx response" ); return Err(LlmError::BadGateway { @@ -359,7 +360,7 @@ impl NearAiChatProvider { }); } - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); return Err(LlmError::RequestFailed { provider: "nearai_chat".to_string(), reason: format!("HTTP {}: {}", status, truncated), @@ -367,7 +368,7 @@ impl NearAiChatProvider { } serde_json::from_str(&response_text).map_err(|e| { - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); LlmError::InvalidResponse { provider: "nearai_chat".to_string(), reason: format!("JSON parse error: {}. Raw: {}", e, truncated), @@ -419,7 +420,7 @@ impl NearAiChatProvider { provider: "nearai_chat".to_string(), }); } - let truncated = crate::agent::truncate_for_preview(&response_text, 512); + let truncated = ironclaw_common::truncate_for_preview(&response_text, 512); return Err(LlmError::RequestFailed { provider: "nearai_chat".to_string(), reason: format!("HTTP {}: {}", status, truncated), @@ -511,7 +512,7 @@ impl NearAiChatProvider { provider: "nearai_chat".to_string(), reason: format!( "No model names found in response: {}", - &response_text[..crate::util::floor_char_boundary(&response_text, 300)] + ironclaw_common::truncate_preview(&response_text, 300) ), }) } @@ -524,7 +525,7 @@ impl LlmProvider for NearAiChatProvider { .take_model_override() .unwrap_or_else(|| self.active_model_name()); let mut raw_messages = req.messages; - crate::llm::provider::sanitize_tool_messages(&mut raw_messages); + crate::provider::sanitize_tool_messages(&mut raw_messages); let raw: Vec = raw_messages.into_iter().map(|m| m.into()).collect(); // NEAR AI rejects `role:"tool"` messages even on text-only completion paths. @@ -597,7 +598,7 @@ impl LlmProvider for NearAiChatProvider { .take_model_override() .unwrap_or_else(|| self.active_model_name()); let mut raw_messages = req.messages; - crate::llm::provider::sanitize_tool_messages(&mut raw_messages); + crate::provider::sanitize_tool_messages(&mut raw_messages); let messages: Vec = raw_messages.into_iter().map(|m| m.into()).collect(); @@ -653,6 +654,7 @@ impl LlmProvider for NearAiChatProvider { name: tc.function.name, arguments, reasoning: None, + signature: None, } }) .collect(); @@ -693,6 +695,7 @@ impl LlmProvider for NearAiChatProvider { output_tokens, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } @@ -765,7 +768,7 @@ struct ChatCompletionRequest { #[derive(Debug, Clone)] enum MessageContent { Text(String), - Parts(Vec), + Parts(Vec), } impl Serialize for MessageContent { @@ -1053,11 +1056,11 @@ impl From for ChatCompletionMessage { None } else if !msg.content_parts.is_empty() { // Build multimodal content array: text + image parts - let mut parts = vec![crate::llm::ContentPart::Text { text: msg.content }]; + let mut parts = vec![crate::ContentPart::Text { text: msg.content }]; parts.extend(msg.content_parts.into_iter().map(|part| match part { - crate::llm::ContentPart::ImageUrl { mut image_url } => { + crate::ContentPart::ImageUrl { mut image_url } => { image_url.detail = Some(image_url.normalized_openai_detail()); - crate::llm::ContentPart::ImageUrl { image_url } + crate::ContentPart::ImageUrl { image_url } } other => other, })); @@ -1096,7 +1099,7 @@ struct ChatCompletionFunction { /// /// Chat Completions is non-strict by default, but this boundary still flattens /// top-level combinators that OpenAI-compatible tool APIs reject. -fn convert_tool_definition(tool: crate::llm::provider::ToolDefinition) -> ChatCompletionTool { +fn convert_tool_definition(tool: crate::provider::ToolDefinition) -> ChatCompletionTool { let mut description = tool.description.clone(); let parameters = shape_tool_schema( ToolSchemaPolicy::FlattenOnly, @@ -1117,7 +1120,7 @@ fn convert_tool_definition(tool: crate::llm::provider::ToolDefinition) -> ChatCo fn build_chat_completion_request( model: String, messages: Vec, - tools: Vec, + tools: Vec, temperature: Option, max_tokens: Option, stop: Option>, @@ -1202,13 +1205,13 @@ fn saturate_u32(val: u64) -> u32 { /// response type. /// /// Subscribers attach a `tracing_subscriber::Layer` filtered on target -/// `ironclaw::llm::reasoning`. Emitted at `TRACE` level so default loggers +/// `ironclaw_llm::reasoning`. Emitted at `TRACE` level so default loggers /// don't surface potentially large chain-of-thought traces. /// /// No-op when reasoning is `None` or empty. fn emit_reasoning_trace(reasoning: Option<&str>) { if let Some(rc) = reasoning.filter(|s| !s.is_empty()) { - tracing::trace!(target: "ironclaw::llm::reasoning", "{rc}"); + tracing::trace!(target: "ironclaw_llm::reasoning", "{rc}"); } } @@ -1231,7 +1234,7 @@ fn parse_usage(usage: Option<&ChatCompletionUsage>) -> (u32, u32) { #[cfg(test)] mod tests { use super::*; - use crate::llm::session::SessionConfig; + use crate::session::SessionConfig; use rust_decimal_macros::dec; fn test_nearai_config(base_url: &str) -> NearAiConfig { @@ -1301,8 +1304,8 @@ mod tests { fn test_message_conversion_defaults_missing_image_detail_to_auto() { let msg = ChatMessage::user_with_parts( "describe this", - vec![crate::llm::ContentPart::ImageUrl { - image_url: crate::llm::ImageUrl { + vec![crate::ContentPart::ImageUrl { + image_url: crate::ImageUrl { url: "data:image/jpeg;base64,Zm9v".to_string(), detail: None, }, @@ -1325,8 +1328,8 @@ mod tests { for expected in ["low", "high"] { let msg = ChatMessage::user_with_parts( "describe this", - vec![crate::llm::ContentPart::ImageUrl { - image_url: crate::llm::ImageUrl { + vec![crate::ContentPart::ImageUrl { + image_url: crate::ImageUrl { url: format!("https://example.com/{expected}.png"), detail: Some(expected.to_string()), }, @@ -1349,7 +1352,7 @@ mod tests { #[test] fn test_assistant_with_tool_calls_conversion() { - use crate::llm::ToolCall; + use crate::ToolCall; let tool_calls = vec![ ToolCall { @@ -1357,12 +1360,14 @@ mod tests { name: "list_issues".to_string(), arguments: serde_json::json!({"owner": "foo", "repo": "bar"}), reasoning: None, + signature: None, }, ToolCall { id: "call_2".to_string(), name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }, ]; @@ -1389,7 +1394,7 @@ mod tests { #[test] fn test_convert_tool_definition_preserves_optional_fields() { - use crate::llm::provider::ToolDefinition; + use crate::provider::ToolDefinition; let tool = ToolDefinition { name: "message".to_string(), @@ -1421,7 +1426,7 @@ mod tests { #[test] fn test_convert_tool_definition_flattens_top_level_oneof_without_strictifying() { - use crate::llm::provider::ToolDefinition; + use crate::provider::ToolDefinition; let tool = ToolDefinition { name: "lookup".to_string(), @@ -1469,13 +1474,14 @@ mod tests { #[test] fn test_tool_call_arguments_serialized_to_string() { - use crate::llm::ToolCall; + use crate::ToolCall; let tc = ToolCall { id: "call_1".to_string(), name: "test".to_string(), arguments: serde_json::json!({"key": "value"}), reasoning: None, + signature: None, }; let msg = ChatMessage::assistant_with_tool_calls(None, vec![tc]); let chat_msg: ChatCompletionMessage = msg.into(); @@ -1727,6 +1733,7 @@ mod tests { name: tc.function.name, arguments, reasoning: None, + signature: None, } }) .collect(); @@ -1784,6 +1791,7 @@ mod tests { name: tc.function.name, arguments, reasoning: None, + signature: None, } }) .collect(); @@ -1884,6 +1892,7 @@ mod tests { name: tc.function.name, arguments, reasoning: None, + signature: None, } }) .collect(); @@ -1902,7 +1911,7 @@ mod tests { } /// Smoke test: non-empty reasoning content produces a trace event on the - /// dedicated `ironclaw::llm::reasoning` target. + /// dedicated `ironclaw_llm::reasoning` target. #[test] #[tracing_test::traced_test] fn reasoning_content_emits_trace_event() { @@ -1921,7 +1930,7 @@ mod tests { emit_reasoning_trace(None); emit_reasoning_trace(Some("")); assert!( - !logs_contain("ironclaw::llm::reasoning"), + !logs_contain("ironclaw_llm::reasoning"), "empty/absent reasoning should not emit any event" ); } @@ -1933,8 +1942,8 @@ mod tests { fn reasoning_emission_uses_dedicated_target() { emit_reasoning_trace(Some("trace-target-marker")); assert!( - logs_contain("ironclaw::llm::reasoning"), - "emission should use ironclaw::llm::reasoning target" + logs_contain("ironclaw_llm::reasoning"), + "emission should use ironclaw_llm::reasoning target" ); assert!(logs_contain("trace-target-marker")); } @@ -2036,7 +2045,7 @@ mod tests { impl EnvLockGuard { fn new() -> Self { Self { - _guard: crate::config::helpers::lock_env(), + _guard: ironclaw_common::env_helpers::lock_env(), } } } @@ -2098,7 +2107,7 @@ mod tests { impl EnvLockGuard { fn new() -> Self { Self { - _guard: crate::config::helpers::lock_env(), + _guard: ironclaw_common::env_helpers::lock_env(), } } } @@ -2151,7 +2160,7 @@ mod tests { #[test] fn test_build_chat_completion_request_normalizes_top_level_oneof() { - use crate::llm::provider::ToolDefinition; + use crate::provider::ToolDefinition; let request = build_chat_completion_request( "test-model".to_string(), @@ -2703,6 +2712,7 @@ mod tests { name: "test".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], ); let chat_msg: ChatCompletionMessage = msg.into(); diff --git a/src/llm/openai_codex_provider.rs b/crates/ironclaw_llm/src/openai_codex_provider.rs similarity index 98% rename from src/llm/openai_codex_provider.rs rename to crates/ironclaw_llm/src/openai_codex_provider.rs index 1e0df402cff..12d9402f698 100644 --- a/src/llm/openai_codex_provider.rs +++ b/crates/ironclaw_llm/src/openai_codex_provider.rs @@ -13,7 +13,7 @@ use serde::Deserialize; use tokio::sync::RwLock; use crate::error::LlmError; -use crate::llm::provider::{ +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, LlmProvider, ModelMetadata, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, }; @@ -260,7 +260,7 @@ impl LlmProvider for OpenAiCodexProvider { async fn complete(&self, request: CompletionRequest) -> Result { let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let body = self.build_request_body(&messages, None); let parsed = self.send_request(body).await?; @@ -279,7 +279,7 @@ impl LlmProvider for OpenAiCodexProvider { request: ToolCompletionRequest, ) -> Result { let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); // Build a reverse map so we can translate sanitized names back to originals. // Only needed when sanitization actually changes a name (e.g. MCP tools with dots). @@ -327,6 +327,7 @@ impl LlmProvider for OpenAiCodexProvider { finish_reason, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } @@ -504,7 +505,7 @@ fn sanitize_tool_name(name: &str) -> String { /// can append a hint to the tool description, so we pass an owned clone /// through and read it back. fn convert_tool_definition(tool: &ToolDefinition) -> serde_json::Value { - use crate::llm::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; + use crate::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; let mut description = tool.description.clone(); let parameters = shape_tool_schema( @@ -680,6 +681,7 @@ fn parse_sse_response(body: &str) -> Result { name: state.name, arguments, reasoning: None, + signature: None, }); } else { // Fallback: extract directly from the item @@ -706,6 +708,7 @@ fn parse_sse_response(body: &str) -> Result { name, arguments, reasoning: None, + signature: None, }); } } @@ -784,6 +787,7 @@ fn parse_sse_response(body: &str) -> Result { name: state.name, arguments, reasoning: None, + signature: None, }); } } @@ -815,7 +819,7 @@ fn parse_sse_response(body: &str) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::llm::codex_test_helpers::make_test_jwt; + use crate::codex_test_helpers::make_test_jwt; #[test] fn test_extract_account_id_success() { @@ -880,12 +884,14 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }, ToolCall { id: "call_2".to_string(), name: "read".to_string(), arguments: serde_json::json!({"path": "/tmp"}), reasoning: None, + signature: None, }, ]; let msg = @@ -1242,6 +1248,7 @@ data: {"type":"response.completed","response":{"status":"completed","usage":{"in name: "mcp.server.search".to_string(), arguments: serde_json::json!({"q": "test"}), reasoning: None, + signature: None, }]; let msg = ChatMessage::assistant_with_tool_calls(None, tool_calls); let items = super::convert_message(&msg, 0); @@ -1293,6 +1300,7 @@ data: {"type":"response.completed","response":{"status":"completed","usage":{"in name: "mcp_server_search".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; if let Some(original) = name_map.get(&tc.name) { tc.name = original.clone(); @@ -1305,7 +1313,7 @@ data: {"type":"response.completed","response":{"status":"completed","usage":{"in /// HTTP 400 because function_call_output references a non-existent call_id. #[test] fn test_build_request_sanitizes_orphaned_tool_results() { - use crate::llm::provider::sanitize_tool_messages; + use crate::provider::sanitize_tool_messages; // An orphaned tool result: no preceding assistant message with a // matching tool_call for "call_orphan". diff --git a/src/llm/openai_codex_session.rs b/crates/ironclaw_llm/src/openai_codex_session.rs similarity index 99% rename from src/llm/openai_codex_session.rs rename to crates/ironclaw_llm/src/openai_codex_session.rs index 75c5e961de4..dac023e5695 100644 --- a/src/llm/openai_codex_session.rs +++ b/crates/ironclaw_llm/src/openai_codex_session.rs @@ -637,7 +637,7 @@ impl OpenAiCodexSessionManager { #[cfg(test)] mod tests { use super::*; - use crate::llm::codex_test_helpers::test_codex_config as test_config; + use crate::codex_test_helpers::test_codex_config as test_config; use tempfile::tempdir; #[tokio::test] diff --git a/src/llm/provider.rs b/crates/ironclaw_llm/src/provider.rs similarity index 92% rename from src/llm/provider.rs rename to crates/ironclaw_llm/src/provider.rs index 3daffdc3a37..eba40673e94 100644 --- a/src/llm/provider.rs +++ b/crates/ironclaw_llm/src/provider.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -use crate::llm::error::LlmError; +use crate::error::LlmError; /// Role in a conversation. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -78,6 +78,14 @@ pub struct ChatMessage { /// to appear on the assistant message preceding tool result messages). #[serde(skip_serializing_if = "Option::is_none")] pub tool_calls: Option>, + /// Provider-emitted reasoning artifacts (DeepSeek's `reasoning_content`, + /// Gemini's `thought_signature` parts, OpenRouter's `reasoning_details`) + /// captured from the previous response. Required to be echoed back on + /// the next request — DeepSeek thinking-mode and Gemini 2.5+ both reject + /// the next turn with HTTP 400 when the prior assistant message had + /// reasoning that was dropped (#3201, #3225). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning: Option, } impl ChatMessage { @@ -90,6 +98,7 @@ impl ChatMessage { tool_call_id: None, name: None, tool_calls: None, + reasoning: None, } } @@ -102,6 +111,7 @@ impl ChatMessage { tool_call_id: None, name: None, tool_calls: None, + reasoning: None, } } @@ -116,6 +126,7 @@ impl ChatMessage { tool_call_id: None, name: None, tool_calls: None, + reasoning: None, } } @@ -128,6 +139,7 @@ impl ChatMessage { tool_call_id: None, name: None, tool_calls: None, + reasoning: None, } } @@ -147,9 +159,24 @@ impl ChatMessage { } else { Some(tool_calls) }, + reasoning: None, } } + /// Attach provider-emitted reasoning artifacts to an assistant message. + /// + /// Required for thinking-mode tool calling on DeepSeek (`reasoning_content`), + /// Gemini 2.5+ (`thought_signature`), and OpenRouter (`reasoning_details`). + /// The provider rejects the next turn with HTTP 400 when the prior + /// assistant message had reasoning that wasn't echoed back. See #3201, #3225. + /// + /// Empty / whitespace-only reasoning is dropped (treated as None) so we + /// don't send `reasoning_content: ""` and trip strict-mode validators. + pub fn with_reasoning(mut self, reasoning: Option) -> Self { + self.reasoning = reasoning.filter(|r| !r.trim().is_empty()); + self + } + /// Create a tool result message. pub fn tool_result( tool_call_id: impl Into, @@ -163,6 +190,7 @@ impl ChatMessage { tool_call_id: Some(tool_call_id.into()), name: Some(name.into()), tool_calls: None, + reasoning: None, } } } @@ -235,12 +263,13 @@ pub struct CompletionResponse { } /// Why the completion finished. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum FinishReason { Stop, Length, ToolUse, ContentFilter, + #[default] Unknown, } @@ -262,6 +291,14 @@ pub struct ToolCall { /// or derived from the shared response content as a fallback. #[serde(default, skip_serializing_if = "Option::is_none")] pub reasoning: Option, + /// Provider-emitted per-tool-call cryptographic signature (Gemini's + /// `thought_signature`, Anthropic's reasoning signature). Required to be + /// echoed on the next request — Gemini 2.5+ rejects tool-loop turns with + /// HTTP 400 ("Function call is missing a thought_signature in + /// functionCall parts") when the prior tool call's signature was dropped. + /// See #3225. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature: Option, } /// Generate a tool-call ID that satisfies all providers. @@ -393,6 +430,12 @@ pub struct ToolCompletionResponse { pub cache_read_input_tokens: u32, /// Tokens written to the provider's server-side prompt cache (Anthropic). pub cache_creation_input_tokens: u32, + /// Provider-emitted reasoning content (DeepSeek `reasoning_content`, + /// Gemini `thought_signature` parts, OpenRouter `reasoning_details`). + /// Callers MUST attach this to the assistant `ChatMessage` they store + /// for the next turn — otherwise the provider rejects the follow-up with + /// HTTP 400 (#3201, #3225). `None` when the model produced no reasoning. + pub reasoning: Option, } /// Metadata about a model returned by the provider's API. @@ -492,7 +535,7 @@ mod model_override_tests { use rust_decimal::Decimal; use super::*; - use crate::llm::error::LlmError; + use crate::error::LlmError; struct StubProvider; @@ -805,6 +848,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let mut messages = vec![ ChatMessage::user("hello"), @@ -849,6 +893,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let mut messages = vec![ ChatMessage::user("test"), @@ -875,12 +920,14 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"q": "test"}), reasoning: None, + signature: None, }; let tc2 = ToolCall { id: "call_sel_2".to_string(), name: "http".to_string(), arguments: serde_json::json!({"url": "https://example.com"}), reasoning: None, + signature: None, }; let mut messages = vec![ ChatMessage::system("You are a helpful assistant."), diff --git a/src/llm/reasoning.rs b/crates/ironclaw_llm/src/reasoning.rs similarity index 90% rename from src/llm/reasoning.rs rename to crates/ironclaw_llm/src/reasoning.rs index 8b54477e3d4..fb6879f7c35 100644 --- a/src/llm/reasoning.rs +++ b/crates/ironclaw_llm/src/reasoning.rs @@ -5,9 +5,9 @@ use std::sync::{Arc, LazyLock}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::llm::error::LlmError; +use crate::error::LlmError; -use crate::llm::{ +use crate::{ ChatMessage, CompletionRequest, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolDefinition, }; @@ -429,6 +429,12 @@ pub enum RespondResult { ToolCalls { tool_calls: Vec, content: Option, + /// Provider-emitted reasoning artifacts (DeepSeek `reasoning_content`, + /// Gemini reasoning, OpenRouter `reasoning_details`). Must be attached + /// to the assistant `ChatMessage` the caller pushes into context for + /// the next turn — otherwise the provider rejects the follow-up + /// request with HTTP 400. See #3201, #3225. + reasoning: Option, }, } @@ -460,7 +466,7 @@ pub struct Reasoning { /// This is passed to the LLM to provide clarity about who/group it's talking to. conversation_context: std::collections::HashMap, /// Platform identity and runtime metadata for self-awareness. - platform_info: Option, + platform_info: Option, } impl Reasoning { @@ -518,7 +524,7 @@ impl Reasoning { } /// Set platform metadata for self-awareness in system prompts. - pub fn with_platform_info(mut self, info: ironclaw_engine::PlatformInfo) -> Self { + pub fn with_platform_info(mut self, info: ironclaw_common::PlatformInfo) -> Self { self.platform_info = Some(info); self } @@ -819,6 +825,7 @@ Respond in JSON format: let pre_truncated = truncate_at_tool_tags(&c); clean_response(&pre_truncated) }); + let provider_reasoning = response.reasoning; // Populate per-tool reasoning from the shared narrative when the // provider did not supply per-tool rationale. let tool_calls: Vec = response @@ -845,6 +852,7 @@ Respond in JSON format: result: RespondResult::ToolCalls { tool_calls, content: narrative, + reasoning: provider_reasoning, }, usage, finish_reason: response.finish_reason, @@ -872,6 +880,10 @@ Respond in JSON format: } else { Some(cleaned) }, + // XML-tag-recovered tool calls don't come with native + // reasoning artifacts — those would have been on the + // structured tool_calls path instead. + reasoning: response.reasoning, }, usage, finish_reason: response.finish_reason, @@ -1077,7 +1089,7 @@ Respond with a JSON plan in this format: let needs_tags = self .model_name .as_ref() - .is_some_and(|n| crate::llm::reasoning_models::requires_think_final_tags(n)); + .is_some_and(|n| crate::reasoning_models::requires_think_final_tags(n)); let response_format = if needs_tags { r#"## Response Format — CRITICAL @@ -1586,7 +1598,7 @@ fn is_recoverable_tool_call_segment( /// - `...` (function_call variant) /// /// Only returns calls whose name matches an available tool. -pub(crate) fn recover_tool_calls_from_content( +pub fn recover_tool_calls_from_content( content: &str, available_tools: &[ToolDefinition], ) -> Vec { @@ -1640,6 +1652,7 @@ pub(crate) fn recover_tool_calls_from_content( name: name.to_string(), arguments, reasoning: None, + signature: None, }); continue; } @@ -1655,6 +1668,7 @@ pub(crate) fn recover_tool_calls_from_content( name: name.to_string(), arguments: serde_json::Value::Object(Default::default()), reasoning: None, + signature: None, }); } } @@ -1693,6 +1707,7 @@ pub(crate) fn recover_tool_calls_from_content( name: name.to_string(), arguments, reasoning: None, + signature: None, }); remaining = &args_start[bracket_end + 1..]; continue; @@ -1705,11 +1720,124 @@ pub(crate) fn recover_tool_calls_from_content( name: name.to_string(), arguments: serde_json::Value::Object(Default::default()), reasoning: None, + signature: None, }); remaining = after_name; } } + // Markdown fenced format that some models (notably some + // OpenAI-compatible chat-completions endpoints) emit instead of + // structured `tool_calls`: + // + // ```tool_call + // {"name": "get_balances", "arguments": {}} + // ``` + // + // Also accepts `function_call` and `tool_calls` as the fence info + // string. The opening fence itself must be an exact `\`\`\`{tag}` + // at the start of a line, followed by whitespace or a newline — + // extra backticks or a space before `{tag}` are not recognised. + for tag in TOOL_TAGS { + let mut search_from = 0; + while search_from < content.len() { + // Find opening fence with this tag as info string. Must + // be on its own line — the fence must come right after + // a newline or at content start. + let opening_pat = format!("```{tag}"); + let Some(rel_offset) = content[search_from..].find(&opening_pat) else { + break; + }; + let abs_open = search_from + rel_offset; + // Confirm the opening backticks are at the start of a + // line (avoid matching `inline ```tool_call` references + // inside prose). Either the absolute offset is 0 or the + // preceding character is a newline. + let at_line_start = abs_open == 0 + || content[..abs_open] + .chars() + .last() + .is_some_and(|c| c == '\n'); + // The character after the tag must be whitespace or + // newline — otherwise `tool_callX` would falsely match. + let after_tag = &content[abs_open + opening_pat.len()..]; + let valid_terminator = after_tag + .chars() + .next() + .is_none_or(|c| c == '\n' || c.is_whitespace()); + if !at_line_start || !valid_terminator { + search_from = abs_open + opening_pat.len(); + continue; + } + // Skip the rest of the opening line. + let body_start = match after_tag.find('\n') { + Some(nl) => abs_open + opening_pat.len() + nl + 1, + None => { + search_from = abs_open + opening_pat.len(); + continue; + } + }; + // Find the closing fence (a line that starts with ```). + let close_search = &content[body_start..]; + let mut close_rel: Option = None; + let mut idx = 0usize; + while idx <= close_search.len() { + let line_start = idx; + let line_end = close_search[idx..] + .find('\n') + .map(|n| idx + n) + .unwrap_or(close_search.len()); + let line = &close_search[line_start..line_end]; + if line.trim_start().starts_with("```") { + close_rel = Some(line_start); + break; + } + if line_end == close_search.len() { + break; + } + idx = line_end + 1; + } + let Some(close_rel) = close_rel else { + break; + }; + let body_end = body_start + close_rel; + // Advance past the closing fence line so the next + // iteration doesn't re-match the same block. + let after_close = body_end + + close_search[close_rel..] + .find('\n') + .map(|n| n + 1) + .unwrap_or_else(|| close_search.len() - close_rel); + search_from = after_close; + + let inner = content[body_start..body_end].trim(); + if inner.is_empty() { + continue; + } + + // Try JSON: {"name": "x", "arguments": {...}} + if let Ok(parsed) = serde_json::from_str::(inner) + && let Some(name) = parsed.get("name").and_then(|v| v.as_str()) + && tool_names.contains(name) + { + let arguments = parsed + .get("arguments") + .cloned() + .unwrap_or(serde_json::Value::Object(Default::default())); + calls.push(ToolCall { + id: super::provider::generate_tool_call_id( + calls.len(), + RECOVERED_TOOL_CALL_SEED, + ), + name: name.to_string(), + arguments, + reasoning: None, + signature: None, + }); + } + } + } + calls } @@ -1726,7 +1854,7 @@ pub(crate) fn recover_tool_calls_from_content( /// 5. Strip pipe-delimited reasoning tags (code-aware) /// 6. Strip tool tags (string matching — no code-awareness needed) /// 7. Collapse triple+ newlines, trim -pub(crate) fn clean_response(text: &str) -> String { +pub fn clean_response(text: &str) -> String { // 1. Quick-check let mut result = if !QUICK_TAG_RE.is_match(text) { text.to_string() @@ -1756,10 +1884,101 @@ pub(crate) fn clean_response(text: &str) -> String { // 6b. Strip bracket-format inline tool calls: [Called tool `name` with arguments: {...}] result = strip_bracket_tool_calls(&result); + // 6c. Strip markdown-fenced tool calls: ```tool_call\n{json}\n``` + // These pass cleanly through the XML/pipe strippers because they + // use backticks instead of angle brackets, but they're still + // tool-call syntax that should never reach the user. Recovery + // (`recover_tool_calls_from_content`) extracts the JSON above; this + // strips any leftover residue (malformed JSON, repeated emissions, + // model echo) so the user-visible text is clean. + for tag in TOOL_TAGS { + result = strip_markdown_fence_block(&result, tag); + } + // 7. Collapse triple+ newlines, trim collapse_newlines(&result) } +/// Strip markdown-fenced tool-call blocks like ` ```tool_call\n{...}\n``` `. +/// +/// Mirrors the recovery pass in `recover_tool_calls_from_content` so the +/// user-visible text has no fence residue when the LLM emits a markdown +/// fence instead of a structured tool call. Only fences with the exact +/// `tag` info string at line start are removed — inline backtick spans +/// (`` `like this` ``) and other unrelated fenced code stay intact. +fn strip_markdown_fence_block(text: &str, tag: &str) -> String { + let opening_pat = format!("```{tag}"); + let mut result = String::with_capacity(text.len()); + let mut remaining = text; + loop { + let Some(rel_offset) = remaining.find(&opening_pat) else { + result.push_str(remaining); + return result; + }; + let abs_open = rel_offset; + // Opening must be at line start (avoid inline backtick spans + // and code-comment references like ` ```tool_call ` shown + // inside another fenced block). + let at_line_start = abs_open == 0 + || remaining[..abs_open] + .chars() + .last() + .is_some_and(|c| c == '\n'); + // Character right after the tag must be whitespace/newline so + // we don't accidentally match `tool_callX`. + let after_tag = &remaining[abs_open + opening_pat.len()..]; + let valid_terminator = after_tag + .chars() + .next() + .is_none_or(|c| c == '\n' || c.is_whitespace()); + if !at_line_start || !valid_terminator { + // Skip past this false match and keep scanning. + let consumed = abs_open + opening_pat.len(); + result.push_str(&remaining[..consumed]); + remaining = &remaining[consumed..]; + continue; + } + + // Push everything before the fence opener (including the + // newline that put us at line start) so we don't leave a + // stray blank line. + let trim_to = remaining[..abs_open].trim_end_matches('\n').len(); + result.push_str(&remaining[..trim_to]); + + // Walk forward to the closing fence line. + let body_start = match after_tag.find('\n') { + Some(nl) => abs_open + opening_pat.len() + nl + 1, + None => { + // Unterminated opener; drop the rest. + return result; + } + }; + let close_search = &remaining[body_start..]; + let mut idx = 0usize; + let mut consumed_to = remaining.len(); + while idx <= close_search.len() { + let line_start = idx; + let line_end = close_search[idx..] + .find('\n') + .map(|n| idx + n) + .unwrap_or(close_search.len()); + let line = &close_search[line_start..line_end]; + if line.trim_start().starts_with("```") { + // Skip past the closing fence's trailing newline (if + // any) so the next chunk starts cleanly. + consumed_to = body_start + line_end + usize::from(line_end < close_search.len()); + break; + } + if line_end == close_search.len() { + // Reached EOF without a closing fence — drop the rest. + return result; + } + idx = line_end + 1; + } + remaining = &remaining[consumed_to..]; + } +} + /// Strip bracket-format inline tool calls produced by `flatten_tool_messages`. /// /// Removes patterns like `[Called tool `name` with arguments: {...}]` from text @@ -2566,6 +2785,96 @@ That's my plan."#; assert!(calls.is_empty()); } + /// Markdown-fenced form is what some OpenAI-compatible models emit + /// when they aren't given native `tool_calls` output. The recovery + /// must extract the call so the engine can dispatch it instead of + /// passing the fence through to the user (PR #3122 review report). + #[test] + fn test_recover_markdown_fence_tool_call() { + let tools = make_tools(&["get_balances"]); + let content = + "Here you go:\n\n```tool_call\n{\"name\": \"get_balances\", \"arguments\": {}}\n```\n"; + let calls = recover_tool_calls_from_content(content, &tools); + assert_eq!(calls.len(), 1, "markdown fence must produce 1 call"); + assert_eq!(calls[0].name, "get_balances"); + assert_eq!(calls[0].arguments, serde_json::json!({})); + } + + #[test] + fn test_recover_markdown_fence_with_arguments() { + let tools = make_tools(&["lookup"]); + let content = "```tool_call\n{\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"}}\n```"; + let calls = recover_tool_calls_from_content(content, &tools); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({"q": "x"})); + } + + #[test] + fn test_recover_markdown_fence_function_call_alias() { + let tools = make_tools(&["lookup"]); + let content = "```function_call\n{\"name\":\"lookup\",\"arguments\":{}}\n```"; + let calls = recover_tool_calls_from_content(content, &tools); + assert_eq!(calls.len(), 1, "function_call alias should also recover"); + assert_eq!(calls[0].name, "lookup"); + } + + #[test] + fn test_recover_markdown_fence_unknown_tool_ignored() { + let tools = make_tools(&["lookup"]); + let content = "```tool_call\n{\"name\":\"nonexistent\",\"arguments\":{}}\n```"; + let calls = recover_tool_calls_from_content(content, &tools); + assert!( + calls.is_empty(), + "unknown tool name in fence must be ignored" + ); + } + + #[test] + fn test_recover_markdown_fence_inline_reference_ignored() { + // An inline reference like `text ```tool_call ... ``` text` + // is NOT an opening fence at line start, so we should not + // accidentally recover from it. + let tools = make_tools(&["lookup"]); + let content = "Use ```tool_call\\n{\\\"name\\\":\\\"lookup\\\"}\\n``` to call it"; + let calls = recover_tool_calls_from_content(content, &tools); + assert!( + calls.is_empty(), + "inline reference must not be treated as a fence" + ); + } + + /// `clean_response` must strip a markdown-fenced tool-call block + /// even when the JSON inside is malformed (so recovery would have + /// skipped it). Otherwise the fence syntax leaks to the user. + #[test] + fn test_clean_response_strips_markdown_fenced_tool_call() { + let input = + "Here you go:\n\n```tool_call\n{\"name\":\"get_balances\",\"arguments\":{}}\n```\n"; + let cleaned = clean_response(input); + assert!( + !cleaned.contains("```"), + "markdown fence must be stripped: {cleaned:?}" + ); + assert!( + !cleaned.contains("get_balances"), + "JSON body inside fence must be stripped: {cleaned:?}" + ); + assert!( + cleaned.contains("Here you go"), + "prose outside the fence must remain: {cleaned:?}" + ); + } + + #[test] + fn test_clean_response_strips_malformed_markdown_fence() { + // Even malformed JSON inside the fence must be stripped — + // recovery skips it but the user must not see it. + let input = "Reply\n```tool_call\nNOT JSON\n```\n"; + let cleaned = clean_response(input); + assert!(!cleaned.contains("```"), "fence must go: {cleaned:?}"); + assert!(!cleaned.contains("NOT JSON"), "body must go: {cleaned:?}"); + } + #[test] fn test_recover_multiple_tool_calls() { let tools = make_tools(&["tool_list", "tool_auth"]); @@ -3361,9 +3670,7 @@ That's my plan."#; #[tokio::test] async fn test_respond_with_tools_flags_empty_tool_completion_when_content_is_none() { - use crate::llm::{ - FinishReason, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, - }; + use crate::{FinishReason, LlmProvider, ToolCompletionRequest, ToolCompletionResponse}; use async_trait::async_trait; use rust_decimal::Decimal; @@ -3381,15 +3688,15 @@ That's my plan."#; async fn complete( &self, - _request: crate::llm::CompletionRequest, - ) -> Result { + _request: crate::CompletionRequest, + ) -> Result { unreachable!("tool-mode test should not call complete()") } async fn complete_with_tools( &self, _request: ToolCompletionRequest, - ) -> Result { + ) -> Result { Ok(ToolCompletionResponse { content: None, tool_calls: Vec::new(), @@ -3398,6 +3705,7 @@ That's my plan."#; finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -3549,6 +3857,7 @@ That's my plan."#; RespondResult::ToolCalls { tool_calls, content, + reasoning: _, } => { assert_eq!(tool_calls.len(), 1); assert_eq!(tool_calls[0].name, "tool_list"); @@ -3582,6 +3891,7 @@ That's my plan."#; RespondResult::ToolCalls { tool_calls, content, + reasoning: _, } => { assert_eq!(tool_calls.len(), 1); assert_eq!(tool_calls[0].name, "tool_list"); @@ -3600,7 +3910,7 @@ That's my plan."#; #[test] fn test_openai_reasoning_models_not_detected() { - use crate::llm::reasoning_models::has_native_thinking; + use crate::reasoning_models::has_native_thinking; assert!(!has_native_thinking("o1")); assert!(!has_native_thinking("o1-mini")); assert!(!has_native_thinking("o1-preview")); @@ -3715,11 +4025,11 @@ That's my plan."#; /// Mock provider that returns tool calls with a configurable finish_reason. struct TruncatingLlm { - finish_reason: crate::llm::FinishReason, + finish_reason: crate::FinishReason, } #[async_trait::async_trait] - impl crate::llm::LlmProvider for TruncatingLlm { + impl crate::LlmProvider for TruncatingLlm { fn model_name(&self) -> &str { "truncating-stub" } @@ -3728,27 +4038,29 @@ That's my plan."#; } async fn complete( &self, - _request: crate::llm::CompletionRequest, - ) -> Result { + _request: crate::CompletionRequest, + ) -> Result { unimplemented!() } async fn complete_with_tools( &self, - _request: crate::llm::ToolCompletionRequest, - ) -> Result { - Ok(crate::llm::ToolCompletionResponse { + _request: crate::ToolCompletionRequest, + ) -> Result { + Ok(crate::ToolCompletionResponse { content: Some("I'll write the report.".to_string()), tool_calls: vec![ToolCall { id: "call_1".to_string(), name: "memory_write".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], input_tokens: 5000, output_tokens: 1024, finish_reason: self.finish_reason, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } diff --git a/src/llm/reasoning_models.rs b/crates/ironclaw_llm/src/reasoning_models.rs similarity index 100% rename from src/llm/reasoning_models.rs rename to crates/ironclaw_llm/src/reasoning_models.rs diff --git a/src/llm/recording.rs b/crates/ironclaw_llm/src/recording.rs similarity index 98% rename from src/llm/recording.rs rename to crates/ironclaw_llm/src/recording.rs index b66360c7120..902d4ff9e17 100644 --- a/src/llm/recording.rs +++ b/crates/ironclaw_llm/src/recording.rs @@ -21,8 +21,8 @@ use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, Role, ToolCompletionRequest, ToolCompletionResponse, }; @@ -378,7 +378,7 @@ fn redact_json_value(value: &mut serde_json::Value) { } } -pub(crate) fn redact_body(body: &str) -> String { +pub fn redact_body(body: &str) -> String { // Try JSON first (most common body format for API calls). if let Ok(mut parsed) = serde_json::from_str::(body) { redact_json_value(&mut parsed); @@ -896,35 +896,23 @@ impl RecordingLlm { Arc::clone(&self.http_interceptor) as Arc } - /// Snapshot all memory documents from a workspace. + /// Append memory-document snapshot entries. /// - /// Call this once after creation, before the agent starts processing. - pub async fn snapshot_memory(&self, workspace: &crate::workspace::Workspace) { - match workspace.list_all().await { - Ok(paths) => { - let mut snapshot = self.memory_snapshot.lock().await; - for path in paths { - match workspace.read(&path).await { - Ok(doc) => { - snapshot.push(MemorySnapshotEntry { - path: doc.path, - content: doc.content, - }); - } - Err(e) => { - tracing::debug!(path = %path, error = %e, "Skipped memory doc in snapshot"); - } - } - } - tracing::info!( - documents = snapshot.len(), - "Captured memory snapshot for trace recording" - ); - } - Err(e) => { - tracing::warn!("Failed to snapshot memory for trace recording: {}", e); - } + /// Caller (main crate) collects entries from whatever workspace + /// implementation it owns and hands materialised data here. Keeping the + /// workspace type out of `ironclaw_llm` avoids a heavy reverse dep. + pub async fn snapshot_memory(&self, entries: Vec) { + if entries.is_empty() { + return; } + let mut snapshot = self.memory_snapshot.lock().await; + let added = entries.len(); + snapshot.extend(entries); + tracing::info!( + documents = snapshot.len(), + added = added, + "Captured memory snapshot for trace recording" + ); } /// Flush accumulated steps, memory snapshot, and HTTP exchanges to the output file. @@ -1356,7 +1344,7 @@ mod tests { #[test] fn from_env_returns_none_when_unset() { - let _env_lock = crate::config::helpers::lock_env(); + let _env_lock = ironclaw_common::env_helpers::lock_env(); let prior = std::env::var_os("IRONCLAW_RECORD_TRACE"); // SAFETY: Tests serialize env access with lock_env(). unsafe { std::env::remove_var("IRONCLAW_RECORD_TRACE") }; diff --git a/src/llm/registry.rs b/crates/ironclaw_llm/src/registry.rs similarity index 63% rename from src/llm/registry.rs rename to crates/ironclaw_llm/src/registry.rs index 9e2ee7f5a81..f2ea87e1598 100644 --- a/src/llm/registry.rs +++ b/crates/ironclaw_llm/src/registry.rs @@ -26,7 +26,11 @@ use serde::{Deserialize, Serialize}; /// API protocol a provider speaks. /// -/// Determines which rig-core client constructor to use. +/// Determines which provider constructor to use. Most variants identify +/// a rig-core client; the trailing four (`Bedrock`, `OpenAiCodex`, +/// `GeminiOauth`, `NearAi`) identify dedicated provider implementations +/// that don't fit the OpenAI-compat shape and have their own typed +/// config struct on [`crate::config::LlmConfig`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ProviderProtocol { @@ -39,6 +43,62 @@ pub enum ProviderProtocol { Ollama, /// GitHub Copilot API (OpenAI-compatible with token exchange). GithubCopilot, + /// DeepSeek API. Routes through rig-core's dedicated DeepSeek client, + /// which round-trips `reasoning_content` for thinking-mode models — + /// the generic OpenAI client strips it. (#3201) + DeepSeek, + /// Google Gemini native API. Routes through rig-core's dedicated Gemini + /// client, which round-trips `thought_signature` on tool calls — + /// the OpenAI-compat shim strips it. (#3225) + Gemini, + /// OpenRouter (multi-model gateway). Routes through rig-core's dedicated + /// OpenRouter client, which round-trips `reasoning`, `reasoning_details` + /// (Summary / Encrypted / Text), and per-tool-call signatures — + /// the generic OpenAI client strips all of them, breaking thinking-mode + /// tool calling on every reasoning model OpenRouter exposes (Claude with + /// thinking, OpenAI o-series, DeepSeek-R1, Gemini 2.5+, Qwen QwQ, …). + OpenRouter, + /// AWS Bedrock native Converse API (via `aws-sdk-bedrockruntime`). + /// Reads its config from [`crate::config::LlmConfig::bedrock`]. + /// Feature-gated behind `--features bedrock`. + Bedrock, + /// OpenAI Codex Responses API (ChatGPT subscription OAuth). + /// Reads its config from [`crate::config::LlmConfig::openai_codex`]. + /// + /// Wire name is `"openai_codex"` (matches the backend identifier + /// used by `LlmConfig::backend` and the gateway adapter field) — + /// the snake_case derivation `"open_ai_codex"` is also accepted as + /// an alias for forward compatibility. + #[serde(rename = "openai_codex", alias = "open_ai_codex")] + OpenAiCodex, + /// Gemini OAuth via Cloud Code API (`generativelanguage.googleapis.com` + /// or `cloudcode-pa.googleapis.com` depending on model). + /// Reads its config from [`crate::config::LlmConfig::gemini_oauth`]. + GeminiOauth, + /// NEAR AI Chat Completions with session-token or API-key auth. + /// Reads its config from [`crate::config::LlmConfig::nearai`]. + /// + /// Wire name is `"nearai"` (matches the historical backend + /// identifier and gateway adapter string) — the snake_case + /// derivation `"near_ai"` is also accepted as an alias. + #[serde(rename = "nearai", alias = "near_ai")] + NearAi, +} + +impl ProviderProtocol { + /// Returns true for protocols whose runtime configuration lives in a + /// dedicated `LlmConfig` field rather than `LlmConfig::provider` + /// (`RegistryProviderConfig`). + /// + /// Used by the resolver to decide which sub-config to populate, and by + /// the wizard to recognise non-OpenAI-shape backends without matching + /// on backend strings. + pub fn has_dedicated_config(self) -> bool { + matches!( + self, + Self::Bedrock | Self::OpenAiCodex | Self::GeminiOauth | Self::NearAi + ) + } } /// How the setup wizard should collect credentials for this provider. @@ -74,6 +134,48 @@ pub enum SetupHint { #[serde(default)] can_list_models: bool, }, + /// AWS Bedrock setup: prompt for region (default us-east-1), optional + /// cross-region prefix (us/eu/apac/global), and AWS named profile. + /// Authentication delegates to the standard AWS credential chain + /// (env, profile, instance role) — no API key collected. + AwsCredentials { + display_name: String, + /// Whether the wizard should offer the cross-region inference prompt. + #[serde(default)] + supports_cross_region: bool, + /// Whether the wizard should offer the AWS_PROFILE prompt. + #[serde(default)] + supports_profile: bool, + }, + /// OAuth device-code or PKCE flow handled by [`crate::auth::start_login`]. + /// The wizard renders a `WizardAuthPrompt` and resumes after token return. + OAuthDeviceCode { + display_name: String, + /// Identifier for [`crate::auth::AuthBackend`] in the auth facade. + backend: String, + }, + /// Read credentials from a JSON / token file on disk (e.g. Gemini Cloud + /// OAuth, where the user logs in once via `gemini auth` and we pick up + /// `~/.gemini/oauth_creds.json`). + FileBasedCredentials { + display_name: String, + #[serde(default)] + default_path_hint: Option, + }, + /// Interactive session-token login (NEAR AI). The wizard delegates to + /// the auth facade for the OAuth-style session flow. + SessionToken { + display_name: String, + /// URL where the user can manually obtain a session token. + #[serde(default)] + key_url: Option, + /// Whether this provider supports `/v1/models` listing. + /// NEAR AI's `/v1/models` endpoint works with either a session + /// token or an API key, so the configure UI should expose the + /// Fetch models button. + #[serde(default)] + can_list_models: bool, + }, } impl SetupHint { @@ -82,6 +184,10 @@ impl SetupHint { Self::ApiKey { display_name, .. } => display_name, Self::Ollama { display_name, .. } => display_name, Self::OpenAiCompatible { display_name, .. } => display_name, + Self::AwsCredentials { display_name, .. } => display_name, + Self::OAuthDeviceCode { display_name, .. } => display_name, + Self::FileBasedCredentials { display_name, .. } => display_name, + Self::SessionToken { display_name, .. } => display_name, } } @@ -96,6 +202,12 @@ impl SetupHint { Self::OpenAiCompatible { can_list_models, .. } => *can_list_models, + Self::SessionToken { + can_list_models, .. + } => *can_list_models, + Self::AwsCredentials { .. } + | Self::OAuthDeviceCode { .. } + | Self::FileBasedCredentials { .. } => false, } } @@ -103,7 +215,11 @@ impl SetupHint { match self { Self::ApiKey { secret_name, .. } => Some(secret_name), Self::OpenAiCompatible { secret_name, .. } => Some(secret_name), - Self::Ollama { .. } => None, + Self::Ollama { .. } + | Self::AwsCredentials { .. } + | Self::OAuthDeviceCode { .. } + | Self::FileBasedCredentials { .. } + | Self::SessionToken { .. } => None, } } @@ -113,6 +229,39 @@ impl SetupHint { _ => None, } } + + /// Wire-stable snake_case discriminator for this setup hint. + /// + /// Matches the `#[serde(tag = "kind", rename_all = "snake_case")]` + /// representation, so the same string can be used as a typed + /// identifier in JSON payloads (e.g. the web LLM providers payload's + /// `credential_kind` field) without going through + /// `serde_json::to_value`. Useful for callers that need to branch + /// on which credential flow a backend uses (api_key, session_token, + /// file_based_credentials, …) so the answer doesn't drift from + /// what the wizard dispatches on. + pub fn kind(&self) -> &'static str { + match self { + Self::ApiKey { .. } => "api_key", + Self::Ollama { .. } => "ollama", + Self::OpenAiCompatible { .. } => "open_ai_compatible", + Self::AwsCredentials { .. } => "aws_credentials", + Self::OAuthDeviceCode { .. } => "o_auth_device_code", + Self::FileBasedCredentials { .. } => "file_based_credentials", + Self::SessionToken { .. } => "session_token", + } + } + + /// For [`SetupHint::FileBasedCredentials`], the default path hint + /// the wizard offers (may contain `~`); `None` for other variants. + pub fn default_path_hint(&self) -> Option<&str> { + match self { + Self::FileBasedCredentials { + default_path_hint, .. + } => default_path_hint.as_deref(), + _ => None, + } + } } /// Validates unsupported_params during deserialization. @@ -220,7 +369,7 @@ impl ProviderRegistry { /// with later entries overriding earlier ones by ID/alias. pub fn load() -> Self { let builtins: Vec = - serde_json::from_str(include_str!("../../providers.json")) + serde_json::from_str(include_str!("../../../providers.json")) .expect("built-in providers.json must be valid JSON"); // safety: compile-time embedded file let mut all = builtins; @@ -297,22 +446,20 @@ impl ProviderRegistry { result } - /// Check whether a backend string is a known provider (NearAI or registry). + /// Check whether a backend string is a known provider. + /// + /// Includes both OpenAI-shape registry providers and the dedicated + /// backends (NearAI, Bedrock, OpenAI Codex, Gemini OAuth) whose + /// protocol returns `has_dedicated_config() == true`. pub fn is_known(&self, backend: &str) -> bool { - backend == "nearai" - || backend == "near_ai" - || backend == "near" - || self.find(backend).is_some() + self.find(backend).is_some() } /// Get the model env var for a backend string. /// - /// Returns the registry provider's `model_env` if found, - /// or `"NEARAI_MODEL"` for the NearAI backend. + /// Returns the registry provider's `model_env`, or `"LLM_MODEL"` for + /// unknown backends (the generic openai-compatible fallback path). pub fn model_env_var(&self, backend: &str) -> &str { - if backend == "nearai" || backend == "near_ai" || backend == "near" { - return "NEARAI_MODEL"; - } self.find(backend) .map(|def| def.model_env.as_str()) .unwrap_or("LLM_MODEL") @@ -320,7 +467,7 @@ impl ProviderRegistry { } fn user_providers_path() -> Option { - Some(crate::bootstrap::ironclaw_base_dir().join("providers.json")) + Some(ironclaw_common::paths::ironclaw_base_dir().join("providers.json")) } #[cfg(test)] @@ -330,7 +477,7 @@ mod tests { #[test] fn test_builtin_registry_loads() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert!( registry.all().len() >= 5, @@ -341,7 +488,7 @@ mod tests { #[test] fn test_find_by_id() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); let openai = registry.find("openai").expect("openai should exist"); assert_eq!(openai.id, "openai"); @@ -351,7 +498,7 @@ mod tests { #[test] fn test_find_by_alias() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); let openai = registry .find("open_ai") @@ -362,7 +509,7 @@ mod tests { #[test] fn test_find_case_insensitive() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert!(registry.find("OpenAI").is_some()); assert!(registry.find("GROQ").is_some()); @@ -372,7 +519,7 @@ mod tests { #[test] fn test_find_unknown_returns_none() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert!(registry.find("nonexistent_provider").is_none()); } @@ -380,7 +527,7 @@ mod tests { #[test] fn test_selectable_has_setup_hints() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); let selectable = registry.selectable(); assert!(!selectable.is_empty()); @@ -396,7 +543,7 @@ mod tests { #[test] fn test_user_override_wins() { let builtins: Vec = - serde_json::from_str(include_str!("../../providers.json")).unwrap(); + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); let mut all = builtins; // Simulate user overriding tinfoil with a different default model all.push(ProviderDefinition { @@ -420,10 +567,59 @@ mod tests { assert_eq!(tf.default_model, "custom-model", "user override should win"); } + /// Regression for nearai/ironclaw#3734: NEAR AI is a dual-auth + /// (session token + API key) provider whose `/v1/models` endpoint + /// accepts either credential. Its `SetupHint::SessionToken` entry + /// in `providers.json` must carry `can_list_models: true` so the + /// configure UI exposes the "Fetch available models" button. Layer + /// C (PR #3416) silently lost this when NEAR AI moved into the + /// generic registry path because `SessionToken` did not yet expose + /// the `can_list_models` field. + #[test] + fn test_nearai_setup_hint_can_list_models() { + let registry = ProviderRegistry::new( + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), + ); + let def = registry.find("nearai").expect("nearai should exist"); + let setup = def + .setup + .as_ref() + .expect("nearai must carry a SetupHint after Layer C"); + assert!( + matches!(setup, SetupHint::SessionToken { .. }), + "nearai setup hint must remain SessionToken" + ); + assert!( + setup.can_list_models(), + "nearai setup must report can_list_models=true so the \ + configure UI shows the Fetch models button (issue #3734)", + ); + } + + /// SessionToken's `can_list_models` field has to be honoured by the + /// `SetupHint::can_list_models()` accessor — the handler reads + /// through that method, not the field directly. + #[test] + fn test_session_token_can_list_models_accessor() { + let with = SetupHint::SessionToken { + display_name: "T".into(), + key_url: None, + can_list_models: true, + }; + assert!(with.can_list_models()); + + let without = SetupHint::SessionToken { + display_name: "T".into(), + key_url: None, + can_list_models: false, + }; + assert!(!without.can_list_models()); + } + #[test] fn test_model_env_var_nearai() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert_eq!(registry.model_env_var("nearai"), "NEARAI_MODEL"); assert_eq!(registry.model_env_var("near_ai"), "NEARAI_MODEL"); @@ -432,7 +628,7 @@ mod tests { #[test] fn test_model_env_var_registry_provider() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert_eq!(registry.model_env_var("groq"), "GROQ_MODEL"); assert_eq!(registry.model_env_var("tinfoil"), "TINFOIL_MODEL"); @@ -442,7 +638,7 @@ mod tests { #[test] fn test_model_env_var_unknown_fallback() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert_eq!(registry.model_env_var("nonexistent"), "LLM_MODEL"); } @@ -450,7 +646,7 @@ mod tests { #[test] fn test_is_known() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); assert!(registry.is_known("nearai")); assert!(registry.is_known("openai")); @@ -461,7 +657,7 @@ mod tests { #[test] fn test_all_providers_have_required_fields() { let providers: Vec = - serde_json::from_str(include_str!("../../providers.json")).unwrap(); + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); for def in &providers { assert!(!def.id.is_empty(), "provider must have an id"); assert!(!def.model_env.is_empty(), "{}: model_env required", def.id); @@ -478,10 +674,52 @@ mod tests { } } + /// Regression for #3201 / #3225 and the OpenRouter generalisation: + /// providers whose APIs return reasoning artifacts (DeepSeek's + /// `reasoning_content`, Gemini's `thought_signature`, OpenRouter's + /// `reasoning_details` + signatures) must NOT use the generic + /// `OpenAiCompletions` protocol. The OpenAI-compat path goes through + /// rig-core's OpenAI client, which strips those fields, breaking + /// multi-turn tool calling for every thinking-mode model these + /// providers expose. They must route through the dedicated rig-core + /// clients which round-trip the artifacts on the next request. + #[test] + fn reasoning_aware_providers_use_dedicated_protocol_not_openai_compat() { + let providers: Vec = + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); + let by_id = |id: &str| providers.iter().find(|p| p.id == id).cloned(); + + let deepseek = by_id("deepseek").expect("deepseek entry must exist"); + assert_eq!( + deepseek.protocol, + ProviderProtocol::DeepSeek, + "deepseek must use DeepSeek protocol — OpenAiCompletions strips \ + reasoning_content and breaks thinking-mode tool calling (#3201)", + ); + + let gemini = by_id("gemini").expect("gemini entry must exist"); + assert_eq!( + gemini.protocol, + ProviderProtocol::Gemini, + "gemini must use Gemini protocol — OpenAiCompletions strips \ + thought_signature and breaks tool calling on thinking models (#3225)", + ); + + let openrouter = by_id("openrouter").expect("openrouter entry must exist"); + assert_eq!( + openrouter.protocol, + ProviderProtocol::OpenRouter, + "openrouter must use OpenRouter protocol — OpenAiCompletions \ + strips reasoning_details and tool-call signatures, breaking \ + every thinking-mode model OpenRouter exposes (Claude with \ + thinking, OpenAI o-series, DeepSeek-R1, Gemini 2.5+, Qwen QwQ)", + ); + } + #[test] fn test_openai_compatible_providers_have_base_url() { let providers: Vec = - serde_json::from_str(include_str!("../../providers.json")).unwrap(); + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); for def in &providers { if def.protocol == ProviderProtocol::OpenAiCompletions && def.id != "openai" @@ -501,7 +739,7 @@ mod tests { #[test] fn test_models_filter_accessor() { let registry = ProviderRegistry::new( - serde_json::from_str(include_str!("../../providers.json")).unwrap(), + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), ); // Groq has models_filter: "chat" let groq = registry.find("groq").expect("groq should exist"); @@ -755,7 +993,7 @@ mod tests { #[test] fn test_unsupported_params_deserialized() { let providers: Vec = - serde_json::from_str(include_str!("../../providers.json")).unwrap(); + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); // Tinfoil should have temperature in unsupported_params let tinfoil = providers.iter().find(|p| p.id == "tinfoil").unwrap(); @@ -804,6 +1042,99 @@ mod tests { } } + /// The dedicated-config backends (nearai/bedrock/codex/gemini_oauth) + /// must be in the registry so: + /// - `is_known()` returns true (no string-list duplication elsewhere), + /// - `find()` resolves their aliases, + /// - `model_env_var()` returns the right env var, + /// - their protocol's `has_dedicated_config()` returns true (so the + /// OpenAI-shape resolver skips them), + /// - they appear in `selectable()` with the `SetupHint` variant the + /// wizard dispatches on (Layer C). + #[test] + fn dedicated_config_backends_are_in_registry_and_selectable() { + let registry = ProviderRegistry::new( + serde_json::from_str(include_str!("../../../providers.json")).unwrap(), + ); + + for (id, expected_protocol, alias_to_check, model_env, expected_hint) in [ + ( + "nearai", + ProviderProtocol::NearAi, + "near", + "NEARAI_MODEL", + "session_token", + ), + ( + "bedrock", + ProviderProtocol::Bedrock, + "aws_bedrock", + "BEDROCK_MODEL", + "aws_credentials", + ), + ( + "openai_codex", + ProviderProtocol::OpenAiCodex, + "codex", + "OPENAI_CODEX_MODEL", + "o_auth_device_code", // SetupHint kind, not protocol name + ), + ( + "gemini_oauth", + ProviderProtocol::GeminiOauth, + "gemini-oauth", + "GEMINI_MODEL", + "file_based_credentials", + ), + ] { + assert!(registry.is_known(id), "{id} should be is_known"); + assert!( + registry.is_known(alias_to_check), + "alias '{alias_to_check}' should resolve to {id}", + ); + let def = registry + .find(id) + .unwrap_or_else(|| panic!("{id} not found")); + assert_eq!(def.protocol, expected_protocol); + assert!( + def.protocol.has_dedicated_config(), + "{id} protocol must report has_dedicated_config()" + ); + assert_eq!(registry.model_env_var(id), model_env); + let setup = def + .setup + .as_ref() + .unwrap_or_else(|| panic!("{id} must carry a SetupHint after Layer C")); + let actual_hint = match setup { + SetupHint::ApiKey { .. } => "api_key", + SetupHint::Ollama { .. } => "ollama", + SetupHint::OpenAiCompatible { .. } => "open_ai_compatible", + SetupHint::AwsCredentials { .. } => "aws_credentials", + SetupHint::OAuthDeviceCode { .. } => "o_auth_device_code", + SetupHint::FileBasedCredentials { .. } => "file_based_credentials", + SetupHint::SessionToken { .. } => "session_token", + }; + assert_eq!( + actual_hint, expected_hint, + "{id} must use the {expected_hint} setup hint" + ); + } + + // The four dedicated-config backends now appear in `selectable()` + // and the wizard menu can iterate it without manual additions. + let selectable_ids: Vec<&str> = registry + .selectable() + .iter() + .map(|d| d.id.as_str()) + .collect(); + for id in ["nearai", "bedrock", "openai_codex", "gemini_oauth"] { + assert!( + selectable_ids.contains(&id), + "{id} must appear in selectable() after Layer C" + ); + } + } + #[test] fn test_unsupported_params_validation_rejects_invalid() { // Invalid parameter names should cause deserialization error @@ -832,7 +1163,7 @@ mod tests { // Every built-in provider with SetupHint::ApiKey must have api_key_env // set, otherwise inject_llm_keys_from_secrets can't map the secret. let providers: Vec = - serde_json::from_str(include_str!("../../providers.json")).unwrap(); + serde_json::from_str(include_str!("../../../providers.json")).unwrap(); for def in &providers { if let Some(SetupHint::ApiKey { .. }) = &def.setup { assert!( diff --git a/src/llm/response_cache.rs b/crates/ironclaw_llm/src/response_cache.rs similarity index 99% rename from src/llm/response_cache.rs rename to crates/ironclaw_llm/src/response_cache.rs index d7746f606b8..f88f050b697 100644 --- a/src/llm/response_cache.rs +++ b/crates/ironclaw_llm/src/response_cache.rs @@ -25,8 +25,8 @@ use async_trait::async_trait; use rust_decimal::Decimal; use sha2::{Digest, Sha256}; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; @@ -311,12 +311,12 @@ mod tests { use rust_decimal::Decimal; use tracing_test::traced_test; - use crate::llm::error::LlmError; - use crate::llm::provider::{ + use crate::error::LlmError; + use crate::provider::{ ChatMessage, CompletionResponse, FinishReason, ToolCompletionRequest, ToolCompletionResponse, }; - use crate::llm::response_cache::*; + use crate::response_cache::*; use crate::testing::StubLlm; /// Minimal provider stub that supports `set_model()` — used to test @@ -381,6 +381,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } diff --git a/src/llm/retry.rs b/crates/ironclaw_llm/src/retry.rs similarity index 97% rename from src/llm/retry.rs rename to crates/ironclaw_llm/src/retry.rs index cc4af3af6f9..2f57ce0e83d 100644 --- a/src/llm/retry.rs +++ b/crates/ironclaw_llm/src/retry.rs @@ -13,8 +13,8 @@ use async_trait::async_trait; use rand::Rng; use rust_decimal::Decimal; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; @@ -42,7 +42,7 @@ pub(crate) const MAX_RETRY_AFTER_SECS: u64 = 3600; /// /// See also `circuit_breaker::is_transient()` which answers a different /// question: "does this error indicate the backend is degraded?" -pub(crate) fn is_retryable(err: &LlmError) -> bool { +pub fn is_retryable(err: &LlmError) -> bool { matches!( err, LlmError::RequestFailed { .. } @@ -95,7 +95,7 @@ pub(crate) fn cap_retry_after(duration: Duration) -> Duration { /// headers can fall through to exponential backoff instead — must use /// [`parse_retry_after_value`] on the `&HeaderValue` extracted from the /// headers map directly. -pub(crate) fn parse_retry_after(header: Option<&reqwest::header::HeaderValue>) -> Duration { +pub fn parse_retry_after(header: Option<&reqwest::header::HeaderValue>) -> Duration { header .map(parse_retry_after_value) .unwrap_or(Duration::from_secs(DEFAULT_RETRY_AFTER_SECS)) @@ -108,7 +108,7 @@ pub(crate) fn parse_retry_after(header: Option<&reqwest::header::HeaderValue>) - /// fall through to [`retry_backoff_delay`] instead of the 60-second default /// that [`parse_retry_after`] returns for rate-limit semantics. Unparseable /// values still fall back to `DEFAULT_RETRY_AFTER` (60 s). -pub(crate) fn parse_retry_after_value(header: &reqwest::header::HeaderValue) -> Duration { +pub fn parse_retry_after_value(header: &reqwest::header::HeaderValue) -> Duration { let parsed = header.to_str().ok().and_then(|v| { if let Ok(secs) = v.trim().parse::() { return Some(cap_retry_after(Duration::from_secs(secs))); @@ -285,11 +285,11 @@ mod tests { use crate::testing::StubLlm; fn make_request() -> CompletionRequest { - CompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")]) + CompletionRequest::new(vec![crate::ChatMessage::user("hello")]) } fn make_tool_request() -> ToolCompletionRequest { - ToolCompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")], vec![]) + ToolCompletionRequest::new(vec![crate::ChatMessage::user("hello")], vec![]) } fn fast_config(max_retries: u32) -> RetryConfig { diff --git a/src/llm/rig_adapter.rs b/crates/ironclaw_llm/src/rig_adapter.rs similarity index 88% rename from src/llm/rig_adapter.rs rename to crates/ironclaw_llm/src/rig_adapter.rs index 3ddb5b1f6db..141299e02ef 100644 --- a/src/llm/rig_adapter.rs +++ b/crates/ironclaw_llm/src/rig_adapter.rs @@ -3,7 +3,7 @@ //! This lets us use any rig-core provider (OpenAI, Anthropic, Ollama, etc.) as an //! `Arc` without changing any of the agent, reasoning, or tool code. -use crate::llm::config::CacheRetention; +use crate::config::CacheRetention; use async_trait::async_trait; use rig::OneOrMany; use rig::completion::{ @@ -24,17 +24,17 @@ use sha2::{Digest, Sha256}; use std::collections::HashSet; use std::str::FromStr; -use crate::llm::costs; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::costs; +use crate::error::LlmError; +use crate::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCall as IronToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition as IronToolDefinition, strip_unsupported_completion_params, strip_unsupported_tool_params, }; -use crate::llm::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; +use crate::tool_schema::{ToolSchemaPolicy, shape_tool_schema}; #[cfg(test)] -use crate::llm::tool_schema::{normalize_schema_strict, serialize_json_capped}; +use crate::tool_schema::{normalize_schema_strict, serialize_json_capped}; /// Adapter that wraps a rig-core `CompletionModel` and implements `LlmProvider`. pub struct RigAdapter { @@ -150,7 +150,7 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec { + crate::Role::System => { // Concatenate system messages into preamble match preamble { Some(ref mut p) => { @@ -160,7 +160,7 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec preamble = Some(msg.content.clone()), } } - crate::llm::Role::User => { + crate::Role::User => { if msg.content_parts.is_empty() { // Skip empty user messages — some providers (e.g. Kimi) reject "content": "" if msg.content.is_empty() { @@ -171,7 +171,7 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec = vec![UserContent::text(&msg.content)]; for part in &msg.content_parts { - if let crate::llm::ContentPart::ImageUrl { image_url } = part { + if let crate::ContentPart::ImageUrl { image_url } = part { let detail = ImageDetail::from_str(&image_url.normalized_openai_detail()) .unwrap_or_default(); @@ -204,23 +204,45 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec { + crate::Role::Assistant => { if let Some(ref tool_calls) = msg.tool_calls { // Assistant message with tool calls let mut contents: Vec = Vec::new(); if !msg.content.is_empty() { contents.push(AssistantContent::text(&msg.content)); } + // Round-trip provider-emitted reasoning artifacts. rig-core's + // dedicated DeepSeek/Gemini/OpenRouter clients consume + // `AssistantContent::Reasoning` on the message and re-emit + // it as the wire-format reasoning field on the next request. + // Without this, DeepSeek/Gemini reject the follow-up turn + // with HTTP 400. See #3201, #3225. + if let Some(ref reasoning) = msg.reasoning + && !reasoning.is_empty() + { + contents.push(AssistantContent::Reasoning(rig::message::Reasoning::new( + reasoning, + ))); + } for (idx, tc) in tool_calls.iter().enumerate() { let tool_call_id = normalized_tool_call_id(Some(tc.id.as_str()), history.len() + idx); - contents.push(AssistantContent::ToolCall( - rig::message::ToolCall::new( - tool_call_id.clone(), - ToolFunction::new(tc.name.clone(), tc.arguments.clone()), - ) - .with_call_id(tool_call_id), - )); + let mut rig_tc = rig::message::ToolCall::new( + tool_call_id.clone(), + ToolFunction::new(tc.name.clone(), tc.arguments.clone()), + ) + .with_call_id(tool_call_id); + // Echo provider-emitted per-tool-call signatures back + // (Gemini's `thought_signature`). The reviewer's + // motivating example: a signed Gemini `functionCall` + // returned in turn N must carry the same signature + // when sent back in turn N+1, otherwise the API + // rejects with "Function call is missing a + // thought_signature in functionCall parts" (#3225). + if tc.signature.is_some() { + rig_tc = rig_tc.with_signature(tc.signature.clone()); + } + contents.push(AssistantContent::ToolCall(rig_tc)); } if let Ok(many) = OneOrMany::many(contents) { history.push(RigMessage::Assistant { @@ -231,6 +253,27 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec = Vec::new(); + if !msg.content.is_empty() { + contents.push(AssistantContent::text(&msg.content)); + } + contents.push(AssistantContent::Reasoning(rig::message::Reasoning::new( + reasoning, + ))); + if let Ok(many) = OneOrMany::many(contents) { + history.push(RigMessage::Assistant { + id: None, + content: many, + }); + } } else { // Skip empty assistant messages — these occur when thinking-tag stripping // leaves a blank response; sending "content": "" causes 400 on strict @@ -241,7 +284,7 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec { + crate::Role::Tool => { // Tool result message: wrap as User { ToolResult }. // Merge consecutive tool results into a single User message // so the API sees one multi-result message instead of @@ -356,13 +399,29 @@ fn convert_tool_choice(choice: Option<&str>) -> Option { } } -/// Extract text and tool calls from a rig-core completion response. +/// Extract text, tool calls, and provider-emitted reasoning artifacts from a +/// rig-core completion response. +/// +/// The returned `reasoning` is the concatenation of every +/// `AssistantContent::Reasoning` chunk in the response. Callers MUST attach it +/// to the assistant `ChatMessage` they store for the next turn — DeepSeek's +/// thinking mode and Gemini 2.5+ both reject the next request with HTTP 400 +/// when the prior message had reasoning that wasn't echoed back. See #3201, +/// #3225, and the rig-core deepseek client source where +/// `last_reasoning_content` is round-tripped onto the last assistant message +/// of the next request. fn extract_response( choice: &OneOrMany, _usage: &RigUsage, -) -> (Option, Vec, FinishReason) { +) -> ( + Option, + Vec, + FinishReason, + Option, +) { let mut text_parts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); + let mut reasoning_parts: Vec = Vec::new(); for content in choice.iter() { match content { @@ -376,9 +435,17 @@ fn extract_response( name: tc.function.name.clone(), arguments: tc.function.arguments.clone(), reasoning: None, + // Capture Gemini `thought_signature` (and any other + // per-tool-call signatures) so the next turn can echo + // them. Without this, Gemini 2.5+ rejects the next + // request with HTTP 400. See #3225. + signature: tc.signature.clone(), }); } - // Reasoning and Image variants are not mapped to IronClaw types + AssistantContent::Reasoning(r) if !r.reasoning.is_empty() => { + reasoning_parts.push(r.reasoning.join("\n")); + } + // Image variants are not mapped to IronClaw types _ => {} } } @@ -389,13 +456,19 @@ fn extract_response( Some(text_parts.join("")) }; + let reasoning = if reasoning_parts.is_empty() { + None + } else { + Some(reasoning_parts.join("\n")) + }; + let finish = if !tool_calls.is_empty() { FinishReason::ToolUse } else { FinishReason::Stop }; - (text, tool_calls, finish) + (text, tool_calls, finish, reasoning) } /// Saturate u64 to u32 for token counts. @@ -573,7 +646,7 @@ where self.strip_unsupported_completion_params(&mut request); let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let (preamble, history) = convert_messages(&messages); let mut rig_req = build_rig_request( @@ -595,7 +668,8 @@ where .await .map_err(|e| map_rig_error(&self.model_name, e))?; - let (text, _tool_calls, finish) = extract_response(&response.choice, &response.usage); + let (text, _tool_calls, finish, _reasoning) = + extract_response(&response.choice, &response.usage); let resp = CompletionResponse { content: text.unwrap_or_default(), @@ -631,7 +705,7 @@ where request.tools.iter().map(|t| t.name.clone()).collect(); let mut messages = request.messages; - crate::llm::provider::sanitize_tool_messages(&mut messages); + crate::provider::sanitize_tool_messages(&mut messages); let (preamble, history) = convert_messages(&messages); let tools = convert_tools(&request.tools); let tool_choice = convert_tool_choice(request.tool_choice.as_deref()); @@ -655,7 +729,8 @@ where .await .map_err(|e| map_rig_error(&self.model_name, e))?; - let (text, mut tool_calls, finish) = extract_response(&response.choice, &response.usage); + let (text, mut tool_calls, finish, reasoning) = + extract_response(&response.choice, &response.usage); // Normalize tool call names: some proxies prepend "proxy_" prefixes. for tc in &mut tool_calls { @@ -678,6 +753,7 @@ where finish_reason: finish, cache_read_input_tokens: saturate_u32(response.usage.cached_input_tokens), cache_creation_input_tokens: extract_cache_creation(&response.raw_response), + reasoning, }; if resp.cache_read_input_tokens > 0 { @@ -1540,6 +1616,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }; let msg = ChatMessage::assistant_with_tool_calls(Some("thinking".to_string()), vec![tc]); let messages = vec![msg]; @@ -1562,12 +1639,13 @@ mod tests { #[test] fn test_convert_messages_tool_result_without_id_gets_fallback() { let messages = vec![ChatMessage { - role: crate::llm::Role::Tool, + role: crate::Role::Tool, content: "result text".to_string(), content_parts: Vec::new(), tool_call_id: None, name: Some("search".to_string()), tool_calls: None, + reasoning: None, }]; let (_preamble, history) = convert_messages(&messages); match &history[0] { @@ -1593,8 +1671,8 @@ mod tests { fn test_convert_messages_data_url_without_detail_defaults_to_auto() { let messages = vec![ChatMessage::user_with_parts( "describe this", - vec![crate::llm::ContentPart::ImageUrl { - image_url: crate::llm::ImageUrl { + vec![crate::ContentPart::ImageUrl { + image_url: crate::ImageUrl { url: "data:image/jpeg;base64,Zm9v".to_string(), detail: None, }, @@ -1621,8 +1699,8 @@ mod tests { fn test_convert_messages_image_detail_preserves_explicit_values() { let low_messages = vec![ChatMessage::user_with_parts( "low detail", - vec![crate::llm::ContentPart::ImageUrl { - image_url: crate::llm::ImageUrl { + vec![crate::ContentPart::ImageUrl { + image_url: crate::ImageUrl { url: "https://example.com/image-low.png".to_string(), detail: Some("low".to_string()), }, @@ -1630,8 +1708,8 @@ mod tests { )]; let high_messages = vec![ChatMessage::user_with_parts( "high detail", - vec![crate::llm::ContentPart::ImageUrl { - image_url: crate::llm::ImageUrl { + vec![crate::ContentPart::ImageUrl { + image_url: crate::ImageUrl { url: "https://example.com/image-high.png".to_string(), detail: Some("high".to_string()), }, @@ -1705,7 +1783,7 @@ mod tests { fn test_extract_response_text_only() { let content = OneOrMany::one(AssistantContent::text("Hello world")); let usage = RigUsage::new(); - let (text, calls, finish) = extract_response(&content, &usage); + let (text, calls, finish, _reasoning) = extract_response(&content, &usage); assert_eq!(text, Some("Hello world".to_string())); assert!(calls.is_empty()); assert_eq!(finish, FinishReason::Stop); @@ -1716,7 +1794,7 @@ mod tests { let tc = AssistantContent::tool_call("call_1", "search", serde_json::json!({"q": "test"})); let content = OneOrMany::one(tc); let usage = RigUsage::new(); - let (text, calls, finish) = extract_response(&content, &usage); + let (text, calls, finish, _reasoning) = extract_response(&content, &usage); assert!(text.is_none()); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "search"); @@ -1730,6 +1808,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }; let messages = vec![ChatMessage::assistant_with_tool_calls(None, vec![tc])]; let (_preamble, history) = convert_messages(&messages); @@ -1762,6 +1841,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }; let messages = vec![ChatMessage::assistant_with_tool_calls(None, vec![tc])]; let (_preamble, history) = convert_messages(&messages); @@ -1796,15 +1876,17 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "test"}), reasoning: None, + signature: None, }; let assistant_msg = ChatMessage::assistant_with_tool_calls(None, vec![tc]); let tool_result_msg = ChatMessage { - role: crate::llm::Role::Tool, + role: crate::Role::Tool, content: "search results here".to_string(), content_parts: Vec::new(), tool_call_id: None, name: Some("search".to_string()), tool_calls: None, + reasoning: None, }; let messages = vec![assistant_msg, tool_result_msg]; let (_preamble, history) = convert_messages(&messages); @@ -2116,12 +2198,14 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"q": "rust"}), reasoning: None, + signature: None, }; let tc2 = IronToolCall { id: "call_b".to_string(), name: "fetch".to_string(), arguments: serde_json::json!({"url": "https://example.com"}), reasoning: None, + signature: None, }; let assistant = ChatMessage::assistant_with_tool_calls(None, vec![tc1, tc2]); let result_a = ChatMessage::tool_result("call_a", "search", "search results"); @@ -2198,9 +2282,10 @@ mod tests { #[test] fn test_empty_assistant_message_is_skipped() { let empty_asst = ChatMessage { - role: crate::llm::Role::Assistant, + role: crate::Role::Assistant, content: String::new(), tool_calls: None, + reasoning: None, tool_call_id: None, name: None, content_parts: vec![], @@ -2218,9 +2303,10 @@ mod tests { fn test_mixed_empty_and_non_empty_messages_filtered_correctly() { let user1 = ChatMessage::user("first"); let empty_asst = ChatMessage { - role: crate::llm::Role::Assistant, + role: crate::Role::Assistant, content: String::new(), tool_calls: None, + reasoning: None, tool_call_id: None, name: None, content_parts: vec![], @@ -2481,4 +2567,133 @@ mod tests { other => panic!("Expected ContextLengthExceeded, got: {other:?}"), } } + + /// Regression for #3201 / #3225 (the high-severity gap in PR #3326): + /// the dedicated rig-core DeepSeek/Gemini/OpenRouter clients only fix the + /// reasoning round-trip *inside* rig-core. IronClaw's RigAdapter sits + /// between the agent loop and rig-core, and previously dropped both + /// `AssistantContent::Reasoning` (DeepSeek `reasoning_content`) and + /// per-tool-call `signature` (Gemini `thought_signature`) on the response → + /// IronClaw conversion. On the next request it rebuilt rig messages + /// without either field, so the provider rejected the follow-up turn. + /// + /// This test simulates a 2-turn tool loop: + /// 1. extract a rig response carrying both reasoning and a signed tool call + /// 2. round-trip those onto an IronClaw `ChatMessage` + /// 3. convert that message back into rig format + /// 4. assert both reasoning and signature appear on the rebuilt rig message + /// + /// If any layer drops a field, the next turn would fail with HTTP 400. + #[test] + fn reasoning_and_signature_round_trip_through_chat_message() { + // --- Turn 1: provider returned reasoning + signed tool call --- + let rig_response = OneOrMany::many(vec![ + AssistantContent::Reasoning(rig::message::Reasoning::new( + "Let me check the weather first.", + )), + AssistantContent::ToolCall( + rig::message::ToolCall::new( + "call_abc123".to_string(), + ToolFunction::new( + "get_weather".to_string(), + serde_json::json!({"city": "London"}), + ), + ) + .with_signature(Some("thought-sig-deadbeef".to_string())), + ), + ]) + .unwrap(); + let usage = RigUsage::new(); + let (text, tool_calls, finish, reasoning) = extract_response(&rig_response, &usage); + + assert_eq!(finish, FinishReason::ToolUse); + assert_eq!(text, None); + assert_eq!( + reasoning.as_deref(), + Some("Let me check the weather first."), + "extract_response must capture AssistantContent::Reasoning so the \ + next request can echo DeepSeek's reasoning_content (#3201)", + ); + assert_eq!(tool_calls.len(), 1); + assert_eq!( + tool_calls[0].signature.as_deref(), + Some("thought-sig-deadbeef"), + "extract_response must capture ToolCall.signature so the next \ + request can echo Gemini's thought_signature (#3225)", + ); + + // --- IronClaw stores the assistant message + tool result --- + let assistant = ChatMessage::assistant_with_tool_calls(text, tool_calls) + .with_reasoning(reasoning.clone()); + let tool_result = + ChatMessage::tool_result("call_abc123", "get_weather", "{\"temp_c\": 14}"); + + // --- Turn 2: IronClaw rebuilds the rig request from stored messages --- + let messages = vec![ + ChatMessage::user("What's the weather?"), + assistant, + tool_result, + ]; + let (_preamble, history) = convert_messages(&messages); + + // The rebuilt rig assistant message must carry both reasoning and + // signature; otherwise the dedicated DeepSeek/Gemini/OpenRouter rig + // clients would emit an empty `reasoning_content` / unsigned + // `functionCall` and the API would reject with HTTP 400. + let assistant_msg = history + .iter() + .find(|m| matches!(m, RigMessage::Assistant { .. })) + .expect("rebuilt rig history should contain the assistant message"); + let RigMessage::Assistant { content, .. } = assistant_msg else { + unreachable!() + }; + + let mut found_reasoning = false; + let mut found_signed_tool_call = false; + for c in content.iter() { + match c { + AssistantContent::Reasoning(r) => { + assert_eq!(r.reasoning, vec!["Let me check the weather first."]); + found_reasoning = true; + } + AssistantContent::ToolCall(tc) => { + assert_eq!( + tc.signature.as_deref(), + Some("thought-sig-deadbeef"), + "rebuilt rig tool call must carry the original \ + thought_signature (#3225)", + ); + found_signed_tool_call = true; + } + _ => {} + } + } + assert!( + found_reasoning, + "convert_messages must emit AssistantContent::Reasoning when \ + ChatMessage carries reasoning — without this, DeepSeek thinking \ + mode rejects the next turn (#3201)", + ); + assert!( + found_signed_tool_call, + "convert_messages must propagate ToolCall.signature when \ + rebuilding rig tool calls — without this, Gemini 2.5+ rejects \ + the next turn (#3225)", + ); + } + + /// `with_reasoning` must drop empty/whitespace-only strings rather than + /// echoing `reasoning_content: ""` (some strict-mode providers reject + /// empty reasoning fields, and an empty echo carries no signal anyway). + #[test] + fn chat_message_with_reasoning_drops_empty_input() { + let msg = ChatMessage::assistant("hi").with_reasoning(Some(String::new())); + assert!(msg.reasoning.is_none()); + let msg = ChatMessage::assistant("hi").with_reasoning(Some(" ".to_string())); + assert!(msg.reasoning.is_none()); + let msg = ChatMessage::assistant("hi").with_reasoning(None); + assert!(msg.reasoning.is_none()); + let msg = ChatMessage::assistant("hi").with_reasoning(Some("real".to_string())); + assert_eq!(msg.reasoning.as_deref(), Some("real")); + } } diff --git a/src/llm/runtime.rs b/crates/ironclaw_llm/src/runtime.rs similarity index 98% rename from src/llm/runtime.rs rename to crates/ironclaw_llm/src/runtime.rs index 8d417b12650..bf42d50aac0 100644 --- a/src/llm/runtime.rs +++ b/crates/ironclaw_llm/src/runtime.rs @@ -29,8 +29,8 @@ use std::sync::{Arc, Mutex, OnceLock, RwLock}; use async_trait::async_trait; use rust_decimal::Decimal; -use crate::llm::error::LlmError; -use crate::llm::provider::{ +use crate::error::LlmError; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; @@ -283,12 +283,12 @@ impl LlmReloadHandle { /// Reloads are serialized so two concurrent callers cannot race. pub async fn reload( &self, - config: &crate::llm::LlmConfig, - session: Arc, + config: &crate::LlmConfig, + session: Arc, ) -> Result<(), LlmError> { let _guard = self.reload_lock.lock().await; - let components = crate::llm::build_provider_chain_components(config, session).await?; + let components = crate::build_provider_chain_components(config, session).await?; self.primary.swap(components.primary); @@ -315,7 +315,7 @@ impl LlmReloadHandle { #[cfg(test)] mod tests { use super::*; - use crate::llm::provider::{CompletionRequest, ToolCompletionRequest}; + use crate::provider::{CompletionRequest, ToolCompletionRequest}; use std::sync::RwLock as StdRwLock; /// Simple stub that supports `set_model()` so we can exercise the diff --git a/src/llm/session.rs b/crates/ironclaw_llm/src/session.rs similarity index 66% rename from src/llm/session.rs rename to crates/ironclaw_llm/src/session.rs index c0d5c888ab7..ceb52331f0d 100644 --- a/src/llm/session.rs +++ b/crates/ironclaw_llm/src/session.rs @@ -7,15 +7,17 @@ use std::path::PathBuf; use std::sync::Arc; -use crate::llm::oauth_helpers::OAUTH_CALLBACK_PORT; - use chrono::{DateTime, Utc}; use reqwest::Client; use secrecy::SecretString; use serde::{Deserialize, Serialize}; use tokio::sync::{Mutex, RwLock}; -use crate::llm::error::LlmError; +use crate::error::LlmError; +use crate::host::{ + NoopKeyPersistor, NoopSessionRenewer, SharedSessionDb, SharedSessionKeyPersistor, + SharedSessionRenewer, SharedSessionSecrets, +}; /// Session data persisted to disk. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -47,6 +49,11 @@ impl Default for SessionConfig { } /// Manages NEAR AI session tokens with persistence and automatic renewal. +/// +/// The DB / encrypted-secrets / interactive-renewal / env-persist hooks are +/// abstracted behind traits in [`crate::host`] so this crate doesn't need to +/// depend on the embedding application. Headless deployments can leave them +/// unset; the CLI build wires in real impls. pub struct SessionManager { config: SessionConfig, client: Client, @@ -55,17 +62,20 @@ pub struct SessionManager { /// Prevents thundering herd during concurrent 401s. renewal_lock: Mutex<()>, /// Optional database store for persisting session to the settings table. - store: RwLock>>, + store: RwLock>, /// User ID for DB settings (default: "default"). user_id: RwLock, /// Optional encrypted secrets store — preferred over plaintext settings when present. - secrets: RwLock>>, + secrets: RwLock>, + /// Interactive renewal hook. Defaults to a no-op that returns `SessionRenewalFailed`. + renewer: RwLock, + /// Persistor for one-shot API key entry (runtime env + .env file). Defaults to no-op. + key_persistor: RwLock, } impl SessionManager { - /// Create a new session manager and load any existing token from disk. - pub fn new(config: SessionConfig) -> Self { - let manager = Self { + fn empty(config: SessionConfig) -> Self { + Self { config, client: Client::builder() .timeout(std::time::Duration::from_secs(30)) @@ -75,23 +85,27 @@ impl SessionManager { renewal_lock: Mutex::new(()), store: RwLock::new(None), // Placeholder; overwritten by attach_store() with the real owner_id at startup. - // TODO(ownership): thread owner_id through SessionManager constructors user_id: RwLock::new("".to_string()), secrets: RwLock::new(None), - }; + renewer: RwLock::new(Arc::new(NoopSessionRenewer) as SharedSessionRenewer), + key_persistor: RwLock::new(Arc::new(NoopKeyPersistor) as SharedSessionKeyPersistor), + } + } + + /// Create a new session manager and load any existing token from disk. + pub fn new(config: SessionConfig) -> Self { + let manager = Self::empty(config); // Try to load existing session synchronously during construction if let Ok(data) = std::fs::read_to_string(&manager.config.session_path) && let Ok(session) = serde_json::from_str::(&data) + && let Ok(mut guard) = manager.token.try_write() { - // We can't await here, so we use try_write - if let Ok(mut guard) = manager.token.try_write() { - *guard = Some(SecretString::from(session.session_token)); - tracing::info!( - "Loaded session token from {}", - manager.config.session_path.display() - ); - } + *guard = Some(SecretString::from(session.session_token)); + tracing::info!( + "Loaded session token from {}", + manager.config.session_path.display() + ); } manager @@ -99,20 +113,7 @@ impl SessionManager { /// Create a session manager and load token asynchronously. pub async fn new_async(config: SessionConfig) -> Self { - let manager = Self { - config, - client: Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build() - .unwrap_or_else(|_| Client::new()), - token: RwLock::new(None), - renewal_lock: Mutex::new(()), - store: RwLock::new(None), - // Placeholder; overwritten by attach_store() with the real owner_id at startup. - // TODO(ownership): thread owner_id through SessionManager constructors - user_id: RwLock::new("".to_string()), - secrets: RwLock::new(None), - }; + let manager = Self::empty(config); if let Err(e) = manager.load_session().await { tracing::debug!("No existing session found: {}", e); @@ -126,7 +127,7 @@ impl SessionManager { /// When a store is attached, session tokens are saved to the `settings` /// table (key: `nearai.session_token`) in addition to the disk file. /// On load, DB is preferred over disk. - pub async fn attach_store(&self, store: Arc, user_id: &str) { + pub async fn attach_store(&self, store: SharedSessionDb, user_id: &str) { *self.store.write().await = Some(store); *self.user_id.write().await = user_id.to_string(); @@ -141,10 +142,7 @@ impl SessionManager { /// When attached, `save_session` writes to the secrets store in addition /// to the disk file, and `load_session_from_db` prefers the secrets store /// over the plaintext settings table. - pub async fn attach_secrets( - &self, - secrets: Arc, - ) { + pub async fn attach_secrets(&self, secrets: SharedSessionSecrets) { *self.secrets.write().await = Some(secrets); // Try to load from encrypted secrets (preferred over settings table) @@ -153,6 +151,44 @@ impl SessionManager { } } + /// Attach an interactive renewer used when a session expires. + /// + /// Headless deployments can skip this; the default `NoopSessionRenewer` + /// returns `SessionRenewalFailed` and the caller is expected to set + /// `NEARAI_SESSION_TOKEN` or `NEARAI_API_KEY` ahead of time. + pub async fn attach_renewer(&self, renewer: SharedSessionRenewer) { + *self.renewer.write().await = renewer; + } + + /// Attach a persistor used by the API-key entry path inside a renewer. + pub async fn attach_key_persistor(&self, persistor: SharedSessionKeyPersistor) { + *self.key_persistor.write().await = persistor; + } + + /// Read-only access to the configured auth base URL (used by renewer impls). + pub fn auth_base_url(&self) -> &str { + &self.config.auth_base_url + } + + /// Returns the persistor most recently set via `attach_key_persistor`. + pub async fn key_persistor(&self) -> SharedSessionKeyPersistor { + Arc::clone(&*self.key_persistor.read().await) + } + + /// Public hook used by `SessionRenewer` impls to write a freshly received + /// session token back through the same disk + DB + secrets pipeline as + /// the internal flow. + pub async fn save_session_for_renewer( + &self, + token: &str, + auth_provider: Option<&str>, + ) -> Result<(), LlmError> { + self.save_session(token, auth_provider).await?; + let mut guard = self.token.write().await; + *guard = Some(SecretString::from(token.to_string())); + Ok(()) + } + /// Get the current session token, returning an error if not authenticated. pub async fn get_token(&self) -> Result { let guard = self.token.read().await; @@ -166,18 +202,16 @@ impl SessionManager { self.token.read().await.is_some() } - /// Ensure we have a valid session, triggering login flow if needed. + /// Ensure we have a valid session, triggering the renewer if needed. /// - /// If no token exists, triggers the OAuth login flow. If a token exists, - /// validates it by making a test API call. If validation fails, triggers - /// the login flow. + /// If no token exists, asks the registered `SessionRenewer` for one. If a + /// token exists, validates it by hitting `/v1/users/me`. If validation + /// fails, asks the renewer for a fresh token. pub async fn ensure_authenticated(&self) -> Result<(), LlmError> { if !self.has_token().await { - // No token, need to authenticate - return self.initiate_login().await; + return self.run_renewer().await; } - // Token exists, validate it by calling /v1/users/me tracing::debug!("Validating session..."); match self.validate_token().await { Ok(()) => { @@ -186,11 +220,16 @@ impl SessionManager { } Err(e) => { tracing::info!("Session expired or invalid: {}", e); - self.initiate_login().await + self.run_renewer().await } } } + async fn run_renewer(&self) -> Result<(), LlmError> { + let renewer = Arc::clone(&*self.renewer.read().await); + renewer.renew(self).await + } + /// Validate the current token by calling the /v1/users/me endpoint. async fn validate_token(&self) -> Result<(), LlmError> { use secrecy::ExposeSecret; @@ -221,7 +260,7 @@ impl SessionManager { let status = response.status(); let body = response.text().await.unwrap_or_default(); - let preview = crate::agent::truncate_for_preview(&body, 200); + let preview = ironclaw_common::truncate_for_preview(&body, 200); Err(LlmError::SessionRenewalFailed { provider: "nearai".to_string(), reason: format!("Validation failed: HTTP {status}: {preview}"), @@ -230,223 +269,13 @@ impl SessionManager { /// Handle an authentication failure (401 response). /// - /// Triggers the OAuth login flow to get a new session token. + /// Acquires the renewal lock to prevent a thundering herd, then asks the + /// registered `SessionRenewer` for a fresh token. pub async fn handle_auth_failure(&self) -> Result<(), LlmError> { - // Acquire renewal lock to prevent thundering herd let _guard = self.renewal_lock.lock().await; tracing::info!("Session expired or invalid, re-authenticating..."); - self.initiate_login().await - } - - /// Start the login flow. - /// - /// Shows the auth method menu FIRST (before binding any listener), so - /// that the API-key path can skip network binding entirely. This is - /// important for remote/headless servers where `127.0.0.1` is - /// unreachable from the user's browser. - /// - /// For OAuth paths (GitHub, Google): - /// 1. Bind the callback listener - /// 2. Print the auth URL and attempt to open browser - /// 3. Wait for OAuth callback with session token - /// 4. Save and return the token - /// - /// For NEAR AI Cloud API key: - /// 1. Prompt user for API key from cloud.near.ai - /// 2. Set NEARAI_API_KEY env var and save to bootstrap .env - /// 3. No session token saved (different auth model) - async fn initiate_login(&self) -> Result<(), LlmError> { - use crate::llm::oauth_helpers; - - let cb_url = oauth_helpers::callback_url(); - let host = oauth_helpers::callback_host(); - - // Show auth provider menu BEFORE binding the listener - println!(); - println!("╔════════════════════════════════════════════════════════════════╗"); - println!("║ NEAR AI Authentication ║"); - println!("╠════════════════════════════════════════════════════════════════╣"); - println!("║ Choose an authentication method: ║"); - println!("║ ║"); - println!("║ [1] GitHub (requires localhost browser access) ║"); - println!("║ [2] Google (requires localhost browser access) ║"); - println!("║ [3] NEAR Wallet (coming soon) ║"); - println!("║ [4] NEAR AI Cloud API key ║"); - println!("║ ║"); - println!("╚════════════════════════════════════════════════════════════════╝"); - println!(); - print!("Enter choice [1-4]: "); - - // Flush stdout to ensure prompt is displayed - use std::io::Write; - std::io::stdout().flush().ok(); - - // Read user choice - let mut choice = String::new(); - std::io::stdin() - .read_line(&mut choice) - .map_err(|e| LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: format!("Failed to read input: {}", e), - })?; - - match choice.trim() { - "4" => return self.api_key_login().await, - "3" => { - println!(); - println!("NEAR Wallet authentication is not yet implemented."); - println!("Please use GitHub or Google for now."); - return Err(LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: "NEAR Wallet auth not yet implemented".to_string(), - }); - } - "1" | "" | "2" => {} // handled below after listener bind - other => { - return Err(LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: format!("Invalid choice: {}", other), - }); - } - } - - // Warn about plain-HTTP token transmission only for OAuth paths (1, 2) - // where the callback URL actually carries the session token. - if !oauth_helpers::is_loopback_host(&host) { - println!(); - println!("Warning: OAuth callback is using plain HTTP to a remote host ({host})."); - println!(" The session token will be transmitted unencrypted."); - println!(" Consider SSH port forwarding instead:"); - println!( - " ssh -L {OAUTH_CALLBACK_PORT}:127.0.0.1:{OAUTH_CALLBACK_PORT} user@{host}" - ); - } - - // OAuth paths: bind the callback listener now - let listener = oauth_helpers::bind_callback_listener().await.map_err(|e| { - LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: e.to_string(), - } - })?; - - let (auth_provider, auth_url) = match choice.trim() { - "2" => { - let url = format!( - "{}/v1/auth/google?frontend_callback={}", - self.config.auth_base_url, - urlencoding::encode(&cb_url) - ); - ("google", url) - } - _ => { - // "1" or "" (default) - let url = format!( - "{}/v1/auth/github?frontend_callback={}", - self.config.auth_base_url, - urlencoding::encode(&cb_url) - ); - ("github", url) - } - }; - - println!(); - println!("Opening {} authentication...", auth_provider); - println!(); - println!(" {}", auth_url); - println!(); - - // Try to open browser automatically - if let Err(e) = open::that(&auth_url) { - tracing::debug!("Could not open browser automatically: {}", e); - println!("(Could not open browser automatically, please copy the URL above)"); - } else { - println!("(Opening browser...)"); - } - println!(); - println!("Waiting for authentication..."); - - // The NEAR AI API redirects to: {frontend_callback}/auth/callback?token=X&... - let session_token = - oauth_helpers::wait_for_callback(listener, "/auth/callback", "token", "NEAR AI", None) - .await - .map_err(|e| LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: e.to_string(), - })?; - - let auth_provider = Some(auth_provider.to_string()); - - // Save the token - self.save_session(&session_token, auth_provider.as_deref()) - .await?; - - // Update in-memory token - { - let mut guard = self.token.write().await; - *guard = Some(SecretString::from(session_token)); - } - - println!(); - println!("✓ Authentication successful!"); - println!(); - - Ok(()) - } - - /// NEAR AI Cloud API key entry flow. - /// - /// Prompts the user to enter a NEAR AI Cloud API key from - /// cloud.near.ai. The key is stored in the thread-safe runtime - /// env overlay (via `set_runtime_env`) so `LlmConfig::resolve()` - /// auto-selects ChatCompletions mode, and persisted to - /// `~/.ironclaw/.env` for survival across restarts. - /// No session token is saved and no `/v1/users/me` validation is - /// performed (different auth model). - async fn api_key_login(&self) -> Result<(), LlmError> { - println!(); - println!("NEAR AI Cloud API key"); - println!("─────────────────────"); - println!(); - println!(" 1. Open https://cloud.near.ai in your browser"); - println!(" 2. Sign in and navigate to API Keys"); - println!(" 3. Create or copy an existing API key"); - println!(); - - let key_secret = - crate::setup::secret_input("API key").map_err(|e| LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: format!("Failed to read input: {}", e), - })?; - - use secrecy::ExposeSecret; - let key = key_secret.expose_secret().to_string(); - if key.is_empty() { - return Err(LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: "API key cannot be empty".to_string(), - }); - } - - // Make the key visible to Config resolution and `env_or_override()` - // callers for the remainder of this process. Uses a thread-safe - // overlay instead of `std::env::set_var`, which is UB in - // multi-threaded programs (Rust 1.82+). - crate::config::helpers::set_runtime_env("NEARAI_API_KEY", &key); - - // Persist to ~/.ironclaw/.env so the key survives restarts - // (bootstrap layer — available before DB is connected). - // Uses upsert to avoid clobbering existing bootstrap vars. - if let Err(e) = crate::bootstrap::upsert_bootstrap_var("NEARAI_API_KEY", &key) { - tracing::warn!("Failed to save API key to bootstrap .env: {}", e); - } - - println!(); - crate::setup::print_success("NEAR AI Cloud API key saved."); - println!(); - - Ok(()) + self.run_renewer().await } /// Save session data to disk and (if available) to the database. @@ -512,10 +341,15 @@ impl SessionManager { let user_id = self.user_id.read().await.clone(); let session_json_str = serde_json::to_string(&session).unwrap_or_else(|_| token.to_string()); - let params = - crate::secrets::CreateSecretParams::new("nearai_session_token", session_json_str) - .with_provider("nearai"); - if let Err(e) = secrets.create(&user_id, params).await { + if let Err(e) = secrets + .create( + &user_id, + "nearai_session_token", + session_json_str, + Some("nearai"), + ) + .await + { tracing::warn!("Failed to save session to encrypted secrets: {}", e); } else { tracing::debug!("Session saved to encrypted secrets store"); @@ -620,12 +454,14 @@ impl SessionManager { reason: format!("Secrets lookup failed: {}", e), })?; - let session: SessionData = serde_json::from_str(decrypted.expose()).map_err(|e| { - LlmError::SessionRenewalFailed { - provider: "nearai".to_string(), - reason: format!("Failed to parse session from secrets: {}", e), - } - })?; + use secrecy::ExposeSecret as _; + let session: SessionData = + serde_json::from_str(decrypted.expose_secret()).map_err(|e| { + LlmError::SessionRenewalFailed { + provider: "nearai".to_string(), + reason: format!("Failed to parse session from secrets: {}", e), + } + })?; let mut guard = self.token.write().await; *guard = Some(SecretString::from(session.session_token)); @@ -699,9 +535,7 @@ pub async fn create_session_manager(config: SessionConfig) -> Arc SmartRoutingConfig { @@ -1585,7 +1585,7 @@ mod tests { content: "I'm not sure.".to_string(), input_tokens: 10, output_tokens: 5, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: crate::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }; @@ -1598,7 +1598,7 @@ mod tests { content: "".to_string(), input_tokens: 10, output_tokens: 0, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: crate::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }; @@ -1611,7 +1611,7 @@ mod tests { content: "Yes.".to_string(), input_tokens: 10, output_tokens: 1, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: crate::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }; @@ -1625,7 +1625,7 @@ mod tests { .to_string(), input_tokens: 10, output_tokens: 20, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: crate::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }; diff --git a/src/testing/fault_injection.rs b/crates/ironclaw_llm/src/testing/fault_injection.rs similarity index 98% rename from src/testing/fault_injection.rs rename to crates/ironclaw_llm/src/testing/fault_injection.rs index f9f8d23bd9b..2c9a673b383 100644 --- a/src/testing/fault_injection.rs +++ b/crates/ironclaw_llm/src/testing/fault_injection.rs @@ -20,7 +20,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; -use crate::llm::error::LlmError; +use crate::error::LlmError; /// The type of fault to inject. #[derive(Debug, Clone)] @@ -152,11 +152,10 @@ impl FaultInjector { /// Panics if `error_rate` is not in `0.0..=1.0` or is NaN. /// /// The seed is guarded against zero, which is a fixed point for xorshift. + #[rustfmt::skip] pub fn random(error_rate: f64, fault: FaultType, seed: u64) -> Self { - assert!( - !error_rate.is_nan() && (0.0..=1.0).contains(&error_rate), - "error_rate must be in 0.0..=1.0 and not NaN, got {error_rate}" - ); + let valid = !error_rate.is_nan() && (0.0..=1.0).contains(&error_rate); + assert!(valid, "error_rate must be in 0.0..=1.0 and not NaN, got {error_rate}"); // safety: test-only helper gated on the `testing` cargo feature let seed = if seed == 0 { 1 } else { seed }; Self { actions: Vec::new(), diff --git a/crates/ironclaw_llm/src/testing/mod.rs b/crates/ironclaw_llm/src/testing/mod.rs new file mode 100644 index 00000000000..b16c620bf09 --- /dev/null +++ b/crates/ironclaw_llm/src/testing/mod.rs @@ -0,0 +1,257 @@ +//! Test helpers for `ironclaw_llm`. +//! +//! Gated behind the `testing` feature (or `cfg(test)`). Downstream test code +//! can opt in by depending on this crate with `features = ["testing"]`. + +pub mod fault_injection; + +// ── Config builders ───────────────────────────────────────────────────── + +/// Build an [`crate::config::LlmConfig`] wired to the NEAR AI backend with +/// the given model name and all caps/timeouts/cache settings collapsed to +/// test-friendly defaults (no retries, no circuit breaker, no response +/// cache, no failover). +/// +/// Designed for hot-reload, smart-routing, and provider-chain tests where +/// the only field a test cares about is the active model. Callers that +/// want different behaviour should clone the result and override fields. +/// +/// Replaces the inline `LlmConfig { ... NearAiConfig { ... } }` literal +/// that several downstream tests had duplicated — using this helper keeps +/// them shielded from `NearAiConfig` field churn. +pub fn nearai_test_config(model: impl Into) -> crate::config::LlmConfig { + crate::config::LlmConfig { + backend: "nearai".to_string(), + session: crate::session::SessionConfig::default(), + nearai: crate::config::NearAiConfig { + model: model.into(), + cheap_model: None, + base_url: "https://api.near.ai".to_string(), + api_key: None, + fallback_model: None, + max_retries: 0, + circuit_breaker_threshold: None, + circuit_breaker_recovery_secs: 30, + response_cache_enabled: false, + response_cache_ttl_secs: 3600, + response_cache_max_entries: 1000, + failover_cooldown_secs: 300, + failover_cooldown_threshold: 3, + smart_routing_cascade: true, + }, + provider: None, + bedrock: None, + gemini_oauth: None, + request_timeout_secs: 120, + cheap_model: None, + smart_routing_cascade: true, + openai_codex: None, + max_retries: 0, + circuit_breaker_threshold: None, + circuit_breaker_recovery_secs: 30, + response_cache_enabled: false, + response_cache_ttl_secs: 3600, + response_cache_max_entries: 1000, + } +} + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use async_trait::async_trait; +use rust_decimal::Decimal; + +use crate::error::LlmError; +use crate::provider::{ + CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, +}; + +// ── Session test constants ────────────────────────────────────────────── + +/// Generic session token for persistence tests. +pub const TEST_SESSION_TOKEN: &str = "test_token_123"; + +/// NEAR AI session token variant A. +pub const TEST_SESSION_NEARAI_ABC: &str = "sess_abc123"; + +/// NEAR AI session token variant B. +pub const TEST_SESSION_NEARAI_XYZ: &str = "sess_xyz789"; + +// ── StubLlm ───────────────────────────────────────────────────────────── + +/// What kind of error the stub should produce when failing. +#[derive(Clone, Copy, Debug)] +pub enum StubErrorKind { + /// Transient/retryable error (`LlmError::RequestFailed`). + Transient, + /// Non-transient error (`LlmError::ContextLengthExceeded`). + NonTransient, +} + +/// A configurable LLM provider stub for tests. +/// +/// Supports: +/// - Fixed response content +/// - Call counting via [`calls()`](Self::calls) +/// - Runtime failure toggling via [`set_failing()`](Self::set_failing) +/// - Configurable error kinds (transient vs non-transient) +/// +/// Use this in tests instead of creating ad-hoc stub implementations. +pub struct StubLlm { + model_name: String, + response: String, + call_count: AtomicU32, + should_fail: AtomicBool, + error_kind: StubErrorKind, + /// Optional fault injector for fine-grained failure control. + /// When set, takes precedence over the `should_fail` / `error_kind` fields. + fault_injector: Option>, +} + +impl StubLlm { + /// Create a new stub that returns the given response. + pub fn new(response: impl Into) -> Self { + Self { + model_name: "stub-model".to_string(), + response: response.into(), + call_count: AtomicU32::new(0), + should_fail: AtomicBool::new(false), + error_kind: StubErrorKind::Transient, + fault_injector: None, + } + } + + /// Create a stub that always fails with a transient error. + pub fn failing(name: impl Into) -> Self { + Self { + model_name: name.into(), + response: String::new(), + call_count: AtomicU32::new(0), + should_fail: AtomicBool::new(true), + error_kind: StubErrorKind::Transient, + fault_injector: None, + } + } + + /// Create a stub that always fails with a non-transient error. + pub fn failing_non_transient(name: impl Into) -> Self { + Self { + model_name: name.into(), + response: String::new(), + call_count: AtomicU32::new(0), + should_fail: AtomicBool::new(true), + error_kind: StubErrorKind::NonTransient, + fault_injector: None, + } + } + + /// Set the model name. + pub fn with_model_name(mut self, name: impl Into) -> Self { + self.model_name = name.into(); + self + } + + /// Get the number of times `complete` or `complete_with_tools` was called. + pub fn calls(&self) -> u32 { + self.call_count.load(Ordering::Relaxed) + } + + /// Attach a fault injector for fine-grained failure control. + /// + /// When set, the injector's `next_action()` is consulted on every call, + /// taking precedence over the `should_fail` / `error_kind` fields. + pub fn with_fault_injector(mut self, injector: Arc) -> Self { + self.fault_injector = Some(injector); + self + } + + /// Toggle whether calls should fail at runtime. + pub fn set_failing(&self, fail: bool) { + self.should_fail.store(fail, Ordering::Relaxed); + } + + /// Check the fault injector or should_fail flag, returning an error if + /// the call should fail, or None if it should succeed. + async fn check_faults(&self) -> Option { + if let Some(ref injector) = self.fault_injector { + match injector.next_action() { + fault_injection::FaultAction::Fail(fault) => { + return Some(fault.to_llm_error(&self.model_name)); + } + fault_injection::FaultAction::Delay(duration) => { + tokio::time::sleep(duration).await; + } + fault_injection::FaultAction::Succeed => {} + } + } else if self.should_fail.load(Ordering::Relaxed) { + return Some(self.make_error()); + } + None + } + + fn make_error(&self) -> LlmError { + match self.error_kind { + StubErrorKind::Transient => LlmError::RequestFailed { + provider: self.model_name.clone(), + reason: "server error".to_string(), + }, + StubErrorKind::NonTransient => LlmError::ContextLengthExceeded { + used: 100_000, + limit: 50_000, + }, + } + } +} + +impl Default for StubLlm { + fn default() -> Self { + Self::new("OK") + } +} + +#[async_trait] +impl LlmProvider for StubLlm { + fn model_name(&self) -> &str { + &self.model_name + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete(&self, _request: CompletionRequest) -> Result { + self.call_count.fetch_add(1, Ordering::Relaxed); + if let Some(err) = self.check_faults().await { + return Err(err); + } + Ok(CompletionResponse { + content: self.response.clone(), + input_tokens: 10, + output_tokens: 5, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + _request: ToolCompletionRequest, + ) -> Result { + self.call_count.fetch_add(1, Ordering::Relaxed); + if let Some(err) = self.check_faults().await { + return Err(err); + } + Ok(ToolCompletionResponse { + content: Some(self.response.clone()), + tool_calls: Vec::new(), + input_tokens: 10, + output_tokens: 5, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + reasoning: None, + }) + } +} diff --git a/src/llm/token_refreshing.rs b/crates/ironclaw_llm/src/token_refreshing.rs similarity index 95% rename from src/llm/token_refreshing.rs rename to crates/ironclaw_llm/src/token_refreshing.rs index c39ad3243cc..2db07a79d8b 100644 --- a/src/llm/token_refreshing.rs +++ b/crates/ironclaw_llm/src/token_refreshing.rs @@ -13,9 +13,9 @@ use rust_decimal::Decimal; use secrecy::ExposeSecret; use crate::error::LlmError; -use crate::llm::openai_codex_provider::OpenAiCodexProvider; -use crate::llm::openai_codex_session::OpenAiCodexSessionManager; -use crate::llm::provider::{ +use crate::openai_codex_provider::OpenAiCodexProvider; +use crate::openai_codex_session::OpenAiCodexSessionManager; +use crate::provider::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; @@ -147,8 +147,8 @@ impl LlmProvider for TokenRefreshingProvider { #[cfg(test)] mod tests { use super::*; - use crate::llm::codex_test_helpers::{make_test_jwt, test_codex_config}; - use crate::llm::openai_codex_session::OpenAiCodexSessionManager; + use crate::codex_test_helpers::{make_test_jwt, test_codex_config}; + use crate::openai_codex_session::OpenAiCodexSessionManager; use tempfile::tempdir; fn make_provider_and_session() -> (TokenRefreshingProvider, tempfile::TempDir) { diff --git a/src/llm/tool_schema.rs b/crates/ironclaw_llm/src/tool_schema.rs similarity index 97% rename from src/llm/tool_schema.rs rename to crates/ironclaw_llm/src/tool_schema.rs index 735e2c7327d..010528f7cd6 100644 --- a/src/llm/tool_schema.rs +++ b/crates/ironclaw_llm/src/tool_schema.rs @@ -56,18 +56,12 @@ fn normalize_schema( normalize_schema_recursive(&mut schema, strict_objects); } - if strict_objects - && let Err(violations) = - crate::tools::schema_validator::validate_strict_schema(&schema, "") - { - tracing::debug!( - violations = ?violations, - "normalize_schema_strict output has {} strict-mode violation(s) — \ - the tool is still usable but the LLM provider may reject the schema", - violations.len() - ); - } - + // Diagnostic strict-mode validation lives in the main crate's + // `tools::schema_validator` and is exercised in CI tests there. The + // run-time `normalize_schema_strict` itself does not need it — invalid + // schemas remain usable (the LLM provider may reject them at request + // time) so the previous debug-log call has been intentionally dropped + // during the LLM-crate extraction. schema } diff --git a/src/llm/transcription/chat_completions.rs b/crates/ironclaw_llm/src/transcription/chat_completions.rs similarity index 100% rename from src/llm/transcription/chat_completions.rs rename to crates/ironclaw_llm/src/transcription/chat_completions.rs diff --git a/src/llm/transcription/mod.rs b/crates/ironclaw_llm/src/transcription/mod.rs similarity index 78% rename from src/llm/transcription/mod.rs rename to crates/ironclaw_llm/src/transcription/mod.rs index d6750d7bd39..6e5a2c0edac 100644 --- a/src/llm/transcription/mod.rs +++ b/crates/ironclaw_llm/src/transcription/mod.rs @@ -93,17 +93,26 @@ impl TranscriptionMiddleware { Self { provider } } - /// Process an incoming message, transcribing any audio attachments with data. + /// Transcribe any audio attachments with inline data. /// - /// Modifies the message in place: - /// - Sets `extracted_text` on audio attachments that have inline data - /// - If the message content is empty, sets it to the transcription - pub async fn process(&self, msg: &mut crate::channels::IncomingMessage) { - use crate::channels::AttachmentKind; + /// Modifies the slice in place: + /// - Sets `extracted_text` on audio attachments that have inline data. + /// - If `content` is empty (or the placeholder `[Voice note]`), replaces + /// it with the first successful transcription. + /// + /// The signature deliberately takes the attachments and content + /// separately rather than the channel-layer `IncomingMessage`, so this + /// crate doesn't have to depend on channel transport types. + pub async fn process( + &self, + attachments: &mut [ironclaw_common::IncomingAttachment], + content: &mut String, + ) { + use ironclaw_common::AttachmentKind; let mut transcriptions = Vec::new(); - for (i, attachment) in msg.attachments.iter().enumerate() { + for (i, attachment) in attachments.iter().enumerate() { if attachment.kind != AttachmentKind::Audio { continue; } @@ -119,6 +128,7 @@ impl TranscriptionMiddleware { Some(f) => f, None => { tracing::warn!( + attachment_id = %attachment.id, mime = %attachment.mime_type, "Skipping audio attachment with unsupported format" ); @@ -128,11 +138,6 @@ impl TranscriptionMiddleware { match self.provider.transcribe(&attachment.data, format).await { Ok(text) => { - tracing::info!( - attachment_id = %attachment.id, - text_len = text.len(), - "Transcribed audio attachment" - ); transcriptions.push((i, text)); } Err(e) => { @@ -147,16 +152,16 @@ impl TranscriptionMiddleware { } for (i, text) in &transcriptions { - msg.attachments[*i].extracted_text = Some(text.clone()); + attachments[*i].extracted_text = Some(text.clone()); } // If message has no text content, use the first successful transcription - if (msg.content.is_empty() || msg.content == "[Voice note]") + if (content.is_empty() || content == "[Voice note]") && let Some((_, text)) = transcriptions .iter() .find(|(_, t)| !t.starts_with("[Transcription failed")) { - msg.content = text.clone(); + *content = text.clone(); } } } @@ -164,7 +169,7 @@ impl TranscriptionMiddleware { #[cfg(test)] mod tests { use super::*; - use crate::channels::{AttachmentKind, IncomingAttachment, IncomingMessage}; + use ironclaw_common::{AttachmentKind, IncomingAttachment}; struct MockProvider { result: Result, @@ -206,16 +211,16 @@ mod tests { result: Ok("Hello world".to_string()), })); - let mut msg = IncomingMessage::new("telegram", "user1", "[Voice note]") - .with_attachments(vec![voice_attachment(vec![1, 2, 3])]); + let mut attachments = vec![voice_attachment(vec![1, 2, 3])]; + let mut content = "[Voice note]".to_string(); - middleware.process(&mut msg).await; + middleware.process(&mut attachments, &mut content).await; assert_eq!( - msg.attachments[0].extracted_text.as_deref(), + attachments[0].extracted_text.as_deref(), Some("Hello world") ); - assert_eq!(msg.content, "Hello world"); + assert_eq!(content, "Hello world"); } #[tokio::test] @@ -224,13 +229,13 @@ mod tests { result: Ok("Should not be called".to_string()), })); - let mut msg = IncomingMessage::new("telegram", "user1", "text message") - .with_attachments(vec![voice_attachment(Vec::new())]); + let mut attachments = vec![voice_attachment(Vec::new())]; + let mut content = "text message".to_string(); - middleware.process(&mut msg).await; + middleware.process(&mut attachments, &mut content).await; - assert!(msg.attachments[0].extracted_text.is_none()); - assert_eq!(msg.content, "text message"); + assert!(attachments[0].extracted_text.is_none()); + assert_eq!(content, "text message"); } #[tokio::test] @@ -242,13 +247,13 @@ mod tests { let mut attachment = voice_attachment(vec![1, 2, 3]); attachment.extracted_text = Some("Already done".to_string()); - let mut msg = - IncomingMessage::new("telegram", "user1", "").with_attachments(vec![attachment]); + let mut attachments = vec![attachment]; + let mut content = String::new(); - middleware.process(&mut msg).await; + middleware.process(&mut attachments, &mut content).await; assert_eq!( - msg.attachments[0].extracted_text.as_deref(), + attachments[0].extracted_text.as_deref(), Some("Already done") ); } @@ -259,16 +264,16 @@ mod tests { result: Ok("Transcription".to_string()), })); - let mut msg = IncomingMessage::new("telegram", "user1", "User typed this") - .with_attachments(vec![voice_attachment(vec![1, 2, 3])]); + let mut attachments = vec![voice_attachment(vec![1, 2, 3])]; + let mut content = "User typed this".to_string(); - middleware.process(&mut msg).await; + middleware.process(&mut attachments, &mut content).await; assert_eq!( - msg.attachments[0].extracted_text.as_deref(), + attachments[0].extracted_text.as_deref(), Some("Transcription") ); - assert_eq!(msg.content, "User typed this"); + assert_eq!(content, "User typed this"); } #[test] diff --git a/src/llm/transcription/openai.rs b/crates/ironclaw_llm/src/transcription/openai.rs similarity index 100% rename from src/llm/transcription/openai.rs rename to crates/ironclaw_llm/src/transcription/openai.rs diff --git a/src/llm/vision_models.rs b/crates/ironclaw_llm/src/vision_models.rs similarity index 100% rename from src/llm/vision_models.rs rename to crates/ironclaw_llm/src/vision_models.rs diff --git a/crates/ironclaw_oauth/CLAUDE.md b/crates/ironclaw_oauth/CLAUDE.md new file mode 100644 index 00000000000..ec85189d801 --- /dev/null +++ b/crates/ironclaw_oauth/CLAUDE.md @@ -0,0 +1,7 @@ +# ironclaw_oauth guardrails + +- Owns the loopback OAuth callback listener (port 9876), branded landing pages, and `OAUTH_CALLBACK_HOST` binding rules. That is the entire scope. +- Do **not** add provider-specific OAuth (Anthropic, Gemini, GitHub Copilot, OpenAI Codex, NEAR AI, MCP) — those flows live with the consumer that owns the credential and depend on this crate for the transport. +- Do **not** add token storage, refresh logic, PKCE/device-code orchestration, secrets handling, or HTTP client work — keep those concerns in the calling crate. This crate is a callback transport. +- Do **not** depend on `ironclaw_llm`, `ironclaw_secrets`, `ironclaw_authorization`, or any upper substrate. Only `ironclaw_common` for env helpers. +- Wildcard host binds (`0.0.0.0`, `::`) must remain rejected — the listener carries session tokens over plain HTTP. diff --git a/crates/ironclaw_oauth/Cargo.toml b/crates/ironclaw_oauth/Cargo.toml new file mode 100644 index 00000000000..6200c537e96 --- /dev/null +++ b/crates/ironclaw_oauth/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ironclaw_oauth" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +description = "Shared OAuth callback server, landing pages, and host-binding utilities used by every IronClaw OAuth flow" +authors = ["NEAR AI "] +license = "MIT OR Apache-2.0" +homepage = "https://github.com/nearai/ironclaw" +repository = "https://github.com/nearai/ironclaw" +publish = false + +[package.metadata.dist] +dist = false + +[dependencies] +ironclaw_common = { path = "../ironclaw_common", version = "0.4.1" } + +thiserror = "2" +tokio = { version = "1", features = ["io-util", "net", "time", "macros", "rt"] } +urlencoding = "2" diff --git a/src/llm/oauth_helpers.rs b/crates/ironclaw_oauth/src/lib.rs similarity index 95% rename from src/llm/oauth_helpers.rs rename to crates/ironclaw_oauth/src/lib.rs index a4a54b4a094..7a5e5c714aa 100644 --- a/src/llm/oauth_helpers.rs +++ b/crates/ironclaw_oauth/src/lib.rs @@ -1,8 +1,12 @@ -//! OAuth callback infrastructure used by the NEAR AI session login flow. +//! Shared OAuth callback infrastructure used by every IronClaw OAuth flow +//! (NEAR AI session login, WASM tool auth, MCP server auth, registry-provider +//! OAuth backends). //! -//! These utilities (callback server, landing pages, hostname detection) were -//! originally in the old CLI OAuth module and are moved here so the `llm` -//! module is self-contained. +//! This crate owns the loopback callback listener, the branded landing pages, +//! and the host-binding rules that keep the callback safe on remote hosts. +//! It is deliberately small and free of LLM/auth-provider concerns — the +//! provider-specific OAuth flows live in their owning crates and depend on +//! this crate for the transport. use std::collections::HashMap; use std::time::Duration; @@ -38,7 +42,7 @@ pub enum OAuthCallbackError { /// deployments where `127.0.0.1` is unreachable from the user's browser), /// then falls back to `http://{callback_host()}:{OAUTH_CALLBACK_PORT}`. pub fn callback_url() -> String { - crate::config::helpers::env_or_override("IRONCLAW_OAUTH_CALLBACK_URL") + ironclaw_common::env_helpers::env_or_override("IRONCLAW_OAUTH_CALLBACK_URL") .unwrap_or_else(|| format!("http://{}:{}", callback_host(), OAUTH_CALLBACK_PORT)) } @@ -54,7 +58,7 @@ pub fn callback_url() -> String { /// Note: this transmits the session token over plain HTTP — prefer SSH port /// forwarding (`ssh -L 9876:127.0.0.1:9876 user@host`) when possible. pub fn callback_host() -> String { - crate::config::helpers::env_or_override("OAUTH_CALLBACK_HOST") + ironclaw_common::env_helpers::env_or_override("OAUTH_CALLBACK_HOST") .unwrap_or_else(|| "127.0.0.1".to_string()) } @@ -360,7 +364,7 @@ pub fn landing_html(provider_name: &str, success: bool) -> String { #[cfg(test)] mod tests { use super::*; - use crate::config::helpers::lock_env; + use ironclaw_common::env_helpers::lock_env; #[test] fn loopback_detection() { diff --git a/crates/ironclaw_silk_decoder/Cargo.lock b/crates/ironclaw_silk_decoder/Cargo.lock new file mode 100644 index 00000000000..703aaeba72d --- /dev/null +++ b/crates/ironclaw_silk_decoder/Cargo.lock @@ -0,0 +1,283 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "ironclaw_silk_decoder" +version = "0.1.0" +dependencies = [ + "silk-codec", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "silk-codec" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378ab490d55e4d7a0e0e1ed0b7518e4126f70ea595ac7b37cbc62dd933327b9a" +dependencies = [ + "bindgen", + "bytes", + "cc", + "glob", + "thiserror", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/crates/ironclaw_silk_decoder/Cargo.toml b/crates/ironclaw_silk_decoder/Cargo.toml new file mode 100644 index 00000000000..32419568c55 --- /dev/null +++ b/crates/ironclaw_silk_decoder/Cargo.toml @@ -0,0 +1,19 @@ +# Standalone — explicitly NOT part of the IronClaw workspace so the main +# `cargo build` does not pull in `silk-codec` (which requires `bindgen` + +# `libclang` and a C toolchain). See `crates/ironclaw_silk_decoder/README.md`. +[workspace] + +[package] +name = "ironclaw_silk_decoder" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false +description = "Standalone helper that decodes WeChat raw SILK v3 voice notes to WAV. Isolated here so the main IronClaw build does not require libclang." + +[[bin]] +name = "ironclaw-silk-decoder" +path = "src/main.rs" + +[dependencies] +silk-codec = "0.2.0" diff --git a/crates/ironclaw_silk_decoder/README.md b/crates/ironclaw_silk_decoder/README.md new file mode 100644 index 00000000000..73823a19df6 --- /dev/null +++ b/crates/ironclaw_silk_decoder/README.md @@ -0,0 +1,60 @@ +# ironclaw-silk-decoder + +Standalone helper that converts WeChat raw SILK v3 voice notes into a +WAV-wrapped 16‑bit mono PCM stream that downstream consumers (the +transcription pipeline, the agent, audio players) can use. + +The decoder lives in its own crate for one reason: SILK decoding requires +[`silk-codec`](https://crates.io/crates/silk-codec), which compiles a vendored +C implementation and pulls in `bindgen` + `clang-sys`. Anything that uses +`silk-codec` therefore needs `libclang` and a C toolchain at build time. By +keeping that dependency in a separate, workspace-excluded crate, the main +IronClaw build does **not** need `libclang`. + +## Build + +```bash +./crates/ironclaw_silk_decoder/build.sh +``` + +The build script runs `cargo build --release` with this crate's manifest +and produces `target/release/ironclaw-silk-decoder` (relative to the +crate). It is excluded from the top-level workspace so it does not +participate in `cargo build --workspace`. + +## Install + +The IronClaw host looks for the binary in this order: + +1. `IRONCLAW_SILK_DECODER` environment variable (a path to the binary) +2. A sibling of the running `ironclaw` executable named + `ironclaw-silk-decoder` (with `.exe` on Windows) +3. `ironclaw-silk-decoder` on `$PATH` + +If none of those resolve, WeChat voice notes are delivered as raw +`audio/silk` attachments and the agent's transcription pipeline skips +them. This is intentional — the decoder is optional. + +## Protocol + +``` +stdin <- raw SILK v3 bytes (the bytes WeChat ships as `audio/silk`) +stdout <- a complete WAV file (RIFF/WAVE, 16‑bit LE PCM, mono) +stderr <- human-readable diagnostics +exit <- 0 success, 1 IO, 2 invalid argument, 3 decode failure +``` + +Default sample rate is 24 000 Hz to match WeChat's voice-note encoding. +Override with `--sample-rate ` (8 000–48 000). + +## Why a separate process? + +- **No `libclang` in the main build.** Removing `silk-rs` from the host + means contributors and CI no longer need a Clang toolchain just to + compile IronClaw. +- **Crash isolation.** Untrusted SILK bytes from a remote messaging + server are decoded in a child process. A bug in the C decoder kills + the child, not the host. +- **Optional install.** Distributions that don't care about WeChat voice + transcription can ship without the decoder; everything else still + works. diff --git a/crates/ironclaw_silk_decoder/build.sh b/crates/ironclaw_silk_decoder/build.sh new file mode 100755 index 00000000000..a4f52ee12ed --- /dev/null +++ b/crates/ironclaw_silk_decoder/build.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Standalone build script for the optional WeChat-voice SILK decoder helper. +# +# This crate is intentionally excluded from the IronClaw workspace so the +# main `cargo build` does not require libclang. It is built separately: +# +# ./crates/ironclaw_silk_decoder/build.sh +# +# After building, the binary lands in `target/release/ironclaw-silk-decoder` +# (relative to this crate). Install it next to your `ironclaw` binary, on +# `$PATH`, or point the `IRONCLAW_SILK_DECODER` environment variable at it. + +set -euo pipefail + +cd "$(dirname "$0")" + +if ! command -v cargo >/dev/null 2>&1; then + echo "Error: cargo not found on PATH" >&2 + exit 1 +fi + +echo "Building ironclaw-silk-decoder (requires libclang + a C toolchain)..." +cargo build --release + +OUT_BIN="target/release/ironclaw-silk-decoder" + +if [ ! -f "$OUT_BIN" ]; then + echo "Error: build did not produce $OUT_BIN" >&2 + exit 1 +fi + +echo "" +echo "Built: $OUT_BIN ($(du -h "$OUT_BIN" | cut -f1))" +echo "" +echo "To install (one of the following):" +echo " cp $OUT_BIN \"\$(dirname \"\$(command -v ironclaw)\")/\" # sibling install" +echo " cp $OUT_BIN /usr/local/bin/ # system PATH" +echo " export IRONCLAW_SILK_DECODER=\"\$(pwd)/$OUT_BIN\" # explicit path" diff --git a/crates/ironclaw_silk_decoder/src/main.rs b/crates/ironclaw_silk_decoder/src/main.rs new file mode 100644 index 00000000000..b6178c18620 --- /dev/null +++ b/crates/ironclaw_silk_decoder/src/main.rs @@ -0,0 +1,214 @@ +//! `ironclaw-silk-decoder` — reads raw SILK v3 bytes on stdin, writes a WAV-wrapped +//! 16-bit little-endian mono PCM stream on stdout. +//! +//! Isolated here so the main IronClaw build does not need libclang. The host +//! invokes this binary as a subprocess; if it isn't installed, WeChat voice +//! notes simply remain `audio/silk` blobs (graceful degradation). +//! +//! Usage: +//! ironclaw-silk-decoder [--sample-rate 24000] + +use std::io::{self, Read, Write}; +use std::process::ExitCode; + +use silk_codec::decode_silk; + +const DEFAULT_SAMPLE_RATE_HZ: i32 = 24_000; +const MIN_SAMPLE_RATE_HZ: i32 = 8_000; +const MAX_SAMPLE_RATE_HZ: i32 = 48_000; + +/// Cap on input SILK bytes — matches the host attachment-size cap (20 MiB). +const MAX_INPUT_BYTES: usize = 20 * 1024 * 1024; +/// Cap on output PCM. SILK→PCM expands ~25× at 24 kHz mono; 60 s of voice +/// produces ~3 MiB. 50 MiB allows generous headroom while preventing +/// decompression-bomb behavior. +const MAX_PCM_BYTES: usize = 50 * 1024 * 1024; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let sample_rate = match parse_sample_rate(&args) { + Ok(rate) => rate, + Err(message) => { + eprintln!("ironclaw-silk-decoder: {message}"); + return ExitCode::from(2); + } + }; + + let input = match read_bounded_stdin(MAX_INPUT_BYTES) { + Ok(bytes) => bytes, + Err(message) => { + eprintln!("ironclaw-silk-decoder: {message}"); + return ExitCode::from(1); + } + }; + if input.is_empty() { + eprintln!("ironclaw-silk-decoder: empty input on stdin"); + return ExitCode::from(1); + } + + let pcm = match decode_silk(&input, sample_rate) { + Ok(pcm) => pcm, + Err(error) => { + eprintln!("ironclaw-silk-decoder: SILK decode failed: {error}"); + return ExitCode::from(3); + } + }; + if pcm.is_empty() { + eprintln!("ironclaw-silk-decoder: SILK decoder returned empty PCM"); + return ExitCode::from(3); + } + if pcm.len() > MAX_PCM_BYTES { + eprintln!( + "ironclaw-silk-decoder: decoded PCM exceeds {MAX_PCM_BYTES} bytes ({} bytes)", + pcm.len() + ); + return ExitCode::from(3); + } + + let wav = match pcm_s16le_to_wav(&pcm, sample_rate as u32) { + Ok(bytes) => bytes, + Err(error) => { + eprintln!("ironclaw-silk-decoder: {error}"); + return ExitCode::from(3); + } + }; + + let mut stdout = io::stdout().lock(); + if let Err(error) = stdout.write_all(&wav) { + eprintln!("ironclaw-silk-decoder: failed to write WAV to stdout: {error}"); + return ExitCode::from(1); + } + if let Err(error) = stdout.flush() { + eprintln!("ironclaw-silk-decoder: failed to flush stdout: {error}"); + return ExitCode::from(1); + } + ExitCode::SUCCESS +} + +fn parse_sample_rate(args: &[String]) -> Result { + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--sample-rate" => { + let raw = iter + .next() + .ok_or_else(|| "--sample-rate requires a value".to_string())?; + let parsed: i32 = raw + .parse() + .map_err(|e| format!("--sample-rate '{raw}' is not an integer: {e}"))?; + if !(MIN_SAMPLE_RATE_HZ..=MAX_SAMPLE_RATE_HZ).contains(&parsed) { + return Err(format!( + "--sample-rate must be between {MIN_SAMPLE_RATE_HZ} and {MAX_SAMPLE_RATE_HZ} Hz, got {parsed}" + )); + } + return Ok(parsed); + } + "--help" | "-h" => { + eprintln!( + "Usage: ironclaw-silk-decoder [--sample-rate HZ]\n\nReads raw SILK v3 bytes on stdin, writes 16-bit mono PCM\nwrapped in a WAV container on stdout." + ); + return Err("help".to_string()); + } + other => return Err(format!("unknown argument '{other}'")), + } + } + Ok(DEFAULT_SAMPLE_RATE_HZ) +} + +fn read_bounded_stdin(cap: usize) -> Result, String> { + let mut buffer = Vec::new(); + let stdin = io::stdin(); + let mut handle = stdin.lock().take((cap as u64).saturating_add(1)); + handle + .read_to_end(&mut buffer) + .map_err(|e| format!("failed to read stdin: {e}"))?; + if buffer.len() > cap { + return Err(format!("input exceeds {cap} bytes")); + } + Ok(buffer) +} + +fn pcm_s16le_to_wav(pcm: &[u8], sample_rate_hz: u32) -> Result, String> { + if !pcm.len().is_multiple_of(2) { + return Err("PCM buffer length must be even for 16-bit mono audio".to_string()); + } + + let data_len = u32::try_from(pcm.len()) + .map_err(|_| "PCM buffer exceeds WAV container size limits".to_string())?; + let total_len = 44u32 + .checked_add(data_len) + .ok_or_else(|| "WAV container size overflowed".to_string())?; + let byte_rate = sample_rate_hz + .checked_mul(2) + .ok_or_else(|| "WAV byte rate overflowed".to_string())?; + + let mut wav = Vec::with_capacity(total_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(total_len - 8).to_le_bytes()); + wav.extend_from_slice(b"WAVE"); + wav.extend_from_slice(b"fmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&sample_rate_hz.to_le_bytes()); + wav.extend_from_slice(&byte_rate.to_le_bytes()); + wav.extend_from_slice(&2u16.to_le_bytes()); + wav.extend_from_slice(&16u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.extend_from_slice(pcm); + Ok(wav) +} + +#[cfg(test)] +mod tests { + use super::{ + DEFAULT_SAMPLE_RATE_HZ, MAX_SAMPLE_RATE_HZ, MIN_SAMPLE_RATE_HZ, parse_sample_rate, + pcm_s16le_to_wav, + }; + + #[test] + fn parse_sample_rate_default_when_no_flag() { + assert_eq!(parse_sample_rate(&[]).unwrap(), DEFAULT_SAMPLE_RATE_HZ); + } + + #[test] + fn parse_sample_rate_accepts_explicit_value() { + let args = vec!["--sample-rate".to_string(), "16000".to_string()]; + assert_eq!(parse_sample_rate(&args).unwrap(), 16_000); + } + + #[test] + fn parse_sample_rate_rejects_out_of_range() { + let too_low = vec!["--sample-rate".to_string(), "1000".to_string()]; + assert!(parse_sample_rate(&too_low).is_err()); + let too_high = vec![ + "--sample-rate".to_string(), + (MAX_SAMPLE_RATE_HZ + 1).to_string(), + ]; + assert!(parse_sample_rate(&too_high).is_err()); + let _ = MIN_SAMPLE_RATE_HZ; + } + + #[test] + fn parse_sample_rate_rejects_unknown_arg() { + let args = vec!["--what".to_string()]; + assert!(parse_sample_rate(&args).is_err()); + } + + #[test] + fn pcm_s16le_to_wav_writes_riff_wave_header() { + let wav = pcm_s16le_to_wav(&[0x00, 0x00, 0x01, 0x00], 24_000).expect("wav wrap"); + assert!(wav.starts_with(b"RIFF")); + assert_eq!(&wav[8..12], b"WAVE"); + assert_eq!(&wav[12..16], b"fmt "); + assert_eq!(&wav[36..40], b"data"); + assert_eq!(&wav[40..44], &(4u32).to_le_bytes()); + assert_eq!(&wav[44..], &[0x00, 0x00, 0x01, 0x00]); + } + + #[test] + fn pcm_s16le_to_wav_rejects_odd_length() { + assert!(pcm_s16le_to_wav(&[0x00], 24_000).is_err()); + } +} diff --git a/crates/ironclaw_tui/Cargo.toml b/crates/ironclaw_tui/Cargo.toml index 53dcff7220e..951b2d829d8 100644 --- a/crates/ironclaw_tui/Cargo.toml +++ b/crates/ironclaw_tui/Cargo.toml @@ -15,7 +15,7 @@ default = ["clipboard"] clipboard = ["dep:arboard", "dep:image"] [dependencies] -ironclaw_common = { path = "../ironclaw_common", version = "0.4.0" } +ironclaw_common = { path = "../ironclaw_common", version = "0.4.2" } ratatui = { version = "0.29", features = ["crossterm"] } tui-textarea = { version = "0.7", features = ["crossterm"] } serde = { version = "1", features = ["derive"] } @@ -31,3 +31,4 @@ image = { version = "0.25", default-features = false, features = ["png"], option [dev-dependencies] tokio = { version = "1", features = ["full"] } +tempfile = "3" diff --git a/crates/ironclaw_tui/src/app.rs b/crates/ironclaw_tui/src/app.rs index 592f687c0ec..5639b2480be 100644 --- a/crates/ironclaw_tui/src/app.rs +++ b/crates/ironclaw_tui/src/app.rs @@ -473,6 +473,17 @@ async fn handle_event( ActiveTab::Logs => ActiveTab::Conversation, }; } + InputAction::DownloadLogs => { + let (message, kind) = match download_logs(state) { + Ok(path) => (format!("Logs saved to {path}"), ToastKind::Success), + Err(err) => (format!("Save logs failed: {err}"), ToastKind::Error), + }; + state.toasts.push(Toast { + message, + kind, + created_at: chrono::Utc::now(), + }); + } InputAction::ScrollUp => match state.active_tab { ActiveTab::Conversation => { let page = state.conversation_height.max(2).saturating_sub(2) as i16; @@ -2379,23 +2390,14 @@ fn render_toasts( let theme = layout.resolve_theme(); let max_toasts = 3usize; - let toast_width = 40u16.min(size.width.saturating_sub(2)); + let icon_pad = 4u16; // " ICON " + let border_pad = 2u16; // left + right border // Stack toasts from bottom up, above status bar - let start_y = size.height.saturating_sub(3); // above status bar + input - let visible_toasts = state.toasts.iter().rev().take(max_toasts); - - for (i, toast) in visible_toasts.enumerate() { - let y = start_y.saturating_sub((i as u16) * 3); - let x = size.width.saturating_sub(toast_width + 1); - let area = Rect::new(x, y, toast_width, 3); - - if area.y == 0 { - continue; - } - - Clear.render(area, frame.buffer_mut()); + let mut next_bottom = size.height.saturating_sub(3); + let visible_toasts: Vec<_> = state.toasts.iter().rev().take(max_toasts).collect(); + for toast in visible_toasts { let (icon, border_style) = match toast.kind { ToastKind::Info => ("\u{2139}", theme.accent_style()), ToastKind::Success => ("\u{2713}", theme.success_style()), @@ -2403,34 +2405,54 @@ fn render_toasts( ToastKind::Error => ("\u{2717}", theme.error_style()), }; + // Width fits the message in full (e.g. saved-log file paths), capped + // at the terminal width. If even the cap isn't enough, the paragraph + // wraps onto extra lines and the toast grows in height. + let max_outer_width = size.width.saturating_sub(1); + let message_chars = u16::try_from(toast.message.chars().count()).unwrap_or(u16::MAX); + let toast_width = message_chars + .saturating_add(icon_pad + border_pad) + .min(max_outer_width) + .max(20); + + let inner_msg_width = toast_width.saturating_sub(border_pad + icon_pad).max(1) as usize; + let wrap_lines = toast + .message + .chars() + .count() + .div_ceil(inner_msg_width) + .max(1); + let toast_height = (wrap_lines as u16 + border_pad).min(size.height.saturating_sub(1)); + + if next_bottom < toast_height { + break; + } + let y = next_bottom.saturating_sub(toast_height); + let x = size.width.saturating_sub(toast_width + 1); + let area = Rect::new(x, y, toast_width, toast_height); + + Clear.render(area, frame.buffer_mut()); + let block = Block::default() .borders(Borders::ALL) .border_style(border_style); let inner = block.inner(area); block.render(area, frame.buffer_mut()); - let msg_width = inner.width as usize; - let display_msg = if toast.message.len() > msg_width.saturating_sub(3) { - format!( - "{}...", - &toast.message[..msg_width.saturating_sub(6).min(toast.message.len())] - ) - } else { - toast.message.clone() - }; - let line = Line::from(vec![ Span::styled( format!(" {icon} "), border_style.add_modifier(Modifier::BOLD), ), Span::styled( - display_msg, + toast.message.clone(), ratatui::style::Style::default().fg(theme.fg.to_color()), ), ]); - let paragraph = Paragraph::new(line); + let paragraph = Paragraph::new(line).wrap(ratatui::widgets::Wrap { trim: false }); paragraph.render(inner, frame.buffer_mut()); + + next_bottom = y; } } @@ -2499,6 +2521,91 @@ fn try_paste_clipboard_image(_state: &AppState) -> Option { None } +/// Write the current log ring buffer to a timestamped file under `~/.ironclaw/logs/` +/// (falls back to `$IRONCLAW_HOME/logs/` or the current directory). Returns the +/// human-readable target path on success. +fn download_logs(state: &AppState) -> Result { + let dir = log_output_dir().ok_or_else(|| "could not resolve log directory".to_string())?; + download_logs_to_dir(state, &dir) +} + +/// Inner write step, parameterized on output dir so tests can use a tempdir +/// without racing `IRONCLAW_HOME` env mutation across threads. +fn download_logs_to_dir(state: &AppState, dir: &std::path::Path) -> Result { + use std::io::Write; + + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + + let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + let (mut file, path) = open_unique_log_file(dir, &stamp)?; + + for entry in state.log_entries.iter() { + writeln!( + file, + "{} {:<5} {} {}", + entry.timestamp, entry.level, entry.target, entry.message + ) + .map_err(|e| e.to_string())?; + } + + Ok(path.display().to_string()) +} + +/// Open a new log file under `dir` with owner-only permissions on Unix and a +/// collision-free name. Tries `tui-logs-.log` first, then appends a +/// numeric suffix on collision. Surfaces "already exists" as a hard error +/// once the candidate budget is exhausted, instead of silently truncating — +/// the timestamp has second precision, so two presses inside the same UTC +/// second would otherwise destroy the earlier export. +fn open_unique_log_file( + dir: &std::path::Path, + stamp: &str, +) -> Result<(std::fs::File, std::path::PathBuf), String> { + /// Bound on the in-second retry budget. Realistically a user is not + /// pressing Ctrl-S a thousand times per second; this caps the loop. + const MAX_RETRIES: u32 = 1000; + + for n in 0..MAX_RETRIES { + let path = if n == 0 { + dir.join(format!("tui-logs-{stamp}.log")) + } else { + dir.join(format!("tui-logs-{stamp}-{n}.log")) + }; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + // Owner-only perms from creation, so a permissive umask on a + // shared host never produces group/world-readable log dumps. + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&path) { + Ok(file) => return Ok((file, path)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(format!("{}: {e}", path.display())), + } + } + Err(format!( + "exhausted {MAX_RETRIES} candidate filenames in {} for stamp {stamp}", + dir.display() + )) +} + +fn log_output_dir() -> Option { + if let Ok(custom) = std::env::var("IRONCLAW_HOME") { + return Some(std::path::PathBuf::from(custom).join("logs")); + } + if let Ok(home) = std::env::var("HOME") { + return Some( + std::path::PathBuf::from(home) + .join(".ironclaw") + .join("logs"), + ); + } + std::env::current_dir().ok().map(|d| d.join("logs")) +} + /// Encode raw RGBA pixel data to PNG. Returns `None` on invalid dimensions or /// encoding failure. #[cfg(feature = "clipboard")] @@ -2566,6 +2673,83 @@ mod tests { } } + fn push_log_entry(state: &mut AppState, message: &str) { + state.log_entries.push(crate::event::TuiLogEntry { + level: "INFO".to_string(), + target: "test".to_string(), + message: message.to_string(), + timestamp: "2026-05-15T00:00:00Z".to_string(), + }); + } + + /// Regression for serrrfirat's Medium #1 (PR #3658): exported log files + /// must not inherit umask. Two presses of Ctrl-S on a permissive host + /// previously produced 0664/0666 files containing every tracing line — + /// readable by anyone in the user's primary group. + #[cfg(unix)] + #[test] + fn download_logs_writes_owner_only_permissions() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let mut state = AppState::default(); + push_log_entry(&mut state, "hello"); + + let path = download_logs_to_dir(&state, dir.path()).expect("download"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + assert_eq!( + mode & 0o777, + 0o600, + "expected 0600, got {:o} for {path}", + mode & 0o777 + ); + } + + /// Regression for serrrfirat's Medium #2 (PR #3658): the filename + /// timestamp has second precision and the writer used to call + /// `File::create`, which truncates. Two Ctrl-S presses in the same UTC + /// second silently overwrote the first export. + #[test] + fn download_logs_does_not_overwrite_on_same_second_collision() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut state_a = AppState::default(); + push_log_entry(&mut state_a, "first"); + let mut state_b = AppState::default(); + push_log_entry(&mut state_b, "second"); + + let path_a = download_logs_to_dir(&state_a, dir.path()).expect("first write"); + let path_b = download_logs_to_dir(&state_b, dir.path()).expect("second write"); + + assert_ne!( + path_a, path_b, + "second write must not collide with the first" + ); + let body_a = std::fs::read_to_string(&path_a).expect("read a"); + let body_b = std::fs::read_to_string(&path_b).expect("read b"); + assert!(body_a.contains("first"), "first export was overwritten"); + assert!(body_b.contains("second")); + assert!( + !body_a.contains("second"), + "first export must be untouched by the second call" + ); + } + + /// Direct check on the collision-resolution helper: feeding it the same + /// stamp repeatedly must keep yielding fresh paths until the budget runs + /// out, never reopening an existing file. + #[test] + fn open_unique_log_file_appends_suffix_on_collision() { + let dir = tempfile::tempdir().expect("tempdir"); + let stamp = "20260515T000000Z"; + + let (_f1, p1) = open_unique_log_file(dir.path(), stamp).expect("first"); + let (_f2, p2) = open_unique_log_file(dir.path(), stamp).expect("second"); + let (_f3, p3) = open_unique_log_file(dir.path(), stamp).expect("third"); + + assert_eq!(p1, dir.path().join("tui-logs-20260515T000000Z.log")); + assert_eq!(p2, dir.path().join("tui-logs-20260515T000000Z-1.log")); + assert_eq!(p3, dir.path().join("tui-logs-20260515T000000Z-2.log")); + } + #[cfg(feature = "clipboard")] #[test] fn encode_rgba_to_png_valid() { diff --git a/crates/ironclaw_tui/src/input.rs b/crates/ironclaw_tui/src/input.rs index df59607393c..66ea4d95581 100644 --- a/crates/ironclaw_tui/src/input.rs +++ b/crates/ironclaw_tui/src/input.rs @@ -17,6 +17,8 @@ pub enum InputAction { ToggleSidebar, /// Toggle between Conversation and Logs tabs. ToggleLogs, + /// Write the current log buffer to a file on disk (Ctrl-S in Logs tab). + DownloadLogs, /// Scroll conversation up. ScrollUp, /// Scroll conversation down. @@ -124,6 +126,11 @@ pub fn map_key( return action; } + // Save logs only when the Logs tab is the focused view. + if logs_active && key.code == KeyCode::Char('s') && key.modifiers == KeyModifiers::CONTROL { + return InputAction::DownloadLogs; + } + match (key.code, key.modifiers) { (KeyCode::Enter, KeyModifiers::NONE) => InputAction::Submit, (KeyCode::Enter, KeyModifiers::SHIFT) | (KeyCode::Enter, KeyModifiers::ALT) => { @@ -517,4 +524,13 @@ mod tests { assert_eq!(parse_slash_command(" /quit "), Some("/quit")); assert_eq!(parse_slash_command("hello"), None); } + + #[test] + fn ctrl_s_triggers_download_only_in_logs_tab() { + let key = KeyEvent::new(KeyCode::Char('s'), KeyModifiers::CONTROL); + assert_eq!(map_logs(key), InputAction::DownloadLogs); + // Outside the Logs tab Ctrl-S must fall through to default text input, + // not silently dump the ring buffer from arbitrary contexts. + assert_eq!(map_default(key), InputAction::Forward); + } } diff --git a/crates/ironclaw_tui/src/widgets/help_overlay.rs b/crates/ironclaw_tui/src/widgets/help_overlay.rs index 420ab2dc2fc..6f1adc6e03f 100644 --- a/crates/ironclaw_tui/src/widgets/help_overlay.rs +++ b/crates/ironclaw_tui/src/widgets/help_overlay.rs @@ -29,6 +29,7 @@ const KEYBINDINGS: &[(&str, &str)] = &[ ("Esc", "Interrupt / cancel"), ("y / n / a", "Approval shortcuts"), ("1-5", "Log level filter (Logs tab)"), + ("Ctrl-S", "Download logs (Logs tab)"), ]; pub struct HelpOverlayWidget { diff --git a/crates/ironclaw_tui/src/widgets/status_bar.rs b/crates/ironclaw_tui/src/widgets/status_bar.rs index 0d9fa3a7e4b..04d443621af 100644 --- a/crates/ironclaw_tui/src/widgets/status_bar.rs +++ b/crates/ironclaw_tui/src/widgets/status_bar.rs @@ -199,7 +199,10 @@ impl TuiWidget for StatusBarWidget { left_spans.push(sep); left_spans.push(Span::styled(duration_str, self.theme.dim_style())); - let right_text = "^L logs ^B sidebar ^C quit"; + let right_text = match state.active_tab { + ActiveTab::Logs => "^S save ^L logs ^B sidebar ^C quit", + ActiveTab::Conversation => "^L logs ^B sidebar ^C quit", + }; let right_span = Span::styled(format!("{right_text} "), self.theme.dim_style()); // Render left-aligned portion diff --git a/crates/ironclaw_wasm/Cargo.toml b/crates/ironclaw_wasm/Cargo.toml index 08765690d94..3503f8e6a42 100644 --- a/crates/ironclaw_wasm/Cargo.toml +++ b/crates/ironclaw_wasm/Cargo.toml @@ -9,8 +9,8 @@ ironclaw_host_api = { path = "../ironclaw_host_api" } serde_json = "1" thiserror = "2" tracing = "0.1" -wasmtime = { version = "43.0.2", features = ["component-model"] } -wasmtime-wasi = "43.0.2" +wasmtime = { version = "44.0.2", features = ["component-model"] } +wasmtime-wasi = "44.0.2" [dev-dependencies] async-trait = "0.1" diff --git a/deny.toml b/deny.toml index ce6a35d5b01..917fde89412 100644 --- a/deny.toml +++ b/deny.toml @@ -62,4 +62,8 @@ allow-git = [ # Pulls in ruff_* crates from astral-sh/ruff at a pinned revision. "https://github.com/pydantic/monty.git", "https://github.com/astral-sh/ruff.git", + # Matrix pilot branch: pinned until the released SDK line carries the + # recursion-limit fix needed by current Rust stable. + "https://github.com/matrix-org/matrix-rust-sdk", + "https://github.com/ruma/ruma", ] diff --git a/docs/api/responses.mdx b/docs/api/responses.mdx new file mode 100644 index 00000000000..657f1cac549 --- /dev/null +++ b/docs/api/responses.mdx @@ -0,0 +1,524 @@ +--- +title: "Responses API" +description: "OpenAI-compatible Responses endpoint backed by the full IronClaw agent loop" +icon: "code" +--- + +IronClaw exposes an OpenAI-compatible Responses API. Unlike a raw LLM passthrough, requests route through the full agent loop, so callers get tools, memory, safety, and server-side conversation state via a standard wire format. + +``` +POST /api/v1/responses +GET /api/v1/responses/{id} +``` + +The legacy `/v1/responses` path is still accepted as an alias for clients pinned to it. + +## When to use this vs. the Chat Completions proxy + +| Endpoint | Behavior | +|----------|----------| +| `/api/v1/responses` | Routes through the agent. The model can call IronClaw tools (file, web, memory, MCP, extensions), retain conversation state via `previous_response_id`, and pause for caller-supplied tools. | +| `/v1/chat/completions` | Raw LLM proxy. No tools, no memory, no agent loop. Use it when you only need a model call. | + +If you want IronClaw's capabilities behind an OpenAI-shaped wire contract, this is the endpoint. + +--- + +## Authentication + +Every protected route requires a bearer token in the `Authorization` header: + +``` +Authorization: Bearer +``` + +You have two options: + + + + For single-user deployments, set `GATEWAY_AUTH_TOKEN` in the environment. The same token authenticates the web UI, the Telegram bridge, and the Responses API. + + If `GATEWAY_AUTH_TOKEN` is not set, IronClaw generates 32 random bytes and prints the hex-encoded value (64 characters) to the console on boot. + + ```bash + export GATEWAY_AUTH_TOKEN="your-secure-token" + ``` + + + + For multi-user deployments, create a per-user API token through the self-service `/api/tokens` endpoint. Each token is scoped to one user, can carry an expiry, and is revocable. + + ```bash + # Create + curl -X POST https://your-host/api/tokens \ + -H "Authorization: Bearer $GATEWAY_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"name":"my-integration","expires_in_days":90}' + # → {"id":"...","token":"","token_prefix":"...","name":"...","expires_at":"..."} + + # List + curl https://your-host/api/tokens \ + -H "Authorization: Bearer $TOKEN" + + # Revoke + curl -X DELETE https://your-host/api/tokens/<id> \ + -H "Authorization: Bearer $TOKEN" + ``` + + The plaintext token is shown exactly once. Store it; you cannot retrieve it again. Admin users may pass `"user_id"` in the create body to mint a token on behalf of another user. + </Tab> +</Tabs> + +A missing or wrong token returns `401`. Chat send endpoints (including this one) are rate-limited to 30 requests per 60 seconds per user. + +--- + +## Quickstart + +The simplest request, with no session state and no tools: + +```bash +curl -X POST https://your-host/api/v1/responses \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "input": "What time is it?" + }' +``` + +Response: + +```json +{ + "id": "resp_<64hex>", + "object": "response", + "created_at": 1715846400, + "model": "default", + "status": "completed", + "output": [ + { + "type": "message", + "id": "item_...", + "role": "assistant", + "content": [ + { "type": "output_text", "text": "It is 14:23 UTC." } + ] + } + ], + "usage": { "input_tokens": 42, "output_tokens": 11, "total_tokens": 53 } +} +``` + +The OpenAI Python and TypeScript SDKs work as-is. Point `base_url` at `https://your-host/api/v1` (or `/v1` for the alias). + +```python +from openai import OpenAI + +client = OpenAI(base_url="https://your-host/api/v1", api_key=TOKEN) +resp = client.responses.create(input="What time is it?") +print(resp.output_text) +``` + +--- + +## Request fields + +```json +{ + "input": "...", + "model": "default", + "instructions": null, + "previous_response_id": null, + "stream": false, + "tools": null, + "x_context": null +} +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `input` | yes | A string, or an array of `message` / `function_call_output` items. See [External tools](#external-tools) for the array form. | +| `model` | no | Must be `"default"`. Per-request model override is not yet supported; configure providers via settings. | +| `instructions` | no | System/developer instructions. Injected ahead of `input` as an `<instructions>` block and stored as part of the persisted user message. See [Per-request instructions](#per-request-instructions) for the IronClaw-specific replay caveat. | +| `previous_response_id` | no | The `id` from a prior response. Resumes the same thread; see [Session continuity](#session-continuity). | +| `stream` | no | `true` for SSE; `false` (default) for a single JSON response. | +| `tools` | no | Caller-supplied function tools. See [External tools](#external-tools). | +| `x_context` | no | IronClaw extension. Structured JSON context (≤ 10 KB) prepended as `<user-context>`. The alias `context` is also accepted but may collide with future OpenAI fields. Prefer `x_context`. | + +These fields are rejected with `400` so callers know they were ignored rather than silently dropped: + +- `tool_choice` (no per-request tool surface to enforce against) +- `temperature` (configure via settings) +- `max_output_tokens` (not yet wired) +- Any `model` other than `"default"` + +--- + +## Session continuity + +Each response embeds its thread UUID in the `id`. Pass the previous `id` back as `previous_response_id` to continue the same conversation: + +```json +{ + "input": "What did I just ask?", + "previous_response_id": "resp_abc...xyz" +} +``` + +IronClaw replays the thread's history from the conversation store and runs the new turn against it. There is no client-side state to manage. Threads are user-scoped, so a token belonging to a different user cannot resume someone else's thread (the lookup returns `404`). + +Each POST mints a fresh `response_uuid`, so two turns on the same thread produce different `id`s. + +### Retrieving a past response + +```bash +curl https://your-host/api/v1/responses/$ID \ + -H "Authorization: Bearer $TOKEN" +``` + +Returns the output items reconstructed from the stored conversation. Token usage is not retained per-message; the `usage` field on a retrieved response is zero. + +--- + +## Streaming + +Set `"stream": true` to receive Server-Sent Events: + +```bash +curl -N -X POST https://your-host/api/v1/responses \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"input":"Summarize the news.","stream":true}' +``` + +Events follow the OpenAI Responses streaming format. The SSE `event:` field matches the JSON `type`: + +| `event:` | `type` | Payload | +|----------|--------|---------| +| `response.created` | `response.created` | Initial in-progress shell. | +| `response.output_item.added` | `response.output_item.added` | A new `message` or `function_call` item began. | +| `response.output_text.delta` | `response.output_text.delta` | Token delta inside an `output_text` content block. | +| `response.output_item.done` | `response.output_item.done` | An item is finalised. | +| `response.completed` | `response.completed` | Terminal success. | +| `response.failed` | `response.failed` | Terminal failure (turn-level error). | + +A keepalive frame fires every 15 seconds to prevent intermediate proxies from closing idle connections. + +The non-streaming path has a 120-second turn timeout. Long-running tool work (sandbox jobs, multi-step agentic flows) should use `stream: true` so the connection stays responsive. + +--- + +## External tools + +You can register your own function tools per request. The agent treats them as first-class actions alongside built-in tools and pauses execution when it wants to call one. Your client executes the call and feeds the result back on the next request. + +This is a function-calling round-trip, not a prompt-level convention. The wire shape matches the OpenAI Responses API spec. + +<Note> +External tools require **engine v2** to be enabled on the server (`ENGINE_V2=true`). Requests with `tools[]` against an engine v1 server return `400`. +</Note> + +### Define tools + +```json +{ + "input": "What is the weather in NYC?", + "tools": [ + { + "type": "function", + "name": "lookup_weather", + "description": "Return the current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { "type": "string", "description": "City name" } + }, + "required": ["city"] + } + } + ] +} +``` + +Constraints (validated up-front, `400` on violation): + +- Only `type: "function"` is accepted. `web_search`, `file_search`, `code_interpreter`, etc. are rejected. IronClaw routes those through its own registry, not caller-provided definitions. +- Names: `^[A-Za-z0-9_-]{1,64}$`, unique within the request. +- Names must not shadow a registered IronClaw action (built-in tool, extension tool, or engine v2 capability like `mission_*`, `skill_*`, `memory_*`). Shadowing is rejected to prevent confused-deputy behavior. +- The entire `tools[]` payload caps at 16 KiB of canonical JSON. + +### The round trip + +When the model calls one of your tools, the response completes with a `function_call` output item and the thread sits in `Waiting`: + +```json +{ + "id": "resp_...", + "status": "completed", + "output": [ + { + "type": "function_call", + "id": "item_...", + "call_id": "call_ext_lookup_1", + "name": "lookup_weather", + "arguments": "{\"city\":\"NYC\"}" + } + ] +} +``` + +The model may emit prose before the call. Any pre-call text is flushed as a leading `message` item so you see both pieces in order. + +Execute the call locally, then POST a follow-up with a `function_call_output` item. Include `previous_response_id` so the resume targets the same thread: + +```json +{ + "previous_response_id": "resp_...", + "input": [ + { + "type": "function_call_output", + "call_id": "call_ext_lookup_1", + "output": "{\"temp_f\":72,\"conditions\":\"sunny\"}" + } + ], + "tools": [ /* same tools array */ ] +} +``` + +The agent resumes, the LLM sees your tool result, and the final answer comes back as a normal `message` output item. + +Pass the same `tools[]` on the resume request. The catalog is per-thread; passing the definitions keeps the tool available for any follow-up calls the model makes. + +### Resume validation + +The resume request must satisfy the bridge: + +- A `function_call_output` item without a live external-tool gate on this thread returns `400`. +- A `function_call_output` item whose `call_id` does not match the pending gate returns `400`. +- A `function_call_output` item with an empty `call_id` or missing `output` returns `400`. +- If the thread is paused on an unrelated gate (OAuth, approval, pairing), resolve that first. + +You can interleave a fresh user message with the tool output by adding a `message` item. Both will be visible to the agent: + +```json +{ + "input": [ + { "type": "function_call_output", "call_id": "...", "output": "..." }, + { "type": "message", "role": "user", "content": "Also, what about tomorrow?" } + ] +} +``` + +### Multi-call batching + +The engine pauses on the **first** external tool call in an assistant turn. If the model wants to invoke `tool_a` and `tool_b` together, only `tool_a` surfaces on the first response. After you resume, `tool_b` is emitted on the next turn. This is a known limitation; OpenAI-style "post all results together" is a follow-up. + +### Streaming flow + +With `stream: true`, the external-tool flow looks like: + +1. `response.created` +2. Optional `response.output_text.delta` events for any leading prose. +3. `response.output_item.added` with the `function_call` item. +4. `response.output_item.done` with the same item. +5. `response.completed`. + +The stream closes after `response.completed`. Send the resume as a fresh POST with `previous_response_id`. + +--- + +## Structured context (`x_context`) + +For integrations that need to pass structured state alongside the user message (notification approval, webhook payload, environment hints), use `x_context`: + +```json +{ + "input": "Process the latest webhook.", + "x_context": { + "webhook": { "source": "stripe", "event": "invoice.paid", "amount_cents": 4200 } + } +} +``` + +The handler renders it as a `<user-context>` block ahead of the user message. Total serialized size caps at 10 KB. Pass a flat `{key: {object}}` structure; deeper nesting is serialized as raw JSON. + +`x_context` is an IronClaw extension and is not part of the OpenAI Responses API spec. + +--- + +## Per-request instructions + +Use `instructions` to inject a one-turn system/developer message: + +```json +{ + "input": "Summarize this.", + "instructions": "Respond in three bullet points, no preamble." +} +``` + +**IronClaw caveat — diverges from the OpenAI spec.** The OpenAI Responses API contract is that `instructions` apply only to the current turn and are not carried by `previous_response_id`. IronClaw currently prepends the `<instructions>` block into the user message and persists it as part of the conversation, so it *is* visible on later turns when history is replayed. Until the handler stores instructions out-of-band, treat them as sticky for the thread and re-send (or override) them explicitly on each turn that needs different behaviour. + +The agent's persistent identity files (`AGENTS.md`, `SOUL.md`, `USER.md`, `IDENTITY.md`) come from workspace memory and are not affected by this field. + +--- + +## Errors + +Errors from the Responses handler use the OpenAI envelope: + +```json +{ + "error": { + "message": "function_call_output items must include a non-empty `call_id` field", + "type": "invalid_request_error", + "code": null + } +} +``` + +Common codes: + +| Status | `type` | When | +|--------|--------|------| +| `400` | `invalid_request_error` | Schema, validation, shadowed tool name, missing pending gate, unsupported field. | +| `429` | `rate_limit_error` | More than 30 requests in 60 seconds for this user. | +| `503` | `server_error` | Agent loop not started, database unavailable, or connection cap reached. | + +A turn that completes but fails mid-flight (the model errored, a required tool raised) returns `200` with `"status": "failed"` and a populated `error` field. Inspect `status` before reading `output`. + +Two cases do **not** use the JSON envelope and need separate handling: + +- **`401` Unauthorized** comes from the gateway auth middleware before the request reaches the handler, and the body is a plain-text string (`Invalid or missing auth token`). Same for `403` (`Forbidden` for OIDC domain violations) and the `503` (`Database unavailable`) emitted by the middleware when the token store is down. +- **`GET /api/v1/responses/{id}`** returns `404` (still JSON-enveloped) when the response id is unknown or the thread does not belong to the authenticated user. `POST` with a foreign `previous_response_id` does **not** return `404`; the handler only decodes the UUID and dispatches into the agent, where the cross-user resume surfaces as a turn-level failure rather than an HTTP error. Treat cross-user resume as undefined and avoid relying on the response shape. + +The Responses API does **not** support interactive approvals or authentication gates in the response stream. If the agent hits a tool that requires user approval (e.g. shell with a destructive command) or an extension OAuth flow, the turn fails with a clear error directing you to resolve the gate via the web UI or a different channel. + +--- + +## SDK usage + +The OpenAI SDKs are the easiest way to use this endpoint. Point them at IronClaw and they "just work": + +<CodeGroup> + +```python Python +from openai import OpenAI + +client = OpenAI( + base_url="https://your-host/api/v1", + api_key="<your-bearer-token>", +) + +# Basic +resp = client.responses.create(input="Hello.") +print(resp.output_text) + +# With session continuity +follow_up = client.responses.create( + input="What did I just say?", + previous_response_id=resp.id, +) + +# With caller tools +import json + +resp = client.responses.create( + input="What is the weather in NYC?", + tools=[ + { + "type": "function", + "name": "lookup_weather", + "description": "Return current weather.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], +) + +# Inspect the function_call item, run it locally, resume. +call = next(item for item in resp.output if item.type == "function_call") +result = {"temp_f": 72, "conditions": "sunny"} # your code + +final = client.responses.create( + previous_response_id=resp.id, + input=[ + { + "type": "function_call_output", + "call_id": call.call_id, + "output": json.dumps(result), + } + ], + tools=[ ... ], # same definitions +) +print(final.output_text) +``` + +```typescript TypeScript +import OpenAI from "openai"; + +const client = new OpenAI({ + baseURL: "https://your-host/api/v1", + apiKey: process.env.IRONCLAW_TOKEN, +}); + +const resp = await client.responses.create({ + input: "What is the weather in NYC?", + tools: [ + { + type: "function", + name: "lookup_weather", + description: "Return current weather.", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + ], +}); + +const call = resp.output.find((it) => it.type === "function_call"); +if (call) { + const result = { temp_f: 72, conditions: "sunny" }; + const final = await client.responses.create({ + previous_response_id: resp.id, + input: [ + { + type: "function_call_output", + call_id: call.call_id, + output: JSON.stringify(result), + }, + ], + tools: [ /* same definitions */ ], + }); + console.log(final.output_text); +} +``` + +</CodeGroup> + +--- + +## Limits and quirks + +- **Rate limit**: 30 requests per 60 seconds per user (shared with `/api/chat/send`). +- **Body size**: 14 MiB request limit at the gateway. +- **Turn timeout (non-streaming)**: 120 seconds. Use `stream: true` for long-running work. +- **Tool batching**: one external-tool call per round trip. The engine resumes the next call on the next turn. +- **Approvals and auth gates**: not surfaced over the Responses API. Resolve them via the web UI before retrying. +- **`tool_choice`, `temperature`, `max_output_tokens`**: not yet supported. Requests carrying these fields are rejected so callers see they were not honoured. +- **`model`**: must be `"default"`. Provider and model selection is server-side via settings. +- **Token usage on retrieval**: `GET /api/v1/responses/{id}` returns reconstructed output items but `usage` is zero because per-message token counts are not persisted. + +--- + +## Related + +- [Inference Providers](/capabilities/llm-providers) — configure the model that backs `"default"`. +- [Configuration](/capabilities/configuration) — `GATEWAY_AUTH_TOKEN`, engine v2, body and rate limits. +- [Sandboxed Tools](/capabilities/sandboxed-tools) — how IronClaw's built-in tools execute when the agent calls them. +- [MCP](/capabilities/mcp) — connect Model Context Protocol servers to extend the agent's tool surface server-side instead of supplying caller tools per-request. diff --git a/docs/architecture-video/package-lock.json b/docs/architecture-video/package-lock.json index 8a1ae73d3c0..56cd3cc4ff6 100644 --- a/docs/architecture-video/package-lock.json +++ b/docs/architecture-video/package-lock.json @@ -2863,9 +2863,9 @@ "dev": true }, "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", "funding": [ { "type": "github", @@ -2875,7 +2875,8 @@ "type": "opencollective", "url": "https://opencollective.com/fastify" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/fastq": { "version": "1.20.1", diff --git a/docs/architecture-video/package.json b/docs/architecture-video/package.json index d37fc8fe29f..c52fe930000 100644 --- a/docs/architecture-video/package.json +++ b/docs/architecture-video/package.json @@ -30,5 +30,8 @@ }, "sideEffects": [ "*.css" - ] + ], + "overrides": { + "fast-uri": "^3.1.1" + } } diff --git a/docs/capabilities/configuration.mdx b/docs/capabilities/configuration.mdx new file mode 100644 index 00000000000..4b7aa8885d2 --- /dev/null +++ b/docs/capabilities/configuration.mdx @@ -0,0 +1,578 @@ +--- +title: Configuration +sidebarTitle: Configuration +description: Common operator configuration reference for IronClaw +--- + +IronClaw is configured through bootstrap environment variables, database-backed settings, and a small set of advanced or provider-specific environment variables. This page documents the primary options used during setup and normal operation. + +<Info> +This operator-focused reference is checked against the current `src/config/` modules and `providers.json`. Test-only variables, one-off debugging hooks, and lower-level runtime internals may still exist outside this page; when changing configuration behavior, treat `src/config/`, `providers.json`, and `src/setup/README.md` as the source of truth. +</Info> + +## Two-Layer Configuration + +IronClaw uses a two-layer configuration system: + +<AccordionGroup> + <Accordion title="Layer 1: ~/.ironclaw/.env (Bootstrap)" icon="file"> + Contains settings needed **before** database connection: + + - `DATABASE_BACKEND` — Which database to use + - `DATABASE_URL` — PostgreSQL connection string + - `LIBSQL_PATH` — libSQL database file path + - `LLM_BACKEND` — Which LLM provider to use + - `NEARAI_API_KEY` — NEAR AI Cloud API key (if using that mode) + + Written automatically by the onboarding wizard. + </Accordion> + + <Accordion title="Layer 2: Database Settings" icon="database"> + Most user-facing runtime settings are stored in the database and loaded at runtime: + + - Channel configuration + - Model selection + - Embeddings settings + - Skills configuration + - Heartbeat settings + + Some security-sensitive and advanced options remain environment-only. Manage database-backed values through the wizard or `ironclaw config` command. + </Accordion> +</AccordionGroup> + +## Common Configuration Categories + +<CardGroup cols={2}> + <Card title="Agent Settings" icon="bot" href="#agent-settings"> + AGENT_NAME, MAX_PARALLEL_JOBS, timeouts, cost limits + </Card> + + <Card title="Database" icon="database" href="#database"> + DATABASE_BACKEND, DATABASE_URL, LIBSQL_PATH + </Card> + + <Card title="LLM / Inference" icon="brain" href="#llm--inference"> + NEARAI_*, ANTHROPIC_*, OPENAI_*, OLLAMA_* + </Card> + + <Card title="Channels" icon="message-square" href="#channels"> + GATEWAY_*, HTTP_*, TELEGRAM_*, SIGNAL_* + </Card> + + <Card title="Embeddings" icon="search" href="#embeddings"> + EMBEDDING_*, OPENAI_API_KEY + </Card> + + <Card title="Sandbox" icon="container" href="#sandbox"> + SANDBOX_*, CLAUDE_CODE_* + </Card> + + <Card title="Skills" icon="puzzle" href="#skills"> + SKILLS_ENABLED, SKILLS_MAX_CONTEXT_TOKENS + </Card> + + <Card title="Security" icon="shield" href="#security"> + SECRETS_MASTER_KEY, IRONCLAW_BASE_DIR + </Card> +</CardGroup> + +## Agent Settings + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `AGENT_NAME` | string | `ironclaw` | Agent name displayed in responses | +| `AGENT_MAX_PARALLEL_JOBS` | int | `5` | Maximum concurrent jobs | +| `AGENT_JOB_TIMEOUT_SECS` | int | `3600` | Job timeout in seconds (1 hour) | +| `AGENT_STUCK_THRESHOLD_SECS` | int | `300` | Time before job considered stuck (5 min) | +| `SELF_REPAIR_CHECK_INTERVAL_SECS` | int | `60` | Self-repair check frequency (1 min) | +| `SELF_REPAIR_MAX_ATTEMPTS` | int | `3` | Max repair attempts per job | +| `AGENT_USE_PLANNING` | bool | `true` | Enable planning before tool execution | +| `SESSION_IDLE_TIMEOUT_SECS` | int | `604800` | Session idle timeout (7 days) | +| `ALLOW_LOCAL_TOOLS` | bool | `false` | Allow filesystem/shell tools directly | +| `MAX_COST_PER_DAY_CENTS` | int | — | Daily spend limit (cents, e.g., 10000 = $100) | +| `MAX_ACTIONS_PER_HOUR` | int | — | Hourly action limit | +| `AGENT_MAX_TOOL_ITERATIONS` | int | `50` | Max tool calls per loop | +| `AGENT_AUTO_APPROVE_TOOLS` | bool | `false` | Skip tool approval (for benchmarks) | + +## Database + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DATABASE_BACKEND` | enum | `postgres` | Backend: `postgres` or `libsql` | +| `DATABASE_URL` | string | — | PostgreSQL connection URL | +| `DATABASE_POOL_SIZE` | int | `30` | Connection pool size | +| `DATABASE_SSLMODE` | enum | `prefer` | TLS mode: `disable`, `prefer`, `require` | +| `LIBSQL_PATH` | path | `~/.ironclaw/ironclaw.db` | libSQL database file | +| `LIBSQL_URL` | URL | — | Turso cloud sync URL | +| `LIBSQL_AUTH_TOKEN` | string | — | Turso auth token | + +### PostgreSQL Example + +```bash +export DATABASE_BACKEND=postgres +export DATABASE_URL="postgres://user:pass@localhost/ironclaw" +export DATABASE_SSLMODE=require +``` + +### libSQL Example + +```bash +export DATABASE_BACKEND=libsql +export LIBSQL_PATH="/home/user/.ironclaw/ironclaw.db" +``` + +### Turso Example + +```bash +export DATABASE_BACKEND=libsql +export LIBSQL_PATH="/home/user/.ironclaw/ironclaw.db" +export LIBSQL_URL="libsql://your-db.turso.io" +export LIBSQL_AUTH_TOKEN="your-auth-token" +``` + +## LLM / Inference + +### NEAR AI + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `NEARAI_BASE_URL` | URL | `https://private.near.ai` | NEAR AI Chat API base URL | +| `NEARAI_SESSION_TOKEN` | string | — | Session token for OAuth mode | +| `NEARAI_API_KEY` | string | — | API key for Cloud mode | +| `NEARAI_MODEL` | string | — | Default model (e.g., `claude-sonnet-4-20250514`) | +| `NEARAI_CHEAP_MODEL` | string | — | Cheaper model for simple tasks | +| `NEARAI_FALLBACK_MODEL` | string | — | Fallback if primary fails | + +### Anthropic + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `ANTHROPIC_API_KEY` | string | — | API key from console.anthropic.com | +| `ANTHROPIC_BASE_URL` | URL | — | Custom base URL (optional) | + +### OpenAI + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `OPENAI_API_KEY` | string | — | API key from platform.openai.com | +| `OPENAI_BASE_URL` | URL | — | Custom base URL (optional) | + +### Ollama + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `OLLAMA_BASE_URL` | URL | `http://localhost:11434` | Ollama server URL | + +### OpenAI-Compatible + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `LLM_BACKEND` | string | — | Set to `openai_compatible` | +| `LLM_BASE_URL` | URL | — | API endpoint (e.g., `https://api.openrouter.ai`) | +| `LLM_API_KEY` | string | — | API key | +| `LLM_EXTRA_HEADERS` | string | — | Extra headers (format: `Key:Value,Key2:Value2`) | + +### Tinfoil + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `TINFOIL_API_KEY` | string | — | Tinfoil API key | +| `TINFOIL_MODEL` | string | `kimi-k2-5` | Model to use | + +## Channels + +### Web Gateway + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `GATEWAY_ENABLED` | bool | `true` | Enable web UI | +| `GATEWAY_HOST` | string | `127.0.0.1` | Bind host | +| `GATEWAY_PORT` | int | `3000` | Port number | +| `GATEWAY_AUTH_TOKEN` | string | random | Bearer token for auth | + +### HTTP Webhook + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `HTTP_HOST` | string | `127.0.0.1` | Bind host | +| `HTTP_PORT` | int | `8080` | Port number | +| `HTTP_WEBHOOK_SECRET` | string | — | Shared secret for validation | + +<Warning> +The HTTP webhook binds to `127.0.0.1:8080` by default. To receive webhooks from external services, set `HTTP_HOST=0.0.0.0`. +</Warning> + +### Terminal UI + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CLI_ENABLED` | bool | `true` | Enable TUI on startup | + +### WASM Channels + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `WASM_CHANNELS_ENABLED` | bool | `true` | Enable WASM channels | +| `WASM_CHANNELS_DIR` | path | `~/.ironclaw/channels` | Channel modules directory | +| `TELEGRAM_OWNER_ID` | int | — | Telegram owner user ID (legacy) | + +### Signal + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `SIGNAL_HTTP_URL` | URL | — | signal-cli daemon URL | +| `SIGNAL_ACCOUNT` | string | — | Phone number (+1234567890) | +| `SIGNAL_ALLOW_FROM` | list | — | Allowed senders (comma-separated) | +| `SIGNAL_ALLOW_FROM_GROUPS` | list | — | Allowed groups | +| `SIGNAL_DM_POLICY` | enum | `pairing` | DM policy: `open`, `allowlist`, `pairing` | +| `SIGNAL_GROUP_POLICY` | enum | `allowlist` | Group policy: `allowlist`, `open`, `disabled` | +| `SIGNAL_IGNORE_ATTACHMENTS` | bool | `false` | Skip attachment-only messages | +| `SIGNAL_IGNORE_STORIES` | bool | `true` | Skip story messages | + +## Embeddings + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `EMBEDDING_ENABLED` | bool | `false` | Enable semantic search | +| `EMBEDDING_PROVIDER` | enum | `nearai` | Provider: `nearai` or `openai` | +| `EMBEDDING_MODEL` | string | `text-embedding-3-small` | Embedding model | +| `OPENAI_API_KEY` | string | — | Required if using OpenAI embeddings | + +## Sandbox + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `SANDBOX_ENABLED` | bool | `true` | Enable Docker sandbox | +| `SANDBOX_POLICY` | enum | `readonly` | Policy: `readonly`, `workspace_write`, `full_access` | +| `SANDBOX_TIMEOUT_SECS` | int | `120` | Command timeout | +| `SANDBOX_MEMORY_LIMIT_MB` | int | `2048` | Memory limit per container | +| `SANDBOX_CPU_SHARES` | int | `1024` | CPU shares (relative weight) | +| `SANDBOX_IMAGE` | string | `ironclaw-worker:latest` | Docker image | +| `SANDBOX_AUTO_PULL` | bool | `true` | Auto-pull missing images | +| `SANDBOX_EXTRA_DOMAINS` | list | — | Additional allowed domains | + +## Claude Code + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CLAUDE_CODE_ENABLED` | bool | `false` | Enable Claude Code mode | +| `CLAUDE_CONFIG_DIR` | path | `~/.claude` | Claude config directory | +| `CLAUDE_CODE_MODEL` | string | `sonnet` | Claude model | +| `CLAUDE_CODE_MAX_TURNS` | int | `50` | Max agentic turns | +| `CLAUDE_CODE_MEMORY_LIMIT_MB` | int | `4096` | Container memory limit | +| `CLAUDE_CODE_ALLOWED_TOOLS` | list | — | Allowed tool patterns | + +## Skills + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `SKILLS_ENABLED` | bool | `true` | Enable skills system | +| `SKILLS_MAX_CONTEXT_TOKENS` | int | `4000` | Max prompt budget for skills | + +## Heartbeat + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `HEARTBEAT_ENABLED` | bool | `false` | Enable periodic execution | +| `HEARTBEAT_INTERVAL_SECS` | int | `1800` | Interval in seconds (30 min) | + +## Routines + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `ROUTINES_ENABLED` | bool | `true` | Enable scheduled/reactive tasks | +| `ROUTINES_CRON_INTERVAL` | int | `15` | Cron tick interval (seconds) | +| `ROUTINES_MAX_CONCURRENT` | int | `10` | Max concurrent routines | + +## Security + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `SECRETS_MASTER_KEY` | string | — | Master key for encryption (env var mode) | +| `IRONCLAW_BASE_DIR` | path | `~/.ironclaw` | Data directory | +| `IRONCLAW_OAUTH_CALLBACK_URL` | URL | `http://127.0.0.1:9876` | OAuth callback URL | + +## Provider Registry Variables + +The provider registry in `providers.json` defines additional LLM provider environment variables. Use these with `LLM_BACKEND=<provider id>` unless the provider is selected through database-backed settings or the UI. + +| Provider | API key | Base URL | Model | Extra headers | +|----------|---------|----------|-------|---------------| +| `openai` | `OPENAI_API_KEY` | `OPENAI_BASE_URL` | `OPENAI_MODEL` | — | +| `anthropic` | `ANTHROPIC_API_KEY` | `ANTHROPIC_BASE_URL` | `ANTHROPIC_MODEL` | — | +| `ollama` | — | `OLLAMA_BASE_URL` | `OLLAMA_MODEL` | — | +| `openai_compatible` | `LLM_API_KEY` | `LLM_BASE_URL` | `LLM_MODEL` | `LLM_EXTRA_HEADERS` | +| `github_copilot` | `GITHUB_COPILOT_TOKEN` | — | `GITHUB_COPILOT_MODEL` | `GITHUB_COPILOT_EXTRA_HEADERS` | +| `tinfoil` | `TINFOIL_API_KEY` | — | `TINFOIL_MODEL` | — | +| `openrouter` | `OPENROUTER_API_KEY` | — | `OPENROUTER_MODEL` | — | +| `groq` | `GROQ_API_KEY` | — | `GROQ_MODEL` | — | +| `nvidia` | `NVIDIA_API_KEY` | — | `NVIDIA_MODEL` | — | +| `venice` | `VENICE_API_KEY` | — | `VENICE_MODEL` | — | +| `together` | `TOGETHER_API_KEY` | — | `TOGETHER_MODEL` | — | +| `fireworks` | `FIREWORKS_API_KEY` | — | `FIREWORKS_MODEL` | — | +| `deepseek` | `DEEPSEEK_API_KEY` | — | `DEEPSEEK_MODEL` | — | +| `zai` | `ZAI_API_KEY` | — | `ZAI_MODEL` | — | +| `cerebras` | `CEREBRAS_API_KEY` | — | `CEREBRAS_MODEL` | — | +| `sambanova` | `SAMBANOVA_API_KEY` | — | `SAMBANOVA_MODEL` | — | +| `gemini` | `GEMINI_API_KEY` | — | `GEMINI_MODEL` | — | +| `ionet` | `IONET_API_KEY` | — | `IONET_MODEL` | — | +| `mistral` | `MISTRAL_API_KEY` | — | `MISTRAL_MODEL` | — | +| `yandex` | `YANDEX_API_KEY` | — | `YANDEX_MODEL` | `YANDEX_EXTRA_HEADERS` | +| `minimax` | `MINIMAX_API_KEY` | `MINIMAX_BASE_URL` | `MINIMAX_MODEL` | — | +| `cloudflare` | `CLOUDFLARE_API_KEY` | `CLOUDFLARE_BASE_URL` | `CLOUDFLARE_MODEL` | — | + +## Advanced Configuration Variables + +These variables are resolved by the current `src/config/` modules but are less commonly needed during initial setup. + +### Agent, tenancy, and execution limits + +| Variable | Description | +|----------|-------------| +| `DEFAULT_TIMEZONE` | Default IANA timezone for new sessions. | +| `AGENT_MAX_TOKENS_PER_JOB` | Maximum tokens per job; `0` means unlimited. | +| `AGENT_MULTI_TENANT` | Marks the deployment as multi-tenant. | +| `MAX_COST_PER_USER_PER_DAY_CENTS` | Per-user daily cost limit. | +| `MAX_JOBS_PER_USER` | Per-user job limit. | +| `TENANT_MAX_LLM_CONCURRENT` | Per-user concurrent LLM call limit. | +| `TENANT_MAX_JOBS_CONCURRENT` | Per-user concurrent job limit. | +| `ENGINE_V2` | Enables Engine V2 routing. | + +### Gateway, memory layers, and TUI + +| Variable | Description | +|----------|-------------| +| `HTTP_ENABLED` | Enables HTTP webhook channel even without host/port env overrides. | +| `CLI_MODE` | CLI mode; `tui` enables the TUI channel. | +| `TUI_THEME` | TUI theme name. | +| `TUI_SIDEBAR` | Controls TUI sidebar visibility. | +| `GATEWAY_MAX_CONNECTIONS` | Maximum concurrent SSE/WebSocket connections. | +| `SSE_BROADCAST_BUFFER` | SSE broadcast channel buffer size, clamped by the gateway. | +| `GATEWAY_OIDC_ENABLED` | Enables gateway OIDC/JWT authentication. | +| `GATEWAY_OIDC_JWKS_URL` | JWKS URL required when OIDC is enabled. | +| `GATEWAY_OIDC_HEADER` | Header containing the gateway JWT. | +| `GATEWAY_OIDC_ISSUER` | Optional expected JWT issuer. | +| `GATEWAY_OIDC_AUDIENCE` | Optional expected JWT audience. | +| `MEMORY_LAYERS` | JSON memory layer definitions. | +| `WORKSPACE_READ_SCOPES` | Comma-separated extra workspace scopes readable by the gateway/workspace. | + +### Signal channel + +| Variable | Description | +|----------|-------------| +| `SIGNAL_ENABLED` | Enables Signal channel configuration. | +| `SIGNAL_GROUP_ALLOW_FROM` | Allowed group message senders; falls back to `SIGNAL_ALLOW_FROM` when unset. | + +### LLM reliability, routing, cache, and special backends + +| Variable | Description | +|----------|-------------| +| `LLM_MODEL` | Generic selected model for compatible/custom providers. | +| `LLM_CHEAP_MODEL` | Generic cheap model override. | +| `LLM_REQUEST_TIMEOUT_SECS` | LLM request timeout. | +| `LLM_MAX_RETRIES` | Generic retry count. | +| `LLM_CIRCUIT_BREAKER_THRESHOLD` | Generic circuit breaker threshold. | +| `LLM_CIRCUIT_BREAKER_RECOVERY_SECS` | Generic circuit breaker recovery window. | +| `LLM_FAILOVER_THRESHOLD` | Failover threshold. | +| `LLM_FAILOVER_COOLDOWN_SECS` | Failover cooldown window. | +| `LLM_RESPONSE_CACHE_ENABLED` | Generic response cache enable flag. | +| `LLM_RESPONSE_CACHE_TTL_SECS` | Generic response cache TTL. | +| `LLM_RESPONSE_CACHE_MAX_ENTRIES` | Generic response cache capacity. | +| `SMART_ROUTING_CASCADE` | Enables smart-routing cascade behavior. | +| `NEARAI_AUTH_URL` | NEAR AI auth/session base URL. | +| `NEARAI_SESSION_PATH` | NEAR AI session file path. | +| `NEARAI_MAX_RETRIES` | NEAR AI retry count. | +| `ANTHROPIC_OAUTH_TOKEN` | Anthropic OAuth token. | +| `ANTHROPIC_CACHE_RETENTION` | Anthropic cache retention mode. | +| `BEDROCK_REGION` | AWS Bedrock region. | +| `BEDROCK_MODEL` | AWS Bedrock model. | +| `BEDROCK_CROSS_REGION` | Bedrock cross-region inference prefix. | +| `AWS_PROFILE` | AWS profile used for Bedrock credentials. | +| `GEMINI_CREDENTIALS_PATH` | Gemini OAuth credentials path. | +| `LLM_USE_CODEX_AUTH` | Enables Codex auth flow. | +| `CODEX_AUTH_PATH` | Codex auth state path. | +| `OPENAI_CODEX_MODEL` | OpenAI Codex model. | +| `OPENAI_CODEX_AUTH_URL` | OpenAI Codex auth endpoint. | +| `OPENAI_CODEX_API_URL` | OpenAI Codex API endpoint. | +| `OPENAI_CODEX_CLIENT_ID` | OpenAI Codex OAuth client ID. | +| `OPENAI_CODEX_SESSION_PATH` | OpenAI Codex session path. | +| `OPENAI_CODEX_REFRESH_MARGIN_SECS` | OpenAI Codex refresh margin. | +| `CIRCUIT_BREAKER_THRESHOLD` | Legacy NearAI circuit breaker threshold. | +| `CIRCUIT_BREAKER_RECOVERY_SECS` | Legacy NearAI circuit breaker recovery window. | +| `RESPONSE_CACHE_ENABLED` | Legacy NearAI response cache flag. | +| `RESPONSE_CACHE_TTL_SECS` | Legacy NearAI response cache TTL. | +| `RESPONSE_CACHE_MAX_ENTRIES` | Legacy NearAI response cache capacity. | + +### Embeddings and search + +| Variable | Description | +|----------|-------------| +| `EMBEDDING_DIMENSION` | Explicit embedding vector dimension. | +| `EMBEDDING_BASE_URL` | OpenAI-compatible embedding base URL. | +| `EMBEDDING_CACHE_SIZE` | Embedding LRU cache capacity. | +| `SEARCH_FUSION_STRATEGY` | Workspace search fusion strategy. | +| `SEARCH_VECTOR_WEIGHT` | Vector score weight. | +| `SEARCH_FTS_WEIGHT` | Full-text score weight. | +| `SEARCH_RRF_K` | Reciprocal-rank-fusion constant. | +| `SEARCH_REASONING_ENABLED` | Enables reasoning over search results. | + +### Sandbox, Claude Code, ACP, and WASM tools + +| Variable | Description | +|----------|-------------| +| `SANDBOX_ALLOW_FULL_ACCESS` | Required second opt-in for `SANDBOX_POLICY=full_access`. | +| `SANDBOX_REAPER_INTERVAL_SECS` | Sandbox orphan reaper interval. | +| `SANDBOX_ORPHAN_THRESHOLD_SECS` | Age threshold for orphaned sandbox containers. | +| `ACP_ENABLED` | Enables ACP mode. | +| `ACP_TIMEOUT_SECS` | ACP command timeout. | +| `ACP_MEMORY_LIMIT_MB` | ACP memory limit. | +| `WASM_ENABLED` | Enables WASM tools. | +| `WASM_TOOLS_DIR` | WASM tools directory. | +| `WASM_CACHE_DIR` | WASM compiled cache directory. | +| `WASM_CACHE_COMPILED` | Enables compiled WASM caching. | +| `WASM_DEFAULT_TIMEOUT_SECS` | Default WASM execution timeout. | +| `WASM_DEFAULT_MEMORY_LIMIT` | Default WASM memory limit. | +| `WASM_DEFAULT_FUEL_LIMIT` | Default WASM fuel limit. | + +### Skills, routines, heartbeat, and builder mode + +| Variable | Description | +|----------|-------------| +| `SKILLS_DIR` | Local trusted skills directory. | +| `SKILLS_INSTALLED_DIR` | Registry-installed skills directory. | +| `SKILLS_MAX_ACTIVE` | Maximum active skills. | +| `SKILLS_MAX_SCAN_DEPTH` | Skill directory scan depth. | +| `HEARTBEAT_FIRE_AT` | Time-of-day heartbeat schedule. | +| `HEARTBEAT_TIMEZONE` | Heartbeat timezone. | +| `HEARTBEAT_QUIET_START` | Quiet-hours start. | +| `HEARTBEAT_QUIET_END` | Quiet-hours end. | +| `HEARTBEAT_NOTIFY_CHANNEL` | Notification channel for heartbeat output. | +| `HEARTBEAT_NOTIFY_USER` | Notification user for heartbeat output. | +| `GATEWAY_USER_TOKENS` | Enables heartbeat integration with gateway user tokens. | +| `ROUTINES_DEFAULT_COOLDOWN` | Default routine cooldown. | +| `ROUTINES_MAX_TOKENS` | Routine token budget. | +| `ROUTINES_LIGHTWEIGHT_TOOLS` | Tool allowlist for lightweight routines. | +| `ROUTINES_LIGHTWEIGHT_MAX_ITERATIONS` | Max lightweight routine iterations. | +| `BUILDER_ENABLED` | Enables builder mode. | +| `BUILDER_DIR` | Builder workspace directory. | +| `BUILDER_TIMEOUT_SECS` | Builder timeout. | +| `BUILDER_MAX_ITERATIONS` | Builder max iterations. | +| `BUILDER_AUTO_REGISTER` | Auto-register builder artifacts. | + +### OAuth, relay, tunnel, transcription, safety, and hygiene + +| Variable | Description | +|----------|-------------| +| `OAUTH_ENABLED` | Enables web OAuth/social login. | +| `OAUTH_BASE_URL` | External OAuth base URL. | +| `OAUTH_ALLOWED_DOMAINS` | Allowed email/domain list for OAuth login. | +| `GOOGLE_CLIENT_ID` | Google OAuth client ID. | +| `GOOGLE_CLIENT_SECRET` | Google OAuth client secret. | +| `GOOGLE_ALLOWED_HD` | Google hosted-domain restriction. | +| `GITHUB_CLIENT_ID` | GitHub OAuth client ID. | +| `GITHUB_CLIENT_SECRET` | GitHub OAuth client secret. | +| `APPLE_CLIENT_ID` | Apple OAuth client ID. | +| `APPLE_TEAM_ID` | Apple team ID. | +| `APPLE_KEY_ID` | Apple key ID. | +| `APPLE_PRIVATE_KEY_PATH` | Apple private key path. | +| `APPLE_PRIVATE_KEY_PEM` | Apple private key PEM. | +| `NEAR_AUTH_ENABLED` | Enables NEAR auth. | +| `NEAR_AUTH_NETWORK` | NEAR network. | +| `NEAR_AUTH_RPC_URL` | NEAR RPC URL. | +| `TUNNEL_PROVIDER` | Tunnel provider selection. | +| `TUNNEL_URL` | Public tunnel URL override. | +| `TUNNEL_NGROK_TOKEN` | ngrok token. | +| `TUNNEL_NGROK_DOMAIN` | ngrok custom domain. | +| `TUNNEL_CF_TOKEN` | Cloudflare tunnel token. | +| `TUNNEL_TS_HOSTNAME` | Tailscale hostname. | +| `TUNNEL_TS_FUNNEL` | Tailscale funnel toggle. | +| `TUNNEL_CUSTOM_COMMAND` | Custom tunnel command. | +| `TUNNEL_CUSTOM_URL_PATTERN` | Custom tunnel URL extraction pattern. | +| `TUNNEL_CUSTOM_HEALTH_URL` | Custom tunnel health URL. | +| `CHANNEL_RELAY_URL` | External channel relay URL. | +| `CHANNEL_RELAY_API_KEY` | External channel relay API key. | +| `IRONCLAW_INSTANCE_ID` | Relay instance ID. | +| `RELAY_WEBHOOK_PATH` | Relay webhook path. | +| `RELAY_REQUEST_TIMEOUT_SECS` | Relay request timeout. | +| `TRANSCRIPTION_ENABLED` | Enables transcription. | +| `TRANSCRIPTION_PROVIDER` | Transcription provider. | +| `TRANSCRIPTION_MODEL` | Transcription model. | +| `TRANSCRIPTION_BASE_URL` | Transcription base URL. | +| `TRANSCRIPTION_API_KEY` | Transcription API key. | +| `SAFETY_MAX_OUTPUT_LENGTH` | Max tool/output length before safety truncation. | +| `SAFETY_INJECTION_CHECK_ENABLED` | Enables prompt-injection checks. | +| `MEMORY_HYGIENE_ENABLED` | Enables memory hygiene. | +| `MEMORY_HYGIENE_CADENCE_HOURS` | Memory hygiene cadence. | +| `MEMORY_HYGIENE_VERSION_KEEP_COUNT` | Memory hygiene versions to keep. | +| `MISSION_INSIGHTS_INTERVAL` | Mission insight generation interval. | + +### Bootstrap and process-level variables + +| Variable | Description | +|----------|-------------| +| `IRONCLAW_BASE_DIR` | Overrides the IronClaw data directory. | +| `IRONCLAW_OWNER_ID` | Stable owner scope for the instance. | +| `IRONCLAW_PROFILE` | Built-in profile selected during onboarding. | +| `IRONCLAW_OAUTH_CALLBACK_URL` | OAuth callback URL used by auth flows. | +| `NEARAI_SESSION_TOKEN` | NEAR AI session token; takes precedence over session file. | +| `IRONCLAW_PID_LOCK_PATH` | Process PID lock path override. | +| `IRONCLAW_PID_LOCK_HOLD_MS` | PID lock hold duration. | +| `IRONCLAW_PID_LOCK_CHILD` | Internal child-process PID lock marker. | + + +## Environment File Example + +Create `~/.ironclaw/.env`: + +```bash +# Database +DATABASE_BACKEND=libsql +LIBSQL_PATH=/home/user/.ironclaw/ironclaw.db + +# LLM (NEAR AI) +LLM_BACKEND=nearai + +# Web Gateway +GATEWAY_ENABLED=true +GATEWAY_HOST=127.0.0.1 +GATEWAY_PORT=3000 + +# Optional: Persistent auth token +GATEWAY_AUTH_TOKEN=your-secure-token-here + +# Sandbox +SANDBOX_ENABLED=true +SANDBOX_POLICY=workspace_write + +# Heartbeat +HEARTBEAT_ENABLED=true +HEARTBEAT_INTERVAL_SECS=1800 +``` + +## Configuration Commands + +```bash +# View current config +ironclaw config list + +# Get specific value +ironclaw config get llm.backend + +# Set value +ironclaw config set llm.backend nearai + +# Delete value (reset to default) +ironclaw config delete llm.backend +``` + +## Next Steps + +<CardGroup cols={2}> + <Card title="Database Backends" icon="database" href="/capabilities/database"> + PostgreSQL vs libSQL comparison + </Card> + + <Card title="LLM Providers" icon="brain" href="/capabilities/llm-providers"> + Provider-specific configuration + </Card> +</CardGroup> diff --git a/docs/capabilities/database.mdx b/docs/capabilities/database.mdx new file mode 100644 index 00000000000..0f6f8a6773c --- /dev/null +++ b/docs/capabilities/database.mdx @@ -0,0 +1,313 @@ +--- +title: Database Backends +description: PostgreSQL vs libSQL — choosing your database +--- + +IronClaw supports two database backends: **PostgreSQL** and **libSQL** (embedded SQLite). **PostgreSQL is the default** — if you don't set `DATABASE_BACKEND`, IronClaw will use PostgreSQL and require a `DATABASE_URL`. Choose libSQL for zero-config personal use. + +<Warning> +If you start IronClaw without configuring a database, you'll see: `DATABASE_URL is required`. Run `ironclaw onboard` or set the environment variables manually. +</Warning> + +## Quick Comparison + +| Feature | PostgreSQL (default) | libSQL | +|---------|----------------------|--------| +| **Setup** | Requires PostgreSQL server | Zero-dependency, auto-created | +| **Best For** | Production, multi-user | Personal use, single-user | +| **Search** | Hybrid (FTS + vector) | FTS only (vector via Turso) | +| **Scaling** | Horizontal (read replicas) | Single node | +| **Backup** | pg_dump, replication | File copy, Turso sync | +| **Size** | 100MB+ installed | ~5MB binary | + +## PostgreSQL + +Recommended for production deployments, multi-user scenarios, and high-throughput use cases. + +### Requirements + +- PostgreSQL 15 or later +- pgvector extension for embeddings + +### Installation + +<AccordionGroup> + <Accordion title="Docker Compose (recommended)" icon="container"> + The fastest way to get started. A `docker-compose.yml` is included in the IronClaw repository: + + ```bash + cd ironclaw + docker compose up -d postgres + ``` + + This starts `pgvector/pgvector:pg16` on `127.0.0.1:5432` with database, user, and password all set to `ironclaw`. + + <Warning> + These credentials are for local development only. Change `POSTGRES_PASSWORD` for any non-local deployment. + </Warning> + </Accordion> + + <Accordion title="Ubuntu/Debian" icon="terminal"> + ```bash + sudo apt install postgresql-15 postgresql-15-pgvector + sudo systemctl enable --now postgresql + ``` + </Accordion> + + <Accordion title="macOS (Homebrew)" icon="terminal"> + ```bash + brew install postgresql + brew install pgvector + brew services start postgresql + ``` + </Accordion> +</AccordionGroup> + +### Configuration + +```bash +# Create database +sudo -u postgres psql -c "CREATE DATABASE ironclaw;" +sudo -u postgres psql -c "CREATE USER ironclaw WITH PASSWORD 'your-password';" +sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE ironclaw TO ironclaw;" + +# Enable pgvector +sudo -u postgres psql -d ironclaw -c "CREATE EXTENSION IF NOT EXISTS vector;" +``` + +### IronClaw Configuration + +```bash +# Select backend (optional — postgres is the default) +# Accepted values: postgres, postgresql, pg | libsql, turso, sqlite +export DATABASE_BACKEND=postgres + +# Required for PostgreSQL +export DATABASE_URL="postgres://ironclaw:***@localhost/ironclaw" +export DATABASE_POOL_SIZE=30 # default; increase for high-throughput workloads +``` + +<Warning> +`DATABASE_URL` is **required** when using PostgreSQL. If not set, IronClaw will fail to start with the error: `DATABASE_URL is required — Run 'ironclaw onboard' or set DATABASE_URL environment variable`. +</Warning> + +Or in the wizard: +1. Select "PostgreSQL" +2. Enter connection string +3. Test connection + +### SSL Modes + +| Mode | Behavior | Use Case | +|------|----------|----------| +| `disable` | Never use TLS | Local development | +| `prefer` | Try TLS, fallback to plaintext | **Default** — works everywhere | +| `require` | Require TLS | Production with TLS | + +```bash +export DATABASE_SSLMODE=require +``` + +## libSQL + +Recommended for personal use, development, and single-user deployments. Zero setup required. + +### How It Works + +libSQL is an embedded SQLite-compatible database: +- Database is a single file (`~/.ironclaw/ironclaw.db`) +- No separate server process +- Auto-created on first connection +- Full SQLite feature set + +### IronClaw Configuration + +```bash +export DATABASE_BACKEND=libsql +export LIBSQL_PATH="/home/user/.ironclaw/ironclaw.db" +``` + +Or just use the wizard defaults: +1. Select "libSQL" +2. Accept default path +3. Done! + +### Turso Cloud Sync + +libSQL supports syncing to Turso for cloud backup: + +```bash +export DATABASE_BACKEND=libsql +export LIBSQL_PATH="/home/user/.ironclaw/ironclaw.db" +export LIBSQL_URL="libsql://your-db.turso.io" +export LIBSQL_AUTH_TOKEN="***" +``` + +<Warning> +`LIBSQL_AUTH_TOKEN` is **required** when `LIBSQL_URL` is set. IronClaw will fail to start without it. +</Warning> + +This keeps a local copy with automatic cloud sync. + +## Feature Comparison + +### Hybrid Search + +**PostgreSQL:** Full hybrid search (FTS + vector via RRF) +``` +Keyword matches + semantic similarity +Reciprocal Rank Fusion ranking +``` + +**libSQL:** FTS only (text search) +``` +Keyword matching via FTS5 +Vector search via Turso cloud only +``` + +### Embeddings + +Both backends support embeddings, but with different implementations: + +| Backend | Embeddings | Notes | +|---------|------------|-------| +| PostgreSQL | Yes | pgvector for vector storage | +| libSQL local | FTS only | No local vector storage | +| libSQL + Turso | Yes | Via Turso vector indexes | + +<Warning> +**Encryption at rest:** The local SQLite database stores conversation and workspace data in plaintext. Only secrets (API tokens) are encrypted with AES-256-GCM. If you handle sensitive data, use full-disk encryption (FileVault, LUKS, BitLocker) or choose PostgreSQL with TDE. +</Warning> + +## Migration + +### From libSQL to PostgreSQL + +<Warning> +SQLite and PostgreSQL have different SQL dialects. A raw `sqlite3 .dump` output may fail on PostgreSQL due to PRAGMA statements, type differences, and quoting. Use a conversion tool like [pgloader](https://pgloader.io/) or [pgloader Docker](https://hub.docker.com/r/dimitri/pgloader) for reliable migration. +</Warning> + +1. **Using pgloader (recommended):** + ```bash + pgloader ~/.ironclaw/ironclaw.db postgresql://ironclaw:***@localhost/ironclaw + ``` + +2. **Manual export (may require editing):** + ```bash + sqlite3 ~/.ironclaw/ironclaw.db ".dump" > ironclaw.sql + # Edit ironclaw.sql to remove PRAGMAs and fix type incompatibilities + psql -d ironclaw -f ironclaw.sql + ``` + +3. **Update IronClaw config:** + ```bash + export DATABASE_BACKEND=postgres + export DATABASE_URL="postgres://user:***@localhost/ironclaw" + ``` + +4. **Restart IronClaw** + +### From PostgreSQL to libSQL + +1. **Export:** + ```bash + pg_dump -h localhost -U ironclaw ironclaw > ironclaw.sql + ``` + +2. **Convert and import to SQLite** (requires conversion tools) + +3. **Update IronClaw config** + +## When to Choose Which + +### Choose libSQL if: + +- Running IronClaw on a personal laptop/desktop +- Single-user deployment +- Want zero database administration +- Don't need horizontal scaling +- FTS-only search is sufficient + +### Choose PostgreSQL if: + +- Production multi-user deployment +- Need hybrid (FTS + vector) search locally +- High-throughput scenario +- Existing PostgreSQL infrastructure +- Require advanced backup/recovery +- Team or shared deployment + +## Backup + +### PostgreSQL + +```bash +# Backup +pg_dump -h localhost -U ironclaw ironclaw > backup.sql + +# Restore +psql -d ironclaw -f backup.sql +``` + +### libSQL + +```bash +# Backup (simple file copy) +cp ~/.ironclaw/ironclaw.db ~/.ironclaw/ironclaw.db.backup + +# Restore +cp ~/.ironclaw/ironclaw.db.backup ~/.ironclaw/ironclaw.db + +# With Turso: automatic cloud backup +``` + +## Troubleshooting + +<AccordionGroup> + <Accordion title="pgvector not found" icon="x-circle"> + ```bash + # Install pgvector + sudo apt install postgresql-15-pgvector + + # Or compile manually + git clone https://github.com/pgvector/pgvector.git + cd pgvector + make + sudo make install + ``` + </Accordion> + + <Accordion title="Database file is locked" icon="lock"> + ```bash + # Find and kill process + lsof ~/.ironclaw/ironclaw.db + kill -9 <pid> + + # Or wait for it to release + ``` + </Accordion> + + <Accordion title="Connection refused (PostgreSQL)" icon="network"> + ```bash + # Check PostgreSQL is running + sudo systemctl status postgresql + + # Check listen addresses + sudo -u postgres psql -c "SHOW listen_addresses;" + + # Should be '*' or 'localhost' + ``` + </Accordion> +</AccordionGroup> + +## Next Steps + +<CardGroup cols={2}> + <Card title="Configuration" icon="settings" href="/capabilities/configuration"> + Full environment variable reference + </Card> + + <Card title="VPS Install" icon="server" href="/infrastructure/droplet"> + Production deployment guide with PostgreSQL + </Card> +</CardGroup> diff --git a/docs/docs.json b/docs/docs.json index aec033a6dc7..773f97a95a4 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -51,6 +51,8 @@ "group": "Core Capabilities", "pages": [ "security", + "capabilities/database", + "capabilities/configuration", { "group": "Memory", "pages": [ @@ -98,6 +100,13 @@ } ] }, + { + "group": "API", + "icon": "code", + "pages": [ + "api/responses" + ] + }, { "group": "Extensions (Tools)", "icon": "hammer", diff --git a/docs/drafts/install/docker.mdx b/docs/drafts/install/docker.mdx index c6486f440d5..3d4a74c3bb1 100644 --- a/docs/drafts/install/docker.mdx +++ b/docs/drafts/install/docker.mdx @@ -31,8 +31,8 @@ docker run -d \ # -v /var/run/docker.sock:/var/run/docker.sock \ -p 3000:3000 \ -p 8080:8080 \ - # Pin to a specific IronClaw version for reproducible, rollback-friendly deployments. - nearai/ironclaw:latest + # Replace :latest with a specific version tag for reproducible, rollback-friendly deployments. + nearaidev/ironclaw:latest ``` ## Docker Compose @@ -44,8 +44,8 @@ version: '3.8' services: ironclaw: - # Pin to a specific IronClaw version for reproducible, rollback-friendly deployments. - image: nearai/ironclaw:latest + # Replace :latest with a specific version tag for reproducible, rollback-friendly deployments. + image: nearaidev/ironclaw:latest container_name: ironclaw restart: unless-stopped @@ -121,7 +121,7 @@ docker run -d \ -v ~/.ironclaw:/home/ironclaw/.ironclaw \ -v /var/run/docker.sock:/var/run/docker.sock \ -p 3000:3000 \ - nearai/ironclaw:latest + nearaidev/ironclaw:latest ``` See [Configuration Reference](/setup/configuration) for all options. @@ -195,7 +195,7 @@ server { ```bash # Pull latest image -docker pull nearai/ironclaw:latest +docker pull nearaidev/ironclaw:latest # Recreate container docker stop ironclaw @@ -205,7 +205,7 @@ docker run -d \ -v ~/.ironclaw:/home/ironclaw/.ironclaw \ -v /var/run/docker.sock:/var/run/docker.sock \ -p 3000:3000 \ - nearai/ironclaw:latest + nearaidev/ironclaw:latest # Or with docker compose docker compose pull diff --git a/docs/drafts/install/uninstalling.mdx b/docs/drafts/install/uninstalling.mdx index 046218b7477..83b77c96b48 100644 --- a/docs/drafts/install/uninstalling.mdx +++ b/docs/drafts/install/uninstalling.mdx @@ -51,7 +51,7 @@ brew services stop ironclaw # Homebrew docker rm ironclaw # Remove image - docker rmi nearai/ironclaw:latest + docker rmi nearaidev/ironclaw:latest ``` </Tab> </Tabs> diff --git a/docs/drafts/install/updating.mdx b/docs/drafts/install/updating.mdx index b27fe806717..c58d615aab4 100644 --- a/docs/drafts/install/updating.mdx +++ b/docs/drafts/install/updating.mdx @@ -43,7 +43,7 @@ ironclaw --version <Tab title="Docker"> ```bash # Pull latest image - docker pull nearai/ironclaw:latest + docker pull nearaidev/ironclaw:latest # Recreate container docker stop ironclaw @@ -115,7 +115,7 @@ If you need to rollback: curl -fsSL https://install.ironclaw.ai | bash -s -- --version 0.12.0 # Docker -docker pull nearai/ironclaw:v0.12.0 +docker pull nearaidev/ironclaw:v0.12.0 ``` <Warning> diff --git a/docs/drafts/platforms/docker-compose.mdx b/docs/drafts/platforms/docker-compose.mdx index f84d0e123f5..7982c8e4ced 100644 --- a/docs/drafts/platforms/docker-compose.mdx +++ b/docs/drafts/platforms/docker-compose.mdx @@ -21,7 +21,7 @@ version: "3.9" services: ironclaw: - image: nearai/ironclaw:latest + image: nearaidev/ironclaw:latest container_name: ironclaw restart: unless-stopped env_file: diff --git a/docs/drafts/setup/configuration.mdx b/docs/drafts/setup/configuration.mdx index 8c86049a78a..167dfb442e8 100644 --- a/docs/drafts/setup/configuration.mdx +++ b/docs/drafts/setup/configuration.mdx @@ -4,6 +4,8 @@ sidebarTitle: Configuration description: Complete environment variable reference for IronClaw --- +<!-- ⚠️ SUPERSEDED — this page has been promoted to capabilities/configuration.mdx with corrections (10 wrong defaults fixed, 6 phantom env vars removed). Do not edit this file. --> + IronClaw is configured primarily through environment variables. This page documents all available configuration options. ## Two-Layer Configuration diff --git a/docs/drafts/setup/database.mdx b/docs/drafts/setup/database.mdx index da1a79c8008..d600b5c19f3 100644 --- a/docs/drafts/setup/database.mdx +++ b/docs/drafts/setup/database.mdx @@ -3,6 +3,8 @@ title: Database Backends description: PostgreSQL vs libSQL — choosing your database --- +<!-- ⚠️ SUPERSEDED — this page has been promoted to capabilities/database.mdx with corrections. Do not edit this file. --> + IronClaw supports two database backends: **PostgreSQL** and **libSQL** (embedded SQLite). Choose based on your deployment needs. ## Quick Comparison diff --git a/docs/onboard.mdx b/docs/onboard.mdx index 5214e90bdf4..64988da8092 100644 --- a/docs/onboard.mdx +++ b/docs/onboard.mdx @@ -33,9 +33,9 @@ ironclaw onboard </Step> -<Step title="Select the Database Path"> +<Step title="Select the Database"> -The wizard will first ask you to select a path for the agent's database, by default `/home/agent/.ironclaw/ironclaw.db`. This is where the agent will store your configuration. +IronClaw supports two database backends: **PostgreSQL** (default, recommended for production) and **libSQL** (embedded SQLite, zero-config). The wizard will guide you through setup — see the [Database Backends](/capabilities/database) page for details. </Step> diff --git a/docs/plans/2026-03-25-wechat-integration-design.md b/docs/plans/2026-03-25-wechat-integration-design.md new file mode 100644 index 00000000000..af3adbdb432 --- /dev/null +++ b/docs/plans/2026-03-25-wechat-integration-design.md @@ -0,0 +1,264 @@ +# WeChat Integration Design + +**Date:** 2026-03-25 +**Status:** Ready for implementation +**Goal:** Add WeChat support to IronClaw using the same upstream iLink Bot protocol as `@tencent-weixin/openclaw-weixin`, while keeping the implementation aligned with IronClaw's extension-first channel architecture. + +--- + +## Upstream Baseline + +The current upstream npm package is `@tencent-weixin/openclaw-weixin` version `2.0.1`. + +From the package README and source, the upstream WeChat channel does all of the following: + +- logs in by QR code against `https://ilinkai.weixin.qq.com` +- receives inbound messages by long-polling `ilink/bot/getupdates` +- sends outbound messages through `ilink/bot/sendmessage` +- uses `ilink/bot/getconfig` and `ilink/bot/sendtyping` for typing indicators +- uses `ilink/bot/getuploadurl` for media uploads +- persists `get_updates_buf` for long-poll resume +- persists `context_token` so replies stay attached to the right WeChat session +- supports multiple logged-in WeChat bot accounts +- treats WeChat as a direct-message-only channel +- block-sends replies instead of token streaming + +This design treats that upstream behavior as the capability boundary. We should not add scope based on features the upstream plugin does not have. + +--- + +## Implementation Direction + +IronClaw should **not** try to load the upstream OpenClaw plugin directly. + +Instead, IronClaw should implement a **native channel extension** under `channels-src/wechat/` and only extend the host/runtime where that support is generic and reusable. + +### Why not host the npm plugin directly + +- The upstream package depends on `openclaw/plugin-sdk/*` APIs and runtime contracts that IronClaw does not have. +- It assumes OpenClaw-specific lifecycle concepts such as `gateway.startAccount`. +- Recreating an OpenClaw-compatible Node plugin host inside IronClaw would be more work and more fragile than implementing the protocol directly. + +### Why `channels-src/wechat/` + +- It matches the existing layering used by other platform channels. +- It keeps platform protocol logic out of host-owned core modules. +- It leaves room for the channel to move outside this repo later without changing the host model. + +Recommended layout: + +```text +channels-src/ + wechat/ + Cargo.toml + build.sh + wechat.capabilities.json + src/ + lib.rs + api.rs + auth.rs + state.rs + types.rs +``` + +--- + +## Phase 1 Scope + +Phase 1 is a **single-account** implementation of the upstream WeChat channel. + +The point of this phase is to keep the channel aligned with upstream behavior while removing the one biggest source of host/runtime complexity: multi-account lifecycle. + +### Must-have in Phase 1 + +- QR code login +- one connected WeChat bot account +- direct-message text receive/send +- `getupdates` long-poll loop +- `sendmessage` outbound replies +- typing indicators via `getconfig` and `sendtyping` +- inbound image download/decrypt for vision +- outbound image upload/send via `getuploadurl` +- inbound file, voice, and video attachment handling +- outbound video and generic file attachment sending +- `context_token` persistence +- `get_updates_buf` persistence +- login persistence across restart +- extension-first packaging under `channels-src/wechat/` + +### Explicit simplification from upstream + +- multi-account support is deferred + +### Follow-up after Phase 1 + +These are upstream features, so they belong on the roadmap, but they do not need to block the first implementation cut: + +- multi-account lifecycle and account management UX + +We should not spend time listing non-goals that come from outside the upstream capability boundary. + +--- + +## Proposed Architecture + +```mermaid +flowchart LR + A["Core WASM channel host"] --> B["WeChat channel extension"] + C["Generic login UI/API"] --> D["QR login session"] + D --> E["Secret storage"] + E --> B + B --> F["getupdates long-poll"] + F --> G["IncomingMessage"] + G --> H["ChannelManager -> Agent"] + H --> I["sendmessage reply"] +``` + +### Extension responsibilities + +`channels-src/wechat/` should own: + +- iLink API request/response types +- QR login protocol calls +- long-polling `getupdates` +- `context_token` storage and lookup +- outbound `sendmessage` +- WeChat-specific status/error mapping + +### Host responsibilities + +IronClaw core should only own reusable pieces: + +- installing and activating the WASM channel +- generic secret persistence +- generic QR/device-login session handling for channels +- exposing login flow through authenticated UI/API +- starting and polling the channel runtime + +--- + +## Data And State Model + +Phase 1 is single-account, so state should stay simple. + +### Secrets + +- `wechat_bot_token` + +This is written after QR login succeeds and reused on restart. + +### Channel state + +Under the channel workspace prefix, persist: + +- `state/get_updates_buf.json` +- `state/context_tokens.json` + +`context_tokens.json` maps the WeChat peer to its latest `context_token`. + +### Inbound message mapping + +For each inbound WeChat DM: + +- `channel = "wechat"` +- `user_id = <wechat sender id>` or owner scope if it is the bound owner +- `thread_id = Some("wechat:<sender_id>")` +- `conversation_scope_id = Some("wechat:<sender_id>")` + +`metadata_json` should include: + +- `from_user_id` +- `to_user_id` +- `message_id` +- `context_token` + +That is enough for `on_respond()` to send the reply back to the right peer. + +--- + +## Minimal Host Uplift + +The current extension host is close, but Phase 1 still needs one important addition: a generic interactive login flow for channels. + +Minimum host support needed: + +1. Start a channel login session. +2. Return QR payload plus a session identifier. +3. Poll login session status. +4. On success, write the returned token to channel secrets. +5. Reload or reactivate the channel so polling starts automatically. + +This should be added as a generic channel-auth capability, not as WeChat-specific core logic. + +--- + +## User Flow + +Phase 1 should be **web-first**, because the target user is a normal WeChat user rather than a CLI-only operator. + +1. Install or enable the `wechat` channel extension. +2. Click "Connect WeChat". +3. Web UI requests a login session from the host. +4. Web UI displays the QR code. +5. User scans and confirms on their phone. +6. Host stores `wechat_bot_token`. +7. Channel reloads and starts polling. +8. User sends a DM in WeChat and receives IronClaw replies there. + +CLI support can still exist for development, but it should not be the primary Phase 1 UX. + +--- + +## Message Handling Semantics + +### Inbound + +On each poll: + +1. load `get_updates_buf` +2. call `getupdates` +3. persist the new cursor if present +4. normalize inbound text messages into `IncomingMessage` +5. persist the latest `context_token` for that peer +6. emit the message to the agent + +### Outbound + +On response: + +1. read peer routing info from `metadata_json` +2. load the latest `context_token` +3. convert the response to plain text if needed +4. send one coalesced text reply via `sendmessage` + +This matches the upstream channel's block-send behavior. + +--- + +## Testing Plan + +### Unit tests + +- QR login response parsing +- `get_updates_buf` round-trip +- `context_token` round-trip +- inbound message normalization +- outbound metadata routing + +### Integration tests + +Use a mock iLink server to cover: + +- QR login success and expiry +- restart without re-login +- inbound poll -> agent -> outbound text reply +- cursor resume after restart + +--- + +## Phase 2 + +After Phase 1 is stable, add the upstream features we intentionally deferred: + +- multi-account support +- media upload/send diff --git a/docs/plans/2026-04-01-wecom-channel-plan.md b/docs/plans/2026-04-01-wecom-channel-plan.md new file mode 100644 index 00000000000..8c318470783 --- /dev/null +++ b/docs/plans/2026-04-01-wecom-channel-plan.md @@ -0,0 +1,57 @@ +# WeCom Channel Plan + +## Current PR Scope + +This PR keeps WeCom focused on the intelligent bot path: + +- inbound messages and events arrive through the WeCom AI Bot WebSocket protocol +- outbound replies use the same bot WebSocket route +- generated media is uploaded and sent with bot upload/send commands +- pairing, `allow_from`, private chat, and group chat session isolation are handled in the channel +- self-built application callback inbound and Agent API outbound are deferred to a separate PR + +The goal is to match the OpenClaw community shape at the product level, where the chat bot is the primary user-facing entry point, without mixing the self-built app transport into the first reviewable slice. + +## Implemented + +- Standalone `wecom` WASM channel scaffold and registry entry. +- Bundled channel wiring and setup flow for `wecom_bot_id` and `wecom_bot_secret`. +- Host WebSocket runtime protocol support for WeCom AI Bot sessions. +- WebSocket inbound handling for text, markdown-like text, image, file, video, mixed messages, quote context, and selected interactive events. +- Inbound media hydration from WeCom-provided encrypted media URLs into IronClaw attachments. +- Attachment-only merge window so a file/image followed by text becomes one agent turn. +- Bot outbound text streaming replies. +- Bot outbound media upload/send for generated image, voice, video, and file attachments, with size guards and chunked upload state. +- Pairing flow that hides pairing codes in groups and asks users to DM the bot for the code. +- Conversation scoping that separates direct chats from group chats. +- Status/error notifications back through the active WebSocket route when model/provider failures happen. + +## Security Model + +- Only `wecom_bot_id` and `wecom_bot_secret` are required for this PR. +- Unknown users are blocked by default through pairing unless `dm_policy = "open"` or `allow_from` permits them. +- Group chats do not reveal pairing codes. A group message from an unapproved user only gets a "please DM the bot" prompt. +- Group conversations are scoped by WeCom `chatid`; private conversations are scoped by WeCom `userid`. +- The channel does not expose a WeCom HTTP callback endpoint in this PR. +- HTTP egress is limited to WeCom OpenWS media URLs and related object storage hosts needed for bot media retrieval. + +## Deferred + +- Self-built application callback verification and encrypted XML callback parsing. +- Agent API proactive send and media send. +- More exhaustive real-payload E2E coverage for every WeCom event type. +- Full account-level multi-bot isolation across all channels. +- TUI/UI affordances for editing `allow_from` and advanced group authorization policy. + +## Validation Targets + +- `cargo fmt --all -- --check` +- `cargo check --manifest-path channels-src/wecom/Cargo.toml --target wasm32-wasip2 -q --offline` +- `cargo clippy --manifest-path channels-src/wecom/Cargo.toml --target wasm32-wasip2 -- -D warnings` +- `cargo test --manifest-path channels-src/wecom/Cargo.toml -q --offline` +- Manual local gateway test with a real WeCom intelligent bot: + - private text reply + - generated image reply + - image/file inbound hydration + - group pairing prompt without leaking the code + - private pairing code approval path diff --git a/docs/plans/2026-05-01-codeact-inline-gate-await.md b/docs/plans/2026-05-01-codeact-inline-gate-await.md new file mode 100644 index 00000000000..1a72bc138c4 --- /dev/null +++ b/docs/plans/2026-05-01-codeact-inline-gate-await.md @@ -0,0 +1,531 @@ +# Inline Gate Await (Tier 0 + Tier 1) + +**Status:** Implemented — 2026-05-01 +**Date:** 2026-05-01 +**Owner:** engine v2 / bridge + +## Problem + +When a CodeAct (Tier 1, Monty) script makes a tool call that requires +approval, the user sees: + +``` +Error: RuntimeError: execution paused by gate 'approval' +``` + +instead of an approval prompt. The script aborts, the gate is never +surfaced to the user. + +Concretely: in `crates/ironclaw_engine/src/executor/scripting.rs`, the +async tool-resolve path (`resolve_tool_future`, line 1740-1766) catches +`Err(EngineError::GatePaused { .. })` from `EffectExecutor::execute_action`, +emits an `ApprovalRequested` event, and converts the gate to a +`MontyException::new(ExcType::RuntimeError, "execution paused by gate +'approval'")`. Python sees a catchable `RuntimeError`. If user code +doesn't catch, the script crashes with the message above. + +The synchronous preflight path (line 841-852) handles this correctly +for policy-level approval gates, but only by aborting the entire +CodeAct turn and returning `need_approval: Some(outcome)` to the +orchestrator. On approval the orchestrator re-runs the LLM step from +scratch, regenerates code, and re-executes CodeAct from the top — +which **double-executes any non-idempotent tool calls that ran before +the gated one** in the same script. + +## Goals + +1. Eliminate the `RuntimeError` leak. Gates never reach Python as + exceptions. +2. Eliminate double-execution of side effects on resume. A tool that + ran successfully before the gate must not run a second time. +3. Stay within the existing `PendingGateStore` / `/api/chat/gate/resolve` + / SSE machinery. Don't fork a parallel UI surface for CodeAct gates. + +## Non-goals + +- Monty VM serialization across process restarts. Out of scope. +- Surviving an IronClaw process restart while a CodeAct gate is + pending. Accepted loss: stranded gates expire after 30 min and the + user retries. +- Combining multiple parallel gate prompts into one approval card. + Future work — for now, gates are surfaced one at a time. + +## Design + +**Keep the Monty VM alive while the gate is pending.** The script's +local state, frame stack, and prior tool results are all in memory. +On approval, deliver the result back via the same `call.resume(...)` +path the VM was already going to use; the script continues from the +exact suspension point. No replay, no restart, no double execution. + +### Core mechanism: a `GateController` callback on `ThreadExecutionContext` + +```rust +// crates/ironclaw_engine/src/gate/mod.rs + +#[derive(Debug, Clone)] +pub struct GatePauseRequest { + pub thread_id: ThreadId, + pub user_id: String, + pub gate_name: String, + pub action_name: String, + pub call_id: String, + pub parameters: serde_json::Value, + pub resume_kind: ResumeKind, +} + +#[async_trait] +pub trait GateController: Send + Sync { + /// Pause execution until the gate is resolved by the user or + /// external system. The implementation is responsible for any + /// persistence, UI/SSE emission, and channel registration needed + /// to surface the gate. + async fn pause(&self, req: GatePauseRequest) -> GateResolution; +} + +/// Default impl that immediately cancels every pause request. Use for +/// post-resolution replay paths, mission protected writes, and tests. +pub struct CancellingGateController; + +impl CancellingGateController { + pub fn arc() -> Arc<dyn GateController> { Arc::new(Self) } +} + +#[async_trait] +impl GateController for CancellingGateController { + async fn pause(&self, _: GatePauseRequest) -> GateResolution { + GateResolution::Cancelled + } +} +``` + +`ThreadExecutionContext` gets: + +```rust +pub gate_controller: Arc<dyn GateController>, +``` + +**Required, not optional.** A previous iteration made it optional to +avoid touching test fixtures, but that left a fall-back path in the +executors that re-emitted the original `"execution paused by gate"` +RuntimeError when the field was `None`. Removing the `Option` makes it +a compile error to forget to wire a controller, and `CancellingGateController` +is the explicit drop-in for paths that don't pause — gates surface as +typed denials there, never as the legacy bug message. + +### Bridge implementation: `BridgeGateController` + +Wraps the existing `PendingGateStore` and adds an in-memory channel +registry: + +```rust +// src/bridge/gate_controller.rs + +pub struct BridgeGateController { + pending: Arc<PendingGateStore>, + sse: Option<Arc<SseManager>>, + channels: Arc<ChannelManager>, + auth_manager: Option<Arc<AuthManager>>, + extension_manager: Option<Arc<ExtensionManager>>, + tools: Arc<ToolRegistry>, + pending_resolutions: Mutex<HashMap<Uuid, oneshot::Sender<GateResolution>>>, + // … plus the user_id / channel / conversation_id that + // construct_pending_gate needs; supplied per controller instance, + // one controller per active thread execution. +} + +#[async_trait] +impl GateController for BridgeGateController { + async fn pause(&self, req: GatePauseRequest<'_>) -> GateResolution { + let request_id = Uuid::new_v4(); + let pending = self.build_pending_gate(request_id, &req); + + // 1. Store in PendingGateStore (DB-backed) — gives us the + // existing UI rendering, history rehydration, expiry, and + // channel-mismatch protection for free. + let _ = self.pending.insert(pending.clone()).await; + + // 2. Register an in-memory resolution channel keyed by request_id. + let (tx, rx) = oneshot::channel(); + self.pending_resolutions.lock().await.insert(request_id, tx); + + // 3. Emit the SSE / channel-native gate prompt (existing + // `send_pending_gate_status` flow). + self.emit_gate_status(&pending).await; + + // 4. Await resolution. + match rx.await { + Ok(resolution) => resolution, + // Sender dropped (process shutting down or channel was + // displaced) → treat as cancel. + Err(_) => GateResolution::Cancelled, + } + } +} + +impl BridgeGateController { + pub async fn try_deliver(&self, request_id: Uuid, resolution: GateResolution) -> bool { + if let Some(tx) = self.pending_resolutions.lock().await.remove(&request_id) { + let _ = tx.send(resolution); + true + } else { + false + } + } +} +``` + +### Gate-resolve endpoint integration + +In `src/bridge/router.rs::resolve_pending_gate`, before falling +through to the existing `execute_pending_gate_action` / +`thread_manager.resume_thread` path, **try the in-memory channel +first**: + +```rust +// existing: take verified gate from store +let pending = self.pending_gates.take_verified(...).await?; + +// NEW: try in-memory delivery +if let Some(controller) = state.bridge_gate_controller_for(&key) { + if controller.try_deliver(pending.request_id, resolution.clone()).await { + // The CodeAct VM is alive and waiting; it will continue + // execution itself. We just need to emit the GateResolved SSE + // for the UI and return Pending. + self.emit_gate_resolved_sse(state, message, &pending, &resolution); + return Ok(BridgeOutcome::Pending); + } +} + +// fall through to legacy path: re-enter the thread via +// execute_pending_gate_action / resume_thread (Tier 0 flow) +``` + +Result: gates fired from CodeAct keep the VM alive and resolve via the +channel. Gates fired from Tier 0 (no live VM) take the existing +re-entry path. + +### `scripting.rs` changes + +Replace the two broken sites: + +**Sync preflight (line 841-852)** — the early return: + +```rust +PreflightResult::GatePaused(outcome) => { + let resolution = match &context.gate_controller { + Some(controller) => controller.pause(GatePauseRequest { + gate_name: &outcome.gate_name, + action_name: &outcome.action_name, + call_id: &outcome.call_id, + parameters: &outcome.parameters, + resume_kind: outcome.resume_kind.clone(), + paused_lease: outcome.paused_lease.as_deref().cloned(), + resume_output: outcome.resume_output.clone(), + }).await, + None => { + // Tests / legacy callers without a controller — preserve + // the current behavior (return need_approval). + return Ok(CodeExecutionResult { + need_approval: Some(outcome), + /* … */ + }); + } + }; + + match resolution { + GateResolution::Approved { always } => { + // Auto-approve registration, lease re-consume, continue + // through the Approved path. + … + } + GateResolution::Denied { reason } => { + // Resume Monty with a TYPED exception so user code can + // catch it but it's not the misleading RuntimeError. Use + // PermissionError; the message is the deny reason. + let ext_result = ExtFunctionResult::Error(MontyException::new( + ExcType::PermissionError, + Some(reason.unwrap_or_else(|| "denied by user".into())), + )); + // resume Monty with the exception, continue loop + } + GateResolution::Cancelled | GateResolution::ExternalCallback { .. } => { + // Same as Denied — script gets PermissionError. + } + GateResolution::CredentialProvided { .. } => { + // Auth gate completed; rebuild lease/credential state and + // retry the action through the Approved path. + } + } +} +``` + +**Async output gate (line 1740-1766)** — the bug site. Same shape: +on `Err(EngineError::GatePaused { .. })` from `effects.execute_action`, +call `controller.pause(...)`, branch on resolution. On `Approved`, +re-execute the action with the refunded lease (or use `resume_output` +if the gate handed back a held result). On `Denied/Cancelled`, surface +`PermissionError` to the script. + +### Resource limit adjustment + +Monty's `ResourceLimits::max_duration` is wall-clock from start of +`runner.start(...)`. It ticks during gate awaits. The default stays +at **30 seconds** — the same as before this change. + +Why not bump to 30 minutes (which would match `PendingGate.expires_at` +and let humans approve at human latency)? Because the existing +`sandbox_enforces_cpu_limits` test relies on `max_duration` to +terminate `while True: x += 1` (a CPU-bound script with no +allocations to trip the allocation/memory caps). Raising the cap +hangs that test for the new value. + +**Tradeoff**: an approval that takes longer than 30 s times out the +script. The user re-prompts and the LLM re-issues the action. Most +approvals come back in seconds; this is acceptable for the common +case. + +**Follow-up**: a proper "active CPU vs paused" timer split — only +count CPU time during VM execution, not during gate-await futures. +Either mutate `LimitedTracker::set_max_duration` around each await +to extend the budget, or expose a per-call tracker handle. Out of +scope for this PR. + +### Restart behavior + +If IronClaw restarts while a CodeAct gate is pending: + +- The DB-stored `PendingGate` still exists. +- The in-memory `oneshot::Sender` is gone. +- User clicks approve → `resolve_pending_gate` → `try_deliver` returns + `false` (no channel) → falls through to legacy `execute_pending_gate_action`. +- `execute_pending_gate_action` re-enters the thread → re-runs LLM → + CodeAct re-runs from the top. **This is the bug we're trying to + prevent.** + +Cleanup: on startup, iterate `PendingGateStore`, find gates whose +`gate_name == "approval"` and whose source thread was in `Running` +state at shutdown, mark them expired with reason +`"interrupted by restart"`. The user gets a clean error and retries. + +This requires a flag on `PendingGate` distinguishing "live-VM gate" +from "Tier 0 re-entry gate", since the latter is genuinely +restart-survivable. Add `requires_live_vm: bool` (default `false`). +CodeAct sets it to `true`. Startup sweep targets only `requires_live_vm` +gates. + +### Wall-clock semantics for signals + +A `Stop` signal during a gate await must cancel the pause. Use +`tokio::select!` in `BridgeGateController::pause`: + +```rust +tokio::select! { + res = rx => res.unwrap_or(GateResolution::Cancelled), + _ = stop_signal.notified() => GateResolution::Cancelled, +} +``` + +`InjectMessage` during a gate await is queued and surfaces only after +the pause resolves — the script is mid-statement, can't accept a new +message inline. + +### Concurrency cap + +A paused VM holds its frame stack and closed-over `Arc`s in memory — +small (kilobytes per script), but should be capped to prevent a stuck +user from accumulating dozens. Per-user cap (default 8) on +concurrent in-script gate pauses; ninth attempt rejects with a clean +"too many pending approvals" error. Tunable via env var. + +## Scope: which gates are unified + +| Resume kind | Tier 0 path | Tier 1 path | +|---|---|---| +| `Approval` | `GateController::pause` (NEW, this PR) | `GateController::pause` (NEW, this PR) | +| `Authentication` | Legacy (`execute_pending_gate_action` + `AuthManager`) | Legacy (returns `need_approval`, orchestrator pauses) | +| `External` | Legacy | Legacy | + +Auth and External gates stay on the existing re-entry path because: + +- Auth completion installs a credential in the secrets store; the + *new* credential availability is what makes the retried action + succeed. No live in-flight state to hand back to. +- External callbacks may arrive long after the originating script + has cleaned up; they're inherently async-via-DB. + +Approval is the only resume kind where re-entry causes the +double-execution bug (the user already gave the answer; we just +need to deliver it back to the suspended call). + +## Migration / blast radius + +- **Tier 0** (structured tool calls): both gate sites in + `structured.rs` (preflight at line 185, mid-execution at line 457) + call `GateController::pause` for `Approval` gates. The loop stays + inside `execute_action_calls` until the gate resolves. On approval, + the gated action is re-executed (lease re-consumed, credential + re-injected); on denial, an `ActionFailed` event is emitted and the + batch continues with that single call marked failed. +- **Tier 1** (CodeAct): same callback used. VM stays alive across + the gate. +- **Bridge**: `handle_with_engine_inner`'s + `ThreadOutcome::GatePaused` arm continues to handle `Authentication` + and `External` resume kinds via the existing path. `Approval` no + longer flows through this arm because the engine handles it + inline. +- **Resolve endpoint**: `resolve_pending_gate` checks + `controller.try_deliver` first. On success (in-flight `Approval` + gate), short-circuits with a UI event. On miss, falls through to + the existing `execute_pending_gate_action` path — preserved as a + fall-through but never hit in normal operation. + +## Restart semantics + +If IronClaw restarts while an `Approval` gate is pending: + +- DB-stored `PendingGate` row still exists. +- In-memory `oneshot::Sender` is gone. +- User clicks approve → `try_deliver` returns false → fall-through + to `execute_pending_gate_action` re-entry → re-runs LLM step → + the bug we're trying to prevent recurs. + +Mitigation in this PR: on startup, sweep `PendingGateStore` for +`Approval`-kind gates and mark them expired (emit `GateExpired` SSE, +remove from store). User sees "approval expired due to restart" and +retries. Auth/External rows are untouched. + +This is simpler than the `requires_live_vm` flag I proposed earlier: +since `Approval` is now always handled inline, **every** unresolved +`Approval` row at startup represents a stranded in-flight gate and +should be invalidated. + +## Testing + +Test mapping to the as-shipped code: + +- **Unit (engine, `scripting.rs`):** `codeact_gate_inline_await_approved_delivers_result` + — tool returns `Err(EngineError::GatePaused)` mid-execution; a stub + `GateController` returns `Approved`; asserts the script completes + with the tool's result. +- **Unit (engine, `scripting.rs`):** `codeact_gate_inline_await_denied_raises_in_script` + — denial surfaces as a typed `RuntimeError("user denied tool 'X': <reason>")`, + catchable in Python. +- **Unit (engine, `scripting.rs`):** `codeact_default_controller_cancels_approval_gates` + — locks in the `CancellingGateController` default. Verifies the + pre-fix `"execution paused by gate"` message never appears even + when the controller is the inert default. +- **Unit (engine, `scripting.rs`):** `bridge::gate_controller::tests::*` + — 4 tests cover `GateResolutions` registry semantics (unknown / + registered / dropped-receiver / one-shot). +- **Live regression (`tests/engine_v2_gate_integration.rs`):** + `codeact_inline_gate_await_resumes_user_reproducer` drives the + user-reported reproducer through `ThreadManager`; asserts no + RuntimeError leak and the post-approval retry runs. + `codeact_inline_gate_await_denial_does_not_retry` covers denial. +- **Unit (bridge, `router.rs`):** `invalidate_stranded_approval_gates_evicts_only_approval_kind` + — boot sweep evicts only `Approval` rows; Auth/External rows + survive. +- **Open follow-ups:** integration test covering `resolve_gate`'s + auto-approve install + rollback path (caller-level coverage per + `.claude/rules/testing.md`); E2E click-through tests for the live + approve/deny/restart flows in the UI. + +`PermissionError` was the originally proposed Python exception kind +but the as-shipped code uses `RuntimeError` with a descriptive message +(`"user denied tool 'X': <reason>"`) — `RuntimeError` is what scripts +already catch, and the message is specific enough to distinguish from +other runtime errors. + +## As-shipped shape + +The code shipped differs from the original sketch in a few places worth +recording so a future reader doesn't get confused: + +- **`gate_controller` is required, not optional.** The original sketch + used `Option<Arc<dyn GateController>>` and let executors fall back + to the V1 unwind path when `None`. That fallback re-emitted the bug + message, so it was removed. `CancellingGateController` is the + explicit drop-in for paths that don't pause (post-resolution replay, + mission protected writes, tests). +- **Bounded retry on Approval.** Both `scripting::drive_inline_gate` + (Tier 1 async) and `structured::execute_with_inline_gate_retry` + (Tier 0) cap the re-pause loop at `MAX_INLINE_GATE_RETRIES = 3`. + Well-behaved chains (auto-approve installed before delivery) + converge in 1–2 iterations; the cap only ever fires on a buggy host + controller. +- **`denial_reason_for_resolution` helper.** Pulled into + `executor::scripting` and used by Tier 0 + Tier 1 sites so denial + messages can't drift between executors. +- **`max_duration` stays at 30 s.** The original plan considered + bumping it to 30 min so human approvals fit; the existing + `sandbox_enforces_cpu_limits` test relied on the 30 s cap to + terminate a CPU-bound script. The active-CPU vs paused-clock split + is on the follow-up list. + +## Build order + +1. `GatePauseRequest` / `GateController` / `CancellingGateController` + in `crates/ironclaw_engine/src/gate/mod.rs`. Required field + (`Arc<dyn GateController>`, not `Option`) on + `traits/effect.rs::ThreadExecutionContext`. Restricted to + `Approval` resume-kind for this PR. +2. `BridgeGateController` in `src/bridge/gate_controller.rs` — + construction, `pause`, `try_deliver`, `emit_gate_status`, + per-execution context registry. +3. Wire one controller per thread execution from + `src/bridge/router.rs::handle_with_engine_inner` into + `ThreadManager::set_gate_controller`. Subsequent thread spawns pick + it up via `ExecutionLoop::new`. `ThreadManager` defaults to + `CancellingGateController` so unwired hosts fail loud rather than + silently. +4. **Tier 1 sites:** rewrite the sync preflight `PreflightResult::GatePaused` + arm and the async output `resolve_tool_future` `EngineError::GatePaused` + arm to call the controller. Async path delegates to + `drive_inline_gate` (bounded retry). +5. **Tier 0 sites:** rewrite the preflight `RequireApproval` arm in + `structured::execute_action_calls` to call the controller, and the + mid-execution loop in `structured::execute_with_inline_gate_retry` + (also bounded). Authentication/External keep the legacy re-entry + path unchanged. +6. Update `resolve_pending_gate` to `try_deliver` the controller's + in-memory channel first; install/rollback auto-approve preference + around the delivery so chained gates short-circuit. Legacy + `execute_pending_gate_action` path stays as a fall-through for + Auth/External and post-restart stragglers. +7. Startup sweep: `invalidate_stranded_approval_gates` evicts every + `Approval`-kind `PendingGate` row on boot and emits a + `GateResolved` SSE with `resolution = "expired"` per row. +8. Tests for both tiers (sync preflight, async output, denial, + approval, fallback-controller default, restart cleanup). +9. Wire `CancellingGateController` into the remaining production paths + that build `ThreadExecutionContext` directly (mission protected + writes, post-resolution replay) — required field, no implicit + default. + +## Open questions / follow-ups (not in this PR) + +- **Active-CPU vs paused-clock split for `max_duration`.** Currently + 30 s and ticks during gate awaits — long approvals time out. A + proper split would only count CPU time during VM execution. +- **Multi-gate single-prompt UX** (`asyncio.gather` of two gating + tools). +- **`InjectMessage` during a gate wait** — current behavior queues; + may want to surface a marker in the script so the LLM knows the + user said something. +- **Migration of Auth/External to the controller.** Would let us + delete the thread re-entry path entirely. Each needs new state + installed (credential, callback) before the suspended call can + succeed, so the controller's contract would need to grow. +- **Bridge integration test for `resolve_gate` rollback** — the + auto-approve install/rollback dance in `resolve_gate` deserves a + caller-level test per `.claude/rules/testing.md`. +- **Per-user concurrency cap on in-flight gates.** Documented but not + implemented. A stuck user could accumulate paused VMs. +- **`mission.rs::dispatch_protected_write` controller wiring.** Currently + uses `CancellingGateController`; if a protected write ever surfaces + a gate the user gets a silent denial. Decide whether to wire it + through the live controller or surface as a clearer engine error. +- **Tier 0 parallel-batch mid-execution gates.** `execute_with_inline_gate_retry` + handles the single-call path; multi-call parallel batches with + simultaneous gates need the same treatment (rare). diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index b353403cacc..7a0a286d33e 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -46,7 +46,7 @@ This guide will get you from zero to a running IronClaw instance in under 10 min </Tab> <Tab title="Local"> - Best for personal use on your own machine. Uses libSQL (embedded SQLite) — no separate database server required. + Best for personal use on your own machine. Uses libSQL (embedded SQLite) by default — no separate database server required. For production or multi-user setups, see [Database Backends](/capabilities/database). ```bash # Install IronClaw diff --git a/docs/reborn/harness/e2e.md b/docs/reborn/harness/e2e.md new file mode 100644 index 00000000000..65b3db8062e --- /dev/null +++ b/docs/reborn/harness/e2e.md @@ -0,0 +1,146 @@ +# Reborn E2E Harness + +This document is the branch-local map for the dedicated Reborn E2E gate. Reborn is not missing tests: `main` already contains extensive crate-level contract and integration coverage. The gap this branch closes is a single, named E2E workflow that runs the Reborn architecture spine together and keeps a small product-surface smoke check beside it. + +## What already exists + +| Area | Existing coverage | +| --- | --- | +| Architecture boundaries | `crates/ironclaw_architecture/tests/reborn_dependency_boundaries.rs` | +| Host runtime facade and outcomes | `crates/ironclaw_host_runtime/tests/host_runtime_contract.rs` | +| Host runtime production composition | `crates/ironclaw_host_runtime/tests/host_runtime_services_contract.rs` | +| Dedicated Reborn E2E spine | `crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs` | +| Capability host invoke/resume/spawn | `crates/ironclaw_capabilities/tests/capability_host_*` | +| Dispatcher adapter selection | `crates/ironclaw_dispatcher/tests/vertical_slice_contract.rs` | +| WASM runtime lane | `crates/ironclaw_wasm/tests/wasm_dispatch_integration.rs` and `wasm_http_adapter_contract.rs` | +| Script runtime lane | `crates/ironclaw_scripts/tests/script_dispatch_integration.rs` and `script_http_adapter_contract.rs` | +| MCP runtime lane | `crates/ironclaw_mcp/tests/mcp_dispatch_integration.rs` and `mcp_adapter_contract.rs` | +| Process lifecycle | `crates/ironclaw_processes/tests/process_dispatch_integration.rs` and process service/store contracts | +| Network policy and host HTTP egress | `crates/ironclaw_network/tests/*` plus host-runtime HTTP egress tests | +| Secret storage/leases | `crates/ironclaw_secrets/tests/secret_store_contract.rs` plus host-runtime staged-secret tests | +| Events/audit replay | `crates/ironclaw_events/tests/durable_log_contract.rs` and host-runtime durable-event tests | +| Gateway product smoke | Existing Playwright scenarios under `tests/e2e/scenarios/`, especially `test_v2_*` | + +## Dedicated Reborn E2E goal + +The dedicated gate should answer one question: + +```text +Can the Reborn architecture path still execute happy, blocked, denied, failed, +background, network, secret, event, and product-smoke paths after a change? +``` + +It intentionally reuses existing deterministic contract/integration tests instead of duplicating them in a second test framework. + +## Happy path spine + +The Reborn E2E happy path is covered by a dedicated `reborn_e2e_gate.rs` spine test plus the broader host-runtime and runtime-lane tests: + +```text +Extension manifests +-> ExtensionRegistry +-> HostRuntimeServices +-> DefaultHostRuntime / HostRuntime facade +-> CapabilityHost authorization and run-state lifecycle +-> RuntimeDispatcher adapter selection +-> WASM / Script / MCP runtime adapters +-> resource reservation and reconciliation +-> durable runtime events +-> structured outcome returned to caller +``` + +Required assertions across the suite: + +- visible capability surfaces include expected descriptors and stable surface version; +- health reports missing runtime backends fail-closed and configured backends ready; +- authorized invocations reach the selected runtime adapter; +- resource reservations are reconciled or released; +- run-state reaches the expected terminal or blocked state; +- durable events are replayable and metadata-only; +- runtime output is structured JSON and redacted where obligations require it. + +## Other paths + +The dedicated gate includes new Reborn E2E gate tests plus existing tests for these non-happy paths: + +### Authorization and approval + +- denied authorization fails before dispatch; +- approval-required invocation blocks with a persisted approval request id; +- approved resume consumes the exact lease once; +- changed input, wrong scope/user, expired lease, or missing stores fail before dispatch; +- unsupported obligations fail closed. + +### Runtime availability and adapter failures + +- missing runtime backend reports a stable missing-runtime failure; +- runtime lane errors map to stable failure categories; +- dispatcher/runtime adapter boundaries remain dependency-clean. + +### Resource, process, and cancellation + +- reservations are released on failure and reconciled on success; +- spawned background processes publish started/completed/failed/killed transitions; +- cancellation reaches the process graph; +- late completion after kill does not publish a misleading success; +- process handoffs for the same scoped capability fail closed while active. + +### Network and secrets + +- runtime HTTP egress is host-mediated; +- missing staged network policy fails before transport; +- staged secret material is consumed once; +- runtime-supplied manual credentials are rejected; +- raw secrets, credential-shaped values, and private host paths are not exposed in runtime-visible output, errors, events, or audit records. + +### Event and observability + +- durable event cursors replay runtime events; +- stale/gap cursor behavior is deterministic; +- event/audit records carry correlation metadata without raw payload leaks. + +### Product smoke + +A small Playwright smoke scenario starts an isolated `ENGINE_V2=true` gateway with the mock LLM and verifies: + +- authenticated web shell loads; +- text-only chat completes and persists; +- tool-capable prompt completes through the gateway history path; +- no duplicate assistant response is emitted for a single user turn. + +This smoke test does not replace the full browser E2E workflow. It proves the branch remains product-bootable while the Rust Reborn gate proves architecture behavior. + +## Local commands + +Run the deterministic Rust Reborn gate: + +```bash +# Full gate +scripts/reborn-e2e-rust.sh + +# Or run one CI matrix group at a time +scripts/reborn-e2e-rust.sh architecture +scripts/reborn-e2e-rust.sh runtimes +scripts/reborn-e2e-rust.sh substrates +``` + +The script expands to the dedicated `reborn_e2e_gate.rs` tests plus the current Reborn boundary, host-runtime, capability-host, dispatcher, WASM, Script, MCP, process, event, filesystem, network, secret, resource, run-state, approval, and authorization contract tests. Use the script as the source of truth for local/CI parity rather than copying individual `cargo test` commands. + +Run the gateway smoke test: + +```bash +cargo build --no-default-features --features libsql +cd tests/e2e +pip install -e . +playwright install --with-deps chromium # on Linux CI; local macOS can omit --with-deps +pytest scenarios/test_reborn_gateway_smoke.py -v --timeout=120 +``` + +## CI ownership + +`reborn-e2e.yml` is intentionally separate from `e2e.yml`: + +- Reborn changes can run a focused architecture gate without destabilizing the main browser E2E matrix. +- The workflow is advisory by default and intentionally does not run on `merge_group`; add a merge-queue trigger only after the gate proves stable and is deliberately promoted to branch protection. +- The Rust jobs should stay deterministic and avoid live providers. +- The gateway job should remain a smoke test, not a second full browser matrix. diff --git a/providers.json b/providers.json index 517e2a264f1..56af05df33e 100644 --- a/providers.json +++ b/providers.json @@ -1,4 +1,57 @@ [ + { + "id": "nearai", + "aliases": [ + "near_ai", + "near" + ], + "protocol": "nearai", + "api_key_env": "NEARAI_API_KEY", + "api_key_required": false, + "base_url_env": "NEARAI_BASE_URL", + "model_env": "NEARAI_MODEL", + "default_model": "auto", + "description": "multi-model access via NEAR account", + "setup": { + "kind": "session_token", + "display_name": "NEAR AI", + "key_url": "https://app.near.ai", + "can_list_models": true + } + }, + { + "id": "gemini_oauth", + "aliases": [ + "gemini-oauth" + ], + "protocol": "gemini_oauth", + "api_key_required": false, + "model_env": "GEMINI_MODEL", + "default_model": "gemini-2.5-flash", + "description": "Official Gemini API via Gemini CLI OAuth", + "setup": { + "kind": "file_based_credentials", + "display_name": "Gemini CLI", + "default_path_hint": "~/.gemini/oauth_creds.json" + } + }, + { + "id": "openai_codex", + "aliases": [ + "openai-codex", + "codex" + ], + "protocol": "openai_codex", + "api_key_required": false, + "model_env": "OPENAI_CODEX_MODEL", + "default_model": "gpt-5.3-codex", + "description": "ChatGPT subscription (Plus/Pro/Max)", + "setup": { + "kind": "o_auth_device_code", + "display_name": "OpenAI Codex", + "backend": "openai_codex" + } + }, { "id": "openai", "aliases": [ @@ -124,13 +177,14 @@ "aliases": [ "open_router" ], - "protocol": "open_ai_completions", - "default_base_url": "https://openrouter.ai/api/v1", + "protocol": "open_router", + "default_base_url": "", "api_key_env": "OPENROUTER_API_KEY", "api_key_required": true, "model_env": "OPENROUTER_MODEL", + "extra_headers_env": "OPENROUTER_EXTRA_HEADERS", "default_model": "openai/gpt-4o", - "description": "OpenRouter multi-provider gateway (200+ models)", + "description": "OpenRouter multi-provider gateway (200+ models, preserves reasoning across turns)", "setup": { "kind": "api_key", "secret_name": "llm_openrouter_api_key", @@ -246,13 +300,13 @@ "aliases": [ "deep_seek" ], - "protocol": "open_ai_completions", - "default_base_url": "https://api.deepseek.com/v1", + "protocol": "deep_seek", + "default_base_url": "", "api_key_env": "DEEPSEEK_API_KEY", "api_key_required": true, "model_env": "DEEPSEEK_MODEL", "default_model": "deepseek-chat", - "description": "DeepSeek inference API", + "description": "DeepSeek inference API (preserves reasoning_content for thinking-mode models)", "setup": { "kind": "api_key", "secret_name": "llm_deepseek_api_key", @@ -325,19 +379,19 @@ "google_gemini", "google" ], - "protocol": "open_ai_completions", - "default_base_url": "https://generativelanguage.googleapis.com/v1beta/openai", + "protocol": "gemini", + "default_base_url": "", "api_key_env": "GEMINI_API_KEY", "api_key_required": true, "model_env": "GEMINI_MODEL", "default_model": "gemini-2.5-flash", - "description": "Google Gemini (via OpenAI-compatible endpoint)", + "description": "Google Gemini native API (preserves thought_signature on tool calls)", "setup": { "kind": "api_key", "secret_name": "llm_gemini_api_key", "key_url": "https://aistudio.google.com/app/apikey", "display_name": "Google Gemini", - "can_list_models": true + "can_list_models": false } }, { @@ -445,5 +499,23 @@ "display_name": "Cloudflare Workers AI", "can_list_models": false } + }, + { + "id": "bedrock", + "aliases": [ + "aws_bedrock", + "aws" + ], + "protocol": "bedrock", + "api_key_required": false, + "model_env": "BEDROCK_MODEL", + "default_model": "anthropic.claude-sonnet-4-20250514-v1:0", + "description": "Claude & other models via AWS (IAM, SSO)", + "setup": { + "kind": "aws_credentials", + "display_name": "AWS Bedrock", + "supports_cross_region": true, + "supports_profile": true + } } ] diff --git a/registry/_bundles.json b/registry/_bundles.json index 322b075741f..38ea578bb58 100644 --- a/registry/_bundles.json +++ b/registry/_bundles.json @@ -16,13 +16,15 @@ }, "messaging": { "display_name": "Messaging Channels", - "description": "Discord, Telegram, Slack, and WhatsApp channels", + "description": "Discord, Telegram, Slack, WhatsApp, and WeChat channels", "extensions": [ "channels/discord", "channels/telegram", "channels/slack", "channels/whatsapp", - "channels/feishu" + "channels/feishu", + "channels/wecom", + "channels/wechat" ], "shared_auth": null }, diff --git a/registry/channels/slack.json b/registry/channels/slack.json index 532f3164151..0fcb79f070c 100644 --- a/registry/channels/slack.json +++ b/registry/channels/slack.json @@ -2,7 +2,7 @@ "name": "slack", "display_name": "Slack Channel", "kind": "channel", - "version": "0.3.2", + "version": "0.3.4", "wit_version": "0.3.0", "description": "Talk to your agent in Slack", "keywords": [ diff --git a/registry/channels/telegram.json b/registry/channels/telegram.json index 4a02d4ecd16..b8b4bf424a4 100644 --- a/registry/channels/telegram.json +++ b/registry/channels/telegram.json @@ -2,7 +2,7 @@ "name": "telegram", "display_name": "Telegram Channel", "kind": "channel", - "version": "0.2.10", + "version": "0.2.11", "wit_version": "0.3.0", "description": "Talk to your agent through a Telegram bot", "keywords": [ diff --git a/registry/channels/wechat.json b/registry/channels/wechat.json new file mode 100644 index 00000000000..a281a56ca70 --- /dev/null +++ b/registry/channels/wechat.json @@ -0,0 +1,37 @@ +{ + "name": "wechat", + "display_name": "WeChat Channel", + "kind": "channel", + "version": "0.1.0", + "wit_version": "0.3.0", + "description": "Talk to your agent through a WeChat iLink bot account", + "keywords": [ + "messaging", + "chat", + "wechat", + "qr" + ], + "source": { + "dir": "channels-src/wechat", + "capabilities": "wechat.capabilities.json", + "crate_name": "wechat-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/download/ironclaw-v0.28.0/channel-wechat-0.1.0-wasm32-wasip2.tar.gz", + "sha256": "1538dfd3554b533f5617475d4c88d2666615bf87c7f0a732df520d442f29e7c2" + } + }, + "auth_summary": { + "method": "interactive", + "provider": "WeChat", + "secrets": [ + "wechat_bot_token" + ], + "shared_auth": null, + "setup_url": "https://ilinkai.weixin.qq.com" + }, + "tags": [ + "messaging" + ] +} diff --git a/registry/channels/wecom.json b/registry/channels/wecom.json new file mode 100644 index 00000000000..77b83901807 --- /dev/null +++ b/registry/channels/wecom.json @@ -0,0 +1,38 @@ +{ + "name": "wecom", + "display_name": "WeCom Channel", + "kind": "channel", + "version": "0.1.0", + "wit_version": "0.3.1", + "description": "Talk to your agent through a WeCom intelligent bot with WebSocket inbound and direct Bot media sends", + "keywords": [ + "messaging", + "chat", + "wecom", + "enterprise-wechat" + ], + "source": { + "dir": "channels-src/wecom", + "capabilities": "wecom.capabilities.json", + "crate_name": "wecom-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/download/ironclaw-v0.29.0/channel-wecom-0.1.0-wasm32-wasip2.tar.gz", + "sha256": "25d674b3d68c7689ab269be9b40b944c1d5a0f9b20378b81688c25c8b7859bda" + } + }, + "auth_summary": { + "method": "manual", + "provider": "WeCom", + "secrets": [ + "wecom_bot_id", + "wecom_bot_secret" + ], + "shared_auth": null, + "setup_url": "https://open.work.weixin.qq.com/" + }, + "tags": [ + "messaging" + ] +} diff --git a/registry/tools/telegram_mtproto.json b/registry/tools/telegram_mtproto.json index 38521ceb836..3ebcdea9c62 100644 --- a/registry/tools/telegram_mtproto.json +++ b/registry/tools/telegram_mtproto.json @@ -4,6 +4,7 @@ "kind": "tool", "version": "0.2.1", "wit_version": "0.3.0", + "hidden": true, "description": "Your agent uses your Telegram account to read and send messages", "keywords": [ "messaging", diff --git a/scripts/auth_live_canary/.gitignore b/scripts/auth_live_canary/.gitignore new file mode 100644 index 00000000000..2549b3ddaac --- /dev/null +++ b/scripts/auth_live_canary/.gitignore @@ -0,0 +1 @@ +config.env diff --git a/scripts/auth_live_canary/run_live_canary.py b/scripts/auth_live_canary/run_live_canary.py index dd2f8cb524d..592ccd00eac 100644 --- a/scripts/auth_live_canary/run_live_canary.py +++ b/scripts/auth_live_canary/run_live_canary.py @@ -325,6 +325,17 @@ async def seed_non_oauth_credentials( value=github_token, provider="github", ) + # Companion scopes record — without it, needs_scope_expansion() in + # src/extensions/manager.rs treats the seeded PAT as a legacy token + # and forces re-auth, leaving the extension stuck at + # authenticated=False. Match the github tool's merged_scopes. + await put_secret( + base_url, token, + user_id=owner_user_id, + name="github_token_scopes", + value="read:org repo workflow", + provider="github", + ) notion_access = env_str("AUTH_LIVE_NOTION_ACCESS_TOKEN") notion_refresh = env_str("AUTH_LIVE_NOTION_REFRESH_TOKEN") diff --git a/scripts/live-canary/emit_results_json.py b/scripts/live-canary/emit_results_json.py new file mode 100755 index 00000000000..9120682e85f --- /dev/null +++ b/scripts/live-canary/emit_results_json.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Scrape `cargo test --nocapture` output into a results.json file that +notify_slack.py::parse_results_json can already consume. + +Invoked from scripts/live-canary/run.sh at the end of every cargo-based +lane. Skipped (early-exits as no-op) when: + + * --out already exists — workflow-canary writes its own results.json + via scripts/workflow_canary/run_workflow_canary.py, and that file + must not be overwritten. + * The log contains no `test result:` line — auth-* lanes use pytest + + JUnit XML and produce no cargo-style output, so there is nothing to + scrape. + +Schema matches the workflow-canary contract (see parse_results_json): + + {"results": [ + {"provider": "...", "mode": "<test_name>", + "success": bool, "latency_ms": int, + "details": {"error": "<short panic msg>"}}, + ... + ]} + +We only emit one entry per executed test (cargo `ok` or `FAILED`). +`ignored` tests are not results — they didn't run — so they are skipped +entirely. + +Per-test latency is unknowable from cargo's plain stdout, so we leave +``latency_ms`` at 0 and put the lane-level wall-clock duration on a +single ``meta`` entry consumed only by future tooling. notify_slack.py +sums ``latency_ms`` per entry so the 0 values are harmless — lane-level +duration is captured by the workflow itself. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +# Cargo test status line — emitted by cargo at the end of each test +# binary invocation, after all per-test output. Its presence is also our +# gate for "is this a cargo lane". +# +# test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 14 filtered out; finished in 236.39s +# test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 12.34s +RESULT_RE = re.compile( + r"test result: (?P<outcome>ok|FAILED)\. " + r"(?P<passed>\d+) passed; " + r"(?P<failed>\d+) failed; " + r"(?P<ignored>\d+) ignored" +) + +# Test-start line. With --nocapture, the test's own stdout gets glued +# onto the same line right after the `...`, so we can't expect a clean +# trailing outcome here — only the test name on the left. +# +# test live_tests::zizmor_scan ... [LiveTest] Mode: LIVE — recording to ... +TEST_START_RE = re.compile(r"^test (?P<name>[\w:]+) \.\.\. ?(?P<trailer>.*)$") + +# Standalone end-of-test outcome lines (cargo emits these on a line of +# their own after the test's stdout in --nocapture mode). With +# --test-threads=1 the most recent TEST_START_RE match owns the next +# standalone outcome. +OUTCOME_RE = re.compile(r"^(?P<outcome>ok|FAILED|ignored)\s*$") + +# Panic header. Two shapes in the wild: +# +# Rust >= 1.73 (current — message on next line): +# thread 'live_tests::zizmor_scan' (27813) panicked at tests/e2e_live.rs:85:9: +# Expected shell tool to be used for running zizmor, got: [] +# +# Rust < 1.73 (legacy — message inline on the header): +# thread 'foo' panicked at 'expected X, got Y', src/lib.rs:1:1 +# +# `.*?` (lazy) is critical: it lets the regex match the legacy form +# where there's nothing between the closing quote and ` panicked at `. +# Greedy `.*` here would have required at least one character of +# between-text (a worker-id like `(27813)`) and missed the legacy form. +PANIC_RE = re.compile(r"^thread '(?P<name>[\w:]+)'.*? panicked at ") + +MAX_ERROR_LEN = 240 + +# Defense in depth: panic messages from e2e tests *could* embed a real +# token if an assertion happens to dump a captured response body. Redact +# the obvious shapes before writing so a token can never reach the +# artifact store via results.json, regardless of what scrub-artifacts.sh +# decides to do downstream. +# +# This list is the union of every shape `scrub-artifacts.sh` rewrites +# plus the provider-specific shapes the canary actually exercises that +# the shell script doesn't carry on its own (OpenAI `sk-…`, AWS access +# keys). Composio has no published key prefix, so its keys are caught +# only via the generic `api_key=…` / `"api_key": "…"` assignment shapes +# below — same gap as scrub-artifacts.sh, documented so nobody assumes +# a bare Composio token survives. +# +# Order matters: Anthropic (`sk-ant-…`) is matched before the broader +# OpenAI `sk-…` rule so an Anthropic key gets the correct label. The +# OpenAI pattern also uses a negative lookahead as belt-and-braces in +# case anyone reorders this list. +REDACT_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"gh[pousr]_[A-Za-z0-9_]{20,}"), "<REDACTED_GITHUB_TOKEN>"), + (re.compile(r"github_pat_[A-Za-z0-9_]{20,}"), "<REDACTED_GITHUB_PAT>"), + (re.compile(r"ya29\.[A-Za-z0-9._-]{20,}"), "<REDACTED_GOOGLE_TOKEN>"), + (re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}"), "<REDACTED_SLACK_TOKEN>"), + (re.compile(r"sk-ant-[A-Za-z0-9_-]{10,}"), "<REDACTED_ANTHROPIC_KEY>"), + (re.compile(r"sk-(?!ant-)[A-Za-z0-9_-]{20,}"), "<REDACTED_OPENAI_KEY>"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "<REDACTED_AWS_ACCESS_KEY>"), + (re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer <REDACTED>"), + # Generic `key=value` / `key: value` assignments. Mirrors the + # corresponding rules in scrub-artifacts.sh so anything that script + # would have stripped from a log file also gets stripped from a + # panic message that landed in results.json. + ( + re.compile(r"(?i)(api[_-]?key)\s*[:=]\s*\S+"), + r"\1=<REDACTED>", + ), + ( + re.compile(r"(?i)(access[_-]?token)\s*[:=]\s*\S+"), + r"\1=<REDACTED>", + ), + ( + re.compile(r"(?i)(refresh[_-]?token)\s*[:=]\s*\S+"), + r"\1=<REDACTED>", + ), + ( + re.compile(r"(?i)(client[_-]?secret)\s*[:=]\s*\S+"), + r"\1=<REDACTED>", + ), + ( + re.compile(r"(?i)(password)\s*[:=]\s*\S+"), + r"\1=<REDACTED>", + ), + ( + re.compile(r"(?i)\bsecret\s*[:=]\s*\S+"), + "secret=<REDACTED>", + ), + # JSON-quoted token shapes — same defensive cases scrub-artifacts.sh + # rewrites. An assertion that dumped an OAuth response body would + # land here. + ( + re.compile( + r'"(access|refresh|id|bearer)_token"\s*:\s*"[^"]+"' + ), + r'"\1_token": "<REDACTED>"', + ), + ( + re.compile( + r'"(api[_-]?key|client[_-]?secret|password)"\s*:\s*"[^"]+"' + ), + r'"\1": "<REDACTED>"', + ), +] + + +def redact(text: str) -> str: + for pattern, replacement in REDACT_PATTERNS: + text = pattern.sub(replacement, text) + return text + + +def parse_log(log_text: str) -> list[dict]: + """Return one entry per executed test (cargo ok / FAILED). + + Tests reported as ``ignored`` are excluded — they did not run, so + they are not results. + + Strategy: walk the log once and pair each `test <name> ...` start + with the next standalone `ok`/`FAILED`/`ignored` token. With + --test-threads=1 (which every cargo lane in run.sh uses) tests run + serially, so each start owns the next outcome unambiguously. + + Inline `... ignored` shortcut (cargo collapses ignored tests onto + one line because they produce no stdout) is also handled. + """ + lines = log_text.splitlines() + + # First pass: panic messages keyed by test name. + panic_messages: dict[str, str] = {} + for i, line in enumerate(lines): + m = PANIC_RE.match(line) + if not m: + continue + + # Rust < 1.73 emits the panic message inline on the header: + # + # thread 'foo' panicked at 'expected X, got Y', src/lib.rs:1:1 + # + # Rust >= 1.73 puts only the location on the header and the + # message on the following line: + # + # thread 'foo' panicked at src/lib.rs:1:1: + # expected X, got Y + # + # A trailing `:` on the suffix is the location-only signal; if + # the suffix carries anything else, that *is* the message. + suffix = line[m.end() :].strip() + if suffix and not suffix.endswith(":") and not suffix.startswith("note:"): + panic_messages[m.group("name")] = redact(suffix)[:MAX_ERROR_LEN] + continue + + for follow in lines[i + 1 : i + 6]: + stripped = follow.strip() + if not stripped: + continue + if stripped.startswith("note: run with"): + break + panic_messages[m.group("name")] = redact(stripped)[:MAX_ERROR_LEN] + break + + # Second pass: pair starts with outcomes. + entries: list[dict] = [] + current: str | None = None + + def record(name: str, outcome: str) -> None: + if outcome == "ignored": + return + success = outcome == "ok" + entry: dict = { + "provider": "", # filled in by caller + "mode": name, + "success": success, + "latency_ms": 0, + } + if not success: + entry["details"] = { + "error": panic_messages.get( + name, "test failed (no panic message captured)" + ), + } + entries.append(entry) + + for line in lines: + start = TEST_START_RE.match(line) + if start: + trailer = start.group("trailer").strip() + # Cargo collapses ignored tests inline: `test foo ... ignored`. + if trailer in {"ok", "FAILED", "ignored"}: + record(start.group("name"), trailer) + current = None + continue + # Two `test foo ...` starts with no intervening standalone + # outcome means cargo is running in parallel (no + # `--test-threads=1`) and the start→outcome pairing this + # parser relies on is broken. Bail loudly so the lane + # fails closed rather than silently misattributing + # outcomes. Reference: scripts/live-canary/run.sh's + # run_cargo_test helper is the single producer of these + # logs and must keep `--test-threads=1` on every + # invocation. + if current is not None: + raise InterleavedOutputError( + f"interleaved cargo test output: saw `test {start.group('name')} ...`" + f" while previous test {current!r} had no standalone outcome." + f" emit_results_json.py requires --test-threads=1." + ) + current = start.group("name") + continue + + outcome = OUTCOME_RE.match(line) + if outcome and current is not None: + record(current, outcome.group("outcome")) + current = None + + return entries + + +class InterleavedOutputError(RuntimeError): + """Raised when cargo test output appears interleaved across parallel + test threads, breaking the parser's start→outcome pairing + assumption. See parse_log() for the rule. + """ + + +def has_cargo_output(log_text: str) -> bool: + """Cheap gate so non-cargo lanes (auth-*, workflow-canary) are no-ops.""" + return RESULT_RE.search(log_text) is not None + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--log", required=True, type=Path) + p.add_argument("--out", required=True, type=Path) + p.add_argument("--lane", required=True) + p.add_argument("--provider", required=True) + args = p.parse_args() + + # Never clobber a results.json written by another tool (workflow-canary + # writes its own, with richer per-probe details). + if args.out.exists(): + print( + f"[emit_results_json] {args.out} already present — leaving untouched", + file=sys.stderr, + ) + return 0 + + if not args.log.exists(): + print(f"[emit_results_json] no log at {args.log} — skipping", file=sys.stderr) + return 0 + + log_text = args.log.read_text(encoding="utf-8", errors="replace") + if not has_cargo_output(log_text): + # Not a cargo lane — auth/workflow lanes have their own count + # files. Silent no-op so this is safe to wire unconditionally. + return 0 + + try: + entries = parse_log(log_text) + except InterleavedOutputError as e: + # Fail closed: the log is real cargo output (has_cargo_output + # passed) but interleaving means the start→outcome pairing is + # untrustworthy. Don't emit a results.json that would + # misattribute outcomes; force the lane to fail visibly so + # whoever changed run.sh sees it. + print(f"[emit_results_json] {e}", file=sys.stderr) + return 2 + for entry in entries: + entry["provider"] = args.provider + + payload = { + "lane": args.lane, + "provider": args.provider, + "results": entries, + } + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print( + f"[emit_results_json] wrote {len(entries)} entries to {args.out}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live-canary/notify_slack.py b/scripts/live-canary/notify_slack.py index f2784131b62..67c05739b2b 100755 --- a/scripts/live-canary/notify_slack.py +++ b/scripts/live-canary/notify_slack.py @@ -181,6 +181,30 @@ def parse_results_json(path: Path, report: LaneReport) -> None: report.duration_s += latency / 1000.0 +SUMMARY_STATUS_RE = re.compile( + r"^\|\s*Status\s*\|\s*`(?P<status>-?\d+)`\s*\|\s*$", re.MULTILINE +) + + +def parse_summary_status(summary_md: str) -> int | None: + """Extract the `| Status | \\`N\\` |` row from a lane summary.md. + + Returns the integer exit code or None if the file doesn't carry that + row. Used as a last-resort fallback for summary-only lanes + (private-oauth) and for any future lane whose results.json is + deleted by strict scrub before upload. + """ + if not summary_md: + return None + m = SUMMARY_STATUS_RE.search(summary_md) + if not m: + return None + try: + return int(m.group("status")) + except ValueError: + return None + + def collect_lane(lane_dir: Path) -> LaneReport | None: parts = lane_dir.parts if len(parts) < 3: @@ -189,19 +213,33 @@ def collect_lane(lane_dir: Path) -> LaneReport | None: provider = parts[-2] r = LaneReport(lane=lane, provider=provider) # Auth-canary lanes write JUnit XML; workflow-canary writes its own - # results.json. Read whichever exists — both populate the same - # LaneReport fields so downstream rendering / Haiku enrichment is - # source-agnostic. + # results.json; cargo lanes get a scraped results.json from + # emit_results_json.py. Read whichever exists — all three populate + # the same LaneReport fields so downstream rendering / Haiku + # enrichment is source-agnostic. parse_junit(lane_dir / "auth-canary-junit.xml", r) parse_results_json(lane_dir / "results.json", r) r.summary_md = read_tail(lane_dir / "summary.md", 4_000) r.log_tail = read_tail(lane_dir / "test-output.log", MAX_LOG_BYTES) - if r.tests == 0 and not r.log_tail: - r.status = "skip" - elif r.failed > 0: + + if r.failed > 0: r.status = "fail" elif r.tests > 0: r.status = "pass" + else: + # No structured counts. Fall back to the lane's exit code from + # summary.md so summary-only lanes (private-oauth) and any lane + # whose results.json got stripped by strict scrub still show + # up as pass/fail instead of misleading "skip". + summary_status = parse_summary_status(r.summary_md) + if summary_status is not None: + r.status = "pass" if summary_status == 0 else "fail" + if summary_status != 0 and not r.reason: + r.reason = f"lane exited with status {summary_status}" + elif r.log_tail: + r.status = "unknown" + else: + r.status = "skip" return r diff --git a/scripts/live-canary/run.sh b/scripts/live-canary/run.sh index 4ac58ce3076..33babe66a05 100755 --- a/scripts/live-canary/run.sh +++ b/scripts/live-canary/run.sh @@ -51,6 +51,7 @@ LOG_FILE="${RUN_DIR}/test-output.log" SUMMARY_FILE="${RUN_DIR}/summary.md" ENV_FILE="${RUN_DIR}/env-summary.txt" TRACE_STATUS_FILE="${RUN_DIR}/trace-fixture-status.txt" +RESULTS_FILE="${RUN_DIR}/results.json" : > "${LOG_FILE}" @@ -65,11 +66,24 @@ finish() { status=$? record_trace_status || true write_summary || true + emit_results_json || true log "[live-canary] summary=${SUMMARY_FILE}" log "[live-canary] log=${LOG_FILE}" exit "${status}" } +emit_results_json() { + # No-op for non-cargo lanes (auth-* uses JUnit XML, workflow-canary + # writes its own results.json from python). The helper bails silently + # when it can't find a `test result:` line or when the output file + # already exists. + python3 "$(dirname "$0")/emit_results_json.py" \ + --log "${LOG_FILE}" \ + --out "${RESULTS_FILE}" \ + --lane "${LANE}" \ + --provider "${PROVIDER}" 2>&1 | tee -a "${LOG_FILE}" >/dev/null +} + write_env_summary() { { echo "lane=${LANE}" diff --git a/scripts/live-canary/test_emit_results_json.py b/scripts/live-canary/test_emit_results_json.py new file mode 100644 index 00000000000..37cf8f35715 --- /dev/null +++ b/scripts/live-canary/test_emit_results_json.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Unit tests for emit_results_json.py. + +The scraper is load-bearing canary alerting code — a regex regression +silently downgrades a failing lane to ``status=skip`` (the original +bug this whole PR exists to fix). Lock the parser behavior with +fixtures for every shape the producers actually emit: + + - cargo `ok` / `FAILED` summary lines + - Rust >=1.73 multi-line panic shape (current codebase) + - Rust <1.73 legacy inline panic shape + - ``ignored`` tests collapsed onto the start line + - ``note: run with`` follow-up line skipped + - Token redaction in panic messages + - Multi-binary logs (one cargo invocation feeding several lanes) + - Interleaved output (parallel test threads) — must fail closed + - Parser counts cross-checked against cargo's own ``test result:`` + +Run with:: + + python3 -m pytest scripts/live-canary/test_emit_results_json.py -v + +Or directly:: + + python3 scripts/live-canary/test_emit_results_json.py +""" + +from __future__ import annotations + +import importlib.util +import re +import sys +import unittest +from pathlib import Path + + +# Load emit_results_json.py as a module without depending on a package +# layout. The script lives next to this test file by design. +_SPEC = importlib.util.spec_from_file_location( + "emit_results_json", + Path(__file__).parent / "emit_results_json.py", +) +emit = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(emit) + + +# --------------------------------------------------------------------------- +# Fixture logs — each block mirrors a real cargo invocation output shape. +# --------------------------------------------------------------------------- + + +MODERN_PANIC_LOG = """\ + +running 2 tests +test live_tests::zizmor_scan ... [LiveTest] Mode: LIVE — recording to /tmp/x + +thread 'live_tests::zizmor_scan' (27813) panicked at tests/e2e_live.rs:85:9: +Expected shell tool to be used for running zizmor, got: [] +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test live_tests::zizmor_scan_v2 ... [LiveTest] Trace recorded successfully +ok + +failures: + +failures: + live_tests::zizmor_scan + +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 14 filtered out; finished in 236.39s +""" + + +LEGACY_PANIC_LOG = """\ + +running 1 tests +test live_tests::legacy ... [stdout] + +thread 'live_tests::legacy' panicked at 'expected X, got Y', src/lib.rs:1:1 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + live_tests::legacy + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.00s +""" + + +IGNORED_INLINE_LOG = """\ + +running 3 tests +test foo::bar ... ignored +test foo::baz ... [stdout] +ok +test foo::qux ... [stdout] +ok + +test result: ok. 2 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.50s +""" + + +TOKEN_REDACTION_LOG = """\ + +running 1 tests +test leaky::tok ... [stdout] + +thread 'leaky::tok' (1) panicked at tests/leaky.rs:1:1: +got header Authorization: Bearer sk-ant-abcdef0123456789xyz instead of expected +FAILED + +failures: + leaky::tok + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s +""" + + +MULTI_BINARY_LOG = """\ + +running 1 tests +test bin_a::first ... [stdout] +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s + +running 2 tests +test bin_b::pass ... [stdout] +ok +test bin_b::fail ... [stdout] + +thread 'bin_b::fail' (2) panicked at tests/b.rs:5:1: +boom +FAILED + +failures: + bin_b::fail + +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.30s +""" + + +PARALLEL_INTERLEAVED_LOG = """\ + +running 2 tests +test bin_a::first ... [stdout chunk 1] +test bin_a::second ... [stdout chunk 2] +ok +ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s +""" + + +NON_CARGO_LOG = """\ +some pytest output +running workflow probes +passed +PASSED tests/foo.py::test_bar +""" + + +# --------------------------------------------------------------------------- +# Helpers — cross-check parsed counts against cargo's own `test result:` summary +# --------------------------------------------------------------------------- + + +def _cargo_summary_totals(log: str) -> tuple[int, int, int]: + """Return cumulative (passed, failed, ignored) across every + ``test result:`` line in a multi-binary log.""" + passed = failed = ignored = 0 + for m in emit.RESULT_RE.finditer(log): + passed += int(m.group("passed")) + failed += int(m.group("failed")) + ignored += int(m.group("ignored")) + return passed, failed, ignored + + +def _counts_from_entries(entries: list[dict]) -> tuple[int, int]: + p = sum(1 for e in entries if e.get("success")) + f = sum(1 for e in entries if not e.get("success")) + return p, f + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class HasCargoOutputTests(unittest.TestCase): + def test_modern_panic_log_is_cargo(self): + self.assertTrue(emit.has_cargo_output(MODERN_PANIC_LOG)) + + def test_non_cargo_log_is_skipped(self): + self.assertFalse(emit.has_cargo_output(NON_CARGO_LOG)) + + def test_empty_log_is_skipped(self): + self.assertFalse(emit.has_cargo_output("")) + + +class ParseLogModernPanicTests(unittest.TestCase): + def test_two_entries_one_failed(self): + entries = emit.parse_log(MODERN_PANIC_LOG) + self.assertEqual(len(entries), 2) + names = [e["mode"] for e in entries] + self.assertIn("live_tests::zizmor_scan", names) + self.assertIn("live_tests::zizmor_scan_v2", names) + + def test_modern_panic_message_extracted(self): + entries = emit.parse_log(MODERN_PANIC_LOG) + failed = [e for e in entries if not e["success"]] + self.assertEqual(len(failed), 1) + self.assertEqual( + failed[0]["details"]["error"], + "Expected shell tool to be used for running zizmor, got: []", + ) + + def test_modern_counts_match_cargo_summary(self): + entries = emit.parse_log(MODERN_PANIC_LOG) + passed, failed = _counts_from_entries(entries) + cargo_passed, cargo_failed, _ = _cargo_summary_totals(MODERN_PANIC_LOG) + self.assertEqual( + (passed, failed), + (cargo_passed, cargo_failed), + f"parser drift: got {passed}p/{failed}f vs cargo {cargo_passed}p/{cargo_failed}f", + ) + + +class ParseLogLegacyPanicTests(unittest.TestCase): + def test_legacy_inline_panic_message_extracted(self): + entries = emit.parse_log(LEGACY_PANIC_LOG) + self.assertEqual(len(entries), 1) + # Legacy header carries the message inline alongside the + # location; we accept the verbose form since we don't try to + # strip the trailing file:line:col from the message. + self.assertIn( + "expected X, got Y", + entries[0]["details"]["error"], + ) + + def test_legacy_counts_match_cargo_summary(self): + entries = emit.parse_log(LEGACY_PANIC_LOG) + passed, failed = _counts_from_entries(entries) + self.assertEqual((passed, failed), (0, 1)) + + +class IgnoredTestsTests(unittest.TestCase): + def test_ignored_inline_excluded_from_results(self): + entries = emit.parse_log(IGNORED_INLINE_LOG) + self.assertEqual(len(entries), 2) + for entry in entries: + self.assertNotIn("ignored", entry["mode"]) + self.assertTrue(entry["success"]) + + def test_ignored_counts_match_cargo_summary(self): + # cargo reports "2 passed; 0 failed; 1 ignored"; the parser + # emits 2 entries (passed). The ignored count is by design + # not represented as entries — ignored tests aren't results. + entries = emit.parse_log(IGNORED_INLINE_LOG) + passed, failed = _counts_from_entries(entries) + cargo_passed, cargo_failed, cargo_ignored = _cargo_summary_totals( + IGNORED_INLINE_LOG + ) + self.assertEqual(passed, cargo_passed) + self.assertEqual(failed, cargo_failed) + # Ignored stays out of entries — sanity-check that contract. + self.assertEqual(len(entries), cargo_passed + cargo_failed) + self.assertEqual(cargo_ignored, 1) + + +class NoteLineSkipTests(unittest.TestCase): + def test_note_run_with_line_not_picked_up_as_message(self): + # Both fixture logs have a `note: run with` line after the + # panic header. The parser must skip past it and find the + # real message on the line above (modern) or accept the + # inline message (legacy). + for log in (MODERN_PANIC_LOG, LEGACY_PANIC_LOG): + entries = emit.parse_log(log) + failed = [e for e in entries if not e["success"]] + for f in failed: + self.assertNotIn( + "note:", + f["details"]["error"], + f"`note: run with` leaked into panic message: {f['details']['error']}", + ) + + +class TokenRedactionTests(unittest.TestCase): + def test_anthropic_key_redacted(self): + entries = emit.parse_log(TOKEN_REDACTION_LOG) + msg = entries[0]["details"]["error"] + self.assertNotIn("sk-ant-abcdef0123456789xyz", msg) + self.assertIn("REDACTED", msg) + + def test_redact_function_covers_documented_shapes(self): + # Belt-and-braces — verify each pattern listed in + # REDACT_PATTERNS actually transforms its input. This list is + # the union of provider-specific shapes the canary actually + # exercises plus the generic assignment / JSON-quoted shapes + # carried by scripts/live-canary/scrub-artifacts.sh — keep them + # in sync so a token can't slip through emit_results_json that + # the shell scrubber would have caught. + cases = [ + ("token ghp_aaaaaaaaaaaaaaaaaaaa", "REDACTED_GITHUB_TOKEN"), + ("token github_pat_bbbbbbbbbbbbbbbbbbbbb", "REDACTED_GITHUB_PAT"), + ("token ya29.cccccccccccccccccc1234", "REDACTED_GOOGLE_TOKEN"), + ("token xoxb-1234567890-abc", "REDACTED_SLACK_TOKEN"), + ("Authorization: Bearer eyJabc.defg+hij/kl=", "Bearer <REDACTED>"), + # OpenAI bare key shape (sk- prefix without `ant-`). + ("token sk-abcDEF0123456789ghij", "REDACTED_OPENAI_KEY"), + # AWS access key ID literal — fixed AKIA prefix + 16 upper/digit. + ("aws AKIAIOSFODNN7EXAMPLE call", "REDACTED_AWS_ACCESS_KEY"), + # Generic env-var / assignment shapes the persona harness + # could leak (e.g. LIVE_CANARY_COMPOSIO_API_KEY=…). The + # generic api_key rule is the catch-all for any provider + # without a published key prefix (e.g. Composio). + ("api_key=composio-abcdef12345", "api_key=<REDACTED>"), + ("ACCESS_TOKEN: abcdef.1234567", "ACCESS_TOKEN=<REDACTED>"), + ("refresh-token = xyz.987654321", "refresh-token=<REDACTED>"), + ("client_secret=hunter2", "client_secret=<REDACTED>"), + ("password=hunter2", "password=<REDACTED>"), + ('"access_token": "abc.def.ghi"', '"access_token": "<REDACTED>"'), + ('"refresh_token": "rt-abc-123"', '"refresh_token": "<REDACTED>"'), + ('"api_key": "composio-abc"', '"api_key": "<REDACTED>"'), + ('"client_secret": "shh"', '"client_secret": "<REDACTED>"'), + ] + for raw, marker in cases: + redacted = emit.redact(raw) + self.assertIn(marker, redacted, f"redact() failed for {raw!r}") + + def test_anthropic_key_wins_over_openai_pattern(self): + # Both `sk-ant-…` and `sk-…` are valid prefixes. The Anthropic + # rule is listed first so the dedicated label survives — and + # the OpenAI rule uses a negative lookahead as belt-and-braces + # for future reorderings. Lock both invariants here. + out = emit.redact("token sk-ant-abcdef0123456789xyz") + self.assertIn("REDACTED_ANTHROPIC_KEY", out) + self.assertNotIn("REDACTED_OPENAI_KEY", out) + + def test_openai_pattern_does_not_swallow_sk_ant(self): + # If someone reorders REDACT_PATTERNS, the negative lookahead + # on the OpenAI rule still keeps `sk-ant-…` strings out of the + # OpenAI bucket. Test the rule in isolation. + for pattern, replacement in emit.REDACT_PATTERNS: + if replacement == "<REDACTED_OPENAI_KEY>": + self.assertIsNone( + pattern.search("sk-ant-abcdef0123456789xyz"), + "OpenAI pattern must not match sk-ant- strings", + ) + self.assertIsNotNone( + pattern.search("sk-abcDEF0123456789ghij"), + "OpenAI pattern must match bare sk- strings", + ) + break + else: + self.fail("no <REDACTED_OPENAI_KEY> rule in REDACT_PATTERNS") + + +class MultiBinaryTests(unittest.TestCase): + def test_multi_binary_log_aggregates_all_entries(self): + entries = emit.parse_log(MULTI_BINARY_LOG) + # bin_a: 1 ok; bin_b: 1 ok + 1 FAILED → 3 entries total + self.assertEqual(len(entries), 3) + names = [e["mode"] for e in entries] + self.assertEqual( + names, + ["bin_a::first", "bin_b::pass", "bin_b::fail"], + ) + + def test_multi_binary_panic_message_attached_to_correct_test(self): + entries = emit.parse_log(MULTI_BINARY_LOG) + failed = [e for e in entries if not e["success"]] + self.assertEqual(len(failed), 1) + self.assertEqual(failed[0]["mode"], "bin_b::fail") + self.assertEqual(failed[0]["details"]["error"], "boom") + + def test_multi_binary_counts_match_cumulative_cargo_summary(self): + entries = emit.parse_log(MULTI_BINARY_LOG) + passed, failed = _counts_from_entries(entries) + cargo_passed, cargo_failed, _ = _cargo_summary_totals(MULTI_BINARY_LOG) + # cargo reports two summaries: 1p/0f then 1p/1f → cumulative 2p/1f + self.assertEqual((passed, failed), (cargo_passed, cargo_failed)) + self.assertEqual((cargo_passed, cargo_failed), (2, 1)) + + +class InterleavedOutputTests(unittest.TestCase): + def test_parallel_log_raises_interleaved_error(self): + with self.assertRaises(emit.InterleavedOutputError) as ctx: + emit.parse_log(PARALLEL_INTERLEAVED_LOG) + # Error message should name both tests so debugging is easy. + msg = str(ctx.exception) + self.assertIn("bin_a::first", msg) + self.assertIn("bin_a::second", msg) + self.assertIn("--test-threads=1", msg) + + +class RegexShapeTests(unittest.TestCase): + """Lock in the regexes themselves — if any of these match + something they shouldn't (or stop matching the canonical form), + the parser silently produces wrong output. Catch that here.""" + + def test_result_re_matches_canonical_failed_line(self): + line = ( + "test result: FAILED. 1 passed; 1 failed; 0 ignored; " + "0 measured; 14 filtered out; finished in 236.39s" + ) + m = emit.RESULT_RE.search(line) + self.assertIsNotNone(m) + self.assertEqual(m.group("outcome"), "FAILED") + self.assertEqual(m.group("passed"), "1") + self.assertEqual(m.group("failed"), "1") + self.assertEqual(m.group("ignored"), "0") + + def test_test_start_re_matches_with_stdout_glue(self): + m = emit.TEST_START_RE.match( + "test live_tests::zizmor_scan ... [LiveTest] Mode: LIVE — recording" + ) + self.assertIsNotNone(m) + self.assertEqual(m.group("name"), "live_tests::zizmor_scan") + # `[LiveTest] Mode: ...` is not a clean outcome word. + self.assertNotIn(m.group("trailer").strip(), {"ok", "FAILED", "ignored"}) + + def test_test_start_re_matches_inline_ignored(self): + m = emit.TEST_START_RE.match("test foo::bar ... ignored") + self.assertIsNotNone(m) + self.assertEqual(m.group("trailer").strip(), "ignored") + + def test_outcome_re_only_matches_bare_lines(self): + for good in ("ok", "FAILED", "ignored"): + self.assertIsNotNone(emit.OUTCOME_RE.match(good)) + for bad in ( + "ok and then some", + " ok", # leading whitespace not allowed + "OK", # case-sensitive + "test foo ... ok", + "", + ): + self.assertIsNone(emit.OUTCOME_RE.match(bad), f"falsely matched: {bad!r}") + + def test_panic_re_matches_modern(self): + m = emit.PANIC_RE.match( + "thread 'live_tests::zizmor_scan' (27813) panicked at tests/e2e_live.rs:85:9:" + ) + self.assertIsNotNone(m) + self.assertEqual(m.group("name"), "live_tests::zizmor_scan") + + def test_panic_re_matches_legacy(self): + # No worker id between thread name and `panicked at` — the + # lazy `.*?` between them must allow this. + m = emit.PANIC_RE.match( + "thread 'live_tests::legacy' panicked at 'expected X', src/lib.rs:1:1" + ) + self.assertIsNotNone(m) + self.assertEqual(m.group("name"), "live_tests::legacy") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/live-canary/test_notify_slack.py b/scripts/live-canary/test_notify_slack.py new file mode 100644 index 00000000000..bf75d84f19c --- /dev/null +++ b/scripts/live-canary/test_notify_slack.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Unit tests for notify_slack.py helpers. + +Focus is on `parse_summary_status` — the `summary.md` → exit-code +fallback that classifies lane status when neither JUnit XML nor +``results.json`` is present (summary-only lanes like private-oauth, +or any lane whose ``results.json`` got stripped by strict scrub before +upload). This path is part of the status-classification surface, so +parser drift would silently mislabel lanes. + +Run with:: + + python3 -m pytest scripts/live-canary/test_notify_slack.py -v + +Or directly:: + + python3 scripts/live-canary/test_notify_slack.py +""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +# Mirror test_emit_results_json.py's loader so this file also runs +# standalone without a package layout. notify_slack.py uses +# ``@dataclass``, which introspects ``sys.modules`` for the owning +# module, so we have to register the module before executing it — +# otherwise dataclass decoration raises an AttributeError on import. +_SPEC = importlib.util.spec_from_file_location( + "notify_slack", + Path(__file__).parent / "notify_slack.py", +) +notify = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = notify +_SPEC.loader.exec_module(notify) + + +# Canonical summary.md produced by scripts/live-canary/run.sh's +# `write_summary` helper. The status row is the single field this +# parser cares about — everything else is decoration that must not +# trigger the regex. +_SUMMARY_TEMPLATE = """\ +## Live Canary Summary + +| Field | Value | +| --- | --- | +| Lane | `private-oauth` | +| Scenario | `<default>` | +| Provider | `dedicated-runner` | +| Status | `{status}` | +| Started | `2026-05-17T12:00:00Z` | +| Finished | `2026-05-17T12:42:13Z` | +| Commit | `abcdef0123456789` | + +Artifacts: +- `test-output.log` +- `env-summary.txt` +- `trace-fixture-status.txt` +""" + + +class ParseSummaryStatusTests(unittest.TestCase): + def test_zero_status_means_pass(self): + self.assertEqual( + notify.parse_summary_status(_SUMMARY_TEMPLATE.format(status="0")), + 0, + ) + + def test_nonzero_status_means_fail(self): + self.assertEqual( + notify.parse_summary_status(_SUMMARY_TEMPLATE.format(status="1")), + 1, + ) + + def test_negative_status_is_preserved(self): + # `run.sh` shouldn't write negatives in practice, but the regex + # allows them and `collect_lane` treats any non-zero as fail — + # confirm the integer flows through unmodified. + self.assertEqual( + notify.parse_summary_status(_SUMMARY_TEMPLATE.format(status="-1")), + -1, + ) + + def test_large_status_is_preserved(self): + # Bash exit codes wrap at 256, but the regex is unbounded; + # ensure no accidental truncation/clamping by the parser. + self.assertEqual( + notify.parse_summary_status(_SUMMARY_TEMPLATE.format(status="137")), + 137, + ) + + def test_missing_status_row_returns_none(self): + # Workflow-canary summary.md (different writer) doesn't carry a + # `| Status | \`N\` |` row — caller falls through to log-tail + # heuristic. Must return None, not raise. + no_status = ( + "## Live Canary Summary\n\n" + "| Field | Value |\n" + "| --- | --- |\n" + "| Lane | `auth-canary` |\n" + ) + self.assertIsNone(notify.parse_summary_status(no_status)) + + def test_empty_string_returns_none(self): + # `read_tail` returns "" when summary.md is missing entirely. + self.assertIsNone(notify.parse_summary_status("")) + + def test_malformed_status_value_returns_none(self): + # If the writer ever emits a non-integer literal in the status + # cell, the parser must degrade to None rather than crash so + # the lane still surfaces (as "unknown") in Slack. + malformed = _SUMMARY_TEMPLATE.replace("`{status}`", "`oops`").format() + self.assertIsNone(notify.parse_summary_status(malformed)) + + def test_status_row_not_at_line_start_is_ignored(self): + # The regex is anchored with `^...$` under MULTILINE. A row + # appearing inline (e.g. quoted inside a prose paragraph) must + # not be picked up — that would let a literal block-quoted + # summary in a comment flip the lane status. + inline = ( + "Some prose mentioning `| Status | `9` |` inline " + "but not as a real table row." + ) + self.assertIsNone(notify.parse_summary_status(inline)) + + def test_status_row_with_extra_whitespace(self): + # `write_summary` uses single-space padding, but accept the + # common variations (no-pad, double-pad) so a future cosmetic + # change to the writer doesn't break classification silently. + for variant in ( + "|Status|`0`|", + "| Status | `0` |", + "| Status |\t`0`\t|", + ): + with self.subTest(variant=variant): + doc = "## summary\n\n" + variant + "\n" + # All variants should resolve to the same exit code. + # If the regex is too strict to match a variant, the + # test fails closed (we'd rather know now than discover + # in prod that a writer tweak silently broke parsing). + got = notify.parse_summary_status(doc) + self.assertEqual(got, 0, f"variant not parsed: {variant!r}") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/live_canary/auth_registry.py b/scripts/live_canary/auth_registry.py index 73edadf5f95..aa7c38c505b 100644 --- a/scripts/live_canary/auth_registry.py +++ b/scripts/live_canary/auth_registry.py @@ -17,6 +17,12 @@ "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_wasm_tool_oauth_exchange_failure_leaves_extension_unauthed", "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_wasm_tool_first_chat_auth_attempt_emits_auth_url", "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_chat_first_gmail_installs_prompts_and_retries", + # ironclaw#3533 — chat-driven `tool_install` raises an approval gate, + # user approves via the approval card, then the auth card surfaces. + # Pairs with `test_chat_first_gmail_installs_prompts_and_retries` + # (auto-approve variant); both must stay green so the regression that + # made "connect my telegram" narrate manual UI steps can't ship again. + "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_chat_install_approval_then_auth_card", "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_settings_first_gmail_auth_then_chat_runs", "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_settings_first_custom_mcp_auth_then_chat_runs", "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_wasm_tool_oauth_refresh_on_demand", @@ -26,6 +32,17 @@ AUTH_CHANNEL_TESTS = [ "tests/e2e/scenarios/test_v2_auth_oauth_matrix.py::test_wasm_channel_oauth_roundtrip", + # ironclaw#3317 — pairing reply must name every IronClaw surface, and + # `approve telegram CODE` typed in chat must complete the pairing. + # Without this lane the whole class of "user pastes code in the wrong + # place, agent improvises an unhelpful answer" regressions would only + # surface in production. + "tests/e2e/scenarios/test_telegram_pairing_chat_claim.py::test_telegram_pairing_reply_names_every_surface", + "tests/e2e/scenarios/test_telegram_pairing_chat_claim.py::test_chat_surface_approves_pairing_code", + # PR #3381 review — `approve telegram CODE` typed in Telegram itself + # must NOT complete pairing (the allowlist gate intercepts before the + # agent parser), and the bot's reply must not promise that surface. + "tests/e2e/scenarios/test_telegram_pairing_chat_claim.py::test_telegram_dm_approve_command_is_intercepted_by_allowlist_gate", ] AUTH_PROFILES: dict[str, list[str]] = { diff --git a/scripts/live_canary/common.py b/scripts/live_canary/common.py index 95ca37ce0a4..5a1e1c660d8 100644 --- a/scripts/live_canary/common.py +++ b/scripts/live_canary/common.py @@ -321,6 +321,18 @@ def build_gateway_env( "WASM_CHANNELS_DIR": str(channels_dir), "ONBOARD_COMPLETED": "true", } + # Propagate select agent-time env vars from the parent process. + # The live-canary workflow YAML sets ALLOW_LOCAL_TOOLS=true and + # AGENT_AUTO_APPROVE_TOOLS=true at the job level expecting them to + # reach the gateway, but Popen(env=env) replaces the env wholesale + # so without this propagation they were silently dropped. First + # probe to notice was tool_install_chat in PR #3682 — chat-driven + # tool dispatches parked on an approval gate instead of + # auto-approving and never reached `installed=true`. + for var in ("ALLOW_LOCAL_TOOLS", "AGENT_AUTO_APPROVE_TOOLS"): + value = os.environ.get(var) + if value: + env[var] = value if extra_env: env.update({key: value for key, value in extra_env.items() if value}) return env @@ -485,7 +497,7 @@ async def start_gateway_stack( # defaults llm_backend to `nearai`, so the env config is ignored # and the agent attempts an interactive NearAI auth flow that # never completes in CI. Mirrors the pattern documented in - # tests/e2e/CLAUDE.md and used by test_v2_tool_activate_surface.py. + # tests/e2e/CLAUDE.md. await _pin_mock_llm_settings(base_url, gateway_token, mock_llm_url) return GatewayStack( base_url=base_url, diff --git a/scripts/pre-commit-safety.sh b/scripts/pre-commit-safety.sh index e704dba087b..f8193d6d4a9 100755 --- a/scripts/pre-commit-safety.sh +++ b/scripts/pre-commit-safety.sh @@ -427,12 +427,51 @@ if [ -n "$PROJECTION_HITS" ]; then echo "$PROJECTION_HITS" | sed 's/^/ /' fi +# 10. Cross-tenant safety: an UNSCOPED `sse.broadcast(...)` call (i.e. not +# `broadcast_for_user`) delivers the event to every connected +# subscriber, regardless of which user owns the underlying state. +# In multi-tenant deployments that pattern leaks tool calls, log +# lines, and onboarding state across tenants — see the cross-tenant +# thread visibility incident and the `dispatch_status_event` fix. +# +# A new `sse.broadcast(...)` line is acceptable only if it is one +# of: +# (a) Transport-only (heartbeat / stream_chunk) — the canonical +# projection-exempt category that already documents the +# empty-payload allowlist. +# (b) Annotated with `// multi-tenant-safe: <reason>` on the same +# line, naming the structural reason the event cannot leak +# tenant-bound state (e.g. "single-tenant fallback inside an +# explicit multi_tenant_mode=false branch"). +# Otherwise prefer `broadcast_for_user(uid, ...)` with a known +# `user_id` derived from the source-log payload. +# +# This check runs only on `src/**` and `crates/**` — `tests/**` and +# `#[cfg(test)]` blocks were already filtered out upstream. +# Marker matching: `//.*multi-tenant-safe: <non-whitespace>` — the +# `//.*` prefix anchors the marker to a Rust comment but allows +# additional annotations on the same line (e.g. when `// projection- +# exempt: ...; multi-tenant-safe: ...` carry both markers in one +# trailing comment because Rust line comments cannot be nested). +MT_BROADCAST_HITS=$(echo "$DIFF_OUTPUT_NO_TESTS" | grep -nE '^\+' \ + | grep -E '(^|[^[:alnum:]_])sse\.broadcast[[:space:]]*\(' \ + | grep -vE '\.broadcast_for_user' \ + | grep -vE '// projection-exempt: transport-only,[[:space:]]*[^[:space:]]' \ + | grep -vE '//.*multi-tenant-safe: [^[:space:]]' \ + | grep -vE '// safety:|:\+\+\+ ' \ + | head -5 || true) +if [ -n "$MT_BROADCAST_HITS" ]; then + warn "MULTITENANT" "Unscoped \`sse.broadcast(...)\` in code reachable in multi-tenant mode. Switch to \`broadcast_for_user(&user_id, ...)\` or annotate with '// multi-tenant-safe: <reason>'. See \`dispatch_status_event\` in \`src/channels/web/mod.rs\` and the cross-tenant thread visibility incident." + echo "$MT_BROADCAST_HITS" | sed 's/^/ /' +fi + if [ "$WARNINGS" -gt 0 ]; then echo "" echo "Found $WARNINGS potential issue(s). Fix them or add '// safety: <reason>' to suppress." echo "(For DISPATCH warnings, use '// dispatch-exempt: <reason>' instead.)" echo "(For CREDNAME warnings, use '// web-identity-exempt: <reason>' instead.)" echo "(For PROJECTION warnings, use '// projection-exempt: <category>, <detail>' instead.)" + echo "(For MULTITENANT warnings, use '// multi-tenant-safe: <reason>' instead.)" echo "" exit 1 fi diff --git a/scripts/reborn-e2e-rust.sh b/scripts/reborn-e2e-rust.sh new file mode 100755 index 00000000000..467178175f7 --- /dev/null +++ b/scripts/reborn-e2e-rust.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the deterministic Rust-side Reborn E2E gate. +# Usage: +# scripts/reborn-e2e-rust.sh # all groups +# scripts/reborn-e2e-rust.sh architecture # boundary + host runtime spine +# scripts/reborn-e2e-rust.sh runtimes # dispatcher/runtime/process lanes +# scripts/reborn-e2e-rust.sh substrates # event/network/secret substrates +# +# Extra cargo test args can be passed through CARGO_TEST_ARGS, for example: +# CARGO_TEST_ARGS='-- --nocapture' scripts/reborn-e2e-rust.sh architecture + +group="${1:-all}" +extra_args=${CARGO_TEST_ARGS:-"-- --nocapture"} + +run_test() { + local package="$1" + local test_name="$2" + echo "::group::cargo test -p ${package} --test ${test_name}" + # shellcheck disable=SC2086 # extra_args intentionally expands into cargo's trailing args. + cargo test -p "${package}" --test "${test_name}" ${extra_args} + echo "::endgroup::" +} + +run_architecture() { + run_test ironclaw_architecture reborn_dependency_boundaries + run_test ironclaw_host_runtime host_runtime_contract + run_test ironclaw_host_runtime host_runtime_services_contract + run_test ironclaw_host_runtime reborn_e2e_gate + run_test ironclaw_host_runtime reborn_invoke_vertical_slice + run_test ironclaw_host_runtime runtime_http_egress_contract + run_test ironclaw_host_runtime builtin_obligation_handler_contract + run_test ironclaw_host_runtime obligation_services_composition_contract + run_test ironclaw_host_runtime production_trust_contract + run_test ironclaw_capabilities capability_boundary_contract + run_test ironclaw_capabilities capability_host_contract + run_test ironclaw_capabilities capability_host_dispatcher_integration + run_test ironclaw_capabilities capability_host_process_integration + run_test ironclaw_capabilities capability_host_run_state_contract + run_test ironclaw_capabilities capability_host_spawn_contract + run_test ironclaw_capabilities capability_obligation_handler_contract +} + +run_runtimes() { + run_test ironclaw_dispatcher boundary_contract + run_test ironclaw_dispatcher dispatch_contract + run_test ironclaw_dispatcher event_dispatch_contract + run_test ironclaw_dispatcher runtime_dispatcher_integration + run_test ironclaw_dispatcher vertical_slice_contract + run_test ironclaw_wasm wasm_dispatch_integration + run_test ironclaw_wasm wasm_http_adapter_contract + run_test ironclaw_wasm wit_tool_runtime_contract + run_test ironclaw_scripts script_dispatch_integration + run_test ironclaw_scripts script_http_adapter_contract + run_test ironclaw_scripts script_runner_contract + run_test ironclaw_mcp mcp_adapter_contract + run_test ironclaw_mcp mcp_dispatch_integration + run_test ironclaw_processes process_dispatch_integration + run_test ironclaw_processes process_host_contract + run_test ironclaw_processes process_services_contract + run_test ironclaw_processes process_store_contract +} + +run_substrates() { + run_test ironclaw_events durable_log_contract + run_test ironclaw_filesystem catalog_contract + run_test ironclaw_filesystem filesystem_contract + run_test ironclaw_network boundary_contract + run_test ironclaw_network network_http_egress_contract + run_test ironclaw_network network_policy_contract + run_test ironclaw_secrets boundary_contract + run_test ironclaw_secrets secret_store_contract + run_test ironclaw_resources resource_governor_contract + run_test ironclaw_run_state approval_resolution_contract + run_test ironclaw_run_state run_state_contract + run_test ironclaw_approvals approval_resolution_contract + run_test ironclaw_approvals boundary_contract + run_test ironclaw_authorization boundary_contract + run_test ironclaw_authorization capability_access_contract + run_test ironclaw_authorization capability_lease_contract +} + +case "${group}" in + architecture) + run_architecture + ;; + runtimes) + run_runtimes + ;; + substrates) + run_substrates + ;; + all) + run_architecture + run_runtimes + run_substrates + ;; + *) + echo "unknown Reborn E2E group: ${group}" >&2 + echo "expected one of: architecture, runtimes, substrates, all" >&2 + exit 2 + ;; +esac diff --git a/scripts/test-pre-commit-safety.sh b/scripts/test-pre-commit-safety.sh index f26459fe7ff..676441d8609 100755 --- a/scripts/test-pre-commit-safety.sh +++ b/scripts/test-pre-commit-safety.sh @@ -165,6 +165,79 @@ assert_flagged "CREDNAME: bare CredentialName reference is flagged" \ "$CREDNAME_POS" \ "$CREDNAME_NEG" +# ── MULTITENANT ─────────────────────────────────────────────── +# A new unscoped `sse.broadcast(...)` must either be transport-only or +# carry an explicit `// multi-tenant-safe: <reason>` annotation. +# `broadcast_for_user(...)` is the safe path and must be exempt. +MT_POS='(^|[^[:alnum:]_])sse\.broadcast[[:space:]]*\(' +MT_NEG='\.broadcast_for_user|// projection-exempt: transport-only,[[:space:]]*[^[:space:]]|//.*multi-tenant-safe: [^[:space:]]|// safety:|:\+\+\+ ' + +assert_filtered "MULTITENANT: diff header line is filtered" \ + "+++ b/src/extensions/manager.rs" \ + "$MT_POS" \ + "$MT_NEG" + +assert_filtered "MULTITENANT: broadcast_for_user is exempt" \ + "+ sse.broadcast_for_user(&user, event);" \ + "$MT_POS" \ + "$MT_NEG" + +assert_filtered "MULTITENANT: heartbeat (transport-only) is exempt" \ + "+ sse.broadcast(AppEvent::Heartbeat); // projection-exempt: transport-only, heartbeat" \ + "$MT_POS" \ + "$MT_NEG" + +# Receiver-prefixed call sites: the boundary regex matches `.sse.broadcast(` +# because the leading `.` is non-alnum-and-non-underscore, so the existing +# check covers production patterns like `state.sse.broadcast(`, +# `gw_state.sse.broadcast(`, and rustfmt-wrapped chains. These tests pin +# that behaviour against a future regex tightening. +assert_flagged "MULTITENANT: state.sse.broadcast (receiver-prefixed) is flagged" \ + "+ state.sse.broadcast(event);" \ + "$MT_POS" \ + "$MT_NEG" + +assert_flagged "MULTITENANT: gw_state.sse.broadcast (snake_case receiver) is flagged" \ + "+ gw_state.sse.broadcast(event);" \ + "$MT_POS" \ + "$MT_NEG" + +assert_filtered "MULTITENANT: state.sse.broadcast with annotation is exempt" \ + "+ state.sse.broadcast(event); // multi-tenant-safe: single-tenant fallback" \ + "$MT_POS" \ + "$MT_NEG" + +assert_filtered "MULTITENANT: explicit multi-tenant-safe annotation is exempt" \ + "+ sse.broadcast(event); // multi-tenant-safe: only reached when multi_tenant_mode=false" \ + "$MT_POS" \ + "$MT_NEG" + +# Compound annotation: a single `// ` comment can carry both +# `projection-exempt:` and `multi-tenant-safe:` because Rust line +# comments don't nest. The marker scanner must accept either marker +# anywhere in the trailing comment, not only when the comment opens +# with it. See `src/channels/web/mod.rs::dispatch_status_event` and +# `src/main.rs` sandbox JobEvent dispatcher. +assert_filtered "MULTITENANT: compound projection-exempt + multi-tenant-safe annotation is exempt" \ + "+ sse.broadcast(event); // projection-exempt: bridge dispatcher, single-tenant unscoped status; multi-tenant-safe: only reached when multi_tenant_mode=false" \ + "$MT_POS" \ + "$MT_NEG" + +assert_flagged "MULTITENANT: bare unscoped sse.broadcast is flagged" \ + "+ sse.broadcast(event);" \ + "$MT_POS" \ + "$MT_NEG" + +assert_flagged "MULTITENANT: unscoped broadcast with bridge-dispatcher projection-exempt is still flagged" \ + "+ sse.broadcast(event); // projection-exempt: bridge dispatcher, status update" \ + "$MT_POS" \ + "$MT_NEG" + +assert_flagged "MULTITENANT: unscoped broadcast with empty multi-tenant-safe detail is still flagged" \ + "+ sse.broadcast(event); // multi-tenant-safe: " \ + "$MT_POS" \ + "$MT_NEG" + echo "" echo "Passed: $PASS, Failed: $FAIL" [ "$FAIL" -eq 0 ] diff --git a/scripts/workflow_canary/run_workflow_canary.py b/scripts/workflow_canary/run_workflow_canary.py index c6a7df60968..0e53ffebd24 100644 --- a/scripts/workflow_canary/run_workflow_canary.py +++ b/scripts/workflow_canary/run_workflow_canary.py @@ -146,6 +146,11 @@ "run", "Unauthenticated tool call surfaces graceful auth path (no 5xx / Error 400)", ), + "tool_install_chat": ( + "scripts.workflow_canary.scenarios.tool_install_chat", + "run", + "Chat-driven tool_install reaches installed=true (regression #3366)", + ), # log_assertions intentionally registered LAST so it scans the # gateway log surface produced by every preceding probe. "log_assertions": ( diff --git a/scripts/workflow_canary/scenarios/tool_install_chat.py b/scripts/workflow_canary/scenarios/tool_install_chat.py new file mode 100644 index 00000000000..46fccb85c22 --- /dev/null +++ b/scripts/workflow_canary/scenarios/tool_install_chat.py @@ -0,0 +1,496 @@ +"""Chat-driven tool_install probe — would have caught the May 8 regression +shipped in #3366, where `tool_install` (and the unified `tool_activate` +it was meant to be subsumed by) was silently dropped from the agent's +callable surface. The HTTP install API kept working, so every existing +canary stayed green for five days while the chat path was broken. + +What this probe asserts +----------------------- + +1. **Chat send succeeds** (HTTP 202 from `/api/chat/send`). +2. **The agent actually invokes `tool_install`** for the requested + extension within ``TIMEOUT_S``. Asserted directly from history — + no inference from secondary effects. +3. **The extension reaches `installed=true`** via `GET /api/extensions`. +4. **No forbidden error/panic substrings** in the rendered history. + +Hooks into the existing mock LLM contract +----------------------------------------- + +The mock LLM at ``tests/e2e/mock_llm.py`` already ships a canned +flow for ``"check gmail unread"`` (mock_llm.py:1257-1292): + +- Turn 1: dispatches a bare ``gmail`` tool call. Engine rejects with + "Extension not installed:" / "is not callable in this execution + context". +- Turn 2: emits ``tool_install(name="gmail")``. **This is exactly + the call #3366 broke** — by hiding ``tool_install`` from the + agent surface, the mock LLM's tool call would be rejected / + ignored, gmail would never install, and the existing + ``auth_recovery`` probe (which only greps for error substrings) + would still pass. +- Turn 3: re-emits ``gmail``; engine raises an OAuth gate. + +We don't drive the OAuth completion here — gmail reaching +``installed=true`` is the failure boundary the regression crosses. + +Why a separate probe and not just tightening ``auth_recovery`` +-------------------------------------------------------------- + +``auth_recovery`` is intentionally lenient: it asserts the *recovery +shape* (no 5xx, no panic) rather than that any specific tool ran. +Tightening it would conflate two distinct guarantees — "the engine +recovers gracefully from unauthenticated calls" and "the agent has +an install primitive on its callable surface." Keep both, isolate +the contracts. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from pathlib import Path +from typing import Any + +import httpx + +from scripts.live_canary.common import ProbeResult + + +# Extension we drive through the install flow. Pinned to gmail because +# the mock LLM's ``check gmail unread`` canned response already encodes +# the full chat→tool_install→retry sequence. Swapping to a different +# extension would require a new mock LLM branch. +TARGET_EXTENSION = "gmail" + +# Trigger phrase that maps to mock_llm.py's gmail-install canned flow. +# Keep in lockstep with ``mock_llm.py:1257-1292`` — if the trigger +# string changes there, change it here. +TRIGGER_PROMPT = "check gmail unread" + +# 60s is generous. Locally the full flow (chat → tool_install → +# install completes → gmail retry → gate) settles in <10s; the budget +# absorbs slow CI runners and any post-install verification the engine +# does (capability seeding, etc). +TIMEOUT_S = 60.0 + +# Polling cadence on /api/extensions. Cheap call, so tight is fine. +POLL_INTERVAL_S = 0.5 + +FORBIDDEN_FRAGMENTS = [ + "Error 400", + "Internal Server Error", + "panicked", + "Traceback", + "rust panic", +] + + +async def _open_thread(client: httpx.AsyncClient, base_url: str) -> str: + response = await client.post(f"{base_url}/api/chat/thread/new", timeout=15.0) + response.raise_for_status() + return response.json()["id"] + + +async def _send_chat( + client: httpx.AsyncClient, base_url: str, thread_id: str, content: str +) -> int: + response = await client.post( + f"{base_url}/api/chat/send", + json={"content": content, "thread_id": thread_id}, + timeout=30.0, + ) + return response.status_code + + +async def _read_history( + client: httpx.AsyncClient, base_url: str, thread_id: str +) -> dict[str, Any]: + response = await client.get( + f"{base_url}/api/chat/history", + params={"thread_id": thread_id}, + timeout=15.0, + ) + response.raise_for_status() + return response.json() + + +async def _get_extension( + client: httpx.AsyncClient, base_url: str, name: str +) -> dict[str, Any] | None: + response = await client.get(f"{base_url}/api/extensions", timeout=15.0) + response.raise_for_status() + for ext in response.json().get("extensions", []): + if ext.get("name") == name: + return ext + return None + + +async def _remove_extension_if_present( + client: httpx.AsyncClient, base_url: str, name: str +) -> bool: + """Best-effort removal of `name` so the probe starts from a clean + slate regardless of prior scenario state. + + The workflow-canary runner shares one gateway stack across all + scenarios; auth_recovery runs immediately before this probe and + uses the same `check gmail unread` prompt, which can leave gmail + installed (or part-installed) by the time we run. Without + isolation, the probe's "gmail registered after our chat send" + signal becomes "gmail registered for *some* reason," which fails + the assertion serrrfirat flagged: the probe stops proving a fresh + chat → tool_install → install path. + + Returns True if removal was attempted (extension was present). + """ + if await _get_extension(client, base_url, name) is None: + return False + response = await client.post( + f"{base_url}/api/extensions/{name}/remove", timeout=30.0 + ) + response.raise_for_status() + # Confirm the gateway no longer lists it before letting the probe + # proceed; otherwise a slow removal race would let our chat-send + # see stale "already installed" state. + for _ in range(20): + if await _get_extension(client, base_url, name) is None: + return True + await asyncio.sleep(0.25) + raise RuntimeError( + f"extension {name!r} still present after remove + 5s grace; " + "cannot guarantee fresh-install precondition" + ) + + +def _history_has_tool_install_call_for( + history: dict[str, Any], target: str +) -> bool: + """True iff a ``tool_install`` invocation in history targets the + given extension by name. + + Walks the tree looking for any dict that names ``tool_install`` and + binds its argument payload to ``target``. Recognises the three + envelope shapes the gateway uses today: + + - Direct: ``{"name": "tool_install", "arguments": {"name": "gmail"}}`` + - OpenAI-style: ``{"function": {"name": "tool_install", + "arguments": "{\"name\": \"gmail\"}"}}`` — arguments here may be + either a dict or a JSON string the model emitted. + - Pending-gate: ``{"tool_name": "tool_install", "parameters": + "{\"name\": \"gmail\"}"}`` — `parameters` is a JSON string the + gate exposes on ``/api/chat/history``. + + Critically: the tool-name check and the target check happen on the + *same* invocation. A bare ``"tool_install" in history`` (anywhere) + combined with ``"gmail" in history`` (elsewhere) is the false + positive serrrfirat flagged — a tool_install for a different + extension followed by gmail appearing for unrelated reasons would + pass that weak check. This walker rejects that. + """ + + def _args_target(args: Any) -> str | None: + if isinstance(args, str): + try: + args = json.loads(args) + except (ValueError, json.JSONDecodeError): + return None + if isinstance(args, dict): + for key in ("name", "extension"): + value = args.get(key) + if isinstance(value, str): + return value + return None + + def _check_call(name_field: str, args_field: str, node: dict) -> bool: + if node.get(name_field) != "tool_install": + return False + return _args_target(node.get(args_field)) == target + + def _walk(node: Any) -> bool: + if isinstance(node, dict): + if _check_call("name", "arguments", node): + return True + if _check_call("tool_name", "parameters", node): + return True + fn = node.get("function") + if isinstance(fn, dict) and fn.get("name") == "tool_install": + if _args_target(fn.get("arguments")) == target: + return True + return any(_walk(v) for v in node.values()) + if isinstance(node, list): + return any(_walk(x) for x in node) + return False + + return _walk(history) + + +# When the probe fails, we want enough breadcrumbs in the artifact to +# diagnose what the agent actually did without re-running the canary. +# The slack notifier surfaces `details.error` plus the structured +# fields it knows about; extra keys we drop into `details` show up in +# the artifact for whoever opens the failing-lane drilldown. +_TOOL_CALL_KEYS = ("tool_name", "name", "function", "action") + + +def _collect_tool_calls(history: dict[str, Any]) -> list[str]: + """Best-effort enumeration of tool-call names found in history. + + Matches a few common envelope shapes the gateway has used: + ``tool_calls: [{"name": ...}]`` on assistant messages, top-level + ``tool_name``/``action`` on turn records, ``<tool_output name="...">`` + on tool-result content. Deduplicates while preserving order so the + diagnostic doesn't double-count parallel dispatches. + """ + seen: list[str] = [] + + def _add(name: Any) -> None: + if isinstance(name, str) and name and name not in seen: + seen.append(name) + + def _walk(node: Any) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "tool_calls" and isinstance(value, list): + for call in value: + if isinstance(call, dict): + _add(call.get("name") or call.get("function")) + elif key in _TOOL_CALL_KEYS and isinstance(value, str): + _add(value) + else: + _walk(value) + elif isinstance(node, list): + for item in node: + _walk(item) + elif isinstance(node, str): + # `<tool_output name="...">` wrapping + import re + + for m in re.finditer(r'<tool_output\s+name="([^"]+)"', node): + _add(m.group(1)) + + _walk(history) + return seen + + +def _last_assistant_text(history: dict[str, Any]) -> str: + """Pull the last assistant-side text we can find for diagnostics. + + Tolerates the ``turns: [{response: "..."}]`` shape the gateway uses + today plus common message-list shapes. Truncated so the artifact + stays small. + """ + candidates: list[str] = [] + + def _walk(node: Any) -> None: + if isinstance(node, dict): + for key in ("response", "content", "text"): + value = node.get(key) + if isinstance(value, str) and value: + candidates.append(value) + for value in node.values(): + if not isinstance(value, str): + _walk(value) + elif isinstance(node, list): + for item in node: + _walk(item) + + _walk(history) + if not candidates: + return "" + return candidates[-1][:300] + + +def _history_text(history: dict[str, Any]) -> str: + chunks: list[str] = [] + + def _walk(node: Any) -> None: + if isinstance(node, str): + chunks.append(node) + elif isinstance(node, list): + for x in node: + _walk(x) + elif isinstance(node, dict): + for v in node.values(): + _walk(v) + + _walk(history) + return "\n".join(chunks) + + +async def _wait_for_install( + client: httpx.AsyncClient, base_url: str, name: str, deadline: float +) -> tuple[bool, dict[str, Any] | None]: + """Poll until the extension appears in /api/extensions or deadline expires. + + The /api/extensions response carries no boolean ``installed`` field — + presence in the list is itself the install confirmation. The runtime + state lives in ``authenticated`` / ``active`` / ``tools`` which can + legitimately stay false for a freshly-installed extension that + still needs OAuth (the natural end state for this probe — gmail + parks on an auth gate after install, which is exactly the flow we + want to verify). + + Returns (registered, last_extension_seen). + """ + last: dict[str, Any] | None = None + while time.perf_counter() < deadline: + ext = await _get_extension(client, base_url, name) + if ext is not None: + return True, ext + await asyncio.sleep(POLL_INTERVAL_S) + return False, last + + +async def run( + *, + stack: Any, + mock_telegram_url: str, + mock_sheets_url: str | None = None, + mock_calendar_url: str | None = None, + mock_hn_url: str | None = None, + mock_gmail_url: str | None = None, + mock_web_search_url: str | None = None, + output_dir: Path, + log_dir: Path, +) -> list[ProbeResult]: + started = time.perf_counter() + mode = "tool_install_chat" + base_url = stack.base_url + token = stack.gateway_token + + # Single client for the whole probe — the install-poll loop calls + # /api/extensions ~120 times across TIMEOUT_S at POLL_INTERVAL_S + # cadence. Reusing the client keeps HTTP keepalives warm and avoids + # the per-call TCP/TLS dance. + auth_headers = {"Authorization": f"Bearer {token}"} + try: + async with httpx.AsyncClient(headers=auth_headers) as client: + # Isolation: the workflow-canary runner shares one gateway + # stack across all probes, and `auth_recovery` (which runs + # immediately before this one) uses the same trigger + # prompt. If gmail is still registered from that run, the + # probe's "gmail registered after our chat send" check + # passes for the wrong reason. Remove gmail first so the + # subsequent install can only succeed via the chat-driven + # path we're trying to verify. + pre_existing = ( + await _get_extension(client, base_url, TARGET_EXTENSION) + is not None + ) + if pre_existing: + await _remove_extension_if_present( + client, base_url, TARGET_EXTENSION + ) + + thread_id = await _open_thread(client, base_url) + send_status = await _send_chat( + client, base_url, thread_id, TRIGGER_PROMPT + ) + if send_status != 202: + return [ + ProbeResult( + provider="extensions", + mode=mode, + success=False, + latency_ms=int((time.perf_counter() - started) * 1000), + details={ + "error": f"chat send returned {send_status}, expected 202", + "thread_id": thread_id, + "trigger_prompt": TRIGGER_PROMPT, + }, + ) + ] + + deadline = time.perf_counter() + TIMEOUT_S + registered, ext = await _wait_for_install( + client, base_url, TARGET_EXTENSION, deadline + ) + + history = await _read_history(client, base_url, thread_id) + text = _history_text(history) + # Target-bound check: assert a ``tool_install`` invocation + # exists in history *and* its argument payload binds the + # target extension. A bare "tool_install" presence check is + # not enough — a tool_install for a different extension + # followed by gmail appearing in /api/extensions for unrelated + # reasons (e.g. seeded by a prior probe) would falsely pass. + # See _history_has_tool_install_call_for for the recognised + # envelope shapes. + tool_install_seen = _history_has_tool_install_call_for( + history, TARGET_EXTENSION + ) + forbidden_hits = [frag for frag in FORBIDDEN_FRAGMENTS if frag in text] + + latency_ms = int((time.perf_counter() - started) * 1000) + success = ( + registered and tool_install_seen and not forbidden_hits + ) + + details: dict[str, Any] = { + "thread_id": thread_id, + "trigger_prompt": TRIGGER_PROMPT, + "target_extension": TARGET_EXTENSION, + "pre_existing_before_probe": pre_existing, + "extension_registered": registered, + "tool_install_seen_in_history": tool_install_seen, + "extension_state": ( + {k: ext.get(k) for k in ("authenticated", "active", "needs_setup")} + if ext is not None + else None + ), + "forbidden_fragments_seen": forbidden_hits, + "history_length_chars": len(text), + # Diagnostic surface — only meaningful on failure but cheap + # enough to always emit. The probe's primary failure mode + # ("agent didn't reach tool_install") has too many possible + # root causes (LLM-surface regression / approval gate parked + # / auth env not propagated / wrong engine version) to + # distinguish without seeing what the agent actually did. + "tool_calls_observed": _collect_tool_calls(history), + "pending_gate": (history.get("pending_gate") if isinstance(history, dict) else None), + "last_assistant_text": _last_assistant_text(history), + "agent_auto_approve_env": os.environ.get("AGENT_AUTO_APPROVE_TOOLS"), + "allow_local_tools_env": os.environ.get("ALLOW_LOCAL_TOOLS"), + } + if not success: + # Build a short, structured error string so the slack + # reason field surfaces the actual failure mode and not + # just "False". + reasons: list[str] = [] + if not registered: + reasons.append( + f"{TARGET_EXTENSION} did not appear in /api/extensions " + f"within {TIMEOUT_S:.0f}s — install never reached the " + "extension manager" + ) + if not tool_install_seen: + reasons.append( + "no tool_install invocation observed in history — " + "agent surface regression (tool_install hidden from " + "callable surface)" + ) + if forbidden_hits: + reasons.append(f"forbidden fragments: {forbidden_hits}") + details["error"] = "; ".join(reasons) + + return [ + ProbeResult( + provider="extensions", + mode=mode, + success=success, + latency_ms=latency_ms, + details=details, + ) + ] + except Exception as exc: # noqa: BLE001 + return [ + ProbeResult( + provider="extensions", + mode=mode, + success=False, + latency_ms=int((time.perf_counter() - started) * 1000), + details={"error": f"{type(exc).__name__}: {exc}"}, + ) + ] diff --git a/skills/linear/SKILL.md b/skills/linear/SKILL.md index 6d892681bd5..8039944817e 100644 --- a/skills/linear/SKILL.md +++ b/skills/linear/SKILL.md @@ -1,29 +1,39 @@ --- name: linear -version: "1.0.0" -description: Linear issue tracker API integration +version: "1.2.0" +description: Linear issue tracker API integration. Covers first-use identity bootstrap (viewer + teams cached), raw GraphQL for list/search/create/update, and the rules for handling "my issues" / "assigned to me" requests. activation: keywords: - "linear" - - "ticket" - - "sprint" - - "backlog" - - "roadmap" + - "my linear" + - "linear issue" + - "linear issues" + - "linear ticket" + - "linear tickets" + - "linear backlog" + - "linear assignments" + - "my linear issues" + - "my linear tickets" + - "assigned in linear" + - "linear.app" exclude_keywords: - "jira" - "asana" + - "github issue" patterns: - - "(?i)(create|list|show|assign|close|update)\\s.*(issue|ticket|task|bug)" - - "(?i)linear\\.app" + - "(?i)linear\\.(?:app|com)" + - "(?i)\\blinear\\b.+(issue|ticket|task|backlog|board)" + - "(?i)(create|show|list|close|update).+linear\\s+(issue|ticket)" tags: - "project-management" - "issue-tracking" - max_context_tokens: 2000 + max_context_tokens: 1600 credentials: - name: linear_api_key provider: linear location: - type: bearer + type: header + name: Authorization hosts: - "api.linear.app" setup_instructions: "Create an API key at https://linear.app/settings/api" @@ -31,7 +41,57 @@ credentials: # Linear API Skill -You have access to the Linear GraphQL API via the `http` tool. Credentials are automatically injected — **never construct Authorization headers manually**. When the URL host is `api.linear.app`, the system injects `Authorization: Bearer {linear_api_key}` transparently. +You have access to the Linear GraphQL API via the `http` tool. Credentials are automatically injected — **never construct Authorization headers manually**. When the URL host is `api.linear.app`, the system injects `Authorization: {linear_api_key}` transparently (no Bearer prefix — Linear API keys are sent raw). + +## Identity bootstrap (first use) + +Linear's API key does not tell you who the user IS inside Linear. Before running any "my issues" / "assigned to me" / "my tickets" request, make sure the user's Linear identity is cached. This avoids re-fetching `viewer` on every request and makes filter-by-assignee queries deterministic. + +### Cache file + +Path: `context/intel/linear-identity.md` + +Shape: + +```yaml +--- +type: linear-identity +bootstrapped_at: 2026-04-21 +refreshed_at: 2026-04-21 +stale_after: 2026-05-21 +--- +# Linear identity +user_id: 8a7f...-uuid +display_name: Tobias Holenstein +email: tobias@... +timezone: Europe/Zurich + +## Teams +- id: team-uuid-a, key: ENG, name: Engineering +- id: team-uuid-b, key: PROD, name: Product + +## Default team +ENG +``` + +### Bootstrap flow + +1. `memory_read("context/intel/linear-identity.md")`. If the file exists and `stale_after` is in the future, use it and stop. +2. If missing or stale, run one GraphQL call: + ``` + query { viewer { id name displayName email } teams(first: 50) { nodes { id key name } } } + ``` +3. Write the cache via `memory_write` with `stale_after` = today + 30 days. +4. If the returned team list has exactly one team, record it as `Default team`. If more than one, ask the user once: *"I see teams ENG, PROD, OPS. Which one do you default to for new issues?"* and store the answer. +5. On HTTP 401 or an `AuthenticationError` GraphQL error, invalidate the cache and re-prompt the user to check their API key — do not silently retry. + +### Using the cached identity + +- "list my issues" / "what's assigned to me" → filter by `assignee: { id: { eq: "<cached user_id>" } }`, **not** by `assignee: { isMe: true }` (the `isMe` filter is not universally available and `viewer` round-trips are wasteful). +- "create an issue in my team" → use cached `Default team` id without asking. +- "create an issue in <team name>" → match against cached team names; ask only if no match. +- Skills that import external work into Linear must consume this cache rather than re-resolving identity per run. + ## API Patterns @@ -45,6 +105,14 @@ All requests are `POST` with a JSON body containing `query` and optional `variab http(method="POST", url="https://api.linear.app/graphql", body={"query": "{ issues(first: 20, orderBy: updatedAt) { nodes { id identifier title state { name } assignee { name } priority priorityLabel createdAt } } }"}) ``` +### List Issues Assigned to the User (uses identity cache) + +``` +http(method="POST", url="https://api.linear.app/graphql", body={"query": "query($uid: ID!) { issues(filter: { assignee: { id: { eq: $uid } }, state: { type: { nin: [completed, canceled] } } }, first: 50, orderBy: updatedAt) { nodes { id identifier title state { name type } priority priorityLabel url updatedAt } } }", "variables": {"uid": "<cached user_id>"}}) +``` + +Never pass `viewer.id` inline from a fresh round-trip when the cache is valid — consult `context/intel/linear-identity.md`. + ### Get Issue by Identifier ``` diff --git a/src/NETWORK_SECURITY.md b/src/NETWORK_SECURITY.md index 68d5d4f8420..358a9d24982 100644 --- a/src/NETWORK_SECURITY.md +++ b/src/NETWORK_SECURITY.md @@ -272,7 +272,7 @@ Always binds to **loopback only**: `127.0.0.1:9876`. Falls back to `[::1]:9876` Both IPv4 and IPv6 loopback addresses are security-equivalent — they are only reachable from the local machine. -**Reference:** `src/auth/oauth.rs` / `src/llm/oauth_helpers.rs` — `OAUTH_CALLBACK_PORT`, `bind_callback_listener()` +**Reference:** `src/auth/oauth.rs` / `crates/ironclaw_oauth/src/lib.rs` — `OAUTH_CALLBACK_PORT`, `bind_callback_listener()` ### Lifecycle @@ -290,7 +290,7 @@ The listener is **ephemeral** — it is started only when an OAuth flow is initi - **Error parameter checking**: The handler checks for `error=` in the callback query string before extracting the auth code - **URL decoding**: Callback parameters are URL-decoded safely -**Reference:** `src/auth/oauth.rs` / `src/llm/oauth_helpers.rs` — `html_escape()` +**Reference:** `src/auth/oauth.rs` / `crates/ironclaw_oauth/src/lib.rs` — `html_escape()` ### Built-in OAuth Credentials @@ -302,7 +302,7 @@ Google OAuth client ID and secret are compiled into the binary (with compile-tim Implicit. The listener is a raw `TcpListener` (not axum) inside a `tokio::time::timeout` future. Once the authorization code or error is received, the future returns and the `TcpListener` is dropped, closing the port. No explicit shutdown signal is needed. -**Reference:** `src/auth/oauth.rs` / `src/llm/oauth_helpers.rs` — `wait_for_callback()` +**Reference:** `src/auth/oauth.rs` / `crates/ironclaw_oauth/src/lib.rs` — `wait_for_callback()` --- diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 03ccc4bbfe3..1c1f93388b8 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -7,8 +7,10 @@ //! - `commands` - System commands and job handlers //! - `thread_ops` - Thread/session operations (user input, undo, approval, persistence) +use std::collections::HashSet; use std::sync::Arc; +use base64::Engine as _; use futures::StreamExt; use uuid::Uuid; @@ -20,16 +22,19 @@ use crate::agent::session::ThreadState; use crate::agent::session_manager::SessionManager; use crate::agent::submission::{Submission, SubmissionParser, SubmissionResult}; use crate::agent::{HeartbeatConfig as AgentHeartbeatConfig, Router, Scheduler, SchedulerDeps}; -use crate::channels::{ChannelManager, IncomingMessage, OutgoingResponse, StatusUpdate}; +use crate::channels::{ + ChannelManager, IncomingMessage, OutgoingAttachment, OutgoingResponse, StatusUpdate, +}; use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig, SkillsConfig}; use crate::context::ContextManager; use crate::db::Database; use crate::error::{ChannelError, Error}; use crate::extensions::ExtensionManager; +use crate::generated_images::GeneratedImageSentinel; use crate::hooks::HookRegistry; -use crate::llm::LlmProvider; use crate::tools::ToolRegistry; use crate::workspace::Workspace; +use ironclaw_llm::LlmProvider; use ironclaw_safety::SafetyLayer; use ironclaw_skills::SkillRegistry; @@ -47,7 +52,7 @@ pub(crate) enum HandleOutcome { /// Shutdown signal (e.g. `/quit`). Run loop should break. Shutdown, /// Send this content via the channel, then emit terminal `Done`. - Respond(String), + Respond(OutgoingResponse), /// No response to send, but the turn is complete — emit `Done` only. NoResponse, /// Turn is paused (awaiting approval/auth/etc). Do not emit `Done`. @@ -64,7 +69,7 @@ impl HandleOutcome { None => HandleOutcome::Shutdown, Some(s) if s == BRIDGE_PENDING_SENTINEL => HandleOutcome::Pending, Some(s) if s.is_empty() => HandleOutcome::NoResponse, - Some(s) => HandleOutcome::Respond(s), + Some(s) => HandleOutcome::Respond(OutgoingResponse::text(s)), } } } @@ -72,7 +77,9 @@ impl HandleOutcome { impl From<crate::bridge::BridgeOutcome> for HandleOutcome { fn from(outcome: crate::bridge::BridgeOutcome) -> Self { match outcome { - crate::bridge::BridgeOutcome::Respond(s) => HandleOutcome::Respond(s), + crate::bridge::BridgeOutcome::Respond(s) => { + HandleOutcome::Respond(OutgoingResponse::text(s)) + } crate::bridge::BridgeOutcome::NoResponse => HandleOutcome::NoResponse, crate::bridge::BridgeOutcome::Pending => HandleOutcome::Pending, } @@ -148,6 +155,145 @@ fn is_single_message_repl(message: &IncomingMessage) -> bool { .unwrap_or(false) } +fn extension_for_image_media_type(media_type: &str) -> &'static str { + match media_type { + "image/jpeg" => "jpg", + "image/png" => "png", + "image/gif" => "gif", + "image/webp" => "webp", + _ => "bin", + } +} + +fn generated_image_attachment_from_data_url( + data_url: &str, + fallback_media_type: Option<&str>, + index: usize, +) -> Option<OutgoingAttachment> { + let (metadata, encoded) = data_url.split_once(',')?; + let header = metadata.strip_prefix("data:")?; + if !header + .split(';') + .any(|part| part.eq_ignore_ascii_case("base64")) + { + return None; + } + + let media_type = header + .split(';') + .next() + .filter(|value| value.starts_with("image/")) + .or(fallback_media_type) + .unwrap_or("image/png"); + if !media_type.starts_with("image/") { + return None; + } + + let data = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .ok()?; + if data.is_empty() { + return None; + } + + Some(OutgoingAttachment { + filename: format!( + "generated-image-{}.{}", + index + 1, + extension_for_image_media_type(media_type) + ), + mime_type: media_type.to_string(), + data, + }) +} + +fn generated_image_attachments_for_turn( + turn: &crate::agent::session::Turn, +) -> Vec<OutgoingAttachment> { + let mut seen = HashSet::new(); + let mut attachments = Vec::new(); + + for (index, tool_call) in turn.tool_calls.iter().enumerate() { + let Some(result) = tool_call.result.as_ref() else { + continue; + }; + let Some(sentinel) = GeneratedImageSentinel::from_value(result) else { + continue; + }; + let Some(data_url) = sentinel + .data_url() + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + continue; + }; + + if !seen.insert(data_url.to_string()) { + continue; + } + + match generated_image_attachment_from_data_url(data_url, sentinel.media_type(), index) { + Some(attachment) => attachments.push(attachment), + None => tracing::warn!("Generated image data URL could not be decoded for attachment"), + } + } + + attachments +} + +async fn build_outgoing_response_for_thread( + session: &Arc<tokio::sync::Mutex<crate::agent::session::Session>>, + thread_id: Uuid, + content: impl Into<String>, + attachment_paths: Vec<String>, +) -> OutgoingResponse { + let mut response = OutgoingResponse::text(content).with_attachments(attachment_paths); + let attachments = { + let sess = session.lock().await; + sess.threads + .get(&thread_id) + .and_then(|thread| thread.last_turn()) + .map(generated_image_attachments_for_turn) + .unwrap_or_default() + }; + + if response.attachments.is_empty() && !attachments.is_empty() { + response = response.with_inline_attachments(attachments); + } + + response +} + +async fn submission_response_to_handle_outcome( + session: &Arc<tokio::sync::Mutex<crate::agent::session::Session>>, + thread_id: Uuid, + content: String, + attachments: Vec<String>, +) -> HandleOutcome { + let has_attachments = !attachments.is_empty(); + + // Suppress silent replies only when there is truly nothing else to deliver. + // Image-only generated responses intentionally have empty text plus staged + // attachments, and must still reach the originating channel. + if ironclaw_llm::is_silent_reply(&content) { + if !has_attachments { + tracing::debug!("Suppressing silent reply token"); + return HandleOutcome::Shutdown; + } + return HandleOutcome::Respond( + build_outgoing_response_for_thread(session, thread_id, "", attachments).await, + ); + } + + if content.is_empty() && !has_attachments { + HandleOutcome::NoResponse + } else { + HandleOutcome::Respond( + build_outgoing_response_for_thread(session, thread_id, content, attachments).await, + ) + } +} + async fn resolve_channel_notification_user( extension_manager: Option<&Arc<ExtensionManager>>, channel: Option<&str>, @@ -228,9 +374,9 @@ pub struct AgentDeps { /// SSE manager for live job event streaming to the web gateway. pub sse_tx: Option<Arc<crate::channels::web::sse::SseManager>>, /// HTTP interceptor for trace recording/replay. - pub http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + pub http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, /// Audio transcription middleware for voice messages. - pub transcription: Option<Arc<crate::llm::transcription::TranscriptionMiddleware>>, + pub transcription: Option<Arc<ironclaw_llm::transcription::TranscriptionMiddleware>>, /// Document text extraction middleware for PDF, DOCX, PPTX, etc. pub document_extraction: Option<Arc<crate::document_extraction::DocumentExtractionMiddleware>>, /// Sandbox readiness state for full-job routine dispatch. @@ -384,7 +530,11 @@ impl Agent { message: &IncomingMessage, response: OutgoingResponse, ) -> Result<(), ChannelError> { + let staged_generated_attachments = response.attachments.clone(); let respond_result = self.channels.respond(message, response).await; + crate::generated_images::remove_staged_generated_image_attachments( + &staged_generated_attachments, + ); // Always emit Done regardless of whether respond succeeded, so the // client knows the turn is over even when the response delivery fails. if let Err(e) = self @@ -452,6 +602,16 @@ impl Agent { self.deps.workspace.as_ref() } + pub(crate) fn workspace_for_user(&self, user_id: &str) -> Option<Arc<Workspace>> { + self.workspace().map(|ws| { + if ws.user_id() == user_id { + Arc::clone(ws) + } else { + Arc::new(ws.scoped_to_user(user_id)) + } + }) + } + pub(crate) fn hooks(&self) -> &Arc<HookRegistry> { &self.deps.hooks } @@ -606,14 +766,7 @@ impl Agent { // unsatisfied (setup skills can still activate). Errors checking // a marker are logged and treated as unsatisfied. let mut satisfied: std::collections::HashSet<String> = std::collections::HashSet::new(); - if let Some(ws) = self.deps.workspace.as_ref() { - // Scope the workspace to the requesting user so multi-user - // channels check the correct user's marker state. - let scoped_ws = if ws.user_id() == user_id { - std::sync::Arc::clone(ws) - } else { - std::sync::Arc::new(ws.scoped_to_user(user_id)) - }; + if let Some(scoped_ws) = self.workspace_for_user(user_id) { for marker in &distinct_markers { match scoped_ws.exists(marker).await { Ok(true) => { @@ -1170,7 +1323,9 @@ impl Agent { // Apply transcription middleware to audio attachments let mut message = message; if let Some(ref transcription) = self.deps.transcription { - transcription.process(&mut message).await; + transcription + .process(&mut message.attachments, &mut message.content) + .await; } // Apply document extraction middleware to document attachments @@ -1187,12 +1342,15 @@ impl Agent { let event = crate::hooks::HookEvent::Outbound { user_id: message.user_id.clone(), channel: message.channel.clone(), - content: response.clone(), + content: response.content.clone(), thread_id: message.thread_id.as_ref().map(|t| t.as_str().to_string()), }; match self.hooks().run(&event).await { Err(err) => { tracing::warn!("BeforeOutbound hook blocked response: {}", err); + crate::generated_images::remove_staged_generated_image_attachments( + &response.attachments, + ); // Still send Done so the client knows the turn is complete // even though the response was suppressed by the hook. self.send_done(&message).await; @@ -1200,10 +1358,9 @@ impl Agent { Ok(crate::hooks::HookOutcome::Continue { modified: Some(new_content), }) => { - if let Err(e) = self - .respond_then_done(&message, OutgoingResponse::text(new_content)) - .await - { + let mut response = response; + response.content = new_content; + if let Err(e) = self.respond_then_done(&message, response).await { tracing::error!( channel = %message.channel, error = %e, @@ -1212,10 +1369,7 @@ impl Agent { } } _ => { - if let Err(e) = self - .respond_then_done(&message, OutgoingResponse::text(response)) - .await - { + if let Err(e) = self.respond_then_done(&message, response).await { tracing::error!( channel = %message.channel, error = %e, @@ -1226,8 +1380,6 @@ impl Agent { } } Ok(HandleOutcome::NoResponse) => { - // Empty response (e.g. routine consumed the message, silent reply). - // Send Done so the client knows the turn is complete. tracing::debug!( channel = %message.channel, user = %message.user_id, @@ -1236,10 +1388,6 @@ impl Agent { self.send_done(&message).await; } Ok(HandleOutcome::Pending) => { - // Turn paused awaiting user action (approval, auth, etc). - // Do NOT emit Done — the thread is not in a terminal state. - // The relevant ApprovalNeeded/AuthRequired status was already - // sent by the inner handler before returning. tracing::debug!( channel = %message.channel, user = %message.user_id, @@ -1247,7 +1395,6 @@ impl Agent { ); } Ok(HandleOutcome::Shutdown) => { - // Shutdown signal received (/quit, /exit, /shutdown) tracing::debug!("Shutdown command received, exiting..."); break; } @@ -1316,7 +1463,7 @@ impl Agent { /// Store extracted document text in workspace memory for future search/recall. async fn store_extracted_documents(&self, message: &IncomingMessage) { - let workspace = match self.workspace() { + let workspace = match self.workspace_for_user(&message.user_id) { Some(ws) => ws, None => return, }; @@ -1402,7 +1549,9 @@ impl Agent { channel = %message.channel, "Forwarding internal message" ); - return Ok(HandleOutcome::Respond(message.content.clone())); + return Ok(HandleOutcome::Respond(OutgoingResponse::text( + message.content.clone(), + ))); } // Set message tool context for this turn (current channel and target) @@ -1458,16 +1607,16 @@ impl Agent { }; match self.hooks().run(&event).await { Err(crate::hooks::HookError::Rejected { reason }) => { - return Ok(HandleOutcome::Respond(format!( + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( "[Message rejected: {}]", reason - ))); + )))); } Err(err) => { - return Ok(HandleOutcome::Respond(format!( + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( "[Message blocked by hook policy: {}]", err - ))); + )))); } Ok(crate::hooks::HookOutcome::Continue { modified: Some(new_content), @@ -1520,10 +1669,18 @@ impl Agent { .await .map(HandleOutcome::from); } - Submission::ExternalCallback { request_id } => { - return crate::bridge::handle_external_callback(self, message, *request_id) - .await - .map(HandleOutcome::from); + Submission::ExternalCallback { + request_id, + payload, + } => { + return crate::bridge::handle_external_callback( + self, + message, + *request_id, + payload.clone(), + ) + .await + .map(HandleOutcome::from); } Submission::GateAuthResolution { request_id, @@ -1558,6 +1715,11 @@ impl Agent { .await .map(HandleOutcome::from); } + Submission::PairingClaim { channel, code } => { + return crate::bridge::handle_pairing_claim(self, message, channel, code) + .await + .map(HandleOutcome::from); + } // Undo/Redo/Resume/SwitchThread: v1-only (engine has no undo; // thread switching is implicit via ConversationManager). // Compact/Summarize/Suggest: orthogonal to engine (compaction is internal). @@ -1573,14 +1735,14 @@ impl Agent { if !self.config.engine_v2 { match submission { Submission::ExternalCallback { .. } => { - return Ok(HandleOutcome::Respond( + return Ok(HandleOutcome::Respond(OutgoingResponse::text( "Error: External callbacks require ENGINE_V2".to_string(), - )); + ))); } Submission::GateAuthResolution { .. } => { - return Ok(HandleOutcome::Respond( + return Ok(HandleOutcome::Respond(OutgoingResponse::text( "Error: Auth gate resolution requires ENGINE_V2".to_string(), - )); + ))); } _ => {} } @@ -1594,7 +1756,10 @@ impl Agent { "Hydrating thread from DB" ); if let Some(rejection) = self.maybe_hydrate_thread(message, external_thread_id).await { - return Ok(HandleOutcome::Respond(format!("Error: {}", rejection))); + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( + "Error: {}", + rejection + )))); } } @@ -1636,9 +1801,9 @@ impl Agent { "Blocked approval for thread with no pending approval" ); drop(sess); - return Ok(HandleOutcome::Respond( - "Error: no pending approval on this thread".into(), - )); + return Ok(HandleOutcome::Respond(OutgoingResponse::text( + "Error: no pending approval on this thread", + ))); } // ApprovalResponse (bare "yes"/"no"/"always") without a // pending approval: fall through to normal handling so the @@ -1659,9 +1824,9 @@ impl Agent { "Blocked cross-channel approval attempt" ); drop(sess); - return Ok(HandleOutcome::Respond( - "Error: approval not authorized for this channel".into(), - )); + return Ok(HandleOutcome::Respond(OutgoingResponse::text( + "Error: approval not authorized for this channel", + ))); } } sess.active_thread = Some(target_thread_id); @@ -1728,10 +1893,10 @@ impl Agent { // If this was a user message (possibly a pasted token), return an // explicit error instead of forwarding it to the LLM/history. if matches!(submission, Submission::UserInput { .. }) { - return Ok(HandleOutcome::Respond(format!( + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( "Authentication for **{}** expired. Please try again.", pending.extension_name - ))); + )))); } // Control submissions (interrupt, undo, etc.) fall through to normal handling } else { @@ -1822,7 +1987,11 @@ impl Agent { // - `Error`: soft error — draining more messages after an error // would produce confusing interleaved output // - `Err(_)`: hard error - while let Ok(SubmissionResult::Response { content: outgoing }) = &result { + while let Ok(SubmissionResult::Response { + content: outgoing, + attachments, + }) = &result + { let merged = { let mut sess = session.lock().await; sess.threads @@ -1852,10 +2021,14 @@ impl Agent { // identity will attribute every response to the first // message. This is acceptable for the current // single-user-per-thread model. - if let Err(e) = self - .respond_then_done(message, OutgoingResponse::text(outgoing.clone())) - .await - { + let response = build_outgoing_response_for_thread( + &session, + thread_id, + outgoing.clone(), + attachments.clone(), + ) + .await; + if let Err(e) = self.respond_then_done(message, response).await { tracing::warn!( thread_id = %thread_id, "Failed to send intermediate drain-loop response: {e}" @@ -1908,13 +2081,22 @@ impl Agent { .handle_reasoning_command(&args, &session, thread_id) .await; return match result { - SubmissionResult::Response { content } => { - Ok(HandleOutcome::Respond(content)) - } + SubmissionResult::Response { + content, + attachments, + } => Ok(HandleOutcome::Respond( + build_outgoing_response_for_thread( + &session, + thread_id, + content, + attachments, + ) + .await, + )), SubmissionResult::Ok { message } => Ok(HandleOutcome::from_legacy(message)), - SubmissionResult::Error { message } => { - Ok(HandleOutcome::Respond(format!("Error: {}", message))) - } + SubmissionResult::Error { message } => Ok(HandleOutcome::Respond( + OutgoingResponse::text(format!("Error: {}", message)), + )), _ => { if is_single_message_repl(message) { Ok(HandleOutcome::Shutdown) @@ -1928,17 +2110,17 @@ impl Agent { self.handle_system_command(&command, &args, &message.channel, &tenant) .await } - Submission::Undo => self.process_undo(session, thread_id).await, - Submission::Redo => self.process_redo(session, thread_id).await, - Submission::Interrupt => self.process_interrupt(session, thread_id).await, - Submission::Compact => self.process_compact(session, thread_id).await, - Submission::Clear => self.process_clear(session, thread_id).await, + Submission::Undo => self.process_undo(session.clone(), thread_id).await, + Submission::Redo => self.process_redo(session.clone(), thread_id).await, + Submission::Interrupt => self.process_interrupt(session.clone(), thread_id).await, + Submission::Compact => self.process_compact(session.clone(), thread_id).await, + Submission::Clear => self.process_clear(session.clone(), thread_id).await, Submission::NewThread => self.process_new_thread(message).await, - Submission::Heartbeat => self.process_heartbeat().await, - Submission::Summarize => self.process_summarize(session, thread_id).await, - Submission::Suggest => self.process_suggest(session, thread_id).await, + Submission::Heartbeat => self.process_heartbeat(&message.user_id).await, + Submission::Summarize => self.process_summarize(session.clone(), thread_id).await, + Submission::Suggest => self.process_suggest(session.clone(), thread_id).await, Submission::Expected { description } => { - self.process_expected(session, thread_id, &description, &message.user_id) + self.process_expected(session.clone(), thread_id, &description, &message.user_id) .await } Submission::JobStatus { job_id } => { @@ -1950,9 +2132,10 @@ impl Agent { self.process_switch_thread(message, target).await } Submission::Resume { checkpoint_id } => { - self.process_resume(session, thread_id, checkpoint_id).await + self.process_resume(session.clone(), thread_id, checkpoint_id) + .await } - Submission::ListThreads => self.process_list_threads(session, message).await, + Submission::ListThreads => self.process_list_threads(session.clone(), message).await, Submission::ExecApproval { request_id, approved, @@ -1960,7 +2143,7 @@ impl Agent { } => { self.process_approval( message, - session, + session.clone(), thread_id, Some(request_id), approved, @@ -1985,8 +2168,15 @@ impl Agent { // NOTE: TOCTOU possible — state could change between check // and process_approval; process_approval handles stale cases. if should_route_as_approval(thread_state, &message.content) { - self.process_approval(message, session, thread_id, None, approved, always) - .await + self.process_approval( + message, + session.clone(), + thread_id, + None, + approved, + always, + ) + .await } else { // Run BeforeInbound hooks for the downgraded content — // the hook check above only fires for UserInput submissions, @@ -2001,15 +2191,15 @@ impl Agent { let content = match self.hooks().run(&hook_event).await { Err(crate::hooks::HookError::Rejected { reason }) => { // Match the main UserInput path's rejection behavior. - return Ok(HandleOutcome::Respond(format!( + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( "[Message rejected: {reason}]" - ))); + )))); } Err(err) => { // Match the main UserInput path's error behavior. - return Ok(HandleOutcome::Respond(format!( + return Ok(HandleOutcome::Respond(OutgoingResponse::text(format!( "[Message blocked by hook policy: {err}]" - ))); + )))); } Ok(crate::hooks::HookOutcome::Continue { modified: Some(new_content), @@ -2030,7 +2220,11 @@ impl Agent { ) .await; - while let Ok(SubmissionResult::Response { content: outgoing }) = &result { + while let Ok(SubmissionResult::Response { + content: outgoing, + attachments, + }) = &result + { let merged = { let mut sess = session.lock().await; sess.threads @@ -2041,10 +2235,14 @@ impl Agent { break; }; - if let Err(e) = self - .respond_then_done(message, OutgoingResponse::text(outgoing.clone())) - .await - { + let response = build_outgoing_response_for_thread( + &session, + thread_id, + outgoing.clone(), + attachments.clone(), + ) + .await; + if let Err(e) = self.respond_then_done(message, response).await { tracing::warn!( %thread_id, "Failed to send intermediate drain-loop response: {e}" @@ -2074,6 +2272,27 @@ impl Agent { result } } + Submission::PairingClaim { channel, code } => { + // Pairing approval is independent of engine_v2 — it only + // touches the pairing store and the extension manager. + // Reuse the bridge handler so v1 and v2 surfaces behave + // identically (#3317). + match crate::bridge::handle_pairing_claim(self, message, &channel, &code).await { + Ok(crate::bridge::BridgeOutcome::Respond(text)) => { + Ok(SubmissionResult::Response { + content: text, + attachments: Vec::new(), + }) + } + Ok(crate::bridge::BridgeOutcome::NoResponse) + | Ok(crate::bridge::BridgeOutcome::Pending) => { + Ok(SubmissionResult::Ok { message: None }) + } + Err(e) => Ok(SubmissionResult::Error { + message: format!("Pairing approval failed: {e}"), + }), + } + } Submission::Plan { sub } => { use crate::agent::submission::PlanSubcommand; let rewritten = match sub { @@ -2106,25 +2325,23 @@ impl Agent { .to_string() } }; - self.process_user_input(message, tenant, session, thread_id, &rewritten) + self.process_user_input(message, tenant, session.clone(), thread_id, &rewritten) .await } }; // Convert SubmissionResult to a HandleOutcome. match result? { - SubmissionResult::Response { content } => { - // Suppress silent replies (e.g. from group chat "nothing to say" responses). - // Silent replies exit single-message REPL invocations. - if crate::llm::is_silent_reply(&content) { - tracing::debug!("Suppressing silent reply token"); - Ok(HandleOutcome::Shutdown) - } else if content.is_empty() { - Ok(HandleOutcome::NoResponse) - } else { - Ok(HandleOutcome::Respond(content)) - } - } + SubmissionResult::Response { + content, + attachments, + } => Ok(submission_response_to_handle_outcome( + &session, + thread_id, + content, + attachments, + ) + .await), SubmissionResult::Ok { message: output_message, } => { @@ -2145,10 +2362,12 @@ impl Agent { Ok(HandleOutcome::from_legacy(output_message)) } } - SubmissionResult::Error { message } => { - Ok(HandleOutcome::Respond(format!("Error: {}", message))) - } - SubmissionResult::Interrupted => Ok(HandleOutcome::Respond("Interrupted.".into())), + SubmissionResult::Error { message } => Ok(HandleOutcome::Respond( + OutgoingResponse::text(format!("Error: {}", message)), + )), + SubmissionResult::Interrupted => Ok(HandleOutcome::Respond(OutgoingResponse::text( + "Interrupted.", + ))), SubmissionResult::AuthPending => { // Auth-required status already sent by handle_auth_intercept. // Thread is in auth mode — suppress text response and Done. @@ -2175,16 +2394,14 @@ mod tests { use crate::agent::agent_loop::{Agent, AgentDeps, HandleOutcome}; use crate::agent::cost_guard::{CostGuard, CostGuardConfig}; use crate::agent::submission::{AuthGateResolution, Submission}; - use crate::channels::IncomingMessage; + use crate::channels::{AttachmentKind, IncomingAttachment, IncomingMessage}; + use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; use crate::error::ChannelError; use crate::hooks::HookRegistry; use crate::tools::ToolRegistry; - use crate::{ - config::{AgentConfig, SafetyConfig, SkillsConfig}, - llm::{ - CompletionRequest, CompletionResponse, FinishReason, LlmProvider, - ToolCompletionRequest, ToolCompletionResponse, - }, + use ironclaw_llm::{ + CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, }; use ironclaw_safety::SafetyLayer; use rust_decimal::Decimal; @@ -2230,6 +2447,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -2298,6 +2516,65 @@ mod tests { ) } + #[cfg(feature = "libsql")] + #[tokio::test] + async fn store_extracted_documents_writes_to_message_user_workspace() { + let (db, _dir) = crate::agent::test_support::make_libsql_test_db().await; + let owner_workspace = Arc::new(crate::workspace::Workspace::new_with_db( + "owner-scope", + Arc::clone(&db), + )); + let mut agent = make_legacy_handle_message_test_agent(); + agent.deps.store = Some(Arc::clone(&db)); + agent.deps.workspace = Some(owner_workspace); + + let message = IncomingMessage::new("gateway", "alice", "uploaded a document") + .with_attachments(vec![IncomingAttachment { + id: "doc-1".to_string(), + kind: AttachmentKind::Document, + mime_type: "text/plain".to_string(), + filename: Some("conversation-notes.txt".to_string()), + size_bytes: Some(42), + source_url: None, + storage_key: None, + local_path: None, + extracted_text: Some("alice-only extracted conversation text".to_string()), + data: Vec::new(), + duration_secs: None, + }]); + + let before_date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + agent.store_extracted_documents(&message).await; + let after_date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + + let mut candidate_paths = vec![format!("documents/{before_date}/conversation-notes.txt")]; + if after_date != before_date { + candidate_paths.push(format!("documents/{after_date}/conversation-notes.txt")); + } + + let alice_ws = crate::workspace::Workspace::new_with_db("alice", Arc::clone(&db)); + let mut stored = None; + for path in &candidate_paths { + if let Ok(doc) = alice_ws.read(path).await { + stored = Some((path.clone(), doc)); + break; + } + } + let (path, alice_doc) = + stored.expect("extracted document should be stored under the message user"); + assert!( + alice_doc + .content + .contains("alice-only extracted conversation text") + ); + + let owner_ws = crate::workspace::Workspace::new_with_db("owner-scope", Arc::clone(&db)); + assert!( + owner_ws.read(&path).await.is_err(), + "extracted document must not be stored under the startup owner scope" + ); + } + #[test] fn test_truncate_short_input() { assert_eq!(truncate_for_preview("hello", 10), "hello"); @@ -2592,6 +2869,73 @@ mod tests { assert!(!is_single_message_repl(&plain_repl)); // safety: test-only assertion } + #[tokio::test] + async fn build_outgoing_response_for_thread_includes_generated_image_inline_attachments() { + use super::build_outgoing_response_for_thread; + use crate::agent::session::Session; + use std::sync::Arc; + + let session: Arc<tokio::sync::Mutex<Session>> = + Arc::new(tokio::sync::Mutex::new(Session::new("user-123"))); + + let thread_id = { + let mut sess = session.lock().await; + let thread = sess.create_thread(None); + let thread_id = thread.id; + let turn = thread.start_turn("draw a cat"); + turn.record_tool_call("image_generate", serde_json::json!({ "prompt": "cat" })); + turn.record_tool_result(serde_json::json!({ + "type": "image_generated", + "data": "data:image/png;base64,cG5nLWJ5dGVz", + "media_type": "image/png", + })); + thread_id + }; + + let response = + build_outgoing_response_for_thread(&session, thread_id, "done", Vec::new()).await; + + assert_eq!(response.content, "done"); + assert!(response.attachments.is_empty()); + assert_eq!(response.inline_attachments.len(), 1); + assert_eq!( + response.inline_attachments[0].filename, + "generated-image-1.png" + ); + assert_eq!(response.inline_attachments[0].mime_type, "image/png"); + assert_eq!(response.inline_attachments[0].data, b"png-bytes"); + } + + #[tokio::test] + async fn empty_submission_response_with_attachments_is_delivered() { + use super::submission_response_to_handle_outcome; + use crate::agent::session::Session; + use std::sync::Arc; + + let session: Arc<tokio::sync::Mutex<Session>> = + Arc::new(tokio::sync::Mutex::new(Session::new("user-123"))); + let thread_id = { + let mut sess = session.lock().await; + sess.create_thread(None).id + }; + + let outcome = submission_response_to_handle_outcome( + &session, + thread_id, + String::new(), + vec!["/tmp/generated-image.png".to_string()], + ) + .await; + + match outcome { + HandleOutcome::Respond(response) => { + assert!(response.content.is_empty()); + assert_eq!(response.attachments, vec!["/tmp/generated-image.png"]); + } + other => panic!("expected attachment response, got {other:?}"), + } + } + #[tokio::test] async fn v2_only_structured_submissions_do_not_switch_threads_when_engine_v2_disabled() { let agent = make_legacy_handle_message_test_agent(); @@ -2629,7 +2973,7 @@ mod tests { .expect("handle message"); assert!(matches!( outcome, - HandleOutcome::Respond(ref msg) if msg == "Error: Auth gate resolution requires ENGINE_V2" + HandleOutcome::Respond(ref msg) if msg.content == "Error: Auth gate resolution requires ENGINE_V2" )); { let sess = session.lock().await; @@ -2638,6 +2982,7 @@ mod tests { let callback = serde_json::to_string(&Submission::ExternalCallback { request_id: Uuid::new_v4(), + payload: None, }) .expect("serialize external callback"); let callback_message = @@ -2649,7 +2994,7 @@ mod tests { .expect("handle callback"); assert!(matches!( outcome, - HandleOutcome::Respond(ref msg) if msg == "Error: External callbacks require ENGINE_V2" + HandleOutcome::Respond(ref msg) if msg.content == "Error: External callbacks require ENGINE_V2" )); { let sess = session.lock().await; diff --git a/src/agent/agentic_loop.rs b/src/agent/agentic_loop.rs index c2b1022e207..625ad493db5 100644 --- a/src/agent/agentic_loop.rs +++ b/src/agent/agentic_loop.rs @@ -12,7 +12,7 @@ use std::hash::{Hash, Hasher}; use crate::agent::session::PendingApproval; use crate::error::Error; -use crate::llm::{ +use ironclaw_llm::{ ChatMessage, FinishReason, Reasoning, ReasoningContext, RespondResult, ResponseMetadata, ToolCall, }; @@ -105,7 +105,7 @@ pub trait LoopDelegate: Send + Sync { reasoning: &Reasoning, reason_ctx: &mut ReasoningContext, iteration: usize, - ) -> Result<crate::llm::RespondOutput, Error>; + ) -> Result<ironclaw_llm::RespondOutput, Error>; /// Handle a text-only response from the LLM. /// Return `TextAction::Return` to exit the loop, `TextAction::Continue` to proceed. @@ -124,9 +124,10 @@ pub trait LoopDelegate: Send + Sync { /// duplicate tool call detector to escalate repeated identical failures. async fn execute_tool_calls( &self, - tool_calls: Vec<crate::llm::ToolCall>, + tool_calls: Vec<ironclaw_llm::ToolCall>, content: Option<String>, reason_ctx: &mut ReasoningContext, + reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, Error>; /// Called when the LLM expresses tool intent without actually calling a tool. @@ -253,6 +254,7 @@ pub async fn run_agentic_loop( RespondResult::ToolCalls { tool_calls, content, + reasoning: _, } => { let names: Vec<&str> = tool_calls.iter().map(|tc| tc.name.as_str()).collect(); tracing::debug!( @@ -272,7 +274,7 @@ pub async fn run_agentic_loop( && !reason_ctx.available_tools.is_empty() && !reason_ctx.force_text && consecutive_tool_intent_nudges < config.max_tool_intent_nudges - && crate::llm::llm_signals_tool_intent(&text) + && ironclaw_llm::llm_signals_tool_intent(&text) { consecutive_tool_intent_nudges += 1; tracing::info!( @@ -283,13 +285,13 @@ pub async fn run_agentic_loop( reason_ctx.messages.push(ChatMessage::assistant(&text)); reason_ctx .messages - .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + .push(ChatMessage::user(ironclaw_llm::TOOL_INTENT_NUDGE)); delegate.after_iteration(iteration).await; continue; } // Reset nudge counter since we got a non-intent text response - if !crate::llm::llm_signals_tool_intent(&text) { + if !ironclaw_llm::llm_signals_tool_intent(&text) { consecutive_tool_intent_nudges = 0; } @@ -307,6 +309,7 @@ pub async fn run_agentic_loop( RespondResult::ToolCalls { tool_calls, content, + reasoning, } => { // If the response was truncated, tool call parameters are likely // incomplete. Discard them and tell the LLM to try a different @@ -325,7 +328,7 @@ pub async fn run_agentic_loop( } reason_ctx .messages - .push(ChatMessage::user(crate::llm::TRUNCATED_TOOL_CALL_NOTICE)); + .push(ChatMessage::user(ironclaw_llm::TRUNCATED_TOOL_CALL_NOTICE)); // After repeated truncations, force text-only mode so the LLM // stops attempting tool calls it can't fit in the output budget. if truncation_count >= 3 { @@ -345,7 +348,7 @@ pub async fn run_agentic_loop( reason_ctx.last_tool_batch_all_failed = false; if let Some(outcome) = delegate - .execute_tool_calls(tool_calls, content, reason_ctx) + .execute_tool_calls(tool_calls, content, reason_ctx, reasoning) .await? { return Ok(outcome); @@ -401,8 +404,8 @@ pub fn truncate_for_preview(s: &str, max: usize) -> Cow<'_, str> { #[cfg(test)] mod tests { use super::*; - use crate::llm::{RespondOutput, ResponseAnomaly, ResponseMetadata, TokenUsage, ToolCall}; use crate::testing::StubLlm; + use ironclaw_llm::{RespondOutput, ResponseAnomaly, ResponseMetadata, TokenUsage, ToolCall}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::Mutex; @@ -434,6 +437,7 @@ mod tests { result: RespondResult::ToolCalls { tool_calls: calls, content: None, + reasoning: None, }, usage: zero_usage(), finish_reason: FinishReason::ToolUse, @@ -448,6 +452,7 @@ mod tests { tool_exec_count: AtomicUsize, tool_exec_outcome: Mutex<Option<LoopOutcome>>, iterations_seen: Mutex<Vec<usize>>, + call_llm_iterations: Mutex<Vec<usize>>, early_exit: Mutex<Option<(usize, LoopOutcome)>>, nudge_count: AtomicUsize, /// When true, execute_tool_calls sets last_tool_batch_all_failed = true. @@ -462,6 +467,7 @@ mod tests { tool_exec_count: AtomicUsize::new(0), tool_exec_outcome: Mutex::new(None), iterations_seen: Mutex::new(Vec::new()), + call_llm_iterations: Mutex::new(Vec::new()), early_exit: Mutex::new(None), nudge_count: AtomicUsize::new(0), simulate_all_failed: false, @@ -506,8 +512,9 @@ mod tests { &self, _reasoning: &Reasoning, _reason_ctx: &mut ReasoningContext, - _iteration: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + iteration: usize, + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { + self.call_llm_iterations.lock().await.push(iteration); let mut responses = self.llm_responses.lock().await; if responses.is_empty() { panic!("MockDelegate: no more LLM responses queued"); @@ -529,6 +536,7 @@ mod tests { _tool_calls: Vec<ToolCall>, _content: Option<String>, reason_ctx: &mut ReasoningContext, + _reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { self.tool_exec_count.fetch_add(1, Ordering::SeqCst); reason_ctx @@ -550,6 +558,41 @@ mod tests { // --- Tests --- + #[tokio::test] + async fn first_call_llm_iteration_is_one() { + // Regression: ChatDelegate::call_llm gates per-request overrides + // (Responses API `temperature`, settings `temperature`, `selected_model` + // from `/model`) on the first iteration. The gate's expected value must + // match what this loop produces on its first call_llm invocation. An + // off-by-one here silently disables every override and was shipped once + // already (gate was `iteration == 0` while the loop starts at 1). + let tool_call = ToolCall { + id: "call_1".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({}), + reasoning: None, + signature: None, + }; + let delegate = MockDelegate::new(vec![ + tool_calls_output(vec![tool_call]), + text_output("done"), + ]); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + let iterations = delegate.call_llm_iterations.lock().await; + assert_eq!( + iterations.first().copied(), + Some(1), + "first call_llm iteration must be 1 — see dispatcher.rs override gate" + ); + } + #[tokio::test] async fn test_text_response_returns_immediately() { let delegate = MockDelegate::new(vec![text_output("Hello, world!")]); @@ -577,6 +620,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let delegate = MockDelegate::new(vec![ tool_calls_output(vec![tool_call]), @@ -630,9 +674,9 @@ mod tests { assert!(matches!(outcome, LoopOutcome::Response(_))); assert!( - ctx.messages - .iter() - .any(|m| m.role == crate::llm::Role::User && m.content.contains("injected prompt")), + ctx.messages.iter().any( + |m| m.role == ironclaw_llm::Role::User && m.content.contains("injected prompt") + ), "Injected message should appear in context" ); } @@ -660,7 +704,7 @@ mod tests { _: &Reasoning, _: &mut ReasoningContext, _: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { Ok(RespondOutput { result: RespondResult::Text("fallback".to_string()), usage: zero_usage(), @@ -688,6 +732,7 @@ mod tests { _: Vec<ToolCall>, _: Option<String>, _: &mut ReasoningContext, + _: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { Ok(None) } @@ -731,7 +776,7 @@ mod tests { _: &Reasoning, _: &mut ReasoningContext, _: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { Ok(text_output("still working")) } async fn handle_text_response( @@ -748,6 +793,7 @@ mod tests { _: Vec<ToolCall>, _: Option<String>, _: &mut ReasoningContext, + _: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { Ok(None) } @@ -769,7 +815,7 @@ mod tests { let assistant_count = ctx .messages .iter() - .filter(|m| m.role == crate::llm::Role::Assistant) + .filter(|m| m.role == ironclaw_llm::Role::Assistant) .count(); assert_eq!(assistant_count, 3); } @@ -783,7 +829,7 @@ mod tests { ]); let reasoning = stub_reasoning(); let mut ctx = ReasoningContext::new(); - ctx.available_tools.push(crate::llm::ToolDefinition { + ctx.available_tools.push(ironclaw_llm::ToolDefinition { name: "search".to_string(), description: "Search files".to_string(), parameters: serde_json::json!({"type": "object"}), @@ -804,7 +850,7 @@ mod tests { .messages .iter() .filter(|m| { - m.role == crate::llm::Role::User + m.role == ironclaw_llm::Role::User && m.content.contains("you did not include any tool calls") }) .count(); @@ -866,11 +912,13 @@ mod tests { name: "memory_write".to_string(), arguments: serde_json::json!({}), // empty — truncated reasoning: None, + signature: None, }; let truncated_output = RespondOutput { result: RespondResult::ToolCalls { tool_calls: vec![truncated_tool_call], content: Some("I'll write the report.".to_string()), + reasoning: None, }, usage: zero_usage(), finish_reason: FinishReason::Length, // response was truncated @@ -896,14 +944,14 @@ mod tests { assert!( ctx.messages .iter() - .any(|m| m.role == crate::llm::Role::User && m.content.contains("truncated")), + .any(|m| m.role == ironclaw_llm::Role::User && m.content.contains("truncated")), "Should inject truncation notice into context" ); // The partial assistant content should have been preserved assert!( ctx.messages .iter() - .any(|m| m.role == crate::llm::Role::Assistant + .any(|m| m.role == ironclaw_llm::Role::Assistant && m.content.contains("write the report")), "Should preserve partial assistant content" ); @@ -918,8 +966,10 @@ mod tests { name: "memory_write".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], content: None, + reasoning: None, }, usage: zero_usage(), finish_reason: FinishReason::Length, @@ -962,6 +1012,7 @@ mod tests { name: "echo".into(), arguments: serde_json::json!({"msg": "hi"}), reasoning: None, + signature: None, }]; let fp = DuplicateToolCallTracker::fingerprint(&calls); // Tool succeeded — count stays at 0 @@ -977,6 +1028,7 @@ mod tests { name: "http_get".into(), arguments: serde_json::json!({"url": "https://example.com"}), reasoning: None, + signature: None, }]; let fp = DuplicateToolCallTracker::fingerprint(&calls); assert_eq!(tracker.record_with_fingerprint(fp, true), 1); @@ -992,6 +1044,7 @@ mod tests { name: "http_get".into(), arguments: serde_json::json!({"url": "https://example.com"}), reasoning: None, + signature: None, }]; let fp = DuplicateToolCallTracker::fingerprint(&calls); assert_eq!(tracker.record_with_fingerprint(fp, true), 1); @@ -1010,12 +1063,14 @@ mod tests { name: "http_get".into(), arguments: serde_json::json!({"url": "https://a.com"}), reasoning: None, + signature: None, }]; let calls_b = vec![ToolCall { id: "c1".into(), name: "http_get".into(), arguments: serde_json::json!({"url": "https://b.com"}), reasoning: None, + signature: None, }]; let fp_a = DuplicateToolCallTracker::fingerprint(&calls_a); let fp_b = DuplicateToolCallTracker::fingerprint(&calls_b); @@ -1033,12 +1088,14 @@ mod tests { name: "echo".into(), arguments: serde_json::json!({"a": 1, "b": 2}), reasoning: None, + signature: None, }]; let calls_b = vec![ToolCall { id: "c1".into(), name: "echo".into(), arguments: serde_json::json!({"b": 2, "a": 1}), reasoning: None, + signature: None, }]; assert_eq!( DuplicateToolCallTracker::fingerprint(&calls_a), @@ -1055,6 +1112,7 @@ mod tests { name: "http_get".to_string(), arguments: serde_json::json!({"url": "https://broken.example.com"}), reasoning: None, + signature: None, }; // 3 identical failing tool calls, then text response let mut delegate = MockDelegate::new(vec![ @@ -1082,7 +1140,7 @@ mod tests { .messages .iter() .filter(|m| { - m.role == crate::llm::Role::User && m.content.contains("same failing tool call") + m.role == ironclaw_llm::Role::User && m.content.contains("same failing tool call") }) .count(); assert!( @@ -1103,6 +1161,7 @@ mod tests { name: "http_get".to_string(), arguments: serde_json::json!({"url": "https://broken.example.com"}), reasoning: None, + signature: None, }; // 5 identical failing tool calls, then text response let mut delegate = MockDelegate::new(vec![ @@ -1140,6 +1199,7 @@ mod tests { name: "http_get".to_string(), arguments: serde_json::json!({"url": "https://broken.example.com"}), reasoning: None, + signature: None, }; // 2 failing calls, then a text continuation, then 2 more of the same failing calls // The text response in the middle should reset the streak, so we never hit 3. @@ -1166,7 +1226,7 @@ mod tests { _: &Reasoning, _: &mut ReasoningContext, _: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { let mut responses = self.llm_responses.lock().await; if responses.is_empty() { panic!("No more responses"); @@ -1193,6 +1253,7 @@ mod tests { _: Vec<ToolCall>, _: Option<String>, reason_ctx: &mut ReasoningContext, + _reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { self.tool_exec_count.fetch_add(1, Ordering::SeqCst); reason_ctx.messages.push(ChatMessage::user("tool error")); @@ -1230,7 +1291,7 @@ mod tests { .messages .iter() .filter(|m| { - m.role == crate::llm::Role::User && m.content.contains("same failing tool call") + m.role == ironclaw_llm::Role::User && m.content.contains("same failing tool call") }) .count(); assert_eq!( diff --git a/src/agent/attachments.rs b/src/agent/attachments.rs index 5c85e6a2a51..47291d84a6b 100644 --- a/src/agent/attachments.rs +++ b/src/agent/attachments.rs @@ -3,7 +3,7 @@ use base64::Engine; use crate::channels::{AttachmentKind, IncomingAttachment}; -use crate::llm::{ContentPart, ImageUrl}; +use ironclaw_llm::{ContentPart, ImageUrl}; /// Result of processing attachments for the LLM pipeline. pub struct AugmentResult { diff --git a/src/agent/commands.rs b/src/agent/commands.rs index d1219587935..8d3fd23ed42 100644 --- a/src/agent/commands.rs +++ b/src/agent/commands.rs @@ -14,8 +14,8 @@ use crate::agent::{Agent, MessageIntent}; use crate::channels::{IncomingMessage, StatusUpdate}; use crate::context::JobState; use crate::error::Error; -use crate::llm::{ChatMessage, Reasoning}; use crate::ownership::Owned; +use ironclaw_llm::{ChatMessage, Reasoning}; /// Format a count with a suffix, using K/M abbreviations for large numbers. fn format_count(n: u64, suffix: &str) -> String { @@ -353,8 +353,8 @@ impl Agent { } /// Trigger a manual heartbeat check. - pub(super) async fn process_heartbeat(&self) -> Result<SubmissionResult, Error> { - let Some(workspace) = self.workspace() else { + pub(super) async fn process_heartbeat(&self, user_id: &str) -> Result<SubmissionResult, Error> { + let Some(workspace) = self.workspace_for_user(user_id) else { return Ok(SubmissionResult::error( "Heartbeat requires a workspace (database must be connected).", )); @@ -421,7 +421,7 @@ impl Agent { context.extend_from_slice(&messages[start..]); context.push(ChatMessage::user("Summarize this conversation.")); - let request = crate::llm::CompletionRequest::new(context) + let request = ironclaw_llm::CompletionRequest::new(context) .with_max_tokens(512) .with_temperature(0.3); @@ -470,7 +470,7 @@ impl Agent { context.extend_from_slice(&messages[start..]); context.push(ChatMessage::user("What should I do next?")); - let request = crate::llm::CompletionRequest::new(context) + let request = ironclaw_llm::CompletionRequest::new(context) .with_max_tokens(512) .with_temperature(0.5); @@ -1063,7 +1063,7 @@ impl Agent { .handle_system_command(command, args, channel, tenant) .await? { - SubmissionResult::Response { content } => Ok(Some(content)), + SubmissionResult::Response { content, .. } => Ok(Some(content)), SubmissionResult::Ok { message } => Ok(message), SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))), _ => Ok(None), @@ -1114,7 +1114,7 @@ impl Agent { if let Err(e) = tokio::task::spawn_blocking(move || { // 3a. Update the backend-specific model env var in ~/.ironclaw/.env // only if the var already exists (don't inject new vars). - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); let model_env = registry.model_env_var(&backend); let env_var_prefix = format!("{}=", model_env); @@ -1163,6 +1163,157 @@ impl Agent { #[cfg(test)] mod tests { use super::format_vertical_list; + use crate::agent::agent_loop::{Agent, AgentDeps}; + use crate::agent::cost_guard::{CostGuard, CostGuardConfig}; + use crate::agent::submission::SubmissionResult; + use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; + use crate::hooks::HookRegistry; + use crate::tools::ToolRegistry; + use ironclaw_llm::{ + CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, + }; + use ironclaw_safety::SafetyLayer; + use rust_decimal::Decimal; + use std::sync::Arc; + use std::time::Duration; + + struct StaticLlmProvider; + + #[async_trait::async_trait] + impl LlmProvider for StaticLlmProvider { + fn model_name(&self) -> &str { + "static-mock" + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete( + &self, + _request: CompletionRequest, + ) -> Result<CompletionResponse, crate::error::LlmError> { + Ok(CompletionResponse { + content: "owner heartbeat content leaked".to_string(), + input_tokens: 0, + output_tokens: 0, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + _request: ToolCompletionRequest, + ) -> Result<ToolCompletionResponse, crate::error::LlmError> { + Ok(ToolCompletionResponse { + content: Some("owner heartbeat content leaked".to_string()), + tool_calls: Vec::new(), + input_tokens: 0, + output_tokens: 0, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + reasoning: None, + }) + } + } + + fn make_commands_test_agent() -> Agent { + let deps = AgentDeps { + owner_id: "default".to_string(), + store: None, + settings_store: None, + llm: Arc::new(StaticLlmProvider), + cheap_llm: None, + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: true, + })), + tools: Arc::new(ToolRegistry::new()), + workspace: None, + extension_manager: None, + skill_registry: None, + skill_catalog: None, + skills_config: SkillsConfig::default(), + hooks: Arc::new(HookRegistry::new()), + auth_manager: None, + cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), + sse_tx: None, + http_interceptor: None, + transcription: None, + document_extraction: None, + sandbox_readiness: crate::agent::routine_engine::SandboxReadiness::DisabledByConfig, + builder: None, + llm_backend: "nearai".to_string(), + tenant_rates: Arc::new(crate::tenant::TenantRateRegistry::new(4, 3)), + }; + + Agent::new( + AgentConfig { + name: "commands-test-agent".to_string(), + max_parallel_jobs: 1, + job_timeout: Duration::from_secs(60), + stuck_threshold: Duration::from_secs(60), + repair_check_interval: Duration::from_secs(30), + max_repair_attempts: 1, + use_planning: false, + session_idle_timeout: Duration::from_secs(300), + allow_local_tools: false, + max_cost_per_day_cents: None, + max_actions_per_hour: None, + max_cost_per_user_per_day_cents: None, + max_tool_iterations: 50, + auto_approve_tools: false, + default_timezone: "UTC".to_string(), + max_jobs_per_user: None, + max_tokens_per_job: 0, + multi_tenant: false, + max_llm_concurrent_per_user: None, + max_jobs_concurrent_per_user: None, + engine_v2: false, + }, + deps, + Arc::new(crate::channels::ChannelManager::new()), + None, + None, + None, + Some(Arc::new(crate::context::ContextManager::new(1))), + None, + ) + } + + #[cfg(feature = "libsql")] + #[tokio::test] + async fn process_heartbeat_uses_requesting_user_workspace() { + let (db, _dir) = crate::agent::test_support::make_libsql_test_db().await; + let owner_workspace = Arc::new(crate::workspace::Workspace::new_with_db( + "owner-scope", + Arc::clone(&db), + )); + owner_workspace + .write( + crate::workspace::paths::HEARTBEAT, + "- [ ] owner-only private heartbeat check", + ) + .await + .expect("seed owner heartbeat checklist"); + + let mut agent = make_commands_test_agent(); + agent.deps.workspace = Some(owner_workspace); + + let result = agent + .process_heartbeat("alice") + .await + .expect("heartbeat command should run"); + + assert!( + matches!(&result, SubmissionResult::Ok { message: Some(msg) } if msg.contains("Heartbeat skipped")), + "manual heartbeat should use alice's empty workspace, not owner private checklist: {result:?}" + ); + } #[test] fn format_vertical_list_renders_one_item_per_line() { diff --git a/src/agent/compaction.rs b/src/agent/compaction.rs index a52a8271ba7..812cf28c6f5 100644 --- a/src/agent/compaction.rs +++ b/src/agent/compaction.rs @@ -12,8 +12,8 @@ use chrono::Utc; use crate::agent::context_monitor::{CompactionStrategy, ContextBreakdown}; use crate::agent::session::Thread; use crate::error::Error; -use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; use crate::workspace::Workspace; +use ironclaw_llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; /// Result of a compaction operation. #[derive(Debug)] @@ -202,10 +202,10 @@ Be brief but capture all important details. Use bullet points."#, .iter() .map(|m| { let role_str = match m.role { - crate::llm::Role::User => "User", - crate::llm::Role::Assistant => "Assistant", - crate::llm::Role::System => "System", - crate::llm::Role::Tool => { + ironclaw_llm::Role::User => "User", + ironclaw_llm::Role::Assistant => "Assistant", + ironclaw_llm::Role::System => "System", + ironclaw_llm::Role::Tool => { return format!( "Tool {}: {}", m.name.as_deref().unwrap_or("unknown"), @@ -844,9 +844,9 @@ mod tests { // Verify alternating user/assistant pattern for (i, msg) in messages.iter().enumerate() { if i % 2 == 0 { - assert_eq!(msg.role, crate::llm::Role::User); + assert_eq!(msg.role, ironclaw_llm::Role::User); } else { - assert_eq!(msg.role, crate::llm::Role::Assistant); + assert_eq!(msg.role, ironclaw_llm::Role::Assistant); } } diff --git a/src/agent/context_monitor.rs b/src/agent/context_monitor.rs index a915a3d6974..7c97281332a 100644 --- a/src/agent/context_monitor.rs +++ b/src/agent/context_monitor.rs @@ -3,7 +3,7 @@ //! Monitors the size of the conversation context and triggers //! compaction when approaching the limit. -use crate::llm::ChatMessage; +use ironclaw_llm::ChatMessage; /// Default context window limit (conservative estimate). pub(crate) const DEFAULT_CONTEXT_LIMIT: usize = 100_000; @@ -173,10 +173,10 @@ impl ContextBreakdown { breakdown.total_tokens += tokens; match message.role { - crate::llm::Role::System => breakdown.system_tokens += tokens, - crate::llm::Role::User => breakdown.user_tokens += tokens, - crate::llm::Role::Assistant => breakdown.assistant_tokens += tokens, - crate::llm::Role::Tool => breakdown.tool_tokens += tokens, + ironclaw_llm::Role::System => breakdown.system_tokens += tokens, + ironclaw_llm::Role::User => breakdown.user_tokens += tokens, + ironclaw_llm::Role::Assistant => breakdown.assistant_tokens += tokens, + ironclaw_llm::Role::Tool => breakdown.tool_tokens += tokens, } } diff --git a/src/agent/cost_guard.rs b/src/agent/cost_guard.rs index 4885364b7df..ffec29a0f82 100644 --- a/src/agent/cost_guard.rs +++ b/src/agent/cost_guard.rs @@ -12,7 +12,7 @@ use rust_decimal::Decimal; use rust_decimal_macros::dec; use tokio::sync::Mutex; -use crate::llm::costs; +use ironclaw_llm::costs; /// Configuration for cost guardrails. #[derive(Debug, Clone, Default)] diff --git a/src/agent/dispatcher.rs b/src/agent/dispatcher.rs index 6269d545b08..9b0c7deef87 100644 --- a/src/agent/dispatcher.rs +++ b/src/agent/dispatcher.rs @@ -21,12 +21,12 @@ use crate::agent::agentic_loop::{ AgenticLoopConfig, LoopDelegate, LoopOutcome, LoopSignal, TextAction, }; use crate::generated_images::GeneratedImageSentinel; -use crate::llm::{ChatMessage, Reasoning, ReasoningContext, TokenUsage}; use crate::tools::permissions::{PermissionState, effective_permission}; use crate::tools::redact_params; +use ironclaw_llm::{ChatMessage, Reasoning, ReasoningContext, TokenUsage}; fn selected_model_override(value: &serde_json::Value) -> Option<String> { - crate::llm::normalized_model_override(value.as_str()).map(str::to_string) + ironclaw_llm::normalized_model_override(value.as_str()).map(str::to_string) } /// Decide whether a settings-derived temperature should override the @@ -47,6 +47,53 @@ fn resolve_settings_temperature( .map(|t| (t as f32).clamp(0.0, 2.0)) } +/// Apply the full iteration-1 temperature precedence rule: +/// +/// 1. **Existing context value** wins — set by an upstream caller (e.g. a +/// reasoning-model default) before the agentic loop started. +/// 2. **Per-request metadata** — `metadata["temperature"]` from the inbound +/// `IncomingMessage`. Populated by the Responses API handler when a +/// caller passes a `temperature` field. +/// 3. **Settings store fallback** — user/admin default via +/// `resolve_settings_temperature`, clamped to `[0.0, 2.0]`. +/// +/// Returns `Some(t)` when any source supplies a value, `None` when none do. +/// +/// Centralizing all three sources here means the dispatcher's iteration-1 +/// gate has a single call site with all inputs visible — a future refactor +/// that drops the metadata read must change this signature instead of +/// silently deleting a branch inside the gate body. +fn resolve_temperature_overrides( + current: Option<f32>, + metadata: &serde_json::Value, + settings_value: Option<&serde_json::Value>, +) -> Option<f32> { + if current.is_some() { + return current; + } + if let Some(t) = metadata + .get("temperature") + .and_then(|v| v.as_f64()) + .map(|f| f as f32) + { + return Some(t); + } + resolve_settings_temperature(None, settings_value) +} + +fn chat_job_context( + message: &IncomingMessage, + thread_id: Uuid, + user_tz: chrono_tz::Tz, +) -> JobContext { + let mut job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session") + .with_requester_id(&message.sender_id); + job_ctx.conversation_id = Some(thread_id); + job_ctx.user_timezone = user_tz.name().to_string(); + job_ctx.metadata = crate::agent::agent_loop::chat_tool_execution_metadata(message); + job_ctx +} + /// Result of the agentic loop execution. pub(super) enum AgenticLoopResult { /// Completed with a response. @@ -125,26 +172,22 @@ impl Agent { &self.config.default_timezone, ); - let system_prompt = if let Some(ws) = self.workspace() { - let scoped_workspace = if ws.user_id() == message.user_id { - Arc::clone(ws) + let system_prompt = + if let Some(scoped_workspace) = self.workspace_for_user(&message.user_id) { + match scoped_workspace + .system_prompt_for_context_tz(is_group_chat, user_tz) + .await + { + Ok(prompt) if !prompt.is_empty() => Some(prompt), + Ok(_) => None, + Err(e) => { + tracing::debug!("Could not load workspace system prompt: {}", e); + None + } + } } else { - Arc::new(ws.scoped_to_user(&message.user_id)) + None }; - match scoped_workspace - .system_prompt_for_context_tz(is_group_chat, user_tz) - .await - { - Ok(prompt) if !prompt.is_empty() => Some(prompt), - Ok(_) => None, - Err(e) => { - tracing::debug!("Could not load workspace system prompt: {}", e); - None - } - } - } else { - None - }; // Select active skills. Explicit /skill-name mentions are force-activated // and replaced with the skill's description in the rewritten message. @@ -247,12 +290,8 @@ impl Agent { } // Create a JobContext for tool execution (chat doesn't have a real job) - let mut job_ctx = - JobContext::with_user(&message.user_id, "chat", "Interactive chat session") - .with_requester_id(&message.sender_id); + let mut job_ctx = chat_job_context(message, thread_id, user_tz); job_ctx.http_interceptor = self.deps.http_interceptor.clone(); - job_ctx.user_timezone = user_tz.name().to_string(); - job_ctx.metadata = crate::agent::agent_loop::chat_tool_execution_metadata(message); // Build system prompts once for this turn. Two variants: with tools // (normal iterations) and without (force_text final iteration). @@ -293,7 +332,7 @@ impl Agent { if let Some(last_user) = msgs .iter_mut() .rev() - .find(|m| m.role == crate::llm::Role::User) + .find(|m| m.role == ironclaw_llm::Role::User) { *last_user = ChatMessage::user(&user_content); } @@ -485,7 +524,6 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { self.tenant.user_id(), admin_policy, ); - // Apply per-user tool permission filtering. // // Load tool_permissions from the per-user DB settings store (same @@ -550,7 +588,6 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { sess.auto_approve_tool(name); } } - // Update context for this iteration reason_ctx.available_tools = tool_defs; // Preserve force_text if already set (e.g. by truncation escalation). @@ -587,7 +624,7 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { reasoning: &Reasoning, reason_ctx: &mut ReasoningContext, iteration: usize, - ) -> Result<crate::llm::RespondOutput, Error> { + ) -> Result<ironclaw_llm::RespondOutput, Error> { // Enforce cost guardrails before the LLM call (global + per-user) if let Err(limit) = self.tenant.check_cost_allowed().await { return Err(crate::error::LlmError::InvalidResponse { @@ -597,31 +634,49 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { .into()); } - // Apply per-user overrides from settings (first iteration only - // to avoid repeated DB lookups within the same agentic loop). + // Apply per-user overrides from settings on the first iteration + // only, to avoid repeated DB lookups within the same agentic loop. // Uses admin-fallback so admin-set defaults propagate to members // who haven't overridden the value themselves. - if iteration == 0 - && let Some(store) = self.tenant.store() - { + // + // The loop in `run_agentic_loop` starts iterations at 1 (see + // `agentic_loop.rs`), so this gate must match that first value. + // Off-by-one here silently disables per-request temperature, the + // `temperature` setting, and the `selected_model` override. + if iteration == 1 { + // Resolve temperature precedence in one place so all three input + // sources (existing context value, per-request metadata, settings + // store) are visible at a single call site. See + // `resolve_temperature_overrides` for the precedence rule. + let settings_temperature = if let Some(store) = self.tenant.store() { + store + .get_setting_with_admin_fallback("temperature") + .await + .ok() + .flatten() + } else { + None + }; + if let Some(t) = resolve_temperature_overrides( + reason_ctx.temperature, + &self.message.metadata, + settings_temperature.as_ref(), + ) { + reason_ctx.temperature = Some(t); + } + // Model override: "selected_model" — the same key the /model command // persists to via SettingsStore (per-user scoped via TenantScope). - if let Ok(Some(value)) = store - .get_setting_with_admin_fallback("selected_model") - .await + // Kept separate from temperature precedence — different override + // category, only a settings-level signal exists. + if let Some(store) = self.tenant.store() + && let Ok(Some(value)) = store + .get_setting_with_admin_fallback("selected_model") + .await && let Some(model) = selected_model_override(&value) { reason_ctx.model_override = Some(model); } - - // Temperature override from user or admin settings. Per-request - // values already on the context take precedence over settings. - if let Ok(setting) = store.get_setting_with_admin_fallback("temperature").await - && let Some(t) = - resolve_settings_temperature(reason_ctx.temperature, setting.as_ref()) - { - reason_ctx.temperature = Some(t); - } } let llm_call_start = std::time::Instant::now(); @@ -742,7 +797,7 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { async fn handle_text_response( &self, text: &str, - _metadata: crate::llm::ResponseMetadata, + _metadata: ironclaw_llm::ResponseMetadata, _reason_ctx: &mut ReasoningContext, ) -> TextAction { // Strip internal "[Called tool ...]" text that can leak when @@ -754,9 +809,10 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { async fn execute_tool_calls( &self, - tool_calls: Vec<crate::llm::ToolCall>, + tool_calls: Vec<ironclaw_llm::ToolCall>, content: Option<String>, reason_ctx: &mut ReasoningContext, + reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, Error> { // Extract and sanitize the narrative before consuming `content`. let narrative = content @@ -773,12 +829,12 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { // Add the assistant message with tool_calls to context. // OpenAI protocol requires this before tool-result messages. - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); + // Carry reasoning so the next request can echo it back — required for + // DeepSeek thinking-mode and Gemini 2.5+ to validate the chain (#3201, #3225). + reason_ctx.messages.push( + ChatMessage::assistant_with_tool_calls(content, tool_calls.clone()) + .with_reasoning(reasoning), + ); // Execute tools and add results to context let _ = self @@ -866,11 +922,11 @@ impl<'a> LoopDelegate for ChatDelegate<'a> { // Walk tool_calls checking approval and hooks. Classify // each tool as Rejected (by hook) or Runnable. Stop at the // first tool that needs approval. - let mut preflight: Vec<(crate::llm::ToolCall, PreflightOutcome)> = Vec::new(); - let mut runnable: Vec<(usize, crate::llm::ToolCall)> = Vec::new(); + let mut preflight: Vec<(ironclaw_llm::ToolCall, PreflightOutcome)> = Vec::new(); + let mut runnable: Vec<(usize, ironclaw_llm::ToolCall)> = Vec::new(); let mut approval_needed: Option<( usize, - crate::llm::ToolCall, + ironclaw_llm::ToolCall, Arc<dyn crate::tools::Tool>, bool, // allow_always )> = None; @@ -1466,7 +1522,7 @@ pub(super) fn restore_selected_auth_prompt( )) } -/// Extract auth prompt fields from a tool_auth/tool_activate result JSON string. +/// Extract auth prompt fields from a tool_auth/tool_install result JSON string. pub(super) fn parse_auth_result(result: &Result<String, Error>) -> ParsedAuthData { let parsed = result .as_ref() @@ -1504,12 +1560,12 @@ pub(super) fn parse_auth_result(result: &Result<String, Error>) -> ParsedAuthDat } } -/// Extract actionable auth prompt data from a tool_auth/tool_activate result. +/// Extract actionable auth prompt data from a tool_auth/tool_install result. pub(super) fn extract_auth_prompt( tool_name: &str, result: &Result<String, Error>, ) -> Option<ParsedAuthData> { - if tool_name != "tool_auth" && tool_name != "tool_activate" { + if tool_name != "tool_auth" && tool_name != "tool_install" { return None; } @@ -1607,7 +1663,7 @@ fn preflight_rejection_tool_message( /// Instead of a generic "Executing 2 tool(s)..." this returns messages like /// "Running command..." or "Fetching page..." for single-tool calls, falling /// back to "Executing N tool(s)..." for multi-tool calls. -fn contextual_tool_message(tool_calls: &[crate::llm::ToolCall]) -> String { +fn contextual_tool_message(tool_calls: &[ironclaw_llm::ToolCall]) -> String { if tool_calls.len() == 1 { match tool_calls[0].name.as_str() { "shell" => "Running command...".into(), @@ -1633,7 +1689,7 @@ fn contextual_tool_message(tool_calls: &[crate::llm::ToolCall]) -> String { /// (the current turn's assistant tool calls and tool results). A short note is /// inserted so the LLM knows earlier history was dropped. fn compact_messages_for_retry(messages: &[ChatMessage]) -> Vec<ChatMessage> { - use crate::llm::Role; + use ironclaw_llm::Role; let mut compacted = Vec::new(); @@ -1781,23 +1837,12 @@ fn image_generation_summary_tool_message( sentinel: &GeneratedImageSentinel, ) -> ChatMessage { let media_type = sentinel.media_type().unwrap_or("image"); - let path = sentinel.path(); - let summary = if let Some(path) = path { - serde_json::json!({ - "type": "image_generated", - "status": "ok", - "media_type": media_type, - "path": path, - }) - .to_string() - } else { - serde_json::json!({ - "type": "image_generated", - "status": "ok", - "media_type": media_type, - }) - .to_string() - }; + let summary = serde_json::json!({ + "type": "image_generated", + "status": "ok", + "media_type": media_type, + }) + .to_string(); let sanitized = safety.sanitize_tool_output(tool_name, &summary); let content = safety.wrap_for_llm(tool_name, &sanitized.content); ChatMessage::tool_result(tool_call_id, tool_name, content) @@ -1818,22 +1863,23 @@ mod tests { use crate::agent::agent_loop::{Agent, AgentDeps}; use crate::agent::cost_guard::{CostGuard, CostGuardConfig}; use crate::agent::session::Session; - use crate::channels::ChannelManager; + use crate::channels::{ChannelManager, IncomingMessage}; use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; use crate::context::{ContextManager, JobContext}; use crate::error::Error; use crate::hooks::HookRegistry; - use crate::llm::{ + use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput, ToolRegistry}; + use ironclaw_llm::{ CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCall, ToolCompletionRequest, ToolCompletionResponse, }; - use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput, ToolRegistry}; use ironclaw_safety::SafetyLayer; + use uuid::Uuid; use super::{ capture_auth_prompt, check_auth_required, extract_auth_prompt, parse_auth_result, - persist_selected_auth_prompt, resolve_settings_temperature, restore_selected_auth_prompt, - selected_model_override, + persist_selected_auth_prompt, resolve_settings_temperature, resolve_temperature_overrides, + restore_selected_auth_prompt, selected_model_override, }; use crate::agent::session::PendingAuthPrompt; use crate::generated_images::GeneratedImageSentinel; @@ -1878,6 +1924,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -1920,6 +1967,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -1963,6 +2011,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }); } @@ -1970,16 +2019,18 @@ mod tests { content: None, tool_calls: vec![ ToolCall { - id: crate::llm::generate_tool_call_id(0, 0), - name: "tool_activate".to_string(), + id: ironclaw_llm::generate_tool_call_id(0, 0), + name: "tool_install".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }, ToolCall { - id: crate::llm::generate_tool_call_id(0, 1), + id: ironclaw_llm::generate_tool_call_id(0, 1), name: "approval_tool".to_string(), arguments: serde_json::json!({"target": "danger"}), reasoning: None, + signature: None, }, ], input_tokens: 0, @@ -1987,6 +2038,7 @@ mod tests { finish_reason: FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -1996,7 +2048,7 @@ mod tests { #[async_trait] impl Tool for OAuthPromptTool { fn name(&self) -> &str { - "tool_activate" + "tool_install" } fn description(&self) -> &str { @@ -2324,12 +2376,14 @@ mod tests { name: "http".to_string(), arguments: serde_json::json!({"url": "https://example.com"}), reasoning: None, + signature: None, }, ToolCall { id: "call_3".to_string(), name: "echo".to_string(), arguments: serde_json::json!({"message": "done"}), reasoning: None, + signature: None, }, ], selected_auth_prompt: Some(crate::agent::session::PendingAuthPrompt::new( @@ -2364,7 +2418,7 @@ mod tests { async fn test_need_approval_persists_first_auth_prompt_for_resume() { use crate::agent::session::Session; use crate::channels::IncomingMessage; - use crate::llm::ChatMessage; + use ironclaw_llm::ChatMessage; use tokio::sync::Mutex; let registry = Arc::new(ToolRegistry::new()); @@ -2553,7 +2607,7 @@ mod tests { }) .to_string()); - let auth_data = extract_auth_prompt("tool_activate", &result).expect("auth prompt"); + let auth_data = extract_auth_prompt("tool_install", &result).expect("auth prompt"); assert_eq!( auth_data.extension_name.as_ref().map(|e| e.as_str()), Some("gmail") @@ -2563,7 +2617,7 @@ mod tests { Some("https://accounts.google.com/o/oauth2/v2/auth?client_id=test") ); assert!(!auth_data.awaiting_token); - assert!(check_auth_required("tool_activate", &result).is_none()); + assert!(check_auth_required("tool_install", &result).is_none()); } #[test] @@ -2576,7 +2630,7 @@ mod tests { }) .to_string()); - assert!(extract_auth_prompt("tool_activate", &result).is_none()); + assert!(extract_auth_prompt("tool_install", &result).is_none()); } // Helper-level sanitize_auth_url tests live alongside the helper itself @@ -2640,7 +2694,7 @@ mod tests { .to_string()); let mut selected = None; - capture_auth_prompt(&mut selected, "tool_activate", &first); + capture_auth_prompt(&mut selected, "tool_install", &first); capture_auth_prompt(&mut selected, "tool_auth", &second); let (ext_name, auth_data) = selected.expect("selected auth prompt"); @@ -2670,7 +2724,7 @@ mod tests { let mut selected = None; capture_auth_prompt(&mut selected, "tool_auth", &first); - capture_auth_prompt(&mut selected, "tool_activate", &second); + capture_auth_prompt(&mut selected, "tool_install", &second); let (ext_name, auth_data) = selected.expect("selected auth prompt"); assert_eq!(ext_name, "notion"); @@ -2714,7 +2768,7 @@ mod tests { } #[test] - fn test_detect_auth_awaiting_tool_activate() { + fn test_detect_auth_awaiting_tool_install() { let result: Result<String, Error> = Ok(serde_json::json!({ "name": "slack", "kind": "McpServer", @@ -2724,7 +2778,7 @@ mod tests { }) .to_string()); - let detected = check_auth_required("tool_activate", &result); + let detected = check_auth_required("tool_install", &result); assert!(detected.is_some()); let (name, instructions) = detected.unwrap(); assert_eq!(name, "slack"); @@ -2732,7 +2786,7 @@ mod tests { } #[test] - fn test_detect_auth_awaiting_tool_activate_not_awaiting() { + fn test_detect_auth_awaiting_tool_install_not_awaiting() { let result: Result<String, Error> = Ok(serde_json::json!({ "name": "slack", "tools_loaded": ["slack_post_message"], @@ -2740,7 +2794,18 @@ mod tests { }) .to_string()); - assert!(check_auth_required("tool_activate", &result).is_none()); + assert!(check_auth_required("tool_install", &result).is_none()); + } + + #[test] + fn test_chat_job_context_includes_thread_id_for_sse_scoped_tools() { + let thread_id = Uuid::new_v4(); + let message = IncomingMessage::new("web", "test-user", "/plan Ship it"); + + let job_ctx = super::chat_job_context(&message, thread_id, chrono_tz::UTC); + + assert_eq!(job_ctx.conversation_id, Some(thread_id)); + assert_eq!(job_ctx.user_timezone, "UTC"); } #[tokio::test] @@ -2804,7 +2869,7 @@ mod tests { // ---- compact_messages_for_retry tests ---- use super::compact_messages_for_retry; - use crate::llm::{ChatMessage, Role}; + use ironclaw_llm::{ChatMessage, Role}; #[test] fn test_compact_keeps_system_and_last_user_exchange() { @@ -2822,6 +2887,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({"message": "hi"}), reasoning: None, + signature: None, }], ), ChatMessage::tool_result("call_1", "echo", "hi"), @@ -2915,12 +2981,14 @@ mod tests { name: "http".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }, ToolCall { id: "c2".to_string(), name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }, ], ), @@ -2955,6 +3023,7 @@ mod tests { name: "echo".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], ), ChatMessage::tool_result("c1", "echo", "done"), @@ -2987,8 +3056,8 @@ mod tests { // 1. Provider returns ContextLengthExceeded // 2. compact_messages_for_retry reduces context // 3. Retry with compacted messages succeeds - use crate::llm::Reasoning; use crate::testing::StubLlm; + use ironclaw_llm::Reasoning; let stub = Arc::new(StubLlm::failing_non_transient("ctx-bomb")); @@ -3006,7 +3075,7 @@ mod tests { ChatMessage::user("Current request"), ]; - let context = crate::llm::ReasoningContext::new().with_messages(messages.clone()); + let context = ironclaw_llm::ReasoningContext::new().with_messages(messages.clone()); // Step 1: First call fails with ContextLengthExceeded. let err = reasoning.respond_with_tools(&context).await.unwrap_err(); @@ -3025,7 +3094,7 @@ mod tests { // Step 3: Switch provider to success and retry. stub.set_failing(false); - let retry_context = crate::llm::ReasoningContext::new().with_messages(compacted); + let retry_context = ironclaw_llm::ReasoningContext::new().with_messages(compacted); let result = reasoning.respond_with_tools(&retry_context).await; assert!(result.is_ok(), "Retry after compaction should succeed"); @@ -3076,22 +3145,25 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }); } // Tools available: always call one. Ok(ToolCompletionResponse { content: None, tool_calls: vec![ToolCall { - id: crate::llm::generate_tool_call_id(0, 0), + id: ironclaw_llm::generate_tool_call_id(0, 0), name: "echo".to_string(), arguments: serde_json::json!({"message": "looping"}), reasoning: None, + signature: None, }], input_tokens: 0, output_tokens: 5, finish_reason: FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -3100,7 +3172,7 @@ mod tests { async fn force_text_prevents_infinite_tool_call_loop() { // Verify that Reasoning with force_text=true returns text even when // the provider would normally return tool calls. - use crate::llm::{Reasoning, ReasoningContext, RespondResult, ToolDefinition}; + use ironclaw_llm::{Reasoning, ReasoningContext, RespondResult, ToolDefinition}; let provider = Arc::new(AlwaysToolCallProvider); let reasoning = Reasoning::new(provider); @@ -3229,6 +3301,113 @@ mod tests { ); } + // ------------------------------------------------------------------- + // resolve_temperature_overrides — full iteration-1 precedence rule + // + // These tests lock in the wiring `ChatDelegate::call_llm` depends on + // (PR #3641, serrrfirat's Medium-severity follow-up). The + // `resolve_settings_temperature_*` tests above only cover the + // settings-side helper in isolation — they would still pass if the + // dispatcher dropped the metadata read entirely. These tests cover + // the precedence contract at the seam the call site uses, so any + // future refactor that loses one of the three inputs has to change + // a signature instead of silently deleting a branch. + // ------------------------------------------------------------------- + + #[test] + fn temperature_overrides_existing_context_wins_over_metadata_and_settings() { + // Highest precedence: a value already on the reasoning context + // (e.g. set by an upstream caller before the agentic loop ran). + assert_eq!( + resolve_temperature_overrides( + Some(0.11), + &serde_json::json!({ "temperature": 0.42 }), + Some(&serde_json::json!(0.9)), + ), + Some(0.11), + ); + } + + #[test] + fn temperature_overrides_per_request_wins_over_settings_fallback() { + // The bug serrrfirat's review targets: per-request `temperature` + // from the Responses API arrives in `metadata["temperature"]` and + // must override the user/admin settings default. Without this + // ordering, API callers cannot set a per-call value. + assert_eq!( + resolve_temperature_overrides( + None, + &serde_json::json!({ "temperature": 0.42 }), + Some(&serde_json::json!(0.9)), + ), + Some(0.42), + ); + } + + #[test] + fn temperature_overrides_settings_used_when_no_per_request_value() { + assert_eq!( + resolve_temperature_overrides( + None, + &serde_json::json!({}), + Some(&serde_json::json!(0.9)) + ), + Some(0.9), + ); + } + + #[test] + fn temperature_overrides_settings_clamped_when_used() { + // The settings-fallback path must still clamp out-of-range DB + // values to `[0.0, 2.0]`, mirroring `resolve_settings_temperature`. + assert_eq!( + resolve_temperature_overrides( + None, + &serde_json::json!({}), + Some(&serde_json::json!(9.0)) + ), + Some(2.0), + ); + assert_eq!( + resolve_temperature_overrides( + None, + &serde_json::json!({}), + Some(&serde_json::json!(-1.0)), + ), + Some(0.0), + ); + } + + #[test] + fn temperature_overrides_returns_none_when_no_source_supplies_value() { + assert_eq!( + resolve_temperature_overrides(None, &serde_json::json!({}), None), + None, + ); + // A non-numeric metadata value falls through to settings (also None). + assert_eq!( + resolve_temperature_overrides(None, &serde_json::json!({ "temperature": "hot" }), None,), + None, + ); + } + + #[test] + fn temperature_overrides_per_request_not_clamped_at_this_layer() { + // Per-request values are accepted verbatim here; clamping for the + // metadata path is owned by `Reasoning::respond_with_tools`, which + // clamps the final value just before the provider call. Locking + // this in so a future "let's clamp everywhere" change is a + // deliberate decision, not an accident. + assert_eq!( + resolve_temperature_overrides( + None, + &serde_json::json!({ "temperature": 9.0 }), + Some(&serde_json::json!(0.5)), + ), + Some(9.0), + ); + } + #[test] fn selected_model_override_ignores_default_sentinel() { assert_eq!(selected_model_override(&serde_json::json!("default")), None); @@ -3285,22 +3464,25 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }); } // Always call a tool that does not exist in the registry. Ok(ToolCompletionResponse { content: None, tool_calls: vec![ToolCall { - id: crate::llm::generate_tool_call_id(0, 0), + id: ironclaw_llm::generate_tool_call_id(0, 0), name: "nonexistent_tool".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }], input_tokens: 0, output_tokens: 5, finish_reason: FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -3351,6 +3533,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -3429,7 +3612,7 @@ mod tests { async fn test_dispatcher_terminates_with_all_tool_calls_failing() { use crate::agent::session::Session; use crate::channels::IncomingMessage; - use crate::llm::ChatMessage; + use ironclaw_llm::ChatMessage; use tokio::sync::Mutex; let agent = make_test_agent_with_llm(Arc::new(FailingToolCallProvider), 5); @@ -3475,9 +3658,9 @@ mod tests { async fn test_admin_policy_filter_happens_before_auto_approval_and_llm_call() { use crate::agent::session::Session; use crate::channels::IncomingMessage; - use crate::llm::ChatMessage; use crate::tools::builtin::{EchoTool, TimeTool}; use crate::tools::permissions::{ADMIN_SETTINGS_USER_ID, ADMIN_TOOL_POLICY_KEY}; + use ironclaw_llm::ChatMessage; use tokio::sync::Mutex; let (db, _tmp_dir) = crate::testing::test_db().await; @@ -3629,8 +3812,8 @@ mod tests { async fn test_dispatcher_terminates_with_max_iterations() { use crate::agent::session::Session; use crate::channels::IncomingMessage; - use crate::llm::ChatMessage; use crate::tools::builtin::EchoTool; + use ironclaw_llm::ChatMessage; use tokio::sync::Mutex; // Use AlwaysToolCallProvider which calls "echo" on every turn. @@ -3758,7 +3941,7 @@ mod tests { async fn test_dispatcher_response_usage_is_per_turn_not_cumulative() { use crate::agent::session::Session; use crate::channels::IncomingMessage; - use crate::llm::ChatMessage; + use ironclaw_llm::ChatMessage; use tokio::sync::Mutex; let agent = make_test_agent_with_llm(Arc::new(FixedUsageTextProvider), 3); @@ -4129,8 +4312,8 @@ mod tests { /// Disabled tools must be excluded from the LLM's tool definition list. #[test] fn test_permission_disabled_tool_excluded_from_definitions() { - use crate::llm::ToolDefinition; use crate::tools::permissions::{PermissionState, effective_permission}; + use ironclaw_llm::ToolDefinition; use std::collections::HashMap; let mut tool_permissions: HashMap<String, PermissionState> = HashMap::new(); diff --git a/src/agent/heartbeat.rs b/src/agent/heartbeat.rs index 5f7364cc555..67d79830fe8 100644 --- a/src/agent/heartbeat.rs +++ b/src/agent/heartbeat.rs @@ -31,10 +31,10 @@ use chrono_tz::Tz; use tokio::sync::mpsc; use crate::channels::OutgoingResponse; -use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; use crate::tenant::SystemScope; use crate::workspace::Workspace; use crate::workspace::hygiene::HygieneConfig; +use ironclaw_llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; /// Configuration for the heartbeat runner. #[derive(Debug, Clone)] @@ -440,6 +440,7 @@ impl HeartbeatRunner { // because it was not supplied by a channel adapter. thread_id: thread_id.map(ironclaw_common::ExternalThreadId::from_trusted), attachments: Vec::new(), + inline_attachments: Vec::new(), metadata: serde_json::json!({ "source": "heartbeat", "owner_id": self.workspace.user_id(), @@ -906,7 +907,7 @@ mod tests { HeartbeatConfig, HygieneConfig, Arc<crate::workspace::Workspace>, - Arc<dyn crate::llm::LlmProvider>, + Arc<dyn ironclaw_llm::LlmProvider>, Option<tokio::sync::mpsc::Sender<crate::channels::OutgoingResponse>>, Option<SystemScope>, ) -> tokio::task::JoinHandle<()> = spawn_heartbeat; diff --git a/src/agent/mod.rs b/src/agent/mod.rs index c1e201a5c4f..c6377f69356 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -32,6 +32,26 @@ pub mod task; mod thread_ops; pub mod undo; +#[cfg(all(test, feature = "libsql"))] +pub(crate) mod test_support { + use std::sync::Arc; + + use crate::db::Database; + + pub(crate) async fn make_libsql_test_db() -> (Arc<dyn Database>, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("failed to create temp dir"); // safety: test-only setup helper + let path = dir.path().join("test.db"); + let backend = crate::db::libsql::LibSqlBackend::new_local(&path) + .await + .expect("failed to create test LibSqlBackend"); // safety: test-only setup helper + backend + .run_migrations() + .await + .expect("failed to run migrations"); // safety: test-only setup helper + (Arc::new(backend) as Arc<dyn Database>, dir) + } +} + pub(crate) use agent_loop::truncate_for_preview; pub use agent_loop::{Agent, AgentDeps}; pub(crate) use attachments::augment_with_attachments; diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index 9a30823ac2c..a87a4bfa02c 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -31,9 +31,6 @@ use crate::config::RoutineConfig; use crate::context::{JobContext, JobState}; use crate::error::RoutineError; use crate::extensions::ExtensionManager; -use crate::llm::{ - ChatMessage, CompletionRequest, FinishReason, LlmProvider, ToolCall, ToolCompletionRequest, -}; use crate::ownership::Owned; use crate::tenant::SystemScope; use crate::tools::{ @@ -41,6 +38,9 @@ use crate::tools::{ prepare_tool_params, }; use crate::workspace::Workspace; +use ironclaw_llm::{ + ChatMessage, CompletionRequest, FinishReason, LlmProvider, ToolCall, ToolCompletionRequest, +}; use ironclaw_safety::SafetyLayer; enum EventMatcher { @@ -130,7 +130,7 @@ pub struct RoutineEngine { /// `EngineContext` and on into the `JobContext` that drives /// Lightweight tool dispatches, so routine-fired tools see the /// same interceptor the chat path does. - http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, /// Timestamp when this engine instance was created. Used by /// `sync_dispatched_runs` to distinguish orphaned runs (from a previous /// process) from actively-watched runs (from this process). @@ -150,7 +150,7 @@ impl RoutineEngine { tools: Arc<ToolRegistry>, safety: Arc<SafetyLayer>, sandbox_readiness: SandboxReadiness, - http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, ) -> Self { Self { config, @@ -1126,7 +1126,7 @@ struct EngineContext { /// tools called from a Lightweight routine action reach the real /// network even when the rest of the system is configured to /// route through mocks. - http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, } /// Execute a routine run. Handles both lightweight and full_job modes. @@ -1704,7 +1704,7 @@ async fn execute_lightweight_no_tools( .with_temperature(0.3); let response = ctx.llm.complete(request).await.map_err(|e| { - let retryable = crate::llm::retry::is_retryable(&e); + let retryable = ironclaw_llm::retry::is_retryable(&e); RoutineError::LlmFailed { reason: e.to_string(), // No partial tokens: the LLM call itself failed, so the response @@ -1882,7 +1882,7 @@ async fn execute_lightweight_with_tools( .with_temperature(0.3); let response = ctx.llm.complete(request).await.map_err(|e| { - let retryable = crate::llm::retry::is_retryable(&e); + let retryable = ironclaw_llm::retry::is_retryable(&e); RoutineError::LlmFailed { reason: e.to_string(), partial_tokens: tokens_to_option(total_input_tokens, total_output_tokens), @@ -1915,7 +1915,7 @@ async fn execute_lightweight_with_tools( .with_temperature(0.3); let response = ctx.llm.complete_with_tools(request).await.map_err(|e| { - let retryable = crate::llm::retry::is_retryable(&e); + let retryable = ironclaw_llm::retry::is_retryable(&e); RoutineError::LlmFailed { reason: e.to_string(), partial_tokens: tokens_to_option(total_input_tokens, total_output_tokens), @@ -1937,11 +1937,17 @@ async fn execute_lightweight_with_tools( ); } - // LLM returned tool calls: add assistant message and execute tools - messages.push(ChatMessage::assistant_with_tool_calls( - response.content.clone(), - response.tool_calls.clone(), - )); + // LLM returned tool calls: add assistant message and execute tools. + // Carry reasoning so the next request can echo it — required for + // DeepSeek thinking-mode and Gemini 2.5+ to validate the chain + // (#3201, #3225). + messages.push( + ChatMessage::assistant_with_tool_calls( + response.content.clone(), + response.tool_calls.clone(), + ) + .with_reasoning(response.reasoning.clone()), + ); // Execute tools sequentially for tc in response.tool_calls { @@ -1992,7 +1998,7 @@ fn snapshot_messages_for_tool_iteration(messages: &[ChatMessage]) -> Vec<ChatMes let mut snapshot = Vec::with_capacity(MAX_TOOL_LOOP_MESSAGES); if let Some(first) = messages.first() - && first.role == crate::llm::Role::System + && first.role == ironclaw_llm::Role::System { snapshot.push(first.clone()); let tail_len = MAX_TOOL_LOOP_MESSAGES - 1; @@ -2133,6 +2139,7 @@ async fn send_notification( thread_id: thread_id .map(|s| ironclaw_common::ExternalThreadId::from_trusted(s.to_string())), attachments: Vec::new(), + inline_attachments: Vec::new(), metadata: serde_json::json!({ "source": "routine", "routine_name": routine_name, @@ -2603,22 +2610,22 @@ mod tests { fn test_empty_response_handling() { // Simulate the empty content guard logic let empty_content = ""; - let finish_reason_length = crate::llm::FinishReason::Length; - let finish_reason_stop = crate::llm::FinishReason::Stop; + let finish_reason_length = ironclaw_llm::FinishReason::Length; + let finish_reason_stop = ironclaw_llm::FinishReason::Stop; assert!( empty_content.trim().is_empty(), "Should detect empty content" ); - assert_eq!(finish_reason_length, crate::llm::FinishReason::Length); - assert_eq!(finish_reason_stop, crate::llm::FinishReason::Stop); + assert_eq!(finish_reason_length, ironclaw_llm::FinishReason::Length); + assert_eq!(finish_reason_stop, ironclaw_llm::FinishReason::Stop); } #[test] fn test_handle_text_response_strips_internal_tool_markers() { let result = super::handle_text_response( "Here is the report.\n[Called tool `http` with arguments: {\"url\":\"https://example.com\"}]", - crate::llm::FinishReason::Stop, + ironclaw_llm::FinishReason::Stop, 10, 5, ) @@ -2633,7 +2640,7 @@ mod tests { fn test_handle_text_response_replaces_marker_only_text() { let result = super::handle_text_response( "[Called tool `http` with arguments: {\"url\":\"https://example.com\"}]", - crate::llm::FinishReason::Stop, + ironclaw_llm::FinishReason::Stop, 4, 3, ) @@ -2656,14 +2663,14 @@ mod tests { #[test] fn test_snapshot_messages_keeps_system_and_recent_tail() { - let mut messages = vec![crate::llm::ChatMessage::system("sys")]; + let mut messages = vec![ironclaw_llm::ChatMessage::system("sys")]; for i in 0..80 { - messages.push(crate::llm::ChatMessage::user(format!("u{i}"))); + messages.push(ironclaw_llm::ChatMessage::user(format!("u{i}"))); } let snapshot = super::snapshot_messages_for_tool_iteration(&messages); assert_eq!(snapshot.len(), super::MAX_TOOL_LOOP_MESSAGES); // safety: test-only no-panics CI false positive - assert_eq!(snapshot[0].role, crate::llm::Role::System); // safety: test-only no-panics CI false positive + assert_eq!(snapshot[0].role, ironclaw_llm::Role::System); // safety: test-only no-panics CI false positive assert_eq!(snapshot[0].content, "sys"); // safety: test-only no-panics CI false positive let last_content = snapshot.last().map(|m| m.content.as_str()); assert_eq!(last_content, Some("u79")); // safety: test-only no-panics CI false positive @@ -2672,13 +2679,13 @@ mod tests { #[test] fn test_snapshot_messages_unchanged_when_within_limit() { let messages = vec![ - crate::llm::ChatMessage::system("sys"), - crate::llm::ChatMessage::user("a"), - crate::llm::ChatMessage::assistant("b"), + ironclaw_llm::ChatMessage::system("sys"), + ironclaw_llm::ChatMessage::user("a"), + ironclaw_llm::ChatMessage::assistant("b"), ]; let snapshot = super::snapshot_messages_for_tool_iteration(&messages); assert_eq!(snapshot.len(), messages.len()); // safety: test-only no-panics CI false positive - assert_eq!(snapshot[0].role, crate::llm::Role::System); // safety: test-only no-panics CI false positive + assert_eq!(snapshot[0].role, ironclaw_llm::Role::System); // safety: test-only no-panics CI false positive assert_eq!(snapshot[1].content, "a"); // safety: test-only no-panics CI false positive assert_eq!(snapshot[2].content, "b"); // safety: test-only no-panics CI false positive } diff --git a/src/agent/scheduler.rs b/src/agent/scheduler.rs index 778318ec435..edfd4a3741a 100644 --- a/src/agent/scheduler.rs +++ b/src/agent/scheduler.rs @@ -14,13 +14,13 @@ use crate::context::{ContextManager, JobContext, JobState}; use crate::error::{Error, JobError}; use crate::extensions::ExtensionManager; use crate::hooks::HookRegistry; -use crate::llm::LlmProvider; use crate::tenant::SystemScope; use crate::tools::{ ApprovalContext, ToolRegistry, autonomous_allowed_tool_names, autonomous_unavailable_error, prepare_tool_params, }; use crate::worker::job::{Worker, WorkerDeps}; +use ironclaw_llm::LlmProvider; use ironclaw_safety::SafetyLayer; /// Message to send to a worker. @@ -69,7 +69,7 @@ pub struct Scheduler { /// SSE manager for live job event streaming. sse_tx: Option<Arc<crate::channels::web::sse::SseManager>>, /// HTTP interceptor for trace recording/replay (propagated to workers). - http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, /// Running jobs (main LLM-driven jobs). jobs: Arc<RwLock<HashMap<Uuid, ScheduledJob>>>, /// Running sub-tasks (tool executions, background tasks). @@ -109,7 +109,7 @@ impl Scheduler { /// Set the HTTP interceptor for trace recording/replay. pub fn set_http_interceptor( &mut self, - interceptor: Arc<dyn crate::llm::recording::HttpInterceptor>, + interceptor: Arc<dyn ironclaw_llm::recording::HttpInterceptor>, ) { self.http_interceptor = Some(interceptor); } @@ -744,11 +744,11 @@ impl Scheduler { mod tests { use super::*; use crate::config::SafetyConfig; - use crate::llm::{ + use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput}; + use ironclaw_llm::{ CompletionRequest, CompletionResponse, LlmError, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, }; - use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput}; use ironclaw_safety::SafetyLayer; use rust_decimal_macros::dec; diff --git a/src/agent/session.rs b/src/agent/session.rs index 2e6c92ec00b..3478d9be678 100644 --- a/src/agent/session.rs +++ b/src/agent/session.rs @@ -17,8 +17,8 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::generated_images::GeneratedImageSentinel; -use crate::llm::{ChatMessage, ToolCall, generate_tool_call_id}; use ironclaw_common::{ExtensionName, truncate_preview}; +use ironclaw_llm::{ChatMessage, ToolCall, generate_tool_call_id}; /// A session containing one or more threads. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -553,6 +553,7 @@ impl Thread { name: tc.name.clone(), arguments: tc.parameters.clone(), reasoning: None, + signature: None, }) .collect(); @@ -616,14 +617,14 @@ impl Thread { let mut turn_number = 0; while let Some(msg) = iter.next() { - if msg.role == crate::llm::Role::User { + if msg.role == ironclaw_llm::Role::User { let mut turn = Turn::new(turn_number, &msg.content); // Consume tool call sequences (assistant_with_tool_calls + tool_results). // A single turn may contain multiple rounds of tool calls, so we // track the cumulative base index into turn.tool_calls. while let Some(next) = iter.peek() { - if next.role == crate::llm::Role::Assistant && next.tool_calls.is_some() { + if next.role == ironclaw_llm::Role::Assistant && next.tool_calls.is_some() { let call_base_idx = turn.tool_calls.len(); if let Some(assistant_msg) = iter.next() @@ -643,7 +644,7 @@ impl Thread { // indexing relative to this batch's base offset. let mut pos = 0; while let Some(tr) = iter.peek() { - if tr.role != crate::llm::Role::Tool { + if tr.role != ironclaw_llm::Role::Tool { break; } if let Some(tool_msg) = iter.next() { @@ -665,7 +666,7 @@ impl Thread { // Check if next is the final assistant response for this turn let is_final_assistant = iter.peek().is_some_and(|n| { - n.role == crate::llm::Role::Assistant && n.tool_calls.is_none() + n.role == ironclaw_llm::Role::Assistant && n.tool_calls.is_none() }); if is_final_assistant && let Some(response) = iter.next() { turn.conclude(TurnOutcome::Completed(response.content.clone())); @@ -739,7 +740,7 @@ pub struct Turn { /// Not serialized — images are only needed for the current LLM call. /// The text description in `user_input` persists for compaction/context. #[serde(skip)] - pub image_content_parts: Vec<crate::llm::ContentPart>, + pub image_content_parts: Vec<ironclaw_llm::ContentPart>, } impl Turn { @@ -1472,19 +1473,19 @@ mod tests { // user + assistant_with_tool_calls + tool_result + assistant = 4 assert_eq!(messages.len(), 4); - assert_eq!(messages[0].role, crate::llm::Role::User); + assert_eq!(messages[0].role, ironclaw_llm::Role::User); assert_eq!(messages[0].content, "Search for X"); - assert_eq!(messages[1].role, crate::llm::Role::Assistant); + assert_eq!(messages[1].role, ironclaw_llm::Role::Assistant); assert!(messages[1].tool_calls.is_some()); let tcs = messages[1].tool_calls.as_ref().unwrap(); assert_eq!(tcs.len(), 1); assert_eq!(tcs[0].name, "memory_search"); - assert_eq!(messages[2].role, crate::llm::Role::Tool); + assert_eq!(messages[2].role, ironclaw_llm::Role::Tool); assert!(messages[2].content.contains("Found X")); - assert_eq!(messages[3].role, crate::llm::Role::Assistant); + assert_eq!(messages[3].role, ironclaw_llm::Role::Assistant); assert_eq!(messages[3].content, "I found X in doc.md."); } @@ -1525,6 +1526,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"q": "test"}), reasoning: None, + signature: None, }; let messages = vec![ ChatMessage::user("Find test"), @@ -1556,6 +1558,7 @@ mod tests { name: "http".to_string(), arguments: serde_json::json!({}), reasoning: None, + signature: None, }; let messages = vec![ ChatMessage::user("Fetch URL"), @@ -1622,12 +1625,14 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"q": "data"}), reasoning: None, + signature: None, }; let tc2 = ToolCall { id: "call_b".to_string(), name: "write".to_string(), arguments: serde_json::json!({"path": "out.txt"}), reasoning: None, + signature: None, }; let messages = vec![ ChatMessage::user("Find and save"), diff --git a/src/agent/submission.rs b/src/agent/submission.rs index 31210af98c5..b43440b71aa 100644 --- a/src/agent/submission.rs +++ b/src/agent/submission.rs @@ -155,6 +155,31 @@ impl SubmissionParser { } } + // `approve <channel> <code>` — claim a pairing code from the chat + // surface. The Telegram bot's pairing reply tells users to type + // exactly this; supporting it in chat closes the gap that #3317 + // surfaced (users pasted the pairing code into TUI/CLI and got + // rejected by the LLM). The slash variant is also accepted so the + // command can't be intercepted as a UserInput at higher layers. + if let Some(rest) = lower + .strip_prefix("approve ") + .or_else(|| lower.strip_prefix("/approve ")) + { + let mut parts = rest.split_whitespace(); + if let (Some(channel), Some(code), None) = (parts.next(), parts.next(), parts.next()) { + let channel = channel.to_string(); + // The parser already lowercased the input (`lower`), so both + // `channel` and `code` flow through in lowercase. The store + // layer is case-insensitive + // (`db.libsql.pairing` test_approve_pairing_case_insensitive), + // so the on-the-wire upper-case shape still matches. + let code = code.to_string(); + if !channel.is_empty() && !code.is_empty() { + return Submission::PairingClaim { channel, code }; + } + } + } + // /thread <uuid> - switch thread if let Some(rest) = lower.strip_prefix("/thread ") { let rest = rest.trim(); @@ -302,10 +327,20 @@ pub enum Submission { always: bool, }, - /// External system resolved a pending gate (for example an OAuth callback). + /// External system resolved a pending gate (for example an OAuth + /// callback or a caller-executed tool result). ExternalCallback { /// ID of the pending gate request being resolved. request_id: Uuid, + /// Optional payload supplied alongside the resolution. OAuth-style + /// callbacks (the original use case) pass `None`; caller-executed + /// external tool calls in the Responses API pass a JSON object + /// describing the tool outputs. + /// + /// Defaults to `None` on the wire so existing OAuth callers don't + /// need to be updated. + #[serde(default)] + payload: Option<serde_json::Value>, }, /// Resolve an authentication gate with an exact request id. @@ -404,6 +439,21 @@ pub enum Submission { /// The plan subcommand. sub: PlanSubcommand, }, + + /// Claim a pairing code from any chat surface (e.g. `approve telegram CODE`). + /// + /// The user-facing pairing flow tells users to type exactly this in any + /// IronClaw chat. The handler delegates to the same pairing store and + /// extension-manager hooks that `POST /api/pairing/{channel}/approve` + /// uses, so the TUI/CLI/web/telegram surfaces all behave consistently. + PairingClaim { + /// Channel name (e.g. `telegram`, `slack-relay`). + channel: String, + /// Pairing code, lowercased by `SubmissionParser::parse`. The + /// pairing store is case-insensitive, so the wire-format + /// uppercase shape still matches. + code: String, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -515,6 +565,8 @@ pub enum SubmissionResult { Response { /// The agent's response. content: String, + /// File paths to deliver with the response. + attachments: Vec<String>, }, /// Need approval before continuing. @@ -555,6 +607,15 @@ impl SubmissionResult { pub fn response(content: impl Into<String>) -> Self { Self::Response { content: content.into(), + attachments: Vec::new(), + } + } + + /// Create a response result with file attachments. + pub fn response_with_attachments(content: impl Into<String>, attachments: Vec<String>) -> Self { + Self::Response { + content: content.into(), + attachments, } } @@ -1046,4 +1107,56 @@ mod tests { let submission = SubmissionParser::parse("/expected "); assert!(matches!(submission, Submission::UserInput { .. })); } + + // Pairing-claim parser — regression tests for #3317. The Telegram bot's + // pairing reply tells users to type `approve telegram CODE` in any + // IronClaw chat surface; before this variant existed, the agent + // unhelpfully routed those to the LLM and answered "wrong place". + #[test] + fn test_parser_pairing_claim_bare() { + let submission = SubmissionParser::parse("approve telegram ABC12345"); + assert!(matches!( + submission, + Submission::PairingClaim { ref channel, ref code } + if channel == "telegram" && code == "abc12345" + )); + } + + #[test] + fn test_parser_pairing_claim_slash_prefix() { + let submission = SubmissionParser::parse("/approve telegram XYZ99"); + assert!(matches!( + submission, + Submission::PairingClaim { ref channel, ref code } + if channel == "telegram" && code == "xyz99" + )); + } + + #[test] + fn test_parser_pairing_claim_mixed_case_channel() { + // Channel name should round-trip lowercased (the parser uses `lower`). + let submission = SubmissionParser::parse("approve Telegram ABCD1234"); + assert!(matches!( + submission, + Submission::PairingClaim { ref channel, .. } if channel == "telegram" + )); + } + + #[test] + fn test_parser_pairing_claim_extra_args_falls_through() { + // Don't claim `approve foo bar baz` — extra tokens look like a sentence. + let submission = SubmissionParser::parse("approve telegram ABC EXTRA"); + assert!(matches!(submission, Submission::UserInput { .. })); + } + + #[test] + fn test_parser_approval_alone_is_still_approval() { + // `approve` by itself stays an ApprovalResponse; only the two-arg + // form is interpreted as a pairing claim. + let submission = SubmissionParser::parse("approve"); + assert!(matches!( + submission, + Submission::ApprovalResponse { approved: true, .. } + )); + } } diff --git a/src/agent/thread_ops.rs b/src/agent/thread_ops.rs index 6876d30388f..7549b4d8b7d 100644 --- a/src/agent/thread_ops.rs +++ b/src/agent/thread_ops.rs @@ -18,16 +18,17 @@ use crate::agent::dispatcher::{ persist_selected_auth_prompt, restore_selected_auth_prompt, }; use crate::agent::session::{ - MAX_PENDING_MESSAGES, PendingApproval, Session, ThreadState, TurnOutcome, + MAX_PENDING_MESSAGES, PendingApproval, Session, ThreadState, TurnOutcome, TurnToolCall, }; use crate::agent::submission::SubmissionResult; use crate::channels::{ChatApprovalPrompt, HistoryMessage, IncomingMessage, StatusUpdate}; use crate::context::JobContext; use crate::error::Error; -use crate::generated_images::GeneratedImageSentinel; -use crate::llm::{ChatMessage, ToolCall}; +use crate::generated_images::{GeneratedImageSentinel, stage_generated_image_data_url}; use crate::tools::redact_params; use ironclaw_common::truncate_preview; +use ironclaw_llm::{ChatMessage, ToolCall}; +use ironclaw_safety::{PolicyAction, SafetyLayer, ValidationResult}; const FORGED_THREAD_ID_ERROR: &str = "Invalid or unauthorized thread ID."; const LIVE_STATE_METADATA_KEY: &str = "live_state"; @@ -49,6 +50,43 @@ fn tool_result_preview_for_persistence(result: &serde_json::Value) -> String { } } +fn stage_generated_image_response_attachments(tool_calls: &[TurnToolCall]) -> Vec<String> { + let mut attachments = Vec::new(); + for call in tool_calls { + if !matches!(call.name.as_str(), "image_generate" | "image_edit") { + continue; + } + let Some(result) = call.result.as_ref() else { + continue; + }; + let Some(sentinel) = GeneratedImageSentinel::from_value(result) else { + continue; + }; + let Some(data_url) = sentinel.data_url().filter(|value| !value.is_empty()) else { + continue; + }; + match stage_generated_image_data_url(data_url) { + Ok(path) => attachments.push(path), + Err(error) => tracing::warn!( + tool = %call.name, + error = %error, + "Failed to stage generated image as response attachment" + ), + } + } + attachments +} + +fn strip_markdown_image_lines(content: &str) -> String { + content + .lines() + .filter(|line| !line.trim_start().starts_with("![")) + .collect::<Vec<_>>() + .join("\n") + .trim() + .to_string() +} + fn tool_result_content_for_persistence(result: &serde_json::Value) -> String { if let Some(sentinel) = GeneratedImageSentinel::from_value(result) { // Persist the full image sentinel so web history can reconstruct the @@ -78,6 +116,10 @@ fn requires_preexisting_uuid_thread(channel: &str) -> bool { matches!(channel, "gateway" | "test") } +fn validate_inbound_text_for_message(safety: &SafetyLayer, content: &str) -> ValidationResult { + safety.validate_input(content) +} + fn auth_retry_message_for_error(error: &crate::extensions::ExtensionError) -> Option<String> { matches!( error, @@ -200,6 +242,44 @@ fn pending_approval_message(pending: Option<&PendingApproval>) -> String { } impl Agent { + fn reject_unsafe_inbound_user_message( + &self, + message: &IncomingMessage, + effective_content: &str, + ) -> Option<SubmissionResult> { + let validation = validate_inbound_text_for_message(self.safety(), effective_content); + if !validation.is_valid { + let details = validation + .errors + .iter() + .map(|e| format!("{}: {}", e.field, e.message)) + .collect::<Vec<_>>() + .join("; "); + return Some(SubmissionResult::error(format!( + "Input rejected by safety validation: {details}", + ))); + } + + let violations = self.safety().check_policy(effective_content); + if violations + .iter() + .any(|rule| rule.action == PolicyAction::Block) + { + return Some(SubmissionResult::error("Input rejected by safety policy.")); + } + + if let Some(warning) = self.safety().scan_inbound_for_secrets(effective_content) { + tracing::warn!( + user = %message.user_id, + channel = %message.channel, + "Inbound message blocked: contains leaked secret" + ); + return Some(SubmissionResult::error(warning)); + } + + None + } + /// Hydrate a historical thread from DB into memory if not already present. /// /// Called before `resolve_thread` so that the session manager finds the @@ -428,6 +508,13 @@ impl Agent { "Processing user input" ); + let augmented = + crate::agent::attachments::augment_with_attachments(content, &message.attachments); + let (effective_content, image_parts) = match &augmented { + Some(result) => (result.text.as_str(), result.image_parts.clone()), + None => (content, Vec::new()), + }; + // First check thread state without holding lock during I/O let (thread_state, pending_approval) = { let sess = session.lock().await; @@ -464,36 +551,11 @@ impl Agent { } // Run the same safety checks that the normal path applies - // (validation, policy, secret scan) so that blocked content - // is never stored in pending_messages or serialized. - let validation = self.safety().validate_input(content); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::<Vec<_>>() - .join("; "); - return Ok(SubmissionResult::error(format!( - "Input rejected by safety validation: {details}", - ))); - } - let violations = self.safety().check_policy(content); - if violations - .iter() - .any(|rule| rule.action == ironclaw_safety::PolicyAction::Block) + // so blocked content is never stored in pending_messages. + if let Some(rejection) = + self.reject_unsafe_inbound_user_message(message, effective_content) { - return Ok(SubmissionResult::error( - "Input rejected by safety policy.", - )); - } - if let Some(warning) = self.safety().scan_inbound_for_secrets(content) { - tracing::warn!( - user = %message.user_id, - channel = %message.channel, - "Queued message blocked: contains leaked secret" - ); - return Ok(SubmissionResult::error(warning)); + return Ok(rejection); } if !thread.queue_message(content.to_string()) { @@ -555,50 +617,12 @@ impl Agent { } } - // Attachments can carry the only user-visible payload (for example, - // a files-only send with empty chat text), so validation and policy - // checks must run against the augmented content that will actually - // enter the turn rather than the raw text field alone. - let augmented = - crate::agent::attachments::augment_with_attachments(content, &message.attachments); - let (effective_content, image_parts) = match &augmented { - Some(result) => (result.text.as_str(), result.image_parts.clone()), - None => (content, Vec::new()), - }; - - // Safety validation for user input - let validation = self.safety().validate_input(effective_content); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::<Vec<_>>() - .join("; "); - return Ok(SubmissionResult::error(format!( - "Input rejected by safety validation: {}", - details - ))); - } - - let violations = self.safety().check_policy(effective_content); - if violations - .iter() - .any(|rule| rule.action == ironclaw_safety::PolicyAction::Block) + // Validate inbound content before the turn is created. Attachment-only + // messages are checked after attachment augmentation so extracted text + // and multimodal metadata go through the same safety pipeline. + if let Some(rejection) = self.reject_unsafe_inbound_user_message(message, effective_content) { - return Ok(SubmissionResult::error("Input rejected by safety policy.")); - } - - // Scan inbound messages for secrets (API keys, tokens). - // Catching them here prevents the LLM from echoing them back, which - // would trigger the outbound leak detector and create error loops. - if let Some(warning) = self.safety().scan_inbound_for_secrets(effective_content) { - tracing::warn!( - user = %message.user_id, - channel = %message.channel, - "Inbound message blocked: contains leaked secret" - ); - return Ok(SubmissionResult::error(warning)); + return Ok(rejection); } // Handle explicit commands (starting with /) directly @@ -643,8 +667,9 @@ impl Agent { .await; let compactor = ContextCompactor::new(self.llm().clone()); + let workspace = self.workspace_for_user(&message.user_id); if let Err(e) = compactor - .compact(thread, strategy, self.workspace().map(|w| w.as_ref())) + .compact(thread, strategy, workspace.as_deref()) .await { tracing::warn!("Auto-compaction failed: {}", e); @@ -774,7 +799,8 @@ impl Agent { crate::agent::dispatcher::extract_suggestions(&response); // Hook: TransformResponse — allow hooks to modify or reject the final response - let response = { + let mut response_attachments_allowed = true; + let mut response = { let event = crate::hooks::HookEvent::ResponseTransform { user_id: message.user_id.clone(), thread_id: thread_id.to_string(), @@ -782,9 +808,11 @@ impl Agent { }; match self.hooks().run(&event).await { Err(crate::hooks::HookError::Rejected { reason }) => { + response_attachments_allowed = false; format!("[Response filtered: {}]", reason) } Err(err) => { + response_attachments_allowed = false; format!("[Response blocked by hook policy: {}]", err) } Ok(crate::hooks::HookOutcome::Continue { @@ -794,6 +822,20 @@ impl Agent { } }; + let response_attachments = if response_attachments_allowed { + let current_tool_calls = thread + .turns + .last() + .map(|turn| turn.tool_calls.clone()) + .unwrap_or_default(); + stage_generated_image_response_attachments(&current_tool_calls) + } else { + Vec::new() + }; + if !response_attachments.is_empty() { + response = strip_markdown_image_lines(&response); + } + thread.conclude_turn(TurnOutcome::Completed(response.clone())); let (turn_number, tool_calls, narrative) = thread .turns @@ -837,7 +879,10 @@ impl Agent { self.send_turn_cost_status(&message.channel, &message.metadata, &turn_usage) .await; - Ok(SubmissionResult::response(response)) + Ok(SubmissionResult::response_with_attachments( + response, + response_attachments, + )) } Ok(AgenticLoopResult::NeedApproval { pending, @@ -1386,6 +1431,7 @@ impl Agent { thread_id: Uuid, ) -> Result<SubmissionResult, Error> { let mut sess = session.lock().await; + let user_id = sess.user_id.clone(); let thread = sess .threads .get_mut(&thread_id) @@ -1401,8 +1447,9 @@ impl Agent { ); let compactor = ContextCompactor::new(self.llm().clone()); + let workspace = self.workspace_for_user(&user_id); match compactor - .compact(thread, strategy, self.workspace().map(|w| w.as_ref())) + .compact(thread, strategy, workspace.as_deref()) .await { Ok(result) => { @@ -1445,6 +1492,26 @@ impl Agent { let undo_mgr = self.session_manager.get_undo_manager(thread_id).await; undo_mgr.lock().await.clear(); + // Drain in-flight OAuth flows for this user (#3320). Mirror of the + // engine-v2 cleanup in `bridge::router::clear_engine_conversation` so + // both code paths give `/clear` a consistent "clean slate" contract. + // Pending flows otherwise live until `OAUTH_FLOW_EXPIRY` (5 min) and + // can fool a fresh auth attempt's CSRF dedupe or leave a ghost entry + // in `pending_oauth_flows()` after the user already moved on. + if let Some(ext_mgr) = self.deps.extension_manager.as_ref() { + let mut flows = ext_mgr.pending_oauth_flows().write().await; + let before = flows.len(); + flows.retain(|_state, flow| flow.user_id != user_id); + let removed = before.saturating_sub(flows.len()); + if removed > 0 { + tracing::debug!( + user_id = %user_id, + removed, + "engine v1: drained pending OAuth flows on /clear" + ); + } + } + Ok(SubmissionResult::ok_with_message("Thread cleared.")) } @@ -1747,10 +1814,10 @@ impl Agent { // === Phase 1: Preflight (sequential) === // Walk deferred tools checking approval. Collect runnable // tools; stop at the first that needs approval. - let mut runnable: Vec<crate::llm::ToolCall> = Vec::new(); + let mut runnable: Vec<ironclaw_llm::ToolCall> = Vec::new(); let mut approval_needed: Option<( usize, - crate::llm::ToolCall, + ironclaw_llm::ToolCall, Arc<dyn crate::tools::Tool>, bool, // allow_always )> = None; @@ -1785,7 +1852,8 @@ impl Agent { } // === Phase 2: Parallel execution === - let exec_results: Vec<(crate::llm::ToolCall, Result<String, Error>)> = if runnable.len() + let exec_results: Vec<(ironclaw_llm::ToolCall, Result<String, Error>)> = if runnable + .len() <= 1 { // Single tool (or none): execute inline @@ -1889,7 +1957,7 @@ impl Agent { } // Collect and reorder by original index - let mut ordered: Vec<Option<(crate::llm::ToolCall, Result<String, Error>)>> = + let mut ordered: Vec<Option<(ironclaw_llm::ToolCall, Result<String, Error>)>> = (0..runnable_count).map(|_| None).collect(); while let Some(join_result) = join_set.join_next().await { match join_result { @@ -2732,6 +2800,7 @@ fn rebuild_chat_messages_from_db( .get("rationale") .and_then(|v| v.as_str()) .map(String::from), + signature: None, }) .collect(); @@ -2776,17 +2845,22 @@ fn rebuild_chat_messages_from_db( #[cfg(test)] mod tests { use super::*; + use crate::channels::{AttachmentKind, IncomingAttachment}; + use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; use std::sync::Mutex as StdMutex; + use std::sync::atomic::{AtomicU32, Ordering}; use crate::agent::AgentDeps; use crate::agent::cost_guard::{CostGuard, CostGuardConfig}; use crate::channels::{ Channel, ChannelManager, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate, }; - use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; use crate::context::ContextManager; use crate::error::ChannelError; - use crate::generated_images::{GeneratedImageSentinel, MAX_RECORDED_IMAGE_SENTINEL_BYTES}; + use crate::generated_images::{ + GeneratedImageSentinel, MAX_RECORDED_IMAGE_SENTINEL_BYTES, + remove_staged_generated_image_attachments, + }; use crate::hooks::HookRegistry; use crate::testing::{StubChannel, StubLlm}; use crate::tools::ToolRegistry; @@ -2839,7 +2913,7 @@ mod tests { struct StaticLlmProvider; #[async_trait::async_trait] - impl crate::llm::LlmProvider for StaticLlmProvider { + impl ironclaw_llm::LlmProvider for StaticLlmProvider { fn model_name(&self) -> &str { "static-mock" } @@ -2850,13 +2924,13 @@ mod tests { async fn complete( &self, - _request: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, crate::error::LlmError> { - Ok(crate::llm::CompletionResponse { + _request: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, crate::error::LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: "ok".to_string(), input_tokens: 0, output_tokens: 0, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -2864,20 +2938,29 @@ mod tests { async fn complete_with_tools( &self, - _request: crate::llm::ToolCompletionRequest, - ) -> Result<crate::llm::ToolCompletionResponse, crate::error::LlmError> { - Ok(crate::llm::ToolCompletionResponse { + _request: ironclaw_llm::ToolCompletionRequest, + ) -> Result<ironclaw_llm::ToolCompletionResponse, crate::error::LlmError> { + Ok(ironclaw_llm::ToolCompletionResponse { content: Some("ok".to_string()), tool_calls: Vec::new(), input_tokens: 0, output_tokens: 0, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } + let llm: Arc<dyn ironclaw_llm::LlmProvider> = Arc::new(StaticLlmProvider); + make_thread_ops_test_agent_with(llm, Arc::new(crate::tools::ToolRegistry::new())).await + } + + async fn make_thread_ops_test_agent_with( + llm: Arc<dyn ironclaw_llm::LlmProvider>, + tools: Arc<crate::tools::ToolRegistry>, + ) -> (Agent, Arc<TokioMutex<Vec<StatusUpdate>>>) { let statuses = Arc::new(TokioMutex::new(Vec::new())); let channels = Arc::new(crate::channels::ChannelManager::new()); channels @@ -2890,7 +2973,7 @@ mod tests { owner_id: "default".to_string(), store: None, settings_store: None, - llm: Arc::new(StaticLlmProvider), + llm, cheap_llm: None, safety: Arc::new(ironclaw_safety::SafetyLayer::new( &ironclaw_safety::SafetyConfig { @@ -2898,7 +2981,7 @@ mod tests { injection_check_enabled: true, }, )), - tools: Arc::new(crate::tools::ToolRegistry::new()), + tools, workspace: None, extension_manager: None, skill_registry: None, @@ -2955,6 +3038,179 @@ mod tests { (agent, statuses) } + const TEST_IMAGE_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="; + + struct SequencedImageLlm { + calls: AtomicU32, + } + + impl SequencedImageLlm { + fn new() -> Self { + Self { + calls: AtomicU32::new(0), + } + } + } + + #[async_trait::async_trait] + impl ironclaw_llm::LlmProvider for SequencedImageLlm { + fn model_name(&self) -> &str { + "sequenced-image-mock" + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete( + &self, + _request: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, crate::error::LlmError> { + Ok(ironclaw_llm::CompletionResponse { + content: "unused".to_string(), + input_tokens: 0, + output_tokens: 0, + finish_reason: ironclaw_llm::FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + _request: ironclaw_llm::ToolCompletionRequest, + ) -> Result<ironclaw_llm::ToolCompletionResponse, crate::error::LlmError> { + let call = self.calls.fetch_add(1, Ordering::Relaxed); + if call == 0 { + return Ok(ironclaw_llm::ToolCompletionResponse { + content: None, + tool_calls: vec![ToolCall { + id: "call_img_0".to_string(), + name: "image_generate".to_string(), + arguments: serde_json::json!({"prompt": "cat"}), + reasoning: None, + signature: None, + }], + input_tokens: 0, + output_tokens: 0, + finish_reason: ironclaw_llm::FinishReason::ToolUse, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + reasoning: None, + }); + } + + Ok(ironclaw_llm::ToolCompletionResponse { + content: Some( + "已生成小猫图片:\n\n![小猫](/mnt/data/generated_image.jpg)".to_string(), + ), + tool_calls: Vec::new(), + input_tokens: 0, + output_tokens: 0, + finish_reason: ironclaw_llm::FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + reasoning: None, + }) + } + } + + struct GeneratedImageTool; + + #[async_trait::async_trait] + impl crate::tools::Tool for GeneratedImageTool { + fn name(&self) -> &str { + "image_generate" + } + + fn description(&self) -> &str { + "Generate an image for tests" + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "prompt": {"type": "string"} + }, + "required": ["prompt"] + }) + } + + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &JobContext, + ) -> Result<crate::tools::ToolOutput, crate::tools::ToolError> { + Ok(crate::tools::ToolOutput::success( + serde_json::json!({ + "type": "image_generated", + "data": TEST_IMAGE_DATA_URL, + "media_type": "image/png" + }), + Duration::from_millis(1), + )) + } + } + + #[cfg(feature = "libsql")] + #[tokio::test] + async fn process_compact_writes_summary_to_session_user_workspace() { + let (db, _dir) = crate::agent::test_support::make_libsql_test_db().await; + let owner_workspace = Arc::new(crate::workspace::Workspace::new_with_db( + "owner-scope", + Arc::clone(&db), + )); + let (mut agent, _statuses) = make_thread_ops_test_agent().await; + agent.deps.store = Some(Arc::clone(&db)); + agent.deps.workspace = Some(owner_workspace); + + let session = Arc::new(Mutex::new(Session::new("alice"))); + let thread_id = { + let mut sess = session.lock().await; + let thread = sess.create_thread(Some("gateway")); + for i in 0..6 { + thread.start_turn(format!("msg-{i}")); + thread.conclude_turn(TurnOutcome::Completed(format!("resp-{i}"))); + } + thread.id + }; + + let before_date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + let result = agent + .process_compact(Arc::clone(&session), thread_id) + .await + .expect("manual compaction should run"); + let after_date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + assert!( + matches!(result, SubmissionResult::Ok { .. }), + "unexpected compact result: {result:?}" + ); + + let mut candidate_paths = vec![format!("daily/{before_date}.md")]; + if after_date != before_date { + candidate_paths.push(format!("daily/{after_date}.md")); + } + + let alice_ws = crate::workspace::Workspace::new_with_db("alice", Arc::clone(&db)); + let mut stored = None; + for path in &candidate_paths { + if let Ok(doc) = alice_ws.read(path).await { + stored = Some((path.clone(), doc)); + break; + } + } + let (path, alice_doc) = + stored.expect("compaction summary should be written to the session user's workspace"); + assert!(alice_doc.content.contains("Context Summary")); + + let owner_ws = crate::workspace::Workspace::new_with_db("owner-scope", Arc::clone(&db)); + assert!( + owner_ws.read(&path).await.is_err(), + "compaction summary must not be written to the startup owner workspace" + ); + } + #[test] fn thread_summaries_are_sorted_by_last_activity_descending() { let conversations = vec![ @@ -3014,8 +3270,8 @@ mod tests { ]; let result = rebuild_chat_messages_from_db(&messages); assert_eq!(result.len(), 2); - assert_eq!(result[0].role, crate::llm::Role::User); - assert_eq!(result[1].role, crate::llm::Role::Assistant); + assert_eq!(result[0].role, ironclaw_llm::Role::User); + assert_eq!(result[1].role, ironclaw_llm::Role::Assistant); } /// Regression: a `PendingApproval` deserialized from a row written @@ -3091,7 +3347,7 @@ mod tests { let result = Ok(AgenticLoopResult::Response { text: "done".to_string(), turn_usage: TurnUsageSummary { - usage: crate::llm::TokenUsage { + usage: ironclaw_llm::TokenUsage { input_tokens: 12, output_tokens: 3, cache_read_input_tokens: 0, @@ -3115,7 +3371,7 @@ mod tests { } .into(), turn_usage: TurnUsageSummary { - usage: crate::llm::TokenUsage { + usage: ironclaw_llm::TokenUsage { input_tokens: 7, output_tokens: 2, cache_read_input_tokens: 0, @@ -3169,10 +3425,10 @@ mod tests { assert_eq!(result.len(), 5); // user - assert_eq!(result[0].role, crate::llm::Role::User); + assert_eq!(result[0].role, ironclaw_llm::Role::User); // assistant with tool_calls - assert_eq!(result[1].role, crate::llm::Role::Assistant); + assert_eq!(result[1].role, ironclaw_llm::Role::Assistant); assert!(result[1].tool_calls.is_some()); let tcs = result[1].tool_calls.as_ref().unwrap(); assert_eq!(tcs.len(), 2); @@ -3181,16 +3437,16 @@ mod tests { assert_eq!(tcs[1].name, "echo"); // tool results - assert_eq!(result[2].role, crate::llm::Role::Tool); + assert_eq!(result[2].role, ironclaw_llm::Role::Tool); assert_eq!(result[2].tool_call_id, Some("call_0".to_string())); assert!(result[2].content.contains("Found 3 results")); - assert_eq!(result[3].role, crate::llm::Role::Tool); + assert_eq!(result[3].role, ironclaw_llm::Role::Tool); assert_eq!(result[3].tool_call_id, Some("call_1".to_string())); assert!(result[3].content.contains("timeout")); // final assistant - assert_eq!(result[4].role, crate::llm::Role::Assistant); + assert_eq!(result[4].role, ironclaw_llm::Role::Assistant); assert_eq!(result[4].content, "I found some results."); } @@ -3214,7 +3470,7 @@ mod tests { let result = rebuild_chat_messages_from_db(&messages); assert_eq!(result.len(), 3); - assert_eq!(result[2].role, crate::llm::Role::Tool); + assert_eq!(result[2].role, ironclaw_llm::Role::Tool); assert_eq!(result[2].tool_call_id, Some("call_1".to_string())); assert_eq!(result[2].content, wrapped_error); } @@ -3234,8 +3490,8 @@ mod tests { // Legacy rows are skipped, only user + assistant assert_eq!(result.len(), 2); - assert_eq!(result[0].role, crate::llm::Role::User); - assert_eq!(result[1].role, crate::llm::Role::Assistant); + assert_eq!(result[0].role, ironclaw_llm::Role::User); + assert_eq!(result[1].role, ironclaw_llm::Role::Assistant); } #[test] @@ -3281,15 +3537,57 @@ mod tests { // Verify turn boundaries assert_eq!(result[0].content, "Find X"); assert!(result[1].tool_calls.is_some()); - assert_eq!(result[2].role, crate::llm::Role::Tool); + assert_eq!(result[2].role, ironclaw_llm::Role::Tool); assert_eq!(result[3].content, "Found X"); assert_eq!(result[4].content, "Write it"); assert!(result[5].tool_calls.is_some()); - assert_eq!(result[6].role, crate::llm::Role::Tool); + assert_eq!(result[6].role, ironclaw_llm::Role::Tool); assert_eq!(result[7].content, "Written"); } + #[test] + fn test_validate_inbound_text_rejects_empty_text_without_attachments() { + let safety = SafetyLayer::new(&SafetyConfig { + max_output_length: 10_000, + injection_check_enabled: true, + }); + + let result = validate_inbound_text_for_message(&safety, ""); + assert!(!result.is_valid); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].field, "input"); + assert_eq!(result.errors[0].message, "Input cannot be empty"); + } + + #[test] + fn test_validate_inbound_text_accepts_augmented_attachment_content() { + let safety = SafetyLayer::new(&SafetyConfig { + max_output_length: 10_000, + injection_check_enabled: true, + }); + + let attachments = vec![IncomingAttachment { + id: "image-1".to_string(), + kind: AttachmentKind::Image, + mime_type: "image/jpeg".to_string(), + filename: Some("photo.jpg".to_string()), + size_bytes: Some(128), + source_url: Some("https://example.com/photo.jpg".to_string()), + storage_key: None, + local_path: None, + extracted_text: None, + data: vec![1, 2, 3], + duration_secs: None, + }]; + + let augmented = crate::agent::attachments::augment_with_attachments("", &attachments) + .expect("attachments should augment content"); + let result = validate_inbound_text_for_message(&safety, &augmented.text); + assert!(result.is_valid); + assert!(result.errors.is_empty()); + } + #[test] fn test_rebuild_chat_messages_summarizes_image_generated_result() { let tool_json = serde_json::json!([ @@ -3314,10 +3612,21 @@ mod tests { let result = rebuild_chat_messages_from_db(&messages); assert_eq!(result.len(), 4); - assert_eq!(result[2].role, crate::llm::Role::Tool); + assert_eq!(result[2].role, ironclaw_llm::Role::Tool); assert_eq!(result[2].content, "Generated image (image/jpeg)"); } + #[test] + fn test_strip_markdown_image_lines_can_return_empty_text() { + let content = "![generated](/mnt/data/image.png)"; + + assert_eq!(strip_markdown_image_lines(content), ""); + assert_eq!( + strip_markdown_image_lines("已生成:\n\n![generated](/mnt/data/image.png)"), + "已生成:" + ); + } + #[test] fn test_tool_result_preview_for_persistence_handles_double_stringified_sentinel() { let sentinel = serde_json::json!({ @@ -3702,7 +4011,7 @@ mod tests { .expect("attachment-only message handled"); match result { - SubmissionResult::Response { content } => { + SubmissionResult::Response { content, .. } => { assert_eq!(content.to_ascii_lowercase(), "ok") } other => panic!("expected response result, got {other:?}"), @@ -3729,6 +4038,136 @@ mod tests { ); } + #[tokio::test] + async fn test_process_user_input_attaches_generated_image_from_tool_result() { + use crate::agent::session::{Session, Thread}; + use uuid::Uuid; + + let tools = Arc::new(ToolRegistry::new()); + tools.register(Arc::new(GeneratedImageTool)).await; + let llm: Arc<dyn ironclaw_llm::LlmProvider> = Arc::new(SequencedImageLlm::new()); + let (agent, statuses) = make_thread_ops_test_agent_with(llm, tools).await; + let session_id = Uuid::new_v4(); + let thread_id = Uuid::new_v4(); + let thread = Thread::with_id(thread_id, session_id, Some("test")); + + let mut sess = Session::new("test-user"); + sess.threads.insert(thread_id, thread); + let session = Arc::new(TokioMutex::new(sess)); + let message = IncomingMessage::new("test", "test-user", "生成一张小猫图片"); + + let result = agent + .process_user_input( + &message, + agent.tenant_ctx("test-user").await, + Arc::clone(&session), + thread_id, + "生成一张小猫图片", + ) + .await + .expect("image generation turn handled"); + + let attachments = match result { + SubmissionResult::Response { + content, + attachments, + } => { + assert_eq!(content, "已生成小猫图片:"); + assert!(!content.contains("![")); + attachments + } + other => panic!("expected response result, got {other:?}"), + }; + + assert_eq!(attachments.len(), 1); + assert!( + std::path::Path::new(&attachments[0]).exists(), + "staged generated image attachment should exist" + ); + remove_staged_generated_image_attachments(&attachments); + + let statuses = statuses.lock().await.clone(); + assert!(statuses.iter().any(|status| matches!( + status, + StatusUpdate::ImageGenerated { + event_id, + data_url, + .. + } if event_id == "call_img_0" && data_url == TEST_IMAGE_DATA_URL + ))); + } + + #[tokio::test] + async fn test_response_transform_rejection_blocks_generated_image_attachment() { + use crate::agent::session::{Session, Thread}; + use crate::hooks::{Hook, HookContext, HookError, HookEvent, HookOutcome, HookPoint}; + use uuid::Uuid; + + struct RejectGeneratedImageResponseHook; + + #[async_trait::async_trait] + impl Hook for RejectGeneratedImageResponseHook { + fn name(&self) -> &str { + "reject-generated-image-response" + } + + fn hook_points(&self) -> &[HookPoint] { + const POINTS: &[HookPoint] = &[HookPoint::TransformResponse]; + POINTS + } + + async fn execute( + &self, + _event: &HookEvent, + _ctx: &HookContext, + ) -> Result<HookOutcome, HookError> { + Ok(HookOutcome::reject("blocked by response policy")) + } + } + + let tools = Arc::new(ToolRegistry::new()); + tools.register(Arc::new(GeneratedImageTool)).await; + let llm: Arc<dyn ironclaw_llm::LlmProvider> = Arc::new(SequencedImageLlm::new()); + let (agent, _statuses) = make_thread_ops_test_agent_with(llm, tools).await; + agent + .hooks() + .register(Arc::new(RejectGeneratedImageResponseHook)) + .await; + let session_id = Uuid::new_v4(); + let thread_id = Uuid::new_v4(); + let thread = Thread::with_id(thread_id, session_id, Some("test")); + + let mut sess = Session::new("test-user"); + sess.threads.insert(thread_id, thread); + let session = Arc::new(TokioMutex::new(sess)); + let message = IncomingMessage::new("test", "test-user", "生成一张小猫图片"); + + let result = agent + .process_user_input( + &message, + agent.tenant_ctx("test-user").await, + Arc::clone(&session), + thread_id, + "生成一张小猫图片", + ) + .await + .expect("image generation turn handled"); + + match result { + SubmissionResult::Response { + content, + attachments, + } => { + assert_eq!(content, "[Response filtered: blocked by response policy]"); + assert!( + attachments.is_empty(), + "rejected response must not deliver generated image attachments" + ); + } + other => panic!("expected response result, got {other:?}"), + } + } + #[tokio::test] async fn test_switch_thread_emits_history_with_pending_approval() { use crate::agent::session::{PendingApproval, Thread}; diff --git a/src/agent/undo.rs b/src/agent/undo.rs index 10ab89d5b38..43e01c40501 100644 --- a/src/agent/undo.rs +++ b/src/agent/undo.rs @@ -8,7 +8,7 @@ use std::collections::VecDeque; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::llm::ChatMessage; +use ironclaw_llm::ChatMessage; /// Maximum number of checkpoints to keep by default. const DEFAULT_MAX_CHECKPOINTS: usize = 20; diff --git a/src/app.rs b/src/app.rs index 22a76b2de7f..8dbe1149911 100644 --- a/src/app.rs +++ b/src/app.rs @@ -16,14 +16,15 @@ use crate::context::ContextManager; use crate::db::{Database, UserStore}; use crate::extensions::ExtensionManager; use crate::hooks::HookRegistry; -use crate::llm::recording::HttpInterceptor; -use crate::llm::{LlmProvider, LlmReloadHandle, RecordingLlm, SessionManager}; use crate::secrets::SecretsStore; use crate::tools::ToolRegistry; use crate::tools::mcp::{McpProcessManager, McpSessionManager}; use crate::tools::wasm::SharedCredentialRegistry; use crate::tools::wasm::WasmToolRuntime; -use crate::workspace::{EmbeddingCacheConfig, EmbeddingProvider, Workspace}; +use crate::workspace::Workspace; +use ironclaw_embeddings::{EmbeddingCacheConfig, EmbeddingProvider}; +use ironclaw_llm::recording::HttpInterceptor; +use ironclaw_llm::{LlmProvider, LlmReloadHandle, RecordingLlm, SessionManager}; use ironclaw_safety::SafetyLayer; use ironclaw_skills::SkillRegistry; use ironclaw_skills::catalog::SkillCatalog; @@ -229,8 +230,10 @@ impl AppBuilder { } } + let session_db: ironclaw_llm::host::SharedSessionDb = + std::sync::Arc::new(crate::llm_host::DatabaseSessionDb::new(db.clone())); self.session - .attach_store(db.clone(), &self.config.owner_id) + .attach_store(session_db, &self.config.owner_id) .await; // Fire-and-forget housekeeping — no need to block startup. @@ -422,7 +425,10 @@ impl AppBuilder { // Wire the secrets store into the session manager so future // token saves go to encrypted storage. - self.session.attach_secrets(Arc::clone(secrets)).await; + let session_secrets: ironclaw_llm::host::SharedSessionSecrets = Arc::new( + crate::llm_host::SecretsStoreSessionSecrets::new(Arc::clone(secrets)), + ); + self.session.attach_secrets(session_secrets).await; } self.secrets_store = store; @@ -493,7 +499,7 @@ impl AppBuilder { anyhow::Error, > { let (llm, cheap_llm, recording_handle, reload_handle) = - crate::llm::build_provider_chain(&self.config.llm, self.session.clone()).await?; + ironclaw_llm::build_provider_chain(&self.config.llm, self.session.clone()).await?; Ok((llm, cheap_llm, recording_handle, reload_handle)) } @@ -552,16 +558,27 @@ impl AppBuilder { tools.register_secrets_tools(Arc::clone(ss)); } - // Create embeddings provider using the unified method - let embeddings = self - .config - .embeddings - .create_provider( - &self.config.llm.nearai.base_url, - self.session.clone(), - self.config.llm.bedrock.as_ref(), - ) - .await; + // Create embeddings provider using the unified method. + // Translate the LLM-side `BedrockConfig` into the embeddings-side + // `BedrockEmbeddingSetup` at the boundary so the embeddings layer + // does not depend on `ironclaw_llm` config types. + let bedrock_setup = + self.config + .llm + .bedrock + .as_ref() + .map(|b| ironclaw_embeddings::BedrockEmbeddingSetup { + region: b.region.clone(), + profile: b.profile.clone(), + }); + let embeddings = ironclaw_embeddings::create_provider( + &self.config.embeddings, + ironclaw_embeddings::ProviderDeps { + session: self.session.clone(), + bedrock_setup, + }, + ) + .await; // Register memory tools if database is available let workspace_user_id = self.config.owner_id.as_str(); @@ -660,13 +677,13 @@ impl AppBuilder { .map(|p| p.model.clone()) .unwrap_or_else(|| self.config.llm.nearai.model.clone()); let models = vec![model_name.clone()]; - let gen_model = crate::llm::image_models::suggest_image_model(&models) + let gen_model = ironclaw_llm::image_models::suggest_image_model(&models) .unwrap_or("black-forest-labs/FLUX.2-klein-4B") .to_string(); tools.register_image_tools(api_base.clone(), api_key.clone(), gen_model, None); // Check for vision models - let vision_model = crate::llm::vision_models::suggest_vision_model(&models) + let vision_model = ironclaw_llm::vision_models::suggest_vision_model(&models) .unwrap_or(&model_name) .to_string(); tools.register_vision_tools(api_base, api_key, vision_model, None); @@ -704,6 +721,7 @@ impl AppBuilder { tools: &Arc<ToolRegistry>, hooks: &Arc<HookRegistry>, settings_store_override: Option<Arc<dyn crate::db::SettingsStore + Send + Sync>>, + ownership_cache: Arc<crate::ownership::OwnershipCache>, ) -> Result< ( Arc<McpSessionManager>, @@ -998,8 +1016,21 @@ impl AppBuilder { if let Some(ref ss) = settings_store_override { em = em.with_settings_store(Arc::clone(ss)); } + let pairing_store = if let Some(ref db) = self.db { + let ps = Arc::new(crate::pairing::PairingStore::new( + Arc::clone(db), + Arc::clone(&ownership_cache), + )); + em = em.with_pairing_store(Arc::clone(&ps)); + Some(ps) + } else { + None + }; let manager = Arc::new(em); tools.register_extension_tools(Arc::clone(&manager)); + if let Some(ps) = pairing_store { + tools.register_sync(Arc::new(crate::tools::builtin::PairingApproveTool::new(ps))); + } // Register permission management tool and upgrade tool_list with // builtin registry support. Prefer the workspace-backed adapter @@ -1106,14 +1137,16 @@ impl AppBuilder { self.init_database().await?; self.init_secrets().await?; - // Post-init validation: backends with dedicated config (nearai, gemini_oauth, - // bedrock, openai_codex) handle their own credential resolution. For registry-based - // backends, fail early if no provider config was resolved. - if !matches!( - self.config.llm.backend.as_str(), - "nearai" | "gemini_oauth" | "bedrock" | "openai_codex" - ) && self.config.llm.provider.is_none() - { + // Post-init validation: backends with a dedicated config slot + // (nearai/gemini_oauth/bedrock/openai_codex) read from their own + // sub-struct and don't populate `LlmConfig.provider`. For + // OpenAI-shape registry backends, fail early if no provider + // config was resolved. + let registry = ironclaw_llm::ProviderRegistry::load(); + let has_dedicated_config = registry + .find(self.config.llm.backend.as_str()) + .is_some_and(|d| d.protocol.has_dedicated_config()); + if !has_dedicated_config && self.config.llm.provider.is_none() { let backend = &self.config.llm.backend; anyhow::bail!( "LLM_BACKEND={backend} is configured but no credentials were found. \ @@ -1190,6 +1223,7 @@ impl AppBuilder { _ => (None, None), }; + let ownership_cache = Arc::new(crate::ownership::OwnershipCache::new()); let ( mcp_session_manager, mcp_process_manager, @@ -1198,7 +1232,12 @@ impl AppBuilder { catalog_entries, dev_loaded_tool_names, ) = self - .init_extensions(&tools, &hooks, settings_store.clone()) + .init_extensions( + &tools, + &hooks, + settings_store.clone(), + Arc::clone(&ownership_cache), + ) .await?; // Load bootstrap-completed flag from settings so that existing users @@ -1307,10 +1346,18 @@ impl AppBuilder { tools.count() ); - // Seed per-user tool permission defaults into the database. - // This runs after all tools (built-in, WASM, MCP) are registered so - // that every tool name is known. Existing entries are never overwritten. - seed_tool_permissions(&tools, self.db.as_ref(), &self.config.owner_id).await; + // One-shot cleanup of ghost-seeded tool permission rows for the + // owner. Pre-#3559, `seed_tool_permissions` wrote the code-level + // defaults (e.g. `tool_install` → `AskEachTime`) into the DB so + // the permissions panel could render them. Those rows were + // indistinguishable from user-explicit overrides, so a user + // could not be told from someone who never touched the setting, + // and `AGENT_AUTO_APPROVE_TOOLS=true` ended up bypassing + // user-explicit `AskEachTime` choices (#3559 security review). + // The seeder is gone; this migration deletes ghost rows once, + // after which any remaining row is user-explicit by + // construction and `resolve_permission` can trust its value. + cleanup_ghost_seeded_tool_permissions(self.db.as_ref(), &self.config.owner_id).await; Ok(AppComponents { config: self.config, @@ -1342,7 +1389,7 @@ impl AppBuilder { catalog_entries, dev_loaded_tool_names, builder, - ownership_cache: Arc::new(crate::ownership::OwnershipCache::new()), + ownership_cache, }) } } @@ -1502,67 +1549,103 @@ async fn migrate_session_credential( } } -async fn seed_tool_permissions( - tools: &crate::tools::ToolRegistry, - db: Option<&Arc<dyn Database>>, - owner_id: &str, -) { +/// Sentinel settings key marking that ghost-seeded tool permission rows +/// have been cleaned up for this owner. Reads/writes are idempotent and +/// scoped per-user, so the migration is safe to re-run. +const TOOL_PERMISSION_CLEANUP_SENTINEL: &str = "_internal.tool_permissions_seed_cleanup_v1"; + +/// One-shot migration that removes ghost-seeded `tool_permissions.<name>` +/// rows whose value matches `seeded_default_permission(name)` from the +/// owner's settings. After this runs, any surviving DB row is a +/// user-explicit choice — which lets `ToolPermissionSnapshot` treat all +/// DB rows as explicit again. See `cleanup_ghost_seeded_tool_permissions` +/// call site for context and the #3559 security review. +async fn cleanup_ghost_seeded_tool_permissions(db: Option<&Arc<dyn Database>>, owner_id: &str) { let db = match db { Some(db) => db, None => { - tracing::debug!("seed_tool_permissions: no database available, skipping"); + tracing::debug!( + "cleanup_ghost_seeded_tool_permissions: no database available, skipping" + ); return; } }; - // Load existing tool permission overrides from the DB. + // Skip if migration already ran for this owner. + match db + .get_setting(owner_id, TOOL_PERMISSION_CLEANUP_SENTINEL) + .await + { + Ok(Some(_)) => { + tracing::debug!("cleanup_ghost_seeded_tool_permissions: sentinel present, skipping"); + return; + } + Ok(None) => {} + Err(e) => { + tracing::warn!( + "cleanup_ghost_seeded_tool_permissions: failed to read sentinel: {}", + e + ); + return; + } + } + let db_map = match db.get_all_settings(owner_id).await { Ok(m) => m, Err(e) => { - tracing::warn!("seed_tool_permissions: failed to load settings: {}", e); + tracing::warn!( + "cleanup_ghost_seeded_tool_permissions: failed to load settings: {}", + e + ); return; } }; let existing = crate::settings::Settings::from_db_map(&db_map).tool_permissions; - let registered_names = tools.list().await; - let mut seeded = 0u32; - - for name in &registered_names { - if existing.contains_key(name.as_str()) { - // User has an explicit override — do not touch it. + let mut deleted = 0u32; + for (tool_name, state) in &existing { + let Some(seeded) = crate::tools::permissions::seeded_default_permission(tool_name) else { + continue; + }; + if *state != seeded { continue; } - - // Only insert seed defaults for known built-ins. Unknown/dynamic tools - // stay absent and fall back to AskEachTime at runtime. - if let Some(default_state) = crate::tools::permissions::seeded_default_permission(name) { - let json_value = match serde_json::to_value(default_state) { - Ok(v) => v, - Err(e) => { - tracing::warn!( - "seed_tool_permissions: failed to serialize state for '{}': {}", - name, - e - ); - continue; - } - }; - if let Err(e) = db - .set_setting(owner_id, &format!("tool_permissions.{}", name), &json_value) - .await - { - tracing::warn!("seed_tool_permissions: failed to set '{}': {}", name, e); - } else { - seeded += 1; + match db + .delete_setting(owner_id, &format!("tool_permissions.{}", tool_name)) + .await + { + Ok(_) => deleted += 1, + Err(e) => { + tracing::warn!( + "cleanup_ghost_seeded_tool_permissions: failed to delete '{}': {}", + tool_name, + e + ); } } } - if seeded > 0 { - tracing::debug!( - count = seeded, - "Seeded tool permission defaults into database" + // Record the sentinel even on partial failures so we don't re-scan + // every startup. The deletes are idempotent if a future run does + // re-process the same row. + if let Err(e) = db + .set_setting( + owner_id, + TOOL_PERMISSION_CLEANUP_SENTINEL, + &serde_json::json!(true), + ) + .await + { + tracing::warn!( + "cleanup_ghost_seeded_tool_permissions: failed to write sentinel: {}", + e + ); + } + + if deleted > 0 { + tracing::info!( + count = deleted, + "Cleaned up ghost-seeded tool permission rows for owner" ); } } @@ -1661,60 +1744,88 @@ mod tests { assert!(!session_id.is_empty()); } - /// Verify that `seed_tool_permissions` is idempotent: an existing user - /// override must survive a re-seed. + /// #3559 security review: ghost-seeded rows whose value matches the + /// code-level seeded default are deleted on first run. After cleanup, + /// the row no longer exists in DB and `effective_permission` falls + /// back to the code-level default at read time. Genuine user + /// overrides (value != seeded default) survive untouched. The + /// migration is idempotent — re-running after the sentinel is + /// written is a no-op. #[cfg(feature = "libsql")] #[tokio::test] - async fn seed_tool_permissions_preserves_user_overrides() { + async fn cleanup_ghost_seeded_tool_permissions_removes_seed_matching_rows() { use crate::db::Database; use crate::db::libsql::LibSqlBackend; - use crate::tools::ToolRegistry; use crate::tools::permissions::PermissionState; let dir = tempfile::tempdir().unwrap(); - let db_path = dir.path().join("test_seed.db"); + let db_path = dir.path().join("test_cleanup.db"); let backend = LibSqlBackend::new_local(&db_path).await.unwrap(); backend.run_migrations().await.unwrap(); let db: Arc<dyn Database> = Arc::new(backend); - let registry = ToolRegistry::new(); - registry.register_builtin_tools(); - let owner = "test-user"; - assert_eq!( - crate::tools::permissions::seeded_default_permission("tool_activate"), - Some(PermissionState::AlwaysAllow), - "tool_activate should seed AlwaysAllow so subgates control auth/setup" - ); - // 1. Initial seed: creates defaults for all registered tools. - super::seed_tool_permissions(&registry, Some(&db), owner).await; + // 1. Simulate the old seeder's effect: write seeded-default rows + // for `tool_install` (AskEachTime) and `echo` (AlwaysAllow), + // plus a real user override for `shell` (AlwaysAllow, diverges + // from the seeded AskEachTime). + let install_seed = serde_json::to_value(PermissionState::AskEachTime).unwrap(); + let echo_seed = serde_json::to_value(PermissionState::AlwaysAllow).unwrap(); + let shell_override = serde_json::to_value(PermissionState::AlwaysAllow).unwrap(); + db.set_setting(owner, "tool_permissions.tool_install", &install_seed) + .await + .unwrap(); + db.set_setting(owner, "tool_permissions.echo", &echo_seed) + .await + .unwrap(); + db.set_setting(owner, "tool_permissions.shell", &shell_override) + .await + .unwrap(); + + // 2. Run the cleanup migration. + super::cleanup_ghost_seeded_tool_permissions(Some(&db), owner).await; - // Verify "echo" was seeded as AlwaysAllow. let map = db.get_all_settings(owner).await.unwrap(); let settings = crate::settings::Settings::from_db_map(&map); + + // Ghost-seeded rows are gone. + assert!( + !settings.tool_permissions.contains_key("tool_install"), + "tool_install row matching the seeded default must be removed" + ); + assert!( + !settings.tool_permissions.contains_key("echo"), + "echo row matching the seeded default must be removed" + ); + + // Genuine user override survives. assert_eq!( - settings.tool_permissions.get("echo"), + settings.tool_permissions.get("shell"), Some(&PermissionState::AlwaysAllow), - "echo should be AlwaysAllow after initial seed" + "shell override diverging from the seeded default must survive cleanup" ); - // 2. User overrides echo → Disabled. - let disabled_json = serde_json::to_value(PermissionState::Disabled).unwrap(); - db.set_setting(owner, "tool_permissions.echo", &disabled_json) + // Sentinel is set so subsequent runs are no-ops. + let sentinel = db + .get_setting(owner, super::TOOL_PERMISSION_CLEANUP_SENTINEL) .await .unwrap(); + assert!(sentinel.is_some(), "cleanup sentinel must be written"); - // 3. Re-seed (e.g. after a restart). - super::seed_tool_permissions(&registry, Some(&db), owner).await; - - // 4. Assert the override survived. + // 3. Re-running the migration after the sentinel is a no-op: + // re-seed a ghost row and assert it survives the second pass. + db.set_setting(owner, "tool_permissions.tool_install", &install_seed) + .await + .unwrap(); + super::cleanup_ghost_seeded_tool_permissions(Some(&db), owner).await; let map = db.get_all_settings(owner).await.unwrap(); let settings = crate::settings::Settings::from_db_map(&map); assert_eq!( - settings.tool_permissions.get("echo"), - Some(&PermissionState::Disabled), - "user override to Disabled must survive re-seed" + settings.tool_permissions.get("tool_install"), + Some(&PermissionState::AskEachTime), + "after sentinel is written, a manually re-inserted row must NOT be cleaned up; \ + the migration is one-shot per owner" ); } } diff --git a/src/auth/extension.rs b/src/auth/extension.rs index a4c72aadce8..c841ee0404f 100644 --- a/src/auth/extension.rs +++ b/src/auth/extension.rs @@ -107,7 +107,7 @@ pub struct AuthManager { /// Branches (in precedence order): /// /// 1. **User-influenced** `name` parameter on -/// `tool_install` / `tool_activate` / `tool_auth` actions. This string +/// `tool_install` / `tool_auth` actions. This string /// comes from the model's tool arguments, so it must pass /// `ExtensionName::new` — invalid values (path traversal, uppercase, /// etc.) fall through to the next branch instead of tainting the @@ -138,17 +138,12 @@ pub(crate) async fn resolve_auth_flow_extension_name( extension_manager: Option<&crate::extensions::ExtensionManager>, ) -> CommonExtensionName { // 1. User-influenced: validate via ExtensionName::new, fall through on failure. - // Match both underscore and hyphen variants for every install/activate/auth + // Match both underscore and hyphen variants for every install/auth // action so the hyphenated tool names dispatched from Python land the // same as the canonical underscore form. if matches!( action_name, - "tool_install" - | "tool-install" - | "tool_activate" - | "tool-activate" - | "tool_auth" - | "tool-auth" + "tool_install" | "tool-install" | "tool_auth" | "tool-auth" ) && let Some(raw) = parameters.get("name").and_then(|v| v.as_str()) && let Ok(name) = CommonExtensionName::new(raw) { @@ -256,11 +251,68 @@ impl AuthManager { .await; } - // For non-HTTP tools, we don't have a generic pre-flight mechanism - // yet. Extension-level auth (NeedsAuth/NeedsSetup) is handled by - // check_tool_readiness() for available_actions() filtering and by - // the post-install pipeline. - AuthCheckResult::NoAuthRequired + // Non-HTTP path: ask the tool itself which credentials it + // declares. WASM tools surface this via + // `Tool::required_credentials()` reading their + // `capabilities.http.credentials` block. Built-in tools default + // to empty, so this branch is a no-op for them. When a tool + // declares a credential and it's missing, raise an + // Authentication gate when the tool's declared credentials + // are missing — the call paths flow into the inline-await + // machinery (#3133/#3166) and resume after the user completes + // OAuth, with no separate enablement step required. + let Some(tools) = self.tools.as_ref() else { + return AuthCheckResult::NoAuthRequired; + }; + let Some(tool) = tools.get(action_name).await else { + return AuthCheckResult::NoAuthRequired; + }; + let required = tool.required_credentials(); + if required.is_empty() { + return AuthCheckResult::NoAuthRequired; + } + + let role_lookup = tools.role_lookup().map(Arc::as_ref); + let mut missing = Vec::new(); + for secret_name in required { + let oauth_refresh = credential_registry.oauth_refresh_for_secret(&secret_name); + match resolve_secret_for_runtime( + self.secrets_store.as_ref(), + user_id, + &secret_name, + role_lookup, + oauth_refresh.as_ref(), + DefaultFallback::AdminOnly, + ) + .await + { + Ok(_) => {} + Err(error) if error.requires_authentication() => { + missing.push( + self.describe_missing_credential(&secret_name, user_id) + .await, + ); + } + Err(error) => { + tracing::debug!( + action = %action_name, + secret = %secret_name, + error = ?error, + "Failed to resolve credential during pre-flight auth — assuming missing" + ); + missing.push(MissingCredential { + credential_name: CredentialName::from_trusted(secret_name.clone()), + setup_instructions: None, + auth_url: None, + }); + } + } + } + if missing.is_empty() { + AuthCheckResult::Ready + } else { + AuthCheckResult::MissingCredentials(missing) + } } /// Check HTTP tool credentials by extracting the host and querying diff --git a/src/auth/oauth.rs b/src/auth/oauth.rs index c41d05daded..6718be1a703 100644 --- a/src/auth/oauth.rs +++ b/src/auth/oauth.rs @@ -25,9 +25,11 @@ use crate::secrets::{CreateSecretParams, SecretsStore}; // ── Shared callback server ────────────────────────────────────────────── -// Core OAuth callback infrastructure is defined in `crate::llm::oauth_helpers` -// and re-exported here for backward compatibility. -pub use crate::llm::oauth_helpers::{ +// Core OAuth callback infrastructure lives in the standalone `ironclaw_oauth` +// crate so non-LLM OAuth flows (WASM tools, MCP, NEAR AI session login) don't +// have to depend on `ironclaw_llm` for transport. Re-exported here so existing +// `crate::auth::oauth::...` call sites continue to compile. +pub use ironclaw_oauth::{ OAUTH_CALLBACK_PORT, OAuthCallbackError, bind_callback_listener, callback_host, callback_url, is_loopback_host, landing_html, wait_for_callback, }; @@ -41,7 +43,7 @@ pub use crate::llm::oauth_helpers::{ /// /// Both the v1 dispatcher (`src/agent/dispatcher.rs`) and the v2 effect adapter /// (`src/bridge/effect_adapter.rs`) call this on every `auth_url` extracted -/// from `tool_activate`/`tool_auth` output before surfacing it to the client. +/// from `tool_install`/`tool_auth` output before surfacing it to the client. /// Keeping the helper in one place ensures the v1/v2 invariants stay symmetric. pub(crate) fn sanitize_auth_url(url: Option<&str>) -> Option<String> { url.map(str::trim).and_then(|u| { diff --git a/src/bootstrap.rs b/src/bootstrap.rs index ed0b4266b2b..a706c31ccc8 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -7,71 +7,10 @@ //! File: `~/.ironclaw/.env` (standard dotenvy format) use std::path::PathBuf; -use std::sync::LazyLock; -const IRONCLAW_BASE_DIR_ENV: &str = "IRONCLAW_BASE_DIR"; - -/// Lazily computed IronClaw base directory, cached for the lifetime of the process. -static IRONCLAW_BASE_DIR: LazyLock<PathBuf> = LazyLock::new(compute_ironclaw_base_dir); - -/// Compute the IronClaw base directory from environment. -/// -/// This is the underlying implementation used by both the public -/// `ironclaw_base_dir()` function (which caches the result) and tests -/// (which need to verify different configurations). -pub fn compute_ironclaw_base_dir() -> PathBuf { - std::env::var(IRONCLAW_BASE_DIR_ENV) - .map(PathBuf::from) - .map(|path| { - if path.as_os_str().is_empty() { - default_base_dir() - } else if !path.is_absolute() { - eprintln!( - "Warning: IRONCLAW_BASE_DIR is a relative path '{}', resolved against current directory", - path.display() - ); - path - } else { - path - } - }) - .unwrap_or_else(|_| default_base_dir()) -} - -/// Get the default IronClaw base directory (~/.ironclaw). -/// -/// Logs a warning if the home directory cannot be determined and falls back to -/// the current directory. -fn default_base_dir() -> PathBuf { - if let Some(home) = dirs::home_dir() { - home.join(".ironclaw") - } else { - eprintln!("Warning: Could not determine home directory, using current directory"); - std::env::current_dir() - .unwrap_or_else(|_| PathBuf::from("/tmp")) - .join(".ironclaw") - } -} - -/// Get the IronClaw base directory. -/// -/// Override with `IRONCLAW_BASE_DIR` environment variable. -/// Defaults to `~/.ironclaw` (or `./.ironclaw` if home directory cannot be determined). -/// -/// Thread-safe: the value is computed once and cached in a `LazyLock`. -/// -/// # Environment Variable Behavior -/// - If `IRONCLAW_BASE_DIR` is set to a non-empty path, that path is used. -/// - If `IRONCLAW_BASE_DIR` is set to an empty string, it is treated as unset. -/// - If `IRONCLAW_BASE_DIR` contains null bytes, a warning is printed and the default is used. -/// - If the home directory cannot be determined, a warning is printed and the current directory is used. -/// -/// # Returns -/// A `PathBuf` pointing to the base directory. The path is not validated -/// for existence. -pub fn ironclaw_base_dir() -> PathBuf { - IRONCLAW_BASE_DIR.clone() -} +// Base-directory resolution lives in `ironclaw_common::paths`. Re-exported +// from this module's path for back-compat with existing callers. +pub use ironclaw_common::paths::{compute_ironclaw_base_dir, ironclaw_base_dir}; /// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`. pub fn ironclaw_env_path() -> PathBuf { diff --git a/src/bridge/CLAUDE.md b/src/bridge/CLAUDE.md index e098acc86ba..ca9347a6063 100644 --- a/src/bridge/CLAUDE.md +++ b/src/bridge/CLAUDE.md @@ -20,16 +20,31 @@ resolution — they call through these adapters. ## Engine-v2 enablement contract -For engine v2, blocked managed integrations are not part of the normal -model-facing callable action list. They surface in capability background -under `Activatable Integrations`, and the single model-facing enablement -path is `tool_activate(name=...)`. - -`tool_activate` may internally install, authenticate, or activate the -integration as needed. `tool_install` and `tool_auth` still exist as -runtime/compatibility surfaces, but they are not the normal v2 prompt -contract. After `tool_activate` succeeds, the newly ready tools appear on -the next top-level turn rather than mid-CodeAct step. +For engine v2, installed-but-unauthed provider tools are direct-callable. +The model calls them like any ready action; the engine's auth preflight +raises an `Authentication` gate at execute time, the inline-await machinery +parks the VM, and the OAuth callback delivers the resolved credential to +retry the action. `tool_activate` was removed in favor of this contract; +its install + auto-activation behavior is covered by: + +- `tool_install` (callable; agent-callable) — installs an extension and + registers its tools with the engine registry. After install, the new + tools appear on the next top-level turn (CodeAct does not hot-refresh + callable tools mid-step). User consent is mediated by the tool's + `ApprovalRequirement::UnlessAutoApproved` and the seeded `AskEachTime` + permission rather than by hiding the tool from the agent surface. +- `tool_auth` (callable; v1-only) — manual auth flow surface for non-OAuth + credential types. +- The auth-preflight + inline-await pipeline (see #3133 / PR #3157) for + the OAuth gate path. + +Integrations that need user setup (`NeedsSetup`, `Inactive`, +`AvailableNotInstalled`) surface in the prompt under `Activatable +Integrations`. The model installs them by calling `tool_install` directly; +the engine's auth preflight handles any credential prompt at execute time. +(Restored in issue #3533 / PR #3559 — `tool_install` was previously +hidden from the model surface, which left "connect my telegram" +narrating manual UI steps instead of running the actual install.) ## Auth-flow extension resolution: one place, no re-derivation @@ -39,7 +54,7 @@ The single authority that maps an auth gate or tool-call context to the installe Its precedence order: -1. **User-influenced** — explicit `name` param on enablement/auth tool invocations such as `tool_activate` (and internal/compat paths like `tool_install` / `tool_auth`). This comes from the model or caller arguments, so it's validated via `ExtensionName::new`; invalid values fall through. +1. **User-influenced** — explicit `name` param on enablement/auth tool invocations like `tool_install` / `tool_auth`. This comes from the model or caller arguments, so it's validated via `ExtensionName::new`; invalid values fall through. 2. The action's provider extension, via `ToolRegistry::provider_extension_for_tool`. 3. Canonicalized `action_name` if the extension manager has an installed extension by that name. 4. The caller-supplied `credential_fallback` — last-resort, used only when no extension owns the action. diff --git a/src/bridge/action_projector.rs b/src/bridge/action_projector.rs index 1fde37e6f3d..0564e394058 100644 --- a/src/bridge/action_projector.rs +++ b/src/bridge/action_projector.rs @@ -204,9 +204,6 @@ fn classify_registered_tool( if crate::bridge::effect_adapter::is_v1_auth_tool(tool.name()) { return ProjectedAction::Hidden; } - if hidden_from_model_callable_surface(tool.name()) { - return ProjectedAction::Hidden; - } if tool_permissions.resolve_permission(tool.name()).effective == PermissionState::Disabled { return ProjectedAction::Hidden; } @@ -240,10 +237,6 @@ fn classify_registered_tool( } } -fn hidden_from_model_callable_surface(tool_name: &str) -> bool { - matches!(tool_name, "tool_install" | "tool-install") -} - fn supports_pre_activation_discovery( kind: SurfaceSubjectKind, invocation_mode: InvocationMode, @@ -367,6 +360,7 @@ mod tests { tools: vec![format!("{name}_search")], needs_setup: false, has_auth: true, + requires_binding: false, installed: true, activation_error: None, version: None, @@ -654,6 +648,10 @@ mod tests { thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } @@ -787,7 +785,18 @@ mod tests { } #[tokio::test] - async fn needs_auth_provider_tools_omitted_from_available_actions() { + async fn needs_auth_provider_tools_stay_in_available_actions() { + // Post-#3133/#3166: a NeedsAuth provider tool (e.g. installed- + // but-unauthed gmail) stays on the callable surface. The + // engine's auth preflight (`AuthManager::check_action_auth`) + // raises an `Authentication` gate at execute time when a + // declared credential is missing, the inline-await machinery + // parks the VM, and the OAuth callback delivers `Approved` + // to retry the action against the now-present secret. The + // model calls the tool directly — there is no separate + // enablement step. Pre-#3133 the contract was inverted: the + // tool was hidden until auth completed and the LLM had to + // navigate via the now-removed `tool_activate` first. let inventory = projected_inventory( "gmail_send", "Send a Gmail message", @@ -797,21 +806,14 @@ mod tests { .await; assert!( - !inventory + inventory .inline .iter() .any(|action| action.name == "gmail_send"), - "NeedsAuth provider tool should be omitted from available_actions, got: {:?}", + "NeedsAuth provider tool should be callable; auth resolves at \ + execute time via inline-await. inline={:?}", inventory.inline ); - assert!( - inventory - .discoverable - .iter() - .any(|action| action.name == "gmail_send"), - "NeedsAuth provider tool should remain discoverable, got: {:?}", - inventory.discoverable - ); } #[tokio::test] @@ -932,7 +934,12 @@ mod tests { } #[tokio::test] - async fn tool_install_hidden_from_model_callable_surface() { + async fn tool_install_is_callable_by_agent() { + // Regression for #3533: the agent must be able to call tool_install + // so "connect my telegram" runs an actual install + auth gate, rather + // than narrating manual UI steps. The hidden gate added in #2868 is + // gone; approval gating in tool_install::requires_approval() is what + // mediates user consent now. let tools = std::sync::Arc::new(ToolRegistry::new()); tools .register(std::sync::Arc::new(BuiltinTool { @@ -941,7 +948,7 @@ mod tests { .await; tools .register(std::sync::Arc::new(BuiltinTool { - name: "tool_activate", + name: "tool_search", })) .await; @@ -961,8 +968,8 @@ mod tests { .map(|action| action.name) .collect::<Vec<_>>(); - assert!(!action_names.iter().any(|name| name == "tool_install")); - assert!(action_names.iter().any(|name| name == "tool_activate")); + assert!(action_names.iter().any(|name| name == "tool_install")); + assert!(action_names.iter().any(|name| name == "tool_search")); } #[tokio::test] diff --git a/src/bridge/capability_projector.rs b/src/bridge/capability_projector.rs index 6bcbf0be210..e7ef15b3b04 100644 --- a/src/bridge/capability_projector.rs +++ b/src/bridge/capability_projector.rs @@ -367,6 +367,7 @@ mod tests { tools: vec![format!("{name}_send")], needs_setup: false, has_auth: true, + requires_binding: false, installed: true, activation_error: None, version: None, diff --git a/src/bridge/effect_adapter.rs b/src/bridge/effect_adapter.rs index 6e71af5d38d..e0f352c8291 100644 --- a/src/bridge/effect_adapter.rs +++ b/src/bridge/effect_adapter.rs @@ -33,7 +33,6 @@ use crate::bridge::sandbox::{InterceptOutcome, maybe_intercept}; use crate::bridge::tool_permissions::{ToolPermissionResolution, ToolPermissionSnapshot}; use crate::context::JobContext; use crate::extensions::InstalledExtension; -use crate::extensions::naming::extension_name_candidates; use crate::hooks::{HookEvent, HookOutcome, HookRegistry}; use crate::tools::ToolRegistry; use crate::tools::permissions::PermissionState; @@ -67,7 +66,7 @@ pub struct EffectBridgeAdapter { /// outbound requests through the interceptor. Without this, engine v2 tool /// calls bypass the recorder entirely — recorded traces end up with zero /// `http_exchanges` and replay can't substitute responses. - http_interceptor: RwLock<Option<Arc<dyn crate::llm::recording::HttpInterceptor>>>, + http_interceptor: RwLock<Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>>, /// Engine v2 store used to mirror live-installed v1 skills into `DocType::Skill`. engine_store: RwLock<Option<Arc<dyn Store>>>, /// V1 skill registry used to load the just-installed skill for v2 sync. @@ -84,6 +83,12 @@ pub struct EffectBridgeAdapter { /// capabilities like `missions` are registered here in `router.rs` and /// would otherwise be invisible to the LLM despite having active leases. capability_registry: RwLock<Option<Arc<CapabilityRegistry>>>, + /// Per-thread catalog of caller-supplied external tools (Responses + /// API). When set, `execute_action` short-circuits to a + /// `GatePaused { resume_kind: External { ext_tool: <call_id> } }` + /// for any action name in the catalog, and `available_actions` + /// merges the catalog into the LLM-visible action surface. + external_tool_catalog: RwLock<Option<Arc<crate::bridge::ExternalToolCatalog>>>, } struct ToolApprovalContext<'a> { @@ -127,9 +132,42 @@ impl EffectBridgeAdapter { skill_registry: RwLock::new(None), workspace_mounts: RwLock::new(None), capability_registry: RwLock::new(None), + external_tool_catalog: RwLock::new(None), } } + /// Install the per-thread external-tool catalog. Set once at bridge + /// init; the Responses API handler registers tools onto the same + /// catalog instance via its own `Arc` clone. + pub async fn set_external_tool_catalog( + &self, + catalog: Arc<crate::bridge::ExternalToolCatalog>, + ) { + *self.external_tool_catalog.write().await = Some(catalog); + } + + /// Look up the catalog (if installed) for read-only use. + async fn external_tool_catalog(&self) -> Option<Arc<crate::bridge::ExternalToolCatalog>> { + self.external_tool_catalog.read().await.clone() + } + + /// Resolve all catalog keys this `ThreadExecutionContext` may have + /// caller-supplied tools registered under. The engine `thread_id` + /// is the canonical key after the bridge's post-spawn `transfer`, + /// but the executor task can run before that transfer completes — + /// `conversation_scope` (stamped into thread metadata by the + /// bridge) is the original caller-side key the catalog was + /// registered under, used as a race-window fallback. + fn external_tool_catalog_keys( + context: &ThreadExecutionContext, + ) -> impl Iterator<Item = ironclaw_engine::ThreadId> { + let scope = context + .conversation_scope + .filter(|uuid| *uuid != context.thread_id.0) + .map(ironclaw_engine::ThreadId); + std::iter::once(context.thread_id).chain(scope) + } + /// Install a per-project workspace mount table on this adapter. When set, /// sandbox-eligible tool calls (`file_read`, `file_write`, `list_dir`, /// `apply_patch`, `shell`) whose path argument resolves into a mount get @@ -154,7 +192,7 @@ impl EffectBridgeAdapter { /// interceptor, so http-aware tools will record/replay through it. pub async fn set_http_interceptor( &self, - interceptor: Arc<dyn crate::llm::recording::HttpInterceptor>, + interceptor: Arc<dyn ironclaw_llm::recording::HttpInterceptor>, ) { *self.http_interceptor.write().await = Some(interceptor); } @@ -274,49 +312,6 @@ impl EffectBridgeAdapter { .resolve_permission(lookup_name) } - async fn tool_activate_requires_install_approval( - &self, - lookup_name: &str, - parameters: &serde_json::Value, - context: &ThreadExecutionContext, - ) -> bool { - if !matches!(lookup_name, "tool_activate" | "tool-activate") { - return false; - } - - let Some(requested_name) = parameters - .get("name") - .and_then(|value| value.as_str()) - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return false; - }; - - let auth_manager = self.auth_manager.read().await; - let Some(auth_manager) = auth_manager.as_ref() else { - return true; - }; - - let extensions = match auth_manager - .list_capability_extensions(&context.user_id) - .await - { - Ok(extensions) => extensions, - Err(error) => { - debug!( - user_id = %context.user_id, - extension_name = requested_name, - error = %error, - "failed to load extension inventory for tool_activate approval; requiring approval" - ); - return true; - } - }; - - matching_extension_requires_install_approval(requested_name, &extensions).unwrap_or(false) - } - fn ensure_tool_not_disabled( action_name: &str, user_permission: ToolPermissionResolution, @@ -346,31 +341,18 @@ impl EffectBridgeAdapter { return Ok(()); } - if matches!( - tool.requires_approval(approval.parameters), - ApprovalRequirement::Always - ) { - return Err(Self::gate_paused( - "approval", - approval.action_name, - approval.context.current_call_id.as_deref(), - approval.parameters.clone(), - ironclaw_engine::ResumeKind::Approval { - allow_always: false, - }, - None, - Some(approval.lease.clone()), - )); + let approval_requirement = tool.requires_approval(approval.parameters); + // `skill_install` is parameter-sensitive: duplicate installs are a + // guaranteed no-op and deliberately return `ApprovalRequirement::Never`. + // Preserve that v1 contract even though the tool's default permission is + // ask-each-time for real installs. + if matches!(approval.lookup_name, "skill_install" | "skill-install") + && matches!(approval_requirement, ApprovalRequirement::Never) + { + return Ok(()); } - if self - .tool_activate_requires_install_approval( - approval.lookup_name, - approval.parameters, - approval.context, - ) - .await - { + if matches!(approval_requirement, ApprovalRequirement::Always) { return Err(Self::gate_paused( "approval", approval.action_name, @@ -669,7 +651,12 @@ impl EffectBridgeAdapter { output_value.get("auth_url").and_then(|v| v.as_str()), ), }, - None, + // Carry the install/auth tool's already-computed output + // through the gate so the inline-await retry can return + // it directly instead of re-running `tool_install` (which + // would re-download the WASM and re-raise approval). + // Tracked by #3533. + Some(output_value.clone()), Some(lease.clone()), )), _ => None, @@ -1226,6 +1213,35 @@ impl EffectBridgeAdapter { .unwrap_or(canonical_action_name) .to_string(); + // ── Schema-guided parameter coercion ── + // + // Engine actions (`mission_*`, routine aliases, `tool_info`) and + // host tools both declare JSON Schemas for their parameters. Run + // both kinds through the same coercion that `prepare_tool_params` + // applies for the v1 path so the LLM can pass stringified scalars + // (`"120"` for an integer field) without breaking the handler. + // Schema sources, in order: orchestrator-populated action + // snapshot, bridge-known engine action defs, host tool registry + // (via `discovery_schema()` to match `prepare_tool_params`). + // Once this runs, downstream sites in this method (sandbox-path + // validator, `execute_tool_with_safety`'s second `prepare_tool_params`) + // see already-coerced input — the second pass is idempotent. + let action_schema = context + .available_actions_snapshot + .as_ref() + .and_then(|snapshot| { + ActionDiscovery::resolve(snapshot.as_ref(), canonical_action_name) + .map(|action| action.discovery_schema().clone()) + }) + .or_else(|| engine_action_schema(canonical_action_name)); + let parameters = if let Some(schema) = action_schema { + crate::tools::prepare_params_for_schema(&parameters, &schema) + } else if let Some(tool) = self.tools.get(&lookup_name).await { + crate::tools::prepare_params_for_schema(&parameters, &tool.discovery_schema()) + } else { + parameters + }; + // ── Per-step call limit (prevent amplification loops) ── const MAX_CALLS_PER_STEP: u32 = 50; let count = self @@ -1394,7 +1410,7 @@ impl EffectBridgeAdapter { credential = %cred.credential_name, tool = %lookup_name, user = %context.user_id, - "Pre-flight auth: credential missing — blocking tool call" + "Pre-flight auth: credential missing — raising Authentication gate" ); return Err(Self::gate_paused( "authentication", @@ -1538,15 +1554,10 @@ impl EffectBridgeAdapter { // prompt-injection / param validation must run). let mounts_snapshot = self.workspace_mounts.read().await.as_ref().map(Arc::clone); let sandbox_result = if let Some(mounts) = mounts_snapshot { - // Normalize parameters the same way the host path does - // (`execute_tool_with_safety` → `prepare_tool_params`) so - // validation sees consistent types (e.g. string "true" → bool). - let normalized = if let Some(tool) = self.tools.get(&lookup_name).await { - crate::tools::prepare_tool_params(tool.as_ref(), &parameters) - } else { - parameters.clone() - }; - let validation = self.safety.validator().validate_tool_params(&normalized); + // `parameters` was already coerced by the schema-guided + // pre-amble above; both the validator and the mount backend + // see the same shape that `execute_tool_with_safety` would. + let validation = self.safety.validator().validate_tool_params(&parameters); if !validation.is_valid { let details = validation .errors @@ -1561,7 +1572,7 @@ impl EffectBridgeAdapter { }, ))) } else { - match maybe_intercept(&lookup_name, &normalized, context.project_id, &mounts).await + match maybe_intercept(&lookup_name, &parameters, context.project_id, &mounts).await { Ok(InterceptOutcome::Handled(s)) => Some(Ok(s)), Ok(InterceptOutcome::FellThrough) => None, @@ -1617,8 +1628,7 @@ impl EffectBridgeAdapter { let output_value = serde_json::from_str::<serde_json::Value>(&output) .unwrap_or(serde_json::Value::String(wrapped)); - if (lookup_name == "tool_activate" - || lookup_name == "tool_auth" + if (lookup_name == "tool_auth" || lookup_name == "tool_install" || lookup_name == "tool-install") && let Some(err) = Self::auth_gate_from_extension_result( @@ -1804,34 +1814,6 @@ impl EffectBridgeAdapter { } } -fn extension_name_matches(extension_name: &str, requested_name: &str) -> bool { - let requested_candidates = extension_name_candidates(requested_name) - .into_iter() - .collect::<HashSet<_>>(); - extension_name_candidates(extension_name) - .into_iter() - .any(|candidate| requested_candidates.contains(&candidate)) -} - -fn matching_extension_requires_install_approval( - requested_name: &str, - extensions: &[InstalledExtension], -) -> Option<bool> { - let mut saw_match = false; - - for extension in extensions { - if !extension_name_matches(&extension.name, requested_name) { - continue; - } - if extension.installed { - return Some(false); - } - saw_match = true; - } - - saw_match.then_some(true) -} - #[async_trait::async_trait] impl EffectExecutor for EffectBridgeAdapter { async fn execute_action( @@ -1841,8 +1823,74 @@ impl EffectExecutor for EffectBridgeAdapter { lease: &CapabilityLease, context: &ThreadExecutionContext, ) -> Result<ActionResult, EngineError> { - self.execute_action_internal(action_name, parameters, lease, context, false) - .await + // External-tool short-circuit. If the per-thread catalog claims + // this action name, the caller will execute it; we pause the + // thread with `ResumeKind::External { ext_tool:<call_id> }` and + // wait for the resume payload. + // + // Parameters skip dispatch-time validation (the caller's tool + // schema isn't registered with the host), but the resume + // payload is run through `SafetyLayer::sanitize_tool_output` + // in `bridge::router` before reaching the LLM — see the + // `is_external_tool_callback` branch in `resolve_gate`. + // + // Known limitation (multi-call batching): the engine pauses on + // the first external-tool invocation in an assistant turn. If + // the LLM emits N caller-tool calls together, only the first + // surfaces as `AppEvent::ExternalToolCall`; subsequent calls + // re-emit on the next assistant turn after the caller posts + // back the first result. The OpenAI Responses contract allows + // "post all N results together"; matching that needs an engine + // change to collect N pauses before unwinding (tracked as a + // follow-up to PR #3122). + if let Some(catalog) = self.external_tool_catalog().await { + let mut hit = false; + for key in Self::external_tool_catalog_keys(context) { + if catalog.contains(key, action_name).await { + hit = true; + break; + } + } + if hit { + // Synthesize a call_id when the executor didn't stamp + // one (Tier 1 / CodeAct paths can reach here without a + // structured call envelope). Without a stable id, the + // resume payload can't be correlated back to the + // originating action — the `function_call_output` would + // arrive with the caller's id but the gate would carry + // `ext_tool:` with no suffix. + let call_id = match context.current_call_id.as_deref() { + Some(id) if !id.is_empty() => id.to_string(), + _ => format!("call_ext_{}", uuid::Uuid::new_v4().simple()), + }; + return Err(Self::gate_paused( + "external_tool", + action_name, + Some(call_id.as_str()), + parameters, + ironclaw_engine::ResumeKind::External { + callback_id: crate::bridge::external_tool_callback_id(&call_id), + }, + None, + Some(lease.clone()), + )); + } + } + + // Honor the engine's one-shot approval flag. Set by inline + // gate-await retry paths after the user resolves the gate; + // mirrors the legacy `execute_resolved_pending_action` path + // that passes `approval_already_granted=true` to skip the + // per-call approval check that would otherwise re-fire. + let approval_already_granted = context.call_approval_granted; + self.execute_action_internal( + action_name, + parameters, + lease, + context, + approval_already_granted, + ) + .await } async fn available_actions( @@ -1866,7 +1914,7 @@ impl EffectExecutor for EffectBridgeAdapter { let extensions = self .fetch_extension_map(auth_manager.as_deref(), context) .await; - ActionProjector::project_inventory( + let mut inventory = ActionProjector::project_inventory( self.tools.as_ref(), auth_manager.as_deref(), capability_registry, @@ -1874,7 +1922,44 @@ impl EffectExecutor for EffectBridgeAdapter { context, extensions.as_ref(), ) - .await + .await?; + + // Merge per-thread external tools (Responses API caller-supplied + // `tools[]`) into the inline action surface so the LLM sees them + // as callable. Caller tools are not gated by leases or admin + // policy: they're owned by the caller end-to-end. Names are + // de-duplicated against the existing inline set; collisions are + // rejected up-front by the Responses API handler, but the + // dedup keeps a defensive ordering invariant: internal beats + // external if they ever collide. + // + // The lookup walks both the engine `thread_id` and the + // caller-side `conversation_scope` because the responses_api + // handler registers under the latter and the bridge re-keys + // post-spawn. The executor task can poll + // `available_action_inventory` before the re-key lands, so we + // need both keys to close the race. + if let Some(catalog) = self.external_tool_catalog().await { + let mut external: Vec<ActionDef> = Vec::new(); + for key in Self::external_tool_catalog_keys(context) { + let entries = catalog.list(key).await; + if !entries.is_empty() { + external = entries; + break; + } + } + if !external.is_empty() { + let existing: std::collections::HashSet<&str> = + inventory.inline.iter().map(|a| a.name.as_str()).collect(); + let extras: Vec<ActionDef> = external + .into_iter() + .filter(|a| !existing.contains(a.name.as_str())) + .collect(); + inventory.inline.extend(extras); + } + } + + Ok(inventory) } async fn available_capabilities( @@ -2714,6 +2799,31 @@ fn extract_credential_name(error_msg: &str) -> Option<String> { None } +/// Look up the bridge-canonical schema for `mission_*` actions, the only +/// engine-native action category that has no corresponding host `Tool` +/// registration. `routine_*` (legacy v1 host tools, intercepted by the +/// alias path before they execute) and `tool_info` (a v1/v2 host tool) +/// are present in the host `ToolRegistry`, so they reach +/// `execute_action_internal`'s registry branch directly and don't need +/// this helper. Used in two places: +/// +/// 1. As a fallback in `execute_action_internal` when the orchestrator +/// has not populated `available_actions_snapshot` — primarily tests +/// that drive `execute_action` without setting up the snapshot. +/// 2. To stay coupled to the `mission_capability_actions()` definitions +/// so coercion in #1 always uses the same JSON Schema the engine +/// advertises to the LLM. +/// +/// In production paths the orchestrator always populates the snapshot, +/// so the snapshot branch wins and this helper is a defense-in-depth +/// fallback. +fn engine_action_schema(action_name: &str) -> Option<serde_json::Value> { + crate::bridge::engine_actions::mission_capability_actions() + .into_iter() + .find(|action| action.matches_name(action_name)) + .map(|action| action.parameters_schema) +} + pub(crate) fn is_v1_only_tool(name: &str) -> bool { // routine_* tools are surfaced in v2 too, but are intercepted by // `handle_mission_call`'s routine alias path *before* this check fires — @@ -2889,6 +2999,15 @@ mod tests { struct DefaultAllowNamedApprovalTestTool; + /// Stand-in for `tool_install`. Its `name()` matches the canonical + /// seeded-`AskEachTime` baseline so the explicit-equals-seeded + /// regression in `explicit_ask_each_time_for_seeded_default_tool_still_gates` + /// actually exercises the value-equality codepath. Mirrors the real + /// `tool_install`'s `UnlessAutoApproved` approval requirement so the + /// `enforce_tool_permission` branch under test (AskEachTime → + /// is_explicit_ask check) is reached. + struct SeededAskEachTimeTestTool; + #[async_trait] impl Tool for ApprovalTestTool { fn name(&self) -> &str { @@ -2994,6 +3113,41 @@ mod tests { } } + #[async_trait] + impl Tool for SeededAskEachTimeTestTool { + fn name(&self) -> &str { + "tool_install" + } + + fn description(&self) -> &str { + "Test stand-in for tool_install; name matches a seeded-AskEachTime baseline" + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "name": { "type": "string" } + } + }) + } + + async fn execute( + &self, + params: serde_json::Value, + _ctx: &JobContext, + ) -> Result<ToolOutput, ToolError> { + Ok(ToolOutput::success( + serde_json::json!({ "installed": params }), + std::time::Duration::from_millis(1), + )) + } + + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + ApprovalRequirement::UnlessAutoApproved + } + } + fn lease() -> ironclaw_engine::CapabilityLease { ironclaw_engine::CapabilityLease { id: ironclaw_engine::types::capability::LeaseId::new(), @@ -3025,24 +3179,10 @@ mod tests { thread_goal: Some("test goal".to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, - } - } - - fn installed_extension(name: &str) -> InstalledExtension { - InstalledExtension { - name: name.to_string(), - kind: crate::extensions::ExtensionKind::McpServer, - display_name: Some(name.to_string()), - description: Some(format!("{name} description")), - url: None, - authenticated: true, - active: true, - tools: vec![format!("{name}_search")], - needs_setup: false, - has_auth: true, - installed: true, - activation_error: None, - version: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } @@ -3869,6 +4009,92 @@ mod tests { } } + /// #3559 security review (finding 1): when a user explicitly sets a + /// tool's permission to a value that happens to match the code-level + /// seeded default (e.g. `tool_install` → `AskEachTime`, which is also + /// the seeded baseline), pre-#3559's `ToolPermissionSnapshot::resolve_permission` + /// collapsed the DB row to `explicit = None`, then this function's + /// `is_explicit_ask` check failed, and `auto_approve_tools` bypassed + /// the gate — silently dropping the user's explicit choice. + /// + /// The companion regression at `bridge::tool_permissions::tests::user_explicit_value_matching_seeded_default_stays_explicit` + /// covers the resolver in isolation. Per `.claude/rules/testing.md` + /// "Test Through the Caller", this test additionally drives the + /// side-effecting call site (`execute_action` → `enforce_tool_permission`) + /// with a tool whose name matches a seeded default in `seeded_default_permission`, + /// so the value-equality bug would surface here if it were ever + /// reintroduced in the resolver. + #[tokio::test] + async fn explicit_ask_each_time_for_seeded_default_tool_still_gates() { + let db_path = std::env::temp_dir().join(format!( + "ironclaw-seeded-ask-each-time-{}.db", + uuid::Uuid::new_v4() + )); + let db = crate::db::connect_from_config(&crate::config::DatabaseConfig::from_libsql_path( + db_path.to_str().expect("db path"), + None, + None, + )) + .await + .expect("db"); + // Write a user-explicit `AskEachTime` for `tool_install`. Confirm + // that `seeded_default_permission("tool_install") == AskEachTime` + // — if the seeded baseline ever changes, this test must be + // updated to keep its value-equality coverage meaningful. + assert_eq!( + crate::tools::permissions::seeded_default_permission("tool_install"), + Some(crate::tools::permissions::PermissionState::AskEachTime), + "this regression test assumes tool_install's seeded baseline is AskEachTime; \ + if you changed it, point this test at a different seeded-AskEachTime tool" + ); + db.set_setting( + "test_user", + "tool_permissions.tool_install", + &serde_json::to_value(crate::tools::permissions::PermissionState::AskEachTime) + .expect("serialize permission"), + ) + .await + .expect("save tool permission"); + + let tools = Arc::new(ToolRegistry::new().with_database(db)); + tools.register(Arc::new(SeededAskEachTimeTestTool)).await; + let adapter = EffectBridgeAdapter::new( + tools, + Arc::new(SafetyLayer::new(&ironclaw_safety::SafetyConfig { + max_output_length: 10_000, + injection_check_enabled: false, + })), + Arc::new(HookRegistry::default()), + ) + .with_global_auto_approve(true); + + let err = adapter + .execute_action( + "tool_install", + serde_json::json!({"name": "gmail"}), + &lease(), + &exec_ctx( + ironclaw_engine::ThreadId::new(), + Some("call_seeded_ask_each_time"), + ), + ) + .await + .expect_err( + "user-explicit AskEachTime must gate even when value matches seeded default \ + and AGENT_AUTO_APPROVE_TOOLS=true", + ); + + match err { + EngineError::GatePaused { gate_name, .. } => { + assert_eq!( + gate_name, "approval", + "explicit user choice must surface as approval gate, not bypass" + ); + } + other => panic!("expected GatePaused, got {other:?}"), + } + } + #[tokio::test] async fn explicit_disabled_override_denies_tool_execution() { let adapter = make_approval_test_adapter_with_permission(Some( @@ -4161,7 +4387,7 @@ mod tests { ); } - /// Regression for nearai/ironclaw#2206: a `tool_activate`/`tool_auth` + /// Regression for nearai/ironclaw#2206: a `tool_install`/`tool_auth` /// extension result containing a non-https `auth_url` (e.g. /// `javascript:alert(1)`) must be sanitized to `None` before it reaches /// `ResumeKind::Authentication` and is forwarded onto the gate stream. @@ -4171,7 +4397,7 @@ mod tests { /// isolation, per the "Test Through the Caller, Not Just the Helper" /// rule in `.claude/rules/testing.md`. #[tokio::test] - async fn auth_gate_strips_non_https_auth_url_from_tool_activate_output() { + async fn auth_gate_strips_non_https_auth_url_from_tool_install_output() { use ironclaw_safety::SafetyConfig; struct OAuthPromptTool; @@ -4179,11 +4405,11 @@ mod tests { #[async_trait] impl Tool for OAuthPromptTool { fn name(&self) -> &str { - "tool_activate" + "tool_install" } fn description(&self) -> &str { - "Test stub for tool_activate that returns a malicious auth_url" + "Test stub for tool_install that returns a malicious auth_url" } fn parameters_schema(&self) -> serde_json::Value { @@ -4214,6 +4440,9 @@ mod tests { let tools = Arc::new(ToolRegistry::new()); tools.register(Arc::new(OAuthPromptTool)).await; + // tool_install normally pauses on UnlessAutoApproved before + // reaching the auth-gate path. Skip that approval gate so the + // test exercises only the auth_url sanitization path. let adapter = EffectBridgeAdapter::new( tools, Arc::new(SafetyLayer::new(&SafetyConfig { @@ -4221,11 +4450,12 @@ mod tests { injection_check_enabled: false, })), Arc::new(HookRegistry::default()), - ); + ) + .with_global_auto_approve(true); let result = adapter .execute_action( - "tool_activate", + "tool_install", serde_json::json!({}), &lease(), &exec_ctx( @@ -4261,7 +4491,7 @@ mod tests { /// Sibling regression: a well-formed `https://` auth_url must still /// flow through unmodified. Guards against an over-eager sanitizer. #[tokio::test] - async fn auth_gate_preserves_https_auth_url_from_tool_activate_output() { + async fn auth_gate_preserves_https_auth_url_from_tool_install_output() { use ironclaw_safety::SafetyConfig; struct OAuthPromptTool; @@ -4269,11 +4499,11 @@ mod tests { #[async_trait] impl Tool for OAuthPromptTool { fn name(&self) -> &str { - "tool_activate" + "tool_install" } fn description(&self) -> &str { - "Test stub for tool_activate that returns a valid auth_url" + "Test stub for tool_install that returns a valid auth_url" } fn parameters_schema(&self) -> serde_json::Value { @@ -4304,6 +4534,9 @@ mod tests { let tools = Arc::new(ToolRegistry::new()); tools.register(Arc::new(OAuthPromptTool)).await; + // tool_install normally pauses on UnlessAutoApproved before + // reaching the auth-gate path. Skip that approval gate so the + // test exercises only the auth_url sanitization path. let adapter = EffectBridgeAdapter::new( tools, Arc::new(SafetyLayer::new(&SafetyConfig { @@ -4311,11 +4544,12 @@ mod tests { injection_check_enabled: false, })), Arc::new(HookRegistry::default()), - ); + ) + .with_global_auto_approve(true); let result = adapter .execute_action( - "tool_activate", + "tool_install", serde_json::json!({}), &lease(), &exec_ctx( @@ -4687,9 +4921,14 @@ mod tests { #[test] fn extract_guardrails_rejects_string_typed_integers() { - // Regression: LLMs pass numeric params as strings (e.g. cooldown_secs="0"). - // The old code silently ignored the wrong type, so mission_update - // returned {"status":"updated"} but changed nothing in the database. + // Defense-in-depth: this helper is called directly on the raw + // params object and is the last line of defense if some future + // code path bypasses the schema-guided coercion that + // `execute_action_internal` now runs. A string-typed integer + // here means coercion didn't happen — fail loudly rather than + // silently dropping the value (the bug shape from before #2630). + // End-to-end coercion of `cooldown_secs="120"` is covered by + // `mission_create_string_guardrails_coerced_via_execute_action`. let params = serde_json::json!({"cooldown_secs": "0", "max_concurrent": "2"}); let mut updates = ironclaw_engine::MissionUpdate::default(); let err = extract_guardrails(&params, &mut updates).unwrap_err(); @@ -4791,6 +5030,10 @@ mod tests { ), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; assert!(should_reject_immediate_mission_create(&ctx)); @@ -4812,6 +5055,10 @@ mod tests { ), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; assert!(!should_reject_immediate_mission_create(&ctx)); @@ -4831,6 +5078,10 @@ mod tests { thread_goal: Some("Summarize every product feedback item right now.".to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; assert!(should_reject_immediate_mission_create(&ctx)); @@ -4850,6 +5101,10 @@ mod tests { thread_goal: Some("Set up the product feedback summary right now.".to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; assert!(should_reject_immediate_mission_create(&ctx)); @@ -4872,6 +5127,10 @@ mod tests { thread_goal: Some("Set up monitoring now.".to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; // Should NOT be rejected — "monitoring" implies scheduling intent. @@ -4892,6 +5151,10 @@ mod tests { thread_goal: Some("Summarize feedback immediately.".to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; assert!(!should_reject_immediate_mission_create(&ctx)); @@ -5116,6 +5379,10 @@ mod tests { thread_goal: Some(goal.to_string()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } @@ -5308,8 +5575,6 @@ mod tests { fn auth_tools_are_v1_auth() { assert!(is_v1_auth_tool("tool_auth")); assert!(is_v1_auth_tool("tool-auth")); - assert!(!is_v1_auth_tool("tool_activate")); - assert!(!is_v1_auth_tool("tool-activate")); } #[test] @@ -5321,52 +5586,6 @@ mod tests { assert!(!is_v1_auth_tool("tool_list")); } - #[test] - fn install_approval_prefers_installed_alias_over_registry_only_match() { - let installed = installed_extension("linear-server"); - let registry_only = InstalledExtension { - installed: false, - active: false, - authenticated: false, - has_auth: true, - tools: Vec::new(), - ..installed_extension("linear_server") - }; - - let decision = matching_extension_requires_install_approval( - "linear_server", - &[registry_only, installed], - ); - - assert_eq!(decision, Some(false)); - } - - #[test] - fn install_approval_requires_confirmation_for_uninstalled_match() { - let registry_only = InstalledExtension { - installed: false, - active: false, - authenticated: false, - has_auth: true, - tools: Vec::new(), - ..installed_extension("web_search") - }; - - let decision = matching_extension_requires_install_approval("web_search", &[registry_only]); - - assert_eq!(decision, Some(true)); - } - - #[test] - fn install_approval_ignores_unknown_extension_names() { - let decision = matching_extension_requires_install_approval( - "missing_tool", - &[installed_extension("web_search")], - ); - - assert_eq!(decision, None); - } - // ── Pre-flight auth gate integration test ───────────────── #[tokio::test] @@ -5445,6 +5664,10 @@ mod tests { thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; let result = adapter.execute_action("http", params, &lease, &ctx).await; @@ -5469,20 +5692,17 @@ mod tests { } #[tokio::test] - async fn tool_activate_seeded_always_allow_reaches_auth_gate() { - use crate::secrets::InMemorySecretsStore; - use crate::secrets::SecretsCrypto; - - struct ActivateTool; + async fn tool_install_post_install_auth_gate_preserves_secret_name_for_resume() { + struct InstallTool; #[async_trait] - impl Tool for ActivateTool { + impl Tool for InstallTool { fn name(&self) -> &str { - "tool_activate" + "tool_install" } fn description(&self) -> &str { - "activate" + "install" } fn parameters_schema(&self) -> serde_json::Value { @@ -5501,9 +5721,10 @@ mod tests { ) -> Result<ToolOutput, ToolError> { Ok(ToolOutput::success( serde_json::json!({ - "name": "notion", - "status": "awaiting_authorization", - "auth_url": "https://example.com/oauth", + "name": "telegram", + "status": "awaiting_token", + "credential_name": "telegram_bot_token", + "instructions": "Enter your Telegram Bot API token (from @BotFather)", }), std::time::Duration::from_millis(1), )) @@ -5511,7 +5732,7 @@ mod tests { } let tools = Arc::new(ToolRegistry::new()); - tools.register(Arc::new(ActivateTool)).await; + tools.register(Arc::new(InstallTool)).await; let adapter = EffectBridgeAdapter::new( Arc::clone(&tools), @@ -5520,19 +5741,8 @@ mod tests { injection_check_enabled: false, })), Arc::new(HookRegistry::default()), - ); - let key = secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex()); - let crypto = Arc::new(SecretsCrypto::new(key).expect("crypto")); - let secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = - Arc::new(InMemorySecretsStore::new(crypto)); - adapter - .set_auth_manager(Arc::new(AuthManager::new( - secrets, - None, - None, - Some(Arc::clone(&tools)), - ))) - .await; + ) + .with_global_auto_approve(true); let lease = ironclaw_engine::CapabilityLease { id: ironclaw_engine::types::capability::LeaseId::new(), @@ -5552,333 +5762,22 @@ mod tests { project_id: ironclaw_engine::ProjectId::new(), user_id: "test_user".to_string(), step_id: ironclaw_engine::StepId::new(), - current_call_id: Some("call_123".to_string()), + current_call_id: Some("call_install".to_string()), source_channel: None, user_timezone: None, thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; let result = adapter .execute_action( - "tool_activate", - serde_json::json!({"name": "notion"}), - &lease, - &ctx, - ) - .await; - - match result { - Err(EngineError::GatePaused { - gate_name, - action_name, - resume_kind, - .. - }) => { - assert_eq!(gate_name, "authentication"); - assert_eq!(action_name, "tool_activate"); - match *resume_kind { - ironclaw_engine::ResumeKind::Authentication { - credential_name, - auth_url, - .. - } => { - assert_eq!(credential_name, "notion"); - assert_eq!(auth_url.as_deref(), Some("https://example.com/oauth")); - } - other => panic!("expected authentication resume kind, got {other:?}"), - } - } - other => panic!("expected auth gate pause, got {other:?}"), - } - } - - #[tokio::test] - async fn tool_activate_requires_install_approval_before_auto_installing_integration() { - use crate::extensions::{AuthHint, ExtensionKind, ExtensionSource, RegistryEntry}; - use crate::secrets::InMemorySecretsStore; - use crate::secrets::SecretsCrypto; - use crate::tools::builtin::extension_tools::ToolActivateTool; - use crate::tools::mcp::process::McpProcessManager; - use crate::tools::mcp::session::McpSessionManager; - - let dir = tempfile::tempdir().expect("temp dir"); - - let key = secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex()); - let crypto = Arc::new(SecretsCrypto::new(key).expect("crypto")); - let secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = - Arc::new(InMemorySecretsStore::new(crypto)); - - let tools = Arc::new(ToolRegistry::new()); - let ext_mgr = Arc::new(crate::extensions::ExtensionManager::new( - Arc::new(McpSessionManager::new()), - Arc::new(McpProcessManager::new()), - Arc::clone(&secrets), - Arc::clone(&tools), - None, - None, - dir.path().join("tools"), - dir.path().join("channels"), - None, - "test_user".to_string(), - None, - vec![RegistryEntry { - name: "web_search".to_string(), - display_name: "Web Search".to_string(), - kind: ExtensionKind::WasmTool, - description: "Search the web".to_string(), - keywords: vec!["search".into(), "web".into()], - source: ExtensionSource::WasmDownload { - wasm_url: "https://example.com/web_search.wasm".to_string(), - capabilities_url: None, - }, - fallback_source: None, - auth_hint: AuthHint::CapabilitiesAuth, - version: None, - }], - )); - tools - .register(Arc::new(ToolActivateTool::new(Arc::clone(&ext_mgr)))) - .await; - - let adapter = EffectBridgeAdapter::new( - Arc::clone(&tools), - Arc::new(SafetyLayer::new(&ironclaw_safety::SafetyConfig { - max_output_length: 10_000, - injection_check_enabled: false, - })), - Arc::new(HookRegistry::default()), - ); - adapter - .set_auth_manager(Arc::new(AuthManager::new( - secrets, - None, - Some(ext_mgr), - Some(Arc::clone(&tools)), - ))) - .await; - - let ctx = exec_ctx(ironclaw_engine::ThreadId::new(), Some("call_activate")); - let result = adapter - .execute_action( - "tool_activate", - serde_json::json!({"name": "web_search"}), - &lease(), - &ctx, - ) - .await; - - match result { - Err(EngineError::GatePaused { - gate_name, - action_name, - resume_kind, - .. - }) => { - assert_eq!(gate_name, "approval"); - assert_eq!(action_name, "tool_activate"); - match *resume_kind { - ironclaw_engine::ResumeKind::Approval { allow_always } => { - assert!(!allow_always); - } - other => panic!("expected approval resume kind, got {other:?}"), - } - } - other => panic!("expected approval gate pause, got {other:?}"), - } - } - - #[tokio::test] - async fn tool_activate_respects_ask_each_time_when_install_state_is_unavailable() { - use crate::extensions::{AuthHint, ExtensionKind, ExtensionSource, RegistryEntry}; - use crate::secrets::InMemorySecretsStore; - use crate::secrets::SecretsCrypto; - use crate::tools::builtin::extension_tools::ToolActivateTool; - use crate::tools::mcp::process::McpProcessManager; - use crate::tools::mcp::session::McpSessionManager; - use crate::tools::permissions::PermissionState; - - let dir = tempfile::tempdir().expect("temp dir"); - - let key = secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex()); - let crypto = Arc::new(SecretsCrypto::new(key).expect("crypto")); - let secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = - Arc::new(InMemorySecretsStore::new(crypto)); - - let db_path = std::env::temp_dir().join(format!( - "ironclaw-tool-activate-permissions-{}.db", - uuid::Uuid::new_v4() - )); - let db = crate::db::connect_from_config(&crate::config::DatabaseConfig::from_libsql_path( - db_path.to_str().expect("db path"), - None, - None, - )) - .await - .expect("db"); - db.set_setting( - "test_user", - "tool_permissions.tool_activate", - &serde_json::to_value(PermissionState::AskEachTime).expect("serialize permission"), - ) - .await - .expect("save tool permission"); - - let tools = Arc::new(ToolRegistry::new().with_database(db)); - let ext_mgr = Arc::new(crate::extensions::ExtensionManager::new( - Arc::new(McpSessionManager::new()), - Arc::new(McpProcessManager::new()), - Arc::clone(&secrets), - Arc::clone(&tools), - None, - None, - dir.path().join("tools"), - dir.path().join("channels"), - None, - "test_user".to_string(), - None, - vec![RegistryEntry { - name: "web_search".to_string(), - display_name: "Web Search".to_string(), - kind: ExtensionKind::WasmTool, - description: "Search the web".to_string(), - keywords: vec!["search".into(), "web".into()], - source: ExtensionSource::WasmDownload { - wasm_url: "https://example.com/web_search.wasm".to_string(), - capabilities_url: None, - }, - fallback_source: None, - auth_hint: AuthHint::CapabilitiesAuth, - version: None, - }], - )); - tools - .register(Arc::new(ToolActivateTool::new(Arc::clone(&ext_mgr)))) - .await; - - let adapter = EffectBridgeAdapter::new( - Arc::clone(&tools), - Arc::new(SafetyLayer::new(&ironclaw_safety::SafetyConfig { - max_output_length: 10_000, - injection_check_enabled: false, - })), - Arc::new(HookRegistry::default()), - ); - - let ctx = exec_ctx(ironclaw_engine::ThreadId::new(), Some("call_activate")); - let result = adapter - .execute_action( - "tool_activate", - serde_json::json!({"name": "web_search"}), - &lease(), - &ctx, - ) - .await; - - match result { - Err(EngineError::GatePaused { - gate_name, - action_name, - resume_kind, - .. - }) => { - assert_eq!(gate_name, "approval"); - assert_eq!(action_name, "tool_activate"); - match *resume_kind { - ironclaw_engine::ResumeKind::Approval { allow_always } => { - assert!(!allow_always); - } - other => panic!("expected approval resume kind, got {other:?}"), - } - } - other => panic!("expected approval gate pause, got {other:?}"), - } - } - - #[tokio::test] - async fn tool_install_post_install_auth_gate_preserves_secret_name_for_resume() { - struct InstallTool; - - #[async_trait] - impl Tool for InstallTool { - fn name(&self) -> &str { - "tool_install" - } - - fn description(&self) -> &str { - "install" - } - - fn parameters_schema(&self) -> serde_json::Value { - serde_json::json!({ - "type": "object", - "properties": { - "name": {"type": "string"} - } - }) - } - - async fn execute( - &self, - _params: serde_json::Value, - _ctx: &crate::context::JobContext, - ) -> Result<ToolOutput, ToolError> { - Ok(ToolOutput::success( - serde_json::json!({ - "name": "telegram", - "status": "awaiting_token", - "credential_name": "telegram_bot_token", - "instructions": "Enter your Telegram Bot API token (from @BotFather)", - }), - std::time::Duration::from_millis(1), - )) - } - } - - let tools = Arc::new(ToolRegistry::new()); - tools.register(Arc::new(InstallTool)).await; - - let adapter = EffectBridgeAdapter::new( - Arc::clone(&tools), - Arc::new(SafetyLayer::new(&ironclaw_safety::SafetyConfig { - max_output_length: 10_000, - injection_check_enabled: false, - })), - Arc::new(HookRegistry::default()), - ) - .with_global_auto_approve(true); - - let lease = ironclaw_engine::CapabilityLease { - id: ironclaw_engine::types::capability::LeaseId::new(), - thread_id: ironclaw_engine::ThreadId::new(), - capability_name: "tools".into(), - granted_actions: ironclaw_engine::GrantedActions::All, - granted_at: chrono::Utc::now(), - expires_at: None, - max_uses: None, - uses_remaining: None, - revoked: false, - revoked_reason: None, - }; - let ctx = ironclaw_engine::ThreadExecutionContext { - thread_id: ironclaw_engine::ThreadId::new(), - thread_type: ironclaw_engine::types::thread::ThreadType::Foreground, - project_id: ironclaw_engine::ProjectId::new(), - user_id: "test_user".to_string(), - step_id: ironclaw_engine::StepId::new(), - current_call_id: Some("call_install".to_string()), - source_channel: None, - user_timezone: None, - thread_goal: None, - available_actions_snapshot: None, - available_action_inventory_snapshot: None, - }; - - let result = adapter - .execute_action( - "tool_install", - serde_json::json!({"name": "telegram"}), + "tool_install", + serde_json::json!({"name": "telegram"}), &lease, &ctx, ) @@ -6264,7 +6163,14 @@ mod tests { } #[tokio::test] - async fn available_actions_omit_installed_needs_auth_provider_action() { + async fn available_actions_keep_installed_needs_auth_provider_action() { + // Post-#3133/#3166: an installed-but-unauthenticated provider + // tool (e.g. gmail) STAYS on the callable surface. The engine + // raises an Authentication gate at execute time when the + // declared credential is missing and the inline-await + // machinery resumes the action after OAuth completes. The + // model can call the tool directly with no separate enablement + // step. Pre-#3133 the action was hidden until auth completed. let fixture = make_adapter_with_installed_provider_fixture( "gmail", "gmail_send", @@ -6295,7 +6201,11 @@ mod tests { .available_actions(&[], &exec_ctx(ironclaw_engine::ThreadId::new(), None)) .await .expect("actions"); - assert!(!actions.iter().any(|action| action.name == "gmail_send")); + assert!( + actions.iter().any(|action| action.name == "gmail_send"), + "NeedsAuth provider tool should be callable; auth resolves at \ + execute time via inline-await. actions={actions:?}" + ); } #[tokio::test] @@ -6470,6 +6380,10 @@ mod tests { thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; let capabilities = adapter @@ -7303,11 +7217,14 @@ Use this skill to set up a Pika meeting. ); } - /// Regression: mission_create with string-typed guardrails (e.g. - /// cooldown_secs="0") must be caught before creating the mission. + /// Regression for #3132: string-typed guardrails (e.g. + /// `cooldown_secs="120"`) must be coerced to integers per the action's + /// JSON Schema before reaching the handler. Previously rejected with + /// `'cooldown_secs' must be an integer, got "120"`. #[tokio::test] - async fn mission_create_string_guardrails_rejected_via_execute_action() { - let adapter = make_adapter_with_missions().await; + async fn mission_create_string_guardrails_coerced_via_execute_action() { + let (adapter, _store, dyn_store) = + make_adapter_with_missions_and_store(Arc::new(ToolRegistry::new())).await; let result = adapter .execute_action( "mission_create", @@ -7315,12 +7232,56 @@ Use this skill to set up a Pika meeting. "name": "test", "goal": "do stuff", "cadence": "manual", - "cooldown_secs": "300" + "cooldown_secs": "120", + "max_concurrent": "2", + "dedup_window_secs": "30", + "max_threads_per_day": "5", }), &lease(), &exec_ctx(ironclaw_engine::ThreadId::new(), Some("c3")), ) .await + .expect("string-typed guardrails should be coerced and succeed"); + + assert!(!result.is_error, "got error: {}", result.output); + let mission_id_str = result + .output + .get("mission_id") + .and_then(|v| v.as_str()) + .expect("should have mission_id"); + let mission_id = + ironclaw_engine::MissionId(uuid::Uuid::parse_str(mission_id_str).expect("uuid")); + let mission = dyn_store + .load_mission(mission_id) + .await + .expect("load_mission") + .expect("mission persisted"); + assert_eq!(mission.cooldown_secs, 120); + assert_eq!(mission.max_concurrent, 2); + assert_eq!(mission.dedup_window_secs, 30); + assert_eq!(mission.max_threads_per_day, 5); + } + + /// Non-coercible strings (e.g. `cooldown_secs="abc"`) still surface as + /// a clean error rather than being silently dropped. Coercion leaves + /// the value unchanged when it can't parse to the target type, and + /// `extract_guardrails`'s strict check then rejects loudly. + #[tokio::test] + async fn mission_create_non_coercible_string_guardrail_returns_error() { + let adapter = make_adapter_with_missions().await; + let result = adapter + .execute_action( + "mission_create", + serde_json::json!({ + "name": "test", + "goal": "do stuff", + "cadence": "manual", + "cooldown_secs": "abc", + }), + &lease(), + &exec_ctx(ironclaw_engine::ThreadId::new(), Some("c3b")), + ) + .await .expect("should return Ok with is_error=true"); assert!(result.is_error); @@ -7367,11 +7328,13 @@ Use this skill to set up a Pika meeting. ); } - /// Regression: mission_update with string-typed guardrails must be - /// caught at the execute_action level, not silently ignored. + /// Regression for #3132: `mission_update` with string-typed guardrails + /// must be coerced (not rejected) so LLM calls passing `"5"` for an + /// integer parameter succeed and persist the new value. #[tokio::test] - async fn mission_update_string_guardrails_rejected_via_execute_action() { - let adapter = make_adapter_with_missions().await; + async fn mission_update_string_guardrails_coerced_via_execute_action() { + let (adapter, _store, dyn_store) = + make_adapter_with_missions_and_store(Arc::new(ToolRegistry::new())).await; let ctx = exec_ctx(ironclaw_engine::ThreadId::new(), Some("u1")); // First create a mission to get an ID. @@ -7389,40 +7352,39 @@ Use this skill to set up a Pika meeting. .await .expect("create should succeed"); assert!(!create_result.is_error); - let mission_id = create_result + let mission_id_str = create_result .output .get("mission_id") .and_then(|v| v.as_str()) .expect("should have mission_id"); - // Now update with string-typed guardrails — should fail. + // Update with a string-typed integer — should be coerced and applied. let update_result = adapter .execute_action( "mission_update", serde_json::json!({ - "id": mission_id, + "id": mission_id_str, "max_concurrent": "5" }), &lease(), &ctx, ) .await - .expect("should return Ok with is_error=true"); + .expect("string-typed guardrails should be coerced and succeed"); assert!( - update_result.is_error, - "string guardrails should fail: {}", - update_result.output - ); - assert!( - update_result - .output - .get("error") - .and_then(|v| v.as_str()) - .is_some_and(|s| s.contains("must be an integer")), - "got: {}", + !update_result.is_error, + "update should succeed after coercion: {}", update_result.output ); + let mission_id = + ironclaw_engine::MissionId(uuid::Uuid::parse_str(mission_id_str).expect("uuid")); + let mission = dyn_store + .load_mission(mission_id) + .await + .expect("load_mission") + .expect("mission persisted"); + assert_eq!(mission.max_concurrent, 5); } /// Verify system_event cadence round-trips through mission_list. @@ -8526,4 +8488,142 @@ Use this skill to set up a Pika meeting. "safe_action should surface through the engine capability path: {names:?}" ); } + + /// Race-window regression: when the bridge has registered caller + /// tools under the conversation_scope (the responses_api handler's + /// pre-spawn key) and the engine task starts running before the + /// post-spawn `transfer` rebinds onto the engine `thread_id`, the + /// adapter must still surface those tools — looked up via the + /// `conversation_scope` field plumbed through `ThreadExecutionContext`. + #[tokio::test] + async fn available_action_inventory_falls_back_to_conversation_scope() { + let adapter = make_adapter(); + let catalog = Arc::new(crate::bridge::ExternalToolCatalog::new()); + adapter + .set_external_tool_catalog(Arc::clone(&catalog)) + .await; + + let scope_uuid = uuid::Uuid::new_v4(); + let engine_thread_id = ironclaw_engine::ThreadId::new(); + assert_ne!( + scope_uuid, engine_thread_id.0, + "test setup: scope and engine thread must differ" + ); + + catalog + .register( + ironclaw_engine::ThreadId(scope_uuid), + vec![ironclaw_engine::ActionDef { + name: "lookup_weather".to_string(), + description: "caller tool".to_string(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![ironclaw_engine::EffectType::Compute], + requires_approval: false, + model_tool_surface: ironclaw_engine::ModelToolSurface::FullSchema, + discovery: None, + }], + ) + .await; + + let mut ctx = exec_ctx(engine_thread_id, None); + ctx.conversation_scope = Some(scope_uuid); + + let inventory = adapter + .available_action_inventory(&[], &ctx) + .await + .expect("inventory"); + assert!( + inventory.inline.iter().any(|a| a.name == "lookup_weather"), + "caller tool registered under scope must surface even before \ + post-spawn transfer; inline = {:?}", + inventory + .inline + .iter() + .map(|a| a.name.as_str()) + .collect::<Vec<_>>() + ); + } + + /// Same race-window regression for `execute_action`: a tool name + /// registered under the conversation_scope must short-circuit to a + /// `GatePaused { External }` even when the engine `thread_id` + /// hasn't received the catalog entry yet. + #[tokio::test] + async fn execute_action_short_circuits_via_conversation_scope() { + let adapter = make_adapter(); + let catalog = Arc::new(crate::bridge::ExternalToolCatalog::new()); + adapter + .set_external_tool_catalog(Arc::clone(&catalog)) + .await; + + let scope_uuid = uuid::Uuid::new_v4(); + let engine_thread_id = ironclaw_engine::ThreadId::new(); + catalog + .register( + ironclaw_engine::ThreadId(scope_uuid), + vec![ironclaw_engine::ActionDef { + name: "lookup_weather".to_string(), + description: "caller tool".to_string(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![ironclaw_engine::EffectType::Compute], + requires_approval: false, + model_tool_surface: ironclaw_engine::ModelToolSurface::FullSchema, + discovery: None, + }], + ) + .await; + + let mut ctx = exec_ctx(engine_thread_id, Some("call_xyz")); + ctx.conversation_scope = Some(scope_uuid); + + let result = adapter + .execute_action( + "lookup_weather", + serde_json::json!({"city": "NYC"}), + &lease(), + &ctx, + ) + .await; + + match result { + Err(EngineError::GatePaused { resume_kind, .. }) => { + assert!( + matches!( + &*resume_kind, + ironclaw_engine::ResumeKind::External { callback_id } + if callback_id.starts_with("ext_tool:") + ), + "expected ResumeKind::External(ext_tool:...), got {resume_kind:?}" + ); + } + other => { + panic!("expected GatePaused(External) when catalog hit via scope; got {other:?}") + } + } + } + + /// Sanity guard: the keys helper yields the engine `thread_id` + /// first (so post-rebind lookups stay fast) and only emits the + /// `conversation_scope` when it differs. Same-key contexts must not + /// trigger a duplicate lookup. + #[test] + fn external_tool_catalog_keys_dedupes_when_scope_equals_thread() { + let thread_id = ironclaw_engine::ThreadId::new(); + + let mut ctx_no_scope = exec_ctx(thread_id, None); + ctx_no_scope.conversation_scope = None; + let keys: Vec<_> = EffectBridgeAdapter::external_tool_catalog_keys(&ctx_no_scope).collect(); + assert_eq!(keys, vec![thread_id]); + + let mut ctx_same = exec_ctx(thread_id, None); + ctx_same.conversation_scope = Some(thread_id.0); + let keys: Vec<_> = EffectBridgeAdapter::external_tool_catalog_keys(&ctx_same).collect(); + assert_eq!(keys, vec![thread_id], "scope == thread must dedupe"); + + let mut ctx_diff = exec_ctx(thread_id, None); + let scope = uuid::Uuid::new_v4(); + ctx_diff.conversation_scope = Some(scope); + let keys: Vec<_> = EffectBridgeAdapter::external_tool_catalog_keys(&ctx_diff).collect(); + assert_eq!(keys, vec![thread_id, ironclaw_engine::ThreadId(scope)]); + } } diff --git a/src/bridge/external_tools.rs b/src/bridge/external_tools.rs new file mode 100644 index 00000000000..b5da76ec068 --- /dev/null +++ b/src/bridge/external_tools.rs @@ -0,0 +1,325 @@ +//! Per-thread catalog of caller-provided external tools. +//! +//! The Responses API (`/v1/responses`) lets clients declare their own +//! `function`-typed tools alongside the agent's internal action surface. +//! Those tools are not registered through `ToolRegistry` — they are +//! caller-executed: the LLM emits a structured tool call, the engine +//! pauses with `ResumeKind::External`, and the caller posts the result +//! back as a `function_call_output` item. +//! +//! This catalog is the per-thread mapping the bridge consults to +//! distinguish caller tools from internal actions: +//! +//! - `register(thread_id, actions)` — called by the Responses API +//! handler before the request reaches the agent loop. +//! - `list(thread_id)` — `EffectBridgeAdapter::available_actions` +//! merges these into the LLM-visible action surface so the model +//! sees the caller tools as callable. +//! - `contains(thread_id, name)` — `EffectBridgeAdapter::execute_action` +//! short-circuits to a `GatePaused { resume_kind: External { ... } }` +//! error for any name in the catalog. +//! - `clear(thread_id)` — invoked when a thread reaches a terminal +//! state so the entry doesn't leak. +//! - `sweep_older_than(max_age)` — backstop TTL eviction for threads +//! that get stuck in `Waiting` because the caller never POSTed back. + +use std::collections::HashMap; + +use chrono::{DateTime, Duration, Utc}; +use ironclaw_engine::{ActionDef, ThreadId}; +use tokio::sync::RwLock; + +/// Callback-id prefix used for external-tool pause gates. The bridge +/// router's projection uses this prefix to distinguish caller-tool +/// pauses (which surface as `AppEvent::ExternalToolCall` for the +/// Responses API) from OAuth/pairing pauses (which keep going through +/// the existing `AppEvent::GateRequired` channel). +pub const EXTERNAL_TOOL_CALLBACK_PREFIX: &str = "ext_tool:"; + +/// Build a fully-qualified callback id for an external tool pause. +/// +/// The `call_id` is the LLM-emitted tool call identifier (e.g. +/// `call_AbCd123…`); we stamp it onto the prefix so the resume payload +/// can be matched back to the originating action call without a +/// secondary lookup. +pub fn external_tool_callback_id(call_id: &str) -> String { + format!("{EXTERNAL_TOOL_CALLBACK_PREFIX}{call_id}") +} + +/// Returns true when a callback id was produced by `external_tool_callback_id`. +pub fn is_external_tool_callback_id(callback_id: &str) -> bool { + callback_id.starts_with(EXTERNAL_TOOL_CALLBACK_PREFIX) +} + +/// Strip the external-tool prefix from a callback id, returning the +/// embedded `call_id`. Returns `None` if the callback id was not +/// produced by `external_tool_callback_id`. +pub fn call_id_from_external_callback(callback_id: &str) -> Option<&str> { + callback_id.strip_prefix(EXTERNAL_TOOL_CALLBACK_PREFIX) +} + +/// One catalog entry: the caller-provided action defs plus when they +/// were registered (used by the TTL sweep). +#[derive(Debug, Clone)] +pub struct ExternalToolEntry { + pub actions: Vec<ActionDef>, + pub registered_at: DateTime<Utc>, +} + +/// Per-thread registry of caller-provided external tools. +/// +/// Single instance lives on the bridge, shared via `Arc` between the +/// Responses API handler (writer) and the `EffectBridgeAdapter` +/// (reader). +#[derive(Debug, Default)] +pub struct ExternalToolCatalog { + inner: RwLock<HashMap<ThreadId, ExternalToolEntry>>, +} + +impl ExternalToolCatalog { + pub fn new() -> Self { + Self { + inner: RwLock::new(HashMap::new()), + } + } + + /// Replace the catalog entry for `thread_id` with `actions`. + /// Updating instead of merging matches the Responses API contract: + /// each request restates the full `tools[]` list. + pub async fn register(&self, thread_id: ThreadId, actions: Vec<ActionDef>) { + let mut map = self.inner.write().await; + map.insert( + thread_id, + ExternalToolEntry { + actions, + registered_at: Utc::now(), + }, + ); + } + + /// Snapshot the registered action defs for a thread. Empty vec if + /// nothing is registered. + pub async fn list(&self, thread_id: ThreadId) -> Vec<ActionDef> { + let map = self.inner.read().await; + map.get(&thread_id) + .map(|entry| entry.actions.clone()) + .unwrap_or_default() + } + + /// Whether `action_name` is in this thread's catalog. + pub async fn contains(&self, thread_id: ThreadId, action_name: &str) -> bool { + let map = self.inner.read().await; + map.get(&thread_id) + .map(|entry| entry.actions.iter().any(|a| a.name == action_name)) + .unwrap_or(false) + } + + /// Drop the entry for `thread_id`. Called when a thread reaches a + /// terminal state, or when the caller explicitly cancels. + pub async fn clear(&self, thread_id: ThreadId) { + let mut map = self.inner.write().await; + map.remove(&thread_id); + } + + /// Move the entry registered under `from` to `to`. Used by the + /// engine bridge to bridge the gap between the responses_api + /// handler (which registers under the conversation_scope UUID it + /// generated) and the engine's actual `ThreadId` (which is only + /// known after `ConversationManager::handle_user_message` returns). + /// + /// Semantics: + /// - If `from == to`, no-op. + /// - If `from` has no entry, no-op (the request didn't supply tools). + /// - Otherwise the entry at `from` overwrites whatever was at `to`. + /// The Responses API contract is "each request restates the full + /// tools[] list" — a follow-up request supersedes the prior + /// registration on the same engine thread. + pub async fn transfer(&self, from: ThreadId, to: ThreadId) { + if from == to { + return; + } + let mut map = self.inner.write().await; + let Some(entry) = map.remove(&from) else { + return; + }; + map.insert(to, entry); + } + + /// Evict entries older than `max_age`. Returns the thread ids that + /// were dropped. Backstop for callers that abandon a paused thread. + pub async fn sweep_older_than(&self, max_age: Duration) -> Vec<ThreadId> { + let cutoff = Utc::now() - max_age; + let mut map = self.inner.write().await; + let stale: Vec<ThreadId> = map + .iter() + .filter(|(_, entry)| entry.registered_at < cutoff) + .map(|(id, _)| *id) + .collect(); + for id in &stale { + map.remove(id); + } + stale + } + + /// Number of registered threads. For diagnostics / metrics. + pub async fn len(&self) -> usize { + self.inner.read().await.len() + } + + pub async fn is_empty(&self) -> bool { + self.inner.read().await.is_empty() + } + + /// Whether any registered thread (regardless of key) has an entry + /// for `action_name`. Lets callers verify cleanup of caller tools + /// without needing to know the engine's allocated `ThreadId` — + /// useful when the registration key was a conversation_scope and + /// the bridge has since rebound it via `transfer`. + pub async fn contains_action_anywhere(&self, action_name: &str) -> bool { + let map = self.inner.read().await; + map.values() + .any(|entry| entry.actions.iter().any(|a| a.name == action_name)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ironclaw_engine::{EffectType, ModelToolSurface}; + + fn action(name: &str) -> ActionDef { + ActionDef { + name: name.to_string(), + description: format!("test {name}"), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::Compute], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + } + } + + #[tokio::test] + async fn register_then_list_returns_actions() { + let catalog = ExternalToolCatalog::new(); + let thread_id = ThreadId::new(); + catalog + .register(thread_id, vec![action("lookup"), action("convert")]) + .await; + let listed = catalog.list(thread_id).await; + assert_eq!(listed.len(), 2); + assert_eq!(listed[0].name, "lookup"); + assert_eq!(listed[1].name, "convert"); + } + + #[tokio::test] + async fn contains_matches_registered_name() { + let catalog = ExternalToolCatalog::new(); + let thread_id = ThreadId::new(); + catalog.register(thread_id, vec![action("lookup")]).await; + assert!(catalog.contains(thread_id, "lookup").await); + assert!(!catalog.contains(thread_id, "other").await); + let other = ThreadId::new(); + assert!(!catalog.contains(other, "lookup").await); + } + + #[tokio::test] + async fn register_replaces_existing_entry() { + let catalog = ExternalToolCatalog::new(); + let thread_id = ThreadId::new(); + catalog.register(thread_id, vec![action("a")]).await; + catalog + .register(thread_id, vec![action("b"), action("c")]) + .await; + let listed = catalog.list(thread_id).await; + assert_eq!(listed.len(), 2); + assert!(listed.iter().any(|a| a.name == "b")); + assert!(!listed.iter().any(|a| a.name == "a")); + } + + #[tokio::test] + async fn clear_removes_entry() { + let catalog = ExternalToolCatalog::new(); + let thread_id = ThreadId::new(); + catalog.register(thread_id, vec![action("a")]).await; + assert_eq!(catalog.len().await, 1); + catalog.clear(thread_id).await; + assert!(catalog.is_empty().await); + } + + #[tokio::test] + async fn sweep_evicts_old_entries_only() { + let catalog = ExternalToolCatalog::new(); + let fresh = ThreadId::new(); + let stale = ThreadId::new(); + catalog.register(fresh, vec![action("a")]).await; + catalog.register(stale, vec![action("b")]).await; + // Backdate the stale entry by mutating the inner map directly. + // Tests own the RwLock, so this is fine. + { + let mut map = catalog.inner.write().await; + if let Some(entry) = map.get_mut(&stale) { + entry.registered_at = Utc::now() - Duration::hours(2); + } + } + let evicted = catalog.sweep_older_than(Duration::hours(1)).await; + assert_eq!(evicted, vec![stale]); + assert!(!catalog.contains(stale, "b").await); + assert!(catalog.contains(fresh, "a").await); + } + + #[tokio::test] + async fn transfer_moves_entry_overwriting_destination() { + let catalog = ExternalToolCatalog::new(); + let from = ThreadId::new(); + let to = ThreadId::new(); + catalog.register(from, vec![action("fresh")]).await; + catalog.register(to, vec![action("stale")]).await; + + catalog.transfer(from, to).await; + + // `from` is empty; `to` has the freshly registered entry. + assert!(catalog.list(from).await.is_empty()); + let listed = catalog.list(to).await; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].name, "fresh"); + } + + #[tokio::test] + async fn transfer_noop_when_from_empty() { + let catalog = ExternalToolCatalog::new(); + let from = ThreadId::new(); + let to = ThreadId::new(); + catalog.register(to, vec![action("only")]).await; + + catalog.transfer(from, to).await; + + // The destination keeps its entry; nothing was clobbered by an + // empty-source transfer. + let listed = catalog.list(to).await; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].name, "only"); + } + + #[tokio::test] + async fn transfer_noop_on_self() { + let catalog = ExternalToolCatalog::new(); + let tid = ThreadId::new(); + catalog.register(tid, vec![action("a")]).await; + + catalog.transfer(tid, tid).await; + + let listed = catalog.list(tid).await; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].name, "a"); + } + + #[test] + fn callback_id_round_trip() { + let cb = external_tool_callback_id("call_abc123"); + assert!(is_external_tool_callback_id(&cb)); + assert_eq!(call_id_from_external_callback(&cb), Some("call_abc123")); + assert!(!is_external_tool_callback_id("pairing:telegram")); + assert_eq!(call_id_from_external_callback("pairing:telegram"), None); + } +} diff --git a/src/bridge/gate_controller.rs b/src/bridge/gate_controller.rs new file mode 100644 index 00000000000..1fb0158e84e --- /dev/null +++ b/src/bridge/gate_controller.rs @@ -0,0 +1,992 @@ +//! Inline gate-await bridge controller. +//! +//! Implements [`ironclaw_engine::GateController`] for the bridge layer. +//! When the engine hits an `Approval` gate inside a live execution +//! (Tier 0 batch or Tier 1 CodeAct VM), it calls +//! [`BridgeGateController::pause`] which: +//! +//! 1. Builds and persists a [`PendingGate`] (existing UI machinery +//! discovers the prompt through the same store / SSE / channel +//! flow as before). +//! 2. Registers a [`oneshot::Sender`] keyed by `request_id` in a +//! process-wide registry shared with the resolve endpoint. +//! 3. Awaits the receiver. The future stays parked here, holding the +//! engine's call stack open, until the user resolves the gate. +//! +//! On the resolve side, [`GateResolutions::try_deliver`] looks up the +//! sender by `request_id` and hands the [`GateResolution`] back into +//! the suspended engine. The engine continues from the exact +//! suspension point — no re-entry, no replay, no double-execution of +//! prior side effects in the same step. +//! +//! ## Single instance, per-thread context +//! +//! The controller is a single shared instance (held by `EngineState`, +//! attached to `ThreadManager` at boot). Per-execution data +//! (conversation id, channel metadata, original message, scope thread +//! id) lives in a `HashMap` keyed by `(user_id, thread_id)`. The +//! bridge populates an entry before invoking +//! `ConversationManager::handle_user_message`; if a gate fires during +//! that execution, the controller looks up the entry to construct the +//! `PendingGate`. Stale entries (from a turn that completed without +//! gating) are removed by the bridge after the call. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use ironclaw_common::AppEvent; +use ironclaw_common::ExternalThreadId; +use ironclaw_engine::{ + ConversationId, GateController, GatePauseRequest, GateResolution, ResumeKind, ThreadId, +}; +use serde_json::Value as JsonValue; +use tokio::sync::{Mutex, oneshot}; +use tracing::debug; +use uuid::Uuid; + +use crate::auth::extension::AuthManager; +use crate::channels::ChannelManager; +use crate::channels::StatusUpdate; +use crate::channels::web::sse::SseManager; +use crate::extensions::ExtensionManager; +use crate::gate::pending::PendingGate; +use crate::gate::store::PendingGateStore; +use crate::tools::ToolRegistry; + +/// Per-execution data the controller needs to build a `PendingGate`. +/// Populated by the bridge before invoking the engine for a turn, +/// removed after. +#[derive(Debug, Clone)] +pub struct PerExecutionContext { + pub conversation_id: ConversationId, + pub source_channel: String, + pub scope_thread_id: Option<ExternalThreadId>, + pub channel_metadata: JsonValue, + pub original_message: Option<String>, +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +struct ExecutionKey { + user_id: String, + thread_id: ThreadId, +} + +/// Pre-execution registry key. Keyed by `(user_id, conversation_id)` +/// so two concurrent conversations for the same user (e.g. two browser +/// tabs) don't clobber each other's pre-execution slot before each +/// turn has been promoted to its own `(user_id, thread_id)` entry. +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +struct PreExecKey { + user_id: String, + conversation_id: ConversationId, +} + +/// Process-wide registry of in-flight gate resolution channels. +/// +/// One entry per pending in-flight gate. Inserts come from +/// [`BridgeGateController::pause`]; removes come from +/// [`GateResolutions::try_deliver`] (the resolve endpoint). +/// +/// Authentication gates additionally register their `request_id` +/// under the `(user_id, credential_name)` pair they are waiting on, +/// so the OAuth callback path can wake the parked VM by credential +/// name without having to know the engine-internal request_id — +/// scoped per user so a credential write under one account never +/// wakes a parked gate from a different account that happens to share +/// the same credential name. +/// +/// Stranded entries from a prior crash do not exist — restarting the +/// process drops the registry. Stale `PendingGate` rows surviving +/// restart are cleaned up by the startup sweep in `router.rs`. +#[derive(Default)] +pub struct GateResolutions { + inner: Mutex<HashMap<Uuid, oneshot::Sender<GateResolution>>>, + /// Secondary index: `(user_id, credential_name)` → request_ids + /// parked on it. Used by [`Self::deliver_for_credential`] so the + /// OAuth-callback path can wake a Tier 0/Tier 1 inline-await for + /// the credential that was just written. The user_id component + /// keeps multi-tenant deployments from cross-waking — only the + /// owning user's parked gates fire. + by_credential: Mutex<HashMap<(String, String), HashSet<Uuid>>>, +} + +impl GateResolutions { + pub fn new() -> Self { + Self::default() + } + + /// Deliver a resolution to the suspended caller. Returns `true` if + /// a sender was registered for `request_id` (engine was waiting), + /// `false` if not (no live VM — fall through to legacy re-entry). + pub async fn try_deliver(&self, request_id: Uuid, resolution: GateResolution) -> bool { + let sender = self.inner.lock().await.remove(&request_id); + // Best-effort: drop any credential-index entries that point at + // this request_id (we don't know which credential without + // tracking the reverse mapping; keep the index lazy by sweeping + // sets that contain the id). + { + let mut idx = self.by_credential.lock().await; + idx.retain(|_, set| { + set.remove(&request_id); + !set.is_empty() + }); + } + match sender { + Some(tx) => tx.send(resolution).is_ok(), + None => false, + } + } + + /// Deliver `Approved` to every parked Authentication gate that was + /// waiting on `(user_id, credential_name)`. Returns the count of + /// waiters woken. Used by the OAuth-callback path: when a + /// credential is written, every paused tool call (Tier 0 or + /// Tier 1, foreground or mission child thread) belonging to + /// `user_id` that was blocked on that credential can resume + /// inline and retry the action against the now-present secret. + /// Other users' parked gates on the same credential name are + /// left untouched. + pub async fn deliver_for_credential(&self, user_id: &str, credential_name: &str) -> usize { + let request_ids: Vec<Uuid> = { + let mut idx = self.by_credential.lock().await; + idx.remove(&(user_id.to_string(), credential_name.to_string())) + .map(|set| set.into_iter().collect()) + .unwrap_or_default() + }; + let mut delivered = 0; + for request_id in request_ids { + if self + .try_deliver(request_id, GateResolution::Approved { always: false }) + .await + { + delivered += 1; + } + } + delivered + } + + async fn register(&self, request_id: Uuid, sender: oneshot::Sender<GateResolution>) { + self.inner.lock().await.insert(request_id, sender); + } + + /// Register `request_id` against `(user_id, credential_name)` so + /// an OAuth completion can wake it by credential name later, + /// scoped to the owning user. + async fn register_credential( + &self, + user_id: String, + credential_name: String, + request_id: Uuid, + ) { + self.by_credential + .lock() + .await + .entry((user_id, credential_name)) + .or_default() + .insert(request_id); + } + + async fn forget(&self, request_id: Uuid) { + self.inner.lock().await.remove(&request_id); + let mut idx = self.by_credential.lock().await; + idx.retain(|_, set| { + set.remove(&request_id); + !set.is_empty() + }); + } +} + +/// Single shared controller. Threaded through every +/// `ThreadExecutionContext` the engine builds for a live execution. +pub struct BridgeGateController { + pending_gates: Arc<PendingGateStore>, + sse: Option<Arc<SseManager>>, + tools: Arc<ToolRegistry>, + auth_manager: Option<Arc<AuthManager>>, + extension_manager: Option<Arc<ExtensionManager>>, + channels: Arc<ChannelManager>, + resolutions: Arc<GateResolutions>, + /// Per-(user, thread) registry. Populated once the bridge knows + /// which thread the engine spawned for a turn. The lookup here + /// wins when both this map and `pre_execution` carry an entry — + /// it's the more specific key. + per_execution: Mutex<HashMap<ExecutionKey, PerExecutionContext>>, + /// Pre-execution registry, populated *before* `handle_user_message` + /// returns the thread_id. Closes the race where a fast tool gate + /// reaches `pause()` before the bridge has had a chance to register + /// the (user, thread)-keyed entry. Keyed by `(user_id, + /// conversation_id)` so concurrent conversations for the same user + /// (e.g. two browser tabs) don't clobber each other — each turn's + /// `pause()` matches its own conversation's slot via + /// `GatePauseRequest::conversation_id`. + pre_execution: Mutex<HashMap<PreExecKey, PerExecutionContext>>, + /// Per-(user, thread) serialization lock for `pause()`. Holding + /// this across the `PendingGateStore::insert` + select-await window + /// guarantees only one inline gate per `(user, thread)` is in + /// flight at a time. Without it, a parallel batch where two tool + /// calls both gate concurrently would have the second insert hit + /// the (user, thread) uniqueness check and silently surface as + /// `GateResolution::Cancelled`. With it, the second `pause()` + /// queues until the first resolves. + gate_locks: Mutex<HashMap<ExecutionKey, Arc<Mutex<()>>>>, + /// Per-`ThreadId` registry of in-flight pause request_ids. + /// `pause()` adds its `request_id` here on entry and removes it on + /// exit. `cancel_thread()` walks this set and delivers + /// `GateResolution::Cancelled` to each — wiring `stop_thread()` + /// through to the parked future so a stop request promptly wakes + /// the engine task instead of waiting on the 30-minute gate + /// expiry. + /// + /// A `HashSet<Uuid>` (rather than `Option<Uuid>`) is correct even + /// though the current `gate_locks` serializes one pause per + /// `(user, thread)`: a future change that loosens that + /// serialization (e.g. per-action gates instead of per-thread) + /// would otherwise drop pending request_ids on the floor. + active_pauses: Mutex<HashMap<ThreadId, HashSet<Uuid>>>, +} + +impl BridgeGateController { + #[allow(clippy::too_many_arguments)] + pub fn new( + pending_gates: Arc<PendingGateStore>, + sse: Option<Arc<SseManager>>, + tools: Arc<ToolRegistry>, + auth_manager: Option<Arc<AuthManager>>, + extension_manager: Option<Arc<ExtensionManager>>, + channels: Arc<ChannelManager>, + resolutions: Arc<GateResolutions>, + ) -> Self { + Self { + pending_gates, + sse, + tools, + auth_manager, + extension_manager, + channels, + resolutions, + per_execution: Mutex::new(HashMap::new()), + pre_execution: Mutex::new(HashMap::new()), + gate_locks: Mutex::new(HashMap::new()), + active_pauses: Mutex::new(HashMap::new()), + } + } + + async fn track_active_pause(&self, thread_id: ThreadId, request_id: Uuid) { + self.active_pauses + .lock() + .await + .entry(thread_id) + .or_default() + .insert(request_id); + } + + async fn untrack_active_pause(&self, thread_id: ThreadId, request_id: Uuid) { + let mut map = self.active_pauses.lock().await; + if let Some(set) = map.get_mut(&thread_id) { + set.remove(&request_id); + if set.is_empty() { + map.remove(&thread_id); + } + } + } + + /// Look up (or create) the per-(user, thread) gate-serialization + /// lock. The returned Arc is cloned out so callers can drop the + /// outer registry lock before contending on the inner lock. + async fn gate_lock_for(&self, key: &ExecutionKey) -> Arc<Mutex<()>> { + let mut map = self.gate_locks.lock().await; + map.entry(key.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + /// Bind per-execution data for `(user_id, conversation_id)` BEFORE + /// the engine spawns the thread. Closes the race window where a + /// fast tool gate reaches `pause()` before the (user, thread)-keyed + /// entry has been written. + /// + /// Keying by `conversation_id` (rather than `user_id` alone) keeps + /// concurrent conversations for the same user — multiple browser + /// tabs, background missions firing alongside a foreground turn — + /// from clobbering each other's slot. Each turn's `pause()` matches + /// its own conversation via `GatePauseRequest::conversation_id`. + pub async fn set_pre_execution_context( + &self, + user_id: String, + conversation_id: ConversationId, + context: PerExecutionContext, + ) { + self.pre_execution.lock().await.insert( + PreExecKey { + user_id, + conversation_id, + }, + context, + ); + } + + /// Bind per-execution data for `(user_id, thread_id)` once the + /// engine has allocated a thread_id. Call after + /// [`Self::set_pre_execution_context`]; supersedes the + /// (user, conversation_id)-keyed entry for subsequent lookups. + pub async fn set_execution_context( + &self, + user_id: String, + thread_id: ThreadId, + context: PerExecutionContext, + ) { + let conv_id = context.conversation_id; + self.per_execution.lock().await.insert( + ExecutionKey { + user_id: user_id.clone(), + thread_id, + }, + context, + ); + // Remove the (user, conversation)-keyed pre-execution entry — + // the (user, thread)-keyed entry is now the source of truth. + self.pre_execution.lock().await.remove(&PreExecKey { + user_id, + conversation_id: conv_id, + }); + } + + /// Drop the pre-execution `(user, conversation)`-keyed entry + /// without touching any `(user, thread)`-keyed entry. Used on the + /// bridge error path when `handle_user_message` failed before + /// allocating a thread_id — without this the slot would leak and + /// could mis-route the next gate prompt for the same conversation. + pub async fn clear_pre_execution_context( + &self, + user_id: &str, + conversation_id: ConversationId, + ) { + self.pre_execution.lock().await.remove(&PreExecKey { + user_id: user_id.to_string(), + conversation_id, + }); + } + + /// Drop per-execution data. Idempotent. `conversation_id` is the + /// originating conversation for this turn so any leftover + /// pre-execution slot (e.g. when the bridge bailed before + /// promotion) gets cleared too. + pub async fn clear_execution_context( + &self, + user_id: &str, + thread_id: ThreadId, + conversation_id: ConversationId, + ) { + let key = ExecutionKey { + user_id: user_id.to_string(), + thread_id, + }; + self.per_execution.lock().await.remove(&key); + // Defensive: clear any leftover pre-execution entry too. In + // the happy path `set_execution_context` already removed it, + // but if the bridge bailed before that promotion (engine spawn + // failed) the entry would otherwise leak. + self.pre_execution.lock().await.remove(&PreExecKey { + user_id: user_id.to_string(), + conversation_id, + }); + // Drop the per-(user, thread) gate-serialization lock entry. + // By the time the bridge clears execution context, all `pause` + // futures for this thread have resolved, so the inner lock is + // idle and removing the registry entry simply bounds the map. + self.gate_locks.lock().await.remove(&key); + } + + /// Forward a resolution into the inline-await registry. Returns + /// `true` if the engine was actively awaiting it. + pub async fn try_deliver(&self, request_id: Uuid, resolution: GateResolution) -> bool { + self.resolutions.try_deliver(request_id, resolution).await + } + + async fn lookup_per_execution( + &self, + user_id: &str, + thread_id: ThreadId, + conversation_id: Option<ConversationId>, + ) -> Option<PerExecutionContext> { + // Most specific match first: (user, thread). Falls back to + // the (user, conversation)-keyed pre-execution entry so a + // gate firing before `set_execution_context` lands still + // finds its context. The fallback requires the request to + // carry `conversation_id`; gates from threads with no + // originating conversation (background missions) only match + // via the (user, thread) entry. + if let Some(ctx) = self.per_execution.lock().await.get(&ExecutionKey { + user_id: user_id.to_string(), + thread_id, + }) { + return Some(ctx.clone()); + } + if let Some(conv_id) = conversation_id { + return self + .pre_execution + .lock() + .await + .get(&PreExecKey { + user_id: user_id.to_string(), + conversation_id: conv_id, + }) + .cloned(); + } + None + } + + async fn build_pending_gate( + &self, + request_id: Uuid, + per_exec: &PerExecutionContext, + user_id: &str, + thread_id: ThreadId, + req: &GatePauseRequest, + ) -> PendingGate { + let display_parameters = match self.tools.get(&req.action_name).await { + Some(tool) => Some(crate::tools::redact_params( + &req.parameters, + tool.sensitive_params(), + )), + None => Some(req.parameters.clone()), + }; + + PendingGate { + request_id, + gate_name: req.gate_name.clone(), + user_id: user_id.to_string(), + thread_id, + scope_thread_id: per_exec.scope_thread_id.clone(), + conversation_id: per_exec.conversation_id, + source_channel: per_exec.source_channel.clone(), + action_name: req.action_name.clone(), + call_id: req.call_id.clone(), + parameters: req.parameters.clone(), + display_parameters, + description: format!( + "Tool '{}' requires {} (gate: {})", + req.action_name, + req.resume_kind.kind_name(), + req.gate_name + ), + resume_kind: req.resume_kind.clone(), + created_at: chrono::Utc::now(), + expires_at: chrono::Utc::now() + chrono::Duration::minutes(30), + original_message: per_exec.original_message.clone(), + resume_output: None, + paused_lease: None, + approval_already_granted: false, + } + } + + async fn emit_gate_prompt(&self, pending: &PendingGate, channel_metadata: &JsonValue) { + let extension_name = crate::bridge::router::resolve_auth_gate_extension_name( + self.auth_manager.as_deref(), + self.extension_manager.as_deref(), + self.tools.as_ref(), + pending, + ) + .await; + + let display_parameters = crate::bridge::router::gate_display_parameters(pending); + + if let Some(ref sse) = self.sse { + sse.broadcast_for_user( + &pending.user_id, + AppEvent::GateRequired { + request_id: pending.request_id.to_string(), + gate_name: pending.gate_name.clone(), + tool_name: pending.action_name.clone(), + description: pending.description.clone(), + parameters: serde_json::to_string_pretty(&display_parameters) + .unwrap_or_else(|_| display_parameters.to_string()), + extension_name: extension_name.clone(), + resume_kind: serde_json::to_value(&pending.resume_kind).unwrap_or_default(), + thread_id: Some(pending.effective_wire_thread_id()), + }, + ); // projection-exempt: bridge dispatcher, inline-await gate prompt for live VM waiting on user input + } + + match &pending.resume_kind { + ResumeKind::Approval { allow_always } => { + let _ = self + .channels + .send_status( + &pending.source_channel, + StatusUpdate::ApprovalNeeded { + request_id: pending.request_id.to_string(), + tool_name: pending.action_name.clone(), + description: pending.description.clone(), + parameters: display_parameters, + allow_always: *allow_always, + }, + channel_metadata, + ) + .await; + } + ResumeKind::Authentication { + instructions, + auth_url, + .. + } => { + let Some(extension_name) = extension_name else { + debug!( + gate = %pending.gate_name, + request_id = %pending.request_id, + "Authentication gate reached emit_gate_prompt without a resolved extension name" + ); + return; + }; + let _ = self + .channels + .send_status( + &pending.source_channel, + StatusUpdate::AuthRequired { + extension_name, + instructions: Some(instructions.clone()), + auth_url: auth_url.clone(), + setup_url: None, + request_id: Some(pending.request_id.to_string()), + }, + channel_metadata, + ) + .await; + } + ResumeKind::External { .. } => {} + } + } +} + +#[async_trait] +impl GateController for BridgeGateController { + async fn pause(&self, request: GatePauseRequest) -> GateResolution { + // Inline gate-await handles Approval and Authentication. + // External resume kinds keep the legacy + // `ThreadOutcome::GatePaused` re-entry path because their + // resolution installs callback-payload state that can't be + // handed back to the suspended call without unwinding. Surface + // External as Cancelled so the call returns a clean error + // instead of hanging. + if matches!(request.resume_kind, ResumeKind::External { .. }) { + debug!( + kind = %request.resume_kind.kind_name(), + "BridgeGateController: External resume kind reached inline await; cancelling", + ); + return GateResolution::Cancelled; + } + + let Some(per_exec) = self + .lookup_per_execution(&request.user_id, request.thread_id, request.conversation_id) + .await + else { + // No per-execution context registered. This shouldn't happen + // when invoked through `handle_with_engine`, which always + // populates it before invoking the engine. Mission / + // background threads also reach here today; they fall + // through to the legacy `ThreadOutcome::GatePaused` unwind + // path so `process_mission_outcome_and_notify` (#3133 + // half-1) transitions the mission to Paused. The half-2 + // mission auto-resume path (`resume_paused_for_credential`) + // resumes the mission after OAuth completes. Cancelling + // here is the right wire for that flow — it produces the + // legacy unwind that the mission state machine consumes. + debug!( + user = %request.user_id, + thread = %request.thread_id, + kind = %request.resume_kind.kind_name(), + "BridgeGateController: no per-execution context — cancelling (mission/background path)" + ); + return GateResolution::Cancelled; + }; + + // Serialize concurrent inline gates per (user, thread). A + // parallel batch where two tool calls both gate would otherwise + // race on `PendingGateStore::insert` — the first wins, the + // second hits the (user, thread) uniqueness check and silently + // becomes `Cancelled` without ever prompting the user. Holding + // this lock across insert + select-await queues subsequent + // gates behind the current one so each gets its own prompt. + // + // TODO(#3157 follow-up — design-doc item): bound live inline + // gate awaits per user / globally with a typed semaphore so an + // authenticated user opening many threads each with an + // unresolved approval gate cannot accumulate parked engine + // tasks/pending rows past a budget. The current implicit bound + // is one pending gate per (user, thread) × the existing + // thread-creation budget × the 30-min expiry; that is enough + // to ship the first inline-await slice but not enough as a + // long-term DoS guard. Track in a separate issue once the + // semaphore design (cap UX, fairness, rejection error shape) + // is settled rather than hand-rolling it inside this + // controller. + let exec_key = ExecutionKey { + user_id: request.user_id.clone(), + thread_id: request.thread_id, + }; + let gate_lock = self.gate_lock_for(&exec_key).await; + let _gate_guard = gate_lock.lock().await; + + let request_id = Uuid::new_v4(); + let pending = self + .build_pending_gate( + request_id, + &per_exec, + &request.user_id, + request.thread_id, + &request, + ) + .await; + + if let Err(e) = self.pending_gates.insert(pending.clone()).await { + // With the per-(user, thread) gate lock held above, a + // legitimate concurrent collision can't happen. An insert + // failure here means a stale row from a prior turn hadn't + // been cleaned up. Surface as cancel. + debug!( + user = %request.user_id, + thread = %request.thread_id, + error = %e, + "BridgeGateController: pending_gates.insert rejected; treating as cancelled", + ); + return GateResolution::Cancelled; + } + + let (tx, rx) = oneshot::channel(); + self.resolutions.register(request_id, tx).await; + // For Authentication gates, also index this request_id by + // the credential name we're waiting on so the OAuth callback + // path can wake us by credential without having to know the + // request_id. Forget on exit cleans the index either way. + if let ResumeKind::Authentication { + ref credential_name, + .. + } = request.resume_kind + { + self.resolutions + .register_credential( + request.user_id.clone(), + credential_name.as_str().to_string(), + request_id, + ) + .await; + } + + // Track this in-flight pause so `cancel_thread()` can wake it + // promptly on `ThreadManager::stop_thread()`. Without this, + // a stop request against a thread parked here would have to + // wait for the user (or the 30-min expiry) before the engine + // task observed the stop signal. + self.track_active_pause(request.thread_id, request_id).await; + + self.emit_gate_prompt(&pending, &per_exec.channel_metadata) + .await; + + // Bound the await on `pending.expires_at`. Without this, a user + // who ignores the prompt past expiry strands the engine: the + // pending DB row expires, but the oneshot stays open and the + // VM keeps running until something else (process restart, + // join_thread timeout) tears it down. Race the receiver against + // a sleep; whichever resolves first wins. + let expires_at = pending.expires_at; + let now = chrono::Utc::now(); + let timeout_dur = (expires_at - now) + .to_std() + .unwrap_or(std::time::Duration::ZERO); + let pending_key = pending.key(); + let resolution = tokio::select! { + biased; + received = rx => match received { + Ok(resolution) => resolution, + Err(_) => { + // Sender dropped — process shutting down or registry + // cleared. Discard the pending row so the UI doesn't + // keep showing a stranded prompt and a future + // (user, thread) gate isn't blocked by the + // duplicate-insert guard. Same cleanup as the + // expiry branch below. + self.resolutions.forget(request_id).await; + let _ = self.pending_gates.discard(&pending_key).await; + GateResolution::Cancelled + } + }, + _ = tokio::time::sleep(timeout_dur) => { + // Expiry hit before the user resolved. Drop the + // registry entry and the pending row so a late + // resolve_gate call can't double-deliver, and surface + // as Cancelled to wake the VM. + self.resolutions.forget(request_id).await; + let _ = self.pending_gates.discard(&pending_key).await; + debug!( + user = %request.user_id, + thread = %request.thread_id, + request_id = %request_id, + "BridgeGateController: pause expired before resolution; cancelling", + ); + GateResolution::Cancelled + } + }; + // Always untrack on exit. Idempotent — `cancel_thread` may + // have already removed our entry while delivering the + // cancellation that woke us; re-removing is a no-op. + self.untrack_active_pause(request.thread_id, request_id) + .await; + resolution + } + + async fn cancel_thread(&self, thread_id: ThreadId) { + // Snapshot the in-flight request_ids and pending keys, then + // release the lock before delivering. Holding `active_pauses` + // while calling `try_deliver` (which takes its own lock) and + // `pending_gates.discard` (DB I/O) would gratuitously serialize + // unrelated stops. + let request_ids: Vec<Uuid> = { + let map = self.active_pauses.lock().await; + map.get(&thread_id) + .map(|set| set.iter().copied().collect()) + .unwrap_or_default() + }; + if request_ids.is_empty() { + return; + } + debug!( + thread = %thread_id, + count = request_ids.len(), + "BridgeGateController::cancel_thread: waking parked gates", + ); + for request_id in request_ids { + // Deliver Cancelled to the parked future. Returns false if + // the future has already woken (resolution arrived between + // our snapshot and try_deliver) — that's fine, we just + // skip the discard for the same reason. + let _ = self + .resolutions + .try_deliver(request_id, GateResolution::Cancelled) + .await; + } + // Discard any pending DB rows for this thread so the UI doesn't + // keep showing stranded prompts. We don't have the + // `pending_key` here (only request_id), so use the thread-level + // discard helper if one exists — otherwise this is the cost of + // not threading the key through. The pause() future will run + // its own cleanup when it wakes; this branch is a defence in + // depth for the case where a row was committed but the + // resolution channel was closed. + let _removed = self.pending_gates.discard_for_thread(thread_id).await; + // Clear the active set for this thread now that all parked + // pauses have been notified. + self.active_pauses.lock().await.remove(&thread_id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `try_deliver` returns `false` for an unknown request_id (the + /// engine isn't waiting). The resolve endpoint uses this to decide + /// whether to fall through to the legacy re-entry path. + #[tokio::test] + async fn try_deliver_unknown_request_returns_false() { + let resolutions = GateResolutions::new(); + let delivered = resolutions + .try_deliver(Uuid::new_v4(), GateResolution::Approved { always: false }) + .await; + assert!(!delivered, "unknown request_id must report false"); + } + + /// Round-trip: register a sender, hand it to a spawned task that + /// awaits the receiver, then deliver. The task must observe the + /// resolution and `try_deliver` must report `true`. + #[tokio::test] + async fn try_deliver_routes_to_registered_receiver() { + let resolutions = Arc::new(GateResolutions::new()); + let request_id = Uuid::new_v4(); + let (tx, rx) = oneshot::channel(); + resolutions.register(request_id, tx).await; + + let receiver_task = tokio::spawn(async move { rx.await.ok() }); + + let delivered = resolutions + .try_deliver(request_id, GateResolution::Denied { reason: None }) + .await; + assert!(delivered, "registered request_id must report true"); + + let received = receiver_task.await.expect("task panicked"); + assert!(matches!(received, Some(GateResolution::Denied { .. }))); + } + + /// `cancel_thread()` wakes a `pause()` future parked on the given + /// thread with `GateResolution::Cancelled`. Without this hook, + /// `ThreadManager::stop_thread()` would have to wait up to the + /// 30-minute gate expiry before the engine task observed the stop. + #[tokio::test] + async fn cancel_thread_wakes_parked_pause_with_cancelled_resolution() { + use ironclaw_engine::GateController; + use ironclaw_engine::ResumeKind; + use ironclaw_engine::ThreadId; + + let controller = Arc::new(BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::new(GateResolutions::new()), + )); + let thread_id = ThreadId::new(); + let user_id = "stop-during-wait-user".to_string(); + let conversation_id = ironclaw_engine::ConversationId::new(); + + // Bridge populates the per-execution context before invoking + // the engine. Without it `pause()` cancels immediately + // (no per-execution lookup hit), which would also pass the + // test for the wrong reason. + controller + .set_execution_context( + user_id.clone(), + thread_id, + PerExecutionContext { + conversation_id, + source_channel: "test".into(), + scope_thread_id: None, + channel_metadata: serde_json::json!({}), + original_message: None, + }, + ) + .await; + + // Park a pause() future in a spawned task. It will block on the + // approval-resolution oneshot (and the 30-min sleep) until + // cancel_thread wakes it. + let controller_clone = controller.clone(); + let user_clone = user_id.clone(); + let pause_task = tokio::spawn(async move { + controller_clone + .pause(GatePauseRequest { + thread_id, + user_id: user_clone, + gate_name: "approval".into(), + action_name: "test_tool".into(), + call_id: "call_stop_test".into(), + parameters: serde_json::json!({}), + resume_kind: ResumeKind::Approval { allow_always: true }, + conversation_id: Some(conversation_id), + }) + .await + }); + + // Let pause() reach the select await before we cancel. + // The track_active_pause + register happen synchronously after + // the pending_gates.insert; one tokio yield is sufficient on + // current_thread runtime because pause() yields at .await + // points before reaching the select. + for _ in 0..10 { + tokio::task::yield_now().await; + if !controller + .active_pauses + .lock() + .await + .get(&thread_id) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + break; + } + } + assert!( + !controller + .active_pauses + .lock() + .await + .get(&thread_id) + .map(|s| s.is_empty()) + .unwrap_or(true), + "pause must register an active entry before we cancel" + ); + + // Now stop the thread. The pause future should resolve + // promptly (well under the 30-minute expiry). + controller.cancel_thread(thread_id).await; + + let resolution = tokio::time::timeout(std::time::Duration::from_secs(2), pause_task) + .await + .expect("cancel_thread must wake parked pause within 2s") + .expect("pause task did not panic"); + assert!( + matches!(resolution, GateResolution::Cancelled), + "stop must surface as Cancelled; got {resolution:?}" + ); + + // Active set is cleared. + assert!( + !controller + .active_pauses + .lock() + .await + .contains_key(&thread_id), + "active_pauses must be cleared after cancel_thread" + ); + } + + /// `cancel_thread()` is a no-op when no pause is parked on the + /// thread. `ThreadManager::stop_thread()` always calls it; the + /// happy path (no inline-await waiter) must not panic or block. + #[tokio::test] + async fn cancel_thread_with_no_active_pause_is_a_no_op() { + use ironclaw_engine::GateController; + use ironclaw_engine::ThreadId; + + let controller = BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::new(GateResolutions::new()), + ); + // Should return immediately. + controller.cancel_thread(ThreadId::new()).await; + } + + /// `try_deliver` returns `false` when the receiver was dropped + /// before delivery. The resolve endpoint then falls through, and + /// the corresponding `PendingGate` is treated as stale. + #[tokio::test] + async fn try_deliver_returns_false_when_receiver_dropped() { + let resolutions = GateResolutions::new(); + let request_id = Uuid::new_v4(); + let (tx, rx) = oneshot::channel(); + resolutions.register(request_id, tx).await; + drop(rx); + + let delivered = resolutions + .try_deliver(request_id, GateResolution::Approved { always: false }) + .await; + assert!(!delivered, "dropped receiver must report false"); + } + + /// A second `try_deliver` for the same request_id returns `false` + /// — the entry was consumed by the first delivery. + #[tokio::test] + async fn try_deliver_is_one_shot() { + let resolutions = Arc::new(GateResolutions::new()); + let request_id = Uuid::new_v4(); + let (tx, _rx) = oneshot::channel(); + resolutions.register(request_id, tx).await; + + let first = resolutions + .try_deliver(request_id, GateResolution::Approved { always: false }) + .await; + let second = resolutions + .try_deliver(request_id, GateResolution::Approved { always: false }) + .await; + // Note: `first` may be `false` because we dropped rx — but the + // entry is still consumed, so `second` must always be `false`. + assert!(!second, "second delivery must report false"); + let _ = first; + } +} diff --git a/src/bridge/llm_adapter.rs b/src/bridge/llm_adapter.rs index eebcf8f18a2..a318ac5bfe5 100644 --- a/src/bridge/llm_adapter.rs +++ b/src/bridge/llm_adapter.rs @@ -11,7 +11,7 @@ use ironclaw_engine::{ use rust_decimal::Decimal; use rust_decimal::prelude::ToPrimitive; -use crate::llm::{ +use ironclaw_llm::{ ChatMessage, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolDefinition, clean_response, recover_tool_calls_from_content, sanitize_tool_messages, }; @@ -110,8 +110,20 @@ impl LlmBackend for LlmBridgeAdapter { sanitize_tool_messages(&mut chat_messages); // Convert actions to tool definitions + // + // In disabled-CodeAct mode the model has no Python escape hatch, so + // every callable action MUST be reachable via the provider's + // structured `tool_calls` interface. Filtering down to + // `emits_full_schema_tool()` in that mode would leave compact-info + // actions (e.g. `mission_create`, `gmail_send`, `notion_search`) + // visible in the prompt as "available" but absent from the provider + // tool list — i.e. unreachable. The prompt builder mirrors this by + // omitting the "Enabled Tools" section when CodeAct is disabled + // (see `prompt::build_codeact_system_prompt_inner`). PR #3665 review. let tools: Vec<ToolDefinition> = if config.force_text { vec![] // No tools when forcing text + } else if ironclaw_engine::executor::prompt::codeact_disabled() { + actions.iter().map(action_def_to_tool_def).collect() } else { actions .iter() @@ -126,7 +138,7 @@ impl LlmBackend for LlmBridgeAdapter { if tools.is_empty() { // No tools: use plain completion (matches existing no-tools path) - let mut request = crate::llm::CompletionRequest::new(chat_messages) + let mut request = ironclaw_llm::CompletionRequest::new(chat_messages) .with_max_tokens(max_tokens) .with_temperature(temperature); request.metadata = config.metadata.clone(); @@ -403,6 +415,7 @@ fn thread_msg_to_chat(msg: &ThreadMessage) -> ChatMessage { tool_call_id: msg.action_call_id.clone(), name: msg.action_name.clone(), tool_calls: None, + reasoning: None, }; // Convert action calls if present (assistant message with tool calls) @@ -415,6 +428,7 @@ fn thread_msg_to_chat(msg: &ThreadMessage) -> ChatMessage { name: c.action_name.clone(), arguments: c.parameters.clone(), reasoning: None, + signature: None, }) .collect(), ); @@ -513,6 +527,12 @@ fn extract_code_block(text: &str) -> Option<String> { } fn text_response_from_cleaned_text(cleaned_text: String) -> LlmResponse { + if ironclaw_engine::executor::prompt::codeact_disabled() { + if cleaned_text.trim().is_empty() { + return LlmResponse::Text(EMPTY_CLEANED_RESPONSE_FALLBACK.to_string()); + } + return LlmResponse::Text(cleaned_text); + } match extract_code_block(&cleaned_text) { Some(code) => LlmResponse::Code { code, @@ -620,7 +640,7 @@ mod tests { use ironclaw_engine::{ActionCall, ActionDef, EffectType, LlmResponse, ThreadMessage}; use crate::error::LlmError; - use crate::llm::ToolCompletionResponse; + use ironclaw_llm::ToolCompletionResponse; #[derive(Default)] struct CapturingProviderState { @@ -646,8 +666,8 @@ mod tests { async fn complete( &self, - req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { + req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { self.state.models.lock().await.push(req.model.clone()); self.state .completion_requests @@ -655,11 +675,11 @@ mod tests { .await .push(req.messages); - Ok(crate::llm::CompletionResponse { + Ok(ironclaw_llm::CompletionResponse { content: "ok".to_string(), input_tokens: 1, output_tokens: 1, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -682,9 +702,10 @@ mod tests { tool_calls: Vec::new(), input_tokens: 1, output_tokens: 1, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -834,8 +855,8 @@ mod tests { async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { unreachable!("test only uses complete_with_tools") } @@ -848,9 +869,10 @@ mod tests { tool_calls: Vec::new(), input_tokens: 1, output_tokens: 1, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -966,13 +988,13 @@ mod tests { async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { - Ok(crate::llm::CompletionResponse { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: self.content.clone(), input_tokens: 1, output_tokens: 1, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -1123,15 +1145,27 @@ mod tests { assert_eq!(models[0], None); } + #[allow(clippy::await_holding_lock)] #[tokio::test] async fn complete_with_tools_only_emits_full_schema_provider_tools() { + // Both this test and `complete_emits_compact_actions_when_codeact_disabled` + // read the process-global `IRONCLAW_DISABLE_CODEACT` env var. Serialize + // via lock_env() and pin the value here so the other test setting + // `=true` can't leak across when `cargo test` runs them in parallel. + let _guard = crate::config::helpers::lock_env(); + let original = std::env::var_os("IRONCLAW_DISABLE_CODEACT"); + // SAFETY: serialized via lock_env(). + unsafe { + std::env::remove_var("IRONCLAW_DISABLE_CODEACT"); + } + let state = Arc::new(CapturingProviderState::default()); let provider: Arc<dyn LlmProvider> = Arc::new(CapturingProvider { state: state.clone(), }); let adapter = LlmBridgeAdapter::new(provider, None); - adapter + let result = adapter .complete( &[ThreadMessage::user("hi")], &[ @@ -1156,8 +1190,18 @@ mod tests { ], &LlmCallConfig::default(), ) - .await - .unwrap(); + .await; + + // SAFETY: serialized via lock_env(). + unsafe { + if let Some(value) = original { + std::env::set_var("IRONCLAW_DISABLE_CODEACT", value); + } else { + std::env::remove_var("IRONCLAW_DISABLE_CODEACT"); + } + } + + result.unwrap(); let tool_definitions = state.tool_definitions.lock().await; let emitted = tool_definitions.last().expect("tool completion request"); @@ -1169,6 +1213,78 @@ mod tests { assert_eq!(names, vec!["http"]); } + /// PR #3665 review (serrrfirat). Disabled-CodeAct mode strips the Python + /// escape hatch, so any callable action MUST be reachable via the + /// provider's structured `tool_calls`. Filtering down to FullSchema in + /// that mode left compact actions (`mission_create`, `gmail_send`, ...) + /// visible in the prompt but absent from the provider tool list — i.e. + /// unreachable. This test pins the relaxed filter. + #[allow(clippy::await_holding_lock)] + #[tokio::test] + async fn complete_emits_compact_actions_when_codeact_disabled() { + let _guard = crate::config::helpers::lock_env(); + let original = std::env::var_os("IRONCLAW_DISABLE_CODEACT"); + // SAFETY: serialized via lock_env(). + unsafe { + std::env::set_var("IRONCLAW_DISABLE_CODEACT", "true"); + } + + let state = Arc::new(CapturingProviderState::default()); + let provider: Arc<dyn LlmProvider> = Arc::new(CapturingProvider { + state: state.clone(), + }); + let adapter = LlmBridgeAdapter::new(provider, None); + + let result = adapter + .complete( + &[ThreadMessage::user("hi")], + &[ + ActionDef { + name: "http".into(), + description: "fetch".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::ReadExternal], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }, + ActionDef { + name: "mission_create".into(), + description: "create mission".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::WriteLocal], + requires_approval: false, + model_tool_surface: ModelToolSurface::CompactToolInfo, + discovery: None, + }, + ], + &LlmCallConfig::default(), + ) + .await; + + // SAFETY: serialized via lock_env(). + unsafe { + if let Some(value) = original { + std::env::set_var("IRONCLAW_DISABLE_CODEACT", value); + } else { + std::env::remove_var("IRONCLAW_DISABLE_CODEACT"); + } + } + + result.expect("adapter.complete should succeed"); + + let tool_definitions = state.tool_definitions.lock().await; + let emitted = tool_definitions.last().expect("tool completion request"); + let mut names: Vec<&str> = emitted.iter().map(|t| t.name.as_str()).collect(); + names.sort(); + assert_eq!( + names, + vec!["http", "mission_create"], + "disabled-CodeAct must emit BOTH FullSchema and CompactToolInfo actions \ + — otherwise compact actions are unreachable" + ); + } + // ── extract_code_block tests ──────────────────────────── #[test] @@ -1555,8 +1671,8 @@ And also check the token price:\n\ } async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { unreachable!("should use complete_with_tools") } async fn complete_with_tools( @@ -1567,7 +1683,7 @@ And also check the token price:\n\ // a prior tool result's project_id via template ref. Ok(ToolCompletionResponse { content: Some("Creating mission in the new project".to_string()), - tool_calls: vec![crate::llm::ToolCall { + tool_calls: vec![ironclaw_llm::ToolCall { id: "call-2".to_string(), name: "mission_create".to_string(), arguments: serde_json::json!({ @@ -1576,12 +1692,14 @@ And also check the token price:\n\ "project_id": "{{call-1.project_id}}" }), reasoning: None, + signature: None, }], input_tokens: 10, output_tokens: 10, - finish_reason: crate::llm::FinishReason::ToolUse, + finish_reason: ironclaw_llm::FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -1663,13 +1781,13 @@ And also check the token price:\n\ } async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { - Ok(crate::llm::CompletionResponse { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: "hello".to_string(), input_tokens: 1000, output_tokens: 500, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -1683,9 +1801,10 @@ And also check the token price:\n\ tool_calls: Vec::new(), input_tokens: 1000, output_tokens: 500, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -1752,13 +1871,13 @@ And also check the token price:\n\ } async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { - Ok(crate::llm::CompletionResponse { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: "ok".into(), input_tokens: 1000, output_tokens: 500, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -1811,13 +1930,13 @@ And also check the token price:\n\ } async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { - Ok(crate::llm::CompletionResponse { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: "ok".into(), input_tokens: 10_000, output_tokens: 5_000, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -1879,15 +1998,15 @@ And also check the token price:\n\ } async fn complete( &self, - _req: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, LlmError> { + _req: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, LlmError> { // Total input = 10_000; 2_000 cache-read, 1_000 cache-write, // 7_000 uncached. Output = 500. - Ok(crate::llm::CompletionResponse { + Ok(ironclaw_llm::CompletionResponse { content: "ok".into(), input_tokens: 10_000, output_tokens: 500, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 2_000, cache_creation_input_tokens: 1_000, }) diff --git a/src/bridge/mod.rs b/src/bridge/mod.rs index 25fe3b1f0c4..8dc80adedd3 100644 --- a/src/bridge/mod.rs +++ b/src/bridge/mod.rs @@ -10,6 +10,8 @@ mod capability_projector; mod cost_guard_gate; mod effect_adapter; mod engine_actions; +mod external_tools; +mod gate_controller; mod llm_adapter; mod router; pub mod sandbox; @@ -21,9 +23,14 @@ mod user_facing_errors; mod workspace_reader; pub use cost_guard_gate::CostGuardBudgetGate; +pub use external_tools::{ + EXTERNAL_TOOL_CALLBACK_PREFIX, ExternalToolCatalog, ExternalToolEntry, + call_id_from_external_callback, external_tool_callback_id, is_external_tool_callback_id, +}; pub use workspace_reader::WorkspaceReaderAdapter; pub use effect_adapter::EffectBridgeAdapter; +pub use gate_controller::{BridgeGateController, GateResolutions, PerExecutionContext}; pub use router::{ // DTO types AttentionItem, @@ -36,10 +43,19 @@ pub use router::{ EngineStepInfo, EngineThreadDetail, EngineThreadInfo, + InlineGateError, + InlineGateOutcome, ProjectOverviewEntry, ProjectsOverviewResponse, clear_engine_pending_auth, + clear_engine_pending_auth_for_credential, discard_engine_pending_auth_request, + // Engine internal action names — used by request validators to + // reject caller-supplied tool names that would shadow internal + // capability actions (mission_*, skill_*, memory_*, etc.). + engine_capability_action_names, + // External tool catalog accessor (Responses API) + engine_external_tool_catalog, // Query functions fire_engine_mission, get_engine_mission, @@ -47,6 +63,7 @@ pub use router::{ get_engine_project, get_engine_projects_overview, get_engine_thread, + get_pending_gate_by_request_id, // Action handlers handle_approval, handle_auth_gate_resolution, @@ -56,6 +73,7 @@ pub use router::{ handle_external_callback, handle_interrupt, handle_new_thread, + handle_pairing_claim, handle_with_engine, has_any_pending_gate, has_pending_auth, @@ -70,8 +88,12 @@ pub use router::{ pause_engine_mission, resolve_engine_auth_callback, resolve_gate, + resolve_inline_gates_for_credential, resume_engine_mission, + resume_paused_missions_for_credential, + resume_paused_missions_for_gate_request, transition_engine_pending_auth_request_to_pairing, + try_resolve_inline_approval_gate, }; #[cfg(feature = "libsql")] diff --git a/src/bridge/router.rs b/src/bridge/router.rs index 8b7cf345c61..78d47128862 100644 --- a/src/bridge/router.rs +++ b/src/bridge/router.rs @@ -48,6 +48,18 @@ pub enum BridgeOutcome { use std::collections::HashSet; +/// Cadence of the external-tool catalog sweep. Backstop only — the +/// per-thread terminal-state cleanup in `await_thread_outcome` is +/// the primary cleanup path. Five minutes is short enough to keep +/// memory bounded without producing visible churn for normal usage. +const EXTERNAL_TOOL_CATALOG_SWEEP_INTERVAL: std::time::Duration = + std::time::Duration::from_secs(5 * 60); + +/// Maximum age for a catalog entry before the periodic sweep evicts +/// it. One hour matches typical pending-gate TTLs and gives callers +/// plenty of headroom to resume a paused tool call. +const EXTERNAL_TOOL_CATALOG_TTL: chrono::Duration = chrono::Duration::hours(1); + /// Check if the engine v2 is enabled via `ENGINE_V2=true` environment variable. pub fn is_engine_v2_enabled() -> bool { std::env::var("ENGINE_V2") @@ -354,7 +366,7 @@ async fn save_attachment_index_notes( } } -fn gate_display_parameters(pending: &PendingGate) -> serde_json::Value { +pub(super) fn gate_display_parameters(pending: &PendingGate) -> serde_json::Value { pending .display_parameters .clone() @@ -417,7 +429,7 @@ async fn resolve_extension_for_action( /// resolver delegates to [`resolve_extension_for_action`]. Non-auth /// gate variants (`Approval`, `External`) don't have an extension /// identity and return `None`. -async fn resolve_auth_gate_extension_name( +pub(super) async fn resolve_auth_gate_extension_name( auth_manager: Option<&AuthManager>, extension_manager: Option<&crate::extensions::ExtensionManager>, tools: &crate::tools::ToolRegistry, @@ -512,6 +524,38 @@ fn resumed_action_result_message( ironclaw_engine::ThreadMessage::action_result(call_id, action_name, rendered) } +/// Extract the tool output for `call_id` from a Responses API +/// `function_call_output` resolution payload. The handler builds the +/// payload as `{"outputs": [{"call_id": ..., "output": <string|json>}]}`. +/// Falls back to: +/// - the raw payload when no `outputs` array is present (defensive +/// path for callers that pass a plain JSON value), and +/// - `Value::Null` when the payload doesn't contain a matching call_id +/// at all (lets the LLM see "the caller returned nothing for this +/// call" rather than re-running the tool). +fn extract_external_tool_output(payload: &serde_json::Value, call_id: &str) -> serde_json::Value { + let outputs = payload.get("outputs").and_then(|v| v.as_array()); + if let Some(arr) = outputs { + for entry in arr { + let entry_call_id = entry.get("call_id").and_then(|v| v.as_str()); + if entry_call_id == Some(call_id) + && let Some(out) = entry.get("output") + { + return out.clone(); + } + } + // No matching call_id: surface a typed null so the LLM sees + // an explicit empty result rather than the (possibly stale) + // raw payload. + return serde_json::Value::Null; + } + + // No `outputs` array at all — treat the whole payload as the + // result (matches OAuth callbacks that historically passed a + // raw value as the resolution). + payload.clone() +} + /// Resolve the assistant action `call_id` that a pending gate corresponds to. /// /// Returns `None` when neither the persisted `call_id` nor a history scan can @@ -683,12 +727,51 @@ async fn notify_pending_gate( let extension_name = resolve_auth_gate_extension_name(auth_manager, extension_manager, tools, pending).await; + // External-tool gates (Responses API caller-executed tools) project + // to a dedicated `AppEvent::ExternalToolCall` so the Responses API + // accumulator can surface them as `function_call` items without + // re-rendering them as approval cards. OAuth/pairing callbacks + // (which also use `ResumeKind::External` but with a different + // callback_id prefix) keep flowing through the standard + // `AppEvent::GateRequired` path. if let ironclaw_engine::ResumeKind::External { callback_id } = &pending.resume_kind { tracing::debug!( gate = %pending.gate_name, callback = %callback_id, "GatePaused(External)" ); + if crate::bridge::is_external_tool_callback_id(callback_id) { + if let Some(ref sse) = sse { + let arguments = serde_json::to_string(&pending.parameters) + .unwrap_or_else(|_| pending.parameters.to_string()); + let event = AppEvent::ExternalToolCall { + request_id: pending.request_id.to_string(), + call_id: pending.call_id.clone(), + name: pending.action_name.clone(), + arguments, + thread_id: Some(pending.effective_wire_thread_id()), + }; + sse.broadcast_for_user(&message.user_id, event); // projection-exempt: bridge dispatcher, ResumeKind::External(ext_tool) → Responses API function_call surface + } else { + // Today every external-tool flow runs through the + // gateway, which always wires SSE — so this branch + // means a future channel grew an external-tool surface + // without an SSE-equivalent fan-out, and the caller + // would never learn that the thread paused. Log so + // we can diagnose instead of silently hanging. + tracing::debug!( + user_id = %message.user_id, + callback = %callback_id, + request_id = %pending.request_id, + "external tool gate paused but no broadcaster is wired; \ + caller will not be notified" + ); + } + // Don't run `send_pending_gate_status` — that path is for + // approval-card UX which doesn't apply to caller-executed + // tool calls. + return Ok(BridgeOutcome::Pending); + } } // Send the approval/auth card via the source channel. Each channel @@ -847,6 +930,19 @@ async fn persist_always_allow( agent: &Agent, state: &EngineState, pending: &PendingGate, +) -> Option<serde_json::Value> { + persist_always_allow_with_store(agent.deps.settings_store.as_deref(), state, pending).await +} + +/// Same as [`persist_always_allow`] but takes the settings store directly +/// rather than reaching through `&Agent`. Lets the gateway HTTP fast-path +/// (`try_resolve_inline_approval_gate`) install the AlwaysAllow preference +/// without an `Agent` reference, since the agent-loop mpsc is the very +/// thing that path is bypassing. +async fn persist_always_allow_with_store( + settings_store: Option<&(dyn crate::db::SettingsStore + Send + Sync)>, + state: &EngineState, + pending: &PendingGate, ) -> Option<serde_json::Value> { // Validate tool name before using it as a settings key. Reject names // that contain dots or other characters that could collide with the @@ -888,8 +984,8 @@ async fn persist_always_allow( // stale data until the 5-minute TTL expires. In production the settings // store is always available when the DB is; the fallback was dead code // that actively broke cache coherence in tests and edge deployments. - let store: &(dyn crate::db::SettingsStore + Send + Sync) = match &agent.deps.settings_store { - Some(ss) => ss.as_ref(), + let store: &(dyn crate::db::SettingsStore + Send + Sync) = match settings_store { + Some(ss) => ss, None => return None, }; @@ -939,8 +1035,19 @@ async fn revert_always_allow( pending: &PendingGate, prior: Option<serde_json::Value>, ) { - let store: &(dyn crate::db::SettingsStore + Send + Sync) = match &agent.deps.settings_store { - Some(ss) => ss.as_ref(), + revert_always_allow_with_store(agent.deps.settings_store.as_deref(), pending, prior).await +} + +/// Same as [`revert_always_allow`] but takes the settings store directly. +/// Pairs with [`persist_always_allow_with_store`] for the gateway HTTP +/// fast-path that bypasses the agent-loop mpsc. +async fn revert_always_allow_with_store( + settings_store: Option<&(dyn crate::db::SettingsStore + Send + Sync)>, + pending: &PendingGate, + prior: Option<serde_json::Value>, +) { + let store: &(dyn crate::db::SettingsStore + Send + Sync) = match settings_store { + Some(ss) => ss, None => return, }; @@ -1087,6 +1194,20 @@ async fn execute_pending_gate_action( thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + // Post-resolution replay: the gate has already been resolved + // upstream, so a real controller is unnecessary. The inert + // controller surfaces any unexpected re-gate as a typed denial + // rather than reproducing the pre-fix unwind bug. + gate_controller: ironclaw_engine::CancellingGateController::arc(), + // The legacy resolved-pending path passes its own + // `approval_already_granted` to `execute_resolved_pending_action` + // directly, so this field is irrelevant for that path. Reset + // here to keep the default obvious. + call_approval_granted: false, + // Post-resolution replay never triggers a fresh inline gate; + // the conversation routing is moot here. + conversation_id: None, }; let active_leases = state .thread_manager @@ -1247,6 +1368,40 @@ async fn resolve_user_project( Ok(pid) } +/// Returns a clone of the live `ExternalToolCatalog` if the engine +/// state has been initialized, or `None` if the engine has not started +/// yet (engine_v2 disabled, or first message hasn't arrived). The +/// Responses API handler uses this to register caller-supplied tools +/// before sending the user message into the agent loop. +pub async fn engine_external_tool_catalog() -> Option<Arc<crate::bridge::ExternalToolCatalog>> { + let lock = ENGINE_STATE.get()?; + let guard = lock.read().await; + guard.as_ref().map(|s| Arc::clone(&s.external_tool_catalog)) +} + +/// Action names that are dispatchable via the engine v2 capability +/// registry (`mission_*`, `skill_*`, `memory_*`, etc.). Used by the +/// Responses API handler to reject caller-supplied tools whose names +/// would shadow internal engine actions — the `tool_registry` check +/// alone catches built-in and extension tools but misses capability +/// actions, which can land in the catalog short-circuit even though +/// the LLM-visible inventory dedup hides them. +/// +/// Returns `None` if engine v2 is not initialised; callers treat that +/// the same as "no engine v2 actions to collide with". +pub async fn engine_capability_action_names() -> Option<Vec<String>> { + let lock = ENGINE_STATE.get()?; + let guard = lock.read().await; + let state = guard.as_ref()?; + let names: Vec<String> = state + .capability_registry + .list() + .into_iter() + .flat_map(|cap| cap.actions.iter().map(|a| a.name.clone())) + .collect(); + Some(names) +} + /// Persistent engine state that lives across messages. struct EngineState { thread_manager: Arc<ThreadManager>, @@ -1268,6 +1423,28 @@ struct EngineState { extension_manager: Option<Arc<crate::extensions::ExtensionManager>>, /// Filesystem root for project-local attachment persistence. project_root: PathBuf, + /// Per-thread catalog of caller-provided external tools (Responses + /// API). Shared by `Arc` clone with the effect adapter (which + /// reads it during action listing and dispatch) and the Responses + /// API handler (which writes to it before sending the request to + /// the agent loop). + external_tool_catalog: Arc<crate::bridge::ExternalToolCatalog>, + /// Engine v2 capability registry. Held here (in addition to the + /// `effect_adapter`'s internal handle) so the Responses API + /// handler can enumerate internal action names and reject + /// caller-supplied tools that would shadow them. + capability_registry: Arc<ironclaw_engine::CapabilityRegistry>, + /// Inline gate-await controller. Lets the engine pause Tier 0 and + /// Tier 1 executions in place on `Approval` and `Authentication` + /// gates, rather than unwinding back to the orchestrator and + /// re-entering on resume (which would re-execute earlier + /// non-idempotent tool calls). + gate_controller: Arc<crate::bridge::gate_controller::BridgeGateController>, + /// Process-wide registry of in-flight gate resolution channels. + /// Held alongside `gate_controller` so the OAuth-callback path can + /// wake parked Authentication waiters by credential name without + /// going through the controller's internals. + gate_resolutions: Arc<crate::bridge::gate_controller::GateResolutions>, } /// Global engine state, initialized on first use. @@ -1652,7 +1829,7 @@ pub async fn init_engine(agent: &Agent) -> Result<(), Error> { llm_adapter, effect_adapter.clone(), store_dyn.clone(), - capabilities, + Arc::clone(&capabilities), leases, policy, )); @@ -1904,10 +2081,67 @@ pub async fn init_engine(agent: &Agent) -> Result<(), Error> { if let Err(e) = pending_gates.restore_from_persistence().await { debug!("engine v2: failed to restore pending gates: {e}"); } + // Restart sweep: any in-flight Approval gate from a prior boot has + // lost its in-memory await receiver. Falling through to legacy + // re-entry would re-run the LLM step and double-execute non-idempotent + // earlier tool calls in the same script (the very bug the inline-await + // path exists to prevent). Drop them at startup so the user gets a + // clean retry path instead. + invalidate_stranded_approval_gates(&pending_gates, agent.deps.sse_tx.as_ref()).await; if let Err(e) = reconcile_pending_gate_state(&store_dyn, &pending_gates).await { debug!("engine v2: pending gate reconciliation failed: {e}"); } + // Build the per-thread external tool catalog. Shared by Arc clone + // with the effect adapter (consults it on every action call) and + // exposed on the engine state so the Responses API handler can + // register/clear caller-supplied tools. + let external_tool_catalog = Arc::new(crate::bridge::ExternalToolCatalog::new()); + effect_adapter + .set_external_tool_catalog(Arc::clone(&external_tool_catalog)) + .await; + + // Backstop sweep: in addition to the per-thread terminal-state + // cleanup in `await_thread_outcome`, evict catalog entries that + // are older than `EXTERNAL_TOOL_CATALOG_TTL` to bound memory + // when a caller registers tools and then abandons the + // conversation (e.g. drops the connection without resuming a + // pending gate). Runs on a fixed cadence so a long-lived + // gateway doesn't accumulate stale entries. + { + let catalog = Arc::clone(&external_tool_catalog); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(EXTERNAL_TOOL_CATALOG_SWEEP_INTERVAL); + // Skip the immediate first tick so we don't sweep + // freshly-registered entries on engine boot. + ticker.tick().await; + loop { + ticker.tick().await; + let evicted = catalog.sweep_older_than(EXTERNAL_TOOL_CATALOG_TTL).await; + if !evicted.is_empty() { + debug!( + evicted = evicted.len(), + "engine v2: external tool catalog sweep evicted stale entries" + ); + } + } + }); + } + + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); + let gate_controller = Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::clone(&pending_gates), + agent.deps.sse_tx.clone(), + Arc::clone(effect_adapter.tools()), + auth_manager.clone(), + agent.deps.extension_manager.clone(), + Arc::clone(&agent.channels), + Arc::clone(&resolutions), + )); + thread_manager + .set_gate_controller(gate_controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; + *guard = Some(EngineState { thread_manager, conversation_manager, @@ -1921,11 +2155,48 @@ pub async fn init_engine(agent: &Agent) -> Result<(), Error> { auth_manager, extension_manager: agent.deps.extension_manager.clone(), project_root: resolve_project_root(), + external_tool_catalog, + capability_registry: Arc::clone(&capabilities), + gate_controller, + gate_resolutions: resolutions, }); Ok(()) } +/// Boot-time sweep: invalidate `Approval`-kind pending gates carried +/// over from a prior process. Their inline-await receivers are gone, +/// and re-entry would re-run earlier non-idempotent tool calls. Auth +/// and External gates survive — they don't depend on a live VM. +async fn invalidate_stranded_approval_gates( + pending_gates: &crate::gate::store::PendingGateStore, + sse: Option<&Arc<SseManager>>, +) { + let restored = pending_gates.list_all().await; + for gate in restored { + if !matches!( + gate.resume_kind, + ironclaw_engine::ResumeKind::Approval { .. } + ) { + continue; + } + let _ = pending_gates.discard(&gate.key()).await; + if let Some(sse) = sse { + sse.broadcast_for_user( + &gate.user_id, + ironclaw_common::AppEvent::GateResolved { + request_id: gate.request_id.to_string(), + gate_name: gate.gate_name.clone(), + tool_name: gate.action_name.clone(), + resolution: "expired".into(), + message: "Approval interrupted by restart. Please retry.".into(), + thread_id: Some(gate.effective_wire_thread_id()), + }, + ); // projection-exempt: bridge dispatcher, restart-time gate cleanup before threads exist + } + } +} + async fn resolve_pending_gate_for_user( pending_gates: &crate::gate::store::PendingGateStore, user_id: &str, @@ -1982,6 +2253,25 @@ pub async fn get_engine_pending_gate( } } +/// Read-only lookup of a pending gate by `request_id`, scoped to the +/// requesting user. Used by the chat cancel handler to recover the +/// owning thread when the client omits `thread_id` in the resolution +/// payload — without this, a foreground inline-await gate would be +/// stranded (gate marked cancelled, parked VM never unwound). See PR +/// #3366 review. +pub async fn get_pending_gate_by_request_id( + user_id: &str, + request_id: uuid::Uuid, +) -> Option<crate::gate::pending::PendingGateView> { + let lock = ENGINE_STATE.get()?; + let guard = lock.read().await; + let state = guard.as_ref()?; + state + .pending_gates + .peek_by_request_id(request_id, user_id) + .await +} + /// Check whether the user has *any* pending gate (resolved, ambiguous, or /// otherwise). Unlike `get_engine_pending_gate` which returns `None` for /// ambiguous resolutions, this returns `true` whenever at least one gate @@ -2077,6 +2367,195 @@ pub async fn resolve_engine_auth_callback( }) } +/// Wake any Tier 0/Tier 1 inline-await waiters that paused on the +/// credential `(user_id, credential_name)` pair. +/// +/// Half-2 of #3133, inline-await arm. The Tier 1 (CodeAct) and Tier 0 +/// (structured) paths now keep their VM/batch parked on +/// `GateController::pause()` for Authentication gates the same way +/// they do for Approval. When OAuth lands a credential, this helper +/// delivers `GateResolution::Approved` to every parked waiter for +/// `user_id` so the suspended action retries inline against the +/// now-present secret — no thread re-entry, no replay of earlier +/// side effects in the same step. Other users' parked waiters on the +/// same credential name are left untouched. +/// +/// Returns the number of waiters woken (zero is normal — most +/// credential writes don't unblock any inline VM). +pub async fn resolve_inline_gates_for_credential(user_id: &str, credential_name: &str) -> usize { + let Some(lock) = ENGINE_STATE.get() else { + return 0; + }; + let guard = lock.read().await; + let Some(state) = guard.as_ref() else { + return 0; + }; + let woken = state + .gate_resolutions + .deliver_for_credential(user_id, credential_name) + .await; + if woken > 0 { + tracing::debug!( + user = %user_id, + credential = %credential_name, + woken, + "delivered Approved to parked inline-await waiter(s) on credential write" + ); + // #3533: also drop the matching Authentication pending-gate + // rows from the store. The inline-await retry will run with + // the credential now present and either succeed or raise its + // own follow-up gate; the original Authentication row no + // longer represents live state and would otherwise linger + // until expiry (and surface in `HistoryResponse.pending_gate` + // for users who had no follow-up gate). Without this discard, + // the external-callback path (`resolve_engine_auth_callback`) + // is the only thing that cleans up — and skipping that path + // to avoid the "thread already running" race left the row + // orphaned. + let matching: Vec<_> = state + .pending_gates + .list_for_user(user_id) + .await + .into_iter() + .filter(|gate| { + matches!( + &gate.resume_kind, + ironclaw_engine::ResumeKind::Authentication { + credential_name: gate_credential, + .. + } if gate_credential.as_str() == credential_name + ) + }) + .collect(); + for gate in matching { + let _ = state.pending_gates.discard(&gate.key()).await; + } + } + woken +} + +/// Auto-resume paused missions whose `paused_gate` was waiting for the +/// credential named `credential_name`. +/// +/// Half-2 of #3133, mission arm. Called from the OAuth completion +/// paths in `channels::web::features::oauth::oauth_callback_handler` +/// and `extensions::manager`'s WASM OAuth completion. Walks the +/// engine state to locate the [`MissionManager`] and delegates to +/// [`MissionManager::resume_paused_for_credential`], which transitions +/// every matching mission `Paused → Active`, clears its `paused_gate`, +/// and (for non-Manual cadences) kicks off an immediate fire so the +/// user sees follow-through after completing OAuth. +/// +/// Note: this hook is currently OAuth-only. Manual credential writes +/// (`/api/secrets`, `tool_auth`, gate-resolution `CredentialProvided`) +/// do NOT call this helper today; missions paused on a non-OAuth +/// credential write are auto-resumed only when the user resubmits an +/// OAuth callback against the same secret. Plumbing the manual path +/// through this helper is tracked as a follow-up. +/// +/// Returns the count of missions that were resumed (zero is the normal +/// case — most credential writes are not blocking any paused mission). +/// Errors are downgraded to logs because a failure to resume a paused +/// mission must NOT prevent the credential from being persisted — +/// the mission can be manually resumed later. +pub async fn resume_paused_missions_for_credential(user_id: &str, credential_name: &str) -> usize { + let Some(lock) = ENGINE_STATE.get() else { + return 0; + }; + let guard = lock.read().await; + let Some(state) = guard.as_ref() else { + return 0; + }; + let Some(mission_manager) = state.effect_adapter.mission_manager().await else { + return 0; + }; + let cred = match ironclaw_common::CredentialName::new(credential_name) { + Ok(c) => c, + Err(e) => { + debug!( + error = %e, + credential_name = %credential_name, + "skipping mission auto-resume — credential name failed validation" + ); + return 0; + } + }; + match mission_manager + .resume_paused_for_credential(&cred, user_id) + .await + { + Ok(ids) => { + if !ids.is_empty() { + tracing::debug!( + user_id = %user_id, + credential = %credential_name, + resumed = ids.len(), + "auto-resumed paused mission(s) after credential write" + ); + } + ids.len() + } + Err(e) => { + tracing::warn!( + user_id = %user_id, + credential = %credential_name, + error = %e, + "failed to auto-resume paused missions after credential write" + ); + 0 + } + } +} + +/// Auto-resume the paused mission whose `paused_gate.gate_request_id` +/// matches `gate_request_id`, given a user-driven gate-resolve outcome. +/// +/// Half-2 of #3133 for the approval/external path. Called from +/// `/api/chat/gate/resolve` after the foreground gate has been +/// resolved. On `Approved` the mission is transitioned `Paused → +/// Active` and (for non-Manual cadences) immediately fired. On +/// `Denied`/`Cancelled` the mission is marked `Failed` so the user must +/// fix the underlying issue and resume manually. +/// +/// Returns the resumed/failed mission id, or `None` if no paused +/// mission was waiting on this gate (the foreground gate alone was +/// resolved). +pub async fn resume_paused_missions_for_gate_request( + user_id: &str, + gate_request_id: uuid::Uuid, + outcome: ironclaw_engine::GateResolutionOutcome, +) -> Option<ironclaw_engine::types::mission::MissionId> { + let lock = ENGINE_STATE.get()?; + let guard = lock.read().await; + let state = guard.as_ref()?; + let mission_manager = state.effect_adapter.mission_manager().await?; + match mission_manager + .resume_paused_for_request_id(gate_request_id, outcome, user_id) + .await + { + Ok(Some(id)) => { + tracing::debug!( + user_id = %user_id, + %gate_request_id, + mission_id = %id, + outcome = ?outcome, + "mission auto-resume after gate resolution" + ); + Some(id) + } + Ok(None) => None, + Err(e) => { + tracing::warn!( + user_id = %user_id, + %gate_request_id, + error = %e, + "failed to auto-resume paused mission after gate resolution" + ); + None + } + } +} + /// Handle an approval response (yes/no/always) for engine v2. /// /// Called from `handle_message` when the user responds to an approval request. @@ -2199,13 +2678,16 @@ pub async fn handle_external_callback( agent: &Agent, message: &IncomingMessage, request_id: uuid::Uuid, + payload: Option<serde_json::Value>, ) -> Result<BridgeOutcome, Error> { init_engine(agent).await?; let resolution = ironclaw_engine::GateResolution::ExternalCallback { - payload: serde_json::Value::Null, + payload: payload.unwrap_or(serde_json::Value::Null), }; + // Auth-flavored callback (legacy OAuth/pairing): consult the auth + // predicates first, including the conversation-scope hint shortcut. if let Some(thread_id) = hinted_pending_gate_thread_id( &message.user_id, message.conversation_scope(), @@ -2224,13 +2706,23 @@ pub async fn handle_external_callback( return resolve_gate(agent, message, thread_id, request_id, resolution).await; } + // Non-auth External callback (e.g. Responses API caller-executed tool + // result): the gate's resume_kind is `External` but it is not an + // authentication gate, so the auth predicates above don't match it. + if let Some(thread_id) = + pending_gate_thread_id_for_request(&message.user_id, request_id, gate_resume_is_external) + .await? + { + return resolve_gate(agent, message, thread_id, request_id, resolution).await; + } + debug!( user_id = %message.user_id, request_id = %request_id, - "engine v2: no matching pending auth gate for external callback" + "engine v2: no matching pending gate for external callback" ); Ok(BridgeOutcome::Respond( - "No matching pending authentication gate found.".into(), + "No matching pending gate found.".into(), )) } @@ -2293,6 +2785,17 @@ fn gate_is_authentication(gate: &PendingGate) -> bool { ) } +/// Matches any gate whose resume kind is `External`. Used as a fallback in +/// `handle_external_callback` to resume non-auth tool-call pauses (e.g. +/// the Responses API caller-executed tool result path) which never go +/// through the authentication predicates. +fn gate_resume_is_external(gate: &PendingGate) -> bool { + matches!( + gate.resume_kind, + ironclaw_engine::ResumeKind::External { .. } + ) +} + fn gate_view_is_approval(gate: &crate::gate::pending::PendingGateView) -> bool { matches!( gate.resume_kind, @@ -2363,6 +2866,272 @@ async fn pending_gate_thread_id_for_request( Ok(pending) } +/// Outcome of a fast-path inline gate resolution attempt. +/// +/// See [`try_resolve_inline_approval_gate`]. +#[derive(Debug)] +#[must_use] +pub enum InlineGateOutcome { + /// The resolution was delivered directly to a parked engine VM. The + /// pending gate has been consumed; SSE `GateResolved` was broadcast. + Delivered, + /// No live VM was waiting for this gate (engine uninitialized, no + /// matching parked future, or non-Approval resume kind). The pending + /// gate has been left in place — the caller should fall through to + /// the legacy mpsc dispatch path so the agent loop's `resolve_gate` + /// can resume the thread normally. + NoLiveVm, +} + +/// Verification failures from [`try_resolve_inline_approval_gate`] that +/// must surface as 4xx HTTP responses rather than fall through to the +/// legacy resume path. Variants map to specific status codes at the HTTP +/// boundary (see `chat_approval_handler`): +/// +/// - [`InlineGateError::ChannelMismatch`] → 403 Forbidden +/// - [`InlineGateError::Stale`] → 409 Conflict (request_id already +/// resolved or doesn't match the latest pending row) +/// - [`InlineGateError::Expired`] → 409 Conflict (the pending gate's +/// `expires_at` has passed) +/// - [`InlineGateError::Other`] → 500 Internal Server Error +/// +/// Typed at the API boundary (rather than relying on +/// `error.to_string().contains("authorization")`) so a future change to +/// the error format string can't silently flip a 403 into a 500. +#[derive(Debug, thiserror::Error)] +pub enum InlineGateError { + /// The resolving channel does not match the channel that originated + /// the gate (and is not in the trusted-channel allowlist). + #[error("Channel '{actual}' cannot resolve gates from channel '{expected}'")] + ChannelMismatch { expected: String, actual: String }, + /// The request_id doesn't match the active pending gate (already + /// resolved, dropped, or replaced by a newer gate row). + #[error("Approval request is stale or already resolved")] + Stale, + /// The pending gate's `expires_at` has elapsed. + #[error("Approval request has expired")] + Expired, + /// The pending gate exists but does not belong to the requesting user. + /// Surfaced as a 403 to avoid leaking gate existence across tenants. + #[error("not authorized to resolve this gate")] + Unauthorized, + /// Any other gate-store failure. + #[error("gate error: {0}")] + Other(String), +} + +/// Fast-path inline resolution for an Approval gate, intended to be +/// callable from HTTP handlers without going through the agent-loop +/// mpsc. +/// +/// **Why this exists.** When `BridgeGateController::pause` parks a Tier 0 +/// or Tier 1 execution on an Approval gate, the engine call sits in +/// `await rx`. That await is held by the bridge call invoked from the +/// agent loop's `handle_message`, which means the per-user agent loop +/// is blocked at `match self.handle_message(...).await` and cannot +/// drain new submissions from `msg_tx`. A subsequent `ExecApproval` +/// posted to `/api/chat/approval` and forwarded through `msg_tx` would +/// queue indefinitely behind the parked alpha — so `try_deliver` would +/// never run and alpha would only wake on the 30-minute pause timeout. +/// +/// This function lets the HTTP handler skip the mpsc and call into +/// the gate controller's in-memory delivery channel directly. The +/// engine resumes from its exact suspension point, and the handler +/// returns 202 to the user. +/// +/// On `NoLiveVm` the caller should still dispatch the legacy +/// `ExecApproval` submission so the agent loop can resume the thread +/// via `state.thread_manager.resume_thread`. That path uses +/// `&Agent` for status updates and remains the source of truth for +/// non-inline resolutions (Authentication, External callbacks). +/// +/// Errors are returned only when the gate exists but verification +/// fails (channel mismatch, stale request_id, expired). Those map to +/// 4xx responses via [`InlineGateError`]; the caller should not fall +/// through. +pub async fn try_resolve_inline_approval_gate( + user_id: &str, + channel: &str, + request_id: uuid::Uuid, + resolution: ironclaw_engine::GateResolution, + settings_store: Option<&(dyn crate::db::SettingsStore + Send + Sync)>, +) -> Result<InlineGateOutcome, InlineGateError> { + // Only Approval-shaped resolutions are eligible for inline-await. + // `BridgeGateController::pause` returns Cancelled immediately for + // Authentication and External resume kinds without parking, so + // there's nothing to deliver to and we'd just need the legacy + // resume path. + if !matches!( + resolution, + ironclaw_engine::GateResolution::Approved { .. } + | ironclaw_engine::GateResolution::Denied { .. } + | ironclaw_engine::GateResolution::Cancelled + ) { + return Ok(InlineGateOutcome::NoLiveVm); + } + + let Some(lock) = ENGINE_STATE.get() else { + return Ok(InlineGateOutcome::NoLiveVm); + }; + let guard = lock.read().await; + let Some(state) = guard.as_ref() else { + return Ok(InlineGateOutcome::NoLiveVm); + }; + + // Resolve the gate by `request_id` (system-wide unique) rather + // than by a caller-supplied thread identifier. The wire + // `req.thread_id` on the HTTP surface is the channel-visible + // value — for the web gateway that is the per-conversation UUID + // returned by `/api/chat/thread/new`, recorded on the gate as + // `scope_thread_id` — not the internal engine `ThreadId` that + // keys `PendingGateStore`. Looking up by `request_id` under the + // store's single mutex keeps the lookup + remove atomic and + // avoids the wire-vs.-engine identifier confusion that would + // otherwise miss every gate whose channel scope differs from its + // engine thread. + // + // Verification failures surface as a typed `InlineGateError` so + // the HTTP handler can map to the right 4xx without inspecting + // message strings. `NotFound` is treated as `NoLiveVm` (legacy + // mpsc fall-through) — a `request_id` we don't have means the + // gate was already resolved, never existed, or wasn't restored + // after a process restart, none of which should surface a 5xx + // here. + let pending = match state + .pending_gates + .take_verified_by_request_id(request_id, user_id, channel) + .await + { + Ok(gate) => gate, + Err(e) => { + use crate::gate::store::GateStoreError; + return match e { + GateStoreError::NotFound => Ok(InlineGateOutcome::NoLiveVm), + GateStoreError::ChannelMismatch { expected, actual } => { + Err(InlineGateError::ChannelMismatch { expected, actual }) + } + GateStoreError::Unauthorized => Err(InlineGateError::Unauthorized), + GateStoreError::Expired => Err(InlineGateError::Expired), + GateStoreError::RequestIdMismatch => Err(InlineGateError::Stale), + other => Err(InlineGateError::Other(other.to_string())), + }; + } + }; + let thread_id = pending.thread_id; + + // Only Approval-resume gates are parked by the gate controller. A + // non-Approval gate hitting take_verified here means a different + // resume_kind happened to share the request_id — re-insert and tell + // the caller to fall back to legacy resume. + if !matches!( + pending.resume_kind, + ironclaw_engine::ResumeKind::Approval { .. } + ) { + if let Err(e) = state.pending_gates.insert(pending.clone()).await { + debug!( + user_id = %user_id, + thread_id = %thread_id, + error = %e, + "try_resolve_inline_approval_gate: failed to re-insert non-Approval gate" + ); + } + return Ok(InlineGateOutcome::NoLiveVm); + } + + let always_for_inline = match &resolution { + ironclaw_engine::GateResolution::Approved { always } => { + clamp_always_to_resume_kind(*always, &pending.resume_kind) + } + _ => false, + }; + + let legacy_registry_name = legacy_extension_alias(&pending.action_name); + let prior_permission = if always_for_inline { + state + .effect_adapter + .auto_approve_tool(&pending.action_name) + .await; + if let Some(ref registry_name) = legacy_registry_name { + state.effect_adapter.auto_approve_tool(registry_name).await; + } + persist_always_allow_with_store(settings_store, state, &pending).await + } else { + None + }; + + let inline_resolution = match &resolution { + ironclaw_engine::GateResolution::Approved { .. } => { + ironclaw_engine::GateResolution::Approved { + always: always_for_inline, + } + } + ironclaw_engine::GateResolution::Denied { reason } => { + ironclaw_engine::GateResolution::Denied { + reason: reason.clone(), + } + } + ironclaw_engine::GateResolution::Cancelled => ironclaw_engine::GateResolution::Cancelled, + _ => unreachable!("guarded by outer matches!()"), + }; + + if state + .gate_controller + .try_deliver(request_id, inline_resolution) + .await + { + if let Some(ref sse) = state.sse { + let (label, status_msg) = match &resolution { + ironclaw_engine::GateResolution::Approved { .. } => { + if always_for_inline { + ("approved_always", "Gate approved. Resuming execution.") + } else { + ("approved", "Gate approved. Resuming execution.") + } + } + ironclaw_engine::GateResolution::Denied { .. } => ("denied", "Gate denied."), + ironclaw_engine::GateResolution::Cancelled => ("cancelled", "Gate cancelled."), + _ => unreachable!(), + }; + let event = AppEvent::GateResolved { + request_id: pending.request_id.to_string(), + gate_name: pending.gate_name.clone(), + tool_name: pending.action_name.clone(), + resolution: label.into(), + message: status_msg.into(), + thread_id: Some(pending.effective_wire_thread_id()), + }; + sse.broadcast_for_user(user_id, event); // projection-exempt: bridge dispatcher, inline-await fast-path resolution event + } + return Ok(InlineGateOutcome::Delivered); + } + + // try_deliver returned false: no parked future for this request_id. + // Roll back the auto-approve preference we installed and re-insert + // the pending gate so the legacy mpsc dispatch path can find it. + if always_for_inline { + state + .effect_adapter + .revoke_auto_approve(&pending.action_name) + .await; + if let Some(registry_name) = legacy_registry_name { + state + .effect_adapter + .revoke_auto_approve(&registry_name) + .await; + } + revert_always_allow_with_store(settings_store, &pending, prior_permission).await; + } + if let Err(e) = state.pending_gates.insert(pending).await { + debug!( + user_id = %user_id, + thread_id = %thread_id, + error = %e, + "try_resolve_inline_approval_gate: failed to re-insert pending gate after no-live-VM" + ); + } + Ok(InlineGateOutcome::NoLiveVm) +} + /// Resolve a unified pending gate. /// /// This is the single entry point for resolving gates stored in the @@ -2412,8 +3181,116 @@ pub async fn resolve_gate( } })?; - match resolution { - ironclaw_engine::GateResolution::Approved { always } => { + // Inline gate-await fast path: if the engine is actively awaiting + // this gate (live Tier 0 batch or Tier 1 CodeAct VM), hand the + // resolution back through the controller's in-memory channel. + // The engine continues from the exact suspension point — no + // re-entry, no replay, no double-execution of earlier non-idempotent + // tool calls in the same step. + // + // We still install any auto-approve preference *before* delivery so + // subsequent gates in the same execution see policy `Allow` rather + // than gating again. + if matches!( + resolution, + ironclaw_engine::GateResolution::Approved { .. } + | ironclaw_engine::GateResolution::Denied { .. } + | ironclaw_engine::GateResolution::Cancelled + ) { + let always_for_inline = match &resolution { + ironclaw_engine::GateResolution::Approved { always } => { + clamp_always_to_resume_kind(*always, &pending.resume_kind) + } + _ => false, + }; + + let legacy_registry_name = legacy_extension_alias(&pending.action_name); + let prior_permission = if always_for_inline { + state + .effect_adapter + .auto_approve_tool(&pending.action_name) + .await; + if let Some(ref registry_name) = legacy_registry_name { + state.effect_adapter.auto_approve_tool(registry_name).await; + } + persist_always_allow(agent, state, &pending).await + } else { + None + }; + + // Re-build the resolution clamped to the pending gate's policy. + let inline_resolution = match &resolution { + ironclaw_engine::GateResolution::Approved { .. } => { + ironclaw_engine::GateResolution::Approved { + always: always_for_inline, + } + } + ironclaw_engine::GateResolution::Denied { reason } => { + ironclaw_engine::GateResolution::Denied { + reason: reason.clone(), + } + } + ironclaw_engine::GateResolution::Cancelled => { + ironclaw_engine::GateResolution::Cancelled + } + _ => unreachable!("guarded by outer matches!()"), + }; + + if state + .gate_controller + .try_deliver(request_id, inline_resolution) + .await + { + if let Some(ref sse) = state.sse { + let (label, status_msg) = match &resolution { + ironclaw_engine::GateResolution::Approved { .. } => { + if always_for_inline { + ("approved_always", "Gate approved. Resuming execution.") + } else { + ("approved", "Gate approved. Resuming execution.") + } + } + ironclaw_engine::GateResolution::Denied { .. } => ("denied", "Gate denied."), + ironclaw_engine::GateResolution::Cancelled => ("cancelled", "Gate cancelled."), + _ => unreachable!(), + }; + sse.broadcast_for_user( + &message.user_id, + AppEvent::GateResolved { + request_id: pending.request_id.to_string(), + gate_name: pending.gate_name.clone(), + tool_name: pending.action_name.clone(), + resolution: label.into(), + message: status_msg.into(), + thread_id: Some(pending.effective_wire_thread_id()), + }, + ); // projection-exempt: bridge dispatcher, inline-await fast-path resolution event + } + return Ok(BridgeOutcome::Pending); + } + + // Delivery failed — no live VM was waiting (process restart, or + // gate was created via a code path that didn't register an + // inline-await receiver). Roll back any auto-approve we just + // installed so subsequent calls don't see a stale preference, + // then fall through to the legacy re-entry path below. + if always_for_inline { + state + .effect_adapter + .revoke_auto_approve(&pending.action_name) + .await; + if let Some(registry_name) = legacy_registry_name { + state + .effect_adapter + .revoke_auto_approve(&registry_name) + .await; + } + revert_always_allow(agent, &pending, prior_permission).await; + } + } + + match resolution { + ironclaw_engine::GateResolution::Approved { always } => { // Clamp the caller-supplied `always` flag to what the pending gate // actually permits. A protected `memory_write` (orchestrator code, // prompt overlays) advertises `Approval { allow_always: false }` @@ -2530,7 +3407,7 @@ pub async fn resolve_gate( // require_action_attempt obligation, which then nudges the LLM // to issue another tool call — exactly the opposite of what a // denial should produce. Avoid every phrase in - // `crate::llm::user_signals_execution_intent`'s list (the + // `ironclaw_llm::user_signals_execution_intent`'s list (the // helper is defined in `src/llm/reasoning.rs` and re-exported // from `crate::llm`). let deny_msg = ironclaw_engine::ThreadMessage::user(format!( @@ -2808,7 +3685,7 @@ pub async fn resolve_gate( } } - ironclaw_engine::GateResolution::ExternalCallback { .. } => { + ironclaw_engine::GateResolution::ExternalCallback { ref payload } => { if let Some(ref sse) = state.sse { sse.broadcast_for_user( &message.user_id, @@ -2822,7 +3699,58 @@ pub async fn resolve_gate( }, ); } - if let Some(resume_output) = pending.resume_output.clone() { + + // Caller-tool callbacks (Responses API) carry the tool's + // output in the resolution payload. The pending gate has + // `resume_output: None` because at gate-fire time the + // adapter doesn't have the output yet — so the legacy + // OAuth/pairing branch (which uses `pending.resume_output`) + // would re-run the action and re-pause forever. Instead, + // synthesize an `ActionResult`-shaped ThreadMessage from + // the resolution payload and resume directly. + // + // OAuth/pairing flows keep using the original + // `pending.resume_output` path (their callback_id has the + // `pairing:` prefix, not `ext_tool:`). + let is_external_tool_callback = matches!( + pending.resume_kind, + ironclaw_engine::ResumeKind::External { ref callback_id } + if crate::bridge::is_external_tool_callback_id(callback_id) + ); + + if is_external_tool_callback { + let resolved_call_id = + resolved_or_synthetic_call_id_for_pending_action(state, &pending).await?; + let synthesized_output = extract_external_tool_output(payload, &resolved_call_id); + // External-tool payloads originate outside the + // EffectBridgeAdapter's sanitization pipeline. Run them + // through the same safety pass internal tool outputs + // get — leak detection, length cap, injection sanitizer, + // policy — before they reach the LLM. Caller is not a + // trust boundary; treat the payload like any other + // tool output. + let raw_rendered = serde_json::to_string_pretty(&synthesized_output) + .unwrap_or_else(|_| synthesized_output.to_string()); + let sanitized = state + .effect_adapter + .safety() + .sanitize_tool_output(&pending.action_name, &raw_rendered); + state + .thread_manager + .resume_thread( + pending.thread_id, + message.user_id.clone(), + Some(ironclaw_engine::ThreadMessage::action_result( + &resolved_call_id, + &pending.action_name, + sanitized.content, + )), + None, + Some(resolved_call_id), + ) + .await + .map_err(|e| engine_err("resume error", e))?; + } else if let Some(resume_output) = pending.resume_output.clone() { let resolved_call_id = resolved_or_synthetic_call_id_for_pending_action(state, &pending).await?; state @@ -3077,6 +4005,120 @@ pub async fn handle_expected( } } +/// Handle `approve <channel> <code>` — claim a pairing code from any chat +/// surface (TUI, CLI, web, or Telegram itself). Mirrors what the web +/// `POST /api/pairing/{channel}/approve` handler does, so the same approval +/// works regardless of where the user typed it. This closes #3317, where +/// the Telegram bot's pairing reply pointed users at "IronClaw" without +/// naming a surface and the agent rejected the resulting chat input. +pub async fn handle_pairing_claim( + agent: &Agent, + message: &IncomingMessage, + channel: &str, + code: &str, +) -> Result<BridgeOutcome, Error> { + use ironclaw_common::ExtensionName; + + // Validate the channel name at the boundary, mirroring + // `web::features::pairing::parse_channel`. We discard the canonical + // form and carry the lowercased raw string forward because the pairing + // store keys off the un-folded name (see `pairing/mod.rs` + // `normalize_channel_name`). + let lowered = channel.to_ascii_lowercase(); + if ExtensionName::new(&lowered).is_err() { + // The raw `channel` token comes from chat input and is unbounded. + // Cap the echo at 32 characters and strip non-printable / non- + // alphanumeric characters before rendering, so a hostile or + // accidentally-pasted blob can't blow up the chat reply or smuggle + // control characters / Markdown through to the SSE / Telegram / + // TUI surface. The underlying `IdentityError` variants also carry + // the raw input verbatim, so we render a fixed category message + // rather than `{e}` to keep the reply size bounded by the echo. + let preview: String = channel + .chars() + .filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) + .take(32) + .collect(); + let preview = if preview.is_empty() { + "<empty>".to_string() + } else { + preview + }; + return Ok(BridgeOutcome::Respond(format!( + "Invalid channel name `{preview}` — channel names must be \ + lowercase letters, digits, hyphens, or underscores (e.g. \ + `telegram`, `slack-relay`)." + ))); + } + + let Some(ext_mgr) = agent.deps.extension_manager.as_ref() else { + return Ok(BridgeOutcome::Respond( + "Pairing is not available — extension manager is not configured.".into(), + )); + }; + let Some(pairing_store) = ext_mgr.pairing_store() else { + return Ok(BridgeOutcome::Respond( + "Pairing is not available — pairing store is not configured.".into(), + )); + }; + + // Bind the pairing to the message's user_id. `from_trusted` matches the + // web handler's pattern: the user identity is sourced from the inbound + // channel auth, not user-controlled chat content. Role is irrelevant + // for self-service approval — only the id is recorded on the pairing + // row. + let owner_id = crate::ownership::UserId::from_trusted( + message.user_id.clone(), + crate::ownership::UserRole::Regular, + ); + + let approval = match pairing_store.approve(&lowered, code, &owner_id).await { + Ok(approval) => approval, + Err(crate::error::DatabaseError::NotFound { .. }) => { + return Ok(BridgeOutcome::Respond( + "Invalid or expired pairing code.".into(), + )); + } + Err(e) => { + debug!(channel = %lowered, error = %e, "pairing approval failed"); + return Ok(BridgeOutcome::Respond( + "Internal error processing pairing approval.".into(), + )); + } + }; + + // Propagate to the running channel so the WASM channel picks up the new + // owner binding without a restart. Same shape as the web handler — on + // propagation failure, revert the DB approval so the user can retry. + match ext_mgr + .complete_pairing_approval(&lowered, &approval.external_id) + .await + { + Ok(()) => Ok(BridgeOutcome::Respond(format!( + "Pairing approved — `{lowered}` is now linked to your account." + ))), + Err(e) => { + tracing::warn!( + channel = %lowered, + error = %e, + "pairing approval propagation to running channel failed" + ); + if let Err(revert_err) = pairing_store.revert_approval(&approval).await { + tracing::warn!( + channel = %lowered, + error = %revert_err, + "failed to revert pairing approval after propagation failure" + ); + } + Ok(BridgeOutcome::Respond( + "Pairing was approved, but the running channel could not be updated. \ + Please retry or restart the channel." + .into(), + )) + } + } +} + /// Find the most recent thread in a conversation (checks active threads first, /// then falls back to the last completed thread visible in conversation entries). async fn find_most_recent_thread( @@ -3143,6 +4185,29 @@ async fn clear_engine_conversation(agent: &Agent, message: &IncomingMessage) -> } } + // Drain in-flight OAuth flows for this user (#3320). + // + // Pending flows otherwise live until `OAUTH_FLOW_EXPIRY` (5 min). On a + // user-initiated `/clear`, the user expects a clean slate — leaving a + // ghost flow can: (a) match a stale `state` from a never-completed + // browser tab, (b) fool a fresh auth attempt's CSRF dedupe, or + // (c) cause the next `extension_manager::pending_oauth_flows()` lookup + // to find an entry whose corresponding engine gate has already been + // discarded above. Drain all flows owned by the clearing user. + if let Some(ext_mgr) = agent.deps.extension_manager.as_ref() { + let mut flows = ext_mgr.pending_oauth_flows().write().await; + let before = flows.len(); + flows.retain(|_state, flow| flow.user_id != message.user_id); + let removed = before.saturating_sub(flows.len()); + if removed > 0 { + debug!( + user_id = %message.user_id, + removed, + "engine v2: drained pending OAuth flows on /clear" + ); + } + } + // Clear the conversation entries and active thread list state .conversation_manager @@ -3223,6 +4288,45 @@ pub async fn clear_engine_pending_auth(user_id: &str, thread_id: Option<&str>) { } } +/// Clear pending auth gates for a user that match a specific credential. +/// +/// Used by OAuth failure paths where we need to release the gate that +/// was waiting on *this* OAuth flow without disturbing unrelated +/// authentication gates. A bare `clear_engine_pending_auth(user, None)` +/// would discard every pending Authentication gate for the user — e.g. a +/// failed Gmail callback would also nuke an in-flight Slack/MCP gate +/// running on a different thread. +/// +/// `credential_name` is taken as `&str` so callers in +/// `src/channels/web/**` don't have to construct an +/// `ironclaw_common::CredentialName` at the web boundary (per +/// `web/CLAUDE.md` — credential identity stays backend-side). Invalid +/// credential strings silently no-op rather than erroring; the gate +/// simply stays open and the user retries. +pub async fn clear_engine_pending_auth_for_credential(user_id: &str, credential_name: &str) { + let Ok(target) = ironclaw_common::CredentialName::new(credential_name) else { + return; + }; + let Some(lock) = ENGINE_STATE.get() else { + return; + }; + let guard = lock.read().await; + let Some(state) = guard.as_ref() else { + return; + }; + + for gate in state.pending_gates.list_for_user(user_id).await { + if let ironclaw_engine::ResumeKind::Authentication { + credential_name: gate_credential, + .. + } = &gate.resume_kind + && gate_credential == &target + { + let _ = state.pending_gates.discard(&gate.key()).await; + } + } +} + pub async fn discard_engine_pending_auth_request( user_id: &str, request_id: uuid::Uuid, @@ -3555,14 +4659,61 @@ async fn handle_with_engine_inner( // Detect execution intent and configure obligation accordingly let thread_config = { let mut cfg = ThreadConfig::default(); - if crate::llm::user_signals_execution_intent(content) { + if ironclaw_llm::user_signals_execution_intent(content) { cfg.require_action_attempt = true; } cfg }; - // Handle the message — spawns a new thread or injects into active one - let thread_id = state + // Stamp the conversation scope (parseable as a Uuid) into the + // thread's `initial_metadata`. The engine reads it back into + // `ThreadExecutionContext.conversation_scope`, which lets the + // bridge's `EffectBridgeAdapter` resolve per-conversation state + // (today: caller-supplied external tool catalog) by either the + // engine `thread_id` or the caller-side scope. Without this the + // executor task that starts immediately after spawn would race the + // bridge's post-spawn `transfer` and miss caller tools on the + // first turn. + let scope_uuid = parse_engine_thread_id(scope); + let extra_metadata = scope_uuid.map(|tid| { + let mut map = serde_json::Map::new(); + map.insert( + "conversation_scope".into(), + serde_json::Value::String(tid.0.to_string()), + ); + map + }); + + // Pre-bind per-execution context BEFORE the engine spawns the + // thread. `handle_user_message` allocates and starts the engine + // task internally; if a fast tool gate fires before + // `set_execution_context` lands, the controller's `pause()` would + // otherwise find no entry and cancel the gate silently. The + // pre-execution slot is keyed by user_id and the per-conversation + // lock upstream guarantees at most one bridge turn per + // conversation is in flight. + let scope_thread_id = message + .conversation_scope() + .and_then(|s| ironclaw_common::ExternalThreadId::new(s).ok()); + let per_exec_context = crate::bridge::gate_controller::PerExecutionContext { + conversation_id: conv_id, + source_channel: message.channel.clone(), + scope_thread_id, + channel_metadata: message.metadata.clone(), + original_message: Some(message.content.clone()), + }; + state + .gate_controller + .set_pre_execution_context(message.user_id.clone(), conv_id, per_exec_context.clone()) + .await; + + // Handle the message — spawns a new thread or injects into active one. + // On error we must clear the pre-execution slot we just installed: + // without this, a failed `handle_user_message` (engine spawn / inject + // failed before any thread_id was allocated) leaves a stale entry + // keyed by user_id that would mis-route the next gate prompt for + // the same user. + let thread_id = match state .conversation_manager .handle_user_message( conv_id, @@ -3571,9 +4722,40 @@ async fn handle_with_engine_inner( &message.user_id, thread_config, validated_tz.as_ref().map(|tz| tz.name()), + extra_metadata, ) .await - .map_err(|e| engine_err("thread error", e))?; + { + Ok(tid) => tid, + Err(e) => { + state + .gate_controller + .clear_pre_execution_context(&message.user_id, conv_id) + .await; + return Err(engine_err("thread error", e)); + } + }; + + // Promote the pre-execution entry to (user, thread)-keyed. From + // here on, gates from this thread land on the thread-keyed entry + // first; the per-user fallback covers any gates that fire before + // this promotion lands. + state + .gate_controller + .set_execution_context(message.user_id.clone(), thread_id, per_exec_context) + .await; + + // Re-key the catalog onto the engine's allocated `thread_id` so + // the terminal-state cleanup hook in `await_thread_outcome` finds + // the entry under the canonical key. The race-window protection + // is the conversation_scope plumbing above; this transfer is the + // bookkeeping leg. + if let Some(scope_uuid) = scope_uuid { + state + .external_tool_catalog + .transfer(scope_uuid, thread_id) + .await; + } if !attachment_notes.is_empty() { save_attachment_index_notes( @@ -3617,95 +4799,526 @@ async fn handle_with_engine_inner( } debug!(thread_id = %thread_id, "engine v2: thread spawned"); - await_thread_outcome(agent, state, message, conv_id, thread_id).await + let outcome = await_thread_outcome(agent, state, message, conv_id, thread_id).await; + // Drop per-execution context. The `PendingGate` row (if a gate + // fired) carries everything the resolver needs from here on. + // + // BridgeOutcome::Pending means the request handler hit its deadline + // while the engine was still running (typically parked in + // `BridgeGateController::pause` waiting for an approval). Clearing + // context here would strand the parked thread — its eventual + // resolution would call `pause()` for any subsequent gate with no + // registered context, surfacing as silent `Cancelled`. Defer the + // cleanup to a background task that watches for thread completion + // and clears once the engine is actually done. + if matches!(outcome, Ok(BridgeOutcome::Pending)) + && state.thread_manager.is_running(thread_id).await + { + spawn_deferred_context_cleanup( + Arc::clone(&state.gate_controller), + Arc::clone(&state.thread_manager), + message.user_id.clone(), + thread_id, + conv_id, + ); + } else { + state + .gate_controller + .clear_execution_context(&message.user_id, thread_id, conv_id) + .await; + } + outcome } -/// Fire active OnEvent missions whose pattern matches the inbound message. +/// Watch a still-running thread for completion and clear its +/// per-execution context once the engine task has actually finished. /// -/// Builds a payload containing the message metadata that mission threads -/// can read via `state["trigger_payload"]`. Skips empty content and -/// system-channel messages. Errors are logged at debug level — a failure -/// here must never block the user-facing message flow. -async fn fire_event_missions_for_message( - state: &EngineState, - message: &IncomingMessage, - content: &str, +/// Used when `await_thread_outcome` returned [`BridgeOutcome::Pending`] +/// because the request-level deadline fired while the thread was +/// parked in [`crate::bridge::gate_controller::BridgeGateController::pause`]. +/// The thread is still alive and the (user, thread)-keyed context must +/// stay registered until the eventual gate resolution drives the engine +/// to completion — otherwise a follow-up gate from the same execution +/// surfaces as silent `Cancelled` (no prompt). +/// +/// Polls `is_running` with a coarse cadence; gate `expires_at` (30 min) +/// upper-bounds how long the thread can stay parked, so the watcher is +/// guaranteed to terminate. The cap is a defensive safety against any +/// future code path that could deadlock the engine task. +fn spawn_deferred_context_cleanup( + gate_controller: Arc<crate::bridge::gate_controller::BridgeGateController>, + thread_manager: Arc<ironclaw_engine::ThreadManager>, + user_id: String, + thread_id: ironclaw_engine::ThreadId, + conv_id: ironclaw_engine::ConversationId, ) { - // Skip empty messages — there's nothing to pattern-match against - // and we don't want missions firing on every status update or empty - // user input. - let trimmed = content.trim(); - if trimmed.is_empty() { - return; - } - - // Recursion guards. Channel adapters that echo the agent's own - // outbound text back as inbound events MUST set is_agent_broadcast - // (Slack/Discord-style); messages produced as a side effect of a - // mission firing MUST set triggering_mission_id (chain-recursion - // across distinct missions). Either flag means: do not re-fire. - if message.is_agent_broadcast { - debug!( - channel = %message.channel, - "engine v2: skipping mission firing — message is an agent broadcast echo" - ); - return; - } - if let Some(ref upstream) = message.triggering_mission_id { + tokio::spawn(async move { + // Poll cadence: 30s (cheap; thread completion is on the order + // of seconds-to-minutes once the user resolves). Cap at one + // hour — well past the 30-min PendingGate expiry that bounds + // any pause() call. + let poll_interval = std::time::Duration::from_secs(30); + let max_wait = std::time::Duration::from_secs(60 * 60); + let started = tokio::time::Instant::now(); + loop { + if !thread_manager.is_running(thread_id).await { + break; + } + if started.elapsed() >= max_wait { + tracing::warn!( + thread_id = %thread_id, + "deferred context cleanup hit one-hour cap; clearing context anyway" + ); + break; + } + tokio::time::sleep(poll_interval).await; + } + gate_controller + .clear_execution_context(&user_id, thread_id, conv_id) + .await; debug!( - channel = %message.channel, - upstream_mission_id = %upstream, - "engine v2: skipping mission firing — message originated from a mission" + thread_id = %thread_id, + "engine v2: deferred context cleanup ran" ); - return; - } - - let Some(mission_manager) = state.effect_adapter.mission_manager().await else { - return; - }; - - let payload = serde_json::json!({ - "channel": message.channel, - "user_id": message.user_id, - "content": content, - "metadata": message.metadata, }); - - match mission_manager - .fire_on_message_event(&message.channel, content, &message.user_id, Some(payload)) - .await - { - Ok(spawned) if !spawned.is_empty() => { - debug!( - count = spawned.len(), - channel = %message.channel, - user_id = %message.user_id, - "engine v2: fired {} OnEvent mission(s) from inbound message", - spawned.len() - ); - } - Ok(_) => {} - Err(error) => { - debug!( - channel = %message.channel, - error = %error, - "engine v2: fire_on_message_event failed; continuing with normal handling" - ); - } - } } -async fn await_thread_outcome( - agent: &Agent, +/// Background continuation that takes over event forwarding and final +/// response delivery for a thread that parked at an inline approval +/// gate. Spawned by `await_thread_outcome` once it detects a pending +/// gate row for the (user, thread) — at that point the foreground +/// `handle_message` future is unblocked (returns `Pending`) so the +/// per-user agent loop can dispatch other threads, while this task +/// continues to: +/// +/// 1. Forward `ThreadEvent`s for `thread_id` to SSE and the originating +/// channel — covers both the events emitted before the user +/// resolves the gate and the post-resume events once the engine +/// continues from the parked tool call. +/// 2. Detect thread completion via `is_running`, then call +/// `join_thread` for the final outcome. +/// 3. Broadcast the final response via SSE (`AppEvent::Response`), +/// deliver it through the originating channel +/// (`ChannelManager::respond` + `Done` status), and persist it to +/// the v1 conversation table for the history API. +/// 4. Clear the per-(user, thread) execution context so the gate +/// controller's bookkeeping bounds. +/// +/// Without this task, after early-Pending-on-park the engine resumes +/// invisibly: SSE clients never see the assistant response, non-web +/// channels (Telegram, CLI) never receive the response message, and +/// the history table is missing the final assistant turn. +/// +/// Capped at one hour to bound execution against any pathological +/// post-resume hang; the gate `expires_at` (30 min) upper-bounds the +/// pre-resume wait, and a sane post-resume thread completes well +/// inside the second 30 min. +#[allow(clippy::too_many_arguments)] +fn spawn_post_park_continuation( state: &EngineState, - message: &IncomingMessage, + channels: Arc<crate::channels::ChannelManager>, + message: IncomingMessage, conv_id: ironclaw_engine::ConversationId, thread_id: ironclaw_engine::ThreadId, -) -> Result<BridgeOutcome, Error> { - let mut event_rx = state.thread_manager.subscribe_events(); - let channels = &agent.channels; - let channel_name = &message.channel; - let metadata = &message.metadata; +) { + let thread_manager = Arc::clone(&state.thread_manager); + let conversation_manager = Arc::clone(&state.conversation_manager); + let effect_adapter = Arc::clone(&state.effect_adapter); + let store = Arc::clone(&state.store); + let gate_controller = Arc::clone(&state.gate_controller); + let pending_gates = Arc::clone(&state.pending_gates); + let sse = state.sse.clone(); + let db = state.db.clone(); + let auth_manager = state.auth_manager.clone(); + let extension_manager = state.extension_manager.clone(); + let user_id = message.user_id.clone(); + let channel_name = message.channel.clone(); + let metadata = message.metadata.clone(); + let tid_str = thread_id.to_string(); + + tokio::spawn(async move { + let mut event_rx = thread_manager.subscribe_events(); + // Cap at one hour: gate expiry bounds the pre-resume wait, and + // a sane post-resume thread completes well inside that. + let max_wait = std::time::Duration::from_secs(60 * 60); + let started = tokio::time::Instant::now(); + loop { + tokio::select! { + event = event_rx.recv() => { + match event { + Ok(ref evt) if evt.thread_id == thread_id => { + forward_event_to_channel(evt, &channels, &channel_name, &metadata).await; + if let Some(ref sse) = sse { + let skip_verbose = !sse.has_verbose_receivers(); + let leak_detector = effect_adapter.safety().leak_detector(); + for mut app_event in thread_event_to_app_events(evt, &tid_str) { + if skip_verbose && app_event.is_verbose_only() { + continue; + } + redact_code_executed_secrets(&mut app_event, leak_detector); + sse.broadcast_for_user(&user_id, app_event); // projection-exempt: bridge dispatcher, post-park event forwarding + } + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + _ => {} + } + } + _ = tokio::time::sleep(std::time::Duration::from_millis(500)) => { + if !thread_manager.is_running(thread_id).await { + break; + } + if started.elapsed() >= max_wait { + tracing::warn!( + thread_id = %thread_id, + "post-park continuation hit one-hour cap; abandoning" + ); + gate_controller.clear_execution_context(&user_id, thread_id, conv_id).await; + return; + } + } + } + } + + // Thread completed. Mirror `await_thread_outcome`'s post-loop + // outcome → BridgeOutcome path, but deliver the response + // directly via channel + SSE rather than returning it through + // the bridge return value (the foreground call returned Pending + // long ago). + let outcome = match thread_manager.join_thread(thread_id).await { + Ok(o) => o, + Err(e) => { + tracing::debug!( + thread_id = %thread_id, + error = %e, + "post-park continuation: join_thread failed" + ); + gate_controller + .clear_execution_context(&user_id, thread_id, conv_id) + .await; + return; + } + }; + + if let Err(e) = conversation_manager + .record_thread_outcome(conv_id, thread_id, &outcome) + .await + { + tracing::debug!( + thread_id = %thread_id, + error = %e, + "post-park continuation: record_thread_outcome failed" + ); + } + + let response_text: Option<String> = match &outcome { + ThreadOutcome::Completed { response } => { + if let Some(ref db) = db { + persist_v2_tool_calls(&store, db, thread_id, &message).await; + } + response.clone() + } + ThreadOutcome::Stopped => Some("Thread was stopped.".into()), + ThreadOutcome::MaxIterations => { + Some("Reached maximum iterations without completing.".into()) + } + ThreadOutcome::Failed { + error, + debug_detail, + } => { + let sanitized = + crate::bridge::user_facing_errors::user_facing_thread_failure(error); + let sse_will_deliver_to_user = + sse.is_some() && channel_name == GATEWAY_CHANNEL_NAME; + if let Some(ref sse) = sse { + sse.broadcast_for_user( + // projection-exempt: bridge dispatcher, post-park failed thread error + &user_id, + AppEvent::Error { + message: sanitized.clone(), + thread_id: Some(tid_str.clone()), + }, + ); + } + match bridge_outcome_for_failed_thread( + error, + debug_detail.as_deref(), + &user_id, + &channel_name, + sse_will_deliver_to_user, + ) { + BridgeOutcome::Respond(text) => Some(text), + _ => None, + } + } + ThreadOutcome::GatePaused { + gate_name, + action_name, + call_id, + parameters, + resume_kind, + resume_output, + paused_lease, + } => { + // The post-resume engine hit ANOTHER (legacy) GatePaused + // outcome — typically Authentication or External. Build + // the new pending gate row and surface the prompt; no + // response text to deliver yet. + let redacted_params = + if let Some(tool) = effect_adapter.tools().get(action_name).await { + crate::tools::redact_params(parameters, tool.sensitive_params()) + } else { + parameters.clone() + }; + let pending = PendingGate { + request_id: uuid::Uuid::new_v4(), + gate_name: gate_name.clone(), + user_id: user_id.clone(), + thread_id, + scope_thread_id: message + .conversation_scope() + .and_then(|s| ironclaw_common::ExternalThreadId::new(s).ok()), + conversation_id: conv_id, + source_channel: channel_name.clone(), + action_name: action_name.clone(), + call_id: call_id.clone(), + parameters: parameters.clone(), + display_parameters: Some(redacted_params), + description: format!( + "Tool '{}' requires {} (gate: {gate_name})", + action_name, + resume_kind.kind_name() + ), + resume_kind: resume_kind.clone(), + created_at: chrono::Utc::now(), + expires_at: chrono::Utc::now() + chrono::Duration::minutes(30), + original_message: Some(message.content.clone()), + resume_output: resume_output.clone(), + paused_lease: paused_lease.as_deref().cloned(), + approval_already_granted: false, + }; + // Skip the prompt entirely if we couldn't persist the + // follow-up gate. Without a row backing the + // `request_id`, the user has nothing to resolve against + // — emitting a card here would dead-end as soon as + // they click it. + let insert_succeeded = match pending_gates.insert(pending.clone()).await { + Ok(()) => true, + Err(e) => { + tracing::debug!( + gate = %gate_name, + error = %e, + "post-park continuation: failed to store follow-up pending gate" + ); + false + } + }; + if insert_succeeded { + let extension_name = resolve_auth_gate_extension_name( + auth_manager.as_deref(), + extension_manager.as_deref(), + effect_adapter.tools(), + &pending, + ) + .await; + // Match `send_pending_gate_status` semantics rather + // than collapsing every non-Approval gate into an + // `AuthRequired` card with `pending.description` / + // `auth_url: None`. The previous catch-all dropped + // real `Authentication` instructions and OAuth + // URLs, and surfaced spurious auth prompts for + // `External` callbacks (which the canonical helper + // intentionally ignores). + let status_update = match &pending.resume_kind { + ironclaw_engine::ResumeKind::Approval { allow_always } => { + Some(StatusUpdate::ApprovalNeeded { + request_id: pending.request_id.to_string(), + tool_name: pending.action_name.clone(), + description: pending.description.clone(), + parameters: pending + .display_parameters + .clone() + .unwrap_or_else(|| pending.parameters.clone()), + allow_always: *allow_always, + }) + } + ironclaw_engine::ResumeKind::Authentication { + instructions, + auth_url, + .. + } => Some(StatusUpdate::AuthRequired { + extension_name: extension_name.unwrap_or_else(|| { + ironclaw_common::ExtensionName::from_trusted( + pending.action_name.clone(), + ) + }), + instructions: Some(instructions.clone()), + auth_url: auth_url.clone(), + setup_url: None, + request_id: Some(pending.request_id.to_string()), + }), + ironclaw_engine::ResumeKind::External { .. } => None, + }; + if let Some(status) = status_update { + let _ = channels.send_status(&channel_name, status, &metadata).await; + } + } + None + } + }; + + if let Some(ref text) = response_text { + // SSE Response broadcast (web). + if let Some(ref sse) = sse { + sse.broadcast_for_user( + // projection-exempt: bridge dispatcher, post-park final response + &user_id, + AppEvent::Response { + content: text.clone(), + thread_id: tid_str.clone(), + }, + ); + } + // Channel respond + Done status (Telegram, CLI, gateway). + if let Err(e) = channels + .respond(&message, OutgoingResponse::text(text.clone())) + .await + { + tracing::debug!( + channel = %channel_name, + error = %e, + "post-park continuation: channel respond failed" + ); + } + if let Err(e) = channels + .send_status( + &channel_name, + StatusUpdate::Status("Done".into()), + &metadata, + ) + .await + { + tracing::debug!( + channel = %channel_name, + error = %e, + "post-park continuation: Done status failed" + ); + } + // Persist to v1 DB so the history API renders the final + // assistant message. + if let Some(ref db) = db { + let scope_uuid = message + .conversation_scope() + .and_then(|s| uuid::Uuid::parse_str(s).ok()); + let v1_conv_id = if let Some(uuid) = scope_uuid { + Some(uuid) + } else { + db.get_or_create_assistant_conversation(&user_id, &channel_name) + .await + .ok() + }; + if let Some(cid) = v1_conv_id { + let _ = db.add_conversation_message(cid, "assistant", text).await; + } + } + } + + gate_controller + .clear_execution_context(&user_id, thread_id, conv_id) + .await; + debug!( + thread_id = %thread_id, + "engine v2: post-park continuation ran" + ); + }); +} + +/// Fire active OnEvent missions whose pattern matches the inbound message. +/// +/// Builds a payload containing the message metadata that mission threads +/// can read via `state["trigger_payload"]`. Skips empty content and +/// system-channel messages. Errors are logged at debug level — a failure +/// here must never block the user-facing message flow. +async fn fire_event_missions_for_message( + state: &EngineState, + message: &IncomingMessage, + content: &str, +) { + // Skip empty messages — there's nothing to pattern-match against + // and we don't want missions firing on every status update or empty + // user input. + let trimmed = content.trim(); + if trimmed.is_empty() { + return; + } + + // Recursion guards. Channel adapters that echo the agent's own + // outbound text back as inbound events MUST set is_agent_broadcast + // (Slack/Discord-style); messages produced as a side effect of a + // mission firing MUST set triggering_mission_id (chain-recursion + // across distinct missions). Either flag means: do not re-fire. + if message.is_agent_broadcast { + debug!( + channel = %message.channel, + "engine v2: skipping mission firing — message is an agent broadcast echo" + ); + return; + } + if let Some(ref upstream) = message.triggering_mission_id { + debug!( + channel = %message.channel, + upstream_mission_id = %upstream, + "engine v2: skipping mission firing — message originated from a mission" + ); + return; + } + + let Some(mission_manager) = state.effect_adapter.mission_manager().await else { + return; + }; + + let payload = serde_json::json!({ + "channel": message.channel, + "user_id": message.user_id, + "content": content, + "metadata": message.metadata, + }); + + match mission_manager + .fire_on_message_event(&message.channel, content, &message.user_id, Some(payload)) + .await + { + Ok(spawned) if !spawned.is_empty() => { + debug!( + count = spawned.len(), + channel = %message.channel, + user_id = %message.user_id, + "engine v2: fired {} OnEvent mission(s) from inbound message", + spawned.len() + ); + } + Ok(_) => {} + Err(error) => { + debug!( + channel = %message.channel, + error = %error, + "engine v2: fire_on_message_event failed; continuing with normal handling" + ); + } + } +} + +async fn await_thread_outcome( + agent: &Agent, + state: &EngineState, + message: &IncomingMessage, + conv_id: ironclaw_engine::ConversationId, + thread_id: ironclaw_engine::ThreadId, +) -> Result<BridgeOutcome, Error> { + let mut event_rx = state.thread_manager.subscribe_events(); + let channels = &agent.channels; + let channel_name = &message.channel; + let metadata = &message.metadata; let sse = state.sse.as_ref(); let tid_str = thread_id.to_string(); @@ -3713,6 +5326,17 @@ async fn await_thread_outcome( // break out to avoid hanging the user session forever (e.g. after // a denied approval where the thread fails to resume). let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(300); + let mut timed_out = false; + // Set when we detect the thread has parked at an inline gate. The + // gate-park handoff (below) returns `Pending` immediately so the + // per-user agent loop unblocks and can dispatch other threads, while + // a background task takes over event forwarding and final-response + // delivery for this thread. + let mut gate_parked = false; + let pending_key = PendingGateKey { + user_id: message.user_id.clone(), + thread_id, + }; loop { tokio::select! { @@ -3755,23 +5379,79 @@ async fn await_thread_outcome( if !state.thread_manager.is_running(thread_id).await { break; } + // Inline gate detection: if a pending gate has been + // registered for (user, thread) while the thread is + // still running, the engine is parked inside + // `BridgeGateController::pause` awaiting user + // resolution. Holding `handle_message` here would + // serialize the per-user agent loop behind the parked + // pause — a second thread's `UserInput` queued in + // `msg_tx` cannot dispatch until either the user + // resolves this gate or the 5-minute deadline below + // fires. Hand off to a background continuation task + // (preserves event forwarding + final-response delivery) + // and surface as `Pending` so the agent loop unblocks. + if state.pending_gates.peek(&pending_key).await.is_some() { + gate_parked = true; + break; + } if tokio::time::Instant::now() >= deadline { tracing::warn!( thread_id = %thread_id, "await_thread_outcome timed out after 5 minutes — breaking to avoid hang" ); + timed_out = true; break; } } } } + // If we exited because the thread parked at an inline gate, hand + // off the rest of the lifecycle (event forwarding + final response + // broadcast on completion + per-execution context cleanup) to a + // background task and return `Pending`. join_thread cannot run on + // the foreground task because it would block on the parked future + // for up to the gate's 30-min expiry. + if gate_parked && state.thread_manager.is_running(thread_id).await { + spawn_post_park_continuation( + state, + agent.channels.clone(), + message.clone(), + conv_id, + thread_id, + ); + return Ok(BridgeOutcome::Pending); + } + + // If we hit the deadline and the thread is still running (typically + // because it's parked in `BridgeGateController::pause` waiting for + // an approval the user hasn't acted on), do NOT call `join_thread` + // — that would block the request handler for up to the gate's + // `expires_at` (30 min) on the same parked task. Surface as + // `Pending`: the live `PendingGate` row stays available, the user + // can still resolve it, and the resolver path will deliver the + // resolution into the parked oneshot. + if timed_out && state.thread_manager.is_running(thread_id).await { + return Ok(BridgeOutcome::Pending); + } + let outcome = state .thread_manager .join_thread(thread_id) .await .map_err(|e| engine_err("join error", e))?; + // Drop the external-tool catalog entry on terminal outcomes — + // the thread can never resume from `Completed`, `Stopped`, + // `MaxIterations`, or `Failed`, so the entry would otherwise + // leak forever. `GatePaused` deliberately keeps the entry: a + // follow-up resume request needs the catalog to still know + // about this thread's caller-supplied tools. + if !matches!(outcome, ThreadOutcome::GatePaused { .. }) { + state.external_tool_catalog.clear(thread_id).await; + } + state .conversation_manager .record_thread_outcome(conv_id, thread_id, &outcome) @@ -4055,6 +5735,46 @@ async fn await_thread_outcome( ); } + // Caller-supplied external tool from the Responses API: + // surface as `AppEvent::ExternalToolCall` so the + // /v1/responses handler can emit a `function_call` + // ResponseOutputItem and complete the turn. Without this + // emit the handler times out waiting for a never-arriving + // event and the user sees `response.failed`. The mid-exec + // path (`notify_pending_gate`) emits the same variant, but + // CodeAct converts the gate into a Python RuntimeError — + // the thread ends with a `ThreadOutcome::GatePaused` and + // never traverses `notify_pending_gate`, so we have to + // emit it here too. + if let ironclaw_engine::ResumeKind::External { ref callback_id } = pending.resume_kind + && crate::bridge::is_external_tool_callback_id(callback_id) + { + if let Some(ref sse) = state.sse { + let arguments = serde_json::to_string(&pending.parameters) + .unwrap_or_else(|_| pending.parameters.to_string()); + let event = AppEvent::ExternalToolCall { + request_id: pending.request_id.to_string(), + call_id: pending.call_id.clone(), + name: pending.action_name.clone(), + arguments, + thread_id: Some(pending.effective_wire_thread_id()), + }; + sse.broadcast_for_user(&message.user_id, event); // projection-exempt: bridge dispatcher, ThreadOutcome::GatePaused External-tool projection from CodeAct re-entry path + } else { + tracing::debug!( + user_id = %message.user_id, + callback = %callback_id, + request_id = %pending.request_id, + "external tool gate paused (post-CodeAct) but no broadcaster is wired; \ + caller will not be notified" + ); + } + // Skip the approval-card delivery path below — that + // surface is for human-in-the-loop UX which doesn't + // apply to caller-executed tool calls. + return Ok(BridgeOutcome::Pending); + } + // Send the approval/auth card via the source channel. Each // channel renders this natively (web → SSE card, TUI → widget, // relay → buttons). No text response is returned — the caller @@ -5438,6 +7158,35 @@ pub async fn get_engine_project( })) } +/// Whether `thread` should be surfaced as a user-actionable failure. +/// +/// A thread counts as a "real" failure for the projects "needs attention" +/// feed when: +/// - its state is `Failed`, AND +/// - it failed within the last 24 hours, AND +/// - it was NOT force-failed by `recover_project_threads` on engine +/// restart (those carry the +/// [`ironclaw_engine::ENGINE_RESTART_RECOVERY_METADATA_KEY`] flag and +/// are crash-recovery artifacts, not user errors). +/// +/// Filtering on the metadata flag fixes #3274: an upgrade transitioned +/// every still-running thread to `Failed`, which then flooded the +/// Projects tab with phantom "Thread failed" warnings. +fn is_real_thread_failure( + thread: &ironclaw_engine::types::thread::Thread, + h24_ago: chrono::DateTime<chrono::Utc>, +) -> bool { + matches!( + thread.state, + ironclaw_engine::types::thread::ThreadState::Failed + ) && thread.updated_at >= h24_ago + && !thread + .metadata + .get(ironclaw_engine::ENGINE_RESTART_RECOVERY_METADATA_KEY) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + /// Projects overview — health, stats, attention items for all projects. /// /// Iterates all projects, computes per-project stats from missions and threads, @@ -5533,12 +7282,14 @@ pub async fn get_engine_projects_overview( .map(|t| t.total_cost_usd) .sum(); + // Filter restart-recovery noise: `recover_project_threads` + // force-fails non-terminal threads on engine restart and tags + // them with `engine_restart_recovery`. They aren't actionable + // failures, so we exclude them from both the count and the + // attention feed (#3274). let failures_24h = threads .iter() - .filter(|t| { - matches!(t.state, ironclaw_engine::types::thread::ThreadState::Failed) - && t.updated_at >= h24_ago - }) + .filter(|t| is_real_thread_failure(t, h24_ago)) .count() as u64; let last_activity = threads @@ -5567,11 +7318,7 @@ pub async fn get_engine_projects_overview( }); } for thread in &threads { - if matches!( - thread.state, - ironclaw_engine::types::thread::ThreadState::Failed - ) && thread.updated_at >= h24_ago - { + if is_real_thread_failure(thread, h24_ago) { attention.push(AttentionItem { kind: "failure".to_string(), project_id: pid.to_string(), @@ -6129,6 +7876,12 @@ pub(crate) mod test_support { )); let cm = Arc::new(ConversationManager::new(Arc::clone(&tm), store_dyn.clone())); + // Share a single `Arc<GateResolutions>` between the gate + // controller and the EngineState field so + // `resolve_inline_gates_for_credential` reads the same index + // that `BridgeGateController::pause` writes to. + let test_gate_resolutions = + Arc::new(crate::bridge::gate_controller::GateResolutions::new()); let state = EngineState { thread_manager: tm, conversation_manager: cm, @@ -6141,7 +7894,19 @@ pub(crate) mod test_support { secrets_store: None, auth_manager: None, extension_manager: None, + gate_controller: Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&test_gate_resolutions), + )), + gate_resolutions: test_gate_resolutions, project_root: super::resolve_project_root(), + external_tool_catalog: Arc::new(crate::bridge::ExternalToolCatalog::new()), + capability_registry: Arc::new(ironclaw_engine::CapabilityRegistry::new()), }; let lock = ENGINE_STATE.get_or_init(|| TokioRwLock::new(None)); @@ -6738,7 +8503,7 @@ mod tests { struct StaticLlmProvider; #[async_trait::async_trait] - impl crate::llm::LlmProvider for StaticLlmProvider { + impl ironclaw_llm::LlmProvider for StaticLlmProvider { fn model_name(&self) -> &str { "static-mock" } @@ -6749,13 +8514,13 @@ mod tests { async fn complete( &self, - _request: crate::llm::CompletionRequest, - ) -> Result<crate::llm::CompletionResponse, crate::error::LlmError> { - Ok(crate::llm::CompletionResponse { + _request: ironclaw_llm::CompletionRequest, + ) -> Result<ironclaw_llm::CompletionResponse, crate::error::LlmError> { + Ok(ironclaw_llm::CompletionResponse { content: "ok".to_string(), input_tokens: 0, output_tokens: 0, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }) @@ -6763,16 +8528,17 @@ mod tests { async fn complete_with_tools( &self, - _request: crate::llm::ToolCompletionRequest, - ) -> Result<crate::llm::ToolCompletionResponse, crate::error::LlmError> { - Ok(crate::llm::ToolCompletionResponse { + _request: ironclaw_llm::ToolCompletionRequest, + ) -> Result<ironclaw_llm::ToolCompletionResponse, crate::error::LlmError> { + Ok(ironclaw_llm::ToolCompletionResponse { content: Some("ok".to_string()), tool_calls: Vec::new(), input_tokens: 0, output_tokens: 0, - finish_reason: crate::llm::FinishReason::Stop, + finish_reason: ironclaw_llm::FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -7201,24 +8967,113 @@ mod tests { )); } + /// Boot-time sweep must evict every `Approval` gate row carried + /// over from a prior process — they have no live `oneshot::Sender` + /// to deliver to, and falling through to `execute_pending_gate_action` + /// would re-run the LLM step and replay earlier non-idempotent tool + /// calls (the bug the inline-await path exists to prevent). + /// Authentication and External rows survive because their resume + /// path doesn't depend on a live VM. #[tokio::test] - async fn handle_approval_ignores_pending_gate_from_different_thread() { - let _guard = ENGINE_STATE_TEST_LOCK.lock().await; - let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); - *lock.write().await = None; + async fn invalidate_stranded_approval_gates_evicts_only_approval_kind() { + let store = crate::gate::store::PendingGateStore::in_memory(); - let outcome = async { - let store = Arc::new(TestStore::new()); - let state = make_expected_test_state(store); - let pending_thread_id = ironclaw_engine::ThreadId::new(); - let active_thread_id = ironclaw_engine::ThreadId::new(); - let pending = sample_pending_gate( + let approval_a = uuid::Uuid::new_v4(); + let approval_b = uuid::Uuid::new_v4(); + let auth = uuid::Uuid::new_v4(); + let external = uuid::Uuid::new_v4(); + + store + .insert(sample_pending_gate_with_request_id( "alice", - pending_thread_id, + ironclaw_engine::ThreadId::new(), + approval_a, ironclaw_engine::ResumeKind::Approval { allow_always: true }, - ); - state - .pending_gates + )) + .await + .unwrap(); + store + .insert(sample_pending_gate_with_request_id( + "bob", + ironclaw_engine::ThreadId::new(), + approval_b, + ironclaw_engine::ResumeKind::Approval { + allow_always: false, + }, + )) + .await + .unwrap(); + store + .insert(sample_pending_gate_with_request_id( + "alice", + ironclaw_engine::ThreadId::new(), + auth, + ironclaw_engine::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("github").unwrap(), + instructions: "paste token".into(), + auth_url: None, + }, + )) + .await + .unwrap(); + store + .insert(sample_pending_gate_with_request_id( + "alice", + ironclaw_engine::ThreadId::new(), + external, + ironclaw_engine::ResumeKind::External { + callback_id: "cb-1".into(), + }, + )) + .await + .unwrap(); + + // No SSE wired — exercises the `if let Some(sse)` skip branch. + invalidate_stranded_approval_gates(&store, None).await; + + let surviving: std::collections::HashSet<uuid::Uuid> = store + .list_all() + .await + .into_iter() + .map(|g| g.request_id) + .collect(); + assert!( + !surviving.contains(&approval_a), + "approval gate for alice must be evicted" + ); + assert!( + !surviving.contains(&approval_b), + "approval gate for bob must be evicted" + ); + assert!( + surviving.contains(&auth), + "auth gate must survive: {surviving:?}" + ); + assert!( + surviving.contains(&external), + "external gate must survive: {surviving:?}" + ); + assert_eq!(surviving.len(), 2, "exactly two non-Approval gates remain"); + } + + #[tokio::test] + async fn handle_approval_ignores_pending_gate_from_different_thread() { + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let store = Arc::new(TestStore::new()); + let state = make_expected_test_state(store); + let pending_thread_id = ironclaw_engine::ThreadId::new(); + let active_thread_id = ironclaw_engine::ThreadId::new(); + let pending = sample_pending_gate( + "alice", + pending_thread_id, + ironclaw_engine::ResumeKind::Approval { allow_always: true }, + ); + state + .pending_gates .insert(pending) .await .expect("insert pending gate"); @@ -7989,6 +9844,212 @@ mod tests { *lock.write().await = None; } + /// Regression for review on #3381: the OAuth failure cleanup must + /// scope to the credential of the failed flow, not nuke every + /// pending auth gate the user has open. Concrete failure mode this + /// covers — Gmail OAuth fails while a Slack auth gate is in flight + /// on a different thread; only the Gmail gate should clear. + #[tokio::test] + async fn clear_engine_pending_auth_for_credential_only_clears_matching_credential() { + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let store = Arc::new(TestStore::new()); + let state = make_expected_test_state(store); + let thread_gmail = ironclaw_engine::ThreadId::new(); + let thread_slack = ironclaw_engine::ThreadId::new(); + + state + .pending_gates + .insert(sample_pending_gate( + "alice", + thread_gmail, + ironclaw_engine::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("google_oauth_token") + .unwrap(), + instructions: "complete OAuth".into(), + auth_url: None, + }, + )) + .await + .unwrap(); + state + .pending_gates + .insert(sample_pending_gate( + "alice", + thread_slack, + ironclaw_engine::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("slack_oauth_token") + .unwrap(), + instructions: "complete OAuth".into(), + auth_url: None, + }, + )) + .await + .unwrap(); + + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + *lock.write().await = Some(state); + + clear_engine_pending_auth_for_credential("alice", "google_oauth_token").await; + + let guard = lock.read().await; + let state = guard.as_ref().unwrap(); + let remaining = state.pending_gates.list_for_user("alice").await; + assert_eq!( + remaining.len(), + 1, + "only the Gmail gate should be discarded; Slack must remain" + ); + assert!( + remaining.iter().any(|gate| gate.thread_id == thread_slack), + "Slack gate must survive Gmail OAuth failure" + ); + drop(guard); + *lock.write().await = None; + } + + /// Regression for review on PR #3381: an OAuth callback that arrives + /// after the 5-minute flow expiry is a terminal failure — the engine + /// pending auth gate must be cleared, otherwise the conversation + /// sits paused forever waiting on a callback that will never arrive + /// (#3320). Cleanup must stay scoped to the failed flow's credential + /// so an unrelated auth gate (Slack/MCP) running on a different + /// thread for the same user survives. + #[tokio::test] + async fn oauth_callback_expired_flow_clears_credential_scoped_engine_gate() { + use crate::channels::web::features::oauth::oauth_callback_handler; + use crate::channels::web::test_helpers::{test_ext_mgr, test_gateway_state}; + use crate::testing::credentials::TEST_GATEWAY_CRYPTO_KEY; + use axum::body::Body; + use axum::routing::get; + use tower::ServiceExt; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + + // Two pending auth gates for the same user, on different threads + // and different credentials. The OAuth callback will be for the + // Gmail flow; the Slack gate must survive. + let store = Arc::new(TestStore::new()); + let engine_state = make_expected_test_state(store); + let thread_gmail = ironclaw_engine::ThreadId::new(); + let thread_slack = ironclaw_engine::ThreadId::new(); + engine_state + .pending_gates + .insert(sample_pending_gate( + "expiry-user", + thread_gmail, + ironclaw_engine::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("google_oauth_token") + .unwrap(), + instructions: "complete OAuth".into(), + auth_url: None, + }, + )) + .await + .unwrap(); + engine_state + .pending_gates + .insert(sample_pending_gate( + "expiry-user", + thread_slack, + ironclaw_engine::ResumeKind::Authentication { + credential_name: ironclaw_common::CredentialName::new("slack_oauth_token") + .unwrap(), + instructions: "complete OAuth".into(), + auth_url: None, + }, + )) + .await + .unwrap(); + + let lock = ENGINE_STATE.get_or_init(|| TokioRwLock::new(None)); + *lock.write().await = None; + *lock.write().await = Some(engine_state); + + // GatewayState with an ext_mgr holding an expired pending flow + // for the Gmail credential. The `state` query parameter we send + // below must round-trip through `decode_hosted_oauth_state`, so + // we mint it via the matching encoder. + let secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = + Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( + crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( + TEST_GATEWAY_CRYPTO_KEY.to_string(), + )) + .expect("crypto"), + ))); + let (ext_mgr, _wasm_tools_dir, _wasm_channels_dir) = test_ext_mgr(secrets.clone()); + + let flow_id = "expiry-flow".to_string(); + let encoded_state = crate::auth::oauth::encode_hosted_oauth_state(&flow_id, None); + let expired_created_at = std::time::Instant::now() + .checked_sub(crate::auth::oauth::OAUTH_FLOW_EXPIRY + Duration::from_secs(1)) + .expect("monotonic clock"); + let flow = crate::auth::oauth::PendingOAuthFlow { + extension_name: ironclaw_common::ExtensionName::new("gmail").unwrap(), + display_name: "Gmail".to_string(), + token_url: "https://example.com/token".to_string(), + client_id: "client123".to_string(), + client_secret: None, + redirect_uri: "https://example.com/oauth/callback".to_string(), + code_verifier: None, + access_token_field: "access_token".to_string(), + secret_name: "google_oauth_token".to_string(), + provider: None, + validation_endpoint: None, + scopes: vec![], + user_id: "expiry-user".to_string(), + secrets, + sse_manager: None, + gateway_token: None, + token_exchange_extra_params: std::collections::HashMap::new(), + client_id_secret_name: None, + client_secret_secret_name: None, + client_secret_expires_at: None, + created_at: expired_created_at, + auto_activate_extension: true, + }; + ext_mgr + .pending_oauth_flows() + .write() + .await + .insert(flow_id, flow); + + let gateway_state = test_gateway_state(Some(ext_mgr)); + let app = axum::Router::new() + .route("/oauth/callback", get(oauth_callback_handler)) + .with_state(gateway_state); + + let req = axum::http::Request::builder() + .uri(format!( + "/oauth/callback?code=test_code&state={}", + encoded_state + )) + .body(Body::empty()) + .expect("request"); + let resp = ServiceExt::<axum::http::Request<Body>>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + + // The Gmail gate (matching credential) must be gone; the Slack + // gate (different credential) must survive. + let guard = lock.read().await; + let state = guard.as_ref().unwrap(); + let remaining = state.pending_gates.list_for_user("expiry-user").await; + assert_eq!( + remaining.len(), + 1, + "expired OAuth callback must clear the matching auth gate; \ + unrelated gate must survive. Remaining: {remaining:?}" + ); + assert!( + remaining.iter().any(|gate| gate.thread_id == thread_slack), + "Slack gate must survive Gmail flow expiry" + ); + drop(guard); + *lock.write().await = None; + } + #[tokio::test] async fn discard_engine_pending_auth_request_discards_only_matching_auth_gate() { let _guard = ENGINE_STATE_TEST_LOCK.lock().await; @@ -8225,6 +10286,11 @@ mod tests { let cm = Arc::new(ConversationManager::new(Arc::clone(&tm), store_dyn.clone())); + // Share a single `Arc<GateResolutions>` between the gate + // controller and the EngineState field so + // `resolve_inline_gates_for_credential` reads the same index + // that `BridgeGateController::pause` writes to. + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); EngineState { thread_manager: tm, conversation_manager: cm, @@ -8237,7 +10303,19 @@ mod tests { secrets_store: None, auth_manager: None, extension_manager: None, + gate_controller: Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&resolutions), + )), + gate_resolutions: resolutions, project_root: resolve_project_root(), + external_tool_catalog: Arc::new(crate::bridge::ExternalToolCatalog::new()), + capability_registry: Arc::new(ironclaw_engine::CapabilityRegistry::new()), } } @@ -8375,6 +10453,11 @@ mod tests { let cm = Arc::new(ConversationManager::new(Arc::clone(&tm), store_dyn.clone())); + // Share a single `Arc<GateResolutions>` between the gate + // controller and the EngineState field so + // `resolve_inline_gates_for_credential` reads the same index + // that `BridgeGateController::pause` writes to. + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); EngineState { thread_manager: tm, conversation_manager: cm, @@ -8387,7 +10470,19 @@ mod tests { secrets_store: None, auth_manager: None, extension_manager: None, + gate_controller: Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&resolutions), + )), + gate_resolutions: resolutions, project_root: resolve_project_root(), + external_tool_catalog: Arc::new(crate::bridge::ExternalToolCatalog::new()), + capability_registry: Arc::new(ironclaw_engine::CapabilityRegistry::new()), } } @@ -9856,6 +11951,88 @@ mod tests { ); } + // ── is_real_thread_failure (#3274) ────────────────────────────────── + + /// Helper: build a Failed thread with a configurable updated_at. + fn make_failed_thread(updated_at: chrono::DateTime<chrono::Utc>) -> ironclaw_engine::Thread { + let mut t = ironclaw_engine::Thread::new( + "test goal", + ironclaw_engine::ThreadType::Foreground, + ironclaw_engine::ProjectId::new(), + "alice", + ironclaw_engine::ThreadConfig::default(), + ); + t.transition_to(ironclaw_engine::ThreadState::Running, None) + .unwrap(); + t.transition_to( + ironclaw_engine::ThreadState::Failed, + Some("LLM error".into()), + ) + .unwrap(); + t.updated_at = updated_at; + t + } + + #[test] + fn real_failure_recent_within_window() { + let now = chrono::Utc::now(); + let h24_ago = now - chrono::Duration::hours(24); + let t = make_failed_thread(now); + assert!( + super::is_real_thread_failure(&t, h24_ago), + "recent failed thread should surface as a real failure" + ); + } + + #[test] + fn real_failure_excluded_when_older_than_24h() { + let now = chrono::Utc::now(); + let h24_ago = now - chrono::Duration::hours(24); + let t = make_failed_thread(now - chrono::Duration::hours(25)); + assert!( + !super::is_real_thread_failure(&t, h24_ago), + "stale failure outside the 24h window must not be surfaced" + ); + } + + #[test] + fn real_failure_excludes_engine_restart_recovery() { + let now = chrono::Utc::now(); + let h24_ago = now - chrono::Duration::hours(24); + let mut t = make_failed_thread(now); + // Simulate `recover_project_threads` having tagged the thread. + if let Some(obj) = t.metadata.as_object_mut() { + obj.insert( + ironclaw_engine::ENGINE_RESTART_RECOVERY_METADATA_KEY.to_string(), + serde_json::Value::Bool(true), + ); + } + assert!( + !super::is_real_thread_failure(&t, h24_ago), + "restart-recovery threads must not surface as user-actionable failures" + ); + } + + #[test] + fn real_failure_ignores_non_failed_states() { + let now = chrono::Utc::now(); + let h24_ago = now - chrono::Duration::hours(24); + let mut t = ironclaw_engine::Thread::new( + "still running", + ironclaw_engine::ThreadType::Foreground, + ironclaw_engine::ProjectId::new(), + "alice", + ironclaw_engine::ThreadConfig::default(), + ); + t.transition_to(ironclaw_engine::ThreadState::Running, None) + .unwrap(); + t.updated_at = now; + assert!( + !super::is_real_thread_failure(&t, h24_ago), + "Running thread must not be classified as a failure" + ); + } + // ── persist_always_allow / revert_always_allow ───────────────────── /// Minimal in-memory SettingsStore for persistence tests. @@ -10038,6 +12215,11 @@ mod tests { Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), )); + // Share a single `Arc<GateResolutions>` between the gate + // controller and the EngineState field so + // `resolve_inline_gates_for_credential` reads the same index + // that `BridgeGateController::pause` writes to. + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); EngineState { conversation_manager: Arc::new(ConversationManager::new( Arc::clone(&thread_manager), @@ -10053,7 +12235,19 @@ mod tests { secrets_store: None, auth_manager: None, extension_manager: None, + gate_controller: Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::new(crate::gate::store::PendingGateStore::in_memory()), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&resolutions), + )), + gate_resolutions: resolutions, project_root: resolve_project_root(), + external_tool_catalog: Arc::new(crate::bridge::ExternalToolCatalog::new()), + capability_registry: Arc::new(ironclaw_engine::CapabilityRegistry::new()), } } @@ -10536,12 +12730,15 @@ mod tests { ); } - /// Regression for the bug fixed in commit 652315e8: `persist_v2_tool_calls` - /// must only be called from the `ThreadOutcome::Completed` arm. If a - /// future refactor moves the call out of that arm, partial tool - /// executions on `GatePaused` would orphan a `role="tool_calls"` DB row - /// that then duplicates when the gate resumes. Pin the call-site - /// conditional by inspecting the source of `await_thread_outcome`. + /// Regression for the bug fixed in commit 652315e8: + /// `persist_v2_tool_calls` must only be called from a + /// `ThreadOutcome::Completed` arm. If a future refactor moves the + /// call out of that arm, partial tool executions on `GatePaused` + /// would orphan a `role="tool_calls"` DB row that then duplicates + /// when the gate resumes. Pin the call-site invariant by inspecting + /// the source of `await_thread_outcome` and any sibling arm-driven + /// dispatchers (currently `spawn_post_park_continuation`, which + /// re-runs the same outcome match in a background task). #[test] fn persist_v2_tool_calls_only_called_from_completed_arm() { let source = include_str!("router.rs"); @@ -10549,33 +12746,49 @@ mod tests { .split_once("async fn persist_v2_tool_calls") .expect("persist_v2_tool_calls must exist in router.rs"); - // There should be exactly one call site in the pre-definition body - // (the call inside `await_thread_outcome`). The text below the - // definition is allowed to reference it (doc comments, unit tests). - let call_sites = before_fn.matches("persist_v2_tool_calls(").count(); - assert_eq!( - call_sites, 1, - "expected exactly one call site for persist_v2_tool_calls, found {call_sites}" - ); - - // The call must live inside `ThreadOutcome::Completed` and must not - // appear in any of the terminal arms that represent non-completion - // outcomes. `GatePaused` is the one that triggered the bug. - let completed_idx = before_fn - .find("ThreadOutcome::Completed") - .expect("Completed arm must exist"); - let gate_paused_idx = before_fn - .find("ThreadOutcome::GatePaused") - .expect("GatePaused arm must exist"); - let call_idx = before_fn - .find("persist_v2_tool_calls(") - .expect("call site must exist"); - + // The text below the definition is allowed to reference it + // (doc comments, unit tests). Above the definition there must + // be at least one call site, and every call site must sit + // between a `ThreadOutcome::Completed` opening match arm and + // the nearest non-Completed sibling arm. + let call_sites: Vec<usize> = before_fn + .match_indices("persist_v2_tool_calls(") + .map(|(idx, _)| idx) + .collect(); assert!( - completed_idx < call_idx && call_idx < gate_paused_idx, - "persist_v2_tool_calls call must sit between Completed and GatePaused arms, got \ - completed={completed_idx} call={call_idx} gate_paused={gate_paused_idx}" - ); + !call_sites.is_empty(), + "expected at least one call site for persist_v2_tool_calls" + ); + + let other_outcome_arms = [ + "ThreadOutcome::GatePaused", + "ThreadOutcome::Failed", + "ThreadOutcome::Stopped", + "ThreadOutcome::MaxIterations", + ]; + for call_idx in &call_sites { + // Find the most recent match-arm marker preceding this call. + // Must be `ThreadOutcome::Completed` — anything else means + // the call sits in a non-completion arm and risks the bug. + let prefix = &before_fn[..*call_idx]; + let last_completed = prefix.rfind("ThreadOutcome::Completed"); + let last_other = other_outcome_arms + .iter() + .filter_map(|arm| prefix.rfind(arm)) + .max(); + assert!( + last_completed.is_some(), + "persist_v2_tool_calls call at byte {call_idx} must be inside a \ + ThreadOutcome::Completed arm — no preceding Completed marker" + ); + assert!( + last_other.unwrap_or(0) < last_completed.unwrap_or(0), + "persist_v2_tool_calls call at byte {call_idx} must be inside the \ + closest enclosing ThreadOutcome::Completed arm; a non-Completed arm \ + marker (GatePaused/Failed/Stopped/MaxIterations) appears between \ + the Completed marker and the call" + ); + } } // ── resume_lease_for_pending_gate tests ──────────────────── @@ -10831,4 +13044,1047 @@ mod tests { let info = thread_to_info(&thread); assert_eq!(info.title.as_deref(), Some("Short first line")); } + + /// `extract_external_tool_output` returns the tool result for the + /// matching call_id when the payload follows the canonical + /// `{"outputs": [...]}` shape the responses_api handler builds. + #[test] + fn extract_external_tool_output_matches_call_id() { + let payload = serde_json::json!({ + "outputs": [ + {"call_id": "call_a", "output": "first result"}, + {"call_id": "call_b", "output": {"weather": "sunny"}}, + ] + }); + assert_eq!( + extract_external_tool_output(&payload, "call_a"), + serde_json::Value::String("first result".into()) + ); + assert_eq!( + extract_external_tool_output(&payload, "call_b"), + serde_json::json!({"weather": "sunny"}) + ); + } + + /// When the payload carries an `outputs` array but no entry + /// matches the requested call_id, the helper must surface a typed + /// `null` so the LLM sees an explicit empty result rather than + /// the (possibly stale) raw payload of some other call. Without + /// this, the bridge would echo back unrelated tool output to the + /// model, confusing the next turn. + #[test] + fn extract_external_tool_output_returns_null_when_call_id_missing() { + let payload = serde_json::json!({ + "outputs": [ + {"call_id": "call_other", "output": "wrong call"}, + ] + }); + let result = extract_external_tool_output(&payload, "call_missing"); + assert_eq!( + result, + serde_json::Value::Null, + "missing call_id must produce a typed null, got: {result:?}" + ); + } + + /// When the payload has no `outputs` array at all (defensive path + /// for legacy OAuth-style raw resolutions), the helper falls back + /// to returning the whole payload — preserving the historical + /// `Submission::ExternalCallback { payload: <raw value> }` shape. + #[test] + fn extract_external_tool_output_falls_back_to_raw_payload() { + let payload = serde_json::json!({"token": "abc123"}); + let result = extract_external_tool_output(&payload, "any_call_id"); + assert_eq!(result, payload); + } + + /// An `outputs` entry without a matching call_id but a different + /// matching one further down the array must still be found — + /// guards against an early-return regression in the lookup loop. + #[test] + fn extract_external_tool_output_finds_match_after_misses() { + let payload = serde_json::json!({ + "outputs": [ + {"call_id": "call_a", "output": "a"}, + {"call_id": "call_b", "output": "b"}, + {"call_id": "call_target", "output": "match"}, + ] + }); + assert_eq!( + extract_external_tool_output(&payload, "call_target"), + serde_json::Value::String("match".into()) + ); + } + + /// Regression test for #3317: when a user types a pairing claim into a + /// chat surface but the gateway has no `ExtensionManager` wired up, the + /// handler must produce a clear, user-facing message instead of + /// panicking or returning an internal error. The corresponding + /// happy-path test (with a real `ExtensionManager`) lives in the + /// telegram pairing chat-claim integration test (gated on libsql). + #[tokio::test] + async fn handle_pairing_claim_without_ext_mgr_responds_with_unavailable_message() { + let (agent, _statuses) = make_router_test_agent(None).await; + let message = IncomingMessage::new("tui", "alice", "approve telegram ABC12345"); + + let outcome = handle_pairing_claim(&agent, &message, "telegram", "ABC12345") + .await + .expect("handle_pairing_claim should not error"); + + match outcome { + BridgeOutcome::Respond(text) => { + assert!( + text.contains("Pairing is not available"), + "expected unavailable-message, got: {text}" + ); + } + other => panic!("expected Respond, got {other:?}"), + } + } + + /// Regression test for #3317: malformed channel slugs (path traversal, + /// empty, oversize) must reject at the boundary rather than reaching + /// the pairing store. Mirrors `web::features::pairing::parse_channel`'s + /// `ExtensionName::new` validation. + #[tokio::test] + async fn handle_pairing_claim_rejects_invalid_channel_name() { + let (agent, _statuses) = make_router_test_agent(None).await; + let message = IncomingMessage::new("tui", "alice", "approve ../etc/passwd ABC"); + + let outcome = handle_pairing_claim(&agent, &message, "../etc/passwd", "ABC") + .await + .expect("handle_pairing_claim should not error"); + + match outcome { + BridgeOutcome::Respond(text) => { + assert!( + text.contains("Invalid channel name"), + "expected invalid-channel message, got: {text}" + ); + } + other => panic!("expected Respond, got {other:?}"), + } + } + + /// The invalid-channel reply echoes a slice of what the user typed so + /// they know which token was rejected, but the echo must be bounded + /// and printable: an attacker-controlled chat input shouldn't be able + /// to inject backticks, control characters, or kilobytes of payload + /// into the SSE / Telegram / TUI reply through this path. + #[tokio::test] + async fn handle_pairing_claim_invalid_channel_echo_is_bounded_and_sanitized() { + let (agent, _statuses) = make_router_test_agent(None).await; + let message = IncomingMessage::new("tui", "alice", "approve x ABC"); + + // 200-character payload mixing control chars, backticks, and + // markdown — well beyond any real channel slug. + let hostile = format!( + "{}`malicious`\x07\x1b[31m{}", + "A".repeat(80), + "B".repeat(120) + ); + let outcome = handle_pairing_claim(&agent, &message, &hostile, "ABC") + .await + .expect("handle_pairing_claim should not error"); + + let text = match outcome { + BridgeOutcome::Respond(text) => text, + other => panic!("expected Respond, got {other:?}"), + }; + + assert!( + text.contains("Invalid channel name"), + "expected invalid-channel message, got: {text}" + ); + // Pull the echoed preview out from between the first two backticks + // — the message also embeds backtick-quoted examples + // (`telegram`, `slack-relay`), so a global backtick count is + // fragile and not what the user-controlled-input invariant cares + // about. What matters is that whatever the user typed doesn't + // smuggle anything into the *preview* region. + let mut parts = text.splitn(3, '`'); + parts.next(); + let preview = parts.next().expect("preview region delimited by backticks"); + assert!( + !preview.contains('`'), + "preview must not contain user-injected backticks: {preview:?}" + ); + assert!( + preview + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) + || preview == "<empty>", + "preview must be alphanumeric (with - / _) or the explicit \ + <empty> placeholder, got {preview:?}" + ); + assert!( + !text.contains('\x07') && !text.contains('\x1b'), + "echo must strip control characters: {text:?}" + ); + assert!( + !text.contains("malicious"), + "non-alphanumeric markup must be stripped from the echo: {text}" + ); + // Cap the rendered length to "Invalid channel name `<≤32 chars>`: <err>" + // — generous upper bound here just confirms the echo isn't unbounded. + assert!( + text.len() < 256, + "rendered reply must be bounded, got {} chars: {text}", + text.len() + ); + } + + /// Regression: when an Approval gate is parked in + /// `BridgeGateController::pause`, the agent loop is blocked at + /// `handle_message`, so the legacy mpsc-driven `ExecApproval` + /// submission would never reach `try_deliver`. The new + /// `try_resolve_inline_approval_gate` entry point bypasses the + /// agent loop and delivers the resolution directly to the parked + /// engine VM. Without this fix the parked future would only wake + /// on the 30-minute pause expiry — failure mode #3157 follow-up. + #[tokio::test] + async fn try_resolve_inline_approval_gate_wakes_parked_pause() { + use crate::bridge::PerExecutionContext; + use ironclaw_engine::{ + ConversationId, GateController, GatePauseRequest, ResumeKind, ThreadId, + }; + use std::time::Duration; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + // Build an EngineState whose pending_gates and gate_controller + // share the same Arc<PendingGateStore> + Arc<GateResolutions>, + // matching the production wiring in `init_engine`. + let pending_gates = Arc::new(crate::gate::store::PendingGateStore::in_memory()); + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); + let controller = Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::clone(&pending_gates), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&resolutions), + )); + + let store = Arc::new(TestStore::new()); + let mut state = make_expected_test_state(store); + state.pending_gates = Arc::clone(&pending_gates); + state.gate_controller = Arc::clone(&controller); + *lock.write().await = Some(state); + + let thread_id = ThreadId::new(); + let user_id = "alice".to_string(); + let conversation_id = ConversationId::new(); + + controller + .set_execution_context( + user_id.clone(), + thread_id, + PerExecutionContext { + conversation_id, + source_channel: "gateway".into(), + scope_thread_id: None, + channel_metadata: serde_json::json!({}), + original_message: None, + }, + ) + .await; + + // Spawn alpha: parks in `pause()` until our inline-resolve + // delivers a resolution. This mirrors the production timing + // where the agent loop's handle_message is blocked here. + let controller_for_pause = Arc::clone(&controller); + let user_for_pause = user_id.clone(); + let pause_task = tokio::spawn(async move { + controller_for_pause + .pause(GatePauseRequest { + thread_id, + user_id: user_for_pause, + gate_name: "approval".into(), + action_name: "shell".into(), + call_id: "call-1".into(), + parameters: serde_json::json!({"cmd": "ls"}), + resume_kind: ResumeKind::Approval { allow_always: true }, + conversation_id: Some(conversation_id), + }) + .await + }); + + // Wait until pause() inserts its pending gate and registers + // its oneshot. peek() lifting Some signals insert completed; + // we follow up with a small sleep so the spawned task can + // also advance past register() and reach `rx.await`. + let key = crate::gate::pending::PendingGateKey { + user_id: user_id.clone(), + thread_id, + }; + let mut request_id = None; + for _ in 0..200 { + tokio::task::yield_now().await; + if let Some(view) = pending_gates.peek(&key).await + && let Ok(parsed) = uuid::Uuid::parse_str(&view.request_id) + { + request_id = Some(parsed); + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let request_id = + request_id.expect("pause() must insert a pending gate with a request_id"); + // Yield a few more times so the spawned pause_task moves + // from `register()` past it into `rx.await`. + for _ in 0..5 { + tokio::task::yield_now().await; + } + + // Beta: gateway HTTP fast-path — call try_resolve_inline_approval_gate + // directly, not through the agent-loop mpsc. + let result = super::try_resolve_inline_approval_gate( + &user_id, + "gateway", + request_id, + ironclaw_engine::GateResolution::Approved { always: false }, + None, + ) + .await + .expect("inline resolve must succeed"); + + assert!( + matches!(result, super::InlineGateOutcome::Delivered), + "expected Delivered for parked Approval gate; got {result:?}" + ); + + // Alpha must wake promptly with our resolution — well under the + // 30-minute pause expiry. + let resolution = tokio::time::timeout(Duration::from_secs(2), pause_task) + .await + .expect("inline-resolve must wake parked pause within 2s") + .expect("pause task did not panic"); + assert!( + matches!( + resolution, + ironclaw_engine::GateResolution::Approved { always: false } + ), + "delivered resolution must reach the parked future; got {resolution:?}" + ); + + // The pending gate must be consumed — `take_verified` runs + // inside the inline path and the rollback branch only fires + // when try_deliver returns false. + assert!( + pending_gates.peek(&key).await.is_none(), + "pending gate must be removed after successful inline delivery" + ); + + drop(controller); + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("inline resolve regression"); + } + + /// Helper: park a real `pause()` future for a fresh thread and + /// return the controller-allocated `request_id` once it has reached + /// `rx.await`. Used by the multi-scenario tests below. + async fn park_inline_pause_for_test( + controller: Arc<crate::bridge::gate_controller::BridgeGateController>, + pending_gates: Arc<crate::gate::store::PendingGateStore>, + user_id: &str, + thread_id: ironclaw_engine::ThreadId, + source_channel: &str, + ) -> ( + uuid::Uuid, + tokio::task::JoinHandle<ironclaw_engine::GateResolution>, + ) { + use crate::bridge::PerExecutionContext; + use ironclaw_engine::{ConversationId, GateController, GatePauseRequest, ResumeKind}; + use std::time::Duration; + + let conversation_id = ConversationId::new(); + controller + .set_execution_context( + user_id.to_string(), + thread_id, + PerExecutionContext { + conversation_id, + source_channel: source_channel.to_string(), + scope_thread_id: None, + channel_metadata: serde_json::json!({}), + original_message: None, + }, + ) + .await; + + let controller_for_pause = Arc::clone(&controller); + let user_for_pause = user_id.to_string(); + let pause_task = tokio::spawn(async move { + controller_for_pause + .pause(GatePauseRequest { + thread_id, + user_id: user_for_pause, + gate_name: "approval".into(), + action_name: "shell".into(), + call_id: format!("call-{thread_id}"), + parameters: serde_json::json!({"cmd": "ls"}), + resume_kind: ResumeKind::Approval { allow_always: true }, + conversation_id: Some(conversation_id), + }) + .await + }); + + let key = crate::gate::pending::PendingGateKey { + user_id: user_id.to_string(), + thread_id, + }; + let mut request_id = None; + for _ in 0..200 { + tokio::task::yield_now().await; + if let Some(view) = pending_gates.peek(&key).await + && let Ok(parsed) = uuid::Uuid::parse_str(&view.request_id) + { + request_id = Some(parsed); + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let request_id = + request_id.expect("park_inline_pause_for_test: pause() did not insert a pending gate"); + // Yield so pause() advances past register() into rx.await. + for _ in 0..5 { + tokio::task::yield_now().await; + } + (request_id, pause_task) + } + + /// Build a controller + state pair that share `pending_gates` and + /// `gate_controller`, install into `ENGINE_STATE`, and return both + /// Arcs for the test to reuse. Caller still owns the global + /// `ENGINE_STATE_TEST_LOCK` guard. + async fn install_inline_test_state() -> ( + Arc<crate::gate::store::PendingGateStore>, + Arc<crate::bridge::gate_controller::BridgeGateController>, + ) { + let pending_gates = Arc::new(crate::gate::store::PendingGateStore::in_memory()); + let resolutions = Arc::new(crate::bridge::gate_controller::GateResolutions::new()); + let controller = Arc::new(crate::bridge::gate_controller::BridgeGateController::new( + Arc::clone(&pending_gates), + None, + Arc::new(crate::tools::ToolRegistry::new()), + None, + None, + Arc::new(crate::channels::ChannelManager::new()), + Arc::clone(&resolutions), + )); + let store = Arc::new(TestStore::new()); + let mut state = make_expected_test_state(store); + state.pending_gates = Arc::clone(&pending_gates); + state.gate_controller = Arc::clone(&controller); + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = Some(state); + (pending_gates, controller) + } + + /// In-memory `SettingsStore` used by the always-allow rollback test. + /// Records every write so the test can assert that the rollback path + /// actually deleted the just-installed `tool_permissions.<tool>` key. + struct TestSettingsStore { + data: tokio::sync::RwLock< + std::collections::HashMap<String, std::collections::HashMap<String, serde_json::Value>>, + >, + } + + impl TestSettingsStore { + fn new() -> Self { + Self { + data: tokio::sync::RwLock::new(std::collections::HashMap::new()), + } + } + } + + #[async_trait::async_trait] + impl crate::db::SettingsStore for TestSettingsStore { + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result<Option<serde_json::Value>, crate::error::DatabaseError> { + Ok(self + .data + .read() + .await + .get(user_id) + .and_then(|m| m.get(key).cloned())) + } + async fn get_setting_full( + &self, + _user_id: &str, + _key: &str, + ) -> Result<Option<crate::history::SettingRow>, crate::error::DatabaseError> { + Ok(None) + } + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), crate::error::DatabaseError> { + self.data + .write() + .await + .entry(user_id.to_string()) + .or_default() + .insert(key.to_string(), value.clone()); + Ok(()) + } + async fn delete_setting( + &self, + user_id: &str, + key: &str, + ) -> Result<bool, crate::error::DatabaseError> { + Ok(self + .data + .write() + .await + .get_mut(user_id) + .and_then(|m| m.remove(key)) + .is_some()) + } + async fn list_settings( + &self, + _user_id: &str, + ) -> Result<Vec<crate::history::SettingRow>, crate::error::DatabaseError> { + Ok(vec![]) + } + async fn get_all_settings( + &self, + user_id: &str, + ) -> Result<std::collections::HashMap<String, serde_json::Value>, crate::error::DatabaseError> + { + Ok(self + .data + .read() + .await + .get(user_id) + .cloned() + .unwrap_or_default()) + } + async fn set_all_settings( + &self, + user_id: &str, + settings: &std::collections::HashMap<String, serde_json::Value>, + ) -> Result<(), crate::error::DatabaseError> { + self.data + .write() + .await + .insert(user_id.to_string(), settings.clone()); + Ok(()) + } + async fn has_settings(&self, user_id: &str) -> Result<bool, crate::error::DatabaseError> { + Ok(self + .data + .read() + .await + .get(user_id) + .is_some_and(|m| !m.is_empty())) + } + } + + /// `try_resolve_inline_approval_gate` must roll back any + /// `tool_permissions.<tool>` AlwaysAllow it provisionally installed + /// when `try_deliver` reports no live VM. Without rollback, a + /// caller resolving an Approval gate from a non-engine path would + /// silently install a session-wide auto-approve preference even + /// though the resume never executed. + #[tokio::test] + async fn try_resolve_inline_approval_gate_rolls_back_always_when_no_live_vm() { + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let (pending_gates, controller) = install_inline_test_state().await; + + // Insert a pending Approval gate but DO NOT register a oneshot — + // simulating the post-restart shape where invalidate_stranded + // somehow missed a row, or any future code path that creates a + // gate without parking a VM. + let thread_id = ironclaw_engine::ThreadId::new(); + let user_id = "alice"; + let request_id = uuid::Uuid::new_v4(); + let pending = sample_pending_gate_with_request_id( + user_id, + thread_id, + request_id, + ironclaw_engine::ResumeKind::Approval { allow_always: true }, + ); + let mut pending = pending; + pending.action_name = "http".into(); + pending.source_channel = "gateway".into(); + pending_gates + .insert(pending.clone()) + .await + .expect("insert pending gate"); + + let settings = Arc::new(TestSettingsStore::new()); + let settings_ref: &(dyn crate::db::SettingsStore + Send + Sync) = settings.as_ref(); + + let result = super::try_resolve_inline_approval_gate( + user_id, + "gateway", + request_id, + ironclaw_engine::GateResolution::Approved { always: true }, + Some(settings_ref), + ) + .await + .expect("inline resolve must succeed (no-live-VM path)"); + + assert!( + matches!(result, super::InlineGateOutcome::NoLiveVm), + "no parked future ⇒ NoLiveVm; got {result:?}" + ); + + // Auto-approve preference must be reverted — the resume + // never executed, so a stale always_allow would be a leak. + let perm_key = format!("tool_permissions.{}", pending.action_name); + assert!( + crate::db::SettingsStore::get_setting(settings_ref, user_id, &perm_key) + .await + .expect("settings get") + .is_none(), + "always_allow must NOT be persisted on no-live-VM path" + ); + + // Pending gate must be back in the store so the legacy mpsc + // dispatch path can find it. + let key = crate::gate::pending::PendingGateKey { + user_id: user_id.to_string(), + thread_id, + }; + assert!( + pending_gates.peek(&key).await.is_some(), + "pending gate must be re-inserted on no-live-VM" + ); + + drop(controller); + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("rollback regression"); + } + + /// Cross-channel security: a gate raised on `telegram` (a non-trusted + /// source channel) cannot be resolved by `slack` (also non-trusted). + /// `take_verified` rejects the channel mismatch; the inline-resolve + /// surface must propagate that as an `authorization` error so the + /// HTTP handler returns 403, not silently drop into the legacy + /// fall-through. The parked alpha must remain parked. + #[tokio::test] + async fn try_resolve_inline_approval_gate_rejects_cross_channel_resolve() { + use std::time::Duration; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let (pending_gates, controller) = install_inline_test_state().await; + let thread_id = ironclaw_engine::ThreadId::new(); + let user_id = "alice"; + + let (request_id, mut pause_task) = park_inline_pause_for_test( + Arc::clone(&controller), + Arc::clone(&pending_gates), + user_id, + thread_id, + "telegram", + ) + .await; + + // Slack tries to approve a Telegram-raised gate. Neither is + // in TRUSTED_GATE_CHANNELS, so this is a true cross-channel + // attempt and must be rejected. + let err = super::try_resolve_inline_approval_gate( + user_id, + "slack", + request_id, + ironclaw_engine::GateResolution::Approved { always: false }, + None, + ) + .await + .expect_err("cross-channel resolve must error"); + assert!( + matches!(err, super::InlineGateError::ChannelMismatch { .. }), + "channel-mismatch must surface as InlineGateError::ChannelMismatch; got: {err:?}" + ); + + // Parked alpha must NOT have woken — the rejection happens + // inside take_verified, before any try_deliver call. Borrow + // `pause_task` mutably (rather than moving it into a `{ ... }` + // block expression) so the JoinHandle stays bound through the + // explicit `controller.cancel_thread` cleanup below. + let still_parked = tokio::time::timeout(Duration::from_millis(200), &mut pause_task) + .await + .is_err(); + assert!( + still_parked, + "rejected cross-channel resolve must not wake the parked future" + ); + + // The original gate must still be in the store (take_verified + // bails *before* the remove). + let key = crate::gate::pending::PendingGateKey { + user_id: user_id.to_string(), + thread_id, + }; + assert!( + pending_gates.peek(&key).await.is_some(), + "rejected cross-channel resolve must leave the pending gate in place" + ); + + // Clean up the parked task — cancel via controller so the + // test process doesn't leak the spawned tokio task. + use ironclaw_engine::GateController; + controller.cancel_thread(thread_id).await; + + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("cross-channel rejection regression"); + } + + /// Two concurrent inline-resolve calls for the same `request_id`: + /// the gate-store mutex serializes them so exactly one wins and + /// delivers. The loser's outcome is permitted to be either an error + /// (race observed inside the lock before the winner removed) or + /// `NoLiveVm` (race observed after removal — falls through to the + /// legacy mpsc path which will respond "no matching pending + /// approval"). What must NOT happen is a second `Delivered`. The + /// parked alpha receives the resolution from the winner. + #[tokio::test] + async fn try_resolve_inline_approval_gate_concurrent_resolves_one_wins() { + use std::time::Duration; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let (pending_gates, controller) = install_inline_test_state().await; + let thread_id = ironclaw_engine::ThreadId::new(); + let user_id = "alice"; + + let (request_id, pause_task) = park_inline_pause_for_test( + Arc::clone(&controller), + Arc::clone(&pending_gates), + user_id, + thread_id, + "gateway", + ) + .await; + + // Two callers race on the same request_id. + let resolution = ironclaw_engine::GateResolution::Approved { always: false }; + let (a, b) = tokio::join!( + super::try_resolve_inline_approval_gate( + user_id, + "gateway", + request_id, + resolution.clone(), + None, + ), + super::try_resolve_inline_approval_gate( + user_id, "gateway", request_id, resolution, None, + ), + ); + + let mut delivered_count = 0; + let mut loser_count = 0; + for r in [&a, &b] { + match r { + Ok(super::InlineGateOutcome::Delivered) => delivered_count += 1, + Ok(super::InlineGateOutcome::NoLiveVm) | Err(_) => loser_count += 1, + } + } + assert_eq!( + delivered_count, 1, + "exactly one concurrent resolve must report Delivered" + ); + assert_eq!( + loser_count, 1, + "the loser must NOT also report Delivered (race => NoLiveVm or error, both acceptable)" + ); + + // Alpha woke once with the winning resolution. + let woken = tokio::time::timeout(Duration::from_secs(2), pause_task) + .await + .expect("alpha must wake once after the winner delivers") + .expect("pause task did not panic"); + assert!( + matches!( + woken, + ironclaw_engine::GateResolution::Approved { always: false } + ), + "winner's resolution must reach the parked future; got {woken:?}" + ); + + drop(controller); + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("concurrent-resolves regression"); + } + + /// Two parallel Approval gates for the same user on different + /// threads must each get their own oneshot. Resolving thread A + /// wakes only A; B stays parked until its own resolve. Verifies + /// `try_deliver` routes by `request_id`, not by `(user, thread)`, + /// and that `BridgeGateController`'s per-(user, thread) gate lock + /// doesn't bleed between threads. + #[tokio::test] + async fn try_resolve_inline_approval_gate_parallel_threads_same_user_independent() { + use std::time::Duration; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let (pending_gates, controller) = install_inline_test_state().await; + let user_id = "alice"; + + let thread_a = ironclaw_engine::ThreadId::new(); + let thread_b = ironclaw_engine::ThreadId::new(); + + let (req_a, mut pause_a) = park_inline_pause_for_test( + Arc::clone(&controller), + Arc::clone(&pending_gates), + user_id, + thread_a, + "gateway", + ) + .await; + let (req_b, mut pause_b) = park_inline_pause_for_test( + Arc::clone(&controller), + Arc::clone(&pending_gates), + user_id, + thread_b, + "gateway", + ) + .await; + assert_ne!(req_a, req_b, "each thread must have its own request_id"); + + // Resolve A only. + let res = super::try_resolve_inline_approval_gate( + user_id, + "gateway", + req_a, + ironclaw_engine::GateResolution::Approved { always: false }, + None, + ) + .await + .expect("inline resolve A must succeed"); + assert!( + matches!(res, super::InlineGateOutcome::Delivered), + "thread A must deliver; got {res:?}" + ); + + let woken_a = tokio::time::timeout(Duration::from_secs(2), &mut pause_a) + .await + .expect("A must wake within 2s") + .expect("pause A did not panic"); + assert!(matches!( + woken_a, + ironclaw_engine::GateResolution::Approved { always: false } + )); + + // B must remain parked — explicitly verify with a short + // timeout window. + let still_parked = tokio::time::timeout(Duration::from_millis(200), &mut pause_b) + .await + .is_err(); + assert!( + still_parked, + "thread B must remain parked while only A is resolved" + ); + + // Now resolve B; it should wake independently. + let res = super::try_resolve_inline_approval_gate( + user_id, + "gateway", + req_b, + ironclaw_engine::GateResolution::Denied { reason: None }, + None, + ) + .await + .expect("inline resolve B must succeed"); + assert!( + matches!(res, super::InlineGateOutcome::Delivered), + "thread B must deliver; got {res:?}" + ); + + let woken_b = tokio::time::timeout(Duration::from_secs(2), pause_b) + .await + .expect("B must wake within 2s after its own resolve") + .expect("pause B did not panic"); + assert!( + matches!(woken_b, ironclaw_engine::GateResolution::Denied { .. }), + "thread B must receive its own Denied resolution; got {woken_b:?}" + ); + + drop(controller); + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("parallel-threads regression"); + } + + /// Regression for the gateway approval contract: when a web thread + /// records a `scope_thread_id` (the per-conversation UUID returned + /// by `/api/chat/thread/new`) that differs from the engine's + /// internal `ThreadId`, the inline fast path must still find the + /// pending gate by `request_id` alone — *not* by the wire + /// thread id. Before this fix the handler constructed + /// `ThreadId(scope_thread_id)` and `take_verified` missed the row, + /// returning 500 instead of waking the parked alpha. + #[tokio::test] + async fn try_resolve_inline_approval_gate_resolves_when_scope_id_differs_from_thread_id() { + use crate::bridge::PerExecutionContext; + use ironclaw_common::ExternalThreadId; + use ironclaw_engine::{ConversationId, GateController, GatePauseRequest, ResumeKind}; + use std::time::Duration; + + let _guard = ENGINE_STATE_TEST_LOCK.lock().await; + let lock = ENGINE_STATE.get_or_init(|| RwLock::new(None)); + *lock.write().await = None; + + let outcome = async { + let (pending_gates, controller) = install_inline_test_state().await; + let user_id = "alice"; + let thread_id = ironclaw_engine::ThreadId::new(); + // The wire / scope id the web frontend would send back is + // *not* the engine thread id — it's the conversation UUID + // recorded on the gate as `scope_thread_id`. + let scope_id = ExternalThreadId::new(uuid::Uuid::new_v4().to_string()) + .expect("UUID is a valid ExternalThreadId"); + let conversation_id = ConversationId::new(); + + controller + .set_execution_context( + user_id.to_string(), + thread_id, + PerExecutionContext { + conversation_id, + source_channel: "gateway".into(), + scope_thread_id: Some(scope_id.clone()), + channel_metadata: serde_json::json!({}), + original_message: None, + }, + ) + .await; + + let controller_for_pause = Arc::clone(&controller); + let user_for_pause = user_id.to_string(); + let mut pause_task = tokio::spawn(async move { + controller_for_pause + .pause(GatePauseRequest { + thread_id, + user_id: user_for_pause, + gate_name: "approval".into(), + action_name: "shell".into(), + call_id: "call-1".into(), + parameters: serde_json::json!({"cmd": "ls"}), + resume_kind: ResumeKind::Approval { + allow_always: false, + }, + conversation_id: Some(conversation_id), + }) + .await + }); + + // Wait until pause() inserts the pending gate. + let key = crate::gate::pending::PendingGateKey { + user_id: user_id.to_string(), + thread_id, + }; + let mut request_id = None; + for _ in 0..200 { + tokio::task::yield_now().await; + if let Some(view) = pending_gates.peek(&key).await + && let Ok(parsed) = uuid::Uuid::parse_str(&view.request_id) + { + request_id = Some(parsed); + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let request_id = request_id.expect("pause() must insert a pending gate"); + for _ in 0..5 { + tokio::task::yield_now().await; + } + + // Sanity: the pending gate's wire-effective thread id is + // the scope id, not the engine ThreadId — exactly the + // shape that broke the original implementation. + let view = pending_gates + .peek(&key) + .await + .expect("pending gate present"); + assert_eq!( + view.thread_id, + scope_id.as_str(), + "view must surface the scope id as the wire thread_id" + ); + assert_ne!( + view.thread_id, + thread_id.to_string(), + "scope id must differ from engine thread id for this regression" + ); + + // The inline handler does not pass thread_id at all. It + // must still resolve the gate from `request_id` alone. + let result = super::try_resolve_inline_approval_gate( + user_id, + "gateway", + request_id, + ironclaw_engine::GateResolution::Approved { always: false }, + None, + ) + .await + .expect("inline resolve must succeed even when scope id != engine thread id"); + assert!( + matches!(result, super::InlineGateOutcome::Delivered), + "must Delivered, not NoLiveVm or error; got {result:?}" + ); + + let woken = tokio::time::timeout(Duration::from_secs(2), &mut pause_task) + .await + .expect("parked alpha must wake within 2s") + .expect("pause task did not panic"); + assert!(matches!( + woken, + ironclaw_engine::GateResolution::Approved { always: false } + )); + + drop(controller); + Ok::<(), crate::error::Error>(()) + } + .await; + + *lock.write().await = None; + outcome.expect("scope-thread-id mismatch regression"); + } } diff --git a/src/bridge/store_adapter.rs b/src/bridge/store_adapter.rs index 998bf6765e1..0f6e1c2b668 100644 --- a/src/bridge/store_adapter.rs +++ b/src/bridge/store_adapter.rs @@ -1900,14 +1900,18 @@ impl Store for HybridStore { project_id: ProjectId, user_id: &str, ) -> Result<Vec<Mission>, EngineError> { - Ok(self + let mut missions: Vec<Mission> = self .missions .read() .await .values() .filter(|mission| mission.project_id == project_id && mission.user_id == user_id) .cloned() - .collect()) + .collect(); + // HashMap iteration is non-deterministic; sort by (name, id) + // so callers see a stable order across runs. + missions.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.0.cmp(&b.id.0))); + Ok(missions) } async fn list_all_threads(&self, project_id: ProjectId) -> Result<Vec<Thread>, EngineError> { @@ -1922,14 +1926,16 @@ impl Store for HybridStore { } async fn list_all_missions(&self, project_id: ProjectId) -> Result<Vec<Mission>, EngineError> { - Ok(self + let mut missions: Vec<Mission> = self .missions .read() .await .values() .filter(|mission| mission.project_id == project_id) .cloned() - .collect()) + .collect(); + missions.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.0.cmp(&b.id.0))); + Ok(missions) } async fn update_mission_status( diff --git a/src/bridge/tool_permissions.rs b/src/bridge/tool_permissions.rs index 7a43fd63ef7..c5f4bc2c8c4 100644 --- a/src/bridge/tool_permissions.rs +++ b/src/bridge/tool_permissions.rs @@ -39,13 +39,24 @@ impl ToolPermissionSnapshot { pub(crate) fn resolve_permission(&self, tool_name: &str) -> ToolPermissionResolution { let canonical = canonical_tool_name(tool_name); let hyphenated = canonical.replace('_', "-"); - let explicit = self.explicit_permission_with_names(tool_name, &canonical, &hyphenated); - let effective = explicit - .or_else(|| seeded_default_permission(&canonical)) + let raw_explicit = self.explicit_permission_with_names(tool_name, &canonical, &hyphenated); + let seeded_default = seeded_default_permission(&canonical); + // Any DB row is a user-explicit choice. The original #3533 fix + // here collapsed value-equal-to-seed rows to `explicit = None` + // so `AGENT_AUTO_APPROVE_TOOLS=true` could bypass them, but + // value-equality is not provenance — a user who genuinely picks + // `AskEachTime` for `tool_install` (the seeded default) would + // see their explicit choice silently bypassed. Provenance is + // now handled at write time: pre-#3559 `seed_tool_permissions` + // wrote ghost rows that `cleanup_ghost_seeded_tool_permissions` + // deletes on first startup, and no new seeded rows are written. + // See `src/app.rs::cleanup_ghost_seeded_tool_permissions`. + let effective = raw_explicit + .or(seeded_default) .unwrap_or(PermissionState::AskEachTime); ToolPermissionResolution { effective, - explicit, + explicit: raw_explicit, } } @@ -111,4 +122,50 @@ mod tests { } ); } + + /// #3559 security review: a DB row is always a user-explicit choice. + /// A user who genuinely picks `AskEachTime` for `tool_install` (which + /// happens to match the code-level seeded default) must surface as + /// `explicit = Some(...)` so `effect_adapter::enforce_tool_permission`'s + /// `is_explicit_ask` check fires and `AGENT_AUTO_APPROVE_TOOLS=true` + /// does NOT bypass the gate. The pre-#3559 collapse-to-implicit logic + /// silently dropped this choice; the cleanup migration in + /// `app::cleanup_ghost_seeded_tool_permissions` now removes the + /// historical ghost-seeded rows at boot so any surviving DB row is + /// user-explicit by construction. + #[test] + fn user_explicit_value_matching_seeded_default_stays_explicit() { + let snapshot = ToolPermissionSnapshot { + overrides: HashMap::from([("tool_install".to_string(), PermissionState::AskEachTime)]), + }; + + assert_eq!( + snapshot.resolve_permission("tool_install"), + ToolPermissionResolution { + effective: PermissionState::AskEachTime, + explicit: Some(PermissionState::AskEachTime), + } + ); + } + + /// A user who explicitly opts out of the seeded default (here: + /// `tool_install` set to `AlwaysAllow` instead of the seeded + /// `AskEachTime`) keeps their explicit choice. Same semantics as + /// `user_explicit_value_matching_seeded_default_stays_explicit`, + /// but with a value that diverges from the seeded default — the + /// resolver treats all DB values identically. + #[test] + fn user_override_diverging_from_seeded_default_stays_explicit() { + let snapshot = ToolPermissionSnapshot { + overrides: HashMap::from([("tool_install".to_string(), PermissionState::AlwaysAllow)]), + }; + + assert_eq!( + snapshot.resolve_permission("tool_install"), + ToolPermissionResolution { + effective: PermissionState::AlwaysAllow, + explicit: Some(PermissionState::AlwaysAllow), + } + ); + } } diff --git a/src/bridge/tool_surface.rs b/src/bridge/tool_surface.rs index ccf2e763a14..6b38be30f5e 100644 --- a/src/bridge/tool_surface.rs +++ b/src/bridge/tool_surface.rs @@ -96,7 +96,21 @@ pub(crate) fn assign_surface(subject: SurfacePolicyInput) -> SurfaceAssignment { } const fn is_direct_ready(status: CapabilityStatus) -> bool { - matches!(status, CapabilityStatus::Ready) + // `NeedsAuth` tools (e.g. installed-but-unauthed gmail) stay on + // the callable surface post-#3133/#3166: the engine's auth + // preflight (`AuthManager::check_action_auth`) raises an + // `Authentication` gate when the tool is invoked and any required + // credential is missing, the inline-await machinery parks the VM, + // and the OAuth-callback hook delivers `Approved` to retry the + // action against the now-present secret. The model can therefore + // call the tool directly without a separate enablement step. + // `NeedsSetup` / `Inactive` / `Latent` still fall through to + // the capabilities surface — those need real onboarding work that + // a credential-write hook can't supply. + matches!( + status, + CapabilityStatus::Ready | CapabilityStatus::NeedsAuth + ) } const fn fallback_assignment(status: CapabilityStatus) -> SurfaceAssignment { @@ -171,14 +185,18 @@ mod tests { expected: SurfaceAssignment::actions_only(), }, Case { - name: "needs-auth extension direct action", + // Post-#3133/#3166: NeedsAuth extension tools stay + // on the callable surface — the engine raises an + // Authentication gate at execute time and inline-await + // resumes the action after OAuth completes. + name: "needs-auth extension direct action stays callable", subject: SurfacePolicyInput { kind: SurfaceSubjectKind::ExtensionDirectAction, status: CapabilityStatus::NeedsAuth, invocation_mode: InvocationMode::Direct, leased_and_callable: false, }, - expected: SurfaceAssignment::capabilities_only(), + expected: SurfaceAssignment::actions_only(), }, Case { name: "needs-setup extension direct action", diff --git a/src/channels/channel.rs b/src/channels/channel.rs index 90783c7c925..9dd07587b47 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -11,57 +11,10 @@ use uuid::Uuid; use crate::error::ChannelError; -/// Kind of attachment carried on an incoming message. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AttachmentKind { - /// Audio content (voice notes, audio files). - Audio, - /// Image content (photos, screenshots). - Image, - /// Document content (PDFs, files). - Document, -} - -impl AttachmentKind { - /// Infer attachment kind from MIME type. - pub fn from_mime_type(mime: &str) -> Self { - let base = mime.split(';').next().unwrap_or(mime).trim(); - if base.starts_with("audio/") { - Self::Audio - } else if base.starts_with("image/") { - Self::Image - } else { - Self::Document - } - } -} - -/// A file or media attachment on an incoming message. -#[derive(Debug, Clone)] -pub struct IncomingAttachment { - /// Unique identifier within the channel (e.g., Telegram file_id). - pub id: String, - /// What kind of content this is. - pub kind: AttachmentKind, - /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). - pub mime_type: String, - /// Original filename, if known. - pub filename: Option<String>, - /// File size in bytes, if known. - pub size_bytes: Option<u64>, - /// URL to download the file from the channel's API. - pub source_url: Option<String>, - /// Opaque key for host-side storage (e.g., after download/caching). - pub storage_key: Option<String>, - /// Relative path to a project-local copy saved on disk, if persisted. - pub local_path: Option<String>, - /// Extracted text content (e.g., OCR result, PDF text, audio transcript). - pub extracted_text: Option<String>, - /// Raw file bytes (for small files downloaded by the channel). - pub data: Vec<u8>, - /// Duration in seconds (for audio/video). - pub duration_secs: Option<u32>, -} +// Channel-agnostic attachment types live in `ironclaw_common::attachment`. +// Re-exported here so the existing `crate::channels::AttachmentKind` / +// `crate::channels::IncomingAttachment` import paths keep working. +pub use ironclaw_common::attachment::{AttachmentKind, IncomingAttachment}; /// A message received from an external channel. #[derive(Debug, Clone)] @@ -130,6 +83,19 @@ pub struct IncomingMessage { impl IncomingMessage { /// Create a new incoming message. + /// + /// The default `metadata` carries `{"user_id": <user_id>}` so that any + /// downstream consumer that scopes by `metadata.user_id` (notably + /// `GatewayChannel::send_status` for SSE/WS event routing) has the + /// owning tenant identity even when the producer never calls + /// [`Self::with_metadata`]. Producers replacing metadata wholesale + /// should prefer [`Self::with_metadata`], which overwrites a + /// **string-typed** `user_id` key on the supplied object with + /// `self.user_id` — caller-supplied string values are dropped to + /// keep the SSE recipient scope unforgeable. Non-string values + /// (e.g. Telegram's `i64` chat user ID) are left alone; the SSE + /// routing layer's `as_str()` treats them as missing and fails + /// closed in multi-tenant mode. pub fn new( channel: impl Into<String>, user_id: impl Into<String>, @@ -140,6 +106,7 @@ impl IncomingMessage { id: Uuid::new_v4(), channel: channel.into(), sender_id: user_id.clone(), + metadata: serde_json::json!({ "user_id": &user_id }), user_id, user_name: None, content: content.into(), @@ -147,7 +114,6 @@ impl IncomingMessage { thread_id: None, conversation_scope_id: None, received_at: Utc::now(), - metadata: serde_json::Value::Null, timezone: None, attachments: Vec::new(), is_internal: false, @@ -236,7 +202,50 @@ impl IncomingMessage { } /// Set metadata. + /// + /// A **string-typed** `metadata.user_id` is always overwritten with + /// `self.user_id` — caller-supplied string values are dropped. This + /// makes the SSE/WS recipient scope unforgeable from channel + /// metadata: a WASM extension whose emitted JSON contains + /// `{"user_id":"victim"}` (intentionally or via bug) cannot route a + /// later `ToolStarted` / `ToolResult` event into another tenant's + /// stream, because the SSE routing layer reads the field via + /// `as_str()`. + /// + /// **Non-string** `user_id` values (e.g. Telegram's `i64` chat user + /// ID, which the Telegram WASM channel persists in this same + /// metadata field for its own `on_respond` routing) are left alone: + /// they cannot be exploited because the SSE routing layer + /// (`as_str()`) treats them as missing and fails closed in + /// multi-tenant mode. Stomping them would corrupt channel-private + /// metadata. + /// + /// Non-object inputs (`Null`, array, scalar) are replaced with a + /// fresh object carrying `self.user_id`. A missing `user_id` key is + /// inserted with `self.user_id`. + /// + /// If a caller legitimately needs to forward to a different tenant + /// (e.g. a proactive broadcast) it must mint a separate + /// `IncomingMessage` with the target `user_id` rather than + /// hand-rolling the metadata field. pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self { + let mut metadata = match metadata { + serde_json::Value::Object(_) => metadata, + _ => serde_json::json!({}), + }; + if let Some(obj) = metadata.as_object_mut() { + let should_set = match obj.get("user_id") { + None => true, + Some(serde_json::Value::String(_)) => true, + Some(_) => false, + }; + if should_set { + obj.insert( + "user_id".to_string(), + serde_json::Value::String(self.user_id.clone()), + ); + } + } self.metadata = metadata; self } @@ -319,6 +328,17 @@ pub fn routing_target_from_metadata(metadata: &serde_json::Value) -> Option<Stri /// Stream of incoming messages. pub type MessageStream = Pin<Box<dyn Stream<Item = IncomingMessage> + Send>>; +/// In-memory attachment to send back to a channel. +#[derive(Debug, Clone)] +pub struct OutgoingAttachment { + /// Filename to present to the receiving channel. + pub filename: String, + /// MIME type (e.g., "image/png"). + pub mime_type: String, + /// Raw attachment bytes. + pub data: Vec<u8>, +} + /// Response to send back to a channel. #[derive(Debug, Clone)] pub struct OutgoingResponse { @@ -331,6 +351,8 @@ pub struct OutgoingResponse { pub thread_id: Option<ExternalThreadId>, /// Optional file paths to attach. pub attachments: Vec<String>, + /// Optional in-memory attachments to attach. + pub inline_attachments: Vec<OutgoingAttachment>, /// Channel-specific metadata for the response. pub metadata: serde_json::Value, } @@ -342,6 +364,7 @@ impl OutgoingResponse { content: content.into(), thread_id: None, attachments: Vec::new(), + inline_attachments: Vec::new(), metadata: serde_json::Value::Null, } } @@ -388,6 +411,12 @@ impl OutgoingResponse { self.attachments = paths; self } + + /// Add in-memory attachments to the response. + pub fn with_inline_attachments(mut self, attachments: Vec<OutgoingAttachment>) -> Self { + self.inline_attachments = attachments; + self + } } /// A single tool decision within a reasoning update. @@ -1091,6 +1120,80 @@ mod tests { assert_eq!(msg.timezone.as_deref(), Some("America/New_York")); } + /// Regression: a WASM channel that emits metadata containing a + /// foreign string `user_id` must NOT be able to override the + /// message's owner. `apply_emitted_metadata` in the WASM wrapper + /// feeds parsed JSON straight into `with_metadata`; if a malicious + /// or buggy extension supplies `{"user_id":"victim"}`, downstream + /// `send_status` would route ToolStarted/ToolResult into the + /// victim's SSE stream. `with_metadata` must clobber the field + /// when it is a string. + #[test] + fn with_metadata_overwrites_caller_supplied_string_user_id() { + let msg = IncomingMessage::new("wasm_channel", "alice", "hi") + .with_metadata(serde_json::json!({"user_id": "bob", "chat_id": 42})); + assert_eq!( + msg.metadata.get("user_id").and_then(|v| v.as_str()), + Some("alice"), + "with_metadata must drop caller-supplied string user_id and use the message's own" + ); + // Other caller fields survive. + assert_eq!( + msg.metadata.get("chat_id").and_then(|v| v.as_i64()), + Some(42) + ); + } + + /// Regression: the Telegram WASM channel persists its Telegram + /// user ID as `metadata.user_id: <i64>` and re-parses it in + /// `on_respond` via `TelegramMessageMetadata { user_id: i64, ... }`. + /// `with_metadata` must NOT clobber a non-string `user_id`: the + /// SSE routing layer reads via `as_str()`, treats non-strings as + /// missing, and fails closed in multi-tenant mode — so the forge + /// threat is mitigated without corrupting channel-private metadata. + /// Without this carve-out the Telegram channel cannot deserialize + /// its own metadata back. Reference: PR #3390 follow-up. + #[test] + fn with_metadata_preserves_non_string_user_id() { + let msg = IncomingMessage::new("telegram", "alice", "hi") + .with_metadata(serde_json::json!({"user_id": 999, "chat_id": 999, "message_id": 1})); + assert_eq!( + msg.metadata.get("user_id").and_then(|v| v.as_i64()), + Some(999), + "with_metadata must preserve i64 user_id (channel-private routing)" + ); + // SSE routing reads via as_str() — non-string values must read as None. + assert!( + msg.metadata + .get("user_id") + .and_then(|v| v.as_str()) + .is_none(), + "non-string user_id must read as None via as_str() so SSE routing fails closed" + ); + } + + #[test] + fn with_metadata_inserts_user_id_when_missing() { + let msg = IncomingMessage::new("test", "alice", "hi") + .with_metadata(serde_json::json!({"chat_id": 42})); + assert_eq!( + msg.metadata.get("user_id").and_then(|v| v.as_str()), + Some("alice"), + "with_metadata must insert user_id when the caller's object lacks it" + ); + } + + #[test] + fn with_metadata_replaces_non_object_with_owner_user_id() { + let msg = + IncomingMessage::new("test", "alice", "hi").with_metadata(serde_json::Value::Null); + assert_eq!( + msg.metadata.get("user_id").and_then(|v| v.as_str()), + Some("alice"), + "non-object metadata must be replaced with an object carrying user_id" + ); + } + #[test] fn tool_call_detail_http() { let args = serde_json::json!({"method": "POST", "url": "https://api.example.com/data"}); diff --git a/src/channels/matrix.rs b/src/channels/matrix.rs index 053fdfc0473..0a01188ab32 100644 --- a/src/channels/matrix.rs +++ b/src/channels/matrix.rs @@ -956,10 +956,10 @@ mod sdk { if let Err(e) = room.typing_notice(true).await { debug!(error = %e, "Matrix: failed to send typing indicator"); } - } else if let Some(text) = format_status_message(&status) { - if let Err(e) = self.send_text_to_room(&room_id, &text).await { - debug!(error = %e, "Matrix: failed to send status notification"); - } + } else if let Some(text) = format_status_message(&status) + && let Err(e) = self.send_text_to_room(&room_id, &text).await + { + debug!(error = %e, "Matrix: failed to send status notification"); } Ok(()) } diff --git a/src/channels/mod.rs b/src/channels/mod.rs index dc70065cd3d..82a27628c35 100644 --- a/src/channels/mod.rs +++ b/src/channels/mod.rs @@ -49,8 +49,8 @@ pub(crate) use attachments::{ }; pub use channel::{ AttachmentKind, Channel, ChannelSecretUpdater, ChatApprovalPrompt, EngineThreadSummary, - HistoryMessage, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse, - StatusUpdate, ThreadSummary, ToolDecision, routing_target_from_metadata, + HistoryMessage, IncomingAttachment, IncomingMessage, MessageStream, OutgoingAttachment, + OutgoingResponse, StatusUpdate, ThreadSummary, ToolDecision, routing_target_from_metadata, }; pub use http::{HttpChannel, HttpChannelState}; pub use manager::ChannelManager; diff --git a/src/channels/relay/channel.rs b/src/channels/relay/channel.rs index 192e48d00da..25827cb4b87 100644 --- a/src/channels/relay/channel.rs +++ b/src/channels/relay/channel.rs @@ -6,6 +6,7 @@ //! proxy API (Slack). use std::collections::HashMap; +use std::sync::Arc; use async_trait::async_trait; use tokio::sync::mpsc; @@ -15,6 +16,7 @@ use crate::channels::{ Channel, ChatApprovalPrompt, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate, }; use crate::error::ChannelError; +use crate::pairing::PairingStore; /// Default channel name for the Slack relay integration. pub const DEFAULT_RELAY_NAME: &str = "slack-relay"; @@ -51,6 +53,8 @@ pub struct RelayChannel { event_tx: mpsc::Sender<ChannelEvent>, /// Receiver side — taken once by `start()`. event_rx: tokio::sync::Mutex<Option<mpsc::Receiver<ChannelEvent>>>, + /// Resolves Slack sender_id → internal UserId for multi-tenant support. + pairing_store: Option<Arc<PairingStore>>, } impl RelayChannel { @@ -88,9 +92,16 @@ impl RelayChannel { instance_id, event_tx, event_rx: tokio::sync::Mutex::new(Some(event_rx)), + pairing_store: None, } } + /// Set the pairing store for multi-tenant identity resolution. + pub fn with_pairing_store(mut self, store: Arc<PairingStore>) -> Self { + self.pairing_store = Some(store); + self + } + /// Get a clone of the event sender for wiring into the webhook endpoint. pub fn event_sender(&self) -> mpsc::Sender<ChannelEvent> { self.event_tx.clone() @@ -123,9 +134,10 @@ impl RelayChannel { team_id: &str, method: &str, body: serde_json::Value, + slack_user_id: Option<&str>, ) -> Result<serde_json::Value, crate::channels::relay::client::RelayError> { self.client - .proxy_provider(self.provider.as_str(), team_id, method, body) + .proxy_provider_with_user(self.provider.as_str(), team_id, method, body, slack_user_id) .await } @@ -205,6 +217,8 @@ impl Channel for RelayChannel { let (tx, rx) = mpsc::channel(64); let provider_str = self.provider.as_str().to_string(); let relay_name = channel_name.clone(); + let pairing_store = self.pairing_store.clone(); + let pairing_client = self.client.clone(); // Spawn a task that reads events from the webhook handler and converts to IncomingMessage tokio::spawn(async move { @@ -240,7 +254,80 @@ impl Channel for RelayChannel { "Relay: received message from {}", provider_str ); - let mut msg = IncomingMessage::new(&relay_name, &event.sender_id, event.text()) + // Resolve sender_id → internal UserId via PairingStore. + // External ID is scoped to workspace: "team_id:sender_id". + let scoped_external_id = format!("{}:{}", event.provider_scope, event.sender_id); + let resolved_user_id: String = if let Some(ref store) = pairing_store { + match store + .resolve_identity(&relay_name, &scoped_external_id) + .await + { + Ok(Some(uid)) => { + let user_str = uid.as_str().to_string(); + tracing::debug!( + sender_id = %event.sender_id, + resolved_user = %user_str, + "Relay: resolved sender to internal user" + ); + user_str + } + Ok(None) => { + tracing::info!( + sender_id = %event.sender_id, + "Relay: sender not paired, sending pairing code" + ); + let meta = serde_json::json!({ + "sender_name": event.display_name(), + "channel_id": event.channel_id, + }); + match store + .upsert_request(&relay_name, &scoped_external_id, Some(meta)) + .await + { + Ok(record) => { + let instructions = format!( + "Enter this code in IronClaw to pair your Slack account: `{}`", + record.code + ); + let team_id = event.team_id().to_string(); + let body = serde_json::json!({ + "channel": event.channel_id, + "text": instructions, + "thread_ts": event.thread_id.as_deref().unwrap_or(&event.id), + }); + if let Err(e) = pairing_client + .proxy_provider( + &provider_str, + &team_id, + "chat.postMessage", + body, + ) + .await + { + tracing::warn!(error = %e, "Relay: failed to send pairing code reply"); + } + } + Err(e) => { + tracing::warn!(error = %e, "Relay: failed to create pairing request"); + } + } + continue; + } + Err(e) => { + tracing::warn!( + sender_id = %event.sender_id, + error = %e, + "Relay: pairing resolution failed, dropping message" + ); + continue; + } + } + } else { + event.sender_id.clone() + }; + + let mut msg = IncomingMessage::new(&relay_name, &resolved_user_id, event.text()) + .with_sender_id(event.sender_id.clone()) .with_user_name(event.display_name()) .with_metadata(serde_json::json!({ "team_id": event.team_id(), @@ -323,8 +410,9 @@ impl Channel for RelayChannel { .filter(|s| !s.is_empty()); let (method, body) = self.build_send_body(channel_id, &response.content, thread_id); + let sender_id = metadata.get("sender_id").and_then(|v| v.as_str()); - self.proxy_send(team_id, &method, body) + self.proxy_send(team_id, &method, body, sender_id) .await .map_err(|e| ChannelError::SendFailed { name: channel_name, @@ -384,7 +472,7 @@ impl Channel for RelayChannel { })?; let body = self.build_approval_body(channel_id, thread_id, &prompt, &approval_token); - self.proxy_send(team_id, "chat.postMessage", body) + self.proxy_send(team_id, "chat.postMessage", body, None) .await .map_err(|e| ChannelError::SendFailed { name: self.name().to_string(), @@ -410,7 +498,7 @@ impl Channel for RelayChannel { let (method, body) = self.build_send_body(target, &response.content, thread_id); - self.proxy_send(&self.team_id, &method, body) + self.proxy_send(&self.team_id, &method, body, None) .await .map_err(|e| ChannelError::SendFailed { name: channel_name, diff --git a/src/channels/relay/client.rs b/src/channels/relay/client.rs index 16f40f66474..f09aeab1b13 100644 --- a/src/channels/relay/client.rs +++ b/src/channels/relay/client.rs @@ -81,9 +81,13 @@ impl ChannelEvent { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Connection { pub provider: String, + #[serde(alias = "provider_scope")] pub team_id: String, + #[serde(alias = "provider_scope_name")] pub team_name: Option<String>, + #[serde(default)] pub connected: bool, + pub authed_user_id: Option<String>, } /// HTTP client for the channel-relay service. @@ -237,6 +241,18 @@ impl RelayClient { team_id: &str, method: &str, body: serde_json::Value, + ) -> Result<serde_json::Value, RelayError> { + self.proxy_provider_with_user(provider, team_id, method, body, None) + .await + } + + pub async fn proxy_provider_with_user( + &self, + provider: &str, + team_id: &str, + method: &str, + body: serde_json::Value, + slack_user_id: Option<&str>, ) -> Result<serde_json::Value, RelayError> { let url = format!("{}/proxy/{}/{}", self.base_url, provider, method); tracing::trace!( @@ -245,7 +261,10 @@ impl RelayClient { method = %method, "RelayClient::proxy_provider: sending request" ); - let query: Vec<(&str, &str)> = vec![("team_id", team_id)]; + let mut query: Vec<(&str, &str)> = vec![("team_id", team_id)]; + if let Some(uid) = slack_user_id { + query.push(("slack_user_id", uid)); + } let resp = self .http .post(&url) diff --git a/src/channels/wasm/attachment_hydration.rs b/src/channels/wasm/attachment_hydration.rs new file mode 100644 index 00000000000..deee63b828a --- /dev/null +++ b/src/channels/wasm/attachment_hydration.rs @@ -0,0 +1,922 @@ +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use aes::Aes128; +use aes::cipher::{BlockDecrypt, BlockEncrypt, KeyInit, generic_array::GenericArray}; +use base64::Engine as _; +use futures::StreamExt; +use md5::{Digest, Md5}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; +use tokio::process::Command; + +use crate::channels::wasm::host::{Attachment, ChannelHostState}; + +const AES_BLOCK_SIZE: usize = 16; +const MAX_ATTACHMENT_BYTES: usize = 20 * 1024 * 1024; +const WECHAT_CHANNEL_NAME: &str = "wechat"; +const WECHAT_SILK_SAMPLE_RATE_HZ: u32 = 24_000; +const WECHAT_OUTBOUND_ENVELOPE_MAGIC: &[u8] = b"ICWXENC1"; + +/// Cap on output WAV bytes accepted from the SILK decoder subprocess. SILK→PCM +/// expansion is ~25× at 24 kHz mono; 60 s of voice is ~3 MiB. 50 MiB matches +/// the cap inside the decoder binary and prevents a runaway child from +/// pushing unbounded data into the host. +const MAX_DECODED_WAV_BYTES: usize = 50 * 1024 * 1024; +const SILK_DECODER_TIMEOUT: Duration = Duration::from_secs(15); +const SILK_DECODER_BIN_NAME: &str = "ironclaw-silk-decoder"; +const SILK_DECODER_ENV_VAR: &str = "IRONCLAW_SILK_DECODER"; + +#[derive(Debug, Deserialize)] +struct WechatAttachmentExtras { + wechat_aes_key: Option<String>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct PreparedWechatUpload { + raw_size: u64, + raw_md5: String, + ciphertext_size: u64, + filekey: String, + aes_key_base64: String, + aes_key_hex: String, +} + +pub(crate) async fn hydrate_attachment_for_channel( + host_state: &mut ChannelHostState, + attachment: &mut Attachment, +) { + if !should_hydrate_wechat_attachment(host_state.channel_name(), attachment) { + return; + } + + let Some(source_url) = attachment.source_url.as_deref() else { + return; + }; + let Some(encoded_aes_key) = wechat_aes_key(&attachment.extras_json) else { + tracing::warn!( + channel = %host_state.channel_name(), + attachment_id = %attachment.id, + "Skipping WeChat attachment hydration: missing AES key metadata" + ); + return; + }; + + match download_wechat_attachment_bytes(host_state, source_url).await { + Ok(ciphertext) => match decrypt_wechat_attachment_bytes(&ciphertext, &encoded_aes_key) { + Ok(plaintext) => { + attachment.size_bytes = Some(plaintext.len() as u64); + attachment.data = plaintext; + if attachment.mime_type.starts_with("image/") { + attachment.mime_type = detect_image_mime(&attachment.data).to_string(); + } else if is_wechat_silk_attachment(attachment) + && let Err(error) = maybe_transcode_wechat_silk_attachment(attachment).await + { + tracing::warn!( + channel = %host_state.channel_name(), + attachment_id = %attachment.id, + error = %error, + "Failed to transcode WeChat SILK attachment; preserving raw SILK" + ); + } + } + Err(error) => { + tracing::warn!( + channel = %host_state.channel_name(), + attachment_id = %attachment.id, + error = %error, + "Failed to decrypt WeChat attachment" + ); + } + }, + Err(error) => { + tracing::warn!( + channel = %host_state.channel_name(), + attachment_id = %attachment.id, + error = %error, + "Failed to download WeChat attachment" + ); + } + } +} + +fn is_wechat_silk_attachment(attachment: &Attachment) -> bool { + attachment.mime_type.eq_ignore_ascii_case("audio/silk") + || attachment + .filename + .as_deref() + .and_then(|filename| filename.rsplit_once('.').map(|(_, ext)| ext)) + .is_some_and(|ext| ext.eq_ignore_ascii_case("silk")) +} + +fn should_hydrate_wechat_attachment(channel_name: &str, attachment: &Attachment) -> bool { + channel_name == WECHAT_CHANNEL_NAME + && attachment.data.is_empty() + && attachment.source_url.is_some() +} + +fn wechat_aes_key(extras_json: &str) -> Option<String> { + if extras_json.trim().is_empty() { + return None; + } + + serde_json::from_str::<WechatAttachmentExtras>(extras_json) + .ok() + .and_then(|extras| extras.wechat_aes_key) + .filter(|value| !value.trim().is_empty()) +} + +async fn download_wechat_attachment_bytes( + host_state: &mut ChannelHostState, + source_url: &str, +) -> Result<Vec<u8>, String> { + host_state.check_http_allowed(source_url, "GET")?; + host_state.record_http_request()?; + + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|e| format!("Failed to build HTTP client: {e}"))?; + + let response = client + .get(source_url) + .timeout(Duration::from_secs(15)) + .send() + .await + .map_err(|e| format!("WeChat CDN download failed: {e}"))?; + + if response.status() != reqwest::StatusCode::OK { + return Err(format!( + "WeChat CDN download returned {}", + response.status() + )); + } + if let Some(content_length) = response.content_length() + && content_length > MAX_ATTACHMENT_BYTES as u64 + { + return Err(format!( + "WeChat attachment exceeds {MAX_ATTACHMENT_BYTES} bytes" + )); + } + + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| format!("Failed to read WeChat CDN response body: {e}"))?; + let next_len = bytes.len().saturating_add(chunk.len()); + if next_len > MAX_ATTACHMENT_BYTES { + return Err(format!( + "WeChat attachment exceeds {MAX_ATTACHMENT_BYTES} bytes" + )); + } + bytes.extend_from_slice(&chunk); + } + + if bytes.is_empty() { + return Err("WeChat CDN download returned an empty body".to_string()); + } + if bytes.len() > MAX_ATTACHMENT_BYTES { + return Err(format!( + "WeChat attachment exceeds {MAX_ATTACHMENT_BYTES} bytes" + )); + } + + Ok(bytes) +} + +fn decrypt_wechat_attachment_bytes( + ciphertext: &[u8], + encoded_aes_key: &str, +) -> Result<Vec<u8>, String> { + let key = parse_aes_key(encoded_aes_key)?; + decrypt_aes_ecb_pkcs7(ciphertext, &key) +} + +fn parse_aes_key(encoded: &str) -> Result<Vec<u8>, String> { + let decoded = if encoded.len() == 32 && encoded.bytes().all(|byte| byte.is_ascii_hexdigit()) { + decode_hex(encoded)? + } else { + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| format!("Failed to decode WeChat AES key: {e}"))? + }; + + if decoded.len() == AES_BLOCK_SIZE { + return Ok(decoded); + } + + if decoded.len() == 32 && decoded.iter().all(|byte| byte.is_ascii_hexdigit()) { + return decode_hex( + std::str::from_utf8(&decoded) + .map_err(|e| format!("WeChat AES key hex payload is not valid UTF-8: {e}"))?, + ); + } + + Err(format!( + "WeChat AES key must decode to 16 bytes or a 32-char hex string, got {} bytes", + decoded.len() + )) +} + +pub(crate) fn prepare_outbound_attachment_for_channel( + channel_name: &str, + data: &[u8], +) -> Result<Vec<u8>, String> { + if channel_name != WECHAT_CHANNEL_NAME || data.is_empty() { + return Ok(data.to_vec()); + } + + let prepared = prepare_wechat_outbound_attachment(data)?; + pack_prepared_wechat_upload(&prepared) +} + +fn decode_hex(input: &str) -> Result<Vec<u8>, String> { + if !input.len().is_multiple_of(2) { + return Err("hex input length must be even".to_string()); + } + let mut bytes = Vec::with_capacity(input.len() / 2); + let chars: Vec<u8> = input.as_bytes().to_vec(); + for idx in (0..chars.len()).step_by(2) { + let high = from_hex_digit(chars[idx])?; + let low = from_hex_digit(chars[idx + 1])?; + bytes.push((high << 4) | low); + } + Ok(bytes) +} + +fn from_hex_digit(value: u8) -> Result<u8, String> { + match value { + b'0'..=b'9' => Ok(value - b'0'), + b'a'..=b'f' => Ok(value - b'a' + 10), + b'A'..=b'F' => Ok(value - b'A' + 10), + _ => Err(format!("invalid hex digit '{}'", value as char)), + } +} + +fn decrypt_aes_ecb_pkcs7(ciphertext: &[u8], key: &[u8]) -> Result<Vec<u8>, String> { + if !ciphertext.len().is_multiple_of(AES_BLOCK_SIZE) { + return Err("ciphertext length is not a multiple of 16 bytes".to_string()); + } + + let cipher = Aes128::new_from_slice(key).map_err(|e| format!("Invalid AES key: {e}"))?; + let mut plaintext = ciphertext.to_vec(); + for chunk in plaintext.chunks_exact_mut(AES_BLOCK_SIZE) { + cipher.decrypt_block(GenericArray::from_mut_slice(chunk)); + } + + let pad_len = *plaintext + .last() + .ok_or_else(|| "ciphertext decrypted to an empty buffer".to_string())? + as usize; + if pad_len == 0 || pad_len > AES_BLOCK_SIZE || pad_len > plaintext.len() { + return Err("invalid PKCS7 padding".to_string()); + } + if !plaintext[plaintext.len() - pad_len..] + .iter() + .all(|byte| *byte as usize == pad_len) + { + return Err("invalid PKCS7 padding bytes".to_string()); + } + plaintext.truncate(plaintext.len() - pad_len); + Ok(plaintext) +} + +fn prepare_wechat_outbound_attachment( + data: &[u8], +) -> Result<(PreparedWechatUpload, Vec<u8>), String> { + let raw_size = data.len() as u64; + let raw_md5 = encode_hex(&Md5::digest(data)).to_ascii_lowercase(); + let ciphertext_size = padded_size(raw_size); + let filekey = encode_hex(&random_bytes(16)?).to_ascii_lowercase(); + let aes_key = random_bytes(16)?; + let aes_key_hex = encode_hex(&aes_key).to_ascii_lowercase(); + let aes_key_base64 = base64::engine::general_purpose::STANDARD.encode(&aes_key); + let ciphertext = encrypt_aes_ecb_pkcs7(data, &aes_key)?; + if ciphertext.len() as u64 != ciphertext_size { + return Err(format!( + "WeChat outbound ciphertext size mismatch: expected={} actual={}", + ciphertext_size, + ciphertext.len() + )); + } + + Ok(( + PreparedWechatUpload { + raw_size, + raw_md5, + ciphertext_size, + filekey, + aes_key_base64, + aes_key_hex, + }, + ciphertext, + )) +} + +fn pack_prepared_wechat_upload( + prepared: &(PreparedWechatUpload, Vec<u8>), +) -> Result<Vec<u8>, String> { + let metadata_json = serde_json::to_vec(&prepared.0) + .map_err(|e| format!("Failed to serialize WeChat outbound attachment metadata: {e}"))?; + let metadata_len = u32::try_from(metadata_json.len()) + .map_err(|_| "WeChat outbound attachment metadata exceeds 4 GiB".to_string())?; + + let mut packed = Vec::with_capacity( + WECHAT_OUTBOUND_ENVELOPE_MAGIC.len() + 4 + metadata_json.len() + prepared.1.len(), + ); + packed.extend_from_slice(WECHAT_OUTBOUND_ENVELOPE_MAGIC); + packed.extend_from_slice(&metadata_len.to_le_bytes()); + packed.extend_from_slice(&metadata_json); + packed.extend_from_slice(&prepared.1); + Ok(packed) +} + +#[cfg(test)] +fn unpack_prepared_wechat_upload( + data: &[u8], +) -> Result<Option<(PreparedWechatUpload, Vec<u8>)>, String> { + if !data.starts_with(WECHAT_OUTBOUND_ENVELOPE_MAGIC) { + return Ok(None); + } + + let header_len = WECHAT_OUTBOUND_ENVELOPE_MAGIC.len(); + if data.len() < header_len + 4 { + return Err("WeChat outbound attachment envelope is truncated".to_string()); + } + + let metadata_len = u32::from_le_bytes( + data[header_len..header_len + 4] + .try_into() + .map_err(|_| "Failed to decode WeChat outbound metadata length".to_string())?, + ) as usize; + let metadata_start = header_len + 4; + let metadata_end = metadata_start.saturating_add(metadata_len); + if metadata_end > data.len() { + return Err("WeChat outbound attachment envelope metadata is truncated".to_string()); + } + + let metadata = + serde_json::from_slice::<PreparedWechatUpload>(&data[metadata_start..metadata_end]) + .map_err(|e| format!("Failed to parse WeChat outbound attachment metadata: {e}"))?; + let ciphertext = data[metadata_end..].to_vec(); + if metadata.ciphertext_size != ciphertext.len() as u64 { + return Err(format!( + "WeChat outbound attachment ciphertext size mismatch: metadata={} actual={}", + metadata.ciphertext_size, + ciphertext.len() + )); + } + + Ok(Some((metadata, ciphertext))) +} + +fn detect_image_mime(bytes: &[u8]) -> &'static str { + if bytes.starts_with(&[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A]) { + "image/png" + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + "image/jpeg" + } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + "image/gif" + } else if bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + "image/webp" + } else { + "image/jpeg" + } +} + +async fn maybe_transcode_wechat_silk_attachment(attachment: &mut Attachment) -> Result<(), String> { + if attachment.data.is_empty() { + return Err("SILK attachment has no data".to_string()); + } + let decoder_path = resolve_silk_decoder_command().ok_or_else(|| { + format!( + "{SILK_DECODER_BIN_NAME} not found (set {SILK_DECODER_ENV_VAR}, install on PATH, or place beside the ironclaw binary)" + ) + })?; + + let wav = run_silk_decoder(&decoder_path, &attachment.data, WECHAT_SILK_SAMPLE_RATE_HZ).await?; + if wav.is_empty() { + return Err("SILK decoder returned empty WAV".to_string()); + } + if !wav.starts_with(b"RIFF") { + return Err("SILK decoder did not produce a RIFF/WAVE stream".to_string()); + } + + attachment.size_bytes = Some(wav.len() as u64); + attachment.data = wav; + attachment.mime_type = "audio/wav".to_string(); + if let Some(filename) = attachment.filename.as_mut() { + replace_attachment_extension(filename, "wav"); + } + Ok(()) +} + +/// Locate the optional SILK decoder helper binary. Lookup order: +/// +/// 1. `IRONCLAW_SILK_DECODER` env var, used verbatim as a path. +/// 2. Sibling of the running executable (`<exe-dir>/ironclaw-silk-decoder[.exe]`). +/// 3. Bare `ironclaw-silk-decoder` for `$PATH` resolution by `Command`. +/// +/// Returns `None` only when no candidate looks viable; callers fall back to +/// preserving raw SILK and logging that the decoder is not configured. +fn resolve_silk_decoder_command() -> Option<OsString> { + if let Some(path) = std::env::var_os(SILK_DECODER_ENV_VAR) + && !path.is_empty() + { + return Some(path); + } + + if let Ok(current_exe) = std::env::current_exe() + && let Some(parent) = current_exe.parent() + { + let mut candidate: PathBuf = parent.to_path_buf(); + if cfg!(windows) { + candidate.push(format!("{SILK_DECODER_BIN_NAME}.exe")); + } else { + candidate.push(SILK_DECODER_BIN_NAME); + } + if candidate.is_file() { + return Some(candidate.into_os_string()); + } + } + + Some(OsString::from(SILK_DECODER_BIN_NAME)) +} + +async fn run_silk_decoder( + program: &OsString, + silk_bytes: &[u8], + sample_rate_hz: u32, +) -> Result<Vec<u8>, String> { + let mut command = Command::new(program); + command + .arg("--sample-rate") + .arg(sample_rate_hz.to_string()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + let mut child = command.spawn().map_err(|e| { + format!("failed to spawn {SILK_DECODER_BIN_NAME}: {e} (is the helper binary installed?)") + })?; + + let mut stdin = child + .stdin + .take() + .ok_or_else(|| format!("failed to capture stdin for {SILK_DECODER_BIN_NAME}"))?; + let input = silk_bytes.to_vec(); + let writer = tokio::spawn(async move { + stdin + .write_all(&input) + .await + .map_err(|e| format!("failed to send SILK bytes to decoder: {e}"))?; + stdin + .shutdown() + .await + .map_err(|e| format!("failed to close decoder stdin: {e}")) + }); + + let output_future = child.wait_with_output(); + let output = match tokio::time::timeout(SILK_DECODER_TIMEOUT, output_future).await { + Ok(Ok(output)) => output, + Ok(Err(error)) => return Err(format!("{SILK_DECODER_BIN_NAME} failed: {error}")), + Err(_) => { + return Err(format!( + "{SILK_DECODER_BIN_NAME} timed out after {}s", + SILK_DECODER_TIMEOUT.as_secs() + )); + } + }; + writer + .await + .map_err(|e| format!("decoder stdin task panicked: {e}"))??; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stderr = stderr.trim(); + return Err(format!( + "{SILK_DECODER_BIN_NAME} exited with {} (stderr: {})", + output + .status + .code() + .map(|c| c.to_string()) + .unwrap_or_else(|| "signal".to_string()), + if stderr.is_empty() { "<empty>" } else { stderr } + )); + } + + if output.stdout.len() > MAX_DECODED_WAV_BYTES { + return Err(format!( + "{SILK_DECODER_BIN_NAME} produced {} bytes, exceeds {MAX_DECODED_WAV_BYTES} cap", + output.stdout.len() + )); + } + Ok(output.stdout) +} + +fn replace_attachment_extension(filename: &mut String, replacement: &str) { + if let Some((stem, _)) = filename.rsplit_once('.') { + *filename = format!("{stem}.{replacement}"); + } else { + filename.push('.'); + filename.push_str(replacement); + } +} + +fn encrypt_aes_ecb_pkcs7(plaintext: &[u8], key: &[u8]) -> Result<Vec<u8>, String> { + // WeChat's CDN upload protocol requires AES-128-ECB with PKCS#7 padding for + // outbound media payloads. This is compatibility logic for that protocol, + // not a general recommendation for new encryption schemes. + let cipher = Aes128::new_from_slice(key).map_err(|e| format!("Invalid AES key: {e}"))?; + let mut padded = plaintext.to_vec(); + let pad_len = AES_BLOCK_SIZE - (padded.len() % AES_BLOCK_SIZE); + padded.extend(std::iter::repeat_n(pad_len as u8, pad_len)); + + for chunk in padded.chunks_exact_mut(AES_BLOCK_SIZE) { + cipher.encrypt_block(GenericArray::from_mut_slice(chunk)); + } + + Ok(padded) +} + +fn encode_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(nibble_to_hex(byte >> 4)); + out.push(nibble_to_hex(byte & 0x0F)); + } + out +} + +fn nibble_to_hex(nibble: u8) -> char { + match nibble { + 0..=9 => (b'0' + nibble) as char, + 10..=15 => (b'A' + (nibble - 10)) as char, + _ => '0', + } +} + +fn padded_size(raw_size: u64) -> u64 { + ((raw_size / AES_BLOCK_SIZE as u64) + 1) * AES_BLOCK_SIZE as u64 +} + +fn random_bytes(len: usize) -> Result<Vec<u8>, String> { + let mut bytes = vec![0u8; len]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + if bytes.iter().all(|byte| *byte == 0) { + return Err("OS RNG returned all-zero bytes unexpectedly".to_string()); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::{ + AES_BLOCK_SIZE, Attachment, MAX_DECODED_WAV_BYTES, SILK_DECODER_BIN_NAME, + SILK_DECODER_ENV_VAR, decrypt_wechat_attachment_bytes, detect_image_mime, + encrypt_aes_ecb_pkcs7, hydrate_attachment_for_channel, + maybe_transcode_wechat_silk_attachment, prepare_outbound_attachment_for_channel, + resolve_silk_decoder_command, should_hydrate_wechat_attachment, + unpack_prepared_wechat_upload, + }; + use crate::channels::wasm::{ChannelCapabilities, ChannelHostState}; + use crate::tools::wasm::{Capabilities, EndpointPattern, HttpCapability}; + use base64::Engine as _; + + fn make_attachment() -> Attachment { + Attachment { + id: "wechat-image-1".to_string(), + mime_type: "image/jpeg".to_string(), + filename: Some("wechat-image.jpg".to_string()), + size_bytes: None, + source_url: Some( + "https://novac2c.cdn.weixin.qq.com/c2c/download?encrypted_query_param=test" + .to_string(), + ), + storage_key: None, + local_path: None, + extracted_text: None, + extras_json: String::new(), + data: Vec::new(), + duration_secs: None, + } + } + + fn encode_test_extras_json(aes_key: &str) -> String { + serde_json::json!({ "wechat_aes_key": aes_key }).to_string() + } + + #[test] + fn decrypt_wechat_image_bytes_round_trips() { + let key = [7u8; 16]; + let plaintext = vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x11]; + let ciphertext = encrypt_aes_ecb_pkcs7(&plaintext, &key).unwrap(); + let encoded_key = base64::engine::general_purpose::STANDARD.encode(key); + let decrypted = decrypt_wechat_attachment_bytes(&ciphertext, &encoded_key).unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn wechat_outbound_attachment_preparation_round_trips() { + let plaintext = b"wechat outbound image".to_vec(); + let packed = + prepare_outbound_attachment_for_channel("wechat", &plaintext).expect("prepare"); + assert_ne!(packed, plaintext); + + let (metadata, ciphertext) = unpack_prepared_wechat_upload(&packed) + .expect("parse envelope") + .expect("wechat envelope"); + assert_eq!(metadata.raw_size, plaintext.len() as u64); + assert_eq!(metadata.ciphertext_size, ciphertext.len() as u64); + assert_eq!(metadata.ciphertext_size % AES_BLOCK_SIZE as u64, 0); + + let decrypted = decrypt_wechat_attachment_bytes(&ciphertext, &metadata.aes_key_base64) + .expect("decrypt host-prepared ciphertext"); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn non_wechat_outbound_attachment_preparation_is_passthrough() { + let plaintext = b"plain attachment".to_vec(); + let prepared = + prepare_outbound_attachment_for_channel("telegram", &plaintext).expect("prepare"); + assert_eq!(prepared, plaintext); + } + + #[test] + fn detect_image_mime_prefers_magic_bytes() { + assert_eq!(detect_image_mime(&[0xFF, 0xD8, 0xFF, 0x00]), "image/jpeg"); + assert_eq!( + detect_image_mime(&[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A]), + "image/png" + ); + } + + #[test] + fn wechat_attachment_hydration_applies_to_wechat_encrypted_media() { + let mut attachment = make_attachment(); + attachment.extras_json = encode_test_extras_json("ZmFrZS1rZXk="); + assert!(should_hydrate_wechat_attachment("wechat", &attachment)); + assert!(!should_hydrate_wechat_attachment("telegram", &attachment)); + + attachment.mime_type = "application/pdf".to_string(); + assert!(should_hydrate_wechat_attachment("wechat", &attachment)); + } + + #[tokio::test] + async fn hydration_skips_when_metadata_is_missing() { + let mut attachment = make_attachment(); + let caps = ChannelCapabilities::for_channel("wechat"); + let mut host_state = ChannelHostState::new("wechat", caps); + hydrate_attachment_for_channel(&mut host_state, &mut attachment).await; + assert!(attachment.data.is_empty()); + assert_eq!(attachment.size_bytes, None); + } + + #[test] + fn wechat_attachment_downloads_consume_host_http_budget() { + let caps = ChannelCapabilities::for_channel("wechat").with_tool_capabilities( + Capabilities::default().with_http(HttpCapability::new(vec![ + EndpointPattern::host("novac2c.cdn.weixin.qq.com") + .with_path_prefix("/c2c/download") + .with_methods(vec!["GET".to_string()]), + ])), + ); + let mut host_state = ChannelHostState::new("wechat", caps); + let url = "https://novac2c.cdn.weixin.qq.com/c2c/download?encrypted_query_param=test"; + + for _ in 0..50 { + host_state + .check_http_allowed(url, "GET") + .expect("allowlisted request"); + host_state + .record_http_request() + .expect("request budget available"); + } + + let error = host_state + .record_http_request() + .expect_err("51st request should exceed per-execution budget"); + assert!(error.contains("Too many HTTP requests in single execution")); + } + + #[test] + fn resolve_silk_decoder_command_prefers_env_var() { + // Serialize against any other env-mutating test in the workspace. + // Without this, the cargo-test default thread pool can interleave + // this test with the EnvGuard-protected stub-binary tests below, + // producing flaky results. + let _env_lock = crate::config::helpers::lock_env(); + let previous = std::env::var_os(SILK_DECODER_ENV_VAR); + // SAFETY: env-mutation under the global env-test lock; cargo-test + // workers contending on this var are blocked until the guard drops. + unsafe { + std::env::set_var(SILK_DECODER_ENV_VAR, "/opt/custom/decoder"); + } + let resolved = resolve_silk_decoder_command(); + // SAFETY: restore prior state before releasing the lock. + unsafe { + match previous { + Some(value) => std::env::set_var(SILK_DECODER_ENV_VAR, value), + None => std::env::remove_var(SILK_DECODER_ENV_VAR), + } + } + assert_eq!( + resolved, + Some(std::ffi::OsString::from("/opt/custom/decoder")) + ); + } + + #[test] + fn resolve_silk_decoder_command_falls_back_to_path_lookup() { + let _env_lock = crate::config::helpers::lock_env(); + let previous = std::env::var_os(SILK_DECODER_ENV_VAR); + // SAFETY: env-mutation under the global env-test lock. + unsafe { + std::env::remove_var(SILK_DECODER_ENV_VAR); + } + let resolved = resolve_silk_decoder_command(); + // SAFETY: restore prior state before releasing the lock. + unsafe { + if let Some(value) = previous { + std::env::set_var(SILK_DECODER_ENV_VAR, value); + } + } + // Without env var and (almost certainly) no sibling binary in the + // cargo-test runner directory, the resolver should still hand back + // the bare program name for $PATH resolution. This guarantees + // graceful "binary not installed" handling rather than `None`. + let value = resolved.expect("resolver should always offer a candidate"); + assert!( + value + .to_str() + .is_some_and(|s| s.ends_with(SILK_DECODER_BIN_NAME)) + ); + } + + #[cfg(unix)] + fn write_unix_stub(dir: &std::path::Path, name: &str, body: &str) -> std::path::PathBuf { + use std::io::Write as _; + use std::os::unix::fs::PermissionsExt as _; + + let path = dir.join(name); + let mut file = std::fs::File::create(&path).expect("create stub"); + write!(file, "#!/usr/bin/env bash\nset -eu\n{body}\n").expect("write stub"); + let mut perms = std::fs::metadata(&path).expect("stat stub").permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms).expect("chmod stub"); + path + } + + #[cfg(unix)] + struct EnvGuard { + key: &'static str, + prior: Option<std::ffi::OsString>, + // Serializes env-var mutation across cargo-test's parallel thread + // pool. Held until Drop so the test that set the var also owns the + // process-wide env state for its full duration. Released last so + // the restoration writes happen before another test takes the lock. + _env_lock: std::sync::MutexGuard<'static, ()>, + } + + #[cfg(unix)] + impl EnvGuard { + fn set(key: &'static str, value: &std::path::Path) -> Self { + let env_lock = crate::config::helpers::lock_env(); + let prior = std::env::var_os(key); + // SAFETY: env-mutation under the global env-test lock — held in + // the returned guard for the full lifetime of the test. + unsafe { + std::env::set_var(key, value); + } + Self { + key, + prior, + _env_lock: env_lock, + } + } + } + + #[cfg(unix)] + impl Drop for EnvGuard { + fn drop(&mut self) { + // SAFETY: see EnvGuard::set; lock is still held via _env_lock. + unsafe { + match self.prior.take() { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } + } + } + + fn fake_silk_attachment() -> Attachment { + Attachment { + id: "wechat-voice-1".to_string(), + mime_type: "audio/silk".to_string(), + filename: Some("wechat-voice-1.silk".to_string()), + size_bytes: Some(3), + source_url: None, + storage_key: None, + local_path: None, + extracted_text: None, + extras_json: encode_test_extras_json("ZmFrZS1rZXk="), + data: vec![1, 2, 3], + duration_secs: Some(1), + } + } + + #[cfg(unix)] + #[tokio::test] + async fn silk_transcoder_consumes_decoder_output_and_updates_attachment() { + let temp = tempfile::tempdir().expect("tempdir"); + // Stub: emit a minimal valid RIFF/WAVE header (44 bytes is the well-formed + // empty WAV). Discards stdin to keep the pipe alive for the writer task. + let body = "cat >/dev/null\n\ + printf 'RIFF\\x24\\x00\\x00\\x00WAVEfmt \\x10\\x00\\x00\\x00\\x01\\x00\\x01\\x00\\xc0]\\x00\\x00\\x80\\xbb\\x00\\x00\\x02\\x00\\x10\\x00data\\x00\\x00\\x00\\x00'"; + let stub = write_unix_stub(temp.path(), "stub-silk-decoder", body); + let _guard = EnvGuard::set(SILK_DECODER_ENV_VAR, &stub); + + let mut attachment = fake_silk_attachment(); + maybe_transcode_wechat_silk_attachment(&mut attachment) + .await + .expect("stub decoder should succeed"); + + assert_eq!(attachment.mime_type, "audio/wav"); + assert_eq!(attachment.filename.as_deref(), Some("wechat-voice-1.wav")); + assert!(attachment.data.starts_with(b"RIFF")); + assert_eq!(attachment.size_bytes, Some(attachment.data.len() as u64)); + } + + #[cfg(unix)] + #[tokio::test] + async fn silk_transcoder_propagates_decoder_failure_for_caller_fallback() { + let temp = tempfile::tempdir().expect("tempdir"); + let body = "cat >/dev/null\n\ + echo 'fake decoder failure' >&2\n\ + exit 3"; + let stub = write_unix_stub(temp.path(), "stub-silk-decoder", body); + let _guard = EnvGuard::set(SILK_DECODER_ENV_VAR, &stub); + + let mut attachment = fake_silk_attachment(); + let original = attachment.data.clone(); + + let error = maybe_transcode_wechat_silk_attachment(&mut attachment) + .await + .expect_err("stub decoder failure should bubble"); + assert!( + error.contains(SILK_DECODER_BIN_NAME), + "error mentions decoder name: {error}" + ); + assert!( + error.contains("fake decoder failure"), + "error includes captured stderr: {error}" + ); + + // Caller-level invariant: the attachment must still be raw SILK so + // hydrate_attachment_for_channel's outer warn-and-preserve branch is + // valid. Regression coverage for the fallback path. + assert_eq!(attachment.mime_type, "audio/silk"); + assert_eq!(attachment.filename.as_deref(), Some("wechat-voice-1.silk")); + assert_eq!(attachment.data, original); + } + + #[cfg(unix)] + #[tokio::test] + async fn silk_transcoder_rejects_non_riff_output() { + let temp = tempfile::tempdir().expect("tempdir"); + let body = "cat >/dev/null\n\ + printf 'NOT_A_WAV'"; + let stub = write_unix_stub(temp.path(), "stub-silk-decoder", body); + let _guard = EnvGuard::set(SILK_DECODER_ENV_VAR, &stub); + + let mut attachment = fake_silk_attachment(); + let error = maybe_transcode_wechat_silk_attachment(&mut attachment) + .await + .expect_err("non-RIFF output should be rejected"); + assert!(error.contains("RIFF"), "error mentions RIFF: {error}"); + assert_eq!(attachment.mime_type, "audio/silk"); + } + + #[tokio::test] + async fn silk_transcoder_errors_when_attachment_data_empty() { + let mut attachment = fake_silk_attachment(); + attachment.data = Vec::new(); + let error = maybe_transcode_wechat_silk_attachment(&mut attachment) + .await + .expect_err("empty input should not invoke the decoder"); + assert!(error.contains("no data"), "error: {error}"); + } + + // Compile-time check that the decoded-WAV cap is at least as large as + // the inbound-attachment cap. SILK→PCM expansion is ~25× at 24 kHz + // mono; 60 s of voice is ~3 MiB. A 50 MiB output cap leaves generous + // headroom over the 20 MiB input cap. const_assert form keeps the + // invariant near the constants without spending a runtime test slot. + const _: () = assert!(MAX_DECODED_WAV_BYTES >= super::MAX_ATTACHMENT_BYTES); +} diff --git a/src/channels/wasm/bundled.rs b/src/channels/wasm/bundled.rs index 60fe8f4d1fb..4d80fcb35ca 100644 --- a/src/channels/wasm/bundled.rs +++ b/src/channels/wasm/bundled.rs @@ -23,6 +23,8 @@ const KNOWN_CHANNELS: &[(&str, &str)] = &[ ("discord", "discord_channel"), ("whatsapp", "whatsapp_channel"), ("feishu", "feishu_channel"), + ("wecom", "wecom_channel"), + ("wechat", "wechat_channel"), ]; /// Names of known channels that can be installed. @@ -143,12 +145,14 @@ mod tests { use super::*; #[test] - fn test_known_channels_includes_all_four() { + fn test_known_channels_include_supported_bundles() { let names = bundled_channel_names(); assert!(names.contains(&"telegram")); assert!(names.contains(&"slack")); assert!(names.contains(&"discord")); assert!(names.contains(&"whatsapp")); + assert!(names.contains(&"feishu")); + assert!(names.contains(&"wechat")); } #[test] diff --git a/src/channels/wasm/host.rs b/src/channels/wasm/host.rs index fbec720d84b..524c18f83db 100644 --- a/src/channels/wasm/host.rs +++ b/src/channels/wasm/host.rs @@ -37,6 +37,8 @@ pub struct Attachment { pub local_path: Option<String>, /// Extracted text content (e.g., OCR result, PDF text, audio transcript). pub extracted_text: Option<String>, + /// Extensible metadata from the channel payload. + pub extras_json: String, /// Raw file bytes (for small files downloaded by the channel). pub data: Vec<u8>, /// Duration in seconds (for audio/video). @@ -620,6 +622,52 @@ impl ChannelWorkspaceStore { data.insert(dest_path.to_string(), raw_queue); Ok(true) } + + /// Restore frames from a processing queue back to the live queue. + /// + /// Processing frames are prepended before any newly arrived live frames so + /// retries preserve original websocket delivery order as much as possible. + pub fn restore_json_text_queue( + &self, + source_path: &str, + dest_path: &str, + max_items: usize, + ) -> Result<bool, String> { + let mut data = self + .data + .write() + .map_err(|_| "workspace store lock poisoned".to_string())?; + + let Some(raw_processing_queue) = data.get(dest_path).cloned() else { + return Ok(false); + }; + + let mut restored: Vec<String> = + serde_json::from_str(&raw_processing_queue).map_err(|error| { + format!("failed to deserialize websocket processing queue: {error}") + })?; + let mut live: Vec<String> = data + .get(source_path) + .and_then(|raw| serde_json::from_str(raw).ok()) + .unwrap_or_default(); + + restored.append(&mut live); + if restored.len() > max_items { + restored.truncate(max_items); + } + + if restored.is_empty() { + data.remove(source_path); + } else { + let serialized = serde_json::to_string(&restored).map_err(|error| { + format!("failed to serialize restored websocket queue: {error}") + })?; + data.insert(source_path.to_string(), serialized); + } + data.remove(dest_path); + + Ok(true) + } } impl crate::tools::wasm::WorkspaceReader for ChannelWorkspaceStore { @@ -944,6 +992,83 @@ mod tests { assert_eq!(live, vec!["frame-3".to_string()]); } + #[test] + fn test_channel_workspace_store_restore_json_text_queue_retries_processing_first() { + use crate::channels::wasm::host::ChannelWorkspaceStore; + use crate::tools::wasm::WorkspaceReader; + + let store = ChannelWorkspaceStore::new(); + let live_path = "channels/discord/state/gateway_event_queue"; + let drain_path = "channels/discord/state/gateway_event_queue_processing"; + + store + .append_json_text_queue(live_path, "frame-1", 4) + .unwrap(); + store + .append_json_text_queue(live_path, "frame-2", 4) + .unwrap(); + assert!(store.move_json_text_queue(live_path, drain_path).unwrap()); + + store + .append_json_text_queue(live_path, "frame-3", 4) + .unwrap(); + + assert!( + store + .restore_json_text_queue(live_path, drain_path, 4) + .unwrap() + ); + assert_eq!(store.read(drain_path), None); + + let restored: Vec<String> = serde_json::from_str(&store.read(live_path).unwrap()).unwrap(); + assert_eq!( + restored, + vec![ + "frame-1".to_string(), + "frame-2".to_string(), + "frame-3".to_string() + ] + ); + } + + #[test] + fn test_channel_workspace_store_restore_json_text_queue_drops_live_tail_on_overflow() { + use crate::channels::wasm::host::ChannelWorkspaceStore; + use crate::tools::wasm::WorkspaceReader; + + let store = ChannelWorkspaceStore::new(); + let live_path = "channels/discord/state/gateway_event_queue"; + let drain_path = "channels/discord/state/gateway_event_queue_processing"; + + store + .append_json_text_queue(live_path, "failed-1", 10) + .unwrap(); + store + .append_json_text_queue(live_path, "failed-2", 10) + .unwrap(); + assert!(store.move_json_text_queue(live_path, drain_path).unwrap()); + + for live in ["live-1", "live-2", "live-3"] { + store.append_json_text_queue(live_path, live, 10).unwrap(); + } + + assert!( + store + .restore_json_text_queue(live_path, drain_path, 3) + .unwrap() + ); + + let restored: Vec<String> = serde_json::from_str(&store.read(live_path).unwrap()).unwrap(); + assert_eq!( + restored, + vec![ + "failed-1".to_string(), + "failed-2".to_string(), + "live-1".to_string() + ] + ); + } + // === QA Plan P2 - 2.3: WASM channel lifecycle tests === #[test] @@ -1122,6 +1247,7 @@ mod tests { storage_key: None, local_path: None, extracted_text: None, + extras_json: String::new(), data: Vec::new(), duration_secs: None, } diff --git a/src/channels/wasm/loader.rs b/src/channels/wasm/loader.rs index 4ca408733c5..35b17a0eb4e 100644 --- a/src/channels/wasm/loader.rs +++ b/src/channels/wasm/loader.rs @@ -327,6 +327,14 @@ impl LoadedChannel { .map(|f| f.webhook_secret_managed_by_host()) .unwrap_or(true) } + + /// Get the HTTP methods the host should expose for the default webhook. + pub fn webhook_methods(&self) -> Vec<String> { + self.capabilities_file + .as_ref() + .map(|f| f.webhook_methods()) + .unwrap_or_else(|| vec!["POST".to_string()]) + } } /// Results from loading multiple channels. diff --git a/src/channels/wasm/mod.rs b/src/channels/wasm/mod.rs index bf23ad1fec9..ced28b837ff 100644 --- a/src/channels/wasm/mod.rs +++ b/src/channels/wasm/mod.rs @@ -78,6 +78,7 @@ //! } //! ``` +mod attachment_hydration; mod bundled; mod capabilities; mod error; diff --git a/src/channels/wasm/router.rs b/src/channels/wasm/router.rs index 12d8a7ce19d..f7e928a488e 100644 --- a/src/channels/wasm/router.rs +++ b/src/channels/wasm/router.rs @@ -32,12 +32,31 @@ pub struct RegisteredEndpoint { pub require_secret: bool, } +fn normalize_registered_methods(methods: &[String]) -> Vec<String> { + let mut normalized = Vec::new(); + for method in methods { + let upper = method.trim().to_ascii_uppercase(); + if !matches!(upper.as_str(), "GET" | "POST") || normalized.contains(&upper) { + continue; + } + normalized.push(upper); + } + + if normalized.is_empty() { + normalized.push("POST".to_string()); + } + + normalized +} + /// Router for WASM channel HTTP endpoints. pub struct WasmChannelRouter { /// Registered channels by name. channels: RwLock<HashMap<String, Arc<WasmChannel>>>, /// Path to channel mapping for fast lookup. path_to_channel: RwLock<HashMap<String, String>>, + /// Allowed HTTP methods by registered path. + path_methods: RwLock<HashMap<String, Vec<String>>>, /// Expected webhook secrets by channel name. secrets: RwLock<HashMap<String, String>>, /// Webhook secret header names by channel name (e.g., "X-Telegram-Bot-Api-Secret-Token"). @@ -54,6 +73,7 @@ impl WasmChannelRouter { Self { channels: RwLock::new(HashMap::new()), path_to_channel: RwLock::new(HashMap::new()), + path_methods: RwLock::new(HashMap::new()), secrets: RwLock::new(HashMap::new()), secret_headers: RwLock::new(HashMap::new()), signature_keys: RwLock::new(HashMap::new()), @@ -83,12 +103,30 @@ impl WasmChannelRouter { // Register path mappings let mut path_map = self.path_to_channel.write().await; + let mut method_map = self.path_methods.write().await; + let previous_paths: Vec<String> = path_map + .iter() + .filter_map(|(path, channel_name)| { + if channel_name == &name { + Some(path.clone()) + } else { + None + } + }) + .collect(); + for path in previous_paths { + path_map.remove(&path); + method_map.remove(&path); + } + for endpoint in endpoints { + let methods = normalize_registered_methods(&endpoint.methods); path_map.insert(endpoint.path.clone(), name.clone()); + method_map.insert(endpoint.path.clone(), methods.clone()); tracing::info!( channel = %name, path = %endpoint.path, - methods = ?endpoint.methods, + methods = ?methods, "Registered WASM channel HTTP endpoint" ); } @@ -139,11 +177,28 @@ impl WasmChannelRouter { self.signature_keys.write().await.remove(channel_name); self.hmac_secrets.write().await.remove(channel_name); - // Remove all paths for this channel - self.path_to_channel - .write() - .await - .retain(|_, name| name != channel_name); + // Remove all paths and method metadata for this channel. + let removed_paths = { + let mut path_map = self.path_to_channel.write().await; + let paths: Vec<String> = path_map + .iter() + .filter_map(|(path, name)| { + if name.as_str() == channel_name { + Some(path.clone()) + } else { + None + } + }) + .collect(); + for path in &paths { + path_map.remove(path); + } + paths + }; + let mut method_map = self.path_methods.write().await; + for path in removed_paths { + method_map.remove(&path); + } tracing::info!( channel = %channel_name, @@ -159,6 +214,20 @@ impl WasmChannelRouter { self.channels.read().await.get(channel_name).cloned() } + /// Whether a registered path allows the provided HTTP method. + pub async fn method_allowed_for_path(&self, path: &str, method: &Method) -> bool { + self.path_methods + .read() + .await + .get(path) + .is_some_and(|methods| methods.iter().any(|m| m == method.as_str())) + } + + /// Get a registered channel directly by name. + pub async fn get_channel_by_name(&self, channel_name: &str) -> Option<Arc<WasmChannel>> { + self.channels.read().await.get(channel_name).cloned() + } + /// Validate a secret for a channel. pub async fn validate_secret(&self, channel_name: &str, provided: &str) -> bool { let secrets = self.secrets.read().await; @@ -326,6 +395,26 @@ async fn webhook_handler( } }; + if !state + .router + .method_allowed_for_path(&full_path, &method) + .await + { + tracing::warn!( + path = %full_path, + method = %method, + "Webhook method not allowed for registered WASM channel endpoint" + ); + return ( + StatusCode::METHOD_NOT_ALLOWED, + Json(serde_json::json!({ + "error": "Method not allowed for channel path", + "path": full_path, + "method": method.as_str() + })), + ); + } + tracing::info!( channel = %channel.channel_name(), "Found channel for webhook" @@ -710,6 +799,46 @@ mod tests { // Should not find non-existent path let not_found = router.get_channel_for_path("/webhook/telegram").await; assert!(not_found.is_none()); + + let found_by_name = router.get_channel_by_name("slack").await; + assert!(found_by_name.is_some()); + assert_eq!(found_by_name.unwrap().channel_name(), "slack"); + } + + #[tokio::test] + async fn test_router_tracks_allowed_methods_per_path() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("wecom"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "wecom".to_string(), + path: "/webhook/wecom".to_string(), + methods: vec![ + "post".to_string(), + "GET".to_string(), + "TRACE".to_string(), + "POST".to_string(), + ], + require_secret: false, + }]; + + router.register(channel, endpoints, None, None).await; + + assert!( + router + .method_allowed_for_path("/webhook/wecom", &axum::http::Method::POST) + .await + ); + assert!( + router + .method_allowed_for_path("/webhook/wecom", &axum::http::Method::GET) + .await + ); + assert!( + !router + .method_allowed_for_path("/webhook/wecom", &axum::http::Method::TRACE) + .await + ); } #[tokio::test] @@ -1400,6 +1529,20 @@ mod tests { (wasm_router, app) } + #[tokio::test] + async fn test_webhook_rejects_get_for_post_only_endpoint() { + let (_wasm_router, app) = setup_slack_router().await; + + let req = Request::builder() + .method("GET") + .uri("/webhook/slack") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + } + /// Helper: compute expected Slack signature for testing. fn slack_signature(signing_secret: &str, timestamp: &str, body: &[u8]) -> String { use hmac::{Hmac, Mac}; diff --git a/src/channels/wasm/schema.rs b/src/channels/wasm/schema.rs index 264248a84d2..1fbe37e1e35 100644 --- a/src/channels/wasm/schema.rs +++ b/src/channels/wasm/schema.rs @@ -147,6 +147,13 @@ impl ChannelCapabilitiesFile { serde_json::to_string(&self.config).unwrap_or_else(|_| "{}".to_string()) } + /// Whether this channel declares owner/pairing gating in its config. + pub fn requires_binding(&self) -> bool { + ["owner_id", "dm_policy", "allow_from"] + .iter() + .any(|key| self.config.contains_key(*key)) + } + /// Get the webhook secret header name for this channel. /// /// Returns the configured header name from capabilities, or a sensible default. @@ -207,6 +214,22 @@ impl ChannelCapabilitiesFile { .unwrap_or(true) } + /// Get the HTTP methods exposed by the host webhook router. + /// + /// Defaults to POST and only keeps methods implemented by the generic + /// WASM webhook router. This prevents a capabilities file from widening + /// the effective method surface through typos or unsupported methods. + pub fn webhook_methods(&self) -> Vec<String> { + let methods = self + .capabilities + .channel + .as_ref() + .and_then(|c| c.webhook.as_ref()) + .map(|w| w.methods.as_slice()) + .unwrap_or_default(); + normalize_webhook_methods(methods) + } + /// Return setup.secret_config_mappings after applying security constraints. /// /// Only mappings for declared `setup.required_secrets` are allowed, and @@ -292,6 +315,23 @@ impl ChannelCapabilitiesFile { } } +fn normalize_webhook_methods(methods: &[String]) -> Vec<String> { + let mut normalized = Vec::new(); + for method in methods { + let upper = method.trim().to_ascii_uppercase(); + if !matches!(upper.as_str(), "GET" | "POST") || normalized.contains(&upper) { + continue; + } + normalized.push(upper); + } + + if normalized.is_empty() { + normalized.push("POST".to_string()); + } + + normalized +} + /// Schema for channel capabilities. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ChannelCapabilitiesSchema { @@ -427,6 +467,13 @@ pub struct WebhookSchema { /// in a provider-specific request field rather than the configured header. #[serde(default)] pub managed_by_host: Option<bool>, + + /// HTTP methods the generic webhook router should expose for this path. + /// + /// Defaults to POST. Callback-style providers that perform URL + /// verification can opt into GET explicitly. + #[serde(default)] + pub methods: Vec<String>, } /// Setup configuration schema. @@ -711,6 +758,34 @@ mod tests { assert_eq!(caps.workspace_prefix, "integrations/custom/"); } + #[test] + fn test_requires_binding_detects_dm_owner_fields() { + let telegram = ChannelCapabilitiesFile::from_json( + r#"{ + "name": "telegram", + "config": { + "owner_id": null, + "dm_policy": "pairing", + "allow_from": [] + } + }"#, + ) + .unwrap(); + assert!(telegram.requires_binding()); + + let wechat = ChannelCapabilitiesFile::from_json( + r#"{ + "name": "wechat", + "config": { + "base_url": "https://ilinkai.weixin.qq.com", + "bot_type": "3" + } + }"#, + ) + .unwrap(); + assert!(!wechat.requires_binding()); + } + #[test] fn test_durable_workspace_paths_are_prefixed() { let json = r#"{ @@ -762,7 +837,8 @@ mod tests { "allowed_paths": ["/webhook/telegram"], "webhook": { "secret_header": "X-Telegram-Bot-Api-Secret-Token", - "secret_name": "telegram_webhook_secret" + "secret_name": "telegram_webhook_secret", + "methods": ["post", "GET", "TRACE", "POST"] } } } @@ -775,6 +851,25 @@ mod tests { ); assert_eq!(file.webhook_secret_name(), "telegram_webhook_secret"); assert!(file.webhook_secret_managed_by_host()); + assert_eq!( + file.webhook_methods(), + vec!["POST".to_string(), "GET".to_string()] + ); + } + + #[test] + fn test_webhook_methods_default_to_post() { + let json = r#"{ + "name": "telegram", + "capabilities": { + "channel": { + "allowed_paths": ["/webhook/telegram"] + } + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + assert_eq!(file.webhook_methods(), vec!["POST".to_string()]); } #[test] diff --git a/src/channels/wasm/setup.rs b/src/channels/wasm/setup.rs index 955552c4074..9f9114a2303 100644 --- a/src/channels/wasm/setup.rs +++ b/src/channels/wasm/setup.rs @@ -5,14 +5,13 @@ //! //! # Ownership model //! -//! Boot-time secret lookups use `config.owner_id` because channels are -//! **instance-level resources** — they run as the instance operator, not as -//! individual users. This is intentional and distinct from tool-level -//! credential resolution, which is scoped to the calling user's `user_id`. +//! Boot-time secret lookups use `config.owner_id` for instance-level channels. +//! Single-login channels such as WeChat may carry a persisted bound user and +//! use that user only for their active channel credentials. //! //! See `docs/superpowers/specs/2026-04-01-ownership-model-design.md`. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use crate::channels::wasm::{ @@ -24,6 +23,9 @@ use crate::channels::wasm::{ use crate::config::Config; use crate::db::Database; use crate::extensions::ExtensionManager; +use crate::extensions::wechat_login::{ + WECHAT_BASE_URL_SETTING_PATH, WECHAT_BOUND_USER_SETTING_PATH, WECHAT_CHANNEL_NAME, +}; use crate::pairing::PairingStore; use crate::secrets::SecretsStore; @@ -66,7 +68,7 @@ pub async fn setup_wasm_channels( extension_manager: Option<&Arc<ExtensionManager>>, database: Option<&Arc<dyn Database>>, registered_channel_names: &[String], - startup_active_channel_names: Option<&HashSet<String>>, + startup_active_channel_names: &HashSet<String>, ownership_cache: Arc<crate::ownership::OwnershipCache>, ) -> Option<WasmChannelSetup> { let runtime = match WasmChannelRuntime::new(WasmChannelRuntimeConfig::default()) { @@ -108,9 +110,11 @@ pub async fn setup_wasm_channels( discovered_channels .into_iter() .filter_map(|(name, discovered)| { - startup_active_channel_names - .is_none_or(|active_names| active_names.contains(&name)) - .then_some((name, discovered.wasm_path, discovered.capabilities_path)) + startup_active_channel_names.contains(&name).then_some(( + name, + discovered.wasm_path, + discovered.capabilities_path, + )) }) .collect(); @@ -120,11 +124,6 @@ pub async fn setup_wasm_channels( let load_results = futures::future::join_all(load_futures).await; let mut loaded_channels = Vec::new(); - let startup_load_error_message = if startup_active_channel_names.is_some() { - "Failed to load persisted-active WASM channel at startup" - } else { - "Failed to load WASM channel at startup" - }; for ((name, wasm_path, _), result) in startup_entries.into_iter().zip(load_results) { match result { Ok(loaded) => loaded_channels.push(loaded), @@ -133,7 +132,7 @@ pub async fn setup_wasm_channels( channel = %name, path = %wasm_path.display(), error = %err, - "{startup_load_error_message}" + "Failed to load active WASM channel at startup" ); } } @@ -283,7 +282,7 @@ async fn register_channel( let endpoints = vec![RegisteredEndpoint { channel_name: channel_name.clone(), path: webhook_path, - methods: vec!["POST".to_string()], + methods: loaded.webhook_methods(), require_secret: host_webhook_secret.is_some(), }]; @@ -309,11 +308,23 @@ async fn register_channel( serde_json::json!(username), ); } + + config_updates.extend(load_wasm_channel_runtime_overrides( + &config.channels.wasm_channel_runtime_overrides, + &channel_name, + )); // Inject channel-specific secrets into config for channels that need // credentials in API request bodies (e.g., Feishu token exchange). // The credential injection system only replaces placeholders in URLs // and headers, so channels like Feishu that exchange app_id + app_secret // for a tenant token need the raw values in their config. + inject_channel_settings_into_config( + &channel_name, + &config.owner_id, + settings_store, + &mut config_updates, + ) + .await; if let Some(secrets) = secrets_store { inject_wasm_channel_secret_config_mappings( &channel_name, @@ -383,6 +394,9 @@ async fn register_channel( tracing::info!(channel = %channel_name, "Registered HMAC signing secret"); } + let credential_scope_id = + channel_credential_scope_id(&channel_name, &config.owner_id, settings_store).await; + // Inject credentials from secrets store / environment. match inject_channel_credentials( &channel_arc, @@ -390,7 +404,7 @@ async fn register_channel( .as_ref() .map(|s| s.as_ref() as &dyn SecretsStore), &channel_name, - &config.owner_id, + &credential_scope_id, ) .await { @@ -648,6 +662,140 @@ pub(crate) async fn inject_wasm_channel_secret_config_mappings( } } +fn load_wasm_channel_runtime_overrides( + stored_overrides: &HashMap<String, serde_json::Value>, + channel_name: &str, +) -> HashMap<String, serde_json::Value> { + let mut result = HashMap::new(); + let prefix = format!("{channel_name}:"); + + for (stored_key, value) in stored_overrides { + let Some(config_key) = stored_key.strip_prefix(&prefix) else { + continue; + }; + let config_key = config_key.trim(); + if config_key.is_empty() { + tracing::warn!( + channel = %channel_name, + key = %stored_key, + "Ignoring empty wasm channel runtime override key" + ); + continue; + } + + if is_reserved_wasm_runtime_config_key(config_key) { + tracing::warn!( + channel = %channel_name, + key = %config_key, + "Ignoring reserved wasm channel runtime override key" + ); + continue; + } + + result.insert(config_key.to_string(), value.clone()); + } + + result +} + +fn is_reserved_wasm_runtime_config_key(key: &str) -> bool { + matches!( + key, + crate::channels::wasm::RUNTIME_CONFIG_KEY_TUNNEL_URL + | crate::channels::wasm::RUNTIME_CONFIG_KEY_WEBHOOK_SECRET + | crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID + ) +} + +async fn channel_credential_scope_id( + channel_name: &str, + owner_id: &str, + settings_store: Option<&Arc<dyn crate::db::SettingsStore>>, +) -> String { + if channel_name != WECHAT_CHANNEL_NAME { + return owner_id.to_string(); + } + + let Some(store) = settings_store else { + return owner_id.to_string(); + }; + + wechat_bound_user_id(owner_id, store) + .await + .unwrap_or_else(|| owner_id.to_string()) +} + +async fn wechat_bound_user_id( + owner_id: &str, + store: &Arc<dyn crate::db::SettingsStore>, +) -> Option<String> { + if let Ok(Some(serde_json::Value::String(value))) = store + .get_setting(owner_id, WECHAT_BOUND_USER_SETTING_PATH) + .await + { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + + None +} + +/// Inject channel-specific settings into config for channels that persist +/// runtime-discovered values (for example a custom API base URL after login). +async fn inject_channel_settings_into_config( + channel_name: &str, + owner_id: &str, + settings_store: Option<&Arc<dyn crate::db::SettingsStore>>, + config_updates: &mut std::collections::HashMap<String, serde_json::Value>, +) { + let Some(store) = settings_store else { + return; + }; + + let setting_mappings: &[(&str, &str)] = match channel_name { + WECHAT_CHANNEL_NAME => &[("base_url", WECHAT_BASE_URL_SETTING_PATH)], + _ => return, + }; + + let bound_user_id = if channel_name == WECHAT_CHANNEL_NAME { + wechat_bound_user_id(owner_id, store).await + } else { + None + }; + let setting_scope_id = bound_user_id + .clone() + .unwrap_or_else(|| owner_id.to_string()); + if let Some(bound_user_id) = bound_user_id { + config_updates.insert( + "bound_user_id".to_string(), + serde_json::Value::String(bound_user_id), + ); + } + + for &(config_key, setting_path) in setting_mappings { + if let Ok(Some(serde_json::Value::String(value))) = + store.get_setting(&setting_scope_id, setting_path).await + { + let trimmed = value.trim(); + if trimmed.is_empty() { + continue; + } + config_updates.insert( + config_key.to_string(), + serde_json::Value::String(trimmed.to_string()), + ); + tracing::debug!( + channel = %channel_name, + config_key = %config_key, + setting_path = %setting_path, + "Injected setting into channel config" + ); + } + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -661,11 +809,14 @@ mod tests { WasmChannel, WasmChannelRouter, WasmChannelRuntime, WasmChannelRuntimeConfig, }; use crate::config::Config; + use crate::db::{Database, SettingsStore}; + use crate::extensions::wechat_login::{ + WECHAT_BASE_URL_SETTING_PATH, WECHAT_BOUND_USER_SETTING_PATH, + }; use crate::pairing::PairingStore; - use crate::secrets::{CreateSecretParams, InMemorySecretsStore, SecretsCrypto, SecretsStore}; - use crate::testing::credentials::TEST_CRYPTO_KEY; + use crate::secrets::{CreateSecretParams, SecretsStore}; + use crate::testing::credentials::test_secrets_store; use crate::tools::wasm::ResourceLimits; - use secrecy::SecretString; /// Build the same reserved-name list that `setup_wasm_channels` uses. fn reserved_names() -> Vec<&'static str> { @@ -683,13 +834,27 @@ mod tests { } fn test_loaded_channel(name: &str, capabilities_config: serde_json::Value) -> LoadedChannel { + test_loaded_channel_with_webhook_methods(name, capabilities_config, Vec::new()) + } + + fn test_loaded_channel_with_webhook_methods( + name: &str, + capabilities_config: serde_json::Value, + methods: Vec<&str>, + ) -> LoadedChannel { + let webhook = if methods.is_empty() { + serde_json::json!({}) + } else { + serde_json::json!({ "methods": methods }) + }; let cap_file = ChannelCapabilitiesFile::from_json( &serde_json::json!({ "type": "channel", "name": name, "capabilities": { "channel": { - "allowed_paths": [format!("/webhook/{name}")] + "allowed_paths": [format!("/webhook/{name}")], + "webhook": webhook } }, "config": capabilities_config @@ -745,8 +910,6 @@ mod tests { #[test] fn reserved_names_reject_case_insensitive() { - // The setup logic lowercases the WASM channel name before checking. - // Verify that "Web" or "GATEWAY" would be caught. let reserved = reserved_names(); let test_cases = ["Web", "GATEWAY", "CLI", "Repl", "__BOOTSTRAP__"]; for name in test_cases { @@ -968,12 +1131,51 @@ mod tests { .get_channel_for_path("/webhook/telegram") .await .expect("telegram channel should be registered"); + assert!( + wasm_router + .method_allowed_for_path("/webhook/telegram", &axum::http::Method::POST) + .await, + "startup registration should default webhook methods to POST" + ); + assert!( + !wasm_router + .method_allowed_for_path("/webhook/telegram", &axum::http::Method::GET) + .await, + "startup registration should not expose GET unless capabilities declare it" + ); assert_eq!( registered.owner_actor_id_for_test().await, Some("12345".to_string()) ); } + #[tokio::test] + async fn register_channel_uses_declared_webhook_methods() { + let (config, _temp_dir) = test_config(); + let loaded = test_loaded_channel_with_webhook_methods( + "wecom", + serde_json::json!({ "owner_id": 12345 }), + vec!["GET", "POST"], + ); + let wasm_router = Arc::new(WasmChannelRouter::new()); + let pairing_store = Arc::new(PairingStore::new_noop()); + + let (_name, _channel) = + super::register_channel(loaded, &config, &None, None, &pairing_store, &wasm_router) + .await; + + assert!( + wasm_router + .method_allowed_for_path("/webhook/wecom", &axum::http::Method::GET) + .await + ); + assert!( + wasm_router + .method_allowed_for_path("/webhook/wecom", &axum::http::Method::POST) + .await + ); + } + #[tokio::test] async fn register_channel_propagates_capabilities_owner_id_to_config() { let (config, _temp_dir) = test_config(); @@ -1019,47 +1221,131 @@ mod tests { } #[tokio::test] - async fn inject_channel_secrets_uses_owner_scope() { - let crypto = - Arc::new(SecretsCrypto::new(SecretString::from(TEST_CRYPTO_KEY.to_string())).unwrap()); - let secrets: Arc<dyn SecretsStore + Send + Sync> = - Arc::new(InMemorySecretsStore::new(crypto)); + async fn register_channel_injects_runtime_overrides_for_matching_channel() { + let (mut config, _temp_dir) = test_config(); + config.channels.wasm_channel_runtime_overrides.insert( + "wecom:dm_policy".to_string(), + serde_json::json!("allowlist"), + ); + config.channels.wasm_channel_runtime_overrides.insert( + "wecom:allow_from".to_string(), + serde_json::json!(["zhangsan"]), + ); + config.channels.wasm_channel_runtime_overrides.insert( + format!( + "wecom:{}", + crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID + ), + serde_json::json!("blocked"), + ); + + let loaded = test_loaded_channel("wecom", serde_json::json!({})); + let wasm_router = Arc::new(WasmChannelRouter::new()); + let pairing_store = Arc::new(PairingStore::new_noop()); + + let (_name, _channel) = + super::register_channel(loaded, &config, &None, None, &pairing_store, &wasm_router) + .await; + + let registered = wasm_router + .get_channel_for_path("/webhook/wecom") + .await + .expect("wecom channel should be registered"); + let runtime_config = registered.get_config().await; + assert_eq!( + runtime_config.get("dm_policy"), + Some(&serde_json::json!("allowlist")) + ); + assert_eq!( + runtime_config.get("allow_from"), + Some(&serde_json::json!(["zhangsan"])) + ); + assert_ne!( + runtime_config.get(crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID), + Some(&serde_json::json!("blocked")) + ); + } + + #[tokio::test] + async fn test_inject_channel_settings_uses_wechat_bound_user_scope() -> Result<(), String> { + let dir = tempfile::tempdir().map_err(|e| format!("tempdir failed: {e}"))?; + let db_path = dir.path().join("wechat-settings.db"); + let db = Arc::new( + crate::db::libsql::LibSqlBackend::new_local(&db_path) + .await + .map_err(|e| format!("create local libsql backend failed: {e}"))?, + ); + db.run_migrations() + .await + .map_err(|e| format!("run libsql migrations failed: {e}"))?; + + db.set_setting( + "default", + WECHAT_BOUND_USER_SETTING_PATH, + &serde_json::json!("owner-123"), + ) + .await + .map_err(|e| format!("persist bound user setting failed: {e}"))?; + db.set_setting( + "default", + WECHAT_BASE_URL_SETTING_PATH, + &serde_json::json!("https://default.example"), + ) + .await + .map_err(|e| format!("persist default setting failed: {e}"))?; + db.set_setting( + "owner-123", + WECHAT_BASE_URL_SETTING_PATH, + &serde_json::json!("https://owner.example"), + ) + .await + .map_err(|e| format!("persist owner setting failed: {e}"))?; + + let settings_store: Arc<dyn crate::db::SettingsStore> = db; + let mut config_updates = std::collections::HashMap::new(); + super::inject_channel_settings_into_config( + "wechat", + "default", + Some(&settings_store), + &mut config_updates, + ) + .await; + + assert_eq!( + config_updates.get("base_url"), + Some(&serde_json::json!("https://owner.example")) + ); + assert_eq!( + config_updates.get("bound_user_id"), + Some(&serde_json::json!("owner-123")) + ); + Ok(()) + } + + #[tokio::test] + async fn test_inject_channel_secrets_uses_owner_scope_for_feishu() -> Result<(), String> { + let secrets = test_secrets_store(); secrets .create( - "owner-123", - CreateSecretParams { - name: "feishu_app_id".to_string(), - value: SecretString::from("owner-app-id".to_string()), - provider: None, - expires_at: None, - }, + "default", + CreateSecretParams::new("feishu_app_id", "default-app-id"), ) .await - .unwrap(); + .map_err(|e| format!("persist default feishu_app_id failed: {e}"))?; secrets .create( "owner-123", - CreateSecretParams { - name: "feishu_app_secret".to_string(), - value: SecretString::from("owner-app-secret".to_string()), - provider: None, - expires_at: None, - }, + CreateSecretParams::new("feishu_app_id", "owner-app-id"), ) .await - .unwrap(); + .map_err(|e| format!("persist owner feishu_app_id failed: {e}"))?; secrets .create( - "default", - CreateSecretParams { - name: "feishu_app_id".to_string(), - value: SecretString::from("default-app-id".to_string()), - provider: None, - expires_at: None, - }, + "owner-123", + CreateSecretParams::new("feishu_app_secret", "owner-app-secret"), ) .await - .unwrap(); + .map_err(|e| format!("persist owner feishu_app_secret failed: {e}"))?; let mut config_updates = HashMap::new(); let secret_config_mappings = vec![ @@ -1079,7 +1365,7 @@ mod tests { super::inject_wasm_channel_secret_config_mappings( "feishu", "owner-123", - secrets.as_ref(), + &secrets, &secret_config_mappings, &mut config_updates, ) @@ -1093,5 +1379,141 @@ mod tests { config_updates.get("app_secret"), Some(&serde_json::json!("owner-app-secret")) ); + Ok(()) + } + + /// Stage the real `telegram.wasm` + `telegram.capabilities.json` from + /// `channels-src/telegram/` into the given directory so a test can drive + /// `setup_wasm_channels` through the full discover -> load -> register + /// path. Returns `None` if the source artifacts aren't present (e.g. a + /// release tarball without `channels-src/`), in which case the caller + /// should skip the test rather than fail. + fn stage_real_telegram_channel(dir: &std::path::Path) -> Option<()> { + let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let wasm_src = manifest_dir.join("channels-src/telegram/telegram.wasm"); + let caps_src = manifest_dir.join("channels-src/telegram/telegram.capabilities.json"); + if !wasm_src.exists() || !caps_src.exists() { + return None; + } + std::fs::copy(&wasm_src, dir.join("telegram.wasm")).ok()?; + std::fs::copy(&caps_src, dir.join("telegram.capabilities.json")).ok()?; + Some(()) + } + + /// Headless-startup regression: with `database = None`, `extension_manager = None`, + /// and `secrets_store = None`, `setup_wasm_channels` must still load and register + /// channels named in `startup_active_channel_names`. This pins down the linkage + /// from the `main.rs` config-fallback resolution to the actual filter inside + /// `setup_wasm_channels`. Pairs with the empty-set test below. + #[tokio::test] + async fn setup_wasm_channels_registers_configured_channel_in_headless_mode() { + let temp = tempfile::tempdir().unwrap(); + let channels_dir = temp.path().join("channels"); + std::fs::create_dir_all(&channels_dir).unwrap(); + if stage_real_telegram_channel(&channels_dir).is_none() { + eprintln!("skipping: channels-src/telegram artifacts not present in this checkout"); + return; + } + + let (mut config, _config_temp) = test_config(); + config.channels.wasm_channels_dir = channels_dir; + config.channels.wasm_channels_enabled = true; + config.channels.configured_wasm_channels = vec!["telegram".to_string()]; + + let mut active = std::collections::HashSet::new(); + active.insert("telegram".to_string()); + + let setup = super::setup_wasm_channels( + &config, + &None, + None, + None, + &[], + &active, + Arc::new(crate::ownership::OwnershipCache::new()), + ) + .await + .expect("setup_wasm_channels should return Some when a channel loads"); + + assert!( + setup.channel_names.iter().any(|n| n == "telegram"), + "headless config-fallback path must register telegram, got {:?}", + setup.channel_names + ); + assert!( + setup.webhook_routes.is_some(), + "webhook_routes are always created so hot-activation works post-startup" + ); + } + + /// Empty `startup_active_channel_names` must register zero channels even + /// when discoverable channels exist on disk. This is the regression test + /// for the "empty set lets everything through" bug class — the previous + /// `Option<&HashSet>` filter signature treated `None` as "load all"; the + /// current `&HashSet` signature with an explicit empty set is the fix. + #[tokio::test] + async fn setup_wasm_channels_with_empty_active_set_registers_nothing() { + let temp = tempfile::tempdir().unwrap(); + let channels_dir = temp.path().join("channels"); + std::fs::create_dir_all(&channels_dir).unwrap(); + if stage_real_telegram_channel(&channels_dir).is_none() { + eprintln!("skipping: channels-src/telegram artifacts not present in this checkout"); + return; + } + + let (mut config, _config_temp) = test_config(); + config.channels.wasm_channels_dir = channels_dir; + config.channels.wasm_channels_enabled = true; + + let setup = super::setup_wasm_channels( + &config, + &None, + None, + None, + &[], + &std::collections::HashSet::new(), + Arc::new(crate::ownership::OwnershipCache::new()), + ) + .await + .expect("setup_wasm_channels returns Some even with no channels to load"); + + assert!( + setup.channel_names.is_empty(), + "empty active set must register zero channels, got {:?}", + setup.channel_names + ); + } + + #[test] + fn load_wasm_channel_runtime_overrides_uses_channel_prefix_and_blocks_reserved_keys() { + let mut stored = HashMap::new(); + stored.insert( + "wecom:dm_policy".to_string(), + serde_json::json!("allowlist"), + ); + stored.insert( + "wecom:allow_from".to_string(), + serde_json::json!(["zhangsan", "lisi"]), + ); + stored.insert( + format!( + "wecom:{}", + crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID + ), + serde_json::json!("123"), + ); + stored.insert("telegram:dm_policy".to_string(), serde_json::json!("open")); + + let overrides = super::load_wasm_channel_runtime_overrides(&stored, "wecom"); + assert_eq!( + overrides.get("dm_policy"), + Some(&serde_json::json!("allowlist")) + ); + assert_eq!( + overrides.get("allow_from"), + Some(&serde_json::json!(["zhangsan", "lisi"])) + ); + assert!(!overrides.contains_key(crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID)); + assert!(!overrides.contains_key("telegram:dm_policy")); } } diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 6d8a90abf5d..d21401933ef 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -50,8 +50,14 @@ use crate::channels::wasm::host::{ use crate::channels::wasm::router::RegisteredEndpoint; use crate::channels::wasm::runtime::{PreparedChannelModule, WasmChannelRuntime}; use crate::channels::wasm::schema::ChannelConfig; -use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; +use crate::channels::{ + Channel, IncomingMessage, MessageStream, OutgoingAttachment, OutgoingResponse, StatusUpdate, +}; use crate::error::ChannelError; +use crate::generated_images::{ + is_staged_generated_image_path, remove_staged_generated_image_attachments, + stage_generated_image_data_url, +}; use crate::pairing::PairingStore; use crate::secrets::SecretsStore; use crate::secrets::host_matches_pattern; @@ -68,6 +74,10 @@ const TEST_HTTP_REWRITE_MAP_ENV: &str = "IRONCLAW_TEST_HTTP_REWRITE_MAP"; const WEBSOCKET_EVENT_QUEUE_RELATIVE_PATH: &str = "state/gateway_event_queue"; const WEBSOCKET_EVENT_PROCESSING_QUEUE_RELATIVE_PATH: &str = "state/gateway_event_queue_processing"; const WEBSOCKET_EVENT_QUEUE_MAX_ITEMS: usize = 100; +const WEBSOCKET_OUTBOUND_QUEUE_CAPACITY: usize = 32; +const WEBSOCKET_OUTBOUND_MAX_FRAME_BYTES: usize = 2 * 1024 * 1024; +const WECHAT_CHANNEL_NAME: &str = "wechat"; +const CHANNEL_BOUND_USER_ID_CONFIG_KEY: &str = "bound_user_id"; #[cfg(any(test, debug_assertions))] const TELEGRAM_TEST_API_BASE_ENV: &str = "IRONCLAW_TEST_TELEGRAM_API_BASE_URL"; @@ -120,6 +130,8 @@ struct ChannelStoreData { host_credentials: Vec<ResolvedHostCredential>, /// Pairing store for DM pairing (guest access control). pairing_store: Arc<PairingStore>, + /// Optional websocket outbound sender for channels with a managed runtime. + websocket_outbound_tx: Option<mpsc::Sender<String>>, /// Dedicated tokio runtime for HTTP requests, lazily initialized. /// Reused across multiple `http_request` calls within one execution. http_runtime: Option<tokio::runtime::Runtime>, @@ -133,6 +145,7 @@ impl ChannelStoreData { credentials: HashMap<String, String>, host_credentials: Vec<ResolvedHostCredential>, pairing_store: Arc<PairingStore>, + websocket_outbound_tx: Option<mpsc::Sender<String>>, ) -> Self { // Create a minimal WASI context (no filesystem, no env vars for security) let wasi = WasiCtxBuilder::new().build(); @@ -145,6 +158,7 @@ impl ChannelStoreData { credentials, host_credentials, pairing_store, + websocket_outbound_tx, http_runtime: None, } } @@ -335,6 +349,14 @@ impl near::agent::channel_host::Host for ChannelStoreData { .map_err(|e| e.to_string()) } + fn websocket_send_text(&mut self, payload: String) -> Result<(), String> { + let sender = self + .websocket_outbound_tx + .as_ref() + .ok_or_else(|| "websocket runtime not available for this channel".to_string())?; + queue_websocket_outbound_frame(sender, payload) + } + fn http_request( &mut self, method: String, @@ -648,6 +670,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { storage_key: a.storage_key, local_path: None, extracted_text: a.extracted_text, + extras_json: a.extras_json, data, duration_secs, } @@ -802,6 +825,10 @@ pub struct WasmChannel { /// Websocket runtime shutdown signal sender. websocket_shutdown_tx: RwLock<Option<oneshot::Sender<()>>>, + /// Host-managed websocket outbound sender used by `on_respond` and + /// `on_poll` to emit protocol-specific websocket frames. + websocket_outbound_tx: Arc<RwLock<Option<mpsc::Sender<String>>>>, + /// Serializes websocket-triggered poll executions. websocket_poll_lock: Arc<Mutex<()>>, @@ -817,6 +844,10 @@ pub struct WasmChannel { /// Telegram's "typing..." indicator expires after ~5s, so we refresh it. typing_task: RwLock<Option<tokio::task::JoinHandle<()>>>, + /// Generated images staged from status updates until the final channel + /// response can deliver them together with the assistant's text. + pending_generated_image_attachments: Arc<Mutex<HashMap<String, Vec<String>>>>, + /// Pairing store for DM pairing (guest access control). pairing_store: Arc<PairingStore>, @@ -845,6 +876,9 @@ pub struct WasmChannel { /// value after pairing approval without capturing a stale clone. owner_actor_id: Arc<tokio::sync::RwLock<Option<String>>>, + /// User bound to a single-login channel such as WeChat. + channel_bound_user_id: Arc<RwLock<Option<String>>>, + /// Secrets store for host-based credential injection. /// Used to pre-resolve credentials before each WASM callback. secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>, @@ -1001,10 +1035,21 @@ fn resolve_message_scope( } } +fn parse_channel_bound_user_id(config_json: &str) -> Option<String> { + let value: serde_json::Value = serde_json::from_str(config_json).ok()?; + value + .get(CHANNEL_BOUND_USER_ID_CONFIG_KEY) + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + async fn resolve_message_scope_with_pairing( channel_name: &str, owner_scope_id: &str, owner_actor_id: Option<&str>, + channel_bound_user_id: Option<&str>, sender_id: &str, pairing_store: &PairingStore, ) -> (String, bool) { @@ -1012,6 +1057,14 @@ async fn resolve_message_scope_with_pairing( return (owner_scope_id.to_string(), true); } + if channel_name == WECHAT_CHANNEL_NAME + && let Some(bound_user_id) = channel_bound_user_id + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return (bound_user_id.to_string(), false); + } + match pairing_store .resolve_identity(channel_name, sender_id) .await @@ -1099,6 +1152,7 @@ impl WasmChannel { ) -> Self { let name = prepared.name.clone(); let rate_limiter = ChannelEmitRateLimiter::new(capabilities.emit_rate_limit.clone()); + let channel_bound_user_id = parse_channel_bound_user_id(&config_json); Self { name, @@ -1114,10 +1168,12 @@ impl WasmChannel { poll_shutdown_tx: RwLock::new(None), poll_task: RwLock::new(None), websocket_shutdown_tx: RwLock::new(None), + websocket_outbound_tx: Arc::new(RwLock::new(None)), websocket_poll_lock: Arc::new(Mutex::new(())), endpoints: RwLock::new(Vec::new()), credentials: Arc::new(RwLock::new(HashMap::new())), typing_task: RwLock::new(None), + pending_generated_image_attachments: Arc::new(Mutex::new(HashMap::new())), pairing_store, workspace_store: Arc::new(ChannelWorkspaceStore::new()), callback_lock: Arc::new(tokio::sync::Mutex::new(())), @@ -1125,6 +1181,7 @@ impl WasmChannel { settings_store, owner_scope_id: owner_scope_id.into(), owner_actor_id: Arc::new(tokio::sync::RwLock::new(None)), + channel_bound_user_id: Arc::new(RwLock::new(channel_bound_user_id)), secrets_store: None, } } @@ -1223,6 +1280,13 @@ impl WasmChannel { /// Merges the provided values into the existing config JSON. /// Call this before `start()` to inject runtime values like tunnel_url. pub async fn update_config(&self, updates: HashMap<String, serde_json::Value>) { + let bound_user_update = updates + .get(CHANNEL_BOUND_USER_ID_CONFIG_KEY) + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string); + let has_bound_user_update = updates.contains_key(CHANNEL_BOUND_USER_ID_CONFIG_KEY); let mut config_guard = self.config_json.write().await; // Parse existing config @@ -1242,6 +1306,12 @@ impl WasmChannel { config = %*config_guard, "Updated channel config" ); + + drop(config_guard); + + if self.name == WECHAT_CHANNEL_NAME && has_bound_user_update { + *self.channel_bound_user_id.write().await = bound_user_update; + } } /// Set a credential for URL injection. @@ -1449,13 +1519,14 @@ impl WasmChannel { let last_broadcast_metadata = self.last_broadcast_metadata.clone(); let settings_store = self.settings_store.clone(); let owner_scope_id = self.owner_scope_id.clone(); + let channel_bound_user_id = Arc::clone(&self.channel_bound_user_id); let websocket_secrets_store = self.secrets_store.clone(); let websocket_poll_lock = Arc::clone(&self.websocket_poll_lock); + let websocket_outbound_tx_state = Arc::clone(&self.websocket_outbound_tx); tokio::spawn(async move { let mut shutdown = std::pin::pin!(shutdown_rx); let mut reconnect_attempt = 0u32; - let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel::<String>(); tracing::info!( channel = %channel_name, @@ -1464,27 +1535,27 @@ impl WasmChannel { ); let queue_path = websocket_queue_path(&channel_name); let processing_queue_path = websocket_processing_queue_path(&channel_name); - let identify_payload = resolve_websocket_identify_message( + let websocket_protocol = config.protocol_kind(); + let websocket_auth = match resolve_websocket_auth( &config, websocket_secrets_store.as_deref(), &owner_scope_id, ) - .await; - // Defense in depth: if an identify template or secret cannot be - // resolved here, do not enter the reconnect loop — peer will - // reject with an auth close code. `resolve_websocket_identify_message` - // returns `None` on several distinct failures (missing secret, - // decrypt error, missing identify template, store failure), so the - // message enumerates them rather than naming one. - if config.identify_secret_name.is_some() && identify_payload.is_none() { - tracing::warn!( - channel = %channel_name, - has_identify_template = config.identify.is_some(), - "Websocket runtime exiting: failed to build identify payload (missing secret, decrypt error, or unresolved template)" - ); - return; - } - let mut session_state = WebsocketSessionState::new(identify_payload.as_deref()); + .await + { + Some(auth) => auth, + None => { + tracing::warn!( + channel = %channel_name, + protocol = ?websocket_protocol, + owner_scope_id = %owner_scope_id, + "Websocket runtime missing required auth secrets; not starting" + ); + *websocket_outbound_tx_state.write().await = None; + return; + } + }; + let mut session_state = WebsocketSessionState::new(&config, Some(&websocket_auth)); 'reconnect: loop { let connect_url = session_state.connect_url(&config.url); @@ -1509,6 +1580,7 @@ impl WasmChannel { _ = tokio::time::sleep(backoff) => continue 'reconnect, _ = &mut shutdown => { tracing::info!(channel = %channel_name, "Stopping websocket runtime"); + *websocket_outbound_tx_state.write().await = None; break 'reconnect; } } @@ -1518,6 +1590,90 @@ impl WasmChannel { let (mut write, mut read) = stream.split(); let mut next_heartbeat: Option<std::pin::Pin<Box<tokio::time::Sleep>>> = None; session_state.reset_connection(); + let mut stop_runtime = false; + let mut should_reconnect = false; + + for action in session_state.on_connected(Some(&websocket_auth)) { + match action { + WebsocketFrameAction::SetHeartbeat { interval_ms } => { + next_heartbeat = Some(Box::pin(tokio::time::sleep( + websocket_heartbeat_sleep_duration(interval_ms), + ))); + } + WebsocketFrameAction::Send(payload) => { + if let Err(error) = + write.send(WebsocketMessage::Text(payload.into())).await + { + tracing::warn!( + channel = %channel_name, + error = %error, + "Websocket protocol bootstrap send failed" + ); + should_reconnect = true; + break; + } + } + WebsocketFrameAction::Reconnect { reset_session } => { + if reset_session { + session_state.invalidate_session(); + } + should_reconnect = true; + break; + } + WebsocketFrameAction::StopRuntime => { + stop_runtime = true; + break; + } + WebsocketFrameAction::Enqueue(raw_text) => { + if let Err(error) = workspace_store.append_json_text_queue( + &queue_path, + &raw_text, + WEBSOCKET_EVENT_QUEUE_MAX_ITEMS, + ) { + tracing::warn!( + channel = %channel_name, + error = %error, + "Failed to enqueue websocket bootstrap frame" + ); + } + } + } + } + + if stop_runtime { + tracing::info!( + channel = %channel_name, + protocol = ?websocket_protocol, + "Stopping websocket runtime per protocol request" + ); + *websocket_outbound_tx_state.write().await = None; + break 'reconnect; + } + if should_reconnect { + let backoff = websocket_reconnect_backoff(reconnect_attempt); + reconnect_attempt = reconnect_attempt.saturating_add(1); + tracing::info!( + channel = %channel_name, + protocol = ?websocket_protocol, + backoff_secs = backoff.as_secs(), + "Websocket runtime bootstrap requested reconnect" + ); + tokio::select! { + _ = tokio::time::sleep(backoff) => continue 'reconnect, + _ = &mut shutdown => { + tracing::info!(channel = %channel_name, "Stopping websocket runtime"); + *websocket_outbound_tx_state.write().await = None; + break 'reconnect; + } + } + } + + let (outbound_tx, mut outbound_rx) = + mpsc::channel::<String>(WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + { + let mut guard = websocket_outbound_tx_state.write().await; + *guard = Some(outbound_tx.clone()); + } loop { tokio::select! { @@ -1528,14 +1684,19 @@ impl WasmChannel { std::future::pending::<()>().await; } } => { - if let Some(payload) = build_websocket_heartbeat_message(session_state.last_sequence.clone()) - && let Err(error) = write.send(WebsocketMessage::Text(payload.into())).await - { - tracing::warn!(channel = %channel_name, error = %error, "Websocket heartbeat send failed"); - break; + match session_state.heartbeat_tick(&channel_name) { + WebsocketHeartbeatAction::None => {} + WebsocketHeartbeatAction::Send(payload) => { + if let Err(error) = write.send(WebsocketMessage::Text(payload.into())).await { + tracing::warn!(channel = %channel_name, error = %error, "Websocket heartbeat send failed"); + break; + } + } + WebsocketHeartbeatAction::Reconnect => break, } - next_heartbeat = session_state.heartbeat_interval_ms + next_heartbeat = session_state + .heartbeat_interval_ms() .map(|interval_ms| Box::pin(tokio::time::sleep(websocket_heartbeat_sleep_duration(interval_ms)))); } outbound = outbound_rx.recv() => { @@ -1548,6 +1709,7 @@ impl WasmChannel { } _ = &mut shutdown => { tracing::info!(channel = %channel_name, "Stopping websocket runtime"); + *websocket_outbound_tx_state.write().await = None; break 'reconnect; } message = read.next() => { @@ -1559,7 +1721,7 @@ impl WasmChannel { let actions = session_state.process_text_frame( &text, &channel_name, - identify_payload.as_deref(), + Some(&websocket_auth), workspace_store.as_ref(), pairing_store.as_ref(), ); @@ -1609,7 +1771,9 @@ impl WasmChannel { settings_store: settings_store.clone(), owner_scope_id: owner_scope_id.clone(), owner_actor_id: owner_actor_id.clone(), + channel_bound_user_id: Arc::clone(&channel_bound_user_id), secrets_store: websocket_secrets_store.clone(), + protocol_kind: websocket_protocol, outbound_tx: outbound_tx.clone(), queue_path: queue_path.clone(), processing_queue_path: processing_queue_path.clone(), @@ -1618,10 +1782,18 @@ impl WasmChannel { ); } } - WebsocketFrameAction::InvalidateAndReconnect => { + WebsocketFrameAction::Reconnect { reset_session } => { + if reset_session { + session_state.invalidate_session(); + } should_reconnect = true; break; } + WebsocketFrameAction::StopRuntime => { + stop_runtime = true; + should_break = true; + break; + } } } if should_reconnect { @@ -1668,6 +1840,17 @@ impl WasmChannel { } } + if stop_runtime { + tracing::info!( + channel = %channel_name, + protocol = ?websocket_protocol, + "Stopping websocket runtime per protocol request" + ); + *websocket_outbound_tx_state.write().await = None; + break 'reconnect; + } + + *websocket_outbound_tx_state.write().await = None; let backoff = websocket_reconnect_backoff(reconnect_attempt); reconnect_attempt = reconnect_attempt.saturating_add(1); tracing::info!( @@ -1679,10 +1862,12 @@ impl WasmChannel { _ = tokio::time::sleep(backoff) => {} _ = &mut shutdown => { tracing::info!(channel = %channel_name, "Stopping websocket runtime"); + *websocket_outbound_tx_state.write().await = None; break 'reconnect; } } } + *websocket_outbound_tx_state.write().await = None; }); } @@ -1694,6 +1879,7 @@ impl WasmChannel { credentials: HashMap<String, String>, host_credentials: Vec<ResolvedHostCredential>, pairing_store: Arc<PairingStore>, + websocket_outbound_tx: Option<mpsc::Sender<String>>, ) -> Result<Store<ChannelStoreData>, WasmChannelError> { let engine = runtime.engine(); let limits = &prepared.limits; @@ -1706,6 +1892,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, ); let mut store = Store::new(engine, store_data); @@ -1814,22 +2001,32 @@ impl WasmChannel { committed_paths } - fn log_on_start_host_state(&self, host_state: &mut ChannelHostState) { + fn log_host_state_entries(channel_name: &str, host_state: &mut ChannelHostState) { for entry in host_state.take_logs() { match entry.level { + crate::tools::wasm::LogLevel::Trace => { + tracing::trace!(channel = %channel_name, "{}", entry.message); + } + crate::tools::wasm::LogLevel::Debug => { + tracing::debug!(channel = %channel_name, "{}", entry.message); + } + crate::tools::wasm::LogLevel::Info => { + tracing::info!(channel = %channel_name, "{}", entry.message); + } crate::tools::wasm::LogLevel::Error => { - tracing::error!(channel = %self.name, "{}", entry.message); + tracing::error!(channel = %channel_name, "{}", entry.message); } crate::tools::wasm::LogLevel::Warn => { - tracing::warn!(channel = %self.name, "{}", entry.message); - } - _ => { - tracing::debug!(channel = %self.name, "{}", entry.message); + tracing::warn!(channel = %channel_name, "{}", entry.message); } } } } + fn log_on_start_host_state(&self, host_state: &mut ChannelHostState) { + Self::log_host_state_entries(&self.name, host_state); + } + async fn execute_on_start_with_state( &self, ) -> Result<(Result<ChannelConfig, WasmChannelError>, ChannelHostState), WasmChannelError> { @@ -1851,6 +2048,7 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); let (config_result, host_state, committed_paths) = tokio::time::timeout(timeout, async move { @@ -1862,6 +2060,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -2002,6 +2201,7 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); // Prepare request data let method = method.to_string(); @@ -2022,6 +2222,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -2112,6 +2313,7 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { @@ -2123,23 +2325,26 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Call on_poll using the generated typed interface let channel_iface = instance.near_agent_channel(); - channel_iface + let poll_result = channel_iface .call_on_poll(&mut store) - .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?; + .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel)); let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); - // Commit pending workspace writes to the persistent store - let committed_paths = - Self::commit_callback_workspace_writes(&mut host_state, &workspace_store); + let committed_paths = if poll_result.is_ok() { + Self::commit_callback_workspace_writes(&mut host_state, &workspace_store) + } else { + Vec::new() + }; - Ok(((), host_state, committed_paths)) + Ok((poll_result, host_state, committed_paths)) }) .await .map_err(|e| WasmChannelError::ExecutionPanicked { @@ -2151,10 +2356,11 @@ impl WasmChannel { let channel_name = self.name.clone(); match result { - Ok(Ok(((), mut host_state, committed_paths))) => { + Ok(Ok((poll_result, mut host_state, committed_paths))) => { self.persist_durable_workspace_snapshot_if_needed(&committed_paths) .await; let _ = drain_guest_logs(&channel_name, "on_poll", &mut host_state); + poll_result?; // Process emitted messages let emitted = host_state.take_emitted_messages(); @@ -2184,6 +2390,7 @@ impl WasmChannel { thread_id: Option<&str>, metadata_json: &str, attachments: &[String], + inline_attachments: &[OutgoingAttachment], ) -> Result<(), WasmChannelError> { let _callback_guard = self.callback_lock.lock().await; @@ -2192,7 +2399,7 @@ impl WasmChannel { message_id = %message_id, content_len = content.len(), thread_id = ?thread_id, - attachment_count = attachments.len(), + attachment_count = attachments.len() + inline_attachments.len(), "call_on_respond invoked" ); @@ -2228,6 +2435,7 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); // Prepare response data let message_id_str = message_id.to_string(); @@ -2235,19 +2443,20 @@ impl WasmChannel { let thread_id = thread_id.map(|s| s.to_string()); let metadata_json = metadata_json.to_string(); let attachments = attachments.to_vec(); + let inline_attachments = inline_attachments.to_vec(); // Execute in blocking task with timeout tracing::info!(channel = %channel_name, "Starting on_respond WASM execution"); let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - // Read attachment files from disk before entering WASM - let wit_attachments = read_attachments(&attachments).map_err(|e| { - WasmChannelError::CallbackFailed { - name: prepared.name.clone(), - reason: e, - } - })?; + // Prepare attachment bytes before entering WASM. + let wit_attachments = + prepare_response_attachments(&prepared.name, &attachments, &inline_attachments) + .map_err(|e| WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + })?; tracing::info!("Creating WASM store for on_respond"); let mut store = Self::create_store( @@ -2257,6 +2466,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; tracing::info!("Instantiating WASM component for on_respond"); @@ -2278,32 +2488,28 @@ impl WasmChannel { "Calling WASM on_respond" ); - // Call on_respond using the generated typed interface + // Call on_respond using the generated typed interface. + // Preserve guest logs even if the callback traps so we can + // diagnose failures inside the channel implementation. let channel_iface = instance.near_agent_channel(); - let wasm_result = channel_iface + let respond_result = channel_iface .call_on_respond(&mut store, &wit_response) .map_err(|e| { tracing::error!(error = %e, "WASM on_respond call failed"); Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel) - })?; - - tracing::info!(wasm_result = ?wasm_result, "WASM on_respond returned"); - - // Check for WASM-level errors - if let Err(ref err_msg) = wasm_result { - tracing::error!(error = %err_msg, "WASM on_respond returned error"); - return Err(WasmChannelError::CallbackFailed { - name: prepared.name.clone(), - reason: err_msg.clone(), }); - } let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); - let committed_paths = - Self::commit_callback_workspace_writes(&mut host_state, &workspace_store); - tracing::info!("on_respond WASM execution completed successfully"); - Ok(((), host_state, committed_paths)) + + let committed_paths = if matches!(&respond_result, Ok(Ok(()))) { + tracing::info!("on_respond WASM execution completed successfully"); + Self::commit_callback_workspace_writes(&mut host_state, &workspace_store) + } else { + Vec::new() + }; + + Ok((respond_result, host_state, committed_paths)) }) .await .map_err(|e| { @@ -2318,7 +2524,19 @@ impl WasmChannel { let channel_name = self.name.clone(); match result { - Ok(Ok(((), _host_state, committed_paths))) => { + Ok(Ok((respond_result, mut host_state, committed_paths))) => { + let _ = drain_guest_logs(&channel_name, "on_respond", &mut host_state); + let wasm_result = respond_result?; + tracing::info!(wasm_result = ?wasm_result, "WASM on_respond returned"); + + if let Err(ref err_msg) = wasm_result { + tracing::error!(error = %err_msg, "WASM on_respond returned error"); + return Err(WasmChannelError::CallbackFailed { + name: self.name.clone(), + reason: err_msg.clone(), + }); + } + self.persist_durable_workspace_snapshot_if_needed(&committed_paths) .await; tracing::debug!( @@ -2345,6 +2563,7 @@ impl WasmChannel { content: &str, thread_id: Option<&str>, attachments: &[String], + inline_attachments: &[OutgoingAttachment], ) -> Result<(), WasmChannelError> { let _callback_guard = self.callback_lock.lock().await; @@ -2352,7 +2571,7 @@ impl WasmChannel { channel = %self.name, user_id = %user_id, content_len = content.len(), - attachment_count = attachments.len(), + attachment_count = attachments.len() + inline_attachments.len(), "call_on_broadcast invoked" ); @@ -2379,21 +2598,22 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); let user_id = user_id.to_string(); let content = content.to_string(); let thread_id = thread_id.map(|s| s.to_string()); let attachments = attachments.to_vec(); + let inline_attachments = inline_attachments.to_vec(); let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - // Read attachment files from disk - let wit_attachments = read_attachments(&attachments).map_err(|e| { - WasmChannelError::CallbackFailed { - name: prepared.name.clone(), - reason: e, - } - })?; + let wit_attachments = + prepare_response_attachments(&prepared.name, &attachments, &inline_attachments) + .map_err(|e| WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + })?; let mut store = Self::create_store( &runtime, @@ -2402,6 +2622,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -2493,6 +2714,7 @@ impl WasmChannel { .await; let pairing_store = self.pairing_store.clone(); let workspace_store = self.workspace_store.clone(); + let websocket_outbound_tx = self.websocket_outbound_tx.read().await.clone(); let Some(wit_update) = status_to_wit(status, metadata) else { return Ok(()); @@ -2507,6 +2729,7 @@ impl WasmChannel { credentials, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -2593,6 +2816,7 @@ impl WasmChannel { credentials_snapshot, host_credentials, pairing_store, + None, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -2651,6 +2875,40 @@ impl WasmChannel { } } + async fn stash_generated_image_attachment(&self, metadata: &serde_json::Value, path: String) { + let key = generated_image_delivery_key(metadata); + self.pending_generated_image_attachments + .lock() + .await + .entry(key) + .or_default() + .push(path); + } + + async fn take_generated_image_attachments(&self, metadata: &serde_json::Value) -> Vec<String> { + self.pending_generated_image_attachments + .lock() + .await + .remove(&generated_image_delivery_key(metadata)) + .unwrap_or_default() + } + + async fn clear_generated_image_attachments(&self, metadata: &serde_json::Value) { + let paths = self.take_generated_image_attachments(metadata).await; + remove_staged_generated_image_attachments(&paths); + } + + async fn clear_all_generated_image_attachments(&self) { + let paths: Vec<String> = self + .pending_generated_image_attachments + .lock() + .await + .drain() + .flat_map(|(_, paths)| paths) + .collect(); + remove_staged_generated_image_attachments(&paths); + } + /// Handle a status update, managing the typing repeat timer. /// /// On Thinking: fires on_status once, then spawns a background task @@ -2756,6 +3014,34 @@ impl WasmChannel { StatusUpdate::StreamChunk(_) => { // No-op, too noisy } + StatusUpdate::ImageGenerated { data_url, .. } + if uses_wecom_aibot_protocol(&self.capabilities) => + { + // WeCom AI Bot stream replies are updated by req_id/stream id. + // Sending the image as its own final frame before the assistant's + // final text lets the text response overwrite the image. Stage + // it and deliver it with the final on_respond payload instead. + self.cancel_typing_task().await; + + match stage_generated_image_data_url(data_url) { + Ok(path) => { + tracing::debug!( + channel = %self.name, + path = %path, + "Staged generated image for final channel response" + ); + self.stash_generated_image_attachment(metadata, path).await; + } + Err(e) => { + tracing::warn!( + channel = %self.name, + error = %e, + "Failed to stage generated image for channel delivery" + ); + let _ = self.call_on_status(&status, metadata).await; + } + } + } StatusUpdate::ApprovalNeeded { .. } => { // WASM channels (Telegram, Slack, etc.) cannot render // interactive approval overlays. Send the approval prompt @@ -2769,7 +3055,14 @@ impl WasmChannel { let metadata_json = serde_json::to_string(metadata).unwrap_or_default(); if let Err(e) = self - .call_on_respond(uuid::Uuid::new_v4(), &prompt, None, &metadata_json, &[]) + .call_on_respond( + uuid::Uuid::new_v4(), + &prompt, + None, + &metadata_json, + &[], + &[], + ) .await { tracing::warn!( @@ -2796,6 +3089,7 @@ impl WasmChannel { StatusUpdate::Status(msg) if is_terminal_text_status(msg) => { // Waiting on user or terminal states: stop typing and fire once. self.cancel_typing_task().await; + self.clear_generated_image_attachments(metadata).await; if let Err(e) = self.call_on_status(&status, metadata).await { tracing::debug!( @@ -2850,7 +3144,28 @@ impl WasmChannel { tx.clone() }; + let mut hydration_host_state = ChannelHostState::new(&self.name, self.capabilities.clone()); + for emitted in messages { + if emitted.content.trim().is_empty() && emitted.attachments.is_empty() { + tracing::debug!( + channel = %self.name, + user_id = %emitted.user_id, + "Skipping empty emitted message" + ); + continue; + } + + let EmittedMessage { + user_id, + user_name, + content, + thread_id, + metadata_json, + attachments, + .. + } = emitted; + // Check rate limit — acquire and release the write lock before send().await { let mut rate_limiter = self.rate_limiter.write().await; @@ -2865,64 +3180,49 @@ impl WasmChannel { } } - // Clone the owner_actor_id out of the lock to avoid holding - // the read guard across the async resolve call below. + let channel_bound_user_id = self.channel_bound_user_id.read().await.clone(); let owner_actor_id = self.owner_actor_id.read().await.clone(); let (resolved_user_id, is_owner_sender) = resolve_message_scope_with_pairing( &self.name, &self.owner_scope_id, owner_actor_id.as_deref(), - &emitted.user_id, + channel_bound_user_id.as_deref(), + &user_id, self.pairing_store.as_ref(), ) .await; // Convert to IncomingMessage - let mut msg = IncomingMessage::new(&self.name, &resolved_user_id, &emitted.content) - .with_sender_id(&emitted.user_id); + let mut msg = IncomingMessage::new(&self.name, &resolved_user_id, &content) + .with_sender_id(&user_id); - if let Some(name) = emitted.user_name { + if let Some(name) = user_name { msg = msg.with_user_name(name); } - if let Some(thread_id) = emitted.thread_id { + if let Some(thread_id) = thread_id { msg = msg.with_thread(thread_id); } // Convert attachments - if !emitted.attachments.is_empty() { - let incoming_attachments = emitted - .attachments - .iter() - .map(|a| crate::channels::IncomingAttachment { - id: a.id.clone(), - kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), - mime_type: a.mime_type.clone(), - filename: a.filename.clone(), - size_bytes: a.size_bytes, - source_url: a.source_url.clone(), - storage_key: a.storage_key.clone(), - local_path: a.local_path.clone(), - extracted_text: a.extracted_text.clone(), - data: a.data.clone(), - duration_secs: a.duration_secs, - }) - .collect(); + if !attachments.is_empty() { + let incoming_attachments = + convert_emitted_attachments(&mut hydration_host_state, attachments).await; msg = msg.with_attachments(incoming_attachments); } // Parse metadata JSON - msg = apply_emitted_metadata(msg, &emitted.metadata_json); + msg = apply_emitted_metadata(msg, &metadata_json); if is_owner_sender { // Store for owner-target routing (chat_id etc.). - self.update_broadcast_metadata(&emitted.metadata_json).await; + self.update_broadcast_metadata(&metadata_json).await; } // Send to stream — no locks held across this await tracing::info!( channel = %self.name, - user_id = %emitted.user_id, - content_len = emitted.content.len(), + user_id = %user_id, + content_len = content.len(), attachment_count = msg.attachments.len(), "Sending emitted message to agent" ); @@ -3010,6 +3310,7 @@ impl WasmChannel { let settings_store = self.settings_store.clone(); let poll_secrets_store = self.secrets_store.clone(); let owner_scope_id = self.owner_scope_id.clone(); + let channel_bound_user_id = Arc::clone(&self.channel_bound_user_id); tokio::spawn(async move { let mut interval_timer = tokio::time::interval(interval); @@ -3041,6 +3342,7 @@ impl WasmChannel { &credentials, host_credentials, pairing_store.clone(), + None, &workspace_store, &callback_lock, settings_store.as_ref(), @@ -3053,12 +3355,15 @@ impl WasmChannel { // post-approval changes are visible immediately. let current_owner = owner_actor_id.read().await.clone(); // Process any emitted messages + let bound_user_id = channel_bound_user_id.read().await.clone(); if !emitted_messages.is_empty() && let Err(e) = Self::dispatch_emitted_messages( EmitDispatchContext { channel_name: &channel_name, + capabilities: &capabilities, owner_scope_id: &owner_scope_id, owner_actor_id: current_owner.as_deref(), + channel_bound_user_id: bound_user_id.as_deref(), pairing_store: pairing_store.as_ref(), message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -3124,6 +3429,7 @@ impl WasmChannel { credentials: &RwLock<HashMap<String, String>>, host_credentials: Vec<ResolvedHostCredential>, pairing_store: Arc<PairingStore>, + websocket_outbound_tx: Option<mpsc::Sender<String>>, workspace_store: &Arc<ChannelWorkspaceStore>, callback_lock: &Arc<tokio::sync::Mutex<()>>, settings_store: Option<&Arc<dyn crate::db::SettingsStore>>, @@ -3160,25 +3466,29 @@ impl WasmChannel { credentials_snapshot, host_credentials, pairing_store, + websocket_outbound_tx, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Call on_poll using the generated typed interface let channel_iface = instance.near_agent_channel(); - channel_iface + let poll_result = channel_iface .call_on_poll(&mut store) - .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?; + .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel)); let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); - // Commit pending workspace writes to the persistent store - let committed_paths = Self::commit_callback_workspace_writes( - &mut host_state, - &workspace_store_for_callback, - ); + let committed_paths = if poll_result.is_ok() { + Self::commit_callback_workspace_writes( + &mut host_state, + &workspace_store_for_callback, + ) + } else { + Vec::new() + }; - Ok((host_state, committed_paths)) + Ok((poll_result, host_state, committed_paths)) }) .await .map_err(|e| WasmChannelError::ExecutionPanicked { @@ -3189,7 +3499,7 @@ impl WasmChannel { .await; match result { - Ok(Ok((mut host_state, committed_paths))) => { + Ok(Ok((poll_result, mut host_state, committed_paths))) => { if committed_paths.iter().any(|path| { durable_workspace_paths .iter() @@ -3205,6 +3515,7 @@ impl WasmChannel { .await; } let _ = drain_guest_logs(channel_name, "on_poll", &mut host_state); + poll_result?; let emitted = host_state.take_emitted_messages(); tracing::debug!( channel = %channel_name, @@ -3249,7 +3560,28 @@ impl WasmChannel { tx.clone() }; + let mut hydration_host_state = + ChannelHostState::new(dispatch.channel_name, dispatch.capabilities.clone()); + for emitted in messages { + if emitted.content.trim().is_empty() && emitted.attachments.is_empty() { + tracing::debug!( + channel = %dispatch.channel_name, + "Skipping empty emitted message" + ); + continue; + } + + let EmittedMessage { + user_id, + user_name, + content, + thread_id, + metadata_json, + attachments, + .. + } = emitted; + // Check rate limit — acquire and release the write lock before send().await { let mut limiter = dispatch.rate_limiter.write().await; @@ -3268,73 +3600,49 @@ impl WasmChannel { dispatch.channel_name, dispatch.owner_scope_id, dispatch.owner_actor_id, - &emitted.user_id, + dispatch.channel_bound_user_id, + &user_id, dispatch.pairing_store, ) .await; // Convert to IncomingMessage - let mut msg = - IncomingMessage::new(dispatch.channel_name, &resolved_user_id, &emitted.content) - .with_sender_id(&emitted.user_id); + let mut msg = IncomingMessage::new(dispatch.channel_name, &resolved_user_id, &content) + .with_sender_id(&user_id); - if let Some(name) = emitted.user_name { + if let Some(name) = user_name { msg = msg.with_user_name(name); } - if let Some(thread_id) = emitted.thread_id { + if let Some(thread_id) = thread_id { msg = msg.with_thread(thread_id); } // Convert attachments - if !emitted.attachments.is_empty() { - let incoming_attachments = emitted - .attachments - .iter() - .map(|a| crate::channels::IncomingAttachment { - id: a.id.clone(), - kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), - mime_type: a.mime_type.clone(), - filename: a.filename.clone(), - size_bytes: a.size_bytes, - source_url: a.source_url.clone(), - storage_key: a.storage_key.clone(), - local_path: a.local_path.clone(), - extracted_text: a.extracted_text.clone(), - data: a.data.clone(), - duration_secs: a.duration_secs, - }) - .collect(); + if !attachments.is_empty() { + let incoming_attachments = + convert_emitted_attachments(&mut hydration_host_state, attachments).await; msg = msg.with_attachments(incoming_attachments); } - msg = apply_emitted_metadata(msg, &emitted.metadata_json); + msg = apply_emitted_metadata(msg, &metadata_json); if is_owner_sender { // Store for owner-target routing (chat_id etc.) do_update_broadcast_metadata( dispatch.channel_name, dispatch.owner_scope_id, - &emitted.metadata_json, + &metadata_json, dispatch.last_broadcast_metadata, dispatch.settings_store, ) .await; } - if emitted.content.trim().is_empty() && emitted.attachments.is_empty() { - tracing::debug!( - channel = %dispatch.channel_name, - user_id = %emitted.user_id, - "Skipping empty emitted message" - ); - continue; - } - // Send to stream — no locks held across this await tracing::info!( channel = %dispatch.channel_name, - user_id = %emitted.user_id, - content_len = emitted.content.len(), + user_id = %user_id, + content_len = content.len(), attachment_count = msg.attachments.len(), "Sending polled message to agent" ); @@ -3359,8 +3667,10 @@ impl WasmChannel { struct EmitDispatchContext<'a> { channel_name: &'a str, + capabilities: &'a ChannelCapabilities, owner_scope_id: &'a str, owner_actor_id: Option<&'a str>, + channel_bound_user_id: Option<&'a str>, pairing_store: &'a PairingStore, message_tx: &'a RwLock<Option<mpsc::Sender<IncomingMessage>>>, rate_limiter: &'a RwLock<ChannelEmitRateLimiter>, @@ -3524,15 +3834,36 @@ impl Channel for WasmChannel { ) { self.update_broadcast_metadata(&metadata_json).await; } - self.call_on_respond( - msg.id, - &response.content, - response.thread_id.as_ref().map(|t| t.as_str()), - &metadata_json, - &response.attachments, - ) - .await - .map_err(|e| ChannelError::SendFailed { + let generated_image_attachments = + self.take_generated_image_attachments(&msg.metadata).await; + let mut attachments = response.attachments.clone(); + let response_already_has_generated_image = attachments + .iter() + .any(|path| is_staged_generated_image_path(path)) + || !response.inline_attachments.is_empty(); + if response_already_has_generated_image { + tracing::debug!( + channel = %self.name, + skipped = generated_image_attachments.len(), + "Skipping status-staged generated images because the final response already carries generated-image attachments" + ); + } else { + attachments.extend(generated_image_attachments.iter().cloned()); + } + + let result = self + .call_on_respond( + msg.id, + &response.content, + response.thread_id.as_ref().map(|t| t.as_str()), + &metadata_json, + &attachments, + &response.inline_attachments, + ) + .await; + remove_staged_generated_image_attachments(&generated_image_attachments); + + result.map_err(|e| ChannelError::SendFailed { name: self.name.clone(), reason: e.to_string(), })?; @@ -3567,6 +3898,7 @@ impl Channel for WasmChannel { &response.content, response.thread_id.as_ref().map(|t| t.as_str()), &response.attachments, + &response.inline_attachments, ) .await .map_err(|e| ChannelError::SendFailed { @@ -3598,6 +3930,7 @@ impl Channel for WasmChannel { async fn shutdown(&self) -> Result<(), ChannelError> { // Cancel typing indicator self.cancel_typing_task().await; + self.clear_all_generated_image_attachments().await; // Send shutdown signal if let Some(tx) = self.shutdown_tx.write().await.take() { @@ -3608,6 +3941,7 @@ impl Channel for WasmChannel { // Stop websocket runtime by dropping the sender (receiver will complete) let _ = self.websocket_shutdown_tx.write().await.take(); + let _ = self.websocket_outbound_tx.write().await.take(); // Clear the message sender *self.message_tx.write().await = None; @@ -3621,12 +3955,37 @@ impl Channel for WasmChannel { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WebsocketProtocolKind { + DiscordGateway, + WecomAibot, +} + +#[derive(Debug, Clone, PartialEq)] +struct DiscordGatewayWebsocketConfig { + identify: Option<serde_json::Value>, + identify_secret_name: Option<String>, +} + +#[derive(Debug, Clone, PartialEq)] +struct WecomAibotWebsocketConfig { + bot_id_secret_name: String, + bot_secret_name: String, + heartbeat_interval_ms: u64, + max_missed_heartbeat_acks: u32, +} + +#[derive(Debug, Clone, PartialEq)] +enum WebsocketProtocolConfig { + DiscordGateway(DiscordGatewayWebsocketConfig), + WecomAibot(WecomAibotWebsocketConfig), +} + #[derive(Debug, Clone, PartialEq)] pub(crate) struct WebsocketRuntimeConfig { pub(crate) url: String, pub(crate) connect_on_start: bool, - pub(crate) identify: Option<serde_json::Value>, - pub(crate) identify_secret_name: Option<String>, + protocol: WebsocketProtocolConfig, } impl WebsocketRuntimeConfig { @@ -3654,19 +4013,95 @@ impl WebsocketRuntimeConfig { return None; } + let protocol_name = match raw.get("protocol").and_then(serde_json::Value::as_str) { + Some(value) => Some(normalize_websocket_protocol_name(value)?), + None => None, + }; + + let protocol = match protocol_name { + Some("wecom-aibot") => { + let bot_id_secret_name = raw + .get("bot_id_secret_name") + .and_then(serde_json::Value::as_str)? + .trim() + .to_string(); + let bot_secret_name = raw + .get("bot_secret_name") + .or_else(|| raw.get("bot_secret_secret_name")) + .and_then(serde_json::Value::as_str)? + .trim() + .to_string(); + if bot_id_secret_name.is_empty() || bot_secret_name.is_empty() { + return None; + } + WebsocketProtocolConfig::WecomAibot(WecomAibotWebsocketConfig { + bot_id_secret_name, + bot_secret_name, + heartbeat_interval_ms: raw + .get("heartbeat_interval_ms") + .and_then(serde_json::Value::as_u64) + .unwrap_or(30_000), + max_missed_heartbeat_acks: raw + .get("max_missed_heartbeat_acks") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .unwrap_or(2), + }) + } + Some("discord-gateway") | None => { + WebsocketProtocolConfig::DiscordGateway(DiscordGatewayWebsocketConfig { + identify: raw.get("identify").cloned(), + identify_secret_name: raw + .get("identify_secret_name") + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned), + }) + } + Some(_) => return None, + }; + Some(Self { url: url.to_string(), connect_on_start: raw .get("connect_on_start") .and_then(serde_json::Value::as_bool) .unwrap_or(false), - identify: raw.get("identify").cloned(), - identify_secret_name: raw - .get("identify_secret_name") - .and_then(serde_json::Value::as_str) - .map(ToOwned::to_owned), + protocol, }) } + + fn protocol_kind(&self) -> WebsocketProtocolKind { + match self.protocol { + WebsocketProtocolConfig::DiscordGateway(_) => WebsocketProtocolKind::DiscordGateway, + WebsocketProtocolConfig::WecomAibot(_) => WebsocketProtocolKind::WecomAibot, + } + } +} + +fn uses_wecom_aibot_protocol(capabilities: &ChannelCapabilities) -> bool { + matches!( + WebsocketRuntimeConfig::from_capabilities(capabilities).map(|config| config.protocol), + Some(WebsocketProtocolConfig::WecomAibot(_)) + ) +} + +fn normalize_websocket_protocol_name(protocol: &str) -> Option<&'static str> { + match protocol + .trim() + .to_ascii_lowercase() + .replace('_', "-") + .as_str() + { + "discord-gateway" => Some("discord-gateway"), + "wecom-aibot" => Some("wecom-aibot"), + _ => None, + } +} + +#[derive(Debug, Clone, PartialEq)] +enum ResolvedWebsocketAuth { + DiscordIdentify(String), + WecomAibot { bot_id: String, secret: String }, } fn websocket_queue_path(channel_name: &str) -> String { @@ -3677,22 +4112,77 @@ fn websocket_processing_queue_path(channel_name: &str) -> String { format!("channels/{channel_name}/{WEBSOCKET_EVENT_PROCESSING_QUEUE_RELATIVE_PATH}") } -async fn resolve_websocket_identify_message( - config: &WebsocketRuntimeConfig, - store: Option<&(dyn SecretsStore + Send + Sync)>, - owner_scope_id: &str, -) -> Option<String> { - let identify = config.identify.clone()?; - let secret_name = config.identify_secret_name.as_ref()?; - let store = store?; - // Channel runtime secrets are instance-owned, resolved under the channel's owner scope. - let secret = store - .get_decrypted(owner_scope_id, secret_name) - .await - .ok()?; - build_websocket_identify_message(&identify, secret.expose()) -} - +fn queue_websocket_outbound_frame( + sender: &mpsc::Sender<String>, + payload: String, +) -> Result<(), String> { + let payload_len = payload.len(); + if payload_len > WEBSOCKET_OUTBOUND_MAX_FRAME_BYTES { + return Err(format!( + "websocket outbound frame exceeds max size: {payload_len} > {WEBSOCKET_OUTBOUND_MAX_FRAME_BYTES} bytes" + )); + } + + match sender.try_send(payload) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) => { + Err("websocket outbound queue is full".to_string()) + } + Err(mpsc::error::TrySendError::Closed(_)) => { + Err("websocket runtime is disconnected".to_string()) + } + } +} + +async fn resolve_websocket_auth( + config: &WebsocketRuntimeConfig, + store: Option<&(dyn SecretsStore + Send + Sync)>, + owner_scope_id: &str, +) -> Option<ResolvedWebsocketAuth> { + match &config.protocol { + WebsocketProtocolConfig::DiscordGateway(discord) => { + let identify = discord.identify.clone()?; + let secret_name = discord.identify_secret_name.as_ref()?; + let secret = resolve_websocket_secret(store, owner_scope_id, secret_name).await?; + build_websocket_identify_message(&identify, &secret) + .map(ResolvedWebsocketAuth::DiscordIdentify) + } + WebsocketProtocolConfig::WecomAibot(wecom) => { + let bot_id = + resolve_websocket_secret(store, owner_scope_id, &wecom.bot_id_secret_name).await?; + let secret = + resolve_websocket_secret(store, owner_scope_id, &wecom.bot_secret_name).await?; + Some(ResolvedWebsocketAuth::WecomAibot { bot_id, secret }) + } + } +} + +async fn resolve_websocket_secret( + store: Option<&(dyn SecretsStore + Send + Sync)>, + owner_scope_id: &str, + secret_name: &str, +) -> Option<String> { + if let Some(store) = store + && let Ok(secret) = store.get_decrypted(owner_scope_id, secret_name).await + { + return Some(secret.expose().to_string()); + } + + None +} + +#[cfg(test)] +async fn resolve_websocket_identify_message( + config: &WebsocketRuntimeConfig, + store: Option<&(dyn SecretsStore + Send + Sync)>, + owner_scope_id: &str, +) -> Option<String> { + match resolve_websocket_auth(config, store, owner_scope_id).await { + Some(ResolvedWebsocketAuth::DiscordIdentify(payload)) => Some(payload), + _ => None, + } +} + /// Result of the websocket auth preflight performed before spawning the runtime. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum WebsocketAuthPreflight { @@ -3754,16 +4244,32 @@ async fn websocket_auth_preflight( let Some(credential_name) = credential_name else { return WebsocketAuthPreflight::Ready; }; + let env_name = credential_name.as_str().to_uppercase(); + let env_has_value = || { + std::env::var(&env_name) + .ok() + .is_some_and(|value| !value.is_empty()) + }; let Some(store) = store else { - return WebsocketAuthPreflight::MissingCredential { - credential_name: credential_name.clone(), + return if env_has_value() { + WebsocketAuthPreflight::Ready + } else { + WebsocketAuthPreflight::MissingCredential { + credential_name: credential_name.clone(), + } }; }; match store.exists(owner_scope_id, credential_name.as_str()).await { Ok(true) => WebsocketAuthPreflight::Ready, - Ok(false) => WebsocketAuthPreflight::MissingCredential { - credential_name: credential_name.clone(), - }, + Ok(false) => { + if env_has_value() { + WebsocketAuthPreflight::Ready + } else { + WebsocketAuthPreflight::MissingCredential { + credential_name: credential_name.clone(), + } + } + } Err(error) => { tracing::warn!( owner_scope_id = %owner_scope_id, @@ -3778,68 +4284,99 @@ async fn websocket_auth_preflight( /// Compose the websocket start decision from capabilities + auth state. /// -/// Validates `config.identify_secret_name` (a raw string on the capability -/// wire contract) into a [`CredentialName`] at this boundary so that all -/// internal flow below uses the typed identity. Validation failure surfaces -/// as [`WebsocketStartDecision::MalformedConfig`]. +/// Validates protocol-specific secret names (raw strings on the capability +/// wire contract) into [`CredentialName`] values at this boundary so internal +/// flow below uses canonical names. Validation failure surfaces as +/// [`WebsocketStartDecision::MalformedConfig`]. async fn websocket_start_decision( capabilities: &ChannelCapabilities, store: Option<&(dyn SecretsStore + Send + Sync)>, owner_scope_id: &str, ) -> WebsocketStartDecision { - let Some(config) = WebsocketRuntimeConfig::from_capabilities(capabilities) else { + let Some(mut config) = WebsocketRuntimeConfig::from_capabilities(capabilities) else { return WebsocketStartDecision::NotConfigured; }; if !config.connect_on_start { return WebsocketStartDecision::NotConfigured; } - // Validate the declared credential name once, at the boundary between the - // capability JSON (raw string) and internal flow (typed `CredentialName`). - let credential_name = match config.identify_secret_name.as_deref() { - None => None, - Some(raw) => match CredentialName::new(raw) { - Ok(name) => Some(name), - Err(err) => { - return WebsocketStartDecision::MalformedConfig { - reason: format!("invalid identify_secret_name {raw:?}: {err}"), - }; - } - }, - }; - // `identify` and `identify_secret_name` must be declared together: the - // runtime builds an identify payload by filling the template with the - // resolved secret, so either side on its own produces a connection that - // cannot send a valid Identify and will be kicked by the peer. - match (config.identify.as_ref(), credential_name.as_ref()) { - (None, Some(_)) => { - return WebsocketStartDecision::MalformedConfig { - reason: "identify_secret_name declared without identify template".to_string(), + + let credential_names = match &mut config.protocol { + WebsocketProtocolConfig::DiscordGateway(discord) => { + // `identify` and `identify_secret_name` must be declared together: + // the runtime builds an identify payload by filling the template + // with the resolved secret, so either side on its own produces a + // connection that cannot send a valid Identify. + let credential_name = match discord.identify_secret_name.as_deref() { + None => None, + Some(raw) => match CredentialName::new(raw) { + Ok(name) => Some(name), + Err(err) => { + return WebsocketStartDecision::MalformedConfig { + reason: format!("invalid identify_secret_name {raw:?}: {err}"), + }; + } + }, }; + match (discord.identify.as_ref(), credential_name.as_ref()) { + (None, Some(_)) => { + return WebsocketStartDecision::MalformedConfig { + reason: "identify_secret_name declared without identify template" + .to_string(), + }; + } + (Some(_), None) => { + return WebsocketStartDecision::MalformedConfig { + reason: "identify template declared without identify_secret_name" + .to_string(), + }; + } + _ => {} + } + if let Some(name) = credential_name { + discord.identify_secret_name = Some(name.as_str().to_string()); + vec![name] + } else { + Vec::new() + } } - (Some(_), None) => { - return WebsocketStartDecision::MalformedConfig { - reason: "identify template declared without identify_secret_name".to_string(), + WebsocketProtocolConfig::WecomAibot(wecom) => { + let bot_id_name = match CredentialName::new(&wecom.bot_id_secret_name) { + Ok(name) => name, + Err(err) => { + return WebsocketStartDecision::MalformedConfig { + reason: format!( + "invalid bot_id_secret_name {:?}: {err}", + wecom.bot_id_secret_name + ), + }; + } }; + let bot_secret_name = match CredentialName::new(&wecom.bot_secret_name) { + Ok(name) => name, + Err(err) => { + return WebsocketStartDecision::MalformedConfig { + reason: format!( + "invalid bot_secret_name {:?}: {err}", + wecom.bot_secret_name + ), + }; + } + }; + wecom.bot_id_secret_name = bot_id_name.as_str().to_string(); + wecom.bot_secret_name = bot_secret_name.as_str().to_string(); + vec![bot_id_name, bot_secret_name] } - _ => {} - } - // Normalize `config.identify_secret_name` to the canonicalized form - // (`CredentialName::new` trims whitespace and folds `-` → `_`) before - // returning `Spawn`. Preflight checks existence via the canonical form, - // but `resolve_websocket_identify_message` later reads the raw string - // from the config; without this write-back, a capability declaring - // `"github-token"` against a store holding `"github_token"` would pass - // preflight and then fail in the runtime. - let mut config = config; - if let Some(name) = credential_name.as_ref() { - config.identify_secret_name = Some(name.as_str().to_string()); - } - match websocket_auth_preflight(credential_name.as_ref(), store, owner_scope_id).await { - WebsocketAuthPreflight::Ready => WebsocketStartDecision::Spawn(config), - WebsocketAuthPreflight::MissingCredential { credential_name } => { - WebsocketStartDecision::MissingAuth { credential_name } + }; + + for credential_name in credential_names { + if let WebsocketAuthPreflight::MissingCredential { credential_name } = + websocket_auth_preflight(Some(&credential_name), store, owner_scope_id).await + { + return WebsocketStartDecision::MissingAuth { credential_name }; } } + + WebsocketStartDecision::Spawn(config) } /// True if `url` points at a Discord gateway host. @@ -3909,6 +4446,20 @@ fn build_websocket_identify_message(identify: &serde_json::Value, token: &str) - .ok() } +fn build_wecom_aibot_subscribe_message(bot_id: &str, secret: &str) -> Option<String> { + serde_json::to_string(&serde_json::json!({ + "cmd": "aibot_subscribe", + "headers": { + "req_id": websocket_request_id("aibot_subscribe"), + }, + "body": { + "bot_id": bot_id, + "secret": secret, + } + })) + .ok() +} + fn build_websocket_heartbeat_message(sequence: Option<serde_json::Value>) -> Option<String> { serde_json::to_string(&serde_json::json!({ "op": 1, @@ -3917,6 +4468,20 @@ fn build_websocket_heartbeat_message(sequence: Option<serde_json::Value>) -> Opt .ok() } +fn build_wecom_aibot_ping_message() -> Option<String> { + serde_json::to_string(&serde_json::json!({ + "cmd": "ping", + "headers": { + "req_id": websocket_request_id("ping"), + } + })) + .ok() +} + +fn websocket_request_id(prefix: &str) -> String { + format!("{prefix}-{}", Uuid::new_v4()) +} + fn build_discord_gateway_presence_update(status: &str) -> Option<String> { serde_json::to_string(&serde_json::json!({ "op": 3, @@ -3930,20 +4495,18 @@ fn build_discord_gateway_presence_update(status: &str) -> Option<String> { .ok() } -fn build_gateway_presence_update( +fn build_protocol_post_poll_update( + protocol_kind: WebsocketProtocolKind, channel_name: &str, workspace_store: &crate::channels::wasm::host::ChannelWorkspaceStore, pairing_store: &PairingStore, ) -> Option<String> { - if channel_name != "discord" { - return None; + match protocol_kind { + WebsocketProtocolKind::DiscordGateway => build_discord_gateway_presence_update( + discord_gateway_presence_status(channel_name, workspace_store, pairing_store), + ), + WebsocketProtocolKind::WecomAibot => None, } - - build_discord_gateway_presence_update(discord_gateway_presence_status( - channel_name, - workspace_store, - pairing_store, - )) } fn discord_gateway_presence_status( @@ -4048,6 +4611,61 @@ fn extract_token_from_identify_payload(identify_payload: &str) -> Option<String> .map(ToOwned::to_owned) } +struct WecomAibotAckFrame { + req_id: String, + errcode: i64, + errmsg: Option<String>, +} + +enum WecomAibotIncomingFrame { + Ack(WecomAibotAckFrame), + Callback, + Event { disconnected: bool }, + Unknown, +} + +fn parse_wecom_aibot_frame(text: &str) -> WecomAibotIncomingFrame { + let payload: serde_json::Value = match serde_json::from_str(text) { + Ok(payload) => payload, + Err(_) => return WecomAibotIncomingFrame::Unknown, + }; + + if let Some(cmd) = payload.get("cmd").and_then(serde_json::Value::as_str) { + if cmd == "aibot_msg_callback" { + return WecomAibotIncomingFrame::Callback; + } + + if cmd.ends_with("_event") || cmd == "aibot_event_callback" { + let disconnected = payload + .get("body") + .and_then(|body| body.get("event")) + .and_then(|event| event.get("eventtype")) + .and_then(serde_json::Value::as_str) + == Some("disconnected_event"); + + return WecomAibotIncomingFrame::Event { disconnected }; + } + } + + if let Some(req_id) = payload + .get("headers") + .and_then(|headers| headers.get("req_id")) + .and_then(serde_json::Value::as_str) + && let Some(errcode) = payload.get("errcode").and_then(serde_json::Value::as_i64) + { + return WecomAibotIncomingFrame::Ack(WecomAibotAckFrame { + req_id: req_id.to_string(), + errcode, + errmsg: payload + .get("errmsg") + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned), + }); + } + + WecomAibotIncomingFrame::Unknown +} + fn drain_guest_logs( channel_name: &str, callback: &str, @@ -4098,8 +4716,10 @@ struct WebsocketPollContext { settings_store: Option<Arc<dyn crate::db::SettingsStore>>, owner_scope_id: String, owner_actor_id: Arc<tokio::sync::RwLock<Option<String>>>, + channel_bound_user_id: Arc<RwLock<Option<String>>>, secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>, - outbound_tx: mpsc::UnboundedSender<String>, + protocol_kind: WebsocketProtocolKind, + outbound_tx: mpsc::Sender<String>, queue_path: String, processing_queue_path: String, callback_timeout: Duration, @@ -4145,6 +4765,7 @@ fn spawn_websocket_poll(poll_guard: tokio::sync::OwnedMutexGuard<()>, ctx: Webso &ctx.credentials, host_credentials, ctx.pairing_store.clone(), + Some(ctx.outbound_tx.clone()), &ctx.workspace_store, &ctx.callback_lock, ctx.settings_store.as_ref(), @@ -4156,12 +4777,15 @@ fn spawn_websocket_poll(poll_guard: tokio::sync::OwnedMutexGuard<()>, ctx: Webso // Read the current owner so post-approval changes // are visible immediately. let current_owner = ctx.owner_actor_id.read().await.clone(); + let bound_user_id = ctx.channel_bound_user_id.read().await.clone(); if !emitted_messages.is_empty() && let Err(error) = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: &ctx.channel_name, + capabilities: &ctx.capabilities, owner_scope_id: &ctx.owner_scope_id, owner_actor_id: current_owner.as_deref(), + channel_bound_user_id: bound_user_id.as_deref(), pairing_store: ctx.pairing_store.as_ref(), message_tx: &ctx.message_tx, rate_limiter: &ctx.rate_limiter, @@ -4177,15 +4801,33 @@ fn spawn_websocket_poll(poll_guard: tokio::sync::OwnedMutexGuard<()>, ctx: Webso } Err(error) => { tracing::warn!(channel = %ctx.channel_name, error = %error, "Websocket-triggered poll failed"); + if let Err(restore_error) = ctx.workspace_store.restore_json_text_queue( + &ctx.queue_path, + &ctx.processing_queue_path, + WEBSOCKET_EVENT_QUEUE_MAX_ITEMS, + ) { + tracing::warn!( + channel = %ctx.channel_name, + error = %restore_error, + "Failed to restore websocket queue after poll failure" + ); + } + break; } } - if let Some(payload) = build_gateway_presence_update( + if let Some(payload) = build_protocol_post_poll_update( + ctx.protocol_kind, &ctx.channel_name, ctx.workspace_store.as_ref(), ctx.pairing_store.as_ref(), - ) { - let _ = ctx.outbound_tx.send(payload); + ) && let Err(error) = queue_websocket_outbound_frame(&ctx.outbound_tx, payload) + { + tracing::warn!( + channel = %ctx.channel_name, + error = %error, + "Failed to queue websocket post-poll update" + ); } } }); @@ -4203,16 +4845,20 @@ enum WebsocketFrameAction { Send(String), /// Enqueue the raw text into the workspace event queue. Enqueue(String), - /// Clear session state and reconnect with a fresh identify. - InvalidateAndReconnect, + /// Reconnect, optionally resetting protocol session state first. + Reconnect { reset_session: bool }, + /// Stop the runtime without reconnecting. + StopRuntime, } -/// Tracks websocket session state across reconnects. -/// -/// Keeps heartbeat interval, sequence counter, and Discord Gateway session -/// resumption fields. The [`process_text_frame`] method parses incoming frames -/// and returns a list of [`WebsocketFrameAction`]s the caller should execute. -struct WebsocketSessionState { +enum WebsocketHeartbeatAction { + None, + Send(String), + Reconnect, +} + +/// Tracks Discord Gateway session state across reconnects. +struct DiscordGatewaySessionState { heartbeat_interval_ms: Option<u64>, last_sequence: Option<serde_json::Value>, session_id: Option<String>, @@ -4223,7 +4869,7 @@ struct WebsocketSessionState { attempted_resume: bool, } -impl WebsocketSessionState { +impl DiscordGatewaySessionState { fn new(identify_payload: Option<&str>) -> Self { let token = identify_payload.and_then(extract_token_from_identify_payload); Self { @@ -4312,9 +4958,9 @@ impl WebsocketSessionState { self.session_id = Some(sid); self.resume_gateway_url = resume_url; - if let Some(payload) = - build_gateway_presence_update(channel_name, workspace_store, pairing_store) - { + if let Some(payload) = build_discord_gateway_presence_update( + discord_gateway_presence_status(channel_name, workspace_store, pairing_store), + ) { actions.push(WebsocketFrameAction::Send(payload)); } } @@ -4333,7 +4979,9 @@ impl WebsocketSessionState { "Received non-resumable invalid session; will reconnect with fresh identify" ); self.invalidate_session(); - actions.push(WebsocketFrameAction::InvalidateAndReconnect); + actions.push(WebsocketFrameAction::Reconnect { + reset_session: false, + }); return actions; } @@ -4344,6 +4992,207 @@ impl WebsocketSessionState { } } +struct WecomAibotSessionState { + heartbeat_interval_ms: u64, + max_missed_heartbeat_acks: u32, + missed_heartbeat_acks: u32, +} + +impl WecomAibotSessionState { + fn new(heartbeat_interval_ms: u64, max_missed_heartbeat_acks: u32) -> Self { + Self { + heartbeat_interval_ms: heartbeat_interval_ms.max(1), + max_missed_heartbeat_acks: max_missed_heartbeat_acks.max(1), + missed_heartbeat_acks: 0, + } + } + + fn reset_connection(&mut self) { + self.missed_heartbeat_acks = 0; + } + + fn on_connected(&mut self, auth: Option<&ResolvedWebsocketAuth>) -> Vec<WebsocketFrameAction> { + let Some(ResolvedWebsocketAuth::WecomAibot { bot_id, secret }) = auth else { + return Vec::new(); + }; + + build_wecom_aibot_subscribe_message(bot_id, secret) + .map(|payload| vec![WebsocketFrameAction::Send(payload)]) + .unwrap_or_default() + } + + fn heartbeat_tick(&mut self, channel_name: &str) -> WebsocketHeartbeatAction { + if self.missed_heartbeat_acks >= self.max_missed_heartbeat_acks { + tracing::warn!( + channel = %channel_name, + missed_heartbeat_acks = self.missed_heartbeat_acks, + max_missed_heartbeat_acks = self.max_missed_heartbeat_acks, + "Websocket heartbeat ack threshold exceeded; reconnecting" + ); + return WebsocketHeartbeatAction::Reconnect; + } + + self.missed_heartbeat_acks = self.missed_heartbeat_acks.saturating_add(1); + build_wecom_aibot_ping_message() + .map(WebsocketHeartbeatAction::Send) + .unwrap_or(WebsocketHeartbeatAction::None) + } + + fn process_text_frame(&mut self, text: &str, channel_name: &str) -> Vec<WebsocketFrameAction> { + match parse_wecom_aibot_frame(text) { + WecomAibotIncomingFrame::Ack(ack) => { + if ack.req_id.starts_with("aibot_subscribe") { + if ack.errcode != 0 { + tracing::warn!( + channel = %channel_name, + req_id = %ack.req_id, + errcode = ack.errcode, + errmsg = ack.errmsg.as_deref().unwrap_or(""), + "WeCom websocket auth failed; reconnecting" + ); + return vec![WebsocketFrameAction::Reconnect { + reset_session: false, + }]; + } + + self.missed_heartbeat_acks = 0; + return vec![WebsocketFrameAction::SetHeartbeat { + interval_ms: self.heartbeat_interval_ms, + }]; + } + + if ack.req_id.starts_with("ping") { + if ack.errcode != 0 { + tracing::warn!( + channel = %channel_name, + req_id = %ack.req_id, + errcode = ack.errcode, + errmsg = ack.errmsg.as_deref().unwrap_or(""), + "WeCom websocket heartbeat ack returned error" + ); + } else { + self.missed_heartbeat_acks = 0; + } + return Vec::new(); + } + + vec![WebsocketFrameAction::Enqueue(text.to_string())] + } + WecomAibotIncomingFrame::Event { + disconnected: true, .. + } => { + tracing::warn!( + channel = %channel_name, + "WeCom websocket received disconnected_event; stopping runtime" + ); + vec![ + WebsocketFrameAction::Enqueue(text.to_string()), + WebsocketFrameAction::StopRuntime, + ] + } + WecomAibotIncomingFrame::Callback + | WecomAibotIncomingFrame::Event { .. } + | WecomAibotIncomingFrame::Unknown => { + vec![WebsocketFrameAction::Enqueue(text.to_string())] + } + } + } +} + +enum WebsocketSessionState { + Discord(DiscordGatewaySessionState), + WecomAibot(WecomAibotSessionState), +} + +impl WebsocketSessionState { + fn new(config: &WebsocketRuntimeConfig, auth: Option<&ResolvedWebsocketAuth>) -> Self { + match &config.protocol { + WebsocketProtocolConfig::DiscordGateway(_) => { + let identify_payload = match auth { + Some(ResolvedWebsocketAuth::DiscordIdentify(payload)) => Some(payload.as_str()), + _ => None, + }; + Self::Discord(DiscordGatewaySessionState::new(identify_payload)) + } + WebsocketProtocolConfig::WecomAibot(wecom) => { + Self::WecomAibot(WecomAibotSessionState::new( + wecom.heartbeat_interval_ms, + wecom.max_missed_heartbeat_acks, + )) + } + } + } + + fn connect_url<'a>(&'a self, default_url: &'a str) -> &'a str { + match self { + Self::Discord(state) => state.connect_url(default_url), + Self::WecomAibot(_) => default_url, + } + } + + fn reset_connection(&mut self) { + match self { + Self::Discord(state) => state.reset_connection(), + Self::WecomAibot(state) => state.reset_connection(), + } + } + + fn invalidate_session(&mut self) { + if let Self::Discord(state) = self { + state.invalidate_session(); + } + } + + fn heartbeat_interval_ms(&self) -> Option<u64> { + match self { + Self::Discord(state) => state.heartbeat_interval_ms, + Self::WecomAibot(state) => Some(state.heartbeat_interval_ms), + } + } + + fn on_connected(&mut self, auth: Option<&ResolvedWebsocketAuth>) -> Vec<WebsocketFrameAction> { + match self { + Self::Discord(_) => Vec::new(), + Self::WecomAibot(state) => state.on_connected(auth), + } + } + + fn heartbeat_tick(&mut self, channel_name: &str) -> WebsocketHeartbeatAction { + match self { + Self::Discord(state) => build_websocket_heartbeat_message(state.last_sequence.clone()) + .map(WebsocketHeartbeatAction::Send) + .unwrap_or(WebsocketHeartbeatAction::None), + Self::WecomAibot(state) => state.heartbeat_tick(channel_name), + } + } + + fn process_text_frame( + &mut self, + text: &str, + channel_name: &str, + auth: Option<&ResolvedWebsocketAuth>, + workspace_store: &crate::channels::wasm::host::ChannelWorkspaceStore, + pairing_store: &PairingStore, + ) -> Vec<WebsocketFrameAction> { + match self { + Self::Discord(state) => { + let identify_payload = match auth { + Some(ResolvedWebsocketAuth::DiscordIdentify(payload)) => Some(payload.as_str()), + _ => None, + }; + state.process_text_frame( + text, + channel_name, + identify_payload, + workspace_store, + pairing_store, + ) + } + Self::WecomAibot(state) => state.process_text_frame(text, channel_name), + } + } +} + fn log_websocket_diagnostic(channel_name: &str, message: &WebsocketMessage) { match message { WebsocketMessage::Text(text) => { @@ -4839,6 +5688,15 @@ fn is_loopback_test_rewrite_base(base: &str) -> bool { let Some(host) = parsed.host_str() else { return false; }; + // The `url` crate keeps IPv6 brackets on `host_str()`. Strip them + // so `::1` parses as `IpAddr::V6` — without this, valid IPv6 + // loopback rewrite targets (`http://[::1]:8443`) silently fall + // through and are rejected. Mirror in + // `tools::wasm::wrapper::is_loopback_test_rewrite_base`. + let host = host + .strip_prefix('[') + .and_then(|v| v.strip_suffix(']')) + .unwrap_or(host); host.eq_ignore_ascii_case("localhost") || host .parse::<std::net::IpAddr>() @@ -4982,6 +5840,37 @@ async fn resolve_channel_host_credentials( /// Maximum total attachment size (50 MB). const MAX_TOTAL_ATTACHMENT_BYTES: u64 = 50 * 1024 * 1024; +async fn convert_emitted_attachments( + hydration_host_state: &mut ChannelHostState, + attachments: Vec<crate::channels::wasm::host::Attachment>, +) -> Vec<crate::channels::IncomingAttachment> { + let mut hydrated = attachments; + for attachment in &mut hydrated { + crate::channels::wasm::attachment_hydration::hydrate_attachment_for_channel( + hydration_host_state, + attachment, + ) + .await; + } + + hydrated + .into_iter() + .map(|attachment| crate::channels::IncomingAttachment { + id: attachment.id, + kind: crate::channels::AttachmentKind::from_mime_type(&attachment.mime_type), + mime_type: attachment.mime_type, + filename: attachment.filename, + size_bytes: attachment.size_bytes, + source_url: attachment.source_url, + storage_key: attachment.storage_key, + local_path: attachment.local_path, + extracted_text: attachment.extracted_text, + data: attachment.data, + duration_secs: attachment.duration_secs, + }) + .collect() +} + /// Detect MIME type from file extension using the `mime_guess` crate. fn mime_from_extension(path: &str) -> String { mime_guess::from_path(path) @@ -4989,15 +5878,34 @@ fn mime_from_extension(path: &str) -> String { .to_string() } -/// Read attachment files from disk and build WIT attachment records. +fn generated_image_delivery_key(metadata: &serde_json::Value) -> String { + if let Some(req_id) = metadata.get("ws_req_id").and_then(|v| v.as_str()) + && !req_id.is_empty() + { + return format!("wecom-ws:{req_id}"); + } + if let Some(msg_id) = metadata.get("source_msg_id").and_then(|v| v.as_str()) + && !msg_id.is_empty() + { + return format!("wecom-source:{msg_id}"); + } + + serde_json::to_string(metadata).unwrap_or_default() +} + +/// Build WIT attachment records from file paths and in-memory attachments. /// /// Validates total size against `MAX_TOTAL_ATTACHMENT_BYTES`. -fn read_attachments(paths: &[String]) -> Result<Vec<wit_channel::Attachment>, String> { - if paths.is_empty() { +fn prepare_response_attachments( + channel_name: &str, + paths: &[String], + inline_attachments: &[OutgoingAttachment], +) -> Result<Vec<wit_channel::Attachment>, String> { + if paths.is_empty() && inline_attachments.is_empty() { return Ok(Vec::new()); } - let mut attachments = Vec::with_capacity(paths.len()); + let mut attachments = Vec::with_capacity(paths.len() + inline_attachments.len()); let mut total_bytes: u64 = 0; let tmp_base = std::path::Path::new("/tmp"); let home_base = dirs::home_dir() @@ -5029,6 +5937,11 @@ fn read_attachments(paths: &[String]) -> Result<Vec<wit_channel::Attachment>, St let data = std::fs::read(&validated) .map_err(|e| format!("Failed to read attachment '{}': {}", validated.display(), e))?; + let data = + crate::channels::wasm::attachment_hydration::prepare_outbound_attachment_for_channel( + channel_name, + &data, + )?; let filename = validated .file_name() @@ -5045,11 +5958,34 @@ fn read_attachments(paths: &[String]) -> Result<Vec<wit_channel::Attachment>, St }); } + for attachment in inline_attachments { + total_bytes += attachment.data.len() as u64; + if total_bytes > MAX_TOTAL_ATTACHMENT_BYTES { + return Err(format!( + "Total attachment size exceeds {} MB limit", + MAX_TOTAL_ATTACHMENT_BYTES / (1024 * 1024) + )); + } + + let data = + crate::channels::wasm::attachment_hydration::prepare_outbound_attachment_for_channel( + channel_name, + &attachment.data, + )?; + + attachments.push(wit_channel::Attachment { + filename: attachment.filename.clone(), + mime_type: attachment.mime_type.clone(), + data, + }); + } + Ok(attachments) } #[cfg(test)] mod tests { + use futures::StreamExt; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -5058,23 +5994,29 @@ mod tests { use secrecy::SecretString; use crate::channels::Channel; - use crate::channels::OutgoingResponse; use crate::channels::wasm::capabilities::ChannelCapabilities; use crate::channels::wasm::host::{ChannelHostState, PendingWorkspaceWrite}; use crate::channels::wasm::runtime::{ PreparedChannelModule, WasmChannelRuntime, WasmChannelRuntimeConfig, }; use crate::channels::wasm::wrapper::{ - EmitDispatchContext, HttpResponse, TELEGRAM_TEST_API_BASE_ENV, TEST_HTTP_REWRITE_MAP_ENV, - WasmChannel, WebsocketAuthPreflight, WebsocketCloseDisposition, WebsocketRuntimeConfig, - WebsocketStartDecision, build_discord_gateway_presence_update, - build_websocket_identify_message, build_websocket_resume_message, - classify_websocket_close_code, discord_gateway_presence_status, drain_guest_logs, - parse_websocket_invalid_session, parse_websocket_ready_session, - resolve_websocket_identify_message, rewrite_http_url_for_testing, - should_warn_on_heartbeat_interval, uses_owner_broadcast_target, websocket_auth_preflight, - websocket_heartbeat_sleep_duration, websocket_reconnect_backoff, websocket_start_decision, + DiscordGatewayWebsocketConfig, EmitDispatchContext, HttpResponse, + TELEGRAM_TEST_API_BASE_ENV, TEST_HTTP_REWRITE_MAP_ENV, WEBSOCKET_EVENT_QUEUE_MAX_ITEMS, + WasmChannel, WebsocketAuthPreflight, WebsocketCloseDisposition, WebsocketFrameAction, + WebsocketPollContext, WebsocketProtocolConfig, WebsocketProtocolKind, + WebsocketRuntimeConfig, WebsocketStartDecision, WecomAibotSessionState, + build_discord_gateway_presence_update, build_websocket_identify_message, + build_websocket_resume_message, build_wecom_aibot_ping_message, + build_wecom_aibot_subscribe_message, classify_websocket_close_code, + discord_gateway_presence_status, drain_guest_logs, generated_image_delivery_key, + parse_websocket_invalid_session, parse_websocket_ready_session, parse_wecom_aibot_frame, + prepare_response_attachments, resolve_websocket_identify_message, + rewrite_http_url_for_testing, should_warn_on_heartbeat_interval, spawn_websocket_poll, + uses_owner_broadcast_target, websocket_auth_preflight, websocket_heartbeat_sleep_duration, + websocket_processing_queue_path, websocket_queue_path, websocket_reconnect_backoff, + websocket_start_decision, }; + use crate::channels::{OutgoingAttachment, OutgoingResponse}; use crate::pairing::PairingStore; use crate::secrets::{CreateSecretParams, InMemorySecretsStore, SecretsCrypto, SecretsStore}; use crate::testing::credentials::{TEST_CRYPTO_KEY, TEST_TELEGRAM_BOT_TOKEN}; @@ -5147,6 +6089,117 @@ mod tests { ) } + fn load_or_build_wecom_component_wasm() -> Vec<u8> { + let fixture_path = std::path::Path::new("channels-src/wecom/wecom.wasm"); + let source_paths = [ + std::path::Path::new("channels-src/wecom/Cargo.toml"), + std::path::Path::new("channels-src/wecom/src/lib.rs"), + ]; + let newest_source_mtime = source_paths + .iter() + .filter_map(|path| std::fs::metadata(path).ok()?.modified().ok()) + .max(); + let fixture_is_fresh = std::fs::metadata(fixture_path) + .ok() + .and_then(|metadata| metadata.modified().ok()) + .is_some_and(|fixture_mtime| { + newest_source_mtime.is_none_or(|source_mtime| fixture_mtime >= source_mtime) + }); + if fixture_is_fresh + && let Ok(bytes) = std::fs::read(fixture_path) + && !bytes.is_empty() + { + return bytes; + } + + let status = std::process::Command::new("cargo") + .args([ + "build", + "--manifest-path", + "channels-src/wecom/Cargo.toml", + "--target", + "wasm32-wasip2", + "--release", + ]) + .status() + .expect("failed to invoke cargo to build WeCom wasm fixture"); + assert!( + status.success(), + "failed to build WeCom wasm fixture for component tests" + ); + + let built_path = std::path::Path::new( + "channels-src/wecom/target/wasm32-wasip2/release/wecom_channel.wasm", + ); + std::fs::read(built_path).unwrap_or_else(|err| { + panic!( + "WeCom component fixture is missing and built artifact could not be read at {}: {err}", + built_path.display() + ) + }) + } + + async fn create_wecom_component_test_channel() -> WasmChannel { + let config = WasmChannelRuntimeConfig::for_testing(); + let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap()); + let wasm_bytes = load_or_build_wecom_component_wasm(); + let prepared = runtime + .prepare( + "wecom", + &wasm_bytes, + None, + Some("WeCom test channel".to_string()), + ) + .await + .expect("prepare wecom wasm"); + let capabilities = wecom_websocket_capabilities().with_path("/webhook/wecom"); + + WasmChannel::new( + runtime, + prepared, + capabilities, + "default", + "{}".to_string(), + Arc::new(PairingStore::new_noop()), + None, + ) + } + + async fn spawn_wecom_websocket_poll_for_test( + channel: &WasmChannel, + outbound_tx: tokio::sync::mpsc::Sender<String>, + callback_timeout: Duration, + ) { + let poll_guard = channel.websocket_poll_lock.clone().lock_owned().await; + spawn_websocket_poll( + poll_guard, + WebsocketPollContext { + channel_name: "wecom".to_string(), + runtime: Arc::clone(&channel.runtime), + prepared: Arc::clone(&channel.prepared), + capabilities: channel.capabilities.clone(), + poll_capabilities: channel.capabilities.clone(), + credentials: Arc::clone(&channel.credentials), + pairing_store: Arc::clone(&channel.pairing_store), + workspace_store: Arc::clone(&channel.workspace_store), + callback_lock: Arc::clone(&channel.callback_lock), + message_tx: Arc::clone(&channel.message_tx), + rate_limiter: Arc::clone(&channel.rate_limiter), + last_broadcast_metadata: Arc::clone(&channel.last_broadcast_metadata), + settings_store: channel.settings_store.clone(), + owner_scope_id: channel.owner_scope_id.clone(), + owner_actor_id: Arc::clone(&channel.owner_actor_id), + channel_bound_user_id: Arc::clone(&channel.channel_bound_user_id), + secrets_store: channel.secrets_store.clone(), + protocol_kind: WebsocketProtocolKind::WecomAibot, + outbound_tx, + queue_path: websocket_queue_path("wecom"), + processing_queue_path: websocket_processing_queue_path("wecom"), + callback_timeout, + }, + ); + } + #[cfg(feature = "libsql")] fn create_test_slack_channel_with_settings_store( settings_store: Arc<dyn crate::db::SettingsStore>, @@ -5286,6 +6339,51 @@ mod tests { } } + #[test] + fn test_prepare_response_attachments_prepares_wechat_file_payloads() { + let mut file = tempfile::NamedTempFile::new_in("/tmp").expect("tempfile"); + std::io::Write::write_all(&mut file, b"wechat image bytes").expect("write"); + let path = file.path().to_string_lossy().to_string(); + + let attachments = + prepare_response_attachments("wechat", &[path], &[]).expect("read attachments"); + assert_eq!(attachments.len(), 1); + assert_eq!( + attachments[0].filename, + file.path().file_name().unwrap().to_string_lossy() + ); + assert_eq!(attachments[0].mime_type, "application/octet-stream"); + assert_ne!(attachments[0].data, b"wechat image bytes"); + } + + #[test] + fn test_prepare_response_attachments_passthrough_for_non_wechat_channels() { + let mut file = tempfile::NamedTempFile::new_in("/tmp").expect("tempfile"); + std::io::Write::write_all(&mut file, b"plain bytes").expect("write"); + let path = file.path().to_string_lossy().to_string(); + + let attachments = + prepare_response_attachments("telegram", &[path], &[]).expect("read attachments"); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].data, b"plain bytes"); + } + + #[test] + fn test_prepare_response_attachments_accepts_inline_payloads() { + let inline = OutgoingAttachment { + filename: "generated.png".to_string(), + mime_type: "image/png".to_string(), + data: b"plain bytes".to_vec(), + }; + + let attachments = + prepare_response_attachments("telegram", &[], &[inline]).expect("inline attachment"); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].filename, "generated.png"); + assert_eq!(attachments[0].mime_type, "image/png"); + assert_eq!(attachments[0].data, b"plain bytes"); + } + #[test] fn test_websocket_runtime_config_reads_capability_payload() { let mut tool_capabilities = ToolCapabilities::default(); @@ -5326,21 +6424,94 @@ mod tests { assert_eq!(config.url, "wss://gateway.discord.gg/?v=10&encoding=json"); assert!(config.connect_on_start); - assert_eq!( - config.identify_secret_name.as_deref(), - Some("discord_bot_token") - ); - assert_eq!( - config.identify, - Some(serde_json::json!({ - "intents": 513, - "properties": { - "os": "linux", - "browser": "ironclaw", - "device": "ironclaw" - } - })) - ); + match config.protocol { + WebsocketProtocolConfig::DiscordGateway(discord) => { + assert_eq!( + discord.identify_secret_name.as_deref(), + Some("discord_bot_token") + ); + assert_eq!( + discord.identify, + Some(serde_json::json!({ + "intents": 513, + "properties": { + "os": "linux", + "browser": "ironclaw", + "device": "ironclaw" + } + })) + ); + } + WebsocketProtocolConfig::WecomAibot(_) => panic!("expected discord websocket config"), + } + } + + #[test] + fn test_wecom_websocket_runtime_config_reads_capability_payload() { + let tool_capabilities = ToolCapabilities { + http: Some(HttpCapability::new(vec![EndpointPattern::host( + "openws.work.weixin.qq.com", + )])), + websocket: Some(serde_json::json!({ + "url": "wss://openws.work.weixin.qq.com", + "connect_on_start": true, + "protocol": "wecom_aibot", + "bot_id_secret_name": "wecom_bot_id", + "bot_secret_name": "wecom_bot_secret", + "heartbeat_interval_ms": 45000, + "max_missed_heartbeat_acks": 3 + })), + ..Default::default() + }; + + let capabilities = + ChannelCapabilities::for_channel("wecom").with_tool_capabilities(tool_capabilities); + + let config = WebsocketRuntimeConfig::from_capabilities(&capabilities) + .expect("websocket config should be parsed"); + + assert_eq!(config.url, "wss://openws.work.weixin.qq.com"); + assert!(config.connect_on_start); + match config.protocol { + WebsocketProtocolConfig::WecomAibot(wecom) => { + assert_eq!(wecom.bot_id_secret_name, "wecom_bot_id"); + assert_eq!(wecom.bot_secret_name, "wecom_bot_secret"); + assert_eq!(wecom.heartbeat_interval_ms, 45_000); + assert_eq!(wecom.max_missed_heartbeat_acks, 3); + } + WebsocketProtocolConfig::DiscordGateway(_) => panic!("expected wecom websocket config"), + } + } + + #[test] + fn test_wecom_websocket_runtime_config_accepts_legacy_secret_field_alias() { + let tool_capabilities = ToolCapabilities { + http: Some(HttpCapability::new(vec![EndpointPattern::host( + "openws.work.weixin.qq.com", + )])), + websocket: Some(serde_json::json!({ + "url": "wss://openws.work.weixin.qq.com", + "connect_on_start": true, + "protocol": "wecom-aibot", + "bot_id_secret_name": "wecom_bot_id", + "bot_secret_secret_name": "wecom_bot_secret" + })), + ..Default::default() + }; + + let capabilities = + ChannelCapabilities::for_channel("wecom").with_tool_capabilities(tool_capabilities); + + let config = WebsocketRuntimeConfig::from_capabilities(&capabilities) + .expect("legacy websocket config should still be parsed"); + + match config.protocol { + WebsocketProtocolConfig::WecomAibot(wecom) => { + assert_eq!(wecom.bot_id_secret_name, "wecom_bot_id"); + assert_eq!(wecom.bot_secret_name, "wecom_bot_secret"); + } + WebsocketProtocolConfig::DiscordGateway(_) => panic!("expected wecom websocket config"), + } } #[test] @@ -5398,11 +6569,13 @@ mod tests { let config = WebsocketRuntimeConfig { url: "wss://gateway.discord.gg/?v=10&encoding=json".to_string(), connect_on_start: true, - identify: Some(serde_json::json!({ - "intents": 513, - "properties": { "os": "linux", "browser": "ironclaw", "device": "ironclaw" } - })), - identify_secret_name: Some("discord_bot_token".to_string()), + protocol: WebsocketProtocolConfig::DiscordGateway(DiscordGatewayWebsocketConfig { + identify: Some(serde_json::json!({ + "intents": 513, + "properties": { "os": "linux", "browser": "ironclaw", "device": "ironclaw" } + })), + identify_secret_name: Some("discord_bot_token".to_string()), + }), }; let payload = @@ -5423,22 +6596,176 @@ mod tests { } #[test] - fn test_websocket_runtime_config_requires_allowlisted_host() { - let tool_capabilities = ToolCapabilities { - http: Some(HttpCapability::new(vec![EndpointPattern::host( - "discord.com", - )])), - websocket: Some(serde_json::json!({ - "url": "wss://gateway.discord.gg/?v=10&encoding=json", - "connect_on_start": true - })), - ..Default::default() - }; + fn test_websocket_runtime_config_requires_allowlisted_host() { + let tool_capabilities = ToolCapabilities { + http: Some(HttpCapability::new(vec![EndpointPattern::host( + "discord.com", + )])), + websocket: Some(serde_json::json!({ + "url": "wss://gateway.discord.gg/?v=10&encoding=json", + "connect_on_start": true + })), + ..Default::default() + }; + + let capabilities = + ChannelCapabilities::for_channel("discord").with_tool_capabilities(tool_capabilities); + + assert!(WebsocketRuntimeConfig::from_capabilities(&capabilities).is_none()); + } + + #[test] + fn test_websocket_runtime_config_rejects_unknown_protocol() { + let tool_capabilities = ToolCapabilities { + http: Some(HttpCapability::new(vec![EndpointPattern::host( + "gateway.discord.gg", + )])), + websocket: Some(serde_json::json!({ + "url": "wss://gateway.discord.gg/?v=10&encoding=json", + "connect_on_start": true, + "protocol": "something_else" + })), + ..Default::default() + }; + + let capabilities = + ChannelCapabilities::for_channel("discord").with_tool_capabilities(tool_capabilities); + + assert!(WebsocketRuntimeConfig::from_capabilities(&capabilities).is_none()); + } + + #[test] + fn test_build_wecom_aibot_messages_include_expected_commands() { + let subscribe = build_wecom_aibot_subscribe_message("bot-123", "secret-456").unwrap(); + let subscribe_json: serde_json::Value = serde_json::from_str(&subscribe).unwrap(); + assert_eq!(subscribe_json["cmd"], serde_json::json!("aibot_subscribe")); + assert_eq!( + subscribe_json["body"]["bot_id"], + serde_json::json!("bot-123") + ); + assert_eq!( + subscribe_json["body"]["secret"], + serde_json::json!("secret-456") + ); + assert!( + subscribe_json["headers"]["req_id"] + .as_str() + .is_some_and(|value| value.starts_with("aibot_subscribe-")) + ); + + let ping = build_wecom_aibot_ping_message().unwrap(); + let ping_json: serde_json::Value = serde_json::from_str(&ping).unwrap(); + assert_eq!(ping_json["cmd"], serde_json::json!("ping")); + assert!( + ping_json["headers"]["req_id"] + .as_str() + .is_some_and(|value| value.starts_with("ping-")) + ); + } + + #[test] + fn test_parse_wecom_aibot_frame_classifies_control_and_callback_frames() { + let auth_ack = serde_json::json!({ + "headers": { "req_id": "aibot_subscribe-123" }, + "errcode": 0, + "errmsg": "ok" + }); + assert!(matches!( + parse_wecom_aibot_frame(&auth_ack.to_string()), + super::WecomAibotIncomingFrame::Ack(_) + )); + + let callback = serde_json::json!({ + "cmd": "aibot_msg_callback", + "headers": { "req_id": "req-1" }, + "body": { "msgtype": "text" } + }); + assert!(matches!( + parse_wecom_aibot_frame(&callback.to_string()), + super::WecomAibotIncomingFrame::Callback + )); + + let disconnected_event = serde_json::json!({ + "cmd": "aibot_event_callback", + "body": { "event": { "eventtype": "disconnected_event" } } + }); + assert!(matches!( + parse_wecom_aibot_frame(&disconnected_event.to_string()), + super::WecomAibotIncomingFrame::Event { disconnected: true } + )); + } + + #[test] + fn test_wecom_aibot_session_process_text_frame_handles_auth_ping_and_disconnect() { + let mut state = WecomAibotSessionState::new(45_000, 3); + + let subscribe_ok = serde_json::json!({ + "headers": { "req_id": "aibot_subscribe-123" }, + "errcode": 0, + "errmsg": "ok" + }); + let actions = state.process_text_frame(&subscribe_ok.to_string(), "wecom"); + assert_eq!(actions.len(), 1); + assert!(matches!( + actions[0], + WebsocketFrameAction::SetHeartbeat { + interval_ms: 45_000 + } + )); + + state.missed_heartbeat_acks = 2; + let ping_ok = serde_json::json!({ + "headers": { "req_id": "ping-1" }, + "errcode": 0, + "errmsg": "ok" + }); + assert!( + state + .process_text_frame(&ping_ok.to_string(), "wecom") + .is_empty() + ); + assert_eq!(state.missed_heartbeat_acks, 0); + + let subscribe_error = serde_json::json!({ + "headers": { "req_id": "aibot_subscribe-456" }, + "errcode": 40001, + "errmsg": "invalid secret" + }); + let actions = state.process_text_frame(&subscribe_error.to_string(), "wecom"); + assert_eq!(actions.len(), 1); + assert!(matches!( + actions[0], + WebsocketFrameAction::Reconnect { + reset_session: false + } + )); + + let disconnected_event = serde_json::json!({ + "cmd": "aibot_event_callback", + "body": { "event": { "eventtype": "disconnected_event" } } + }); + let actions = state.process_text_frame(&disconnected_event.to_string(), "wecom"); + assert_eq!(actions.len(), 2); + assert!(matches!(actions[0], WebsocketFrameAction::Enqueue(_))); + assert!(matches!(actions[1], WebsocketFrameAction::StopRuntime)); + } + + #[test] + fn test_websocket_outbound_queue_rejects_oversized_and_full_frames() { + let (tx, mut rx) = tokio::sync::mpsc::channel(1); - let capabilities = - ChannelCapabilities::for_channel("discord").with_tool_capabilities(tool_capabilities); + let too_large = "x".repeat(super::WEBSOCKET_OUTBOUND_MAX_FRAME_BYTES + 1); + let err = super::queue_websocket_outbound_frame(&tx, too_large) + .expect_err("oversized frame should be rejected"); + assert!(err.contains("exceeds max size")); - assert!(WebsocketRuntimeConfig::from_capabilities(&capabilities).is_none()); + super::queue_websocket_outbound_frame(&tx, "first".to_string()) + .expect("first frame should fit"); + let err = super::queue_websocket_outbound_frame(&tx, "second".to_string()) + .expect_err("bounded queue should reject when full"); + assert!(err.contains("queue is full")); + + assert_eq!(rx.try_recv().expect("queued frame"), "first"); } #[test] @@ -5496,6 +6823,24 @@ mod tests { ChannelCapabilities::for_channel("discord").with_tool_capabilities(tool_capabilities) } + fn wecom_websocket_capabilities() -> ChannelCapabilities { + let tool_capabilities = ToolCapabilities { + http: Some(HttpCapability::new(vec![ + EndpointPattern::host("openws.work.weixin.qq.com"), + EndpointPattern::host("*.myqcloud.com").with_methods(vec!["GET".to_string()]), + ])), + websocket: Some(serde_json::json!({ + "url": "wss://openws.work.weixin.qq.com", + "connect_on_start": true, + "protocol": "wecom_aibot", + "bot_id_secret_name": "wecom_bot_id", + "bot_secret_name": "wecom_bot_secret" + })), + ..Default::default() + }; + ChannelCapabilities::for_channel("wecom").with_tool_capabilities(tool_capabilities) + } + fn empty_secrets_store() -> Arc<dyn SecretsStore + Send + Sync> { let crypto = Arc::new(SecretsCrypto::new(SecretString::from(TEST_CRYPTO_KEY.to_string())).unwrap()); @@ -5506,6 +6851,10 @@ mod tests { CredentialName::new("discord_bot_token").expect("valid credential name") } + fn wecom_bot_secret_name() -> CredentialName { + CredentialName::new("wecom_bot_secret").expect("valid credential name") + } + /// Regression test for #2557: websocket preflight must report /// MissingCredential when a required credential is absent under the /// owner scope. @@ -5636,6 +6985,62 @@ mod tests { assert!(matches!(decision, WebsocketStartDecision::Spawn(_))); } + #[tokio::test] + async fn test_websocket_start_decision_missing_auth_when_wecom_secret_absent() { + let store = empty_secrets_store(); + store + .create( + "owner_42", + CreateSecretParams { + name: "wecom_bot_id".to_string(), + value: SecretString::from("bot-id".to_string()), + provider: None, + expires_at: None, + }, + ) + .await + .unwrap(); + let capabilities = wecom_websocket_capabilities(); + + let decision = + websocket_start_decision(&capabilities, Some(store.as_ref()), "owner_42").await; + + assert_eq!( + decision, + WebsocketStartDecision::MissingAuth { + credential_name: wecom_bot_secret_name(), + } + ); + } + + #[tokio::test] + async fn test_websocket_start_decision_spawn_when_wecom_secrets_present() { + let store = empty_secrets_store(); + for (name, value) in [ + ("wecom_bot_id", "bot-id"), + ("wecom_bot_secret", "bot-secret"), + ] { + store + .create( + "owner_42", + CreateSecretParams { + name: name.to_string(), + value: SecretString::from(value.to_string()), + provider: None, + expires_at: None, + }, + ) + .await + .unwrap(); + } + let capabilities = wecom_websocket_capabilities(); + + let decision = + websocket_start_decision(&capabilities, Some(store.as_ref()), "owner_42").await; + + assert!(matches!(decision, WebsocketStartDecision::Spawn(_))); + } + /// The reverse of the existing "secret without identify template" case: /// `identify` present but no `identify_secret_name` must also be flagged /// as `MalformedConfig`. Without this branch, the runtime spawned but @@ -5713,11 +7118,16 @@ mod tests { let WebsocketStartDecision::Spawn(config) = decision else { panic!("expected Spawn, got {decision:?}"); }; - assert_eq!( - config.identify_secret_name.as_deref(), - Some("discord_bot_token"), - "Spawn config must carry the canonicalized name so runtime lookups match preflight" - ); + match config.protocol { + WebsocketProtocolConfig::DiscordGateway(discord) => { + assert_eq!( + discord.identify_secret_name.as_deref(), + Some("discord_bot_token"), + "Spawn config must carry the canonicalized name so runtime lookups match preflight" + ); + } + WebsocketProtocolConfig::WecomAibot(_) => panic!("expected discord websocket config"), + } } #[tokio::test] @@ -6009,6 +7419,7 @@ mod tests { &credentials, Vec::new(), // no host credentials in test Arc::new(PairingStore::new_noop()), + None, &workspace_store, &callback_lock, None, @@ -6125,6 +7536,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("test-channel"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( @@ -6142,8 +7555,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "test-channel", + capabilities: &capabilities, owner_scope_id: "default", owner_actor_id: None, + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -6175,6 +7590,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("test-channel"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( @@ -6193,8 +7610,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "test-channel", + capabilities: &capabilities, owner_scope_id: "default", owner_actor_id: None, + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -6221,6 +7640,8 @@ mod tests { // No sender available (channel not started) let message_tx = Arc::new(tokio::sync::RwLock::new(None)); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("test-channel"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( crate::channels::wasm::host::ChannelEmitRateLimiter::new( @@ -6235,8 +7656,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "test-channel", + capabilities: &capabilities, owner_scope_id: "default", owner_actor_id: None, + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -6620,11 +8043,363 @@ mod tests { .expect("respond should succeed"); let stored_metadata = channel.last_broadcast_metadata.read().await.clone(); - assert_eq!(stored_metadata.as_deref(), Some(r#"{"chat_id":12345}"#)); + // `with_metadata` always sets `user_id` from the IncomingMessage + // (overwriting any caller-supplied value) so downstream + // `send_status` can route SSE events to the owning tenant + // unforgeably. The caller-supplied `chat_id` survives alongside it. + assert_eq!( + stored_metadata.as_deref(), + Some(r#"{"chat_id":12345,"user_id":"owner-scope"}"#) + ); channel.shutdown().await.expect("Shutdown should succeed"); } + #[tokio::test] + async fn test_call_on_respond_uses_websocket_runtime_sender_for_wecom_component() { + let channel = create_wecom_component_test_channel().await; + let (tx, mut rx) = tokio::sync::mpsc::channel(super::WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + *channel.websocket_outbound_tx.write().await = Some(tx); + + let metadata_json = serde_json::json!({ + "to_user": "ZhangSan", + "source_msg_id": "msg-1", + "ws_req_id": "req-1", + "ws_chat_id": null, + "ws_chat_type": "single", + "ws_reply_cmd": "aibot_respond_msg", + }) + .to_string(); + + channel + .call_on_respond( + uuid::Uuid::new_v4(), + "hello from test", + None, + &metadata_json, + &[], + &[], + ) + .await + .expect("wecom websocket respond should succeed"); + + let payload = rx.try_recv().expect("websocket payload"); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("json websocket payload"); + assert_eq!(parsed["cmd"], serde_json::json!("aibot_respond_msg")); + assert_eq!(parsed["headers"]["req_id"], serde_json::json!("req-1")); + assert_eq!( + parsed["body"]["stream"]["id"], + serde_json::json!("stream-req-1") + ); + assert_eq!( + parsed["body"]["stream"]["content"], + serde_json::json!("hello from test") + ); + assert_eq!(parsed["body"]["stream"]["finish"], serde_json::json!(true)); + } + + #[tokio::test] + async fn test_call_on_poll_emits_wecom_websocket_callback_message() { + let channel = create_wecom_component_test_channel().await; + channel + .call_on_start() + .await + .expect("wecom on_start should succeed"); + let mut stream = channel + .ensure_message_channel() + .await + .expect("message stream should be created"); + + channel.workspace_store.commit_writes(&[ + PendingWorkspaceWrite { + path: "channels/wecom/dm_policy".to_string(), + content: "allowlist".to_string(), + }, + PendingWorkspaceWrite { + path: "channels/wecom/allow_from".to_string(), + content: serde_json::json!(["ZhangSan"]).to_string(), + }, + ]); + + let frame = serde_json::json!({ + "cmd": "aibot_msg_callback", + "headers": { "req_id": "req-callback-1" }, + "body": { + "msgid": "msg-callback-1", + "chatid": "group-chat-1", + "chattype": "group", + "from": { "userid": "ZhangSan" }, + "msgtype": "mixed", + "mixed": { + "msg_item": [ + { + "type": "text", + "text": { "content": "hello from group" } + }, + { + "type": "image", + "image": { "url": "https://not-allowlisted.example.com/image.jpg" } + } + ] + } + } + }) + .to_string(); + channel + .workspace_store + .append_json_text_queue( + &websocket_queue_path("wecom"), + &frame, + WEBSOCKET_EVENT_QUEUE_MAX_ITEMS, + ) + .expect("queue websocket callback frame"); + + let (outbound_tx, _outbound_rx) = + tokio::sync::mpsc::channel(super::WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + spawn_wecom_websocket_poll_for_test(&channel, outbound_tx, Duration::from_secs(10)).await; + + let msg = tokio::time::timeout(Duration::from_secs(20), stream.next()) + .await + .expect("poll should emit message before timeout") + .expect("message stream should remain open"); + assert_eq!(msg.channel, "wecom"); + assert_eq!(msg.sender_id, "ZhangSan"); + assert_eq!(msg.content, "hello from group"); + assert_eq!( + msg.thread_id.as_ref().map(|thread_id| thread_id.as_str()), + Some("wecom:group:group-chat-1") + ); + assert_eq!( + msg.metadata["source_msg_id"], + serde_json::json!("msg-callback-1") + ); + assert_eq!( + msg.metadata["ws_req_id"], + serde_json::json!("req-callback-1") + ); + assert_eq!(msg.metadata["chat_type"], serde_json::json!("group")); + assert!( + msg.attachments.is_empty(), + "unhydrated WeCom media must not be emitted as metadata-only attachments" + ); + } + + #[tokio::test] + async fn test_websocket_poll_restores_processing_queue_when_on_poll_fails() { + use crate::tools::wasm::WorkspaceReader; + + let channel = create_wecom_component_test_channel().await; + let frame = serde_json::json!({ + "cmd": "aibot_msg_callback", + "headers": { "req_id": "req-timeout-1" }, + "body": { + "msgid": "msg-timeout-1", + "chattype": "single", + "from": { "userid": "ZhangSan" }, + "msgtype": "text", + "text": { "content": "retry me" } + } + }) + .to_string(); + let queue_path = websocket_queue_path("wecom"); + let processing_queue_path = websocket_processing_queue_path("wecom"); + channel + .workspace_store + .append_json_text_queue(&queue_path, &frame, WEBSOCKET_EVENT_QUEUE_MAX_ITEMS) + .expect("queue websocket callback frame"); + + let (outbound_tx, _outbound_rx) = + tokio::sync::mpsc::channel(super::WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + spawn_wecom_websocket_poll_for_test(&channel, outbound_tx, Duration::ZERO).await; + + let restored = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if channel + .workspace_store + .read(&processing_queue_path) + .is_none() + && let Some(raw) = channel.workspace_store.read(&queue_path) + { + break raw; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("websocket queue should be restored after poll timeout"); + let restored_frames: Vec<String> = + serde_json::from_str(&restored).expect("restored queue json"); + assert_eq!(restored_frames, vec![frame]); + } + + #[tokio::test] + async fn test_image_generated_status_defers_until_wecom_final_response() { + use crate::channels::IncomingMessage; + + let channel = create_wecom_component_test_channel().await; + let (tx, mut rx) = tokio::sync::mpsc::channel(super::WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + *channel.websocket_outbound_tx.write().await = Some(tx); + + let metadata = serde_json::json!({ + "to_user": "ZhangSan", + "source_msg_id": "msg-1", + "ws_req_id": "req-img", + "ws_chat_id": null, + "ws_chat_type": "single", + "ws_reply_cmd": "aibot_respond_msg", + }); + + channel + .send_status( + crate::channels::StatusUpdate::ImageGenerated { + event_id: "image-call-1".to_string(), + data_url: "data:image/png;base64,YWJj".to_string(), + path: None, + }, + &metadata, + ) + .await + .expect("generated image status should stage"); + + assert!( + rx.try_recv().is_err(), + "generated image status should wait for the final response" + ); + + let msg = IncomingMessage::new("wecom", "ZhangSan", "generate an image") + .with_metadata(metadata.clone()); + channel + .respond( + &msg, + crate::channels::OutgoingResponse::text("generated caption"), + ) + .await + .expect("wecom websocket respond should succeed"); + + let payload = rx.try_recv().expect("websocket payload"); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("json websocket payload"); + assert_eq!(parsed["cmd"], serde_json::json!("aibot_upload_media_init")); + assert_eq!(parsed["body"]["type"], serde_json::json!("image")); + assert_eq!( + parsed["body"]["md5"], + serde_json::json!("900150983cd24fb0d6963f7d28e17f72") + ); + assert!(rx.try_recv().is_err(), "only one image upload should start"); + } + + #[tokio::test] + async fn test_done_status_clears_status_staged_wecom_image_without_response() { + let channel = create_wecom_component_test_channel().await; + let metadata = serde_json::json!({ + "to_user": "ZhangSan", + "source_msg_id": "msg-cleanup", + "ws_req_id": "req-cleanup", + "ws_chat_id": null, + "ws_chat_type": "single", + "ws_reply_cmd": "aibot_respond_msg", + }); + + channel + .send_status( + crate::channels::StatusUpdate::ImageGenerated { + event_id: "image-cleanup".to_string(), + data_url: "data:image/png;base64,YWJj".to_string(), + path: None, + }, + &metadata, + ) + .await + .expect("generated image status should stage"); + + let staged_paths = channel + .pending_generated_image_attachments + .lock() + .await + .get(&generated_image_delivery_key(&metadata)) + .cloned() + .expect("image should be staged"); + assert_eq!(staged_paths.len(), 1); + + channel + .send_status( + crate::channels::StatusUpdate::Status("Done".to_string()), + &metadata, + ) + .await + .expect("done status should clear staged image"); + + assert!( + channel + .pending_generated_image_attachments + .lock() + .await + .get(&generated_image_delivery_key(&metadata)) + .is_none() + ); + assert!( + !std::path::Path::new(&staged_paths[0]).exists(), + "staged generated image file should be removed on terminal no-response path" + ); + } + + #[tokio::test] + async fn test_final_generated_image_attachment_dedupes_status_staged_wecom_image() { + use crate::channels::IncomingMessage; + use crate::generated_images::{ + remove_staged_generated_image_attachments, stage_generated_image_data_url, + }; + + let channel = create_wecom_component_test_channel().await; + let (tx, mut rx) = tokio::sync::mpsc::channel(super::WEBSOCKET_OUTBOUND_QUEUE_CAPACITY); + *channel.websocket_outbound_tx.write().await = Some(tx); + + let metadata = serde_json::json!({ + "to_user": "ZhangSan", + "source_msg_id": "msg-2", + "ws_req_id": "req-img-dedupe", + "ws_chat_id": null, + "ws_chat_type": "single", + "ws_reply_cmd": "aibot_respond_msg", + }); + + channel + .send_status( + crate::channels::StatusUpdate::ImageGenerated { + event_id: "image-call-2".to_string(), + data_url: "data:image/png;base64,YWJj".to_string(), + path: None, + }, + &metadata, + ) + .await + .expect("generated image status should stage"); + + let final_attachment = + stage_generated_image_data_url("data:image/png;base64,YWJj").expect("stage image"); + let msg = IncomingMessage::new("wecom", "ZhangSan", "generate an image") + .with_metadata(metadata.clone()); + let mut response = crate::channels::OutgoingResponse::text("generated caption"); + response.attachments = vec![final_attachment.clone()]; + + channel + .respond(&msg, response) + .await + .expect("wecom websocket respond should succeed"); + + let payload = rx.try_recv().expect("websocket payload"); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("json websocket payload"); + assert_eq!(parsed["cmd"], serde_json::json!("aibot_upload_media_init")); + assert!( + rx.try_recv().is_err(), + "duplicate image upload should be skipped" + ); + + remove_staged_generated_image_attachments(&[final_attachment]); + } + #[tokio::test] async fn test_stream_chunk_is_noop() { let channel = create_test_channel(); @@ -7158,6 +8933,7 @@ mod tests { creds, Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let error = format!( @@ -7192,6 +8968,7 @@ mod tests { std::collections::HashMap::new(), Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let input = "some error message"; @@ -7225,6 +9002,7 @@ mod tests { creds, host_creds, Arc::new(PairingStore::new_noop()), + None, ); // Error containing URL-encoded form of the credential @@ -7258,6 +9036,7 @@ mod tests { creds, Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let input = "should not match anything"; @@ -7292,6 +9071,7 @@ mod tests { HashMap::new(), host_creds, Arc::new(PairingStore::new_noop()), + None, ); // Matching host + matching path → inject @@ -7341,6 +9121,7 @@ mod tests { HashMap::new(), host_creds, Arc::new(PairingStore::new_noop()), + None, ); let mut headers = HashMap::new(); @@ -7365,6 +9146,7 @@ mod tests { std::collections::HashMap::new(), Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let result = super::near::agent::channel_host::Host::http_request( @@ -7406,6 +9188,7 @@ mod tests { std::collections::HashMap::new(), Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let result = super::near::agent::channel_host::Host::http_request( @@ -7542,6 +9325,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("test-channel"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( @@ -7560,6 +9345,7 @@ mod tests { storage_key: None, local_path: None, extracted_text: None, + extras_json: String::new(), data: Vec::new(), duration_secs: None, }, @@ -7572,6 +9358,7 @@ mod tests { storage_key: Some("store/doc456".to_string()), local_path: None, extracted_text: Some("Report contents...".to_string()), + extras_json: String::new(), data: Vec::new(), duration_secs: None, }, @@ -7584,8 +9371,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "test-channel", + capabilities: &capabilities, owner_scope_id: "default", owner_actor_id: None, + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -7631,6 +9420,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("telegram"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( crate::channels::wasm::host::ChannelEmitRateLimiter::new( @@ -7647,8 +9438,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "telegram", + capabilities: &capabilities, owner_scope_id: "owner-scope", owner_actor_id: Some("telegram-owner"), + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -7709,6 +9502,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("telegram"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( crate::channels::wasm::host::ChannelEmitRateLimiter::new( @@ -7724,8 +9519,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "telegram", + capabilities: &capabilities, owner_scope_id: "owner-scope", owner_actor_id: Some("telegram-owner"), + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -7745,6 +9542,53 @@ mod tests { assert!(last_broadcast_metadata.read().await.is_none()); // safety: test-only assertion } + #[tokio::test] + async fn test_dispatch_emitted_messages_wechat_sender_uses_bound_user() { + use crate::channels::wasm::host::EmittedMessage; + + let (tx, mut rx) = tokio::sync::mpsc::channel(10); + let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("wechat"); + let pairing_store = PairingStore::new_noop(); + let rate_limiter = Arc::new(tokio::sync::RwLock::new( + crate::channels::wasm::host::ChannelEmitRateLimiter::new( + crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), + ), + )); + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); + + let result = WasmChannel::dispatch_emitted_messages( + EmitDispatchContext { + channel_name: "wechat", + capabilities: &capabilities, + owner_scope_id: "owner-scope", + owner_actor_id: None, + channel_bound_user_id: Some("bound-user"), + pairing_store: &pairing_store, + message_tx: &message_tx, + rate_limiter: &rate_limiter, + last_broadcast_metadata: &last_broadcast_metadata, + settings_store: None, + }, + vec![ + EmittedMessage::new("wx-user-42", "Hello from WeChat") + .with_metadata(r#"{"from_user_id":"wx-user-42"}"#) + .with_thread_id("wechat:wx-user-42"), + ], + ) + .await; + + assert!(result.is_ok()); + + let msg = rx.try_recv().expect("Should receive message"); + assert_eq!(msg.user_id, "bound-user"); + assert_eq!(msg.sender_id, "wx-user-42"); + assert_eq!(msg.conversation_scope(), Some("wechat:wx-user-42")); + assert!(msg.metadata.get("owner_id").is_none()); + assert!(last_broadcast_metadata.read().await.is_none()); + } + #[cfg(feature = "libsql")] #[tokio::test] async fn test_dispatch_emitted_messages_paired_sender_sets_owner_scope() { @@ -7771,6 +9615,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("telegram"); let rate_limiter = Arc::new(tokio::sync::RwLock::new( crate::channels::wasm::host::ChannelEmitRateLimiter::new( crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), @@ -7781,8 +9627,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "telegram", + capabilities: &capabilities, owner_scope_id: "owner-scope", owner_actor_id: Some("telegram-owner"), + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -7853,6 +9701,8 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + let capabilities = + crate::channels::wasm::capabilities::ChannelCapabilities::for_channel("test-channel"); let pairing_store = PairingStore::new_noop(); let rate_limiter = Arc::new(tokio::sync::RwLock::new( @@ -7867,8 +9717,10 @@ mod tests { let result = WasmChannel::dispatch_emitted_messages( EmitDispatchContext { channel_name: "test-channel", + capabilities: &capabilities, owner_scope_id: "default", owner_actor_id: None, + channel_bound_user_id: None, pairing_store: &pairing_store, message_tx: &message_tx, rate_limiter: &rate_limiter, @@ -8019,6 +9871,7 @@ mod tests { credentials, Vec::new(), Arc::new(PairingStore::new_noop()), + None, ); let result = super::near::agent::channel_host::Host::http_request( diff --git a/src/channels/web/features/chat/mod.rs b/src/channels/web/features/chat/mod.rs index adcbcc26cd8..87f2a6f1781 100644 --- a/src/channels/web/features/chat/mod.rs +++ b/src/channels/web/features/chat/mod.rs @@ -187,6 +187,74 @@ pub(crate) async fn chat_approval_handler( ) })?; + // Inline fast-path: when an Approval gate parks the live engine VM + // via `BridgeGateController::pause`, the per-user agent loop is + // blocked at `handle_message` awaiting the bridge call, so an + // ExecApproval submission posted to msg_tx would queue indefinitely + // behind the parked execution. Bypass the mpsc and call into the + // gate controller's in-memory delivery channel directly. On + // `NoLiveVm` we fall through to the legacy mpsc path so engine v1 + // approvals (and any post-restart Approval gates without a parked + // future) still resolve correctly. + // + // The fast path looks the gate up by `request_id` rather than by + // the wire `thread_id`: web's `req.thread_id` is the channel-visible + // identifier (the per-conversation UUID returned by + // `/api/chat/thread/new`) and is recorded on the pending gate as + // `scope_thread_id`, not as the internal engine `ThreadId` that + // keys `PendingGateStore`. Mixing them up would miss every gate + // whose channel scope differs from its engine thread. + let resolution = if approved { + ironclaw_engine::GateResolution::Approved { always } + } else { + ironclaw_engine::GateResolution::Denied { reason: None } + }; + // Match the legacy mpsc path's settings precedence (cache → raw DB) + // so an `action="always"` approval still persists + // `tool_permissions.<tool>=always_allow` whenever any DB-backed + // SettingsStore is configured. Falling back to the raw `state.store` + // covers gateways that wire a database without the cache layer. + let settings_store = + crate::channels::web::features::settings::resolve_settings_store(&state).ok(); + match crate::bridge::try_resolve_inline_approval_gate( + &user.user_id, + "gateway", + request_id, + resolution, + settings_store, + ) + .await + { + Ok(crate::bridge::InlineGateOutcome::Delivered) => { + return Ok(( + StatusCode::ACCEPTED, + Json(SendMessageResponse { + message_id: Uuid::new_v4(), + status: "accepted", + }), + )); + } + Ok(crate::bridge::InlineGateOutcome::NoLiveVm) => { + // Fall through to the legacy mpsc dispatch below. + } + Err(e) => { + // Map typed verification failures to specific 4xx / + // 5xx codes. Matching on the variant — not a substring + // of the rendered message — keeps the HTTP contract + // tied to the typed surface so a future change to the + // error message can't silently flip a 403 → 500. + use crate::bridge::InlineGateError; + let status = match &e { + InlineGateError::ChannelMismatch { .. } | InlineGateError::Unauthorized => { + StatusCode::FORBIDDEN + } + InlineGateError::Stale | InlineGateError::Expired => StatusCode::CONFLICT, + InlineGateError::Other(_) => StatusCode::INTERNAL_SERVER_ERROR, + }; + return Err((status, e.to_string())); + } + } + // Build a structured ExecApproval submission as JSON, sent through the // existing message pipeline so the agent loop picks it up. let approval = crate::agent::submission::Submission::ExecApproval { @@ -238,16 +306,48 @@ pub(crate) async fn chat_gate_resolve_handler( AuthenticatedUser(user): AuthenticatedUser, Json(req): Json<GateResolveRequest>, ) -> Result<Json<ActionResponse>, (StatusCode, String)> { - match req.resolution { + // Half-2 of #3133: a paused background mission may be waiting on + // this same `request_id`. After the foreground gate is resolved we + // fan the disposition out to the mission auto-resume path so a + // paused mission re-fires (Approved / CredentialProvided) or gets + // marked Failed (Denied / Cancelled). For OAuth flows the + // credential-write path also triggers + // `resume_paused_missions_for_credential` from the OAuth callback + // handler — both hooks landing on the same mission are idempotent + // since `resume_paused_for_request_id` and + // `resume_paused_for_credential` re-check `paused_gate` atomically. + // Best-effort dispatch — failures inside the helper are logged and + // never surfaced as a gate-resolve error. + // Validate the request id once up front so every arm — including + // the Approved / Denied paths that delegate to chat_approval_handler + // — surfaces a uniform 400 on malformed UUIDs, and the mission + // auto-resume hook below isn't silently skipped on bad input. + let gate_request_id = Uuid::parse_str(&req.request_id).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid request_id (expected UUID)".to_string(), + ) + })?; + let mission_outcome = match req.resolution { + GateResolutionPayload::Approved { .. } + | GateResolutionPayload::CredentialProvided { .. } => { + Some(ironclaw_engine::GateResolutionOutcome::Approved) + } + GateResolutionPayload::Denied => Some(ironclaw_engine::GateResolutionOutcome::Denied), + GateResolutionPayload::Cancelled => Some(ironclaw_engine::GateResolutionOutcome::Cancelled), + }; + let mission_resume = mission_outcome.map(|outcome| (outcome, gate_request_id)); + + let response: Result<Json<ActionResponse>, (StatusCode, String)> = match req.resolution { GateResolutionPayload::Approved { always } => { let action = if always { "always" } else { "approve" }.to_string(); let _ = chat_approval_handler( - State(state), - AuthenticatedUser(user), + State(state.clone()), + AuthenticatedUser(user.clone()), Json(ApprovalRequest { - request_id: req.request_id, + request_id: req.request_id.clone(), action, - thread_id: req.thread_id, + thread_id: req.thread_id.clone(), }), ) .await?; @@ -255,12 +355,12 @@ pub(crate) async fn chat_gate_resolve_handler( } GateResolutionPayload::Denied => { let _ = chat_approval_handler( - State(state), - AuthenticatedUser(user), + State(state.clone()), + AuthenticatedUser(user.clone()), Json(ApprovalRequest { - request_id: req.request_id, + request_id: req.request_id.clone(), action: "deny".into(), - thread_id: req.thread_id, + thread_id: req.thread_id.clone(), }), ) .await?; @@ -271,14 +371,8 @@ pub(crate) async fn chat_gate_resolve_handler( StatusCode::BAD_REQUEST, "thread_id is required for credential resolution".to_string(), ))?; - let request_id = Uuid::parse_str(&req.request_id).map_err(|_| { - ( - StatusCode::BAD_REQUEST, - "Invalid request_id (expected UUID)".to_string(), - ) - })?; let submission = crate::agent::submission::Submission::GateAuthResolution { - request_id, + request_id: gate_request_id, resolution: crate::agent::submission::AuthGateResolution::CredentialProvided { token, }, @@ -296,30 +390,57 @@ pub(crate) async fn chat_gate_resolve_handler( Ok(Json(ActionResponse::ok("Credential submitted."))) } GateResolutionPayload::Cancelled => { - let thread_id = req.thread_id.ok_or(( - StatusCode::BAD_REQUEST, - "thread_id is required for cancellation".to_string(), - ))?; - let request_id = Uuid::parse_str(&req.request_id).map_err(|_| { - ( - StatusCode::BAD_REQUEST, - "Invalid request_id (expected UUID)".to_string(), - ) - })?; - let submission = crate::agent::submission::Submission::GateAuthResolution { - request_id, - resolution: crate::agent::submission::AuthGateResolution::Cancelled, + // Mission-only gates have no foreground `thread_id` — the + // gate is owned by a background mission's child thread, and + // the gate-card UI doesn't surface a `thread_id` in the + // resolution payload. For foreground inline-await gates, + // dispatch the structured cancellation so the parked VM + // unwinds promptly. The mission auto-resume path + // (`resume_paused_missions_for_gate_request`, fired below) + // independently carries the Cancelled outcome to the + // mission state machine. + // + // If the client omits `thread_id` for a foreground gate + // (regression from PR #3366 review: gate-card UI without + // foreground thread context), recover the owning thread + // from `PendingGateStore` so the parked VM is not stranded. + // Lookup is scoped to the requesting user via the store's + // own ownership check. + let dispatch_thread_id = match req.thread_id.clone() { + Some(t) => Some(t), + None => { + crate::bridge::get_pending_gate_by_request_id(&user.user_id, gate_request_id) + .await + .map(|gate| gate.thread_id) + } }; - crate::channels::web::platform::engine_dispatch::dispatch_engine_submission( - &state, - &user.user_id, - &thread_id, - submission, - ) - .await?; + if let Some(thread_id) = dispatch_thread_id { + let submission = crate::agent::submission::Submission::GateAuthResolution { + request_id: gate_request_id, + resolution: crate::agent::submission::AuthGateResolution::Cancelled, + }; + crate::channels::web::platform::engine_dispatch::dispatch_engine_submission( + &state, + &user.user_id, + &thread_id, + submission, + ) + .await?; + } Ok(Json(ActionResponse::ok("Gate cancelled."))) } + }; + + if let Some((outcome, gate_request_id)) = mission_resume { + let _ = crate::bridge::resume_paused_missions_for_gate_request( + &user.user_id, + gate_request_id, + outcome, + ) + .await; } + + response } pub(crate) async fn chat_auth_token_handler( @@ -2640,13 +2761,13 @@ mod tests { } #[tokio::test] - async fn pending_gate_extension_name_uses_install_parameters_for_hyphenated_activate_tool() { + async fn pending_gate_extension_name_uses_install_parameters_for_hyphenated_install_tool() { let state = test_gateway_state(None); let extension_name = pending_gate_extension_name( &state, "test-user", - "tool-activate", + "tool-install", r#"{"name":"telegram"}"#, &ironclaw_engine::ResumeKind::Authentication { credential_name: ironclaw_common::CredentialName::from_trusted( diff --git a/src/channels/web/features/extensions/mod.rs b/src/channels/web/features/extensions/mod.rs index bfb89f77d8f..7e0eb174874 100644 --- a/src/channels/web/features/extensions/mod.rs +++ b/src/channels/web/features/extensions/mod.rs @@ -50,7 +50,7 @@ pub(crate) fn derive_activation_status( has_owner_binding: bool, ) -> Option<ExtensionActivationStatus> { if ext.kind == crate::extensions::ExtensionKind::WasmChannel { - classify_wasm_channel_activation(ext, has_paired, has_owner_binding) + classify_wasm_channel_activation(ext, has_paired, has_owner_binding, ext.requires_binding) } else if ext.kind == crate::extensions::ExtensionKind::ChannelRelay { Some(if ext.active { ExtensionActivationStatus::Active @@ -536,11 +536,117 @@ pub(crate) async fn extensions_setup_handler( kind, secrets: setup.secrets, fields: setup.fields, + interactive_login: setup.interactive_login, onboarding_state: None, onboarding: None, })) } +pub(crate) async fn extensions_login_start_handler( + State(state): State<Arc<GatewayState>>, + AuthenticatedUser(user): AuthenticatedUser, + Path(name): Path<String>, + Json(_req): Json<ExtensionInteractiveLoginStartRequest>, +) -> Result<Json<ExtensionInteractiveLoginResponse>, (StatusCode, String)> { + let name = ironclaw_common::ExtensionName::new(&name).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Invalid extension name: {e}"), + ) + })?; + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + match ext_mgr + .start_interactive_login(name.as_str(), &user.user_id) + .await + { + Ok(result) => Ok(Json(ExtensionInteractiveLoginResponse { + success: true, + status: result.status, + message: result.message, + session_id: Some(result.session_id), + qr_code_url: result.qr_code_url, + instructions: result.instructions, + activated: None, + })), + Err(e) => Ok(Json(ExtensionInteractiveLoginResponse { + success: false, + status: "failed".to_string(), + message: e.to_string(), + session_id: None, + qr_code_url: None, + instructions: None, + activated: Some(false), + })), + } +} + +pub(crate) async fn extensions_login_poll_handler( + State(state): State<Arc<GatewayState>>, + AuthenticatedUser(user): AuthenticatedUser, + Path(name): Path<String>, + Json(req): Json<ExtensionInteractiveLoginPollRequest>, +) -> Result<Json<ExtensionInteractiveLoginResponse>, (StatusCode, String)> { + let name = ironclaw_common::ExtensionName::new(&name).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Invalid extension name: {e}"), + ) + })?; + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + match ext_mgr + .poll_interactive_login(name.as_str(), &req.session_id, &user.user_id) + .await + { + Ok(result) => { + if result.activated == Some(true) { + crate::channels::web::platform::legacy_auth::clear_auth_mode(&state, &user.user_id) + .await; + state.sse.broadcast_for_user( + &user.user_id, + AppEvent::OnboardingState { + extension_name: name.clone(), + state: OnboardingStateDto::Ready, + request_id: None, + message: Some(result.message.clone()), + instructions: None, + auth_url: None, + setup_url: None, + onboarding: None, + thread_id: None, + }, + ); + } + + Ok(Json(ExtensionInteractiveLoginResponse { + success: result.status != "failed", + status: result.status, + message: result.message, + session_id: Some(result.session_id), + qr_code_url: result.qr_code_url, + instructions: None, + activated: result.activated, + })) + } + Err(e) => Ok(Json(ExtensionInteractiveLoginResponse { + success: false, + status: "failed".to_string(), + message: e.to_string(), + session_id: Some(req.session_id), + qr_code_url: None, + instructions: None, + activated: Some(false), + })), + } +} + pub(crate) async fn extensions_setup_submit_handler( State(state): State<Arc<GatewayState>>, AuthenticatedUser(user): AuthenticatedUser, @@ -701,6 +807,7 @@ mod tests { needs_setup: false, has_auth: false, installed: true, + requires_binding: true, activation_error: None, version: None, } @@ -786,11 +893,12 @@ mod tests { needs_setup: true, has_auth: false, installed: true, + requires_binding: true, activation_error: None, version: None, }; - let owner_bound = classify_wasm_channel_activation(&ext, false, true); + let owner_bound = classify_wasm_channel_activation(&ext, false, true, ext.requires_binding); if owner_bound != Some(ExtensionActivationStatus::Active) { return Err(format!( "owner-bound channel should be active, got {:?}", @@ -798,7 +906,7 @@ mod tests { )); } - let unbound = classify_wasm_channel_activation(&ext, false, false); + let unbound = classify_wasm_channel_activation(&ext, false, false, ext.requires_binding); if unbound != Some(ExtensionActivationStatus::Pairing) { return Err(format!( "unbound channel should be pairing, got {:?}", @@ -823,12 +931,13 @@ mod tests { needs_setup: true, has_auth: false, installed: true, + requires_binding: false, activation_error: None, version: None, }; let status = if relay.kind == crate::extensions::ExtensionKind::WasmChannel { - classify_wasm_channel_activation(&relay, false, false) + classify_wasm_channel_activation(&relay, false, false, relay.requires_binding) } else if relay.kind == crate::extensions::ExtensionKind::ChannelRelay { Some(if relay.active { ExtensionActivationStatus::Active @@ -1168,6 +1277,7 @@ mod tests { needs_setup: false, has_auth: true, installed: true, + requires_binding: false, activation_error: Some("boom".to_string()), version: None, }; diff --git a/src/channels/web/features/oauth/mod.rs b/src/channels/web/features/oauth/mod.rs index 8d3ef00ece6..14190f8ed55 100644 --- a/src/channels/web/features/oauth/mod.rs +++ b/src/channels/web/features/oauth/mod.rs @@ -51,6 +51,83 @@ fn oauth_error_page(label: &str) -> axum::response::Response { axum::response::Html(html).into_response() } +/// Failure category for the OAuth callback handler. The text is embedded in +/// `tracing::warn!` so a user-reported correlation ID can be mapped to the +/// actual cause without having to re-enable verbose tracing on the gateway. +#[derive(Copy, Clone)] +enum OauthCallbackFailure { + /// `?error=...` returned by the provider (e.g. user denied consent). + ProviderError, + /// `state` param is missing or empty. + MissingState, + /// `code` param is missing or empty. + MissingCode, + /// `state` couldn't be decoded — likely a tampered or stale token. + MalformedState, + /// No matching pending flow found for the decoded state. + UnknownState, + /// Pending flow exceeded `OAUTH_FLOW_EXPIRY` before the callback fired. + Expired, + /// Token exchange (provider or proxy) returned a non-2xx response. + Exchange, + /// `extension_manager` is unavailable on the gateway state. + NoExtensionManager, +} + +impl OauthCallbackFailure { + fn as_str(self) -> &'static str { + match self { + Self::ProviderError => "provider_error", + Self::MissingState => "missing_state", + Self::MissingCode => "missing_code", + Self::MalformedState => "malformed_state", + Self::UnknownState => "unknown_state", + Self::Expired => "expired", + Self::Exchange => "exchange_failed", + Self::NoExtensionManager => "no_extension_manager", + } + } +} + +/// Generate a short correlation ID from an arbitrary seed plus the current +/// monotonic-ish timestamp. Emitted on every `tracing::warn!` call in the +/// OAuth callback failure paths so an operator can map a timestamp / +/// category / correlation cluster to a single log line. +/// +/// The seed is whatever caller-side string best identifies the failure — +/// in practice the raw `state` query parameter (when present) or the +/// `flow.extension_name` for post-state-resolution failures. The seed is +/// hashed (SHA-256) before any hex output, so it is never logged or +/// exposed verbatim through the returned ID; callers that want the seed +/// itself in logs must pass it separately through +/// [`redact_oauth_state_for_logs`]. +/// +/// Currently logs-only: the user-facing landing page rendered by +/// [`oauth_error_page`] uses [`crate::auth::oauth::landing_html`], which has +/// a fixed failure subtitle and does not embed the correlation. Plumbing +/// the ID into the HTML is a follow-up — see `landing_html` for the +/// fixed-subtitle wiring. +/// +/// 8 hex characters — collision-resistant within a single tracing window +/// (operators grep recent logs), short enough to read aloud over phone. +fn oauth_failure_correlation_id(seed: &str) -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let mut hasher = Sha256::new(); + hasher.update(seed.as_bytes()); + hasher.update(nanos.to_le_bytes()); + let digest = hasher.finalize(); + let mut s = String::with_capacity(8); + for byte in digest.iter().take(4) { + use std::fmt::Write as _; + let _ = write!(&mut s, "{byte:02x}"); + } + s +} + /// Produce a log-safe fingerprint of an OAuth `state` parameter. /// /// The raw `state` is a one-time CSRF token linked to an in-flight flow. @@ -70,6 +147,60 @@ fn redact_oauth_state_for_logs(state: &str) -> String { format!("sha256:{short_hash}:len={}", state.len()) } +/// Resolve the IronClaw user who initiated a Slack relay OAuth flow. +/// +/// `auth_channel_relay` (in `extensions/manager.rs`) stores the +/// initiating user_id under `relay:{name}:oauth_user`, namespaced by +/// the *gateway owner* secret-store user (`owner_id`). The callback +/// handler reads it back here so the completion toast can be +/// broadcast to the actual tenant who started the flow rather than +/// the gateway owner — in multi-tenant deployments those differ, and +/// broadcasting to `owner_id` would deliver the success/failure +/// toast to the wrong browser tab. +/// +/// Falls back to `owner_id` when the secret is missing +/// (single-tenant deployments or flows started before the field was +/// stored). When `multi_tenant_mode` is true, the fallback emits a +/// WARN: in a multi-tenant deployment a missing initiator secret +/// means we have no way to recover the original tenant, and quietly +/// routing the toast to `owner_id` reintroduces the same misroute +/// the rest of this PR closes. The completion still proceeds — +/// the `oauth_user` value is also used to seed the pairing identity, +/// and aborting the callback would leave a dangling Slack install — +/// but the WARN surfaces the lost-initiator case for operators. +/// +/// Returns the resolved user_id as a `String` so the caller owns the +/// value past the secret store's lifetime. +/// +/// This helper is the canonical broadcast-target resolver. The +/// pairing-identity creation path also uses it so the toast and the +/// new identity always agree on which user just completed the flow. +async fn resolve_relay_oauth_user( + secrets: &(dyn crate::secrets::SecretsStore + Send + Sync), + owner_id: &str, + extension_name: &str, + multi_tenant_mode: bool, +) -> String { + let user_key = format!("relay:{extension_name}:oauth_user"); + match secrets.get_decrypted(owner_id, &user_key).await.ok() { + Some(s) => s.expose().to_string(), + None => { + if multi_tenant_mode { + tracing::warn!( + extension = %extension_name, + owner_id = %owner_id, + "relay OAuth callback: missing `relay:{{ext}}:oauth_user` secret in \ + multi-tenant mode — falling back to gateway owner_id for the \ + completion broadcast. The original initiator is unrecoverable \ + (likely a flow started before the field was stored, or a \ + premature secret delete). Toast may reach the wrong tab." + ); + } + owner_id.to_string() + } + } +} + /// OAuth callback handler for the web gateway. /// /// This is a PUBLIC route (no Bearer token required) because OAuth providers @@ -92,12 +223,31 @@ pub(crate) async fn oauth_callback_handler( // here — otherwise it lingers until `OAUTH_FLOW_EXPIRY` (5 min) and any // subsequent auth attempt for the same (extension, user) pair has to // dedupe against a ghost entry. + // + // We also auto-cancel the engine pending auth gate for the same user so + // the user's conversation doesn't sit blocked waiting for an OAuth + // resume that will never arrive (#3320 — the user reported being unable + // to continue the conversation after a failed Gmail OAuth from + // Telegram, even after `/clear`). if let Some(error) = params.get("error") { let description = params .get("error_description") .cloned() .unwrap_or_else(|| error.clone()); - if let Some(state_param) = params.get("state") + let state_seed = params + .get("state") + .cloned() + .unwrap_or_else(|| "no-state".to_string()); + let correlation = oauth_failure_correlation_id(&state_seed); + + // Pull the full flow out so we can mirror the failure handling of + // the exchange-failure and expiry paths: SSE `OnboardingState::Failed` + // for the UI, legacy-v1 session `pending_auth` clear, and + // credential-scoped engine gate clear. Without all three, an + // `?error=access_denied` from the provider would leave the auth + // card spinning and a legacy v1 session intercepting the next + // user message as a token (Copilot review on PR #3381). + let removed_flow = if let Some(state_param) = params.get("state") && !state_param.is_empty() && let Ok(decoded) = oauth::decode_hosted_oauth_state(state_param) && let Some(ext_mgr) = state.extension_manager.as_ref() @@ -106,14 +256,76 @@ pub(crate) async fn oauth_callback_handler( .pending_oauth_flows() .write() .await - .remove(&decoded.flow_id); + .remove(&decoded.flow_id) + } else { + None + }; + + tracing::warn!( + category = OauthCallbackFailure::ProviderError.as_str(), + correlation = %correlation, + error = %error, + description = %description, + user_id = removed_flow + .as_ref() + .map(|f| f.user_id.as_str()) + .unwrap_or("<unknown>"), + "OAuth callback received provider error" + ); + + if let Some(ref flow) = removed_flow { + // Notify the UI so the auth card stops spinning and shows the + // error instead. Same shape as the expiry branch. + if let Some(ref sse) = flow.sse_manager { + sse.broadcast_for_user( + &flow.user_id, + AppEvent::OnboardingState { + extension_name: flow.extension_name.clone(), + state: crate::channels::web::types::OnboardingStateDto::Failed, + request_id: None, + message: Some(description.clone()), + instructions: None, + auth_url: None, + setup_url: None, + onboarding: None, + thread_id: None, + }, + ); + } + // Discard the pending engine auth gate that was waiting on + // *this* OAuth flow (matched by credential name). Without this + // the engine sits paused forever (#3320). Scoping by + // credential keeps the cleanup from nuking unrelated auth + // gates the user may have open in parallel — see PR review + // on #3381. The bridge helper takes the credential as `&str` + // and parses it backend-side, so the web layer keeps its + // string-typed boundary intact. + crate::bridge::clear_engine_pending_auth_for_credential( + &flow.user_id, + &flow.secret_name, + ) + .await; + // Legacy v1 session cleanup: drop `pending_auth` so the next + // user message goes through to the LLM rather than being + // intercepted as a token. Use `clear_session_auth_mode_for_thread` + // (not the broader `clear_auth_mode`) because the latter would + // re-call the unscoped engine cleanup and undo the credential + // scoping above. + let _ = clear_session_auth_mode_for_thread(&state, &flow.user_id, None).await; } + return oauth_error_page(&description); } let state_param = match params.get("state") { Some(s) if !s.is_empty() => s.clone(), _ => { + let correlation = oauth_failure_correlation_id("missing-state"); + tracing::warn!( + category = OauthCallbackFailure::MissingState.as_str(), + correlation = %correlation, + "OAuth callback missing or empty state parameter" + ); return oauth_error_page("IronClaw"); } }; @@ -121,6 +333,13 @@ pub(crate) async fn oauth_callback_handler( let code = match params.get("code") { Some(c) if !c.is_empty() => c.clone(), _ => { + let correlation = oauth_failure_correlation_id(&state_param); + tracing::warn!( + category = OauthCallbackFailure::MissingCode.as_str(), + correlation = %correlation, + state = %redact_oauth_state_for_logs(&state_param), + "OAuth callback missing or empty code parameter" + ); return oauth_error_page("IronClaw"); } }; @@ -129,6 +348,12 @@ pub(crate) async fn oauth_callback_handler( let ext_mgr = match state.extension_manager.as_ref() { Some(mgr) => mgr, None => { + let correlation = oauth_failure_correlation_id(&state_param); + tracing::warn!( + category = OauthCallbackFailure::NoExtensionManager.as_str(), + correlation = %correlation, + "OAuth callback fired but extension manager is not configured" + ); return oauth_error_page("IronClaw"); } }; @@ -137,7 +362,10 @@ pub(crate) async fn oauth_callback_handler( Ok(decoded) => decoded, Err(error) => { let redacted_state = redact_oauth_state_for_logs(&state_param); + let correlation = oauth_failure_correlation_id(&state_param); tracing::warn!( + category = OauthCallbackFailure::MalformedState.as_str(), + correlation = %correlation, state = %redacted_state, error = %error, "OAuth callback received with malformed state" @@ -159,7 +387,10 @@ pub(crate) async fn oauth_callback_handler( None => { let redacted_state = redact_oauth_state_for_logs(&state_param); let redacted_lookup_key = redact_oauth_state_for_logs(&lookup_key); + let correlation = oauth_failure_correlation_id(&state_param); tracing::warn!( + category = OauthCallbackFailure::UnknownState.as_str(), + correlation = %correlation, state = %redacted_state, lookup_key = %redacted_lookup_key, "OAuth callback received with unknown or expired state" @@ -170,8 +401,12 @@ pub(crate) async fn oauth_callback_handler( // Check flow expiry (5 minutes, matching TCP listener timeout) if flow.created_at.elapsed() > oauth::OAUTH_FLOW_EXPIRY { + let correlation = oauth_failure_correlation_id(flow.extension_name.as_str()); tracing::warn!( + category = OauthCallbackFailure::Expired.as_str(), + correlation = %correlation, extension = %flow.extension_name, + user_id = %flow.user_id, "OAuth flow expired" ); // Notify UI so auth card can show error instead of staying stuck @@ -191,7 +426,18 @@ pub(crate) async fn oauth_callback_handler( }, ); } - clear_auth_mode(&state, &flow.user_id).await; + // Expiry is a terminal failure path just like provider-error and + // exchange-failure: discard the engine pending auth gate so the + // conversation isn't blocked on a callback that will never arrive + // (#3320). Scoped to this flow's credential so unrelated auth + // gates the user has open in parallel survive — review feedback + // on #3381. Use `clear_session_auth_mode_for_thread` for the + // legacy v1 cleanup; the broader `clear_auth_mode` helper would + // re-call `clear_engine_pending_auth(user, None)` and undo the + // credential scoping we just applied. + crate::bridge::clear_engine_pending_auth_for_credential(&flow.user_id, &flow.secret_name) + .await; + let _ = clear_session_auth_mode_for_thread(&state, &flow.user_id, None).await; return oauth_error_page(&flow.display_name); } @@ -322,11 +568,30 @@ pub(crate) async fn oauth_callback_handler( ); } Err(e) => { + // Token exchange or downstream persistence failed. Tag the log + // line with a correlation ID so a user-reported "I saw 400" can + // be mapped to this exact failure without re-enabling verbose + // tracing on the gateway. Then auto-cancel the engine pending + // auth gate for this user so subsequent messages aren't blocked + // waiting for an OAuth resume that will never arrive (#3320). + // + // Scope the cleanup to the failed flow's credential so an + // unrelated pending auth gate (e.g. Slack/MCP open in another + // thread) survives — review feedback on #3381. + let correlation = oauth_failure_correlation_id(flow.extension_name.as_str()); tracing::warn!( + category = OauthCallbackFailure::Exchange.as_str(), + correlation = %correlation, extension = %flow.extension_name, + user_id = %flow.user_id, error = %e, "OAuth failed via gateway callback" ); + crate::bridge::clear_engine_pending_auth_for_credential( + &flow.user_id, + &flow.secret_name, + ) + .await; } } @@ -334,12 +599,16 @@ pub(crate) async fn oauth_callback_handler( // user message goes through to the LLM instead of being intercepted // as a token. // - // Do NOT clear the engine pending-auth gate here: the successful - // callback path still needs the pending gate so it can resolve and - // replay the paused action (preserving the paused_lease), and failed - // callbacks should leave the gate visible for retry from the UI. - // The gate is cleared by the engine itself when `ExternalCallback` - // resolves (success) or when the user explicitly cancels (failure). + // Engine-gate handling is scoped per failure mode and done earlier in + // each branch — provider-error, expiry, and exchange-failure each + // call `clear_engine_pending_auth_for_credential` at their own + // failure site (#3320 + PR #3381 review). The successful-callback + // path intentionally leaves the gate intact so the `ExternalCallback` + // resume below can resolve it and replay the paused action + // (preserving the paused_lease). This is why we use the legacy-v1-only + // `clear_session_auth_mode_for_thread` here instead of the broader + // `clear_auth_mode` helper — `clear_auth_mode` would also discard + // the engine gate on the *success* path and undo the resume. let _ = clear_session_auth_mode_for_thread(&state, &flow.user_id, None).await; // After successful OAuth, auto-activate the extension so it moves @@ -407,74 +676,114 @@ pub(crate) async fn oauth_callback_handler( } if success { - match crate::bridge::resolve_engine_auth_callback(&flow.user_id, &flow.secret_name).await { - Ok(crate::bridge::AuthCallbackContinuation::ResolveGateExternal { - channel, - thread_scope, - request_id, - }) => { - if let Some(tx) = state.msg_tx.read().await.as_ref().cloned() { - let callback = - crate::agent::submission::Submission::ExternalCallback { request_id }; - match serde_json::to_string(&callback) { - Ok(content) => { - let msg = web_incoming_message( - &channel, - &flow.user_id, - content, - thread_scope.as_deref(), - ); - if let Err(e) = tx.send(msg).await { + // Half-2 of #3133, two-pronged auto-resume: + // + // 1. Wake any Tier 0 / Tier 1 inline-await VMs parked on this + // credential. The CodeAct VM (mission's child thread, in + // the bug-shape #3133 reported) keeps its full state across + // the OAuth round-trip; on Approved it retries the original + // action and continues without unwinding. + // 2. Auto-resume any paused background missions whose + // `paused_gate` matches this credential. This handles the + // case where the mission's child thread already finished + // (Tier 0 unwind, or Tier 1 hit MaxIterations) before + // OAuth completed. + // Both are best-effort — failures are logged inside the bridge + // helpers and never block the OAuth landing page. + let inline_woken = + crate::bridge::resolve_inline_gates_for_credential(&flow.user_id, &flow.secret_name) + .await; + let _ = + crate::bridge::resume_paused_missions_for_credential(&flow.user_id, &flow.secret_name) + .await; + + // #3533: when the inline-await path already woke a parked + // waiter, the engine is already retrying the action that was + // blocked on this credential. Sending an `ExternalCallback` + // submission down the agent loop in addition to that is + // redundant and causes "thread already running" races against + // the in-flight retry (and re-dispatches `tool_install` a + // second time when the mock LLM pattern-matches the user + // turn). Skip the external-callback re-entry in that case; + // it stays in place for paths where no inline waiter exists + // (mission's child thread already finished, or Tier 0 unwound + // before OAuth landed). + let skip_external_callback = inline_woken > 0; + + if !skip_external_callback { + match crate::bridge::resolve_engine_auth_callback(&flow.user_id, &flow.secret_name) + .await + { + Ok(crate::bridge::AuthCallbackContinuation::ResolveGateExternal { + channel, + thread_scope, + request_id, + }) => { + if let Some(tx) = state.msg_tx.read().await.as_ref().cloned() { + let callback = crate::agent::submission::Submission::ExternalCallback { + request_id, + payload: None, + }; + match serde_json::to_string(&callback) { + Ok(content) => { + let msg = web_incoming_message( + &channel, + &flow.user_id, + content, + thread_scope.as_deref(), + ); + if let Err(e) = tx.send(msg).await { + tracing::warn!( + extension = %extension_name, + user_id = %flow.user_id, + error = %e, + "Failed to resolve pending engine auth gate after OAuth callback" + ); + } + } + Err(e) => { tracing::warn!( extension = %extension_name, user_id = %flow.user_id, error = %e, - "Failed to resolve pending engine auth gate after OAuth callback" + "Failed to serialize external callback submission" ); } } - Err(e) => { + } + } + Ok(crate::bridge::AuthCallbackContinuation::ReplayMessage { + channel, + thread_scope, + content, + }) => { + if let Some(tx) = state.msg_tx.read().await.as_ref().cloned() { + let msg = web_incoming_message( + &channel, + &flow.user_id, + content, + thread_scope.as_deref(), + ); + if let Err(e) = tx.send(msg).await { tracing::warn!( extension = %extension_name, user_id = %flow.user_id, error = %e, - "Failed to serialize external callback submission" + "Failed to replay pending engine auth request after OAuth callback" ); } } } - } - Ok(crate::bridge::AuthCallbackContinuation::ReplayMessage { - channel, - thread_scope, - content, - }) => { - if let Some(tx) = state.msg_tx.read().await.as_ref().cloned() { - let msg = web_incoming_message( - &channel, - &flow.user_id, - content, - thread_scope.as_deref(), + Ok(crate::bridge::AuthCallbackContinuation::None) => {} + Err(e) => { + tracing::warn!( + extension = %extension_name, + user_id = %flow.user_id, + error = %e, + "Failed to resume pending engine auth gate after OAuth callback" ); - if let Err(e) = tx.send(msg).await { - tracing::warn!( - extension = %extension_name, - user_id = %flow.user_id, - error = %e, - "Failed to replay pending engine auth request after OAuth callback" - ); - } } } - Ok(crate::bridge::AuthCallbackContinuation::None) => {} - Err(e) => { - tracing::warn!( - extension = %extension_name, - user_id = %flow.user_id, - error = %e, - "Failed to resume pending engine auth gate after OAuth callback" - ); - } } } @@ -709,6 +1018,32 @@ pub(crate) async fn slack_relay_oauth_callback_handler( .into_response(); } + // Resolve the IronClaw user who initiated this OAuth flow BEFORE the + // result block so the completion toast can be routed to that tenant + // even on the failure path. + let oauth_user = resolve_relay_oauth_user( + ext_mgr.secrets().as_ref(), + &state.owner_id, + &relay_extension_name, + state.multi_tenant_mode, + ) + .await; + + // Delete the temporary `relay:{ext}:oauth_user` secret unconditionally + // once we've captured its value — regardless of whether the result + // block below short-circuits, whether `pairing_store` is wired up, + // or whether identity pairing later fails. Leaving it behind would + // let a subsequent OAuth callback for the same extension read a + // stale initiating user and misroute the toast (Copilot review on + // PR #3390 commit d247bde8). The previous cleanup site inside the + // `if let Some(pairing_store)` branch was unreachable on three of + // those failure paths. + let oauth_user_key = format!("relay:{}:oauth_user", relay_extension_name); + let _ = ext_mgr + .secrets() + .delete(&state.owner_id, &oauth_user_key) + .await; + let result: Result<(), String> = async { let store = state.store.as_ref().ok_or_else(|| { "Relay activation requires persistent settings storage; no-db mode is unsupported." @@ -736,7 +1071,8 @@ pub(crate) async fn slack_relay_oauth_callback_handler( format!("Failed to persist relay team_id: {e}") })?; - // Activate the relay channel + // Activate the relay channel first — this creates the relay client and + // verifies the connection is usable. tracing::info!( relay = %relay_extension_name, owner_id = %state.owner_id, @@ -747,6 +1083,81 @@ pub(crate) async fn slack_relay_oauth_callback_handler( .await .map_err(|e| format!("Failed to activate relay channel: {}", e))?; + // Create channel identity pairing: Slack authed_user_id → IronClaw user. + // Fetch authed_user_id from the relay's connections API (server-side, + // not from the redirect URL which could be tampered). + if let Some(pairing_store) = ext_mgr.pairing_store() { + let relay_config = ext_mgr + .relay_config() + .map_err(|e| format!("Relay config not available: {e}"))?; + let effective_url = ext_mgr + .effective_relay_url(&relay_extension_name) + .await + .unwrap_or_else(|| relay_config.url.clone()); + let client = crate::channels::relay::RelayClient::new( + effective_url, + relay_config.api_key.clone(), + relay_config.request_timeout_secs, + ) + .map_err(|e| format!("Failed to create relay client: {e}"))?; + + let connections = client + .list_connections("") + .await + .map_err(|e| format!("Failed to fetch relay connections: {e}"))?; + let authed_user_id = connections + .iter() + .find(|c| c.team_id == team_id) + .and_then(|c| c.authed_user_id.clone()) + .ok_or_else(|| { + "No connection with authed_user_id found for this team".to_string() + })?; + + // The outer `oauth_user` (resolved by `resolve_relay_oauth_user` + // above the result block) is the IronClaw user this OAuth flow + // belongs to. Reuse it here so the pairing identity and the + // completion toast always agree on the target tenant. The + // temporary `relay:{ext}:oauth_user` secret was already + // deleted unconditionally above the result block, so we + // never reach this branch with a stale credential lingering. + let user_record = if let Some(ref db) = state.store { + db.get_user(&oauth_user).await.ok().flatten() + } else { + None + }; + let Some(ref record) = user_record else { + return Err(format!( + "OAuth user '{oauth_user}' not found — cannot create relay identity" + )); + }; + if record.status != "active" { + return Err(format!( + "OAuth user '{oauth_user}' is not active (status: {})", + record.status + )); + } + let role = match record.role.as_str() { + "owner" => crate::ownership::UserRole::Owner, + "admin" => crate::ownership::UserRole::Admin, + _ => crate::ownership::UserRole::Regular, + }; + let Ok(user_id) = crate::ownership::UserId::new(&oauth_user, role) else { + return Err(format!( + "OAuth user '{oauth_user}' has invalid user_id format" + )); + }; + // Scope external_id to workspace: "team_id:slack_user_id" + let scoped_external_id = format!("{}:{}", team_id, authed_user_id); + pairing_store + .create_identity( + crate::channels::relay::channel::DEFAULT_RELAY_NAME, + &scoped_external_id, + &user_id, + ) + .await + .map_err(|e| format!("Failed to create relay identity: {e}"))?; + } + Ok(()) } .await; @@ -762,8 +1173,14 @@ pub(crate) async fn slack_relay_oauth_callback_handler( } }; - // Broadcast event to notify the web UI. - state.sse.broadcast(AppEvent::OnboardingState { + // Broadcast event to notify the web UI. Scope to the resolved + // OAuth flow user (`oauth_user`) rather than `state.owner_id` — + // in multi-tenant mode a non-owner can complete a Slack OAuth + // flow, and the completion toast must reach THAT user's open + // browser tab, not the gateway owner's. Broadcasting to + // `state.owner_id` was a cross-tenant misroute even after the + // global-broadcast leak was closed. + let onboarding_event = AppEvent::OnboardingState { extension_name: ironclaw_common::ExtensionName::from_trusted(relay_extension_name.clone()), state: if success { crate::channels::web::types::OnboardingStateDto::Ready @@ -777,7 +1194,8 @@ pub(crate) async fn slack_relay_oauth_callback_handler( setup_url: None, onboarding: None, thread_id: None, - }); + }; + state.sse.broadcast_for_user(&oauth_user, onboarding_event); // projection-exempt: channel-lifecycle, slack relay onboarding state if success { axum::response::Html( @@ -828,6 +1246,97 @@ mod tests { use crate::testing::credentials::TEST_GATEWAY_CRYPTO_KEY; + /// `auth_channel_relay` stores the initiating user_id under + /// `relay:{ext}:oauth_user` namespaced by the GATEWAY OWNER's + /// secret-store user. The callback must read it back from the same + /// namespace and return it — broadcasting the completion toast to + /// `state.owner_id` instead would deliver the success/failure + /// notification to the wrong tenant in multi-tenant deployments. + #[tokio::test] + async fn resolve_relay_oauth_user_returns_stored_value() { + use crate::secrets::CreateSecretParams; + + let secrets = test_secrets_store(); + let owner_id = "gateway-owner"; + let extension_name = DEFAULT_RELAY_NAME; + let initiating_user = "alice"; // deliberately != owner_id + + secrets + .create( + owner_id, + CreateSecretParams::new( + format!("relay:{extension_name}:oauth_user"), + initiating_user, + ), + ) + .await + .expect("seed oauth_user secret"); // safety: cfg(test) fixture + + // Multi-tenant flag is irrelevant when the secret IS present — + // the resolved value comes from the stored initiator regardless. + let resolved = super::resolve_relay_oauth_user( + secrets.as_ref(), + owner_id, + extension_name, + true, // multi_tenant_mode + ) + .await; + assert_eq!( + resolved, initiating_user, + "callback must broadcast to the initiating user, not the gateway owner" + ); + } + + /// Falls back to `owner_id` when the secret was never written — + /// covers single-tenant deployments and pre-multi-tenant flows. + /// In single-tenant mode the fallback is silent: owner == only user, + /// so routing the toast there is correct, not a misroute. + #[tokio::test] + async fn resolve_relay_oauth_user_falls_back_to_owner_id() { + let secrets = test_secrets_store(); + let owner_id = "single-tenant-owner"; + + let resolved = super::resolve_relay_oauth_user( + secrets.as_ref(), + owner_id, + DEFAULT_RELAY_NAME, + false, // multi_tenant_mode + ) + .await; + assert_eq!( + resolved, owner_id, + "missing secret must fall back to owner_id, not panic or empty-string" + ); + } + + /// In multi-tenant mode a missing initiator secret means the + /// original tenant is unrecoverable. The function still falls back + /// to `owner_id` so the OAuth flow doesn't strand a half-installed + /// Slack relay, but the WARN documented in `resolve_relay_oauth_user` + /// is the operator-visible signal that a toast may have reached the + /// wrong tab. We can't easily assert on tracing output without + /// pulling in `tracing-test`, so this test pins the behavioral + /// contract: even with `multi_tenant_mode=true`, the function does + /// not panic, return empty-string, or block the callback. + #[tokio::test] + async fn resolve_relay_oauth_user_multitenant_fallback_returns_owner_id() { + let secrets = test_secrets_store(); + let owner_id = "gateway-owner"; + + let resolved = super::resolve_relay_oauth_user( + secrets.as_ref(), + owner_id, + DEFAULT_RELAY_NAME, + true, // multi_tenant_mode + ) + .await; + assert_eq!( + resolved, owner_id, + "multi-tenant fallback must still return owner_id (and emit the WARN); \ + returning empty would crash the broadcast call site" + ); + } + fn test_oauth_router(state: Arc<GatewayState>) -> Router { Router::new() .route("/oauth/callback", get(oauth_callback_handler)) @@ -1052,6 +1561,126 @@ mod tests { assert!(html.contains("Authorization Failed")); } + /// Regression test for #3320: when the OAuth provider returns an error, + /// the gateway must remove the pending flow associated with the inbound + /// `state` so a subsequent auth attempt isn't blocked by a ghost entry + /// (the entry would otherwise live until `OAUTH_FLOW_EXPIRY`). + /// + /// The previous code path did `.remove(&decoded.flow_id)` but threw + /// the value away, so we never knew the flow's `user_id` and could not + /// auto-cancel the matching engine pending auth gate. After the fix + /// we capture the user_id; the engine-gate side of the cleanup is + /// covered by integration tests since it requires bringing up the + /// engine state. + #[tokio::test] + async fn test_oauth_callback_provider_error_drains_pending_flow() { + use axum::body::Body; + use tower::ServiceExt; + + let secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = + Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( + crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( + TEST_GATEWAY_CRYPTO_KEY.to_string(), + )) + .expect("crypto"), + ))); + let (ext_mgr, _wasm_tools_dir, _wasm_channels_dir) = test_ext_mgr(secrets.clone()); + + // Insert a pending flow keyed by a known `flow_id`. Encode an + // OAuth `state` value that decodes to this flow_id so the + // error-branch cleanup actually exercises the lookup, not the + // outer "no state param" path. + let flow_id = "drain-me-flow-id"; + let flow = fresh_pending_oauth_flow(secrets.clone(), None, None); + ext_mgr + .pending_oauth_flows() + .write() + .await + .insert(flow_id.to_string(), flow); + + let state_param = crate::auth::oauth::encode_hosted_oauth_state(flow_id, None); + + let state = test_gateway_state(Some(Arc::clone(&ext_mgr))); + let app = test_oauth_router(Arc::clone(&state)); + + let req = axum::http::Request::builder() + .uri(format!( + "/oauth/callback?error=access_denied&error_description=user_denied&state={state_param}" + )) + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::<axum::http::Request<Body>>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + // Pending flow must be drained on provider error so a stale + // `state` can't satisfy a later attempt. + let flows = ext_mgr.pending_oauth_flows().read().await; + assert!( + !flows.contains_key(flow_id), + "pending OAuth flow should be drained on provider error" + ); + } + + /// Regression for Copilot review on PR #3381: the provider-error + /// branch must mirror the exchange-failure / expiry branches — + /// broadcast `OnboardingState::Failed` so the UI auth card stops + /// spinning, not just drain the pending flow and return an error + /// page. Without the SSE emit the legacy v1 web UI gets no signal + /// that the OAuth dance has terminated. + #[tokio::test] + async fn test_oauth_callback_provider_error_broadcasts_onboarding_failed() { + use axum::body::Body; + use tower::ServiceExt; + + let secrets = test_secrets_store(); + let (ext_mgr, _wasm_tools_dir, _wasm_channels_dir) = test_ext_mgr(Arc::clone(&secrets)); + let sse_mgr = Arc::new(SseManager::new()); + let mut receiver = sse_mgr.sender().subscribe(); + let flow = fresh_pending_oauth_flow(Arc::clone(&secrets), Some(Arc::clone(&sse_mgr)), None); + let flow_id = "provider-error-broadcast-flow"; + ext_mgr + .pending_oauth_flows() + .write() + .await + .insert(flow_id.to_string(), flow); + + let state_param = crate::auth::oauth::encode_hosted_oauth_state(flow_id, None); + let state = test_gateway_state(Some(Arc::clone(&ext_mgr))); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri(format!( + "/oauth/callback?error=access_denied&error_description=user_denied&state={}", + urlencoding::encode(&state_param) + )) + .body(Body::empty()) + .expect("request"); + let resp = ServiceExt::<axum::http::Request<Body>>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + match receiver.recv().await.expect("onboarding_state event").event { + crate::channels::web::types::AppEvent::OnboardingState { + extension_name, + state, + message, + .. + } => { + assert_eq!(extension_name, "test_tool"); + assert_eq!( + state, + crate::channels::web::types::OnboardingStateDto::Failed + ); + assert_eq!(message.as_deref(), Some("user_denied")); + } + event => panic!("expected OnboardingState event, got {event:?}"), + } + } + #[tokio::test] async fn test_oauth_callback_unknown_state() { use axum::body::Body; @@ -2054,6 +2683,83 @@ mod tests { assert!(!exists, "Legacy CSRF nonce should be deleted after use"); } + /// Regression: the `relay:{ext}:oauth_user` secret stores the + /// IronClaw user who initiated the OAuth flow. The callback must + /// consume it unconditionally once the value is read, regardless of + /// whether downstream activation, identity-pairing, or a missing + /// `pairing_store` causes the result block to short-circuit. Leaving + /// the secret behind would let a subsequent OAuth callback for the + /// same extension read a stale initiating user and misroute the + /// completion toast. + /// + /// The test fixture has no real relay service and no `pairing_store` + /// wired up, so the result block errors out after the CSRF check + /// passes — exactly the failure path Copilot flagged on PR #3390 + /// (comment id 3211833864) where the previous in-`if-let` cleanup + /// site was unreachable. + #[tokio::test] + async fn test_relay_oauth_callback_consumes_oauth_user_secret_on_failure_path() { + use axum::body::Body; + use tower::ServiceExt; + + let secrets = test_secrets_store(); + let nonce = "test-nonce-oauth-user-cleanup"; + let relay_name = crate::extensions::naming::canonicalize_extension_name(DEFAULT_RELAY_NAME) + .expect("canonical relay name"); + + secrets + .create( + "test", + crate::secrets::CreateSecretParams::new( + format!("relay:{}:oauth_state", relay_name), + nonce, + ), + ) + .await + .expect("store nonce"); // safety: cfg(test) fixture + secrets + .create( + "test", + crate::secrets::CreateSecretParams::new( + format!("relay:{}:oauth_user", relay_name), + "alice", // initiating user, deliberately != owner_id + ), + ) + .await + .expect("store oauth_user secret"); // safety: cfg(test) fixture + + let (ext_mgr, _wasm_tools_dir, _wasm_channels_dir) = test_ext_mgr(secrets.clone()); + let state = test_gateway_state(Some(ext_mgr)); + let app = test_relay_oauth_router(state); + + let req = axum::http::Request::builder() + .uri(format!( + "/oauth/slack/callback?team_id=T123&provider=slack&state={}", + nonce + )) + .body(Body::empty()) + .expect("request"); + + let _resp = ServiceExt::<axum::http::Request<Body>>::oneshot(app, req) + .await + .expect("response"); + // We don't assert on the response body — the result block almost + // certainly errors (no real relay backend), and that's the point: + // the cleanup must still run on the failure path. + + let oauth_user_key = format!("relay:{}:oauth_user", relay_name); + let exists = secrets + .exists("test", &oauth_user_key) + .await + .unwrap_or(true); + assert!( + !exists, + "relay:{{ext}}:oauth_user must be deleted unconditionally — \ + leaving it behind on the failure path lets a subsequent \ + OAuth callback misroute the completion toast" + ); + } + #[tokio::test] async fn test_relay_oauth_callback_nonce_under_different_user_fails() { // why: In hosted mode, the DB user's UUID differs from the gateway diff --git a/src/channels/web/features/pairing/mod.rs b/src/channels/web/features/pairing/mod.rs index b23a63da91b..86b8b0ec9fe 100644 --- a/src/channels/web/features/pairing/mod.rs +++ b/src/channels/web/features/pairing/mod.rs @@ -677,7 +677,7 @@ mod tests { .expect("structured submission sideband"); assert!(matches!( submission, - crate::agent::submission::Submission::ExternalCallback { request_id: rid } + crate::agent::submission::Submission::ExternalCallback { request_id: rid, .. } if rid == request_id )); assert_eq!(callback.content, "[structured external callback]"); diff --git a/src/channels/web/features/settings/mod.rs b/src/channels/web/features/settings/mod.rs index 4b0c4008465..413ce02c7d7 100644 --- a/src/channels/web/features/settings/mod.rs +++ b/src/channels/web/features/settings/mod.rs @@ -249,19 +249,35 @@ pub async fn settings_set_handler( /// Keep this list narrow: every key added here causes an extra /// `Config::from_db_with_toml` round-trip plus a chain rebuild (retry, cache, /// circuit breaker wrappers), so non-LLM settings must not be listed. +/// +/// Both exact-match keys and dotted-path subpaths under +/// `llm_builtin_overrides.*` and `llm_custom_providers.*` trigger the +/// reload — Layer D moved bedrock-specific settings into +/// `llm_builtin_overrides["bedrock"].extras`, so a write to e.g. +/// `llm_builtin_overrides.bedrock.extras.region` must also rebuild the +/// chain. fn llm_setting_requires_reload(key: &str) -> bool { - matches!( - key, - "llm_backend" - | "selected_model" - | "llm_custom_providers" - | "llm_builtin_overrides" - | "ollama_base_url" - | "openai_compatible_base_url" - | "bedrock_region" - | "bedrock_cross_region" - | "bedrock_profile" - ) + const EXACT: &[&str] = &[ + "llm_backend", + "selected_model", + "llm_custom_providers", + "llm_builtin_overrides", + "ollama_base_url", + "openai_compatible_base_url", + // Legacy bedrock keys retained for backward-compat with + // settings.json files written before Layer D. New code writes to + // `llm_builtin_overrides.bedrock.extras.*` instead. + "bedrock_region", + "bedrock_cross_region", + "bedrock_profile", + ]; + const PREFIX: &[&str] = &["llm_builtin_overrides", "llm_custom_providers"]; + if EXACT.contains(&key) { + return true; + } + PREFIX.iter().any(|root| { + key.len() > root.len() + 1 && key.starts_with(root) && key.as_bytes()[root.len()] == b'.' + }) } /// True when writes to `effective_user_id` actually feed the global provider @@ -697,7 +713,11 @@ pub async fn settings_import_handler( fn is_admin_only_setting_key(key: &str) -> bool { // Single source of truth lives in `crate::config::helpers` so the // write-side gate here cannot drift from the read-side strip filter. - crate::config::helpers::ADMIN_ONLY_LLM_SETTING_KEYS.contains(&key) + // Must match dotted subpaths too: a non-admin write to + // `llm_builtin_overrides.bedrock.extras.region` reaches the same + // resolver state as a write to `llm_builtin_overrides`, so the gate + // has to cover both shapes. + crate::config::helpers::is_admin_only_llm_key(key) } fn ensure_setting_write_allowed( @@ -1602,7 +1622,7 @@ mod tests { // What actually prevents such a config from being used at runtime: // 1. Frontend activation guard (isProviderConfigured in // static/js/surfaces/config.js blocks the "Use" button). - // 2. Startup fallback in `LlmConfig::resolve_with_fallback` + // 2. Startup fallback in `crate::config::llm::resolve_with_fallback` // (invoked from `Config::re_resolve_llm_with_secrets`) — // demotes unusable custom providers to NearAI rather than // crash-looping the instance (#2514). @@ -1623,6 +1643,28 @@ mod tests { assert!(!is_admin_only_setting_key("selected_model")); } + /// Regression: dotted subpaths under an admin-only root must also be + /// gated. The read-side strip filter (`strip_admin_only_llm_keys`) + /// supports both exact-match and dotted-prefix matching; the + /// write-side gate used to only check exact match, so a non-admin + /// could write `llm_builtin_overrides.bedrock.extras.region` directly + /// even though the root key was protected. + #[test] + fn test_admin_only_setting_keys_cover_dotted_subpaths() { + assert!(is_admin_only_setting_key( + "llm_builtin_overrides.bedrock.extras.region" + )); + assert!(is_admin_only_setting_key( + "llm_builtin_overrides.bedrock.api_key" + )); + assert!(is_admin_only_setting_key( + "llm_custom_providers.my_provider.base_url" + )); + // Sanity: unrelated dotted subpaths must still be allowed. + assert!(!is_admin_only_setting_key("tool_permissions.http")); + assert!(!is_admin_only_setting_key("agent.name")); + } + #[tokio::test] async fn test_settings_set_rejects_member_for_admin_only_key() { let secrets = test_secrets_store(); @@ -1681,47 +1723,14 @@ mod tests { #[tokio::test] #[allow(clippy::await_holding_lock)] // env guard must span async hot-reload flow async fn settings_set_handler_triggers_llm_provider_hot_reload() { - use crate::llm::{LlmConfig, SessionConfig, SessionManager, build_provider_chain}; + use ironclaw_llm::{SessionConfig, SessionManager, build_provider_chain}; let _env_guard = lock_env(); let secrets = test_secrets_store(); let (db, tmp) = crate::testing::test_db().await; // Starting config: NEAR AI backend with "model-start". - let mut initial = LlmConfig { - backend: "nearai".to_string(), - session: SessionConfig::default(), - nearai: crate::llm::config::NearAiConfig { - model: "model-start".to_string(), - cheap_model: None, - base_url: "https://api.near.ai".to_string(), - api_key: None, - fallback_model: None, - max_retries: 0, - circuit_breaker_threshold: None, - circuit_breaker_recovery_secs: 30, - response_cache_enabled: false, - response_cache_ttl_secs: 3600, - response_cache_max_entries: 1000, - failover_cooldown_secs: 300, - failover_cooldown_threshold: 3, - smart_routing_cascade: true, - }, - provider: None, - bedrock: None, - gemini_oauth: None, - request_timeout_secs: 120, - cheap_model: None, - smart_routing_cascade: true, - openai_codex: None, - max_retries: 0, - circuit_breaker_threshold: None, - circuit_breaker_recovery_secs: 30, - response_cache_enabled: false, - response_cache_ttl_secs: 3600, - response_cache_max_entries: 1000, - }; - initial.nearai.model = "model-start".to_string(); + let initial = ironclaw_llm::testing::nearai_test_config("model-start"); let session = Arc::new(SessionManager::new(SessionConfig::default())); let (primary, _cheap, _recording, reload_handle) = @@ -1831,47 +1840,15 @@ mod tests { /// wrapper so tests can observe swap side effects. async fn hot_reload_harness() -> ( Arc<GatewayState>, - Arc<dyn crate::llm::LlmProvider>, + Arc<dyn ironclaw_llm::LlmProvider>, tempfile::TempDir, ) { - use crate::llm::{LlmConfig, SessionConfig, SessionManager, build_provider_chain}; + use ironclaw_llm::{SessionConfig, SessionManager, build_provider_chain}; let secrets = test_secrets_store(); let (db, tmp) = crate::testing::test_db().await; - let initial = LlmConfig { - backend: "nearai".to_string(), - session: SessionConfig::default(), - nearai: crate::llm::config::NearAiConfig { - model: "model-start".to_string(), - cheap_model: None, - base_url: "https://api.near.ai".to_string(), - api_key: None, - fallback_model: None, - max_retries: 0, - circuit_breaker_threshold: None, - circuit_breaker_recovery_secs: 30, - response_cache_enabled: false, - response_cache_ttl_secs: 3600, - response_cache_max_entries: 1000, - failover_cooldown_secs: 300, - failover_cooldown_threshold: 3, - smart_routing_cascade: true, - }, - provider: None, - bedrock: None, - gemini_oauth: None, - request_timeout_secs: 120, - cheap_model: None, - smart_routing_cascade: true, - openai_codex: None, - max_retries: 0, - circuit_breaker_threshold: None, - circuit_breaker_recovery_secs: 30, - response_cache_enabled: false, - response_cache_ttl_secs: 3600, - response_cache_max_entries: 1000, - }; + let initial = ironclaw_llm::testing::nearai_test_config("model-start"); let session = Arc::new(SessionManager::new(SessionConfig::default())); let (primary, _cheap, _recording, reload_handle) = build_provider_chain(&initial, Arc::clone(&session)) @@ -2430,4 +2407,66 @@ mod tests { assert!(locked_entry.locked); assert!(locked_entry.locked_reason.is_some()); } + + /// Regression for #3034: a new user with no persisted tool-permission + /// override must see the `http` tool as `always_allow` via the + /// `/api/settings/tools` listing — the same code path the web settings + /// UI consumes. The seeded baseline drives this; if it ever flips to + /// `disabled` or `ask_each_time`, every HTTP-dependent workflow stops + /// working out of the box. + #[cfg(feature = "libsql")] + #[tokio::test] + async fn test_http_tool_default_is_always_allow_for_new_user() { + use std::sync::Arc; + + use crate::db::Database; + use crate::tools::ToolRegistry; + use crate::tools::builtin::HttpTool; + use axum::extract::State; + + let registry = Arc::new(ToolRegistry::new()); + registry.register(Arc::new(HttpTool::new())).await; + + let tmp_dir = tempfile::tempdir().expect("tempdir"); + let db_path = tmp_dir.path().join("test.db"); + let db = crate::db::libsql::LibSqlBackend::new_local(&db_path) + .await + .expect("temp db"); + db.run_migrations().await.expect("migrations"); + let db: Arc<dyn Database> = Arc::new(db); + + let state = Arc::new(GatewayState { + tool_registry: Some(registry), + store: Some(db), + ..test_gateway_state(test_secrets_store()) + }); + + let result = settings_tools_list_handler( + State(state), + crate::channels::web::auth::AuthenticatedUser( + crate::channels::web::auth::UserIdentity { + user_id: "fresh-user".to_string(), + role: "regular".to_string(), + workspace_read_scopes: vec![], + }, + ), + ) + .await; + + let axum::Json(response) = result.expect("handler should succeed"); + let http_entry = response + .tools + .iter() + .find(|t| t.name == "http") + .expect("http tool should be in the listing"); + + assert_eq!( + http_entry.current_state, "always_allow", + "http must default to always_allow for a user with no override (issue #3034)" + ); + assert_eq!( + http_entry.default_state, "always_allow", + "http's surfaced default must be always_allow (issue #3034)" + ); + } } diff --git a/src/channels/web/handlers/frontend.rs b/src/channels/web/handlers/frontend.rs index 62c5a452cce..e4d18545ce9 100644 --- a/src/channels/web/handlers/frontend.rs +++ b/src/channels/web/handlers/frontend.rs @@ -260,15 +260,17 @@ pub async fn project_widgets_handler( Path(project_id): Path<String>, ) -> Result<Json<Vec<ResolvedWidget>>, (StatusCode, String)> { // Resolve project name to derive the workspace slug. - let project = crate::bridge::get_engine_project(&project_id, &user.user_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .ok_or((StatusCode::NOT_FOUND, "Project not found".to_string()))?; - - let slug = project - .name - .to_lowercase() - .replace(|c: char| !c.is_ascii_alphanumeric() && c != '-', "-"); + let slug = { + let project = crate::bridge::get_engine_project(&project_id, &user.user_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Project not found".to_string()))?; + + project + .name + .to_lowercase() + .replace(|c: char| !c.is_ascii_alphanumeric() && c != '-', "-") + }; let widgets_dir = format!("projects/{slug}/.system/widgets/"); let workspace = resolve_workspace(&state, &user).await?; @@ -287,8 +289,8 @@ pub async fn project_widgets_handler( if !entry.is_directory { continue; } - let name = entry.name(); - if !is_safe_widget_id(name) { + let name = entry.name().to_string(); + if !is_safe_widget_id(&name) { continue; } let manifest_path = format!("{widgets_dir}{name}/manifest.json"); diff --git a/src/channels/web/handlers/llm.rs b/src/channels/web/handlers/llm.rs index 0462438d504..0975f0426de 100644 --- a/src/channels/web/handlers/llm.rs +++ b/src/channels/web/handlers/llm.rs @@ -393,7 +393,7 @@ pub async fn llm_providers_handler( fn build_llm_providers(nearai_has_session_token: bool) -> serde_json::Value { use crate::config::helpers::optional_env; - use crate::llm::registry::ProviderRegistry; + use ironclaw_llm::registry::ProviderRegistry; let registry = ProviderRegistry::load(); @@ -404,42 +404,13 @@ fn build_llm_providers(nearai_has_session_token: bool) -> serde_json::Value { let mut providers = Vec::new(); - // NEAR AI is not in the registry — add it as a special case. - { - let mut entry = serde_json::Map::new(); - entry.insert("id".into(), "nearai".into()); - entry.insert("name".into(), "NEAR AI".into()); - entry.insert("adapter".into(), "nearai".into()); - entry.insert("base_url".into(), "https://cloud-api.near.ai/v1".into()); - entry.insert("builtin".into(), true.into()); - entry.insert( - "default_model".into(), - serde_json::Value::String(crate::llm::DEFAULT_MODEL.to_string()), - ); - entry.insert("api_key_required".into(), true.into()); - entry.insert("base_url_required".into(), false.into()); - entry.insert("can_list_models".into(), true.into()); - // Env defaults — true if either an env API key OR a loaded session - // token is present; the frontend treats either as "credentials - // configured" because both reach NEAR AI as `Bearer <token>`. - entry.insert( - "has_api_key".into(), - (read_env("NEARAI_API_KEY").is_some() || nearai_has_session_token).into(), - ); - if let Some(model) = read_env("NEARAI_MODEL") { - entry.insert("env_model".into(), serde_json::Value::String(model)); - } - if let Some(url) = read_env("NEARAI_BASE_URL") { - entry.insert("env_base_url".into(), serde_json::Value::String(url)); - } - providers.push(serde_json::Value::Object(entry)); - } - - // Registry-based providers + // Single registry-driven loop. NEAR AI / Bedrock / OpenAI Codex / + // Gemini OAuth are now first-class registry entries (Layer B), so + // the synthetic per-backend blocks that used to live here are gone. for def in registry.all() { let mut entry = serde_json::Map::new(); entry.insert("id".into(), serde_json::Value::String(def.id.clone())); - // Use display_name from setup hint, falling back to titlecased id. + // Use display_name from setup hint, falling back to the id. let name = def .setup .as_ref() @@ -463,12 +434,50 @@ fn build_llm_providers(nearai_has_session_token: bool) -> serde_json::Value { ); entry.insert("api_key_required".into(), def.api_key_required.into()); entry.insert("base_url_required".into(), def.base_url_required.into()); + // `accepts_api_key` answers "should the configure UI show the + // API key input?", which is different from "is the API key + // required to make this provider work?". NEAR AI has dual auth + // (session token OR API key) — `api_key_required: false` but + // we still want the user to be able to enter `NEARAI_API_KEY` + // from the settings dialog. Drive the UI off the presence of + // an `api_key_env` declaration in the registry, which is true + // for every provider that actually consumes an API key. + let accepts_api_key = def.api_key_env.is_some(); + entry.insert("accepts_api_key".into(), accepts_api_key.into()); let can_list = def.setup.as_ref().is_some_and(|s| s.can_list_models()); entry.insert("can_list_models".into(), can_list.into()); - // Env defaults - if let Some(ref api_key_env) = def.api_key_env { - entry.insert("has_api_key".into(), read_env(api_key_env).is_some().into()); + + // Env defaults / has_api_key. NEAR AI is "configured" if either + // its API key env is set OR a session token has been loaded — + // both reach the API as `Bearer <token>`. + let mut has_api_key = def + .api_key_env + .as_ref() + .is_some_and(|env| read_env(env).is_some()); + if def.id == "nearai" && nearai_has_session_token { + has_api_key = true; } + entry.insert("has_api_key".into(), has_api_key.into()); + + // Wire-stable credential discriminator + backend-authoritative + // "configured" flag. For backends with `api_key_required: false` + // (nearai session token, gemini oauth creds file, openai codex + // device-code session, AWS Bedrock profile), `api_key_required` + // alone tells the frontend nothing — it would render the Use + // button on a fresh install with no credentials and a click + // could trigger an interactive OAuth from a settings request. + // The frontend now gates non-`api_key` kinds on + // `has_credentials`. + let credential_kind = def.setup.as_ref().map_or("none", |s| s.kind()); + entry.insert( + "credential_kind".into(), + serde_json::Value::String(credential_kind.to_string()), + ); + entry.insert( + "has_credentials".into(), + backend_has_credentials(def, has_api_key, &read_env).into(), + ); + if let Some(model) = read_env(&def.model_env) { entry.insert("env_model".into(), serde_json::Value::String(model)); } @@ -480,25 +489,80 @@ fn build_llm_providers(nearai_has_session_token: bool) -> serde_json::Value { providers.push(serde_json::Value::Object(entry)); } - // Bedrock is not in the registry — add it as a special case. - { - let mut entry = serde_json::Map::new(); - entry.insert("id".into(), "bedrock".into()); - entry.insert("name".into(), "AWS Bedrock".into()); - entry.insert("adapter".into(), "bedrock".into()); - entry.insert("base_url".into(), "".into()); - entry.insert("builtin".into(), true.into()); - entry.insert( - "default_model".into(), - "anthropic.claude-3-sonnet-20240229-v1:0".into(), - ); - entry.insert("api_key_required".into(), false.into()); - entry.insert("base_url_required".into(), false.into()); - entry.insert("can_list_models".into(), false.into()); - providers.push(serde_json::Value::Object(entry)); + serde_json::Value::Array(providers) +} + +/// Best-effort "are credentials available for this backend?" check. +/// +/// The answer is informational — the settings UI uses it to gate the +/// Use button and avoid kicking off an interactive OAuth flow from a +/// settings GET. It does not replace the resolver's own validation +/// when the chain actually rebuilds. +fn backend_has_credentials( + def: &ironclaw_llm::registry::ProviderDefinition, + has_api_key: bool, + read_env: &dyn Fn(&str) -> Option<String>, +) -> bool { + use ironclaw_llm::registry::SetupHint; + match def.setup.as_ref() { + // No setup hint at all = nothing to configure (Tinfoil, Groq, + // etc. — they all carry SetupHint::ApiKey today, so this arm is + // future-proofing). Treat as configured to keep current + // behaviour where the frontend already shows them as usable. + None => true, + // Ollama is local; no credentials are needed at all. + Some(SetupHint::Ollama { .. }) => true, + // API-key flows: the existing `has_api_key` covers both env + // vars and the per-host session token NEAR AI accepts. + Some(SetupHint::ApiKey { .. }) + | Some(SetupHint::OpenAiCompatible { .. }) + | Some(SetupHint::SessionToken { .. }) => has_api_key, + // Bedrock takes either an AWS profile or env-style credentials. + // Settings-stored `extras.profile` is read by the resolver, but + // this status path is keyed off env + ambient AWS config only + // (matching the old behaviour where `has_api_key: false` + // permanently advertised the backend as configured). + // `AWS_ACCESS_KEY_ID` alone is insufficient — the AWS SDK needs + // both the access key and secret to sign requests, and + // `AWS_SESSION_TOKEN` is supplemental to that pair (temporary + // credentials), never a substitute on its own. + Some(SetupHint::AwsCredentials { .. }) => { + read_env("AWS_PROFILE").is_some() + || (read_env("AWS_ACCESS_KEY_ID").is_some() + && read_env("AWS_SECRET_ACCESS_KEY").is_some()) + } + // OpenAI Codex device-code login persists a session file. The + // resolver reads `OpenAiCodexConfig::session_path`, which can + // be overridden via `OPENAI_CODEX_SESSION_PATH`; honour the env + // override here so the UI doesn't falsely report "not + // configured" when the session lives at a custom path. + Some(SetupHint::OAuthDeviceCode { .. }) => read_env("OPENAI_CODEX_SESSION_PATH") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| ironclaw_llm::OpenAiCodexConfig::default().session_path) + .exists(), + // Gemini OAuth + similar file-based flows. Expand `~` in the + // hint, then test for file existence. + Some(SetupHint::FileBasedCredentials { + default_path_hint, .. + }) => default_path_hint + .as_deref() + .and_then(expand_tilde) + .is_some_and(|p| p.exists()), } +} - serde_json::Value::Array(providers) +/// Expand a leading `~` to the user's home directory. Returns `None` +/// for empty paths; returns the original path verbatim if it doesn't +/// start with `~/`. +fn expand_tilde(path: &str) -> Option<std::path::PathBuf> { + if path.is_empty() { + return None; + } + if let Some(rest) = path.strip_prefix("~/") { + dirs::home_dir().map(|home| home.join(rest)) + } else { + Some(std::path::PathBuf::from(path)) + } } // --------------------------------------------------------------------------- @@ -586,7 +650,7 @@ fn builtin_api_key_env_var(provider_id: &str) -> Option<String> { if provider_id == "nearai" { return Some("NEARAI_API_KEY".to_string()); } - crate::llm::registry::ProviderRegistry::load() + ironclaw_llm::registry::ProviderRegistry::load() .find(provider_id) .and_then(|def| def.api_key_env.clone()) } @@ -775,6 +839,256 @@ mod tests { } } + /// Regression: the three "dedicated auth" backends ship with + /// `api_key_required: false` because they don't authenticate via a + /// bearer-token API key. The frontend previously read that as + /// "needs no credentials" and rendered the Use button on a fresh + /// install, where clicking it could trigger an interactive + /// device-code OAuth from inside a settings request. The backend + /// now ships `credential_kind` + `has_credentials` for every + /// provider so the frontend can gate non-api-key kinds on actual + /// credential availability. + #[tokio::test] + async fn test_llm_providers_expose_credential_kind_and_has_credentials() { + // Hold an env-mutex guard for the full test so no other test + // can race the env reads in `backend_has_credentials`. + let _env_lock = crate::config::helpers::lock_env(); + // SAFETY: scoped to this test via lock_env(); restored below. + unsafe { + std::env::remove_var("AWS_PROFILE"); + std::env::remove_var("AWS_ACCESS_KEY_ID"); + std::env::remove_var("AWS_SECRET_ACCESS_KEY"); + std::env::remove_var("AWS_SESSION_TOKEN"); + std::env::remove_var("OPENAI_CODEX_SESSION_PATH"); + std::env::remove_var("NEARAI_API_KEY"); + } + + let result = build_llm_providers(false); + let arr = result.as_array().expect("should be an array"); + + // nearai (SessionToken kind): no session token loaded and no + // API key set => has_credentials must be false. + let nearai = find_provider(arr, "nearai").expect("nearai"); + assert_eq!( + nearai.get("credential_kind").and_then(|v| v.as_str()), + Some("session_token") + ); + assert_eq!( + nearai.get("has_credentials").and_then(|v| v.as_bool()), + Some(false), + "nearai with no session/key must report has_credentials=false" + ); + + // gemini_oauth (FileBasedCredentials kind): default path + // probably doesn't exist on the test machine; has_credentials + // reflects that. We don't assert on the value because the path + // may legitimately exist for a local dev — just on the kind. + let gemini = find_provider(arr, "gemini_oauth").expect("gemini_oauth"); + assert_eq!( + gemini.get("credential_kind").and_then(|v| v.as_str()), + Some("file_based_credentials") + ); + assert!(gemini.get("has_credentials").is_some()); + + // openai_codex (OAuthDeviceCode kind): same — kind must be + // surfaced; has_credentials presence is asserted. + let codex = find_provider(arr, "openai_codex").expect("openai_codex"); + assert_eq!( + codex.get("credential_kind").and_then(|v| v.as_str()), + Some("o_auth_device_code") + ); + assert!(codex.get("has_credentials").is_some()); + + // bedrock (AwsCredentials kind): no AWS env vars set => + // has_credentials=false. + let bedrock = find_provider(arr, "bedrock").expect("bedrock"); + assert_eq!( + bedrock.get("credential_kind").and_then(|v| v.as_str()), + Some("aws_credentials") + ); + assert_eq!( + bedrock.get("has_credentials").and_then(|v| v.as_bool()), + Some(false), + "bedrock with no AWS env must report has_credentials=false" + ); + + // Ollama (no credentials needed) must report true. + let ollama = find_provider(arr, "ollama").expect("ollama"); + assert_eq!( + ollama.get("credential_kind").and_then(|v| v.as_str()), + Some("ollama") + ); + assert_eq!( + ollama.get("has_credentials").and_then(|v| v.as_bool()), + Some(true) + ); + + // Every entry must carry both fields so the frontend gate is + // never undefined. + for p in arr { + let id = p.get("id").and_then(|v| v.as_str()).unwrap_or("<missing>"); + assert!( + p.get("credential_kind").is_some(), + "{id} missing credential_kind" + ); + assert!( + p.get("has_credentials").is_some(), + "{id} missing has_credentials" + ); + } + } + + /// Regression for nearai/ironclaw#3734: NEAR AI is dual-auth + /// (session token + API key). The configure UI must show the + /// API Key input and the "Fetch available models" button even + /// though `api_key_required: false` (because session_token alone + /// is a valid configuration). PR #3416 collapsed NEAR AI into the + /// generic registry path with the SessionToken setup hint, which + /// silently flipped both flags off on the wire. Lock in: + /// + /// - `accepts_api_key: true` (provider has an api_key_env, so + /// the API key field must be visible) + /// - `can_list_models: true` (SessionToken setup hint propagates + /// the per-provider can_list_models from providers.json) + /// + /// Also covers the corollary that providers without an api_key_env + /// (Bedrock, Codex device-code, Gemini OAuth) emit + /// `accepts_api_key: false`. + #[tokio::test] + async fn test_nearai_configure_ui_flags_3734() { + let _env_lock = crate::config::helpers::lock_env(); + let arr = build_llm_providers(false); + let arr = arr.as_array().expect("array"); + + let nearai = find_provider(arr, "nearai").expect("nearai"); + assert_eq!( + nearai.get("accepts_api_key").and_then(|v| v.as_bool()), + Some(true), + "nearai must expose accepts_api_key=true so the configure \ + UI shows the API Key input (#3734)" + ); + assert_eq!( + nearai.get("can_list_models").and_then(|v| v.as_bool()), + Some(true), + "nearai must expose can_list_models=true so the configure \ + UI shows the Fetch models button (#3734)" + ); + + // Sanity-check the corollary: backends with no api_key_env + // (Bedrock, OpenAI Codex device-code, Gemini OAuth) must NOT + // surface the API Key field. + for id in ["bedrock", "openai_codex", "gemini_oauth"] { + let p = find_provider(arr, id).unwrap_or_else(|| panic!("{id} should exist")); + assert_eq!( + p.get("accepts_api_key").and_then(|v| v.as_bool()), + Some(false), + "{id}: no api_key_env means accepts_api_key must be false" + ); + } + + // Every entry must carry the field so the frontend gate is + // never undefined for any provider. + for p in arr { + let id = p.get("id").and_then(|v| v.as_str()).unwrap_or("<missing>"); + assert!( + p.get("accepts_api_key").is_some(), + "{id} missing accepts_api_key" + ); + } + } + + /// nearai with a loaded session token must report + /// `has_credentials: true` so the frontend lets the user activate it. + #[tokio::test] + async fn test_nearai_has_credentials_true_when_session_token_loaded() { + let _env_lock = crate::config::helpers::lock_env(); + // SAFETY: scoped to this test via lock_env(); restored below. + unsafe { + std::env::remove_var("NEARAI_API_KEY"); + } + let result = build_llm_providers(true); + let nearai = find_provider(result.as_array().unwrap(), "nearai").expect("nearai"); + assert_eq!( + nearai.get("has_credentials").and_then(|v| v.as_bool()), + Some(true) + ); + } + + /// Regression: a host with only `AWS_ACCESS_KEY_ID` set (no + /// `AWS_SECRET_ACCESS_KEY`) cannot actually sign requests — the + /// AWS SDK needs the pair. Previously `has_credentials` returned + /// true on the access key alone (and also on a bare + /// `AWS_SESSION_TOKEN`), so the frontend would show Bedrock as + /// configured and a click would fail at first call. + #[tokio::test] + async fn test_bedrock_partial_aws_env_reports_not_configured() { + let _env_lock = crate::config::helpers::lock_env(); + // SAFETY: scoped to this test via lock_env(); restored below. + unsafe { + std::env::remove_var("AWS_PROFILE"); + std::env::remove_var("AWS_SESSION_TOKEN"); + std::env::remove_var("AWS_SECRET_ACCESS_KEY"); + std::env::set_var("AWS_ACCESS_KEY_ID", "AKIA-test-only"); + } + let arr_partial = build_llm_providers(false); + let bedrock = find_provider(arr_partial.as_array().unwrap(), "bedrock").expect("bedrock"); + assert_eq!( + bedrock.get("has_credentials").and_then(|v| v.as_bool()), + Some(false), + "AWS_ACCESS_KEY_ID alone must not flip has_credentials true" + ); + + // SAFETY: scoped via lock_env(). + unsafe { + std::env::set_var("AWS_SECRET_ACCESS_KEY", "secret-test-only"); + } + let arr_full = build_llm_providers(false); + let bedrock_full = + find_provider(arr_full.as_array().unwrap(), "bedrock").expect("bedrock full"); + assert_eq!( + bedrock_full + .get("has_credentials") + .and_then(|v| v.as_bool()), + Some(true), + "AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY must flip has_credentials true" + ); + + // SAFETY: clean up so neighbouring tests see a pristine env. + unsafe { + std::env::remove_var("AWS_ACCESS_KEY_ID"); + std::env::remove_var("AWS_SECRET_ACCESS_KEY"); + } + } + + /// Regression: Codex stores its session at a custom path when + /// `OPENAI_CODEX_SESSION_PATH` is set. The status helper used to + /// only probe the built-in default, so a user with the env + /// override saw "not configured" even when logged in. + #[tokio::test] + async fn test_openai_codex_honours_session_path_env() { + let _env_lock = crate::config::helpers::lock_env(); + let tmp = tempfile::tempdir().expect("tmp"); + let session_path = tmp.path().join("openai_codex_session.json"); + std::fs::write(&session_path, "{}").expect("write session stub"); + + // SAFETY: scoped via lock_env(); restored below. + unsafe { + std::env::set_var("OPENAI_CODEX_SESSION_PATH", &session_path); + } + let arr = build_llm_providers(false); + let codex = find_provider(arr.as_array().unwrap(), "openai_codex").expect("codex"); + assert_eq!( + codex.get("has_credentials").and_then(|v| v.as_bool()), + Some(true), + "Codex must honour OPENAI_CODEX_SESSION_PATH when probing has_credentials" + ); + + // SAFETY: scoped via lock_env(). + unsafe { + std::env::remove_var("OPENAI_CODEX_SESSION_PATH"); + } + } + #[tokio::test] async fn test_openai_compatible_exposes_base_url_required_true() { // Regression: openai_compatible has base_url_required=true (no default). @@ -1236,7 +1550,7 @@ mod tests { // Build a `SessionManager` with a token seeded directly — same shape // as `~/.ironclaw/session.json` having been loaded at startup. - let session = crate::llm::SessionManager::new_async(crate::llm::SessionConfig { + let session = ironclaw_llm::SessionManager::new_async(ironclaw_llm::SessionConfig { auth_base_url: "https://private.near.ai".to_string(), session_path: std::env::temp_dir().join("ironclaw-test-no-such-file.json"), }) diff --git a/src/channels/web/handlers/memory.rs b/src/channels/web/handlers/memory.rs index b8cc71c3cd5..2a220c9c210 100644 --- a/src/channels/web/handlers/memory.rs +++ b/src/channels/web/handlers/memory.rs @@ -244,7 +244,8 @@ mod tests { use crate::channels::web::sse::SseManager; use crate::config::{WorkspaceConfig, WorkspaceSearchConfig}; use crate::db::Database; - use crate::workspace::{EmbeddingCacheConfig, Workspace}; + use crate::workspace::Workspace; + use ironclaw_embeddings::EmbeddingCacheConfig; use super::memory_read_handler; diff --git a/src/channels/web/mod.rs b/src/channels/web/mod.rs index e875b3996f2..0247c54ef0f 100644 --- a/src/channels/web/mod.rs +++ b/src/channels/web/mod.rs @@ -63,7 +63,8 @@ use crate::error::ChannelError; use crate::extensions::ExtensionManager; use crate::orchestrator::job_manager::ContainerJobManager; use crate::tools::ToolRegistry; -use crate::workspace::{EmbeddingCacheConfig, EmbeddingProvider, Workspace}; +use crate::workspace::Workspace; +use ironclaw_embeddings::{EmbeddingCacheConfig, EmbeddingProvider}; use ironclaw_skills::catalog::SkillCatalog; use ironclaw_skills::registry::SkillRegistry; @@ -380,20 +381,20 @@ impl GatewayChannel { } /// Inject the LLM provider for OpenAI-compatible API proxy. - pub fn with_llm_provider(mut self, llm: Arc<dyn crate::llm::LlmProvider>) -> Self { + pub fn with_llm_provider(mut self, llm: Arc<dyn ironclaw_llm::LlmProvider>) -> Self { self.rebuild_state(|s| s.llm_provider = Some(llm)); self } /// Inject the LLM hot-reload controller for the settings handlers. - pub fn with_llm_reload(mut self, reload: Arc<crate::llm::LlmReloadHandle>) -> Self { + pub fn with_llm_reload(mut self, reload: Arc<ironclaw_llm::LlmReloadHandle>) -> Self { self.rebuild_state(|s| s.llm_reload = Some(reload)); self } /// Inject the LLM session manager so a hot-reload can rebuild the /// provider chain without dropping the current auth session. - pub fn with_llm_session_manager(mut self, sm: Arc<crate::llm::SessionManager>) -> Self { + pub fn with_llm_session_manager(mut self, sm: Arc<ironclaw_llm::SessionManager>) -> Self { self.rebuild_state(|s| s.llm_session_manager = Some(sm)); self } @@ -635,6 +636,58 @@ impl GatewayChannel { /// constant, so the value is compile-time-pinned in both places. pub const GATEWAY_CHANNEL_NAME: &str = "gateway"; +/// Route a status `AppEvent` to the SSE manager based on owner identity. +/// +/// In multi-tenant deployments an unscoped global broadcast would deliver +/// the status (Thinking / ToolStarted / ToolResult / ...) to every +/// connected subscriber, leaking another tenant's tool calls and outputs. +/// Drop the event in that mode and surface a WARN so the upstream +/// producer gets fixed. Single-tenant deployments keep the unscoped +/// fan-out because there is only one subscriber population. +/// +/// An empty-string `user_id` is treated the same as `None` — empty +/// values typically come from a producer that lost the field along the +/// way (default-initialised structs, missing JSON keys converted to +/// empty strings) and must not collapse into a global broadcast in +/// multi-tenant mode. +/// +/// Extracted from `Channel::send_status` so the routing rule can be +/// asserted by unit tests without standing up a `GatewayChannel`. See +/// `tests::status_event_isolation`. +/// +/// **Internal bridge contract.** This is `pub` only so the sandbox +/// `JobEvent` rx loop in `src/main.rs` can route through the same +/// drop/WARN/broadcast policy as `Channel::send_status`. It is not part +/// of a stable public API; downstream consumers must not depend on it. +/// Callers from inside the `web` slice should use `send_status` rather +/// than reaching for this directly. +pub fn dispatch_status_event( + sse: &platform::sse::SseManager, + multi_tenant_mode: bool, + user_id: Option<&str>, + event: AppEvent, +) { + match user_id.filter(|uid| !uid.is_empty()) { + Some(uid) => sse.broadcast_for_user(uid, event), // projection-exempt: bridge dispatcher, scoped status update + None if multi_tenant_mode => { + // Log only the wire-stable variant name. `?event` would emit + // the full Debug payload, which on variants like + // `AppEvent::Response` / `Thinking` / `ToolResult` carries + // user-authored content into operator logs in a multi-tenant + // deployment. The variant name is enough to chase the + // misbehaving producer. + tracing::warn!( + event_kind = event.event_type(), + "dropped unscoped status event in multi-tenant mode — \ + producer must include a non-empty user_id in metadata" + ); + } + None => { + sse.broadcast(event); // projection-exempt: bridge dispatcher, single-tenant unscoped status; multi-tenant-safe: only reached when multi_tenant_mode=false + } + } +} + #[async_trait] impl Channel for GatewayChannel { fn name(&self) -> &str { @@ -940,15 +993,12 @@ impl Channel for GatewayChannel { } }; - // Scope events to the user when user_id is available in metadata. - // When user_id is missing (heartbeat, routines), events go to all - // subscribers. In multi-tenant mode this leaks status across users. - if let Some(uid) = metadata.get("user_id").and_then(|v| v.as_str()) { - self.state.sse.broadcast_for_user(uid, event); - } else { - tracing::debug!("Status event missing user_id in metadata; broadcasting globally"); - self.state.sse.broadcast(event); - } + dispatch_status_event( + &self.state.sse, + self.state.multi_tenant_mode, + metadata.get("user_id").and_then(|v| v.as_str()), + event, + ); Ok(()) } diff --git a/src/channels/web/openai_compat.rs b/src/channels/web/openai_compat.rs index 3121c62562e..a724ab7599c 100644 --- a/src/channels/web/openai_compat.rs +++ b/src/channels/web/openai_compat.rs @@ -17,7 +17,7 @@ use axum::{ }; use serde::{Deserialize, Serialize}; -use crate::llm::{ +use ironclaw_llm::{ ChatMessage, CompletionRequest, FinishReason, Role, ToolCall, ToolCompletionRequest, ToolDefinition, }; @@ -232,6 +232,7 @@ pub fn convert_messages(messages: &[OpenAiMessage]) -> Result<Vec<ChatMessage>, arguments: serde_json::from_str(&tc.function.arguments) .unwrap_or(serde_json::Value::Object(Default::default())), reasoning: None, + signature: None, }) .collect(); Ok(ChatMessage::assistant_with_tool_calls( @@ -249,6 +250,7 @@ pub fn convert_messages(messages: &[OpenAiMessage]) -> Result<Vec<ChatMessage>, tool_call_id: None, name: m.name.clone(), tool_calls: None, + reasoning: None, }), } }) @@ -595,7 +597,7 @@ pub async fn chat_completions_handler( /// proper HTTP errors instead of SSE error events. True token streaming can be /// added later by extending `LlmProvider` with a `complete_stream()` method. async fn handle_streaming( - llm: Arc<dyn crate::llm::LlmProvider>, + llm: Arc<dyn ironclaw_llm::LlmProvider>, req: OpenAiChatRequest, has_tools: bool, ) -> Result<Response, (StatusCode, Json<OpenAiErrorResponse>)> { @@ -610,8 +612,8 @@ async fn handle_streaming( // Since streaming is simulated (LlmProvider returns complete responses), // this lets us return proper HTTP errors on failure. enum LlmResult { - Simple(crate::llm::CompletionResponse), - WithTools(crate::llm::ToolCompletionResponse), + Simple(ironclaw_llm::CompletionResponse), + WithTools(ironclaw_llm::ToolCompletionResponse), } let llm_result = if has_tools { @@ -956,6 +958,7 @@ mod tests { name: "search".to_string(), arguments: serde_json::json!({"query": "rust"}), reasoning: None, + signature: None, }]; let converted = convert_tool_calls_to_openai(&calls); diff --git a/src/channels/web/platform/engine_dispatch.rs b/src/channels/web/platform/engine_dispatch.rs index a1519117e30..f07b1bacf08 100644 --- a/src/channels/web/platform/engine_dispatch.rs +++ b/src/channels/web/platform/engine_dispatch.rs @@ -81,7 +81,10 @@ pub(crate) async fn dispatch_engine_external_callback( "Invalid request_id (expected UUID)".to_string(), ) })?; - let callback = crate::agent::submission::Submission::ExternalCallback { request_id }; + let callback = crate::agent::submission::Submission::ExternalCallback { + request_id, + payload: None, + }; dispatch_engine_submission(state, user_id, thread_id, callback).await } diff --git a/src/channels/web/platform/router.rs b/src/channels/web/platform/router.rs index d4023cbaf9f..684f9a84076 100644 --- a/src/channels/web/platform/router.rs +++ b/src/channels/web/platform/router.rs @@ -78,8 +78,9 @@ use crate::channels::web::features::chat::{ }; use crate::channels::web::features::extensions::{ extensions_activate_handler, extensions_install_handler, extensions_list_handler, - extensions_readiness_handler, extensions_registry_handler, extensions_remove_handler, - extensions_setup_handler, extensions_setup_submit_handler, extensions_tools_handler, + extensions_login_poll_handler, extensions_login_start_handler, extensions_readiness_handler, + extensions_registry_handler, extensions_remove_handler, extensions_setup_handler, + extensions_setup_submit_handler, extensions_tools_handler, }; use crate::channels::web::features::logs::{ logs_events_handler, logs_level_get_handler, logs_level_set_handler, @@ -225,6 +226,14 @@ pub async fn start_server( "/api/extensions/{name}/setup", get(extensions_setup_handler).post(extensions_setup_submit_handler), ) + .route( + "/api/extensions/{name}/login/start", + post(extensions_login_start_handler), + ) + .route( + "/api/extensions/{name}/login/poll", + post(extensions_login_poll_handler), + ) // Pairing .route("/api/pairing/{channel}", get(pairing_list_handler)) .route( diff --git a/src/channels/web/platform/sse.rs b/src/channels/web/platform/sse.rs index 0082f80a053..73b0ec5a6c0 100644 --- a/src/channels/web/platform/sse.rs +++ b/src/channels/web/platform/sse.rs @@ -516,6 +516,105 @@ mod tests { assert!(matches!(e, AppEvent::Heartbeat)); // safety: test assertion } + /// Lock down every quadrant of the (subscriber-scope, event-scope) cross + /// product so a regression in `subscribe_raw` filtering shows up here + /// before it ships. This is the cross-tenant invariant for SSE/WS: + /// once a subscriber declares a `user_id`, only that user's scoped + /// events plus globally-broadcast (unscoped) events reach them. + #[tokio::test] + async fn test_filter_quadrants_scoped_subscriber() { + let manager = SseManager::new(); + let mut alice = Box::pin( + manager + .subscribe_raw(Some("alice".to_string()), false) + .expect("alice subscribe"), + ); + + // (scoped event, matching user) -> deliver + manager.broadcast_for_user( + "alice", + AppEvent::Status { + message: "to_alice".to_string(), + thread_id: None, + }, + ); + // (scoped event, mismatched user) -> drop + manager.broadcast_for_user( + "bob", + AppEvent::Status { + message: "to_bob".to_string(), + thread_id: None, + }, + ); + // (unscoped event, any subscriber) -> deliver (global) + manager.broadcast(AppEvent::Heartbeat); + + // Alice receives her own scoped event and the global heartbeat, + // in broadcast order. The intermediate bob-scoped event is filtered + // out before alice's stream sees it. + let first = alice.next().await.expect("alice receives one event"); + assert!( + matches!(&first, AppEvent::Status { message, .. } if message == "to_alice"), + "alice first event should be her scoped Status, got {first:?}" + ); + let second = alice.next().await.expect("alice receives heartbeat"); + assert!( + matches!(second, AppEvent::Heartbeat), + "alice second event should be heartbeat after the bob-scoped event was filtered" + ); + } + + /// The unscoped-subscriber branch is the single-tenant compatibility + /// path. It delivers every event regardless of scope. Multi-tenant + /// callers must NOT pass `user_id = None` to `subscribe_raw` — this + /// test exists so the day someone removes that branch is an explicit + /// decision, not a stealth refactor. + #[tokio::test] + async fn test_filter_quadrants_unscoped_subscriber_sees_all() { + let manager = SseManager::new(); + let mut everyone = Box::pin( + manager + .subscribe_raw(None, false) + .expect("unscoped subscribe"), + ); + + manager.broadcast_for_user( + "alice", + AppEvent::Status { + message: "alice".to_string(), + thread_id: None, + }, + ); + manager.broadcast_for_user( + "bob", + AppEvent::Status { + message: "bob".to_string(), + thread_id: None, + }, + ); + manager.broadcast(AppEvent::Heartbeat); + + let mut seen: Vec<String> = Vec::new(); + for _ in 0..3 { + match everyone.next().await.expect("event") { + AppEvent::Status { message, .. } => seen.push(message), + AppEvent::Heartbeat => seen.push("heartbeat".to_string()), + other => panic!("unexpected variant: {other:?}"), + } + } + assert_eq!( + seen, + vec![ + "alice".to_string(), + "bob".to_string(), + "heartbeat".to_string() + ], + "unscoped subscriber must observe both scoped events plus the global heartbeat \ + (single-tenant compat). If this fails, the global-broadcast hole is closed and \ + the multi-tenant fix in `mod.rs::send_status` should be re-evaluated." + ); + } + #[tokio::test] async fn test_verbose_filtering() { let manager = SseManager::new(); diff --git a/src/channels/web/platform/state.rs b/src/channels/web/platform/state.rs index a640645cbf7..a5581a9f2de 100644 --- a/src/channels/web/platform/state.rs +++ b/src/channels/web/platform/state.rs @@ -198,8 +198,8 @@ impl PerUserRateLimiter { /// avoiding a separate `PerUserWorkspaceResolver` with duplicated logic. pub struct WorkspacePool { db: Arc<dyn Database>, - embeddings: Option<Arc<dyn crate::workspace::EmbeddingProvider>>, - embedding_cache_config: crate::workspace::EmbeddingCacheConfig, + embeddings: Option<Arc<dyn ironclaw_embeddings::EmbeddingProvider>>, + embedding_cache_config: ironclaw_embeddings::EmbeddingCacheConfig, search_config: crate::config::WorkspaceSearchConfig, workspace_config: crate::config::WorkspaceConfig, cache: tokio::sync::RwLock<std::collections::HashMap<String, Arc<Workspace>>>, @@ -211,8 +211,8 @@ pub struct WorkspacePool { impl WorkspacePool { pub fn new( db: Arc<dyn Database>, - embeddings: Option<Arc<dyn crate::workspace::EmbeddingProvider>>, - embedding_cache_config: crate::workspace::EmbeddingCacheConfig, + embeddings: Option<Arc<dyn ironclaw_embeddings::EmbeddingProvider>>, + embedding_cache_config: ironclaw_embeddings::EmbeddingCacheConfig, search_config: crate::config::WorkspaceSearchConfig, workspace_config: crate::config::WorkspaceConfig, ) -> Self { @@ -250,7 +250,13 @@ impl WorkspacePool { ws = ws.with_additional_read_scopes(self.workspace_config.read_scopes.clone()); } - ws = ws.with_memory_layers(self.workspace_config.memory_layers.clone()); + let mut memory_layers = self.workspace_config.memory_layers.clone(); + for layer in &mut memory_layers { + if layer.sensitivity == crate::workspace::layer::LayerSensitivity::Private { + layer.scope = user_id.to_string(); + } + } + ws = ws.with_memory_layers(memory_layers); ws } @@ -383,15 +389,15 @@ pub struct GatewayState { /// WebSocket connection tracker. pub ws_tracker: Option<Arc<crate::channels::web::ws::WsConnectionTracker>>, /// LLM provider for OpenAI-compatible API proxy. - pub llm_provider: Option<Arc<dyn crate::llm::LlmProvider>>, + pub llm_provider: Option<Arc<dyn ironclaw_llm::LlmProvider>>, /// Hot-reload controller for the LLM provider chain. Populated at /// startup when the chain is built from config (not in test harnesses /// that inject a provider directly). - pub llm_reload: Option<Arc<crate::llm::LlmReloadHandle>>, + pub llm_reload: Option<Arc<ironclaw_llm::LlmReloadHandle>>, /// LLM session manager handed through to `LlmReloadHandle::reload` so /// the rebuilt chain keeps using the same (potentially authenticated) /// NEAR AI / OAuth session without forcing a re-login. - pub llm_session_manager: Option<Arc<crate::llm::SessionManager>>, + pub llm_session_manager: Option<Arc<ironclaw_llm::SessionManager>>, /// Optional TOML config path that produced the current `LlmConfig`. /// Needed so a hot-reload reads the same precedence layers /// (TOML → DB overlay) as startup. @@ -505,7 +511,7 @@ mod tests { let pool = WorkspacePool::new( db, None, - crate::workspace::EmbeddingCacheConfig::default(), + ironclaw_embeddings::EmbeddingCacheConfig::default(), crate::config::WorkspaceSearchConfig::default(), crate::config::WorkspaceConfig::default(), ); diff --git a/src/channels/web/platform/static_files.rs b/src/channels/web/platform/static_files.rs index f695d27dee5..aec04f64982 100644 --- a/src/channels/web/platform/static_files.rs +++ b/src/channels/web/platform/static_files.rs @@ -1214,7 +1214,7 @@ mod tests { use crate::config::{WorkspaceConfig, WorkspaceSearchConfig}; use crate::db::Database as _; use crate::db::libsql::LibSqlBackend; - use crate::workspace::EmbeddingCacheConfig; + use ironclaw_embeddings::EmbeddingCacheConfig; let dir = tempfile::tempdir().expect("tempdir"); let backend = LibSqlBackend::new_local(&dir.path().join("multi_tenant_css.db")) @@ -1363,7 +1363,7 @@ mod tests { use crate::config::{WorkspaceConfig, WorkspaceSearchConfig}; use crate::db::Database as _; use crate::db::libsql::LibSqlBackend; - use crate::workspace::EmbeddingCacheConfig; + use ironclaw_embeddings::EmbeddingCacheConfig; let dir = tempfile::tempdir().expect("tempdir"); let backend = LibSqlBackend::new_local(&dir.path().join("multi_tenant_index.db")) diff --git a/src/channels/web/responses_api.rs b/src/channels/web/responses_api.rs index 238f2a6c66e..3d7dd60c44e 100644 --- a/src/channels/web/responses_api.rs +++ b/src/channels/web/responses_api.rs @@ -9,6 +9,41 @@ //! `/api/...` prefix used by the rest of IronClaw's HTTP surface. The legacy //! `/v1/responses` path is still accepted as an alias for backward //! compatibility with clients configured against it (see ironclaw#2201). +//! +//! ## Externally-provided tools +//! +//! Callers can declare their own tools alongside IronClaw's built-in +//! registry by passing `tools: [{type: "function", name, description, +//! parameters}]` and feeding back results via `function_call_output` items +//! in the next request's `input`. The integration is engine-native, not +//! prompt-level: +//! +//! 1. The handler validates the request, registers the caller's tools in +//! the per-thread [`ExternalToolCatalog`] keyed by the engine +//! `ThreadId`, and routes the user message through the agent loop. The +//! catalog merges into the LLM-visible action surface via +//! `EffectBridgeAdapter::available_actions`, so the model sees caller +//! tools alongside internal ones. +//! 2. When the LLM invokes a caller tool, `EffectBridgeAdapter::execute_action` +//! short-circuits to `EngineError::GatePaused { resume_kind: +//! External { callback_id: ext_tool:<call_id> } }`. The bridge router +//! projects that pause to `AppEvent::ExternalToolCall` carrying the +//! OpenAI-shaped `function_call` fields. This handler emits it as a +//! `function_call` `ResponseOutputItem` (both streaming +//! `output_item.added`+`done` and non-streaming) and returns +//! `status: "completed"`. The thread sits in `Waiting`. +//! 3. The caller resumes by POSTing a follow-up request whose `input` +//! array contains `function_call_output` items. The handler converts +//! them to `Submission::ExternalCallback { request_id, payload }`, +//! routed through `bridge::handle_external_callback`, which +//! materialises an `ActionResult` ThreadMessage from the payload and +//! resumes the thread. The LLM sees the result on its next call. +//! +//! Caller-supplied tool names that shadow registered (built-in or +//! extension) actions are rejected at request validation with 400 — see +//! the confused-deputy note in `create_response_handler`. +//! +//! [`ExternalToolCatalog`]: crate::bridge::ExternalToolCatalog use std::convert::Infallible; use std::sync::Arc; @@ -90,16 +125,47 @@ fn default_model() -> String { #[serde(untagged)] pub enum ResponsesInput { Text(String), - Messages(Vec<ResponsesInputMessage>), + Items(Vec<ResponsesInputItem>), } +/// A single item in the Responses API `input` array. +/// +/// Items without an explicit `type` default to a user message — this preserves +/// backward compatibility with the simpler `[{"role":"user","content":"..."}]` +/// shape that pre-dates external tool support. #[derive(Debug, Clone, Deserialize)] -pub struct ResponsesInputMessage { - pub role: String, - pub content: String, +pub struct ResponsesInputItem { + /// Item type tag: `message`, `function_call`, or `function_call_output`. + /// Absent or empty means `message` (legacy shape). + #[serde(rename = "type", default)] + pub item_type: Option<String>, + /// For `message` items: role (`user`, `assistant`, `system`). + #[serde(default)] + pub role: Option<String>, + /// For `message` items: text content. + #[serde(default)] + pub content: Option<String>, + /// For `function_call` and `function_call_output` items: links a call to its result. + #[serde(default)] + pub call_id: Option<String>, + /// For `function_call` items: the tool name the agent (previously) chose. + #[serde(default)] + pub name: Option<String>, + /// For `function_call` items: stringified JSON arguments. + #[serde(default)] + pub arguments: Option<String>, + /// For `function_call_output` items: the tool result the caller executed externally. + #[serde(default)] + pub output: Option<String>, } -#[derive(Debug, Deserialize)] +/// Externally-provided tool definition. +/// +/// Per the OpenAI Responses API spec, only `type: "function"` is currently +/// honoured. Built-in tool types like `web_search`, `file_search`, or +/// `code_interpreter` are rejected with 400 — IronClaw routes those through +/// its own internal tool registry, not through caller-provided definitions. +#[derive(Debug, Clone, Deserialize)] pub struct ResponsesTool { #[serde(rename = "type")] pub tool_type: String, @@ -344,32 +410,309 @@ fn format_context(ctx: &serde_json::Value) -> String { parts.join("\n") } -/// Extract the user message text from the input. -fn extract_user_content(input: &ResponsesInput) -> Result<String, String> { +/// Outputs of previously-executed tools the caller is feeding back in. +#[derive(Debug, Default, Clone)] +struct ExtractedInput { + /// The latest user message text (the prompt for this turn). + user_text: String, + /// Previous tool outputs (`function_call_output` items) supplied by the + /// caller. Pairs of `(call_id, output)`. Surfaced to the agent as a + /// preamble so it can reason over the results of the tools it asked the + /// caller to run on the prior turn. + tool_outputs: Vec<(String, String)>, +} + +/// Extract the latest user message and any caller-supplied tool outputs from +/// the input. +/// +/// Accepts: +/// - a plain string (legacy text input) +/// - a list of `{role, content}` items (legacy message input, no `type` tag) +/// - a list of typed items: `message`, `function_call`, `function_call_output` +/// +/// `function_call` items are accepted but ignored — they are echoes of what +/// the agent emitted on a previous turn. `function_call_output` items are +/// extracted into `tool_outputs` so we can surface them to the agent. +fn extract_user_content(input: &ResponsesInput) -> Result<ExtractedInput, String> { match input { ResponsesInput::Text(s) => { if s.is_empty() { Err("input must not be empty".to_string()) } else { - Ok(s.clone()) + Ok(ExtractedInput { + user_text: s.clone(), + tool_outputs: Vec::new(), + }) } } - ResponsesInput::Messages(msgs) => { - // Find the last user message. - let last_user = msgs - .iter() - .rev() - .find(|m| m.role == "user") - .ok_or("input messages must contain at least one user message")?; - if last_user.content.is_empty() { - Err("user message content must not be empty".to_string()) - } else { - Ok(last_user.content.clone()) + ResponsesInput::Items(items) => { + let mut last_user_text: Option<String> = None; + let mut tool_outputs: Vec<(String, String)> = Vec::new(); + for item in items { + match item.item_type.as_deref().unwrap_or("message") { + "message" => { + if item.role.as_deref() == Some("user") + && let Some(text) = item.content.as_ref() + && !text.is_empty() + { + last_user_text = Some(text.clone()); + } + } + "function_call" => { + // Echo of a previous turn's output. The agent's own + // history is reconstructed from the conversation + // store, so we don't need to re-inject it. + } + "function_call_output" => { + let Some(output) = item.output.as_ref() else { + return Err( + "function_call_output items must include an `output` field" + .to_string(), + ); + }; + // Defaulting to an empty call_id silently breaks + // resume correlation (the bridge looks up the + // pending external-tool gate by call_id and + // would return Value::Null to the LLM). Reject + // explicitly instead. + let Some(call_id) = item.call_id.as_deref().map(str::trim) else { + return Err( + "function_call_output items must include a non-empty `call_id` field" + .to_string(), + ); + }; + if call_id.is_empty() { + return Err( + "function_call_output items must include a non-empty `call_id` field" + .to_string(), + ); + } + tool_outputs.push((call_id.to_string(), output.clone())); + } + other => { + return Err(format!( + "unsupported input item type: '{other}' \ + (expected 'message', 'function_call', or 'function_call_output')" + )); + } + } } + + // A turn without a user message is allowed *only* when the caller + // is exclusively delivering function_call_output items — i.e. they + // want the agent to react to tool results without saying anything + // new. In that case we synthesise a minimal user prompt so the + // engine still sees a "user turn". + let user_text = match last_user_text { + Some(t) => t, + None if !tool_outputs.is_empty() => { + "[Continue with the tool results above.]".to_string() + } + None => { + return Err( + "input must contain at least one user message or function_call_output" + .to_string(), + ); + } + }; + + Ok(ExtractedInput { + user_text, + tool_outputs, + }) } } } +/// Maximum total serialized size of caller-supplied tool definitions (16 KiB). +/// +/// Caps prompt blow-up from a misconfigured client passing hundreds of tool +/// schemas. Mirrors the existing 10 KiB cap on `x_context`. +const MAX_TOOLS_BYTES: usize = 16 * 1024; + +/// Maximum length of a single external tool name, in characters. +/// +/// Matches the OpenAI Responses API constraint +/// `^[A-Za-z0-9_-]{1,64}$`. Catches accidental over-long names +/// before they propagate into engine action surfaces and SSE +/// payloads where they would corrupt logs and break downstream +/// LLM clients that enforce the same limit. +const MAX_TOOL_NAME_LEN: usize = 64; + +/// `io::Write` sink that counts bytes without storing them. Lets +/// `serde_json::to_writer` measure the serialized size of the +/// caller-supplied tool list against `MAX_TOOLS_BYTES` without +/// allocating an intermediate `Vec<Value>` or `String`. +struct ByteCounter(usize); + +impl std::io::Write for ByteCounter { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + self.0 += buf.len(); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Validate and normalise the externally-provided tool list. +/// +/// Returns an error string on the first violation so the handler can surface a +/// 400 with a precise message. Only `type: "function"` is supported. +fn validate_external_tools(tools: &[ResponsesTool]) -> Result<(), String> { + if tools.is_empty() { + return Ok(()); + } + // Stream the payload through a counting writer instead of + // building a `Vec<Value>` + `String` just to call `.len()`. + // + // What this measures: the canonicalised JSON length the tool + // array would serialise to (single field ordering, no + // pretty-printing, optional fields skipped when absent). It is + // NOT the byte-for-byte length of what the caller put on the + // wire — whitespace and key ordering in the request body can + // make the wire size diverge from this count by a constant + // factor. The 16 KiB cap is therefore on the canonical size, + // which is the meaningful "how much do we have to handle" + // number; the actual request body is already bounded by the + // gateway's 14 MiB body limit. + // + // The `Serializer` import brings the trait into scope so the + // method calls on the concrete `serde_json::Serializer` below + // resolve. `SerializeMap` / `SerializeSeq` do the same for the + // associated types returned by `serialize_map` / `serialize_seq`. + use serde::ser::{SerializeMap, SerializeSeq, Serializer}; + struct ToolEntry<'a>(&'a ResponsesTool, usize); + impl serde::Serialize for ToolEntry<'_> { + fn serialize<S: serde::Serializer>(&self, ser: S) -> Result<S::Ok, S::Error> { + let mut map = ser.serialize_map(Some(self.1))?; + map.serialize_entry("type", &self.0.tool_type)?; + if let Some(ref n) = self.0.name { + map.serialize_entry("name", n)?; + } + if let Some(ref d) = self.0.description { + map.serialize_entry("description", d)?; + } + if let Some(ref p) = self.0.parameters { + map.serialize_entry("parameters", p)?; + } + map.end() + } + } + + let mut counter = ByteCounter(0); + let serialize_result = (|| -> Result<(), serde_json::Error> { + let mut ser = serde_json::Serializer::new(&mut counter); + let mut seq = ser.serialize_seq(Some(tools.len()))?; + for t in tools { + // 4 fields max: type, name, description, parameters. We + // skip absent optional fields so the count matches what + // `serde_json::to_string` would have produced. + let field_count = 1 + + usize::from(t.name.is_some()) + + usize::from(t.description.is_some()) + + usize::from(t.parameters.is_some()); + seq.serialize_element(&ToolEntry(t, field_count))?; + } + SerializeSeq::end(seq)?; + Ok(()) + })(); + // Fail closed: if the serialization stream errored we can't + // trust the byte count, so reject the request as if it had + // exceeded the cap. This is paranoia — `Value::Object` round-trips + // never error in practice — but keeps the size gate from + // silently waving through unmeasurable payloads. + let serialized_size = if serialize_result.is_ok() { + counter.0 + } else { + MAX_TOOLS_BYTES + 1 + }; + if serialized_size > MAX_TOOLS_BYTES { + return Err(format!( + "tools exceed {MAX_TOOLS_BYTES}-byte limit ({serialized_size} bytes)" + )); + } + let mut seen = std::collections::HashSet::new(); + for tool in tools { + if tool.tool_type != "function" { + return Err(format!( + "unsupported tool type '{}'; only 'function' is accepted", + tool.tool_type + )); + } + let name = tool + .name + .as_deref() + .filter(|s| !s.is_empty()) + .ok_or_else(|| "function tool missing 'name'".to_string())?; + // OpenAI Responses API spec for function names: max 64 chars, + // `[A-Za-z0-9_-]` only. Enforce here so non-conformant names + // can't propagate into engine action surfaces, SSE payloads, + // logs, or downstream LLM clients (which would reject them + // anyway). Whitespace and control characters in particular + // would corrupt log lines and confuse pattern-matching + // downstream. + if name.len() > MAX_TOOL_NAME_LEN { + return Err(format!( + "tool name '{name}' exceeds {MAX_TOOL_NAME_LEN}-character limit" + )); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(format!( + "tool name '{name}' contains invalid characters; \ + only ASCII letters, digits, '_', and '-' are allowed" + )); + } + if !seen.insert(name.to_string()) { + return Err(format!("duplicate tool name '{name}'")); + } + } + Ok(()) +} + +/// Convert caller-supplied `tools[]` into engine-native `ActionDef`s for +/// registration in the per-thread external tool catalog. Caller schemas +/// pass through unchanged: `parameters` becomes `parameters_schema`. +/// Effects are stamped as `Compute` (no externally-claimed effect type) +/// and `requires_approval` is false — caller-side gating is the +/// caller's responsibility, not the engine's. +fn responses_tools_to_action_defs(tools: &[ResponsesTool]) -> Vec<ironclaw_engine::ActionDef> { + tools + .iter() + .filter_map(|t| { + let name = t.name.clone()?; + Some(ironclaw_engine::ActionDef { + name, + description: t.description.clone().unwrap_or_default(), + parameters_schema: t + .parameters + .clone() + .unwrap_or_else(|| serde_json::json!({"type": "object"})), + effects: vec![ironclaw_engine::EffectType::Compute], + requires_approval: false, + model_tool_surface: ironclaw_engine::ModelToolSurface::FullSchema, + discovery: None, + }) + }) + .collect() +} + +/// Synthetic engine action names that the orchestrator emits as +/// `ActionStarted`/`ActionFailed` events for internal bookkeeping +/// (CodeAct script execution, etc.) but which are not real +/// caller-visible tool calls. Filtering these out of the Responses +/// API output prevents internal markers like `__codeact__` from +/// surfacing as `function_call` items in the response. +/// +/// Names use the leading-underscore convention reserved for +/// engine-internal markers. +fn is_synthetic_engine_action(name: &str) -> bool { + name.starts_with("__") && name.ends_with("__") +} + /// Check whether an `AppEvent` belongs to the target thread. fn event_matches_thread(event: &AppEvent, target: &str) -> bool { match event { @@ -387,7 +730,8 @@ fn event_matches_thread(event: &AppEvent, target: &str) -> bool { | AppEvent::Status { thread_id, .. } | AppEvent::ApprovalNeeded { thread_id, .. } | AppEvent::GateRequired { thread_id, .. } - | AppEvent::GateResolved { thread_id, .. } => thread_id.as_deref() == Some(target), + | AppEvent::GateResolved { thread_id, .. } + | AppEvent::ExternalToolCall { thread_id, .. } => thread_id.as_deref() == Some(target), // Global or job-scoped events are never matched. _ => false, } @@ -482,7 +826,46 @@ impl ResponseAccumulator { } true // turn complete } + AppEvent::ExternalToolCall { + call_id, + name, + arguments, + .. + } => { + // Caller-supplied tool was invoked by the LLM. Emit a + // `function_call` output item and complete the turn — + // the thread is paused in `Waiting` until the caller + // POSTs the matching `function_call_output`. + // + // If buffered stream chunks accumulated before the + // pause (e.g. the model preceded the call with prose), + // flush them as a leading message so the OpenAI client + // sees both pieces in order. + if !self.text_chunks.is_empty() { + let leading: String = self.text_chunks.drain(..).collect(); + if !leading.is_empty() { + self.output.push(ResponseOutputItem::Message { + id: make_item_id(), + role: "assistant".to_string(), + content: vec![MessageContent::OutputText { text: leading }], + }); + } + } + self.output.push(ResponseOutputItem::FunctionCall { + id: make_item_id(), + call_id, + name, + arguments, + }); + true // turn complete (waiting for caller-supplied result) + } AppEvent::ToolStarted { name, call_id, .. } => { + // Filter synthetic engine markers — these are internal + // bookkeeping events (CodeAct script execution, etc.), + // not real tool calls the caller asked for. + if is_synthetic_engine_action(&name) { + return false; + } // Emit function_call placeholder — arguments filled on ToolCompleted. let call_id = call_id.unwrap_or_else(|| format!("call_{}", Uuid::new_v4().simple())); @@ -502,6 +885,9 @@ impl ResponseAccumulator { call_id, .. } => { + if is_synthetic_engine_action(&name) { + return false; + } // Try to attach arguments to the matching FunctionCall. if let Some(args) = parameters && let Some(idx) = @@ -528,6 +914,9 @@ impl ResponseAccumulator { call_id, .. } => { + if is_synthetic_engine_action(&name) { + return false; + } let call_id = self.resolve_call_id(&name, call_id.as_deref()); self.output.push(ResponseOutputItem::FunctionCallOutput { id: make_item_id(), @@ -712,44 +1101,117 @@ pub async fn create_response_handler( "invalid_request_error", )); } - if req.instructions.is_some() { - return Err(api_error( - StatusCode::BAD_REQUEST, - "The 'instructions' field is not yet supported", - "invalid_request_error", - )); - } - if req.tools.is_some() { - return Err(api_error( - StatusCode::BAD_REQUEST, - "The 'tools' field is not yet supported", - "invalid_request_error", - )); + let external_tools: Vec<ResponsesTool> = req.tools.clone().unwrap_or_default(); + validate_external_tools(&external_tools) + .map_err(|e| api_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; + + // Reject names that shadow internal action names (built-in tools, + // extension tools, or engine v2 capability actions like + // `mission_*` / `skill_*` / `memory_*`). Without this check the + // catalog short-circuits dispatch in `EffectBridgeAdapter`, so an + // LLM call to (say) `shell` or `mission_create` lands in + // caller-side execution even though the LLM saw the internal + // action's description in its action surface — a confused-deputy + // path where the caller crafts any output and the LLM treats it + // as the trusted internal action's reply. + // + // Two collision sources: + // - `ToolRegistry::tool_definitions()` — built-ins + extensions. + // - `engine_capability_action_names()` — capability actions + // registered via the engine v2 CapabilityRegistry. The + // `tool_registry` check alone misses these because they live + // on a different surface. + // + // The check runs only when the tool registry is wired; deployments + // without a registry also lack engine v2 (they are constructed + // together in `init_engine`), and the engine-v2 availability check + // below will reject the request before any external tool can be + // dispatched, so there is no bypass path in those configurations. + if !external_tools.is_empty() + && let Some(registry) = state.tool_registry.as_ref() + { + let mut reserved: std::collections::HashSet<String> = registry + .tool_definitions() + .await + .into_iter() + .map(|t| t.name) + .collect(); + if let Some(capability_names) = crate::bridge::engine_capability_action_names().await { + reserved.extend(capability_names); + } + for tool in &external_tools { + if let Some(name) = tool.name.as_deref() + && reserved.contains(name) + { + return Err(api_error( + StatusCode::BAD_REQUEST, + format!( + "tool '{name}' shadows a built-in, extension, or engine \ + action; pick a different name" + ), + "invalid_request_error", + )); + } + } } + if req.tool_choice.is_some() { + // `tool_choice` (auto / none / required / specific function) is not + // honoured because the engine doesn't have a per-request tool surface + // to enforce it against. Reject explicitly so callers don't get + // silently-ignored steering. return Err(api_error( StatusCode::BAD_REQUEST, "The 'tool_choice' field is not yet supported", "invalid_request_error", )); } - if req.temperature.is_some() { + if req.max_output_tokens.is_some() { return Err(api_error( StatusCode::BAD_REQUEST, - "Per-request 'temperature' is not supported on this endpoint; configure the default via settings", + "The 'max_output_tokens' field is not yet supported", "invalid_request_error", )); } - if req.max_output_tokens.is_some() { + // Reject non-finite or out-of-range temperature at the API boundary. + // The provider-side path clamps to [0, 2] later, but callers modelling + // against the OpenAI Responses contract expect a 400 here rather than + // silent normalization (e.g. 9.0 → 2.0). + if let Some(t) = req.temperature + && (!t.is_finite() || !(0.0..=2.0).contains(&t)) + { return Err(api_error( StatusCode::BAD_REQUEST, - "The 'max_output_tokens' field is not yet supported", + "The 'temperature' field must be a finite number in [0, 2]", "invalid_request_error", )); } - let mut content = extract_user_content(&req.input) + // Caller-supplied tools require engine v2 — the gate machinery that + // pauses execution and emits `AppEvent::ExternalToolCall` only + // exists on the v2 path. The presence of an initialized + // `ExternalToolCatalog` is the canonical "engine v2 is up" signal + // here: the catalog is constructed inside `init_engine` and shared + // between the bridge and this handler. Falling back to the + // `ENGINE_V2` env var would diverge from the agent loop's runtime + // config (`Config::agent::engine_v2`), which is the actual switch. + let catalog = if !external_tools.is_empty() { + let cat = crate::bridge::engine_external_tool_catalog().await; + if cat.is_none() { + return Err(api_error( + StatusCode::BAD_REQUEST, + "Caller-supplied 'tools' require engine v2 to be enabled on the server", + "invalid_request_error", + )); + } + cat + } else { + None + }; + + let extracted = extract_user_content(&req.input) .map_err(|e| api_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; + let mut content = extracted.user_text; // Prepend structured context (e.g. notification approval/rejection). // Enforce a 10 KB size limit to prevent context window exhaustion. @@ -766,6 +1228,18 @@ pub async fn create_response_handler( content = format!("<user-context>\n{prefix}\n</user-context>\n\n{content}"); } + // Prepend per-request instructions. Per the OpenAI Responses API spec, + // `instructions` is a system/developer message inserted at the start of + // the model's context for this turn only — it does NOT carry over via + // `previous_response_id`. We surface it to the agent as an `<instructions>` + // block ahead of everything else so it takes precedence over + // `<user-context>`. + if let Some(instructions) = req.instructions.as_deref().map(str::trim) + && !instructions.is_empty() + { + content = format!("<instructions>\n{instructions}\n</instructions>\n\n{content}"); + } + // Resolve or create thread. let thread_uuid = match &req.previous_response_id { Some(prev_id) => { @@ -780,6 +1254,145 @@ pub async fn create_response_handler( // Each POST gets its own unique response UUID. let response_uuid = Uuid::new_v4(); + // Register caller-supplied tools in the engine's per-thread external + // tool catalog. The engine's `EffectBridgeAdapter` consults this on + // every action call to short-circuit caller tools to a + // `GatePaused { resume_kind: External { ext_tool:<call_id> } }`, + // which the bridge router projects to `AppEvent::ExternalToolCall`. + if let Some(catalog) = catalog.as_ref() { + let action_defs = responses_tools_to_action_defs(&external_tools); + catalog + .register(ironclaw_engine::ThreadId(thread_uuid), action_defs) + .await; + } + + // Resume detection: a request that carries `function_call_output` + // items resolves the most recent `ResumeKind::External` gate for + // this thread. Without a pending gate the request is malformed — + // there's nothing to resume against. + if !extracted.tool_outputs.is_empty() { + let pending = crate::bridge::get_engine_pending_gate(&user.user_id, Some(&thread_id_str)) + .await + .map_err(|e| { + api_error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to look up pending gate: {e}"), + "server_error", + ) + })?; + let pending = pending.ok_or_else(|| { + api_error( + StatusCode::BAD_REQUEST, + "function_call_output supplied but no pending external tool call for \ + this thread. Verify the prior response.output array contained a \ + `function_call` item for this call_id; if it did not, the agent did \ + not actually invoke the caller-supplied tool.", + "invalid_request_error", + ) + })?; + // Verify the pending gate is actually an external-tool gate. + // A thread can be paused on an unrelated approval/auth gate + // (e.g. OAuth callback in progress); without this check, a + // `function_call_output` would route through the wrong gate + // and silently fail to resolve, returning a confusing + // response to the client. + let expected_call_id = match &pending.resume_kind { + ironclaw_engine::ResumeKind::External { callback_id } + if crate::bridge::is_external_tool_callback_id(callback_id) => + { + crate::bridge::call_id_from_external_callback(callback_id) + .unwrap_or("") + .to_string() + } + _ => { + return Err(api_error( + StatusCode::BAD_REQUEST, + "function_call_output supplied but the pending gate for this thread \ + is not an external tool callback (it is an unrelated approval, \ + authentication, or OAuth/pairing gate). Resolve that gate first.", + "invalid_request_error", + )); + } + }; + // The pending gate names exactly one outstanding external + // tool call (call_id is embedded in the callback id). At + // least one of the supplied `function_call_output` items + // must match it — otherwise the resume payload describes a + // call the engine never made. + if !extracted + .tool_outputs + .iter() + .any(|(call_id, _)| call_id == &expected_call_id) + { + return Err(api_error( + StatusCode::BAD_REQUEST, + "function_call_output supplied does not match the pending external \ + tool callback for this thread; verify the call_id matches the \ + `function_call` item from the prior response.", + "invalid_request_error", + )); + } + let request_uuid = uuid::Uuid::parse_str(&pending.request_id).map_err(|_| { + api_error( + StatusCode::INTERNAL_SERVER_ERROR, + "pending gate has malformed request_id", + "server_error", + ) + })?; + let payload = serde_json::json!({ + "outputs": extracted + .tool_outputs + .iter() + .map(|(call_id, output)| serde_json::json!({ + "call_id": call_id, + "output": output, + })) + .collect::<Vec<_>>(), + }); + let submission = crate::agent::submission::Submission::ExternalCallback { + request_id: request_uuid, + payload: Some(payload), + }; + let placeholder = "[external tool callback]".to_string(); + let mut metadata = serde_json::json!({ + "thread_id": &thread_id_str, + "user_id": &user.user_id, + "source": "responses_api", + }); + if let Some(ref ctx) = req.x_context { + metadata["context"] = ctx.clone(); + } + let resume_msg = crate::channels::web::util::web_incoming_message_with_metadata( + "gateway", + &user.user_id, + &placeholder, + Some(&thread_id_str), + metadata, + ) + .with_structured_submission(submission); + + let resp_id = encode_response_id(&response_uuid, &thread_uuid); + let model = req.model.clone(); + let stream = req.stream.unwrap_or(false); + let user_id = user.user_id.clone(); + if stream { + return handle_streaming(state, resume_msg, resp_id, model, thread_id_str, user_id) + .await + .map(IntoResponse::into_response); + } else { + return handle_non_streaming( + state, + resume_msg, + resp_id, + model, + thread_id_str, + &user_id, + ) + .await + .map(IntoResponse::into_response); + } + } + // Build the message for the agent loop. let mut metadata = serde_json::json!({ "thread_id": &thread_id_str, @@ -789,6 +1402,9 @@ pub async fn create_response_handler( if let Some(ref ctx) = req.x_context { metadata["context"] = ctx.clone(); } + if let Some(t) = req.temperature { + metadata["temperature"] = serde_json::json!(t); + } let msg = crate::channels::web::util::web_incoming_message_with_metadata( "gateway", &user.user_id, @@ -993,6 +1609,9 @@ async fn streaming_worker( acc.text_chunks.push(content.clone()); } AppEvent::ToolStarted { name, call_id, .. } => { + if is_synthetic_engine_action(name) { + continue; + } let idx = acc.output.len(); let call_id = call_id .clone() @@ -1021,6 +1640,9 @@ async fn streaming_worker( call_id, .. } => { + if is_synthetic_engine_action(name) { + continue; + } if let Some(args) = parameters && let Some(idx) = acc.find_function_call_index(name, call_id.as_deref(), true) && let Some(ResponseOutputItem::FunctionCall { arguments, .. }) = @@ -1074,6 +1696,9 @@ async fn streaming_worker( call_id, .. } => { + if is_synthetic_engine_action(name) { + continue; + } let call_id = acc.resolve_call_id(name, call_id.as_deref()); let idx = acc.output.len(); let item = ResponseOutputItem::FunctionCallOutput { @@ -1109,6 +1734,121 @@ async fn streaming_worker( _ => {} } + // External tool call (engine pause): emit added+done frames for the + // function_call wire item, then close the response. The thread + // stays in `Waiting` until the caller POSTs the matching + // `function_call_output`. + if let AppEvent::ExternalToolCall { + ref call_id, + ref name, + ref arguments, + .. + } = event + { + // Finalize any in-flight Message placeholder before + // emitting the function_call item. Two shapes to handle: + // + // - StreamChunk-created placeholder: a Message item was + // pushed with `output_item.added` when the first chunk + // arrived (`message_output_index` is `Some`). The + // accumulated text needs to be folded into that item + // and `output_item.done` emitted for the same index. + // Leaving it dangling without a matching `done` event + // would render as "in progress" forever in OpenAI + // clients (mirrors the bug the Response terminal path's + // `streaming_worker_finalizes_item_when_resolved_text_is_empty` + // regression test pins down). + // + // - No placeholder yet: we can have accumulated chunks if + // the worker batched them, or we may have nothing. Only + // push a new Message item when there's actual text. + if let Some(idx) = message_output_index.take() { + let leading: String = acc.text_chunks.drain(..).collect(); + let item_id = match acc.output.get(idx) { + Some(ResponseOutputItem::Message { id, .. }) => id.clone(), + _ => make_item_id(), + }; + let item = ResponseOutputItem::Message { + id: item_id, + role: "assistant".to_string(), + content: vec![MessageContent::OutputText { text: leading }], + }; + if idx < acc.output.len() { + acc.output[idx] = item.clone(); + } else { + acc.output.push(item.clone()); + } + let _ = emit( + &tx, + "response.output_item.done", + &ResponseStreamEvent::OutputItemDone { + output_index: idx, + item, + }, + ); + } else if !acc.text_chunks.is_empty() { + let leading: String = acc.text_chunks.drain(..).collect(); + if !leading.is_empty() { + let idx = acc.output.len(); + let item = ResponseOutputItem::Message { + id: make_item_id(), + role: "assistant".to_string(), + content: vec![MessageContent::OutputText { text: leading }], + }; + let _ = emit( + &tx, + "response.output_item.added", + &ResponseStreamEvent::OutputItemAdded { + output_index: idx, + item: item.clone(), + }, + ); + let _ = emit( + &tx, + "response.output_item.done", + &ResponseStreamEvent::OutputItemDone { + output_index: idx, + item: item.clone(), + }, + ); + acc.output.push(item); + } + } + + let idx = acc.output.len(); + let item = ResponseOutputItem::FunctionCall { + id: make_item_id(), + call_id: call_id.clone(), + name: name.clone(), + arguments: arguments.clone(), + }; + let _ = emit( + &tx, + "response.output_item.added", + &ResponseStreamEvent::OutputItemAdded { + output_index: idx, + item: item.clone(), + }, + ); + let _ = emit( + &tx, + "response.output_item.done", + &ResponseStreamEvent::OutputItemDone { + output_index: idx, + item: item.clone(), + }, + ); + acc.output.push(item); + + let resp = acc.finish(); + let _ = emit( + &tx, + "response.completed", + &ResponseStreamEvent::ResponseCompleted { response: resp }, + ); + return; + } + // Terminal events. let is_terminal = matches!( &event, @@ -1125,7 +1865,14 @@ async fn streaming_worker( } else { content.clone() }; - if !text.is_empty() { + // Finalize whenever there's text to emit OR a message + // item is already in flight from prior StreamChunks. A + // mid-flight item that never receives `output_item.done` + // dangles in the OpenAI client's UI as "in progress" + // forever. + let needs_finalize = !text.is_empty() || message_output_index.is_some(); + if needs_finalize { + let message_text = text.clone(); let idx = match message_output_index { Some(i) => i, None => { @@ -1153,15 +1900,16 @@ async fn streaming_worker( // Emit the full text as a delta so streaming clients // receive it via response.output_text.delta, but only - // when StreamChunks haven't already delivered the content. - if acc.text_chunks.is_empty() { + // when StreamChunks haven't already delivered the content + // and there's actual text to deliver. + if acc.text_chunks.is_empty() && !message_text.is_empty() { emit( &tx, "response.output_text.delta", &ResponseStreamEvent::OutputTextDelta { output_index: idx, content_index: 0, - delta: text.clone(), + delta: message_text.clone(), }, ); } @@ -1176,7 +1924,7 @@ async fn streaming_worker( let item = ResponseOutputItem::Message { id: item_id, role: "assistant".to_string(), - content: vec![MessageContent::OutputText { text }], + content: vec![MessageContent::OutputText { text: message_text }], }; acc.output[idx] = item.clone(); emit( @@ -1420,10 +2168,49 @@ mod tests { assert!(decode_response_id("resp_not_a_uuid").is_err()); } + fn message_item(role: &str, content: &str) -> ResponsesInputItem { + ResponsesInputItem { + item_type: Some("message".to_string()), + role: Some(role.to_string()), + content: Some(content.to_string()), + call_id: None, + name: None, + arguments: None, + output: None, + } + } + + fn legacy_message_item(role: &str, content: &str) -> ResponsesInputItem { + // Backwards-compatible shape: no `type` field. + ResponsesInputItem { + item_type: None, + role: Some(role.to_string()), + content: Some(content.to_string()), + call_id: None, + name: None, + arguments: None, + output: None, + } + } + + fn function_call_output_item(call_id: &str, output: &str) -> ResponsesInputItem { + ResponsesInputItem { + item_type: Some("function_call_output".to_string()), + role: None, + content: None, + call_id: Some(call_id.to_string()), + name: None, + arguments: None, + output: Some(output.to_string()), + } + } + #[test] fn extract_user_content_text() { let input = ResponsesInput::Text("hello".to_string()); - assert_eq!(extract_user_content(&input).unwrap(), "hello"); + let extracted = extract_user_content(&input).expect("extract"); + assert_eq!(extracted.user_text, "hello"); + assert!(extracted.tool_outputs.is_empty()); } #[test] @@ -1434,32 +2221,591 @@ mod tests { #[test] fn extract_user_content_messages_uses_last_user() { - let input = ResponsesInput::Messages(vec![ - ResponsesInputMessage { - role: "user".to_string(), - content: "first".to_string(), - }, - ResponsesInputMessage { - role: "assistant".to_string(), - content: "middle".to_string(), - }, - ResponsesInputMessage { - role: "user".to_string(), - content: "last".to_string(), - }, + let input = ResponsesInput::Items(vec![ + legacy_message_item("user", "first"), + legacy_message_item("assistant", "middle"), + legacy_message_item("user", "last"), ]); - assert_eq!(extract_user_content(&input).unwrap(), "last"); + let extracted = extract_user_content(&input).expect("extract"); + assert_eq!(extracted.user_text, "last"); + assert!(extracted.tool_outputs.is_empty()); } #[test] fn extract_user_content_no_user_message_errors() { - let input = ResponsesInput::Messages(vec![ResponsesInputMessage { - role: "system".to_string(), - content: "hello".to_string(), + let input = ResponsesInput::Items(vec![legacy_message_item("system", "hello")]); + assert!(extract_user_content(&input).is_err()); + } + + #[test] + fn extract_user_content_collects_function_call_outputs() { + let input = ResponsesInput::Items(vec![ + message_item("user", "what is the weather?"), + function_call_output_item("call_abc", "{\"temp\":72}"), + function_call_output_item("call_def", "sunny"), + ]); + let extracted = extract_user_content(&input).expect("extract"); + assert_eq!(extracted.user_text, "what is the weather?"); + assert_eq!( + extracted.tool_outputs, + vec![ + ("call_abc".to_string(), "{\"temp\":72}".to_string()), + ("call_def".to_string(), "sunny".to_string()), + ] + ); + } + + #[test] + fn extract_user_content_function_call_output_only_synthesises_prompt() { + // A follow-up turn that only carries tool results (no new user text) + // should still produce a non-empty `user_text` so the engine sees a + // user turn. + let input = ResponsesInput::Items(vec![function_call_output_item("call_x", "result")]); + let extracted = extract_user_content(&input).expect("extract"); + assert!(!extracted.user_text.is_empty()); + assert_eq!( + extracted.tool_outputs, + vec![("call_x".to_string(), "result".to_string())] + ); + } + + #[test] + fn extract_user_content_rejects_unknown_item_type() { + let input = ResponsesInput::Items(vec![ResponsesInputItem { + item_type: Some("file_search".to_string()), + ..legacy_message_item("user", "hi") }]); assert!(extract_user_content(&input).is_err()); } + #[test] + fn validate_external_tools_accepts_function_type() { + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some("get_weather".to_string()), + description: Some("Look up the weather".to_string()), + parameters: Some(serde_json::json!({"type":"object"})), + }]; + assert!(validate_external_tools(&tools).is_ok()); + } + + #[test] + fn validate_external_tools_rejects_other_types() { + let tools = vec![ResponsesTool { + tool_type: "web_search".to_string(), + name: Some("search".to_string()), + description: None, + parameters: None, + }]; + let err = validate_external_tools(&tools).expect_err("must reject"); + assert!(err.contains("web_search")); + } + + #[test] + fn validate_external_tools_rejects_missing_name() { + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: None, + description: None, + parameters: None, + }]; + assert!(validate_external_tools(&tools).is_err()); + } + + #[test] + fn validate_external_tools_rejects_name_with_invalid_chars() { + // Whitespace, control chars, dots, slashes — anything outside + // ASCII alphanumeric + `_` + `-` must be rejected. + for bad in [ + "has space", + "has\ttab", + "has\nnewline", + "has.dot", + "has/slash", + "has\x07bell", + "has;semicolon", + "🦀rust", + ] { + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some(bad.to_string()), + description: None, + parameters: None, + }]; + let err = validate_external_tools(&tools) + .expect_err(&format!("name {bad:?} must be rejected")); + assert!( + err.contains("invalid characters"), + "wrong rejection message for {bad:?}: {err}" + ); + } + } + + #[test] + fn validate_external_tools_rejects_name_exceeding_length() { + let too_long = "a".repeat(MAX_TOOL_NAME_LEN + 1); + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some(too_long), + description: None, + parameters: None, + }]; + let err = validate_external_tools(&tools).expect_err("over-long name must be rejected"); + assert!( + err.contains(&MAX_TOOL_NAME_LEN.to_string()), + "rejection should cite the limit: {err}" + ); + } + + #[test] + fn validate_external_tools_accepts_max_length_name() { + // Exactly MAX_TOOL_NAME_LEN must be accepted. + let name = "a".repeat(MAX_TOOL_NAME_LEN); + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some(name), + description: None, + parameters: None, + }]; + validate_external_tools(&tools).expect("max-length name must be accepted"); + } + + #[test] + fn validate_external_tools_rejects_duplicate_names() { + let tools = vec![ + ResponsesTool { + tool_type: "function".to_string(), + name: Some("dup".to_string()), + description: None, + parameters: None, + }, + ResponsesTool { + tool_type: "function".to_string(), + name: Some("dup".to_string()), + description: None, + parameters: None, + }, + ]; + let err = validate_external_tools(&tools).expect_err("must reject"); + assert!(err.contains("duplicate")); + } + + #[test] + fn validate_external_tools_rejects_oversized_payload() { + // One tool with a giant description blows past the 16 KiB cap. + let big = "x".repeat(MAX_TOOLS_BYTES + 1024); + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some("big".to_string()), + description: Some(big), + parameters: None, + }]; + let err = validate_external_tools(&tools).expect_err("must reject"); + assert!(err.contains("limit")); + } + + #[test] + fn responses_tools_to_action_defs_basic_round_trip() { + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: Some("get_weather".to_string()), + description: Some("Look up the weather".to_string()), + parameters: Some( + serde_json::json!({"type":"object","properties":{"city":{"type":"string"}}}), + ), + }]; + let defs = responses_tools_to_action_defs(&tools); + assert_eq!(defs.len(), 1); + assert_eq!(defs[0].name, "get_weather"); + assert_eq!(defs[0].description, "Look up the weather"); + assert!(defs[0].parameters_schema.get("properties").is_some()); + assert!(!defs[0].requires_approval); + } + + #[test] + fn responses_tools_to_action_defs_skips_nameless() { + let tools = vec![ResponsesTool { + tool_type: "function".to_string(), + name: None, + description: None, + parameters: None, + }]; + let defs = responses_tools_to_action_defs(&tools); + assert!(defs.is_empty()); + } + + #[test] + fn accumulator_external_tool_call_emits_function_call_item() { + let mut acc = ResponseAccumulator::new("resp_test".to_string(), "m".to_string()); + let done = acc.process(AppEvent::ExternalToolCall { + request_id: "req-1".into(), + call_id: "call_abc".into(), + name: "lookup".into(), + arguments: "{\"q\":\"x\"}".into(), + thread_id: Some("t".into()), + }); + assert!(done, "external tool call must complete the turn"); + let resp = acc.finish(); + assert_eq!(resp.output.len(), 1); + assert!(matches!( + &resp.output[0], + ResponseOutputItem::FunctionCall { call_id, name, arguments, .. } + if call_id == "call_abc" + && name == "lookup" + && arguments == "{\"q\":\"x\"}" + )); + assert_eq!(resp.status, ResponseStatus::Completed); + } + + #[test] + fn accumulator_external_tool_call_flushes_buffered_chunks() { + let mut acc = ResponseAccumulator::new("resp_test".to_string(), "m".to_string()); + acc.process(AppEvent::StreamChunk { + content: "Calling the tool.".into(), + thread_id: Some("t".into()), + }); + let done = acc.process(AppEvent::ExternalToolCall { + request_id: "req-1".into(), + call_id: "call_abc".into(), + name: "lookup".into(), + arguments: "{}".into(), + thread_id: Some("t".into()), + }); + assert!(done); + let resp = acc.finish(); + assert_eq!(resp.output.len(), 2); + assert!(matches!( + &resp.output[0], + ResponseOutputItem::Message { content, .. } + if matches!(&content[0], MessageContent::OutputText { text } if text == "Calling the tool.") + )); + assert!(matches!( + &resp.output[1], + ResponseOutputItem::FunctionCall { name, .. } if name == "lookup" + )); + } + + /// `__codeact__` and other `__double_underscore__` action names are + /// internal engine markers for synthetic events (CodeAct script + /// execution failure, etc.) — they must NOT surface to the caller + /// as `function_call` output items. Filter them in the accumulator. + #[test] + fn accumulator_filters_synthetic_engine_actions() { + let mut acc = ResponseAccumulator::new("resp_test".to_string(), "m".to_string()); + // Simulate the orchestrator's CodeAct-script-failed path: + // ToolStarted + ToolCompleted with the synthetic name. + acc.process(AppEvent::ToolStarted { + name: "__codeact__".into(), + detail: None, + call_id: Some("codeact-step-1".into()), + thread_id: Some("t".into()), + }); + acc.process(AppEvent::ToolCompleted { + name: "__codeact__".into(), + success: false, + error: Some("CodeAct execution failed".into()), + parameters: None, + call_id: Some("codeact-step-1".into()), + duration_ms: Some(1), + thread_id: Some("t".into()), + }); + let resp = acc.finish(); + assert!( + resp.output.is_empty(), + "synthetic engine actions must not produce output items, got: {:?}", + resp.output + ); + } + + #[test] + fn is_synthetic_engine_action_recognizes_double_underscore() { + assert!(is_synthetic_engine_action("__codeact__")); + assert!(is_synthetic_engine_action("__init__")); + assert!(!is_synthetic_engine_action("get_balances")); + assert!(!is_synthetic_engine_action("_private")); + assert!(!is_synthetic_engine_action("__leading_only")); + assert!(!is_synthetic_engine_action("trailing_only__")); + } + + /// Streaming round-trip: when the LLM streams a couple of text + /// chunks then triggers a caller-supplied external tool, the SSE + /// stream must (a) deliver the deltas as `output_text.delta`, (b) + /// flush the buffered prose as a complete Message item *before* + /// the function_call, (c) emit `output_item.added`+`done` for the + /// function_call, and (d) close with `response.completed`. + /// + /// The earlier `accumulator_*` tests cover the in-memory state + /// transitions; this test drives the full `streaming_worker` and + /// verifies the wire-frame sequence that an OpenAI client would + /// observe. + #[tokio::test] + async fn streaming_worker_external_tool_call_emits_correct_frame_sequence() { + use axum::response::Sse; + use axum::response::sse::KeepAlive; + use http_body_util::BodyExt; + use tokio_stream::wrappers::ReceiverStream; + + let thread_id = "thread-stream-test".to_string(); + + // Input: synthetic AppEvent stream the worker will consume. + let (input_tx, input_rx) = tokio::sync::mpsc::channel::<AppEvent>(8); + let input_stream = ReceiverStream::new(input_rx); + + // Output: the worker pushes axum SSE Events here. Buffer + // generously — the external-tool branch can emit up to 7 + // frames (created, delta, item.added/done × 2, completed). + let (out_tx, out_rx) = tokio::sync::mpsc::channel::<axum::response::sse::Event>(16); + + // Drive the worker on a background task so we can feed it + // synchronously from the test body. + let worker = tokio::spawn(streaming_worker( + out_tx, + input_stream, + "resp_stream_test".to_string(), + "test-model".to_string(), + thread_id.clone(), + )); + + // Two text deltas, then the external-tool gate fires. + input_tx + .send(AppEvent::StreamChunk { + content: "Looking up ".to_string(), + thread_id: Some(thread_id.clone()), + }) + .await + .unwrap(); + input_tx + .send(AppEvent::StreamChunk { + content: "the weather.".to_string(), + thread_id: Some(thread_id.clone()), + }) + .await + .unwrap(); + input_tx + .send(AppEvent::ExternalToolCall { + request_id: "req-stream-1".to_string(), + call_id: "call_lookup_weather_1".to_string(), + name: "lookup_weather".to_string(), + arguments: "{\"city\":\"NYC\"}".to_string(), + thread_id: Some(thread_id.clone()), + }) + .await + .unwrap(); + // Closing the input stream lets the worker exit if the + // external-tool branch didn't already terminate it. + drop(input_tx); + + // Collect Events. The external-tool branch returns from the + // worker, which drops out_tx and closes the receiver — so + // pulling from the stream until exhaustion gives us every + // emitted frame. + let out_stream = ReceiverStream::new(out_rx); + let response = Sse::new(out_stream.map(Ok::<_, Infallible>)) + .keep_alive(KeepAlive::new().interval(Duration::from_secs(60)).text("")) + .into_response(); + let body = response.into_body(); + let bytes = tokio::time::timeout(Duration::from_secs(5), body.collect()) + .await + .expect("body collected within timeout") + .expect("body bytes") + .to_bytes(); + let text = std::str::from_utf8(&bytes).expect("utf8"); + + // Worker must finish cleanly (no panic). + worker.await.expect("worker panicked"); + + // Parse SSE frames. axum emits `event: <type>\ndata: <json>\n\n` + // for each event; a leading colon-only line is the keep-alive. + let frames: Vec<(String, String)> = text + .split("\n\n") + .filter_map(|frame| { + let mut event_type: Option<String> = None; + let mut data: Option<String> = None; + for line in frame.lines() { + if let Some(t) = line.strip_prefix("event:") { + event_type = Some(t.trim().to_string()); + } else if let Some(d) = line.strip_prefix("data:") { + data = Some(d.trim().to_string()); + } + } + match (event_type, data) { + (Some(t), Some(d)) => Some((t, d)), + _ => None, + } + }) + .collect(); + + let event_types: Vec<&str> = frames.iter().map(|(t, _)| t.as_str()).collect(); + + // Expected wire-frame sequence: + // response.created + // response.output_item.added (Message placeholder, opened on first StreamChunk) + // response.output_text.delta × 2 (one per chunk, into the placeholder) + // response.output_item.done (Message — placeholder finalized with accumulated text) + // response.output_item.added (FunctionCall) + // response.output_item.done (FunctionCall) + // response.completed + assert_eq!( + event_types, + vec![ + "response.created", + "response.output_item.added", + "response.output_text.delta", + "response.output_text.delta", + "response.output_item.done", + "response.output_item.added", + "response.output_item.done", + "response.completed", + ], + "wire frame sequence does not match expected ordering" + ); + + // Pairing invariant: every `output_item.added` must have a + // matching `output_item.done`. Without this, a streaming + // client sees "in progress" placeholders that never resolve. + let added_count = event_types + .iter() + .filter(|t| **t == "response.output_item.added") + .count(); + let done_count = event_types + .iter() + .filter(|t| **t == "response.output_item.done") + .count(); + assert_eq!( + added_count, done_count, + "every output_item.added must have a matching output_item.done; \ + added={added_count} done={done_count}" + ); + + // Find the function_call frames and assert they carry the + // caller's tool identity. + let function_call_frames: Vec<&(String, String)> = frames + .iter() + .filter(|(t, d)| { + t == "response.output_item.added" && d.contains("\"type\":\"function_call\"") + }) + .collect(); + assert_eq!( + function_call_frames.len(), + 1, + "expected exactly one function_call output_item.added frame" + ); + let payload = &function_call_frames[0].1; + assert!( + payload.contains("\"call_id\":\"call_lookup_weather_1\""), + "function_call frame missing caller-supplied call_id: {payload}" + ); + assert!( + payload.contains("\"name\":\"lookup_weather\""), + "function_call frame missing tool name: {payload}" + ); + assert!( + payload.contains("\"arguments\":\"{\\\"city\\\":\\\"NYC\\\"}\""), + "function_call frame missing arguments: {payload}" + ); + + // The flushed leading message must contain the concatenated + // text from both StreamChunks, in order. + let message_done_frames: Vec<&(String, String)> = frames + .iter() + .filter(|(t, d)| t == "response.output_item.done" && d.contains("\"type\":\"message\"")) + .collect(); + assert_eq!(message_done_frames.len(), 1, "expected one Message done"); + let message_payload = &message_done_frames[0].1; + assert!( + message_payload.contains("Looking up the weather."), + "Message done frame missing concatenated stream text: {message_payload}" + ); + } + + /// Regression: if `StreamChunk`s create a Message item via + /// `output_item.added` and the terminal `Response` event resolves + /// to empty text (chunks were all empty, or the content vacuums + /// out somehow), the worker must still emit `output_item.done` for + /// the item it opened. Without this, OpenAI clients leave a + /// dangling "in-progress" message in the UI forever. + #[tokio::test] + async fn streaming_worker_finalizes_item_when_resolved_text_is_empty() { + use axum::response::Sse; + use axum::response::sse::KeepAlive; + use http_body_util::BodyExt; + use tokio_stream::wrappers::ReceiverStream; + + let thread_id = "thread-stream-empty".to_string(); + let (input_tx, input_rx) = tokio::sync::mpsc::channel::<AppEvent>(4); + let (out_tx, out_rx) = tokio::sync::mpsc::channel::<axum::response::sse::Event>(16); + + let worker = tokio::spawn(streaming_worker( + out_tx, + ReceiverStream::new(input_rx), + "resp_empty_test".to_string(), + "test-model".to_string(), + thread_id.clone(), + )); + + // An empty StreamChunk creates the Message item but contributes + // no text. Without it, message_output_index would stay None and + // the dangling-item bug couldn't manifest. + input_tx + .send(AppEvent::StreamChunk { + content: String::new(), + thread_id: Some(thread_id.clone()), + }) + .await + .unwrap(); + // Terminal Response with empty content. With the buggy gate + // (`!text.is_empty()`), the entire finalize block was skipped. + input_tx + .send(AppEvent::Response { + content: String::new(), + thread_id: thread_id.clone(), + }) + .await + .unwrap(); + drop(input_tx); + + let response = Sse::new(ReceiverStream::new(out_rx).map(Ok::<_, Infallible>)) + .keep_alive(KeepAlive::new().interval(Duration::from_secs(60)).text("")) + .into_response(); + let bytes = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect()) + .await + .expect("body collected within timeout") + .expect("body bytes") + .to_bytes(); + let text = std::str::from_utf8(&bytes).expect("utf8"); + worker.await.expect("worker panicked"); + + let event_types: Vec<&str> = text + .split("\n\n") + .filter_map(|frame| { + frame + .lines() + .find_map(|line| line.strip_prefix("event:").map(|s| s.trim())) + }) + .collect(); + + // Every `output_item.added` for a Message item must be paired + // with a matching `output_item.done`. If the dangling-item + // regression returns, `done_count < added_count`. + let added_count = event_types + .iter() + .filter(|t| **t == "response.output_item.added") + .count(); + let done_count = event_types + .iter() + .filter(|t| **t == "response.output_item.done") + .count(); + assert_eq!( + added_count, done_count, + "every output_item.added must be paired with output_item.done; \ + frames seen: {event_types:?}" + ); + assert!( + added_count >= 1, + "expected at least one output_item.added (from the StreamChunk), got: {event_types:?}" + ); + } + #[test] fn event_matches_thread_filters_correctly() { let target = "abc-123"; diff --git a/src/channels/web/test_helpers.rs b/src/channels/web/test_helpers.rs index ff7a18d0a9d..d3875f7f360 100644 --- a/src/channels/web/test_helpers.rs +++ b/src/channels/web/test_helpers.rs @@ -45,8 +45,9 @@ use crate::tools::ToolRegistry; /// [`start`](Self::start) to also bind an Axum server on a random port. pub struct TestGatewayBuilder { msg_tx: Option<mpsc::Sender<IncomingMessage>>, - llm_provider: Option<Arc<dyn crate::llm::LlmProvider>>, + llm_provider: Option<Arc<dyn ironclaw_llm::LlmProvider>>, user_id: String, + tool_registry: Option<Arc<crate::tools::ToolRegistry>>, } impl Default for TestGatewayBuilder { @@ -55,6 +56,7 @@ impl Default for TestGatewayBuilder { msg_tx: None, llm_provider: None, user_id: "test-user".to_string(), + tool_registry: None, } } } @@ -73,7 +75,7 @@ impl TestGatewayBuilder { } /// Set the LLM provider (needed for OpenAI-compatible API tests). - pub fn llm_provider(mut self, provider: Arc<dyn crate::llm::LlmProvider>) -> Self { + pub fn llm_provider(mut self, provider: Arc<dyn ironclaw_llm::LlmProvider>) -> Self { self.llm_provider = Some(provider); self } @@ -84,6 +86,14 @@ impl TestGatewayBuilder { self } + /// Attach a `ToolRegistry` to the gateway. Tests that need to + /// exercise registry-aware handler paths (collision rejection, + /// permission filtering, etc.) inject one here. + pub fn tool_registry(mut self, registry: Arc<crate::tools::ToolRegistry>) -> Self { + self.tool_registry = Some(registry); + self + } + /// Build the `Arc<GatewayState>` without starting a server. pub fn build(self) -> Arc<GatewayState> { Arc::new(GatewayState { @@ -96,7 +106,7 @@ impl TestGatewayBuilder { log_broadcaster: None, log_level_handle: None, extension_manager: None, - tool_registry: None, + tool_registry: self.tool_registry, store: None, settings_cache: None, job_manager: None, diff --git a/src/channels/web/tests/mod.rs b/src/channels/web/tests/mod.rs index d5c696a8292..e6a1d4edb46 100644 --- a/src/channels/web/tests/mod.rs +++ b/src/channels/web/tests/mod.rs @@ -3,4 +3,5 @@ mod multi_tenant; mod no_silent_drop; mod rebuild_state_preserves_fields; +mod status_event_isolation; mod tool_event_passthrough; diff --git a/src/channels/web/tests/multi_tenant.rs b/src/channels/web/tests/multi_tenant.rs index 2e809eefeaa..1186c62f352 100644 --- a/src/channels/web/tests/multi_tenant.rs +++ b/src/channels/web/tests/multi_tenant.rs @@ -192,8 +192,8 @@ fn make_sandbox_job(user_id: &str, task: &str) -> crate::history::SandboxJobReco mod workspace_pool { use super::*; use crate::config::{WorkspaceConfig, WorkspaceSearchConfig}; - use crate::workspace::EmbeddingCacheConfig; - use crate::workspace::layer::MemoryLayer; + use crate::workspace::layer::{LayerSensitivity, MemoryLayer}; + use ironclaw_embeddings::EmbeddingCacheConfig; #[tokio::test] async fn test_workspace_pool_applies_search_config() { @@ -225,7 +225,7 @@ mod workspace_pool { name: "shared-layer".to_string(), scope: "shared".to_string(), writable: false, - sensitivity: Default::default(), + sensitivity: LayerSensitivity::Shared, }]; let ws_config = WorkspaceConfig { memory_layers: layers, @@ -252,6 +252,74 @@ mod workspace_pool { ); } + #[tokio::test] + async fn test_workspace_pool_rebinds_default_private_layer_to_identity_user() { + let (db, _dir) = test_db().await; + let ws_config = WorkspaceConfig { + memory_layers: MemoryLayer::default_for_user("owner-scope"), + read_scopes: vec![], + }; + let pool = WorkspacePool::new( + db, + None, + EmbeddingCacheConfig::default(), + WorkspaceSearchConfig::default(), + ws_config, + ); + let identity = UserIdentity { + user_id: "alice".to_string(), + role: "admin".to_string(), + workspace_read_scopes: vec![], + }; + + let ws = pool.get_or_create(&identity).await; + + assert_eq!(ws.user_id(), "alice"); + let private = MemoryLayer::find(ws.memory_layers(), "private") + .expect("default private layer should exist"); + assert_eq!(private.scope, "alice"); + assert_eq!(ws.read_user_ids(), &["alice".to_string()]); + assert!( + !ws.read_user_ids().contains(&"owner-scope".to_string()), + "tenant workspace must not inherit the startup owner scope: {:?}", + ws.read_user_ids() + ); + } + + #[tokio::test] + async fn test_workspace_pool_does_not_read_owner_private_memory_by_default() { + let (db, _dir) = test_db().await; + let owner_ws = crate::workspace::Workspace::new_with_db("owner-scope", Arc::clone(&db)); + owner_ws + .write("daily/2099-01-01.md", "owner-only conversation archive") + .await + .expect("seed owner memory"); + let ws_config = WorkspaceConfig { + memory_layers: MemoryLayer::default_for_user("owner-scope"), + read_scopes: vec![], + }; + let pool = WorkspacePool::new( + db, + None, + EmbeddingCacheConfig::default(), + WorkspaceSearchConfig::default(), + ws_config, + ); + let identity = UserIdentity { + user_id: "alice".to_string(), + role: "admin".to_string(), + workspace_read_scopes: vec![], + }; + + let ws = pool.get_or_create(&identity).await; + let result = ws.read("daily/2099-01-01.md").await; + + assert!( + result.is_err(), + "tenant workspace unexpectedly read owner private memory: {result:?}" + ); + } + #[tokio::test] async fn test_workspace_pool_applies_identity_read_scopes() { let (db, _dir) = test_db().await; @@ -1091,6 +1159,50 @@ mod admin_api_contracts { assert_rfc3339(user.last_active_at.as_deref().unwrap()); } + #[tokio::test] + async fn test_admin_user_list_handles_tiny_costs_from_libsql() { + let (db, _dir) = test_db().await; + db.create_user(&test_user( + "carol", + "Carol", + Some("carol@example.com"), + "active", + "member", + serde_json::json!({}), + )) + .await + .unwrap(); + let job_id = db.create_system_job("carol", "test").await.unwrap(); + db.record_llm_call(&crate::history::LlmCallRecord { + job_id: Some(job_id), + conversation_id: None, + provider: "test", + model: "tiny-cost-model", + input_tokens: 1, + output_tokens: 1, + cost: rust_decimal::Decimal::from_str_exact("0.000075").unwrap(), + purpose: Some("test"), + }) + .await + .unwrap(); + + let state = build_state(Some(db), None); + let app = admin_router(state, two_user_auth()); + + let req = Request::builder() + .uri("/api/admin/users") + .header("Authorization", "Bearer tok-alice") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: AdminUserListResponse = parse_json(resp).await; + + let user = body.users.iter().find(|u| u.id == "carol").unwrap(); + assert_eq!(user.job_count, 1); + assert_eq!(user.total_cost, "0.000075"); + } + #[tokio::test] async fn test_admin_user_detail_response_contract() { let (db, _dir) = test_db().await; @@ -1256,7 +1368,7 @@ mod admin_tool_policy { let pool = WorkspacePool::new( Arc::clone(&db), None, - crate::workspace::EmbeddingCacheConfig::default(), + ironclaw_embeddings::EmbeddingCacheConfig::default(), crate::config::WorkspaceSearchConfig::default(), crate::config::WorkspaceConfig::default(), ); diff --git a/src/channels/web/tests/status_event_isolation.rs b/src/channels/web/tests/status_event_isolation.rs new file mode 100644 index 00000000000..2c5c76116c6 --- /dev/null +++ b/src/channels/web/tests/status_event_isolation.rs @@ -0,0 +1,318 @@ +//! Regression tests for cross-tenant SSE/WS status event isolation. +//! +//! Reproduces the report that users in a multi-tenant deployment could +//! see another user's status events (Thinking / ToolStarted / ToolResult / +//! ...). The leak vector was the unscoped global broadcast fallback in +//! `GatewayChannel::send_status` — see `mod.rs::dispatch_status_event`. +//! +//! These tests assert two invariants: +//! +//! 1. **Multi-tenant**: a status event without `metadata.user_id` is +//! DROPPED — no SSE subscriber receives it. Producers that lose +//! `user_id` along the way silently fail-closed; the warning surfaces +//! in logs so the producer can be fixed without exposing tenant data. +//! 2. **Single-tenant**: the same dropped-in-multi-tenant case +//! falls through to a global broadcast — there is one tenant, one +//! subscriber population, and the unscoped fan-out is by design. +//! +//! Plus the cross-cutting case: if `metadata.user_id` IS present, the +//! event is delivered ONLY to that user's stream, regardless of mode. + +use ironclaw_common::AppEvent; +use tokio_stream::StreamExt; + +use crate::channels::web::dispatch_status_event; +use crate::channels::web::sse::SseManager; + +/// Sentinel event used to flush past the dispatch under test. SSE +/// streams have no peek/timeout APIs in this test surface, so we always +/// broadcast a Heartbeat after the call and read until we see it. If +/// the event-under-test arrived, it shows up before the heartbeat; +/// if it was dropped, the heartbeat is the first thing we see. +fn flush_sentinel(manager: &SseManager) { + manager.broadcast(AppEvent::Heartbeat); +} + +#[tokio::test] +async fn unscoped_status_event_is_dropped_in_multi_tenant() { + let manager = SseManager::new(); + let mut alice = Box::pin( + manager + .subscribe_raw(Some("alice".to_string()), false) + .expect("alice subscribe"), + ); + let mut bob = Box::pin( + manager + .subscribe_raw(Some("bob".to_string()), false) + .expect("bob subscribe"), + ); + + // Producer forgot to include user_id in metadata — this is the bug + // shape that previously leaked across tenants. + dispatch_status_event( + &manager, + true, // multi_tenant_mode = ON + None, // no user_id in metadata + AppEvent::Thinking { + message: "secret tool reasoning".to_string(), + thread_id: Some("alice-thread".to_string()), + }, + ); + flush_sentinel(&manager); + + // Both subscribers receive the heartbeat first because the unscoped + // Thinking was dropped. If the leak comes back, alice (or bob) will + // see Thinking before Heartbeat and the assertion below fails. + let alice_first = alice.next().await.expect("alice receives sentinel"); + assert!( + matches!(alice_first, AppEvent::Heartbeat), + "multi-tenant leak: alice received an unscoped event before the heartbeat sentinel: {alice_first:?}" + ); + let bob_first = bob.next().await.expect("bob receives sentinel"); + assert!( + matches!(bob_first, AppEvent::Heartbeat), + "multi-tenant leak: bob received an unscoped event before the heartbeat sentinel: {bob_first:?}" + ); +} + +#[tokio::test] +async fn unscoped_status_event_passes_in_single_tenant() { + let manager = SseManager::new(); + let mut sole = Box::pin( + manager + .subscribe_raw(Some("only-user".to_string()), false) + .expect("subscribe"), + ); + + // In single-tenant mode there is one user and one subscriber + // population. Unscoped events MUST still reach them or background + // producers (heartbeat, routines) lose their UI. + dispatch_status_event( + &manager, + false, // multi_tenant_mode = OFF + None, + AppEvent::Thinking { + message: "single-tenant background work".to_string(), + thread_id: None, + }, + ); + + let event = sole + .next() + .await + .expect("single-tenant subscriber receives unscoped Thinking"); + match event { + AppEvent::Thinking { message, .. } => { + assert_eq!(message, "single-tenant background work") + } + other => panic!("expected Thinking, got {other:?}"), + } +} + +#[tokio::test] +async fn scoped_status_event_only_reaches_owning_user_in_multi_tenant() { + let manager = SseManager::new(); + let mut alice = Box::pin( + manager + .subscribe_raw(Some("alice".to_string()), false) + .expect("alice subscribe"), + ); + let mut bob = Box::pin( + manager + .subscribe_raw(Some("bob".to_string()), false) + .expect("bob subscribe"), + ); + + dispatch_status_event( + &manager, + true, + Some("alice"), + AppEvent::ToolStarted { + name: "telegram_send".to_string(), + detail: Some("alice's tool call".to_string()), + call_id: Some("call-1".to_string()), + thread_id: Some("alice-thread".to_string()), + }, + ); + flush_sentinel(&manager); + + // Alice sees her own scoped event, then the heartbeat. + let first = alice.next().await.expect("alice receives ToolStarted"); + assert!( + matches!(&first, AppEvent::ToolStarted { name, .. } if name == "telegram_send"), + "alice should receive her own ToolStarted, got {first:?}" + ); + let second = alice.next().await.expect("alice receives heartbeat"); + assert!(matches!(second, AppEvent::Heartbeat)); + + // Bob sees only the heartbeat — alice's scoped event was filtered. + let bob_event = bob.next().await.expect("bob receives heartbeat"); + assert!( + matches!(bob_event, AppEvent::Heartbeat), + "cross-tenant leak: bob received alice's ToolStarted: {bob_event:?}" + ); +} + +#[tokio::test] +async fn scoped_status_event_routes_in_single_tenant_too() { + // In single-tenant mode the scoped path still works — it is the + // happy path for chat send. This guards against a regression that + // would route every status event through the unscoped fallback. + let manager = SseManager::new(); + let mut sole = Box::pin( + manager + .subscribe_raw(Some("alice".to_string()), false) + .expect("subscribe"), + ); + + dispatch_status_event( + &manager, + false, + Some("alice"), + AppEvent::ToolCompleted { + name: "memory_write".to_string(), + success: true, + error: None, + parameters: None, + call_id: Some("call-1".to_string()), + duration_ms: Some(12), + thread_id: Some("t".to_string()), + }, + ); + + let event = sole + .next() + .await + .expect("subscriber receives ToolCompleted"); + assert!( + matches!(event, AppEvent::ToolCompleted { name, .. } if name == "memory_write"), + "single-tenant scoped delivery broke" + ); +} + +/// Walk a representative selection of `AppEvent` variants and assert +/// each one is dropped when emitted unscoped in multi-tenant mode. +/// Adding a new status variant should require updating this list — if +/// that is missed, the leak surface grows silently and this is the test +/// that catches it. +/// +/// The `_compile_time_appevent_variant_check` helper below pairs with +/// the runtime `leak_candidates` list as a build-time reminder: it +/// exhaustively matches every `AppEvent` variant, so adding a new +/// variant fails the test compilation until the helper is updated. +/// When you update the helper, also extend `leak_candidates` with the +/// new variant so the runtime drop assertion actually exercises it. +/// This is two-step (compile fails → developer adds to both places) +/// rather than fully automatic, but it converts a silent regression +/// into a loud build break. +#[tokio::test] +async fn unscoped_drop_holds_for_every_status_variant_in_multi_tenant() { + let manager = SseManager::new(); + let mut alice = Box::pin( + manager + .subscribe_raw(Some("alice".to_string()), false) + .expect("subscribe"), + ); + + let leak_candidates = [ + AppEvent::Thinking { + message: "x".into(), + thread_id: None, + }, + AppEvent::ToolStarted { + name: "x".into(), + detail: None, + call_id: None, + thread_id: None, + }, + AppEvent::ToolCompleted { + name: "x".into(), + success: true, + error: None, + parameters: None, + call_id: None, + duration_ms: None, + thread_id: None, + }, + AppEvent::Status { + message: "x".into(), + thread_id: None, + }, + AppEvent::Response { + content: "x".into(), + thread_id: "t".into(), + }, + ]; + + for event in leak_candidates { + dispatch_status_event(&manager, true, None, event); + } + flush_sentinel(&manager); + + let first = alice.next().await.expect("subscriber receives sentinel"); + assert!( + matches!(first, AppEvent::Heartbeat), + "multi-tenant leak: at least one unscoped status variant reached the subscriber \ + before the heartbeat sentinel — got {first:?}. If a new AppEvent variant was \ + added to the dispatch path, add it to `leak_candidates` and re-run." + ); +} + +/// Compile-time reminder for `unscoped_drop_holds_for_every_status_variant_in_multi_tenant`. +/// +/// This function is never called. It exists only so the exhaustive match +/// below fails to compile when a new `AppEvent` variant is added — that +/// failure is the prompt to update both this helper AND the runtime +/// `leak_candidates` list above. Without this, a new variant that bypasses +/// the dispatcher's drop-in-multi-tenant rule could silently ship. +/// +/// The match must list every variant explicitly; do not add a `_` arm. +/// `event_type()` in `crates/ironclaw_common/src/event.rs` follows the +/// same pattern for the same reason. +#[allow(dead_code)] +fn _compile_time_appevent_variant_check(e: AppEvent) { + match e { + AppEvent::Response { .. } + | AppEvent::Thinking { .. } + | AppEvent::ToolStarted { .. } + | AppEvent::ToolCompleted { .. } + | AppEvent::ToolResult { .. } + | AppEvent::StreamChunk { .. } + | AppEvent::Status { .. } + | AppEvent::JobStarted { .. } + | AppEvent::ApprovalNeeded { .. } + | AppEvent::OnboardingState { .. } + | AppEvent::GateRequired { .. } + | AppEvent::GateResolved { .. } + | AppEvent::Error { .. } + | AppEvent::Heartbeat + | AppEvent::JobMessage { .. } + | AppEvent::JobToolUse { .. } + | AppEvent::JobToolResult { .. } + | AppEvent::JobStatus { .. } + | AppEvent::JobResult { .. } + | AppEvent::ImageGenerated { .. } + | AppEvent::Suggestions { .. } + | AppEvent::TurnCost { .. } + | AppEvent::SkillActivated { .. } + | AppEvent::ExtensionStatus { .. } + | AppEvent::ReasoningUpdate { .. } + | AppEvent::JobReasoning { .. } + | AppEvent::ToolResultFull { .. } + | AppEvent::TurnMetrics { .. } + | AppEvent::ThreadStateChanged { .. } + | AppEvent::ChildThreadSpawned { .. } + | AppEvent::ChildThreadCompleted { .. } + | AppEvent::MissionThreadSpawned { .. } + | AppEvent::PlanUpdate { .. } + | AppEvent::CodeExecuted { .. } + | AppEvent::Warning { .. } + | AppEvent::CodeExecutionFailed { .. } + | AppEvent::LeaseGranted { .. } + | AppEvent::LeaseRevoked { .. } + | AppEvent::LeaseExpired { .. } + | AppEvent::SelfImprovement { .. } + | AppEvent::OrchestratorRollback { .. } + | AppEvent::ExternalToolCall { .. } => {} + } +} diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index 72532c6d341..00dcb1d828e 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -443,6 +443,7 @@ pub fn classify_wasm_channel_activation( ext: &crate::extensions::InstalledExtension, has_paired: bool, has_owner_binding: bool, + requires_binding: bool, ) -> Option<ExtensionActivationStatus> { if ext.kind != crate::extensions::ExtensionKind::WasmChannel { return None; @@ -453,7 +454,7 @@ pub fn classify_wasm_channel_activation( } else if !ext.authenticated { ExtensionActivationStatus::Installed } else if ext.active { - if has_paired || has_owner_binding { + if !requires_binding || has_paired || has_owner_binding { ExtensionActivationStatus::Active } else { ExtensionActivationStatus::Pairing @@ -544,6 +545,7 @@ pub struct ExtensionSetupResponse { pub secrets: Vec<SecretFieldInfo>, pub fields: Vec<SetupFieldInfo>, #[serde(skip_serializing_if = "Option::is_none")] + pub interactive_login: Option<crate::extensions::InteractiveLoginInfo>, pub onboarding_state: Option<ChannelOnboardingState>, #[serde(skip_serializing_if = "Option::is_none")] pub onboarding: Option<ChannelOnboardingInfo>, @@ -554,6 +556,8 @@ pub struct SecretFieldInfo { pub name: String, pub prompt: String, pub optional: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub validation: Option<String>, /// Whether this secret is already stored. pub provided: bool, /// Whether the secret will be auto-generated if left empty. @@ -583,6 +587,32 @@ pub struct ExtensionSetupRequest { pub fields: std::collections::HashMap<String, String>, } +#[derive(Debug, Deserialize)] +pub struct ExtensionInteractiveLoginStartRequest { + #[serde(default)] + pub force: bool, +} + +#[derive(Debug, Deserialize)] +pub struct ExtensionInteractiveLoginPollRequest { + pub session_id: String, +} + +#[derive(Debug, Serialize)] +pub struct ExtensionInteractiveLoginResponse { + pub success: bool, + pub status: String, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub session_id: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub qr_code_url: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub activated: Option<bool>, +} + #[derive(Debug, Serialize)] pub struct ActionResponse { pub success: bool, diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index c408323b0d5..866922f4a74 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -275,7 +275,7 @@ fn check_settings_file() -> CheckResult { async fn check_nearai_session(settings: &Settings) -> CheckResult { // Skip entirely when the configured backend is not NEAR AI. - let llm_config = match crate::config::LlmConfig::resolve(settings) { + let llm_config = match crate::config::llm::resolve(settings) { Ok(config) => config, Err(e) => { // check_llm_config will report the full error; just skip here. @@ -315,16 +315,12 @@ async fn check_nearai_session(settings: &Settings) -> CheckResult { // ── LLM configuration ────────────────────────────────────── fn check_llm_config(settings: &Settings) -> CheckResult { - match crate::llm::LlmConfig::resolve(settings) { + match crate::config::llm::resolve(settings) { Ok(config) => { - // Show the model for the active backend, not always nearai.model. - let model = if let Some(ref bedrock) = config.bedrock { - &bedrock.model - } else if let Some(ref provider) = config.provider { - &provider.model - } else { - &config.nearai.model - }; + // `active_model_name` is the crate-side dispatch that handles + // all backends (nearai/bedrock/codex/gemini_oauth + registry) + // — the doctor doesn't need to know which sub-config to read. + let model = config.active_model_name(); CheckResult::Pass(format!("backend={}, model={}", config.backend, model)) } Err(e) => CheckResult::Fail(format!("LLM config error: {e}")), @@ -415,43 +411,75 @@ fn check_workspace_dir() -> CheckResult { // ── Embeddings ────────────────────────────────────────────── fn check_embeddings(settings: &Settings) -> CheckResult { - match crate::config::EmbeddingsConfig::resolve(settings) { - Ok(config) => { - if !config.enabled { - return CheckResult::Skip("disabled (set EMBEDDING_ENABLED=true)".into()); - } - let has_creds = match config.provider.as_str() { - "openai" => config.openai_api_key().is_some(), - "nearai" => { - // NearAiEmbeddings uses SessionManager::get_token() which - // only returns session tokens, NOT NEARAI_API_KEY - // (src/workspace/embeddings.rs:309, src/llm/session.rs:132). - let session_path = crate::config::llm::default_session_path(); - session_path.exists() - && std::fs::read_to_string(&session_path) - .map(|s| !s.trim().is_empty()) - .unwrap_or(false) - } - "ollama" => true, // local, no creds needed - _ => config.openai_api_key().is_some(), - }; - if has_creds { - CheckResult::Pass(format!( - "provider={}, model={}", - config.provider, config.model - )) - } else { - let hint = match config.provider.as_str() { - "nearai" => "run `ironclaw onboard` to create a session", - _ => "set OPENAI_API_KEY", - }; - CheckResult::Fail(format!( - "provider={} but credentials missing ({})", - config.provider, hint - )) - } + // Resolve embeddings with a placeholder URL first. The URL field is + // only consulted at runtime by the NEAR AI provider, so a non-NEAR AI + // provider — even an enabled one — must not report a broken LLM config + // as an embeddings failure. + let placeholder_url = "https://placeholder.invalid"; + let initial = + match crate::config::embeddings::resolve_embeddings_config(settings, placeholder_url) { + Ok(c) => c, + Err(e) => return CheckResult::Fail(format!("config error: {e}")), + }; + if !initial.enabled { + return CheckResult::Skip("disabled (set EMBEDDING_ENABLED=true)".into()); + } + + // Only re-resolve with the real NEAR AI base URL when the provider + // actually needs it — otherwise an unrelated LLM resolve error would + // be reported as an embeddings failure. + let config = if initial.provider == "nearai" { + let nearai_base_url = match crate::config::llm::resolve(settings) { + Ok(llm) => llm.nearai.base_url, + Err(e) => return CheckResult::Fail(format!("could not resolve LLM config: {e}")), + }; + match crate::config::embeddings::resolve_embeddings_config(settings, &nearai_base_url) { + Ok(c) => c, + Err(e) => return CheckResult::Fail(format!("config error: {e}")), } - Err(e) => CheckResult::Fail(format!("config error: {e}")), + } else { + initial + }; + + let has_creds = match config.provider.as_str() { + "openai" => config.openai_api_key().is_some(), + "nearai" => { + // NearAiEmbeddings uses SessionManager::get_token() which only + // returns session tokens, NOT NEARAI_API_KEY. + let session_path = crate::config::llm::default_session_path(); + session_path.exists() + && std::fs::read_to_string(&session_path) + .map(|s| !s.trim().is_empty()) + .unwrap_or(false) + } + "ollama" => true, // local, no creds needed + "bedrock" => { + // AWS SDK credential chain — accept a named profile or static + // access-key + secret. Instance-role / IMDS credentials aren't + // visible from env and will surface here as Fail; that's + // acceptable for a static self-check. Mirrors the gateway's + // Bedrock setup-hint logic in `web/handlers/llm.rs`. + std::env::var("AWS_PROFILE").is_ok() + || (std::env::var("AWS_ACCESS_KEY_ID").is_ok() + && std::env::var("AWS_SECRET_ACCESS_KEY").is_ok()) + } + _ => config.openai_api_key().is_some(), + }; + if has_creds { + CheckResult::Pass(format!( + "provider={}, model={}", + config.provider, config.model + )) + } else { + let hint = match config.provider.as_str() { + "nearai" => "run `ironclaw onboard` to create a session", + "bedrock" => "set AWS_PROFILE or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY", + _ => "set OPENAI_API_KEY", + }; + CheckResult::Fail(format!( + "provider={} but credentials missing ({})", + config.provider, hint + )) } } @@ -1073,6 +1101,188 @@ mod tests { } } + /// Regression: PR #3739 review (P2 #5). The doctor used to resolve LLM + /// config to extract `nearai.base_url` before consulting the + /// `enabled` flag — so a broken LLM env (e.g., public-HTTP NEAR AI + /// base URL that fails SSRF validation) reported the Embeddings + /// check as Fail even when embeddings were disabled. + /// Regression: PR #3739 Copilot review. Once the disabled-skip + /// short-circuit was fixed, an *enabled* non-`nearai` provider + /// (e.g. `ollama`) still resolved LLM config to extract the NEAR AI + /// base URL — so an invalid LLM env reported the Embeddings check + /// as Fail even though embeddings wouldn't have used the URL. The + /// LLM resolve must only run when `provider == "nearai"`. + #[test] + fn check_embeddings_non_nearai_ignores_invalid_llm_config() { + let _guard = crate::config::helpers::lock_env(); + // SAFETY: Under ENV_MUTEX, no concurrent env access. + unsafe { + std::env::set_var("EMBEDDING_ENABLED", "true"); + std::env::set_var("EMBEDDING_PROVIDER", "ollama"); + // Public-HTTP base URL — `validate_operator_base_url` in the + // LLM resolver rejects this. The Ollama embeddings path + // does not consult LLM config, so the doctor must not + // surface this as an embeddings failure. + std::env::set_var("NEARAI_BASE_URL", "http://8.8.8.8/v1"); + } + let settings = Settings::default(); + let result = check_embeddings(&settings); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::remove_var("EMBEDDING_ENABLED"); + std::env::remove_var("EMBEDDING_PROVIDER"); + std::env::remove_var("NEARAI_BASE_URL"); + } + match result { + CheckResult::Pass(msg) => { + assert!( + msg.contains("ollama"), + "expected Pass mentioning ollama, got: {msg}" + ); + } + other => panic!( + "expected Pass for Ollama embeddings regardless of broken LLM env, got: {}", + format_result(&other) + ), + } + } + + #[test] + fn check_embeddings_disabled_skips_even_when_llm_config_invalid() { + let _guard = crate::config::helpers::lock_env(); + // SAFETY: Under ENV_MUTEX, no concurrent env access. + unsafe { + std::env::remove_var("EMBEDDING_ENABLED"); + // Public-HTTP base URL — `validate_operator_base_url` in the + // LLM resolver rejects this. The doctor must not see the + // resulting error, because embeddings are disabled. + std::env::set_var("NEARAI_BASE_URL", "http://8.8.8.8/v1"); + } + let settings = Settings::default(); + let result = check_embeddings(&settings); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::remove_var("NEARAI_BASE_URL"); + } + match result { + CheckResult::Skip(msg) => { + assert!( + msg.contains("disabled"), + "expected 'disabled' Skip even with broken LLM env, got: {msg}" + ); + } + other => panic!( + "expected Skip for disabled embeddings with broken LLM, got: {}", + format_result(&other) + ), + } + } + + /// Snapshot and restore an env var across a single test body — needed + /// because the AWS SDK env vars (`AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, + /// `AWS_SECRET_ACCESS_KEY`) may be set on dev/CI hosts and would + /// otherwise leak between the two Bedrock tests below. + struct EnvSnapshot { + name: &'static str, + prev: Option<String>, + } + impl EnvSnapshot { + fn take(name: &'static str) -> Self { + let prev = std::env::var(name).ok(); + // SAFETY: Under ENV_MUTEX, no concurrent env access. + unsafe { std::env::remove_var(name) }; + Self { name, prev } + } + } + impl Drop for EnvSnapshot { + fn drop(&mut self) { + // SAFETY: Under ENV_MUTEX. + unsafe { + match &self.prev { + Some(v) => std::env::set_var(self.name, v), + None => std::env::remove_var(self.name), + } + } + } + } + + /// Regression: PR #3739 Copilot review. `check_embeddings` used to + /// fall through to the `_` arm for `provider=bedrock`, treating + /// missing `OPENAI_API_KEY` as the credential failure. Bedrock has + /// its own credential chain (AWS profile or static access-key + + /// secret); the doctor must recognise it. + #[test] + fn check_embeddings_bedrock_with_aws_profile_passes() { + let _guard = crate::config::helpers::lock_env(); + // Snapshot ambient AWS env so we restore it after the test. + let _aws_profile = EnvSnapshot::take("AWS_PROFILE"); + let _aws_access = EnvSnapshot::take("AWS_ACCESS_KEY_ID"); + let _aws_secret = EnvSnapshot::take("AWS_SECRET_ACCESS_KEY"); + let _embed_enabled = EnvSnapshot::take("EMBEDDING_ENABLED"); + let _embed_provider = EnvSnapshot::take("EMBEDDING_PROVIDER"); + let _openai_key = EnvSnapshot::take("OPENAI_API_KEY"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::set_var("EMBEDDING_ENABLED", "true"); + std::env::set_var("EMBEDDING_PROVIDER", "bedrock"); + std::env::set_var("AWS_PROFILE", "default"); + } + + let settings = Settings::default(); + let result = check_embeddings(&settings); + match result { + CheckResult::Pass(msg) => { + assert!( + msg.contains("bedrock"), + "expected Pass mentioning bedrock, got: {msg}" + ); + } + other => panic!( + "expected Pass for Bedrock with AWS_PROFILE set, got: {}", + format_result(&other) + ), + } + } + + /// Regression: PR #3739 Copilot review. The credential-missing hint + /// used to suggest `set OPENAI_API_KEY` for any non-nearai provider, + /// including Bedrock. Confirm Bedrock now surfaces an AWS-specific + /// hint. + #[test] + fn check_embeddings_bedrock_without_aws_creds_fails_with_aws_hint() { + let _guard = crate::config::helpers::lock_env(); + let _aws_profile = EnvSnapshot::take("AWS_PROFILE"); + let _aws_access = EnvSnapshot::take("AWS_ACCESS_KEY_ID"); + let _aws_secret = EnvSnapshot::take("AWS_SECRET_ACCESS_KEY"); + let _embed_enabled = EnvSnapshot::take("EMBEDDING_ENABLED"); + let _embed_provider = EnvSnapshot::take("EMBEDDING_PROVIDER"); + let _openai_key = EnvSnapshot::take("OPENAI_API_KEY"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::set_var("EMBEDDING_ENABLED", "true"); + std::env::set_var("EMBEDDING_PROVIDER", "bedrock"); + } + + let settings = Settings::default(); + let result = check_embeddings(&settings); + match result { + CheckResult::Fail(msg) => { + assert!( + msg.contains("AWS_PROFILE") && msg.contains("AWS_ACCESS_KEY_ID"), + "expected Bedrock-specific hint, got: {msg}" + ); + assert!( + !msg.contains("OPENAI_API_KEY"), + "Bedrock failure must not mention OPENAI_API_KEY: {msg}" + ); + } + other => panic!( + "expected Fail for Bedrock with no AWS creds, got: {}", + format_result(&other) + ), + } + } + #[test] fn check_routines_enabled_by_default() { let _guard = crate::config::helpers::lock_env(); diff --git a/src/cli/memory.rs b/src/cli/memory.rs index fca6d03b35d..699ad1b6b85 100644 --- a/src/cli/memory.rs +++ b/src/cli/memory.rs @@ -7,7 +7,8 @@ use std::sync::Arc; use clap::Subcommand; -use crate::workspace::{EmbeddingCacheConfig, EmbeddingProvider, SearchConfig, Workspace}; +use crate::workspace::{SearchConfig, Workspace}; +use ironclaw_embeddings::{EmbeddingCacheConfig, EmbeddingProvider}; /// Run a memory command using the Database trait (works with any backend). pub async fn run_memory_command_with_db( diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 20914edaabb..7dd097dfb12 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -442,22 +442,31 @@ pub async fn run_memory_command(mem_cmd: &MemoryCommand) -> anyhow::Result<()> { .await .map_err(|e| anyhow::anyhow!("{}", e))?; - let session = crate::llm::create_session_manager(config.llm.session.clone()).await; - - let embeddings = config - .embeddings - .create_provider( - &config.llm.nearai.base_url, + let session = ironclaw_llm::create_session_manager(config.llm.session.clone()).await; + + let bedrock_setup = + config + .llm + .bedrock + .as_ref() + .map(|b| ironclaw_embeddings::BedrockEmbeddingSetup { + region: b.region.clone(), + profile: b.profile.clone(), + }); + let embeddings = ironclaw_embeddings::create_provider( + &config.embeddings, + ironclaw_embeddings::ProviderDeps { session, - config.llm.bedrock.as_ref(), - ) - .await; + bedrock_setup, + }, + ) + .await; let db: Arc<dyn crate::db::Database> = crate::db::connect_from_config(&config.database) .await .map_err(|e| anyhow::anyhow!("{}", e))?; - let cache_config = crate::workspace::EmbeddingCacheConfig { + let cache_config = ironclaw_embeddings::EmbeddingCacheConfig { max_entries: config.embeddings.cache_size, }; run_memory_command_with_db(mem_cmd.clone(), db, embeddings, cache_config).await diff --git a/src/cli/models.rs b/src/cli/models.rs index 39cb75a44d0..f7f3db173f3 100644 --- a/src/cli/models.rs +++ b/src/cli/models.rs @@ -8,8 +8,8 @@ use clap::Subcommand; use std::path::Path; -use crate::llm::registry::ProviderRegistry; use crate::settings::Settings; +use ironclaw_llm::registry::ProviderRegistry; #[derive(Subcommand, Debug, Clone)] pub enum ModelsCommand { @@ -164,44 +164,49 @@ async fn try_fetch_models(provider_id: &str, config_path: Option<&Path>) -> Opti let mut llm_config = config.llm.clone(); llm_config.backend = provider_id.to_string(); - // For registry providers, resolve the RegistryProviderConfig if not - // already set for this backend. - if provider_id != "nearai" && provider_id != "bedrock" { - let registry = ProviderRegistry::load(); - if let Some(def) = registry.find(provider_id) - && llm_config - .provider - .as_ref() - .is_none_or(|p| p.provider_id != def.id) - { - // Build a minimal RegistryProviderConfig from env + registry - let api_key = def - .api_key_env - .as_ref() - .and_then(|env| std::env::var(env).ok()); - if def.api_key_required && api_key.is_none() { - return None; - } - let base_url = def.default_base_url.clone().unwrap_or_default(); - llm_config.provider = Some(crate::llm::RegistryProviderConfig { - protocol: def.protocol, - provider_id: def.id.clone(), - model: def.default_model.clone(), - api_key: api_key.map(secrecy::SecretString::from), - base_url, - extra_headers: Vec::new(), - oauth_token: None, - is_codex_chatgpt: false, - refresh_token: None, - auth_path: None, - cache_retention: Default::default(), - unsupported_params: def.unsupported_params.clone(), - }); + // For OpenAI-shape registry providers, resolve the RegistryProviderConfig + // if not already set for this backend. Backends with a dedicated config + // slot (nearai/bedrock/codex/gemini_oauth) read from their own + // `LlmConfig.{nearai,bedrock,openai_codex,gemini_oauth}` sub-struct + // instead of `LlmConfig.provider`, so they're skipped here. + let registry = ProviderRegistry::load(); + let needs_registry_provider_config = registry + .find(provider_id) + .is_some_and(|def| !def.protocol.has_dedicated_config()); + if needs_registry_provider_config + && let Some(def) = registry.find(provider_id) + && llm_config + .provider + .as_ref() + .is_none_or(|p| p.provider_id != def.id) + { + // Build a minimal RegistryProviderConfig from env + registry + let api_key = def + .api_key_env + .as_ref() + .and_then(|env| std::env::var(env).ok()); + if def.api_key_required && api_key.is_none() { + return None; } + let base_url = def.default_base_url.clone().unwrap_or_default(); + llm_config.provider = Some(ironclaw_llm::RegistryProviderConfig { + protocol: def.protocol, + provider_id: def.id.clone(), + model: def.default_model.clone(), + api_key: api_key.map(secrecy::SecretString::from), + base_url, + extra_headers: Vec::new(), + oauth_token: None, + is_codex_chatgpt: false, + refresh_token: None, + auth_path: None, + cache_retention: Default::default(), + unsupported_params: def.unsupported_params.clone(), + }); } - let session = crate::llm::create_session_manager(config.llm.session.clone()).await; - let provider = crate::llm::create_llm_provider(&llm_config, session) + let session = ironclaw_llm::create_session_manager(config.llm.session.clone()).await; + let provider = ironclaw_llm::create_llm_provider(&llm_config, session) .await .ok()?; provider.list_models().await.ok().filter(|m| !m.is_empty()) diff --git a/src/config/channels.rs b/src/config/channels.rs index e8050d0f707..04bc3f5a7ad 100644 --- a/src/config/channels.rs +++ b/src/config/channels.rs @@ -33,6 +33,10 @@ pub struct ChannelsConfig { /// Per-channel owner user IDs. When set, the channel only responds to this user. /// Key: channel name (e.g., "telegram"), Value: owner user ID. pub wasm_channel_owner_ids: HashMap<String, i64>, + /// Runtime config overrides for WASM channels. + /// + /// Key format: `<channel>:<config_key>` (e.g. `wecom:allow_from`). + pub wasm_channel_runtime_overrides: HashMap<String, serde_json::Value>, } #[derive(Debug, Clone)] @@ -562,6 +566,7 @@ impl ChannelsConfig { } ids }, + wasm_channel_runtime_overrides: cs.wasm_channel_runtime_overrides.clone(), }) } } @@ -744,6 +749,7 @@ mod tests { wasm_channels_enabled: true, configured_wasm_channels: Vec::new(), wasm_channel_owner_ids: HashMap::new(), + wasm_channel_runtime_overrides: HashMap::new(), }; assert!(cfg.cli.enabled); assert!(cfg.http.is_none()); @@ -752,6 +758,7 @@ mod tests { assert_eq!(cfg.wasm_channels_dir, PathBuf::from("/tmp/channels")); assert!(cfg.wasm_channels_enabled); assert!(cfg.wasm_channel_owner_ids.is_empty()); + assert!(cfg.wasm_channel_runtime_overrides.is_empty()); } #[test] @@ -771,6 +778,7 @@ mod tests { wasm_channels_enabled: false, configured_wasm_channels: vec!["telegram".to_string()], wasm_channel_owner_ids: ids, + wasm_channel_runtime_overrides: HashMap::new(), }; assert_eq!(cfg.wasm_channel_owner_ids.get("telegram"), Some(&12345)); assert_eq!(cfg.wasm_channel_owner_ids.get("slack"), Some(&67890)); diff --git a/src/config/embeddings.rs b/src/config/embeddings.rs index 59a95f301f2..457e93287cd 100644 --- a/src/config/embeddings.rs +++ b/src/config/embeddings.rs @@ -1,273 +1,112 @@ -use std::sync::Arc; +//! Resolver that builds an `EmbeddingsConfig` from binary-side `Settings`. +//! +//! The `EmbeddingsConfig` data shape, factory, cache, and providers all live +//! in the `ironclaw_embeddings` crate. Only the bit that reads +//! `crate::settings::Settings` (a binary-internal type) and validates +//! env-driven base URLs against the SSRF blocklist (also binary-internal) +//! stays here. -use secrecy::{ExposeSecret, SecretString}; +use secrecy::SecretString; use crate::config::helpers::{ db_first_bool, db_first_or_default, optional_env, parse_optional_env, validate_operator_base_url, }; use crate::error::ConfigError; -use crate::llm::{BedrockConfig, SessionManager}; use crate::settings::Settings; -use crate::workspace::EmbeddingProvider; - -/// Default maximum number of cached embeddings. -pub const DEFAULT_EMBEDDING_CACHE_SIZE: usize = 10_000; - -/// Embeddings provider configuration. -#[derive(Debug, Clone)] -pub struct EmbeddingsConfig { - /// Whether embeddings are enabled. - pub enabled: bool, - /// Provider to use: "openai", "nearai", "ollama", or "bedrock" - pub provider: String, - /// OpenAI API key (for OpenAI provider). - pub openai_api_key: Option<SecretString>, - /// Model to use for embeddings. - pub model: String, - /// Ollama base URL (for Ollama provider). Defaults to http://localhost:11434. - pub ollama_base_url: String, - /// Embedding vector dimension. Inferred from the model name when not set explicitly. - pub dimension: usize, - /// Custom base URL for OpenAI-compatible embedding providers. - /// When set, overrides the default `https://api.openai.com`. - pub openai_base_url: Option<String>, - /// Maximum entries in the embedding LRU cache (default 10,000). - /// - /// Approximate raw embedding payload: `cache_size × dimension × 4 bytes`. - /// 10,000 × 1536 floats ≈ 58 MB (payload only; actual memory is higher - /// due to HashMap buckets, per-entry Vec/timestamp overhead). - pub cache_size: usize, -} - -impl Default for EmbeddingsConfig { - fn default() -> Self { - let model = "text-embedding-3-small".to_string(); - let dimension = default_dimension_for_model(&model); - Self { - enabled: false, - provider: "openai".to_string(), - openai_api_key: None, - model, - ollama_base_url: "http://localhost:11434".to_string(), - dimension, - openai_base_url: None, - cache_size: DEFAULT_EMBEDDING_CACHE_SIZE, - } - } -} +use ironclaw_embeddings::{ + DEFAULT_EMBEDDING_CACHE_SIZE, EmbeddingsConfig, default_dimension_for_model, +}; -/// Infer the embedding dimension from a well-known model name. +/// Resolve embeddings configuration from settings, env vars, and defaults. /// -/// Falls back to 1536 (OpenAI text-embedding-3-small default) for unknown models. -pub(crate) fn default_dimension_for_model(model: &str) -> usize { - match model { - "text-embedding-3-small" => 1536, - "text-embedding-3-large" => 3072, - "text-embedding-ada-002" => 1536, - "amazon.titan-embed-text-v2:0" => 1024, - "nomic-embed-text" => 768, - "mxbai-embed-large" => 1024, - "all-minilm" => 384, - _ => 1536, +/// Precedence: DB/TOML settings > env > default. +/// +/// `nearai_base_url` is copied from `LlmConfig::nearai::base_url` by the +/// caller so embeddings share the LLM's NEAR AI endpoint rather than +/// duplicate the config knob. +pub(crate) fn resolve_embeddings_config( + settings: &Settings, + nearai_base_url: &str, +) -> Result<EmbeddingsConfig, ConfigError> { + let defaults = crate::settings::EmbeddingsSettings::default(); + + let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from); + + let provider = db_first_or_default( + &settings.embeddings.provider, + &defaults.provider, + "EMBEDDING_PROVIDER", + )?; + + let model = if provider == "bedrock" { + optional_env("EMBEDDING_MODEL")? + .unwrap_or_else(|| "amazon.titan-embed-text-v2:0".to_string()) + } else { + db_first_or_default( + &settings.embeddings.model, + &defaults.model, + "EMBEDDING_MODEL", + )? + }; + + // ollama_base_url lives on the top-level Settings, not the embeddings + // sub-struct. Use a manual DB > env > default chain. + let default_ollama_url = "http://localhost:11434".to_string(); + let ollama_base_url = match settings + .ollama_base_url + .as_ref() + .filter(|s| !s.is_empty()) + .cloned() + { + Some(url) => url, + None => optional_env("OLLAMA_BASE_URL")?.unwrap_or(default_ollama_url), + }; + + // Dimension depends on the resolved model, not on a DB setting — env-only. + let dimension = parse_optional_env("EMBEDDING_DIMENSION", default_dimension_for_model(&model))?; + if provider == "bedrock" && !matches!(dimension, 256 | 512 | 1024) { + return Err(ConfigError::InvalidValue { + key: "EMBEDDING_DIMENSION".to_string(), + message: "Bedrock Titan v2 embeddings support only 256, 512, or 1024 dimensions" + .to_string(), + }); } -} - -impl EmbeddingsConfig { - pub(crate) fn resolve(settings: &Settings) -> Result<Self, ConfigError> { - let defaults = crate::settings::EmbeddingsSettings::default(); - - let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from); - - let provider = db_first_or_default( - &settings.embeddings.provider, - &defaults.provider, - "EMBEDDING_PROVIDER", - )?; - - let model = if provider == "bedrock" { - optional_env("EMBEDDING_MODEL")? - .unwrap_or_else(|| "amazon.titan-embed-text-v2:0".to_string()) - } else { - db_first_or_default( - &settings.embeddings.model, - &defaults.model, - "EMBEDDING_MODEL", - )? - }; - - // ollama_base_url lives on the top-level Settings, not the embeddings - // sub-struct. Use a manual DB > env > default chain. - let default_ollama_url = "http://localhost:11434".to_string(); - let ollama_base_url = match settings - .ollama_base_url - .as_ref() - .filter(|s| !s.is_empty()) - .cloned() - { - Some(url) => url, - None => optional_env("OLLAMA_BASE_URL")?.unwrap_or(default_ollama_url), - }; - // Dimension depends on the resolved model, not on a DB setting — env-only. - let dimension = - parse_optional_env("EMBEDDING_DIMENSION", default_dimension_for_model(&model))?; - if provider == "bedrock" && !matches!(dimension, 256 | 512 | 1024) { - return Err(ConfigError::InvalidValue { - key: "EMBEDDING_DIMENSION".to_string(), - message: "Bedrock Titan v2 embeddings support only 256, 512, or 1024 dimensions" - .to_string(), - }); - } - - let enabled = db_first_bool( - settings.embeddings.enabled, - defaults.enabled, - "EMBEDDING_ENABLED", - )?; - - let openai_base_url = optional_env("EMBEDDING_BASE_URL")?; - - // Validate base URLs to prevent SSRF attacks (#1103). - validate_operator_base_url(&ollama_base_url, "OLLAMA_BASE_URL")?; - if let Some(ref url) = openai_base_url { - validate_operator_base_url(url, "EMBEDDING_BASE_URL")?; - } - - let cache_size = parse_optional_env("EMBEDDING_CACHE_SIZE", DEFAULT_EMBEDDING_CACHE_SIZE)?; + let enabled = db_first_bool( + settings.embeddings.enabled, + defaults.enabled, + "EMBEDDING_ENABLED", + )?; - if cache_size == 0 { - return Err(ConfigError::InvalidValue { - key: "EMBEDDING_CACHE_SIZE".to_string(), - message: "must be at least 1".to_string(), - }); - } - - Ok(Self { - enabled, - provider, - openai_api_key, - model, - ollama_base_url, - dimension, - openai_base_url, - cache_size, - }) - } + let openai_base_url = optional_env("EMBEDDING_BASE_URL")?; - /// Get the OpenAI API key if configured. - pub fn openai_api_key(&self) -> Option<&str> { - self.openai_api_key.as_ref().map(|s| s.expose_secret()) + // Validate base URLs to prevent SSRF attacks (#1103). + validate_operator_base_url(&ollama_base_url, "OLLAMA_BASE_URL")?; + if let Some(ref url) = openai_base_url { + validate_operator_base_url(url, "EMBEDDING_BASE_URL")?; } - /// Create the appropriate embedding provider based on configuration. - /// - /// Returns `None` if embeddings are disabled or the required credentials - /// are missing. The `nearai_base_url` and `session` are needed only for - /// the NEAR AI provider but must be passed unconditionally. - pub async fn create_provider( - &self, - nearai_base_url: &str, - session: Arc<SessionManager>, - bedrock_config: Option<&BedrockConfig>, - ) -> Option<Arc<dyn EmbeddingProvider>> { - if !self.enabled { - tracing::debug!("Embeddings disabled (set EMBEDDING_ENABLED=true to enable)"); - return None; - } + let cache_size = parse_optional_env("EMBEDDING_CACHE_SIZE", DEFAULT_EMBEDDING_CACHE_SIZE)?; - match self.provider.as_str() { - "nearai" => { - tracing::debug!( - "Embeddings enabled via NEAR AI (model: {}, dim: {})", - self.model, - self.dimension, - ); - Some(Arc::new( - crate::workspace::NearAiEmbeddings::new(nearai_base_url, session) - .with_model(&self.model, self.dimension), - )) - } - "bedrock" => { - #[cfg(feature = "bedrock")] - { - let Some(bedrock) = bedrock_config else { - tracing::warn!( - "Embeddings configured for Bedrock but no Bedrock config is available" - ); - return None; - }; - tracing::debug!( - "Embeddings enabled via Bedrock (model: {}, region: {}, dim: {})", - self.model, - bedrock.region, - self.dimension, - ); - match crate::workspace::BedrockEmbeddings::new( - bedrock, - &self.model, - self.dimension, - ) - .await - { - Ok(provider) => Some(Arc::new(provider) as Arc<dyn EmbeddingProvider>), - Err(e) => { - tracing::warn!("Failed to initialize Bedrock embeddings provider: {e}"); - None - } - } - } - #[cfg(not(feature = "bedrock"))] - { - let _ = bedrock_config; - tracing::warn!( - "Embeddings configured for Bedrock but the `bedrock` feature is disabled" - ); - None - } - } - "ollama" => { - tracing::debug!( - "Embeddings enabled via Ollama (model: {}, url: {}, dim: {})", - self.model, - self.ollama_base_url, - self.dimension, - ); - Some(Arc::new( - crate::workspace::OllamaEmbeddings::new(&self.ollama_base_url) - .with_model(&self.model, self.dimension), - )) - } - _ => { - if let Some(api_key) = self.openai_api_key() { - let mut provider = crate::workspace::OpenAiEmbeddings::with_model( - api_key, - &self.model, - self.dimension, - ); - if let Some(ref base_url) = self.openai_base_url { - tracing::debug!( - "Embeddings enabled via OpenAI (model: {}, base_url: {}, dim: {})", - self.model, - base_url, - self.dimension, - ); - provider = provider.with_base_url(base_url); - } else { - tracing::debug!( - "Embeddings enabled via OpenAI (model: {}, dim: {})", - self.model, - self.dimension, - ); - } - Some(Arc::new(provider)) - } else { - tracing::warn!("Embeddings configured but OPENAI_API_KEY not set"); - None - } - } - } + if cache_size == 0 { + return Err(ConfigError::InvalidValue { + key: "EMBEDDING_CACHE_SIZE".to_string(), + message: "must be at least 1".to_string(), + }); } + + Ok(EmbeddingsConfig { + enabled, + provider, + openai_api_key, + model, + ollama_base_url, + dimension, + openai_base_url, + nearai_base_url: nearai_base_url.to_string(), + cache_size, + }) } #[cfg(test)] @@ -309,7 +148,8 @@ mod tests { ..Default::default() }; - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert!( !config.enabled, "embeddings should remain disabled when settings.embeddings.enabled=false, \ @@ -335,7 +175,8 @@ mod tests { ..Default::default() }; - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert!( config.enabled, "embeddings should be enabled when settings say so" @@ -362,7 +203,8 @@ mod tests { ..Default::default() }; - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert!( config.enabled, "DB enabled=true should win over env EMBEDDING_ENABLED=false" @@ -395,7 +237,8 @@ mod tests { // Settings left at defaults — no explicit DB/TOML override let settings = Settings::default(); - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert!( config.enabled, "env EMBEDDING_ENABLED should be used when settings at default" @@ -428,7 +271,8 @@ mod tests { } let settings = Settings::default(); - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert_eq!(config.openai_base_url.as_deref(), Some("https://8.8.8.8")); // SAFETY: Under ENV_MUTEX. unsafe { @@ -442,7 +286,8 @@ mod tests { clear_embedding_env(); let settings = Settings::default(); - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert!( config.openai_base_url.is_none(), "openai_base_url should be None when EMBEDDING_BASE_URL is not set" @@ -459,7 +304,7 @@ mod tests { } let settings = Settings::default(); - let result = EmbeddingsConfig::resolve(&settings); + let result = resolve_embeddings_config(&settings, "https://api.near.ai"); assert!(result.is_err(), "cache_size=0 should be rejected"); let err = result.unwrap_err().to_string(); assert!(err.contains("at least 1"), "should mention minimum: {err}"); @@ -481,7 +326,8 @@ mod tests { } let settings = Settings::default(); - let config = EmbeddingsConfig::resolve(&settings).expect("resolve should succeed"); + let config = resolve_embeddings_config(&settings, "https://api.near.ai") + .expect("resolve should succeed"); assert_eq!(config.provider, "bedrock"); assert_eq!(config.model, "amazon.titan-embed-text-v2:0"); assert_eq!(config.dimension, 1024); @@ -506,7 +352,7 @@ mod tests { } let settings = Settings::default(); - let result = EmbeddingsConfig::resolve(&settings); + let result = resolve_embeddings_config(&settings, "https://api.near.ai"); assert!( result.is_err(), "unsupported bedrock dimensions should fail" diff --git a/src/config/helpers.rs b/src/config/helpers.rs index 8880730d684..70b20720b8f 100644 --- a/src/config/helpers.rs +++ b/src/config/helpers.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::{Mutex, OnceLock}; +use std::sync::Mutex; use crate::error::ConfigError; @@ -7,21 +7,15 @@ use crate::config::INJECTED_VARS; /// Crate-wide mutex for tests that mutate process environment variables. /// -/// The process environment is global state shared across all threads. -/// Per-module mutexes do NOT prevent races between modules running in -/// parallel. Every `unsafe { set_var / remove_var }` call in tests -/// MUST hold this single lock. -#[cfg(test)] -pub(crate) static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(()); - -/// Acquire the env-var mutex, recovering from poison. +/// Acquire the workspace-wide env-var mutex, recovering from poison. /// -/// A poisoned mutex means a previous test panicked while holding the lock. -/// The env state might be slightly stale, but cascading every subsequent -/// test into a `PoisonError` panic is far worse. Recover and carry on. +/// Delegates to [`ironclaw_common::env_helpers::lock_env`] so tests across +/// every crate (`ironclaw`, `ironclaw_llm`, `ironclaw_common`) serialize on +/// the same `Mutex`. Per-module mutexes would not prevent races between +/// modules running in parallel. #[cfg(test)] pub(crate) fn lock_env() -> std::sync::MutexGuard<'static, ()> { - ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()) + ironclaw_common::env_helpers::lock_env() } /// Thread-safe mutable overlay for env vars set at runtime. @@ -32,52 +26,27 @@ pub(crate) fn lock_env() -> std::sync::MutexGuard<'static, ()> { /// otherwise be UB in multi-threaded programs (Rust 1.82+). /// /// Priority: real env vars > `RUNTIME_ENV_OVERRIDES` > `INJECTED_VARS`. -static RUNTIME_ENV_OVERRIDES: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new(); - -fn runtime_overrides() -> &'static Mutex<HashMap<String, String>> { - RUNTIME_ENV_OVERRIDES.get_or_init(|| Mutex::new(HashMap::new())) -} - /// Set a runtime environment override (thread-safe alternative to `std::env::set_var`). /// -/// Values set here are visible to `optional_env()`, `env_or_override()`, and -/// all config resolution that goes through those helpers. This avoids the UB -/// of `std::env::set_var` in multi-threaded programs. +/// Delegates to `ironclaw_common::env_helpers::set_runtime_env` so the +/// override is visible to both this crate and `ironclaw_llm` (which reads +/// the same overlay through `ironclaw_common::env_helpers::env_or_override`). pub fn set_runtime_env(key: &str, value: &str) { - runtime_overrides() - .lock() - .unwrap_or_else(|e| e.into_inner()) - .insert(key.to_string(), value.to_string()); + ironclaw_common::env_helpers::set_runtime_env(key, value); } -/// Read an env var, checking the real environment first, then runtime overrides. +/// Read an env var, checking real env first, then the shared runtime +/// overlay, then this crate's `INJECTED_VARS` (secrets injected from DB). /// /// Priority: real env vars > runtime overrides > `INJECTED_VARS`. -/// Empty values are treated as unset at every layer for consistency with -/// `optional_env()`. -/// -/// Use this instead of `std::env::var()` when the value might have been set -/// via `set_runtime_env()` (e.g., `NEARAI_API_KEY` during interactive login). +/// Empty values are treated as unset at every layer. pub fn env_or_override(key: &str) -> Option<String> { - // Real env vars always win - if let Ok(val) = std::env::var(key) - && !val.is_empty() - { - return Some(val); - } - - // Check runtime overrides (skip empty values for consistency with optional_env) - if let Some(val) = runtime_overrides() - .lock() - .unwrap_or_else(|e| e.into_inner()) - .get(key) - .filter(|v| !v.is_empty()) - .cloned() - { + // Real env + runtime overlay (shared with `ironclaw_llm`) + if let Some(val) = ironclaw_common::env_helpers::env_or_override(key) { return Some(val); } - // Check INJECTED_VARS (secrets from DB, set once at startup) + // Main-crate-only INJECTED_VARS overlay (DB-loaded secrets). if let Some(val) = INJECTED_VARS .lock() .unwrap_or_else(|e| e.into_inner()) @@ -104,14 +73,9 @@ pub(crate) fn optional_env(key: &str) -> Result<Option<String>, ConfigError> { } } - // Fall back to runtime overrides (set via set_runtime_env) - if let Some(val) = runtime_overrides() - .lock() - .unwrap_or_else(|e| e.into_inner()) - .get(key) - .filter(|v| !v.is_empty()) - .cloned() - { + // Fall back to the shared runtime overrides (set via set_runtime_env; + // also reachable from `ironclaw_llm` via `ironclaw_common::env_helpers`). + if let Some(val) = ironclaw_common::env_helpers::env_or_override(key) { return Ok(Some(val)); } @@ -201,11 +165,15 @@ pub(crate) const ADMIN_ONLY_LLM_SETTING_KEYS: &[&str] = &[ "ollama_base_url", "openai_compatible_base_url", // Provider-selection keys — every member shares one LLM provider chain, - // so the choice of backend and the provider-specific endpoint knobs - // (Bedrock region / cross-region prefix / AWS profile) must be gated - // to admins. Members can still pick their own model via `selected_model`, - // which is intentionally NOT in this list. + // so the choice of backend must be gated to admins. Members can still + // pick their own model via `selected_model`, which is intentionally + // NOT in this list. "llm_backend", + // Legacy bedrock keys retained for backward-compat with + // settings.json files written before Layer D moved this config into + // `llm_builtin_overrides["bedrock"].extras` (which is already covered + // by the `llm_builtin_overrides` prefix-strip below). New code does + // not write to these. "bedrock_region", "bedrock_cross_region", "bedrock_profile", @@ -216,8 +184,31 @@ pub(crate) const ADMIN_ONLY_LLM_SETTING_KEYS: &[&str] = &[ /// Used by config resolution paths that load per-user DB settings for a /// non-operator user, to ensure they cannot inject private/loopback /// provider endpoints into the active LLM/embeddings configuration. +/// +/// Strips both: +/// - exact-match keys (e.g. `llm_backend`, `bedrock_region`), and +/// - any key whose dotted path starts with one of the admin-only roots +/// followed by a `.` (e.g. `llm_builtin_overrides.bedrock.api_key`, +/// `llm_builtin_overrides.bedrock.extras.region`). pub(crate) fn strip_admin_only_llm_keys(map: &mut HashMap<String, serde_json::Value>) { - map.retain(|key, _| !ADMIN_ONLY_LLM_SETTING_KEYS.contains(&key.as_str())); + map.retain(|key, _| !is_admin_only_llm_key(key)); +} + +/// Single source of truth for "is this setting key admin-only?". +/// +/// Matches both exact keys in [`ADMIN_ONLY_LLM_SETTING_KEYS`] and dotted +/// subpaths under those roots (e.g. `llm_builtin_overrides.bedrock.extras +/// .region`). The write-side gate in `channels::web::features::settings` +/// must call this rather than re-implementing `.contains(key)`, or a +/// non-admin can sneak past the gate by addressing the same value +/// through a dotted subpath. +pub(crate) fn is_admin_only_llm_key(key: &str) -> bool { + if ADMIN_ONLY_LLM_SETTING_KEYS.contains(&key) { + return true; + } + ADMIN_ONLY_LLM_SETTING_KEYS.iter().any(|root| { + key.len() > root.len() + 1 && key.starts_with(root) && key.as_bytes()[root.len()] == b'.' + }) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -684,6 +675,8 @@ mod tests { #[test] fn lock_env_recovers_from_poisoned_mutex() { + use ironclaw_common::env_helpers::ENV_MUTEX; + // Simulate a poisoned mutex: spawn a thread that panics while holding the lock. let _ = std::thread::spawn(|| { let _guard = ENV_MUTEX.lock().unwrap(); diff --git a/src/config/llm.rs b/src/config/llm.rs index 0e0a88cddc7..c292e6594a5 100644 --- a/src/config/llm.rs +++ b/src/config/llm.rs @@ -9,871 +9,813 @@ use crate::config::helpers::{ validate_operator_base_url, }; use crate::error::ConfigError; -use crate::llm::config::*; -use crate::llm::registry::{ProviderProtocol, ProviderRegistry}; -use crate::llm::session::SessionConfig; use crate::settings::Settings; +use ironclaw_llm::config::*; +use ironclaw_llm::registry::{ProviderProtocol, ProviderRegistry}; +use ironclaw_llm::session::SessionConfig; static LOG_LLM_BACKEND_RESOLUTION: Once = Once::new(); -impl LlmConfig { - fn selected_model_override(settings: &Settings) -> Option<String> { - crate::llm::normalized_model_override(settings.selected_model.as_deref()) - .map(str::to_string) - } +fn selected_model_override(settings: &Settings) -> Option<String> { + ironclaw_llm::normalized_model_override(settings.selected_model.as_deref()).map(str::to_string) +} - /// Create a test-friendly config without reading env vars. - #[cfg(feature = "libsql")] - pub fn for_testing() -> Self { - Self { - backend: "nearai".to_string(), - session: SessionConfig { - auth_base_url: "http://localhost:0".to_string(), - session_path: std::env::temp_dir().join("ironclaw-test-session.json"), - }, - nearai: NearAiConfig { - model: "test-model".to_string(), - cheap_model: None, - base_url: "http://localhost:0".to_string(), - api_key: None, - fallback_model: None, - max_retries: 0, - circuit_breaker_threshold: None, - circuit_breaker_recovery_secs: 30, - response_cache_enabled: false, - response_cache_ttl_secs: 3600, - response_cache_max_entries: 100, - failover_cooldown_secs: 300, - failover_cooldown_threshold: 3, - smart_routing_cascade: false, - }, - provider: None, - bedrock: None, - gemini_oauth: None, - openai_codex: None, - request_timeout_secs: 120, +/// Create a test-friendly config without reading env vars. +#[cfg(feature = "libsql")] +pub fn for_testing() -> LlmConfig { + LlmConfig { + backend: "nearai".to_string(), + session: SessionConfig { + auth_base_url: "http://localhost:0".to_string(), + session_path: std::env::temp_dir().join("ironclaw-test-session.json"), + }, + nearai: NearAiConfig { + model: "test-model".to_string(), cheap_model: None, - smart_routing_cascade: false, + base_url: "http://localhost:0".to_string(), + api_key: None, + fallback_model: None, max_retries: 0, circuit_breaker_threshold: None, circuit_breaker_recovery_secs: 30, response_cache_enabled: false, response_cache_ttl_secs: 3600, response_cache_max_entries: 100, - } + failover_cooldown_secs: 300, + failover_cooldown_threshold: 3, + smart_routing_cascade: false, + }, + provider: None, + bedrock: None, + gemini_oauth: None, + openai_codex: None, + request_timeout_secs: 120, + cheap_model: None, + smart_routing_cascade: false, + max_retries: 0, + circuit_breaker_threshold: None, + circuit_breaker_recovery_secs: 30, + response_cache_enabled: false, + response_cache_ttl_secs: 3600, + response_cache_max_entries: 100, } +} - /// Resolve a model name from settings.selected_model -> env var -> hardcoded default. - fn resolve_model( - env_var: &str, - settings: &Settings, - default: &str, - ) -> Result<String, ConfigError> { - if let Some(model) = Self::selected_model_override(settings) { - Ok(model) - } else if let Some(model) = optional_env(env_var)? { - Ok(model) - } else { - Ok(default.to_string()) - } - } - - /// Resolve LLM configuration, with NearAI fallback for unusable configs. - /// - /// This entry point is for the **final** resolve after secrets have been - /// hydrated from the encrypted store. If the user-configured backend is - /// not usable (missing API key, missing base URL), we fall back to NearAI - /// rather than crashing — this prevents the #2514 crash-loop when a user - /// activates a provider via the UI without completing all required - /// fields. - /// - /// Do NOT call this during early startup (`Config::build()`) when - /// secrets are not yet hydrated — use [`resolve`] instead, otherwise - /// the fallback fires spuriously and gets overridden by the later - /// re-resolve, spamming operators with misleading error logs. - pub(crate) fn resolve_with_fallback(settings: &Settings) -> Result<Self, ConfigError> { - match Self::resolve(settings) { - Ok(cfg) => { - if let Some(reason) = Self::unusable_reason(&cfg, settings) { - tracing::error!( - backend = %cfg.backend, - reason = %reason, - "Configured LLM backend is not usable. Falling back to NearAI default. \ - Reconfigure in Settings → Inference → Model Providers." - ); - Self::resolve_nearai_fallback(settings, &cfg.backend) - } else { - Ok(cfg) - } - } - Err(e) if Self::is_fallback_recoverable(&e) => { +/// Resolve LLM configuration, with NearAI fallback for unusable configs. +/// +/// This entry point is for the **final** resolve after secrets have been +/// hydrated from the encrypted store. If the user-configured backend is +/// not usable (missing API key, missing base URL), we fall back to NearAI +/// rather than crashing — this prevents the #2514 crash-loop when a user +/// activates a provider via the UI without completing all required +/// fields. +/// +/// Do NOT call this during early startup (`Config::build()`) when +/// secrets are not yet hydrated — use [`resolve`] instead, otherwise +/// the fallback fires spuriously and gets overridden by the later +/// re-resolve, spamming operators with misleading error logs. +pub fn resolve_with_fallback(settings: &Settings) -> Result<LlmConfig, ConfigError> { + match resolve(settings) { + Ok(cfg) => { + if let Some(reason) = unusable_reason(&cfg, settings) { tracing::error!( - error = %e, - configured_backend = ?settings.llm_backend, - "Failed to resolve configured LLM backend. Falling back to NearAI default. \ + backend = %cfg.backend, + reason = %reason, + "Configured LLM backend is not usable. Falling back to NearAI default. \ Reconfigure in Settings → Inference → Model Providers." ); - let attempted = settings - .llm_backend - .clone() - .unwrap_or_else(|| "<unset>".to_string()); - Self::resolve_nearai_fallback(settings, &attempted) + resolve_nearai_fallback(settings, &cfg.backend) + } else { + Ok(cfg) } - Err(e) => Err(e), - } - } - - /// If the resolved config's LLM provider is unusable, return a short - /// reason string. `None` means the config is fine to use. Returns `None` - /// for special backends that don't use the `provider` slot - /// (nearai/bedrock/codex/gemini_oauth) — those have their own validation - /// inside `resolve_once`. - /// - /// This check is deliberately narrow: it only flags configurations that - /// will certainly fail at runtime. Built-in providers like Anthropic may - /// legitimately have an empty `base_url` at config time because the - /// downstream rig-core client hardcodes the canonical endpoint. - fn unusable_reason(cfg: &Self, settings: &Settings) -> Option<&'static str> { - let provider = cfg.provider.as_ref()?; - - let is_custom = settings - .llm_custom_providers - .iter() - .any(|c| c.id == provider.provider_id); - let is_ollama = matches!(provider.protocol, ProviderProtocol::Ollama); - - // Custom providers have no hardcoded base URL in the client layer — - // an empty `base_url` here means requests will be sent to a bare - // path with no host, which always fails. - if is_custom && provider.base_url.trim().is_empty() { - return Some("missing base URL"); - } - - // Ollama runs locally and has no API key concept. Every other - // provider needs at least one form of authentication. - if !is_ollama - && provider.api_key.is_none() - && provider.oauth_token.is_none() - && provider.refresh_token.is_none() - { - return Some("missing API key"); } - - None - } - - /// Errors that indicate a fixable user-config problem (as opposed to a - /// programming bug or environmental failure). We fall back to NearAI on - /// these so the instance can still start. - fn is_fallback_recoverable(err: &ConfigError) -> bool { - matches!(err, ConfigError::MissingRequired { .. }) - } - - /// Re-resolve with `llm_backend` forced to `"nearai"`. - /// `attempted_backend` is only used for logging so operators can see which - /// backend we bailed out of. - fn resolve_nearai_fallback( - settings: &Settings, - attempted_backend: &str, - ) -> Result<Self, ConfigError> { - let mut fallback = settings.clone(); - fallback.llm_backend = Some("nearai".to_string()); - // The previously-selected model was tied to the unusable backend - // (e.g. "openai/gpt-4o" for OpenRouter, "kimi-k2-turbo-preview" for - // a custom kimi provider). Sending it to NearAI would 404. Clear it - // so resolve_model falls through to NearAI's default. The DB sync in - // Config::re_resolve_llm_with_secrets deletes the row persistently; - // this keeps the in-memory config consistent for the current process. - fallback.selected_model = None; - let cfg = Self::resolve(&fallback).map_err(|e| { + Err(e) if is_fallback_recoverable(&e) => { tracing::error!( - attempted = %attempted_backend, - fallback_error = %e, - "NearAI fallback also failed to resolve — surfacing original error" + error = %e, + configured_backend = ?settings.llm_backend, + "Failed to resolve configured LLM backend. Falling back to NearAI default. \ + Reconfigure in Settings → Inference → Model Providers." ); - e - })?; - tracing::warn!( - attempted = %attempted_backend, - active = %cfg.backend, - active_model = %cfg.nearai.model, - "Active LLM backend fell back to NearAI default due to unusable user config" - ); - Ok(cfg) - } - - /// Resolve LLM configuration without any fallback behavior. - /// - /// Returns the config exactly as computed from env/DB/defaults, with no - /// safety net for missing credentials. Use this for: - /// - Early startup (before secrets are hydrated), so a spurious fallback - /// doesn't fire and get overridden by the later re-resolve. - /// - Tests that verify pure resolution mechanics (model/base_url priority - /// chains, alias normalization, etc.). - /// - /// The top-level `AppBuilder` path calls [`resolve_with_fallback`] after - /// hydrating secrets, which handles #2514-style crash-loop prevention. - pub(crate) fn resolve(settings: &Settings) -> Result<Self, ConfigError> { - let registry = ProviderRegistry::load(); - - // Determine backend: db settings > env var > default ("nearai") - let (backend, backend_source) = if let Some(ref b) = settings.llm_backend { - (b.clone(), "db:llm_backend") - } else if let Some(b) = optional_env("LLM_BACKEND")? { - (b, "env:LLM_BACKEND") - } else { - ("nearai".to_string(), "default") - }; - LOG_LLM_BACKEND_RESOLUTION.call_once(|| { - tracing::debug!( - backend = %backend, - source = %backend_source, - db_llm_backend = ?settings.llm_backend, - custom_providers_count = settings.llm_custom_providers.len(), - "Resolving LLM backend" - ); - }); - // Warn operators when a DB-persisted value silently overrides LLM_BACKEND. - // Skip the warning when both values are identical — this is the normal - // state after `ironclaw models set-provider`, which intentionally writes - // to both config.toml and .env for immediate effect. - if backend_source == "db:llm_backend" - && let Ok(env_val) = std::env::var("LLM_BACKEND") - && !env_val.is_empty() - && env_val != backend - { - tracing::warn!( - db_value = %backend, - env_value = %env_val, - "LLM_BACKEND env var is set but DB setting takes priority. \ - Unset llm_backend in the DB (via settings UI) to use the env var." - ); - } - - // Validate the backend is known - let backend_lower = backend.to_lowercase(); - let is_nearai = - backend_lower == "nearai" || backend_lower == "near_ai" || backend_lower == "near"; - let is_bedrock = - backend_lower == "bedrock" || backend_lower == "aws_bedrock" || backend_lower == "aws"; - let is_gemini_oauth = backend_lower == "gemini_oauth" || backend_lower == "gemini-oauth"; - let is_openai_codex = backend_lower == "openai_codex" - || backend_lower == "openai-codex" - || backend_lower == "codex"; - - // Check custom providers defined - let custom_provider = settings - .llm_custom_providers - .iter() - .find(|p| p.id.to_lowercase() == backend_lower); - - if !is_nearai - && !is_bedrock - && !is_gemini_oauth - && !is_openai_codex - && custom_provider.is_none() - && registry.find(&backend_lower).is_none() - { - tracing::warn!( - "Unknown LLM backend '{}'. Will attempt as openai_compatible fallback.", - backend - ); - } - - // Always resolve NEAR AI config (used for embeddings even when not the primary backend) - // Priority: DB (builtin_overrides) > env > default - let nearai_override = settings.llm_builtin_overrides.get("nearai"); - let nearai_override_has_base_url = - nearai_override.and_then(|o| o.base_url.as_ref()).is_some(); - - // Check whether NearAI embeddings are enabled. When they are, the - // NearAI base_url and auth_url are reachable code-paths and must pass - // SSRF validation even when the primary chat backend is not NearAI. - let emb_defaults = crate::settings::EmbeddingsSettings::default(); - let emb_provider = db_first_or_default( - &settings.embeddings.provider, - &emb_defaults.provider, - "EMBEDDING_PROVIDER", - )?; - let emb_enabled = db_first_bool( - settings.embeddings.enabled, - emb_defaults.enabled, - "EMBEDDING_ENABLED", - )?; - let nearai_embeddings_active = emb_enabled && emb_provider.eq_ignore_ascii_case("nearai"); - - // Predicate: NearAI URLs must be validated when: - // - NearAI is the primary chat backend, OR - // - the user/DB explicitly supplied the URL, OR - // - NEARAI_API_KEY is set (implies intent to use NearAI), OR - // - NearAI embeddings are enabled (they use the same base/auth URLs), OR - // - the DB builtin_overrides for NearAI include a base_url. - let nearai_api_key_env = optional_env("NEARAI_API_KEY")?; - - // Session config (used by NearAI provider for OAuth/session-token auth) - let nearai_auth_url_explicit = optional_env("NEARAI_AUTH_URL")?; - let nearai_auth_url = nearai_auth_url_explicit - .clone() - .unwrap_or_else(|| "https://private.near.ai".to_string()); - // Only validate NearAI URLs when NearAI is active or the user explicitly - // set the URL. Default URLs point to private.near.ai which requires DNS - // resolution — this blocks startup in environments without network access - // (CI runners, containers) when a different backend is configured. - if is_nearai - || nearai_auth_url_explicit.is_some() - || nearai_api_key_env.is_some() - || nearai_override_has_base_url - || nearai_embeddings_active - { - validate_base_url(&nearai_auth_url, "NEARAI_AUTH_URL")?; - } - let session = SessionConfig { - auth_base_url: nearai_auth_url, - session_path: optional_env("NEARAI_SESSION_PATH")? - .map(PathBuf::from) - .unwrap_or_else(default_session_path), - }; - - let nearai_api_key = if let Some(key) = nearai_override.and_then(|o| o.api_key.as_ref()) { - Some(SecretString::from(key.clone())) - } else { - nearai_api_key_env.map(SecretString::from) - }; - // Model priority: selected_model (DB) > builtin_overrides (DB) > env > default - let nearai_model = if let Some(model) = Self::selected_model_override(settings) { - model - } else if let Some(model) = nearai_override.and_then(|o| o.model.clone()) { - model - } else if let Some(model) = optional_env("NEARAI_MODEL")? { - model - } else { - crate::llm::DEFAULT_MODEL.to_string() - }; - let nearai_base_url_explicit = optional_env("NEARAI_BASE_URL")?; - let nearai_base_url = if let Some(url) = nearai_override.and_then(|o| o.base_url.clone()) { - url - } else if let Some(url) = nearai_base_url_explicit.clone() { - url - } else if nearai_api_key.is_some() { - "https://cloud-api.near.ai".to_string() - } else { - "https://private.near.ai".to_string() - }; - if is_nearai - || nearai_base_url_explicit.is_some() - || nearai_api_key.is_some() - || nearai_override_has_base_url - || nearai_embeddings_active - { - validate_base_url(&nearai_base_url, "NEARAI_BASE_URL")?; - } - let nearai = NearAiConfig { - model: nearai_model, - cheap_model: optional_env("NEARAI_CHEAP_MODEL")?, - base_url: nearai_base_url, - api_key: nearai_api_key, - fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?, - max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?, - circuit_breaker_threshold: optional_env("CIRCUIT_BREAKER_THRESHOLD")? - .map(|s| s.parse()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "CIRCUIT_BREAKER_THRESHOLD".to_string(), - message: format!("must be a positive integer: {e}"), - })?, - circuit_breaker_recovery_secs: parse_optional_env("CIRCUIT_BREAKER_RECOVERY_SECS", 30)?, - response_cache_enabled: parse_optional_env("RESPONSE_CACHE_ENABLED", false)?, - response_cache_ttl_secs: parse_optional_env("RESPONSE_CACHE_TTL_SECS", 3600)?, - response_cache_max_entries: parse_optional_env("RESPONSE_CACHE_MAX_ENTRIES", 1000)?, - failover_cooldown_secs: parse_optional_env("LLM_FAILOVER_COOLDOWN_SECS", 300)?, - failover_cooldown_threshold: parse_optional_env("LLM_FAILOVER_THRESHOLD", 3)?, - smart_routing_cascade: parse_optional_env("SMART_ROUTING_CASCADE", true)?, - }; - - // Resolve registry provider config (for non-NearAI, non-Bedrock, non-Gemini, non-Codex backends) - let provider = if is_nearai || is_bedrock || is_gemini_oauth || is_openai_codex { - None - } else if let Some(custom) = custom_provider { - Some(Self::resolve_custom_provider(custom, settings)?) - } else { - Some(Self::resolve_registry_provider( - &backend_lower, - &registry, - settings, - )?) - }; - - let bedrock = if is_bedrock { - let explicit_region = settings - .bedrock_region - .clone() - .or(optional_env("BEDROCK_REGION")?); - if explicit_region.is_none() { - tracing::debug!("BEDROCK_REGION not set, defaulting to us-east-1"); - } - let region = explicit_region.unwrap_or_else(|| "us-east-1".to_string()); - let model = Self::selected_model_override(settings) - .or(optional_env("BEDROCK_MODEL")?) - .ok_or_else(|| ConfigError::MissingRequired { - key: "BEDROCK_MODEL".to_string(), - hint: "Set BEDROCK_MODEL or selected_model when LLM_BACKEND=bedrock" - .to_string(), - })?; - let cross_region = settings - .bedrock_cross_region + let attempted = settings + .llm_backend .clone() - .or(optional_env("BEDROCK_CROSS_REGION")?); - if let Some(ref cr) = cross_region - && !matches!(cr.as_str(), "us" | "eu" | "apac" | "global") - { - return Err(ConfigError::InvalidValue { - key: "BEDROCK_CROSS_REGION".to_string(), - message: format!( - "'{}' is not valid, expected one of: us, eu, apac, global", - cr - ), - }); - } - let profile = settings - .bedrock_profile - .clone() - .or(optional_env("AWS_PROFILE")?); - Some(BedrockConfig { - region, - model, - cross_region, - profile, - }) - } else { - None - }; - - // Resolve OpenAI Codex config - let openai_codex = if is_openai_codex { - // Model: settings.selected_model > OPENAI_CODEX_MODEL > OPENAI_MODEL > default - let model = Self::selected_model_override(settings) - .or(optional_env("OPENAI_CODEX_MODEL")?) - .or(optional_env("OPENAI_MODEL")?) - .unwrap_or_else(|| "gpt-5.3-codex".to_string()); - let auth_endpoint = optional_env("OPENAI_CODEX_AUTH_URL")? - .unwrap_or_else(|| "https://auth.openai.com".to_string()); - validate_base_url(&auth_endpoint, "OPENAI_CODEX_AUTH_URL")?; - let api_base_url = optional_env("OPENAI_CODEX_API_URL")? - .unwrap_or_else(|| "https://chatgpt.com/backend-api/codex".to_string()); - validate_base_url(&api_base_url, "OPENAI_CODEX_API_URL")?; - let client_id = optional_env("OPENAI_CODEX_CLIENT_ID")? - .unwrap_or_else(|| "app_EMoamEEZ73f0CkXaXp7hrann".to_string()); - let session_path = optional_env("OPENAI_CODEX_SESSION_PATH")? - .map(PathBuf::from) - .unwrap_or_else(|| ironclaw_base_dir().join("openai_codex_session.json")); - let token_refresh_margin_secs = - parse_optional_env("OPENAI_CODEX_REFRESH_MARGIN_SECS", 300)?; - Some(OpenAiCodexConfig { - model, - auth_endpoint, - api_base_url, - client_id, - session_path, - token_refresh_margin_secs, - }) - } else { - None - }; - - let request_timeout_secs = parse_optional_env("LLM_REQUEST_TIMEOUT_SECS", 120)?; - - let gemini_oauth = if backend_lower == "gemini_oauth" || backend_lower == "gemini-oauth" { - let model = Self::resolve_model("GEMINI_MODEL", settings, "gemini-2.5-flash")?; - let credentials_path = optional_env("GEMINI_CREDENTIALS_PATH")? - .map(PathBuf::from) - .unwrap_or_else(GeminiOauthConfig::default_credentials_path); - Some(GeminiOauthConfig { - model, - credentials_path, - }) - } else { - None - }; + .unwrap_or_else(|| "<unset>".to_string()); + resolve_nearai_fallback(settings, &attempted) + } + Err(e) => Err(e), + } +} - // Generic cheap model (works with any backend). - // Falls back to NearAI-specific cheap_model in provider chain logic. - let cheap_model = optional_env("LLM_CHEAP_MODEL")?; +/// If the resolved config's LLM provider is unusable, return a short +/// reason string. `None` means the config is fine to use. Returns `None` +/// for special backends that don't use the `provider` slot +/// (nearai/bedrock/codex/gemini_oauth) — those have their own validation +/// inside `resolve_once`. +/// +/// This check is deliberately narrow: it only flags configurations that +/// will certainly fail at runtime. Built-in providers like Anthropic may +/// legitimately have an empty `base_url` at config time because the +/// downstream rig-core client hardcodes the canonical endpoint. +fn unusable_reason(cfg: &LlmConfig, settings: &Settings) -> Option<&'static str> { + let provider = cfg.provider.as_ref()?; + + let is_custom = settings + .llm_custom_providers + .iter() + .any(|c| c.id == provider.provider_id); + let is_ollama = matches!(provider.protocol, ProviderProtocol::Ollama); + + // Custom providers have no hardcoded base URL in the client layer — + // an empty `base_url` here means requests will be sent to a bare + // path with no host, which always fails. + if is_custom && provider.base_url.trim().is_empty() { + return Some("missing base URL"); + } + + // Ollama runs locally and has no API key concept. Every other + // provider needs at least one form of authentication. + if !is_ollama + && provider.api_key.is_none() + && provider.oauth_token.is_none() + && provider.refresh_token.is_none() + { + return Some("missing API key"); + } + + None +} - // Generic smart routing cascade flag. - // Defaults to true. Overrides NearAI-specific smart_routing_cascade. - let smart_routing_cascade = parse_optional_env("SMART_ROUTING_CASCADE", true)?; +/// Errors that indicate a fixable user-config problem (as opposed to a +/// programming bug or environmental failure). We fall back to NearAI on +/// these so the instance can still start. +fn is_fallback_recoverable(err: &ConfigError) -> bool { + matches!(err, ConfigError::MissingRequired { .. }) +} - // Decorator chain settings — top-level `LLM_*` vars with fallback to - // existing backend-specific vars for backward compatibility. - let max_retries = optional_env("LLM_MAX_RETRIES")? - .map(|s| s.parse::<u32>()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "LLM_MAX_RETRIES".to_string(), - message: format!("must be a non-negative integer: {e}"), - })? - .unwrap_or(nearai.max_retries); +/// Re-resolve with `llm_backend` forced to `"nearai"`. +/// `attempted_backend` is only used for logging so operators can see which +/// backend we bailed out of. +fn resolve_nearai_fallback( + settings: &Settings, + attempted_backend: &str, +) -> Result<LlmConfig, ConfigError> { + let mut fallback = settings.clone(); + fallback.llm_backend = Some("nearai".to_string()); + // The previously-selected model was tied to the unusable backend + // (e.g. "openai/gpt-4o" for OpenRouter, "kimi-k2-turbo-preview" for + // a custom kimi provider). Sending it to NearAI would 404. Clear it + // so resolve_model falls through to NearAI's default. This is + // in-memory only — the user's DB-persisted selected_model is + // deliberately preserved so a transient hydration failure does not + // destroy their configured selection on next restart (#3229). + fallback.selected_model = None; + let cfg = resolve(&fallback).map_err(|e| { + tracing::error!( + attempted = %attempted_backend, + fallback_error = %e, + "NearAI fallback also failed to resolve — surfacing original error" + ); + e + })?; + tracing::warn!( + attempted = %attempted_backend, + active = %cfg.backend, + active_model = %cfg.nearai.model, + "Active LLM backend fell back to NearAI default due to unusable user config" + ); + Ok(cfg) +} - let circuit_breaker_threshold = optional_env("LLM_CIRCUIT_BREAKER_THRESHOLD")? - .map(|s| s.parse::<u32>()) +/// Resolve LLM configuration without any fallback behavior. +/// +/// Returns the config exactly as computed from env/DB/defaults, with no +/// safety net for missing credentials. Use this for: +/// - Early startup (before secrets are hydrated), so a spurious fallback +/// doesn't fire and get overridden by the later re-resolve. +/// - Tests that verify pure resolution mechanics (model/base_url priority +/// chains, alias normalization, etc.). +/// +/// The top-level `AppBuilder` path calls [`resolve_with_fallback`] after +/// hydrating secrets, which handles #2514-style crash-loop prevention. +pub fn resolve(settings: &Settings) -> Result<LlmConfig, ConfigError> { + let registry = ProviderRegistry::load(); + + // Determine backend: db settings > env var > default ("nearai") + let (backend, backend_source) = if let Some(ref b) = settings.llm_backend { + (b.clone(), "db:llm_backend") + } else if let Some(b) = optional_env("LLM_BACKEND")? { + (b, "env:LLM_BACKEND") + } else { + ("nearai".to_string(), "default") + }; + LOG_LLM_BACKEND_RESOLUTION.call_once(|| { + tracing::debug!( + backend = %backend, + source = %backend_source, + db_llm_backend = ?settings.llm_backend, + custom_providers_count = settings.llm_custom_providers.len(), + "Resolving LLM backend" + ); + }); + // Warn operators when a DB-persisted value silently overrides LLM_BACKEND. + // Skip the warning when both values are identical — this is the normal + // state after `ironclaw models set-provider`, which intentionally writes + // to both config.toml and .env for immediate effect. + if backend_source == "db:llm_backend" + && let Ok(env_val) = std::env::var("LLM_BACKEND") + && !env_val.is_empty() + && env_val != backend + { + tracing::warn!( + db_value = %backend, + env_value = %env_val, + "LLM_BACKEND env var is set but DB setting takes priority. \ + Unset llm_backend in the DB (via settings UI) to use the env var." + ); + } + + // Classify the backend by protocol. The registry knows all built-in + // backends including the dedicated-config ones (nearai/bedrock/codex/ + // gemini_oauth) and their aliases. + let backend_lower = backend.to_lowercase(); + let backend_protocol = registry.find(&backend_lower).map(|d| d.protocol); + let is_nearai = backend_protocol == Some(ProviderProtocol::NearAi); + let is_bedrock = backend_protocol == Some(ProviderProtocol::Bedrock); + let is_gemini_oauth = backend_protocol == Some(ProviderProtocol::GeminiOauth); + let is_openai_codex = backend_protocol == Some(ProviderProtocol::OpenAiCodex); + + // Check custom providers defined + let custom_provider = settings + .llm_custom_providers + .iter() + .find(|p| p.id.to_lowercase() == backend_lower); + + if backend_protocol.is_none() && custom_provider.is_none() { + tracing::warn!( + "Unknown LLM backend '{}'. Will attempt as openai_compatible fallback.", + backend + ); + } + + // Always resolve NEAR AI config (used for embeddings even when not the primary backend) + // Priority: DB (builtin_overrides) > env > default + let nearai_override = settings.llm_builtin_overrides.get("nearai"); + let nearai_override_has_base_url = nearai_override.and_then(|o| o.base_url.as_ref()).is_some(); + + // Check whether NearAI embeddings are enabled. When they are, the + // NearAI base_url and auth_url are reachable code-paths and must pass + // SSRF validation even when the primary chat backend is not NearAI. + let emb_defaults = crate::settings::EmbeddingsSettings::default(); + let emb_provider = db_first_or_default( + &settings.embeddings.provider, + &emb_defaults.provider, + "EMBEDDING_PROVIDER", + )?; + let emb_enabled = db_first_bool( + settings.embeddings.enabled, + emb_defaults.enabled, + "EMBEDDING_ENABLED", + )?; + let nearai_embeddings_active = emb_enabled && emb_provider.eq_ignore_ascii_case("nearai"); + + // Predicate: NearAI URLs must be validated when: + // - NearAI is the primary chat backend, OR + // - the user/DB explicitly supplied the URL, OR + // - NEARAI_API_KEY is set (implies intent to use NearAI), OR + // - NearAI embeddings are enabled (they use the same base/auth URLs), OR + // - the DB builtin_overrides for NearAI include a base_url. + let nearai_api_key_env = optional_env("NEARAI_API_KEY")?; + + // Session config (used by NearAI provider for OAuth/session-token auth) + let nearai_auth_url_explicit = optional_env("NEARAI_AUTH_URL")?; + let nearai_auth_url = nearai_auth_url_explicit + .clone() + .unwrap_or_else(|| "https://private.near.ai".to_string()); + // Only validate NearAI URLs when NearAI is active or the user explicitly + // set the URL. Default URLs point to private.near.ai which requires DNS + // resolution — this blocks startup in environments without network access + // (CI runners, containers) when a different backend is configured. + if is_nearai + || nearai_auth_url_explicit.is_some() + || nearai_api_key_env.is_some() + || nearai_override_has_base_url + || nearai_embeddings_active + { + validate_base_url(&nearai_auth_url, "NEARAI_AUTH_URL")?; + } + let session = SessionConfig { + auth_base_url: nearai_auth_url, + session_path: optional_env("NEARAI_SESSION_PATH")? + .map(PathBuf::from) + .unwrap_or_else(default_session_path), + }; + + let nearai_api_key = if let Some(key) = nearai_override.and_then(|o| o.api_key.as_ref()) { + Some(SecretString::from(key.clone())) + } else { + nearai_api_key_env.map(SecretString::from) + }; + // Model priority: selected_model (DB) > builtin_overrides (DB) > env > default + let nearai_model = if let Some(model) = selected_model_override(settings) { + model + } else if let Some(model) = nearai_override.and_then(|o| o.model.clone()) { + model + } else if let Some(model) = optional_env("NEARAI_MODEL")? { + model + } else { + ironclaw_llm::DEFAULT_MODEL.to_string() + }; + let nearai_base_url_explicit = optional_env("NEARAI_BASE_URL")?; + let nearai_base_url = if let Some(url) = nearai_override.and_then(|o| o.base_url.clone()) { + url + } else if let Some(url) = nearai_base_url_explicit.clone() { + url + } else if nearai_api_key.is_some() { + "https://cloud-api.near.ai".to_string() + } else { + "https://private.near.ai".to_string() + }; + if is_nearai + || nearai_base_url_explicit.is_some() + || nearai_api_key.is_some() + || nearai_override_has_base_url + || nearai_embeddings_active + { + validate_base_url(&nearai_base_url, "NEARAI_BASE_URL")?; + } + let nearai = NearAiConfig { + model: nearai_model, + cheap_model: optional_env("NEARAI_CHEAP_MODEL")?, + base_url: nearai_base_url, + api_key: nearai_api_key, + fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?, + max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?, + circuit_breaker_threshold: optional_env("CIRCUIT_BREAKER_THRESHOLD")? + .map(|s| s.parse()) .transpose() .map_err(|e| ConfigError::InvalidValue { - key: "LLM_CIRCUIT_BREAKER_THRESHOLD".to_string(), + key: "CIRCUIT_BREAKER_THRESHOLD".to_string(), message: format!("must be a positive integer: {e}"), - })? - .or(nearai.circuit_breaker_threshold); - - let circuit_breaker_recovery_secs = optional_env("LLM_CIRCUIT_BREAKER_RECOVERY_SECS")? - .map(|s| s.parse::<u64>()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "LLM_CIRCUIT_BREAKER_RECOVERY_SECS".to_string(), - message: format!("must be a non-negative integer: {e}"), - })? - .unwrap_or(nearai.circuit_breaker_recovery_secs); - - let response_cache_enabled = optional_env("LLM_RESPONSE_CACHE_ENABLED")? - .map(|s| s.parse::<bool>()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "LLM_RESPONSE_CACHE_ENABLED".to_string(), - message: format!("must be true or false: {e}"), - })? - .unwrap_or(nearai.response_cache_enabled); - - let response_cache_ttl_secs = optional_env("LLM_RESPONSE_CACHE_TTL_SECS")? - .map(|s| s.parse::<u64>()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "LLM_RESPONSE_CACHE_TTL_SECS".to_string(), - message: format!("must be a non-negative integer: {e}"), - })? - .unwrap_or(nearai.response_cache_ttl_secs); + })?, + circuit_breaker_recovery_secs: parse_optional_env("CIRCUIT_BREAKER_RECOVERY_SECS", 30)?, + response_cache_enabled: parse_optional_env("RESPONSE_CACHE_ENABLED", false)?, + response_cache_ttl_secs: parse_optional_env("RESPONSE_CACHE_TTL_SECS", 3600)?, + response_cache_max_entries: parse_optional_env("RESPONSE_CACHE_MAX_ENTRIES", 1000)?, + failover_cooldown_secs: parse_optional_env("LLM_FAILOVER_COOLDOWN_SECS", 300)?, + failover_cooldown_threshold: parse_optional_env("LLM_FAILOVER_THRESHOLD", 3)?, + smart_routing_cascade: parse_optional_env("SMART_ROUTING_CASCADE", true)?, + }; + + // Resolve registry provider config (for non-NearAI, non-Bedrock, non-Gemini, non-Codex backends) + let provider = if is_nearai || is_bedrock || is_gemini_oauth || is_openai_codex { + None + } else if let Some(custom) = custom_provider { + Some(resolve_custom_provider(custom, settings)?) + } else { + Some(resolve_registry_provider( + &backend_lower, + &registry, + settings, + )?) + }; + + let bedrock = if is_bedrock { + // Layer D: bedrock-specific settings live in + // `llm_builtin_overrides["bedrock"].extras` rather than + // top-level named fields. Legacy `bedrock_*` fields are + // migrated into this bag on `Settings::load`, so the resolver + // only needs to read from one place. + let bedrock_overrides = settings.llm_builtin_overrides.get("bedrock"); + let extra = |key: &str| -> Option<String> { + bedrock_overrides + .and_then(|o| o.extra(key)) + .map(str::to_string) + }; + let region = extra("region").or(optional_env("BEDROCK_REGION")?); + if region.is_none() { + tracing::debug!( + "BEDROCK_REGION not set, defaulting to {}", + BedrockConfig::DEFAULT_REGION + ); + } + let model = selected_model_override(settings).or(optional_env("BEDROCK_MODEL")?); + let cross_region = extra("cross_region").or(optional_env("BEDROCK_CROSS_REGION")?); + let profile = extra("profile").or(optional_env("AWS_PROFILE")?); + Some(BedrockConfig::build(region, model, cross_region, profile)?) + } else { + None + }; - let response_cache_max_entries = optional_env("LLM_RESPONSE_CACHE_MAX_ENTRIES")? - .map(|s| s.parse::<usize>()) - .transpose() - .map_err(|e| ConfigError::InvalidValue { - key: "LLM_RESPONSE_CACHE_MAX_ENTRIES".to_string(), - message: format!("must be a non-negative integer: {e}"), - })? - .unwrap_or(nearai.response_cache_max_entries); - - Ok(Self { - backend: if is_nearai { - "nearai".to_string() - } else if is_bedrock { - "bedrock".to_string() - } else if is_gemini_oauth { - "gemini_oauth".to_string() - } else if is_openai_codex { - "openai_codex".to_string() - } else if let Some(ref p) = provider { - p.provider_id.clone() - } else { - backend_lower - }, - session, - nearai, - provider, - bedrock, - gemini_oauth, - openai_codex, - request_timeout_secs, - cheap_model, - smart_routing_cascade, - max_retries, - circuit_breaker_threshold, - circuit_breaker_recovery_secs, - response_cache_enabled, - response_cache_ttl_secs, - response_cache_max_entries, - }) - } + // Resolve OpenAI Codex config. Defaults live in `OpenAiCodexConfig::build`; + // this side owns env reading and SSRF validation against the resolved URLs. + let openai_codex = if is_openai_codex { + let cfg = OpenAiCodexConfig::build( + // Model: settings.selected_model > OPENAI_CODEX_MODEL > OPENAI_MODEL > default + selected_model_override(settings) + .or(optional_env("OPENAI_CODEX_MODEL")?) + .or(optional_env("OPENAI_MODEL")?), + optional_env("OPENAI_CODEX_AUTH_URL")?, + optional_env("OPENAI_CODEX_API_URL")?, + optional_env("OPENAI_CODEX_CLIENT_ID")?, + optional_env("OPENAI_CODEX_SESSION_PATH")?.map(PathBuf::from), + optional_env("OPENAI_CODEX_REFRESH_MARGIN_SECS")? + .map(|s| s.parse::<u64>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "OPENAI_CODEX_REFRESH_MARGIN_SECS".to_string(), + message: format!("must be a non-negative integer: {e}"), + })?, + ); + validate_base_url(&cfg.auth_endpoint, "OPENAI_CODEX_AUTH_URL")?; + validate_base_url(&cfg.api_base_url, "OPENAI_CODEX_API_URL")?; + Some(cfg) + } else { + None + }; - /// Resolve a `RegistryProviderConfig` from a user-defined custom provider. - fn resolve_custom_provider( - custom: &crate::settings::CustomLlmProviderSettings, - settings: &Settings, - ) -> Result<RegistryProviderConfig, ConfigError> { - tracing::debug!( - id = %custom.id, - adapter = %custom.adapter, - base_url = ?custom.base_url, - "Resolving custom LLM provider" - ); - let protocol = match custom.adapter.as_str() { - "anthropic" => ProviderProtocol::Anthropic, - "ollama" => ProviderProtocol::Ollama, - _ => ProviderProtocol::OpenAiCompletions, - }; + let request_timeout_secs = parse_optional_env("LLM_REQUEST_TIMEOUT_SECS", 120)?; - let api_key = custom - .api_key - .as_ref() - .filter(|k| !k.is_empty()) - .map(|k| SecretString::from(k.clone())); + let gemini_oauth = if is_gemini_oauth { + Some(GeminiOauthConfig::build( + selected_model_override(settings).or(optional_env("GEMINI_MODEL")?), + optional_env("GEMINI_CREDENTIALS_PATH")?.map(PathBuf::from), + )) + } else { + None + }; + + // Generic cheap model (works with any backend). + // Falls back to NearAI-specific cheap_model in provider chain logic. + let cheap_model = optional_env("LLM_CHEAP_MODEL")?; + + // Generic smart routing cascade flag. + // Defaults to true. Overrides NearAI-specific smart_routing_cascade. + let smart_routing_cascade = parse_optional_env("SMART_ROUTING_CASCADE", true)?; + + // Decorator chain settings — top-level `LLM_*` vars with fallback to + // existing backend-specific vars for backward compatibility. + let max_retries = optional_env("LLM_MAX_RETRIES")? + .map(|s| s.parse::<u32>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_MAX_RETRIES".to_string(), + message: format!("must be a non-negative integer: {e}"), + })? + .unwrap_or(nearai.max_retries); + + let circuit_breaker_threshold = optional_env("LLM_CIRCUIT_BREAKER_THRESHOLD")? + .map(|s| s.parse::<u32>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_CIRCUIT_BREAKER_THRESHOLD".to_string(), + message: format!("must be a positive integer: {e}"), + })? + .or(nearai.circuit_breaker_threshold); + + let circuit_breaker_recovery_secs = optional_env("LLM_CIRCUIT_BREAKER_RECOVERY_SECS")? + .map(|s| s.parse::<u64>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_CIRCUIT_BREAKER_RECOVERY_SECS".to_string(), + message: format!("must be a non-negative integer: {e}"), + })? + .unwrap_or(nearai.circuit_breaker_recovery_secs); + + let response_cache_enabled = optional_env("LLM_RESPONSE_CACHE_ENABLED")? + .map(|s| s.parse::<bool>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_RESPONSE_CACHE_ENABLED".to_string(), + message: format!("must be true or false: {e}"), + })? + .unwrap_or(nearai.response_cache_enabled); + + let response_cache_ttl_secs = optional_env("LLM_RESPONSE_CACHE_TTL_SECS")? + .map(|s| s.parse::<u64>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_RESPONSE_CACHE_TTL_SECS".to_string(), + message: format!("must be a non-negative integer: {e}"), + })? + .unwrap_or(nearai.response_cache_ttl_secs); + + let response_cache_max_entries = optional_env("LLM_RESPONSE_CACHE_MAX_ENTRIES")? + .map(|s| s.parse::<usize>()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "LLM_RESPONSE_CACHE_MAX_ENTRIES".to_string(), + message: format!("must be a non-negative integer: {e}"), + })? + .unwrap_or(nearai.response_cache_max_entries); + + // Canonical backend id: registry-resolved when the backend is known, + // otherwise the resolved registry provider's id, otherwise the raw + // (lowercased) backend string the caller passed. + let canonical_backend = registry + .find(&backend_lower) + .map(|d| d.id.clone()) + .or_else(|| provider.as_ref().map(|p| p.provider_id.clone())) + .unwrap_or(backend_lower); + + Ok(LlmConfig { + backend: canonical_backend, + session, + nearai, + provider, + bedrock, + gemini_oauth, + openai_codex, + request_timeout_secs, + cheap_model, + smart_routing_cascade, + max_retries, + circuit_breaker_threshold, + circuit_breaker_recovery_secs, + response_cache_enabled, + response_cache_ttl_secs, + response_cache_max_entries, + }) +} - let base_url = custom.base_url.clone().unwrap_or_default(); - if base_url.is_empty() { - tracing::warn!(id = %custom.id, "Custom provider has no base_url configured — requests will fail"); - } else { - validate_operator_base_url( - &base_url, - &format!("custom provider '{}' base_url", custom.id), - )?; - } - - let model = Self::selected_model_override(settings) - .or(optional_env("LLM_MODEL")?) - .or_else(|| custom.default_model.clone()) - .unwrap_or_default(); - if model.is_empty() { - tracing::warn!(id = %custom.id, "Custom provider has no model configured — requests may fail"); - } - - Ok(RegistryProviderConfig { - protocol, - provider_id: custom.id.clone(), - api_key, - base_url, - model, - extra_headers: Vec::new(), - oauth_token: None, - is_codex_chatgpt: false, - refresh_token: None, - auth_path: None, - cache_retention: CacheRetention::default(), - unsupported_params: Vec::new(), - }) +/// Resolve a `RegistryProviderConfig` from a user-defined custom provider. +fn resolve_custom_provider( + custom: &crate::settings::CustomLlmProviderSettings, + settings: &Settings, +) -> Result<RegistryProviderConfig, ConfigError> { + tracing::debug!( + id = %custom.id, + adapter = %custom.adapter, + base_url = ?custom.base_url, + "Resolving custom LLM provider" + ); + let protocol = match custom.adapter.as_str() { + "anthropic" => ProviderProtocol::Anthropic, + "ollama" => ProviderProtocol::Ollama, + _ => ProviderProtocol::OpenAiCompletions, + }; + + let api_key = custom + .api_key + .as_ref() + .filter(|k| !k.is_empty()) + .map(|k| SecretString::from(k.clone())); + + let base_url = custom.base_url.clone().unwrap_or_default(); + if base_url.is_empty() { + tracing::warn!(id = %custom.id, "Custom provider has no base_url configured — requests will fail"); + } else { + validate_operator_base_url( + &base_url, + &format!("custom provider '{}' base_url", custom.id), + )?; } - /// Resolve a `RegistryProviderConfig` from the registry and env vars. - fn resolve_registry_provider( - backend: &str, - registry: &ProviderRegistry, - settings: &Settings, - ) -> Result<RegistryProviderConfig, ConfigError> { - // Look up provider definition. Fall back to openai_compatible if unknown. - let def = registry - .find(backend) - .or_else(|| registry.find("openai_compatible")); - - let ( - canonical_id, - protocol, - api_key_env, - base_url_env, - model_env, - default_model, - default_base_url, - extra_headers_env, - api_key_required, - base_url_required, - unsupported_params, - ) = if let Some(def) = def { - ( - def.id.as_str(), - def.protocol, - def.api_key_env.as_deref(), - def.base_url_env.as_deref(), - def.model_env.as_str(), - def.default_model.as_str(), - def.default_base_url.as_deref(), - def.extra_headers_env.as_deref(), - def.api_key_required, - def.base_url_required, - def.unsupported_params.clone(), - ) - } else { - // Absolute fallback: treat as generic openai_completions - ( - backend, - ProviderProtocol::OpenAiCompletions, - Some("LLM_API_KEY"), - Some("LLM_BASE_URL"), - "LLM_MODEL", - "default", - None, - Some("LLM_EXTRA_HEADERS"), - false, - true, - Vec::new(), - ) - }; + let model = selected_model_override(settings) + .or(optional_env("LLM_MODEL")?) + .or_else(|| custom.default_model.clone()) + .unwrap_or_default(); + if model.is_empty() { + tracing::warn!(id = %custom.id, "Custom provider has no model configured — requests may fail"); + } + + Ok(RegistryProviderConfig { + protocol, + provider_id: custom.id.clone(), + api_key, + base_url, + model, + extra_headers: Vec::new(), + oauth_token: None, + is_codex_chatgpt: false, + refresh_token: None, + auth_path: None, + cache_retention: CacheRetention::default(), + unsupported_params: Vec::new(), + }) +} - // Codex auth.json override: when LLM_USE_CODEX_AUTH=true, - // credentials from the Codex CLI's auth.json take highest priority - // (over env vars AND secrets store). In ChatGPT mode, the base URL - // is also overridden to the private ChatGPT backend endpoint. - let mut codex_base_url_override: Option<String> = None; - let codex_creds = if parse_optional_env("LLM_USE_CODEX_AUTH", false)? { - let path = optional_env("CODEX_AUTH_PATH")? - .map(std::path::PathBuf::from) - .unwrap_or_else(crate::llm::codex_auth::default_codex_auth_path); - crate::llm::codex_auth::load_codex_credentials(&path) +/// Resolve a `RegistryProviderConfig` from the registry and env vars. +fn resolve_registry_provider( + backend: &str, + registry: &ProviderRegistry, + settings: &Settings, +) -> Result<RegistryProviderConfig, ConfigError> { + // Look up provider definition. Fall back to openai_compatible if unknown. + let def = registry + .find(backend) + .or_else(|| registry.find("openai_compatible")); + + let ( + canonical_id, + protocol, + api_key_env, + base_url_env, + model_env, + default_model, + default_base_url, + extra_headers_env, + api_key_required, + base_url_required, + unsupported_params, + ) = if let Some(def) = def { + ( + def.id.as_str(), + def.protocol, + def.api_key_env.as_deref(), + def.base_url_env.as_deref(), + def.model_env.as_str(), + def.default_model.as_str(), + def.default_base_url.as_deref(), + def.extra_headers_env.as_deref(), + def.api_key_required, + def.base_url_required, + def.unsupported_params.clone(), + ) + } else { + // Absolute fallback: treat as generic openai_completions + ( + backend, + ProviderProtocol::OpenAiCompletions, + Some("LLM_API_KEY"), + Some("LLM_BASE_URL"), + "LLM_MODEL", + "default", + None, + Some("LLM_EXTRA_HEADERS"), + false, + true, + Vec::new(), + ) + }; + + // Codex auth.json override: when LLM_USE_CODEX_AUTH=true, + // credentials from the Codex CLI's auth.json take highest priority + // (over env vars AND secrets store). In ChatGPT mode, the base URL + // is also overridden to the private ChatGPT backend endpoint. + let mut codex_base_url_override: Option<String> = None; + let codex_creds = if parse_optional_env("LLM_USE_CODEX_AUTH", false)? { + let override_path = optional_env("CODEX_AUTH_PATH")?.map(std::path::PathBuf::from); + ironclaw_llm::auth::load_persisted_credentials( + ironclaw_llm::auth::CredentialSource::CodexCli, + override_path.as_deref(), + ) + } else { + None + }; + + let codex_refresh_token = codex_creds.as_ref().and_then(|c| c.refresh_token.clone()); + let codex_auth_path = codex_creds.as_ref().and_then(|c| c.source_path.clone()); + + let api_key = if let Some(creds) = codex_creds { + if creds.is_subscription { + codex_base_url_override = Some(creds.base_url.clone()); + } + Some(creds.token) + } else if let Some(env_var) = api_key_env { + // Resolve API key: settings override (DB) > env var (including secrets store overlay) + if let Some(key) = settings + .llm_builtin_overrides + .get(backend) + .and_then(|o| o.api_key.as_ref()) + { + Some(SecretString::from(key.clone())) } else { - None - }; + optional_env(env_var)?.map(SecretString::from) + } + } else { + None + }; - let codex_refresh_token = codex_creds.as_ref().and_then(|c| c.refresh_token.clone()); - let codex_auth_path = codex_creds.as_ref().and_then(|c| c.auth_path.clone()); + if api_key_required && api_key.is_none() { + // Don't hard-fail here. The key might be injected later from the secrets store + // via inject_llm_keys_from_secrets(). Log a warning instead. + if let Some(env_var) = api_key_env { + tracing::debug!( + "API key not found in {env_var} for backend '{backend}'. \ + Will be injected from secrets store if available." + ); + } + } - let api_key = if let Some(creds) = codex_creds { - if creds.is_chatgpt_mode { - codex_base_url_override = Some(creds.base_url().to_string()); - } - Some(creds.token) - } else if let Some(env_var) = api_key_env { - // Resolve API key: settings override (DB) > env var (including secrets store overlay) - if let Some(key) = settings + // Resolve base URL: codex override > builtin_overrides (DB) > legacy settings (DB) > env var > registry default + let is_codex_chatgpt = codex_base_url_override.is_some(); + let env_base_url = if let Some(env_var) = base_url_env { + optional_env(env_var)? + } else { + None + }; + let base_url = codex_base_url_override + .or_else(|| { + // DB settings: per-provider base_url override + settings .llm_builtin_overrides .get(backend) - .and_then(|o| o.api_key.as_ref()) - { - Some(SecretString::from(key.clone())) - } else { - optional_env(env_var)?.map(SecretString::from) + .and_then(|o| o.base_url.clone()) + }) + .or_else(|| { + // DB settings: legacy settings fields + match backend { + "ollama" => settings.ollama_base_url.clone(), + "openai_compatible" | "openrouter" => settings.openai_compatible_base_url.clone(), + _ => None, } - } else { - None - }; + }) + .or(env_base_url) + .or_else(|| default_base_url.map(String::from)) + .unwrap_or_default(); + + if base_url_required + && base_url.is_empty() + && let Some(env_var) = base_url_env + { + return Err(ConfigError::MissingRequired { + key: env_var.to_string(), + hint: format!("Set {env_var} when LLM_BACKEND={backend}"), + }); + } - if api_key_required && api_key.is_none() { - // Don't hard-fail here. The key might be injected later from the secrets store - // via inject_llm_keys_from_secrets(). Log a warning instead. - if let Some(env_var) = api_key_env { - tracing::debug!( - "API key not found in {env_var} for backend '{backend}'. \ - Will be injected from secrets store if available." - ); - } - } + // Provider base URLs are explicit operator configuration, so allow + // private/local endpoints while still rejecting unsafe schemes, + // public plaintext HTTP, and special blocked addresses. + if !base_url.is_empty() { + let field = base_url_env.unwrap_or("LLM_BASE_URL"); + validate_operator_base_url(&base_url, field)?; + } - // Resolve base URL: codex override > builtin_overrides (DB) > legacy settings (DB) > env var > registry default - let is_codex_chatgpt = codex_base_url_override.is_some(); - let env_base_url = if let Some(env_var) = base_url_env { - optional_env(env_var)? - } else { - None - }; - let base_url = codex_base_url_override - .or_else(|| { - // DB settings: per-provider base_url override - settings - .llm_builtin_overrides - .get(backend) - .and_then(|o| o.base_url.clone()) - }) - .or_else(|| { - // DB settings: legacy settings fields - match backend { - "ollama" => settings.ollama_base_url.clone(), - "openai_compatible" | "openrouter" => { - settings.openai_compatible_base_url.clone() - } - _ => None, + // Resolve model: selected_model (DB) > per-provider override (DB) > env var > registry default + let model = selected_model_override(settings) + .or_else(|| { + settings + .llm_builtin_overrides + .get(backend) + .and_then(|o| o.model.clone()) + }) + .or(optional_env(model_env)?) + .unwrap_or_else(|| default_model.to_string()); + + // Resolve extra headers + let extra_headers = if let Some(env_var) = extra_headers_env { + optional_env(env_var)? + .map(|val| parse_extra_headers_with_key(&val, env_var)) + .transpose()? + .unwrap_or_default() + } else { + Vec::new() + }; + let extra_headers = if canonical_id == "github_copilot" { + merge_extra_headers( + ironclaw_llm::auth::default_headers(ironclaw_llm::auth::AuthBackend::GithubCopilot), + extra_headers, + ) + } else { + extra_headers + }; + + // Resolve OAuth token (Anthropic-specific: `claude login` flow). + // Only check for OAuth token when the provider is actually Anthropic. + let oauth_token = if canonical_id == "anthropic" { + optional_env("ANTHROPIC_OAUTH_TOKEN")?.map(SecretString::from) + } else { + None + }; + let api_key = if api_key.is_none() && oauth_token.is_some() { + // OAuth token present but no API key: use a placeholder so the + // config block is populated. The provider factory will route to + // the OAuth provider instead of rig-core's x-api-key client. + Some(SecretString::from(OAUTH_PLACEHOLDER.to_string())) + } else { + api_key + }; + + // Resolve Anthropic prompt cache retention from env (default: Short). + let cache_retention: CacheRetention = if canonical_id == "anthropic" { + optional_env("ANTHROPIC_CACHE_RETENTION")? + .and_then(|val| match val.parse::<CacheRetention>() { + Ok(r) => Some(r), + Err(e) => { + tracing::warn!("Invalid ANTHROPIC_CACHE_RETENTION: {e}; defaulting to short"); + None } }) - .or(env_base_url) - .or_else(|| default_base_url.map(String::from)) - .unwrap_or_default(); - - if base_url_required - && base_url.is_empty() - && let Some(env_var) = base_url_env - { - return Err(ConfigError::MissingRequired { - key: env_var.to_string(), - hint: format!("Set {env_var} when LLM_BACKEND={backend}"), - }); - } - - // Provider base URLs are explicit operator configuration, so allow - // private/local endpoints while still rejecting unsafe schemes, - // public plaintext HTTP, and special blocked addresses. - if !base_url.is_empty() { - let field = base_url_env.unwrap_or("LLM_BASE_URL"); - validate_operator_base_url(&base_url, field)?; - } - - // Resolve model: selected_model (DB) > per-provider override (DB) > env var > registry default - let model = Self::selected_model_override(settings) - .or_else(|| { - settings - .llm_builtin_overrides - .get(backend) - .and_then(|o| o.model.clone()) - }) - .or(optional_env(model_env)?) - .unwrap_or_else(|| default_model.to_string()); - - // Resolve extra headers - let extra_headers = if let Some(env_var) = extra_headers_env { - optional_env(env_var)? - .map(|val| parse_extra_headers_with_key(&val, env_var)) - .transpose()? - .unwrap_or_default() - } else { - Vec::new() - }; - let extra_headers = if canonical_id == "github_copilot" { - merge_extra_headers( - crate::llm::github_copilot_auth::default_headers(), - extra_headers, - ) - } else { - extra_headers - }; - - // Resolve OAuth token (Anthropic-specific: `claude login` flow). - // Only check for OAuth token when the provider is actually Anthropic. - let oauth_token = if canonical_id == "anthropic" { - optional_env("ANTHROPIC_OAUTH_TOKEN")?.map(SecretString::from) - } else { - None - }; - let api_key = if api_key.is_none() && oauth_token.is_some() { - // OAuth token present but no API key: use a placeholder so the - // config block is populated. The provider factory will route to - // the OAuth provider instead of rig-core's x-api-key client. - Some(SecretString::from(OAUTH_PLACEHOLDER.to_string())) - } else { - api_key - }; - - // Resolve Anthropic prompt cache retention from env (default: Short). - let cache_retention: CacheRetention = if canonical_id == "anthropic" { - optional_env("ANTHROPIC_CACHE_RETENTION")? - .and_then(|val| match val.parse::<CacheRetention>() { - Ok(r) => Some(r), - Err(e) => { - tracing::warn!( - "Invalid ANTHROPIC_CACHE_RETENTION: {e}; defaulting to short" - ); - None - } - }) - .unwrap_or_default() - } else { - CacheRetention::default() - }; - - Ok(RegistryProviderConfig { - protocol, - provider_id: canonical_id.to_string(), - api_key, - base_url, - model, - extra_headers, - oauth_token, - is_codex_chatgpt, - refresh_token: codex_refresh_token, - auth_path: codex_auth_path, - cache_retention, - unsupported_params, - }) - } + .unwrap_or_default() + } else { + CacheRetention::default() + }; + + Ok(RegistryProviderConfig { + protocol, + provider_id: canonical_id.to_string(), + api_key, + base_url, + model, + extra_headers, + oauth_token, + is_codex_chatgpt, + refresh_token: codex_refresh_token, + auth_path: codex_auth_path, + cache_retention, + unsupported_params, + }) } /// Parse `LLM_EXTRA_HEADERS` value into a list of (key, value) pairs. @@ -971,7 +913,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.model, "openai/gpt-5.1-codex"); @@ -993,7 +935,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( @@ -1115,7 +1057,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.model, "llama3.2"); @@ -1136,7 +1078,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( @@ -1162,7 +1104,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( @@ -1182,7 +1124,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.base_url, "https://localhost:8443/v1"); @@ -1199,7 +1141,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.base_url, "http://100.64.0.10:8000/v1"); @@ -1221,7 +1163,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "groq"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.provider_id, "groq"); @@ -1245,7 +1187,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "tinfoil"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.base_url, "https://inference.tinfoil.sh/v1"); @@ -1274,7 +1216,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "zai"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.provider_id, "zai"); @@ -1298,7 +1240,7 @@ mod tests { let settings = Settings::default(); - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "github_copilot"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.provider_id, "github_copilot"); @@ -1340,7 +1282,7 @@ mod tests { } let settings = Settings::default(); - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "nearai"); assert!(cfg.provider.is_none()); } @@ -1356,7 +1298,7 @@ mod tests { } let settings = Settings::default(); - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "openai", "alias 'open_ai' should be normalized to canonical 'openai'" @@ -1382,7 +1324,7 @@ mod tests { } let settings = Settings::default(); - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "openai_compatible"); let provider = cfg.provider.expect("should have provider config"); assert_eq!(provider.provider_id, "openai_compatible"); @@ -1405,7 +1347,7 @@ mod tests { std::env::set_var("LLM_BACKEND", alias); } let settings = Settings::default(); - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "nearai", "alias '{alias}' should resolve to 'nearai'" @@ -1440,7 +1382,7 @@ mod tests { }; // DB settings should take priority over env var - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("should have provider config"); assert_eq!( provider.base_url, "http://localhost:9000/v1", @@ -1453,7 +1395,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings_no_base).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings_no_base).expect("resolve should succeed"); let provider = cfg.provider.expect("should have provider config"); assert_eq!( provider.base_url, "http://localhost:8000/v1", @@ -1496,7 +1438,7 @@ mod tests { llm_backend: Some("anthropic".to_string()), ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( @@ -1535,7 +1477,7 @@ mod tests { llm_backend: Some("anthropic".to_string()), ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( @@ -1567,7 +1509,7 @@ mod tests { llm_backend: Some("openai".to_string()), ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert!( @@ -1669,7 +1611,7 @@ mod tests { unsafe { std::env::remove_var("LLM_REQUEST_TIMEOUT_SECS"); } - let config = LlmConfig::resolve(&Settings::default()).expect("resolve"); + let config = crate::config::llm::resolve(&Settings::default()).expect("resolve"); assert_eq!(config.request_timeout_secs, 120); } @@ -1680,7 +1622,7 @@ mod tests { unsafe { std::env::set_var("LLM_REQUEST_TIMEOUT_SECS", "300"); } - let config = LlmConfig::resolve(&Settings::default()).expect("resolve"); + let config = crate::config::llm::resolve(&Settings::default()).expect("resolve"); assert_eq!(config.request_timeout_secs, 300); // SAFETY: Cleanup unsafe { @@ -1713,7 +1655,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "myprovider"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!(provider.provider_id, "myprovider"); @@ -1721,7 +1663,7 @@ mod tests { assert_eq!(provider.model, "my-model"); assert_eq!( provider.protocol, - crate::llm::registry::ProviderProtocol::OpenAiCompletions + ironclaw_llm::registry::ProviderProtocol::OpenAiCompletions ); } @@ -1756,7 +1698,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "myprovider", "DB setting should override LLM_BACKEND env var" @@ -1791,6 +1733,7 @@ mod tests { api_key: None, model: Some("llama-3.1-8b-instant".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -1799,7 +1742,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( provider.model, "llama-3.1-8b-instant", @@ -1817,7 +1760,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.backend, "openai_codex"); let codex = cfg.openai_codex.expect("codex config should be present"); assert_eq!(codex.model, "gpt-5.3-codex"); // default @@ -1843,6 +1786,7 @@ mod tests { api_key: None, model: Some("llama-3.1-8b-instant".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -1852,7 +1796,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( provider.model, "llama-3.3-70b-versatile", @@ -1874,7 +1818,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let codex = cfg.openai_codex.expect("codex config should be present"); assert_eq!(codex.model, "o3-pro"); @@ -1901,6 +1845,7 @@ mod tests { api_key: Some("gsk_test_key".to_string()), model: Some("llama-3.3-70b-versatile".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -1909,7 +1854,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); use secrecy::ExposeSecret as _; let key = provider @@ -1936,7 +1881,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let codex = cfg.openai_codex.expect("codex config should be present"); assert_eq!(codex.model, "gpt-4o"); @@ -1957,7 +1902,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let codex = cfg.openai_codex.expect("codex config should be present"); assert_eq!(codex.model, "gpt-4o-mini"); } @@ -1980,7 +1925,7 @@ mod tests { ..Default::default() }; - let err = LlmConfig::resolve(&settings).unwrap_err(); + let err = crate::config::llm::resolve(&settings).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("OPENAI_CODEX_API_URL"), @@ -2008,7 +1953,7 @@ mod tests { ..Default::default() }; - let err = LlmConfig::resolve(&settings).unwrap_err(); + let err = crate::config::llm::resolve(&settings).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("OPENAI_CODEX_AUTH_URL"), @@ -2043,7 +1988,7 @@ mod tests { std::env::set_var("LLM_MAX_RETRIES", "10"); } - let cfg = LlmConfig::resolve(&Settings::default()).expect("resolve"); + let cfg = crate::config::llm::resolve(&Settings::default()).expect("resolve"); assert_eq!(cfg.max_retries, 10); unsafe { @@ -2061,7 +2006,7 @@ mod tests { std::env::set_var("NEARAI_MAX_RETRIES", "7"); } - let cfg = LlmConfig::resolve(&Settings::default()).expect("resolve"); + let cfg = crate::config::llm::resolve(&Settings::default()).expect("resolve"); assert_eq!(cfg.max_retries, 7); unsafe { @@ -2078,7 +2023,7 @@ mod tests { std::env::set_var("LLM_MAX_RETRIES", "not-a-number"); } - let err = LlmConfig::resolve(&Settings::default()).unwrap_err(); + let err = crate::config::llm::resolve(&Settings::default()).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("LLM_MAX_RETRIES"), @@ -2109,6 +2054,7 @@ mod tests { api_key: Some("gsk_from_db".to_string()), model: None, base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -2117,7 +2063,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); use secrecy::ExposeSecret as _; assert_eq!( @@ -2151,6 +2097,7 @@ mod tests { api_key: None, model: Some("model-from-db".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -2159,7 +2106,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( provider.model, "model-from-db", @@ -2196,7 +2143,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( provider.model, "model-from-db", @@ -2224,7 +2171,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let codex = cfg.openai_codex.expect("codex config should be present"); assert_eq!( codex.model, "codex-from-db", @@ -2252,7 +2199,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.model, "nearai-from-db", "DB selected_model must take priority over NEARAI_MODEL env var" @@ -2280,6 +2227,7 @@ mod tests { api_key: None, model: Some("model-from-db-override".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -2288,7 +2236,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.model, "model-from-db-override", "DB builtin_overrides model must take priority over NEARAI_MODEL env var" @@ -2316,6 +2264,7 @@ mod tests { api_key: None, model: Some("model-from-override".to_string()), base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -2325,7 +2274,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.model, "model-from-selected", "selected_model must take priority over builtin_overrides model" @@ -2349,6 +2298,7 @@ mod tests { api_key: None, model: None, base_url: Some("http://localhost:9002".to_string()), + extras: Default::default(), }, ); let settings = Settings { @@ -2357,7 +2307,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.base_url, "http://localhost:9002", "DB builtin_overrides base_url must take priority over NEARAI_BASE_URL env var" @@ -2384,7 +2334,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.base_url, "http://localhost:9001", "NEARAI_BASE_URL env var should be used when no DB override exists" @@ -2412,6 +2362,7 @@ mod tests { api_key: Some("key-from-db".to_string()), model: None, base_url: None, + extras: Default::default(), }, ); let settings = Settings { @@ -2420,7 +2371,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); use secrecy::ExposeSecret as _; assert_eq!( cfg.nearai @@ -2453,7 +2404,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.nearai.base_url, "https://private.near.ai", "Without API key, should default to private.near.ai" @@ -2467,6 +2418,7 @@ mod tests { api_key: Some("some-key".to_string()), model: None, base_url: None, + extras: Default::default(), }, ); let settings_with_key = Settings { @@ -2475,7 +2427,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings_with_key).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings_with_key).expect("resolve should succeed"); assert_eq!( cfg.nearai.base_url, "https://cloud-api.near.ai", "With API key, should default to cloud-api.near.ai" @@ -2500,6 +2452,7 @@ mod tests { api_key: None, model: None, base_url: Some("http://localhost:9004".to_string()), + extras: Default::default(), }, ); let settings = Settings { @@ -2508,7 +2461,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); let provider = cfg.provider.expect("provider config should be present"); assert_eq!( provider.base_url, "http://localhost:9004", @@ -2528,7 +2481,7 @@ mod tests { ..Default::default() }; - assert_eq!(LlmConfig::selected_model_override(&settings), None); + assert_eq!(selected_model_override(&settings), None); } #[test] @@ -2546,7 +2499,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!(cfg.nearai.model, "env-model"); // SAFETY: Under ENV_MUTEX. @@ -2585,7 +2538,7 @@ mod tests { }; // This must succeed without attempting DNS resolution on private.near.ai. - let cfg = LlmConfig::resolve(&settings) + let cfg = crate::config::llm::resolve(&settings) .expect("resolve should succeed for non-NearAI backend without NearAI URL validation"); assert_eq!(cfg.backend, "openai_compatible"); } @@ -2615,7 +2568,7 @@ mod tests { // resolve() would hand back an unusable anthropic config (no key); // resolve_with_fallback() must notice that and fall back to NearAI. - let cfg = LlmConfig::resolve_with_fallback(&settings) + let cfg = crate::config::llm::resolve_with_fallback(&settings) .expect("resolve should succeed via fallback"); assert_eq!( cfg.backend, "nearai", @@ -2651,7 +2604,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings) + let cfg = crate::config::llm::resolve_with_fallback(&settings) .expect("resolve should succeed via fallback"); assert_eq!(cfg.backend, "nearai"); assert_ne!( @@ -2661,7 +2614,7 @@ mod tests { ); assert_eq!( cfg.nearai.model, - crate::llm::DEFAULT_MODEL, + ironclaw_llm::DEFAULT_MODEL, "NearAI fallback should use the built-in default model when the pre-fallback \ selection is cleared" ); @@ -2685,7 +2638,7 @@ mod tests { // openai_compatible has base_url_required=true and no default — this // previously returned Err(MissingRequired), causing main.rs to bail // and the container to crash-loop. resolve() now recovers. - let cfg = LlmConfig::resolve_with_fallback(&settings) + let cfg = crate::config::llm::resolve_with_fallback(&settings) .expect("resolve should succeed via fallback"); assert_eq!(cfg.backend, "nearai"); } @@ -2704,7 +2657,8 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings).expect("resolve should succeed"); + let cfg = + crate::config::llm::resolve_with_fallback(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "groq", "a properly-configured backend must NOT trigger fallback" @@ -2733,7 +2687,8 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings).expect("resolve should succeed"); + let cfg = + crate::config::llm::resolve_with_fallback(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "anthropic", "anthropic with OAuth token must NOT fall back" @@ -2770,7 +2725,7 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings) + let cfg = crate::config::llm::resolve_with_fallback(&settings) .expect("resolve should succeed via fallback"); assert_eq!( cfg.backend, "nearai", @@ -2795,7 +2750,8 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings).expect("resolve should succeed"); + let cfg = + crate::config::llm::resolve_with_fallback(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "ollama", "ollama without api_key must NOT trigger fallback" @@ -2827,7 +2783,8 @@ mod tests { ..Default::default() }; - let cfg = LlmConfig::resolve_with_fallback(&settings).expect("resolve should succeed"); + let cfg = + crate::config::llm::resolve_with_fallback(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "my-ollama", "custom ollama provider without api_key must NOT fall back" @@ -2858,7 +2815,7 @@ mod tests { // Pure resolve keeps the configured backend even when it would be // considered unusable — the caller is responsible for hydrating // secrets and calling resolve_with_fallback afterwards. - let cfg = LlmConfig::resolve(&settings).expect("resolve should succeed"); + let cfg = crate::config::llm::resolve(&settings).expect("resolve should succeed"); assert_eq!( cfg.backend, "anthropic", "pure resolve must not auto-fall-back; that is resolve_with_fallback's job" diff --git a/src/config/mod.rs b/src/config/mod.rs index fc947fbade1..00831d85c50 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -57,7 +57,6 @@ pub use self::channels::{ MatrixConfig, SignalConfig, TuiChannelConfig, }; pub use self::database::{DatabaseBackend, DatabaseConfig, SslMode, default_libsql_path}; -pub use self::embeddings::{DEFAULT_EMBEDDING_CACHE_SIZE, EmbeddingsConfig}; pub use self::heartbeat::HeartbeatConfig; pub use self::hygiene::HygieneConfig; pub use self::llm::default_session_path; @@ -75,11 +74,15 @@ pub use self::transcription::TranscriptionConfig; pub use self::tunnel::TunnelConfig; pub use self::wasm::WasmConfig; pub use self::workspace::WorkspaceConfig; -pub use crate::llm::config::{ +pub use ironclaw_embeddings::{DEFAULT_EMBEDDING_CACHE_SIZE, EmbeddingsConfig}; +// LLM config / session types live in `ironclaw_llm`. Re-exported here so +// existing `crate::config::*Config` callers (notably `LlmConfig::resolve` +// in `src/config/llm.rs`, plus the wizard / doctor) keep compiling without +// being touched in this PR. +pub use ironclaw_llm::{ BedrockConfig, CacheRetention, GeminiOauthConfig, LlmConfig, NearAiConfig, OAUTH_PLACEHOLDER, - OpenAiCodexConfig, RegistryProviderConfig, + OpenAiCodexConfig, RegistryProviderConfig, SessionConfig, }; -pub use crate::llm::session::SessionConfig; // Thread-safe env var override helpers (replaces unsafe `std::env::set_var` // for mid-process env mutations in multi-threaded contexts). @@ -187,7 +190,7 @@ impl Config { libsql_url: None, libsql_auth_token: None, }, - llm: LlmConfig::for_testing(), + llm: crate::config::llm::for_testing(), embeddings: EmbeddingsConfig::default(), tunnel: TunnelConfig::default(), channels: ChannelsConfig { @@ -201,6 +204,7 @@ impl Config { wasm_channels_enabled: false, configured_wasm_channels: Vec::new(), wasm_channel_owner_ids: HashMap::new(), + wasm_channel_runtime_overrides: HashMap::new(), }, agent: AgentConfig::for_testing(), safety: SafetyConfig { @@ -504,59 +508,23 @@ impl Config { // backend is unusable. This prevents the #2514 crash-loop and keeps the // instance runnable while the user fixes their provider configuration. // + // The fallback is in-memory only — the user's DB-persisted + // `llm_backend` and `selected_model` are deliberately left untouched + // so a transient hydration failure (DB read race, secrets decryption + // hiccup) does not destroy their configured provider on next restart + // (#3229). The previous behavior of syncing the fallback into the DB + // turned a one-off fallback into a permanent reversion. + // // Hot-reload path (strict): use pure `resolve` so a bad save fails the // whole call and lets the caller roll back the triggering settings // write. Silently falling back here would be worse UX — the user // saved "openrouter", runtime would switch to NearAI, the UI would // show NearAI, and the user would wonder where their selection went. if strict_db_reads { - return LlmConfig::resolve(&settings); - } - - let configured_backend = settings.llm_backend.clone(); - let cfg = LlmConfig::resolve_with_fallback(&settings)?; - - // If fallback demoted the backend, persist the effective backend to - // the DB so the UI, status endpoint, and any other consumers stay - // consistent with what is actually running. Without this, the user - // would see "Active: openrouter" in Settings while the runtime is - // quietly using NearAI. - if let Some(store) = store - && fallback_fired(configured_backend.as_deref(), &cfg.backend) - { - tracing::warn!( - configured = ?configured_backend, - active = %cfg.backend, - "Syncing llm_backend in DB to reflect post-fallback runtime state" - ); - if let Err(e) = store - .set_setting( - user_id, - "llm_backend", - &serde_json::Value::String(cfg.backend.clone()), - ) - .await - { - tracing::warn!( - error = %e, - "Failed to persist post-fallback llm_backend to DB — UI may \ - display the previously-selected backend until next save" - ); - } - // The previously-selected model is almost certainly wrong for - // the NearAI fallback (e.g. an OpenRouter model name). Clear - // it so resolve_model() picks NearAI's default on next load. - if settings.selected_model.is_some() - && let Err(e) = store.delete_setting(user_id, "selected_model").await - { - tracing::warn!( - error = %e, - "Failed to clear selected_model after fallback" - ); - } + return crate::config::llm::resolve(&settings); } - Ok(cfg) + crate::config::llm::resolve_with_fallback(&settings) } /// Resolve only the LLM configuration from the current source stack. @@ -606,11 +574,15 @@ impl Config { // handled separately by WorkspacePool. let workspace = WorkspaceConfig::resolve(&owner_id)?; + let llm = crate::config::llm::resolve(settings)?; + let embeddings = + self::embeddings::resolve_embeddings_config(settings, &llm.nearai.base_url)?; + Ok(Self { owner_id: owner_id.clone(), database: DatabaseConfig::resolve()?, - llm: LlmConfig::resolve(settings)?, - embeddings: EmbeddingsConfig::resolve(settings)?, + llm, + embeddings, tunnel, channels, agent: AgentConfig::resolve(settings)?, @@ -638,63 +610,6 @@ impl Config { } } -/// Detect whether `resolve_with_fallback` demoted the user-configured backend -/// to NearAI. Returns true when the user explicitly asked for something -/// non-trivial (non-empty, not already nearai) and the resolver landed on a -/// different backend. Aliases like `open_ai` → `openai` are not counted as a -/// fallback — only cross-backend demotion is. -fn fallback_fired(configured: Option<&str>, active: &str) -> bool { - let configured = match configured.map(str::trim).filter(|s| !s.is_empty()) { - Some(c) => c, - None => return false, - }; - // Normalize both sides so alias-only drift (e.g. open_ai → openai, - // near / near_ai → nearai) doesn't spuriously look like a fallback. - normalize_backend(configured) != normalize_backend(active) -} - -/// Normalize a backend id to the canonical form that `LlmConfig::resolve` -/// lands on after alias resolution. Must produce the same canonical id the -/// resolver uses — otherwise `fallback_fired` will mis-fire on every restart -/// for any DB value that's a known alias (e.g. `claude` → `anthropic`, -/// `bigmodel` → `zai`, `github-copilot` → `github_copilot`) and trigger a -/// spurious DB rewrite. -/// -/// Two sources of aliases: -/// 1. Registry-defined aliases — delegated to `ProviderRegistry::find`, which -/// is the same lookup `resolve_registry_provider` uses. -/// 2. Hardcoded aliases for the four "virtual" backends that are not in the -/// registry (nearai / bedrock / gemini_oauth / openai_codex). These must -/// stay in sync with the matching branches in `LlmConfig::resolve`. -fn normalize_backend(raw: &str) -> String { - let lower = raw.to_lowercase(); - - // (1) Virtual backends (not in the registry) — hardcoded alias list - // mirroring LlmConfig::resolve. - match lower.as_str() { - "nearai" | "near" | "near_ai" => return "nearai".to_string(), - "bedrock" | "aws" | "aws_bedrock" => return "bedrock".to_string(), - "gemini_oauth" | "gemini-oauth" => return "gemini_oauth".to_string(), - "openai_codex" | "openai-codex" | "codex" => return "openai_codex".to_string(), - _ => {} - } - - // (2) Registry providers — any alias declared in `providers.json` is - // resolved by `ProviderRegistry::find` to its canonical `id`. This is the - // SAME canonicalization `LlmConfig::resolve_registry_provider` does, so - // DB values like `claude` / `bigmodel` / `github-copilot` / `open_ai` - // won't look like a fallback. - if let Some(def) = crate::llm::ProviderRegistry::load().find(&lower) { - return def.id.clone(); - } - - // Unknown backend — resolve() treats it as openai_compatible at runtime, - // but here we conservatively return the input as-is. A truly unknown id - // won't match the canonical `active` either way; the comparison in - // `fallback_fired` just has to be consistent between both sides. - lower -} - pub(crate) fn load_bootstrap_settings( toml_path: Option<&std::path::Path>, ) -> Result<Settings, ConfigError> { @@ -736,7 +651,7 @@ pub(crate) fn resolve_owner_id(settings: &Settings) -> Result<String, ConfigErro /// Load API keys from the encrypted secrets store into a thread-safe overlay. /// /// This bridges the gap between secrets stored during onboarding and the -/// env-var-first resolution in `LlmConfig::resolve()`. Keys in the overlay +/// env-var-first resolution in `crate::config::llm::resolve()`. Keys in the overlay /// are read by `optional_env()` before falling back to `std::env::var()`, /// so explicit env vars always win. /// @@ -762,7 +677,7 @@ pub async fn inject_llm_keys_from_secrets( // Dynamically discover secret->env mappings from the provider registry. // Uses selectable() which deduplicates user overrides correctly. - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); let dynamic_mappings: Vec<(String, String)> = registry .selectable() .iter() @@ -822,6 +737,7 @@ fn merge_injected_vars(new_entries: HashMap<String, String>) { if new_entries.is_empty() { return; } + register_injected_vars_fallback(); match INJECTED_VARS.lock() { Ok(mut map) => map.extend(new_entries), Err(poisoned) => poisoned.into_inner().extend(new_entries), @@ -833,6 +749,7 @@ fn merge_injected_vars(new_entries: HashMap<String, String>) { /// Used by the setup wizard to make credentials available to `optional_env()` /// without calling `unsafe { std::env::set_var }`. pub fn inject_single_var(key: &str, value: &str) { + register_injected_vars_fallback(); match INJECTED_VARS.lock() { Ok(mut map) => { map.insert(key.to_string(), value.to_string()); @@ -845,6 +762,23 @@ pub fn inject_single_var(key: &str, value: &str) { } } +/// Register a one-time secondary env-lookup fallback with `ironclaw_common` +/// so the workspace-wide `env_or_override` (used from `ironclaw_llm`) can +/// see values populated via `inject_single_var` / the secrets injection +/// pipeline. Idempotent thanks to the underlying `OnceLock`. +fn register_injected_vars_fallback() { + static REGISTERED: std::sync::Once = std::sync::Once::new(); + REGISTERED.call_once(|| { + ironclaw_common::env_helpers::register_secondary_fallback(|key| { + INJECTED_VARS + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(key) + .cloned() + }); + }); +} + /// Remove a single key from the injected-vars overlay. /// /// Tests that exercise production paths calling [`inject_single_var`] @@ -877,7 +811,7 @@ fn inject_os_credential_store_tokens(injected: &mut HashMap<String, String>) { /// Hydrate LLM API keys from the secrets store into the settings struct. /// -/// Called after loading settings from DB but before `LlmConfig::resolve()`. +/// Called after loading settings from DB but before `crate::config::llm::resolve()`. /// Populates `api_key` fields that were stripped from settings during the /// write path and stored encrypted in the secrets store instead. pub async fn hydrate_llm_keys_from_secrets( @@ -1073,6 +1007,7 @@ mod tests { api_key: None, // stripped during write model: Some("gpt-4o".to_string()), base_url: None, + extras: Default::default(), }, ); m @@ -1506,6 +1441,7 @@ mod tests { api_key: Some("sk-existing".to_string()), model: None, base_url: None, + extras: Default::default(), }, ); m @@ -1522,85 +1458,89 @@ mod tests { ); } - // ── fallback_fired / normalize_backend tests ───────────────────────── - // - // These gate the post-fallback DB sync in re_resolve_llm_with_secrets, - // so wrong answers either (a) let stale user intent linger in the DB - // (UI shows openrouter, runtime uses NearAI) or (b) clobber the user's - // selection every startup even though nothing meaningfully changed. - - #[test] - fn fallback_fired_detects_cross_backend_demotion() { - // The #2514 scenario: user picked openrouter, config was unusable, - // resolver demoted to NearAI. DB must be synced. - assert!(fallback_fired(Some("openrouter"), "nearai")); - assert!(fallback_fired(Some("anthropic"), "nearai")); - assert!(fallback_fired(Some("openai_compatible"), "nearai")); - } + // Regression for #3229: a startup-path fallback to NearAI must NOT + // overwrite the user's DB-persisted llm_backend / selected_model. + // Before the fix, a transient hydration failure (DB read race, secrets + // decryption hiccup) would cause the fallback to be persisted, turning + // a one-off into a permanent reversion of the user's configured provider. + #[allow(clippy::await_holding_lock)] + #[tokio::test] + async fn startup_fallback_must_not_overwrite_persisted_user_backend() { + use crate::db::SettingsStore; - #[test] - fn fallback_fired_ignores_alias_normalization() { - // `resolve` canonicalises backend aliases (near → nearai, open_ai → - // openai) but that is not a fallback and must not trigger a DB - // rewrite — doing so would churn the row on every startup. - // - // Virtual backends (not in the registry — alias set hardcoded in - // normalize_backend): - assert!(!fallback_fired(Some("near"), "nearai")); - assert!(!fallback_fired(Some("near_ai"), "nearai")); - assert!(!fallback_fired(Some("aws"), "bedrock")); - assert!(!fallback_fired(Some("aws_bedrock"), "bedrock")); - assert!(!fallback_fired(Some("codex"), "openai_codex")); - assert!(!fallback_fired(Some("openai-codex"), "openai_codex")); - assert!(!fallback_fired(Some("gemini-oauth"), "gemini_oauth")); - } + let _env_guard = crate::config::helpers::lock_env(); + // SAFETY: Under ENV_MUTEX. Strip env-var inputs so we are testing the + // DB-driven path, not values that happen to be set in the test runner. + unsafe { + std::env::remove_var("LLM_BACKEND"); + std::env::remove_var("LLM_API_KEY"); + std::env::remove_var("LLM_BASE_URL"); + std::env::remove_var("OPENAI_API_KEY"); + std::env::remove_var("OPENROUTER_API_KEY"); + std::env::remove_var("ANTHROPIC_API_KEY"); + std::env::remove_var("ANTHROPIC_OAUTH_TOKEN"); + } - #[test] - fn fallback_fired_ignores_registry_aliases() { - // Regression: `providers.json` declares aliases for many registry - // providers (e.g. `claude` → `anthropic`, `bigmodel` → `zai`, - // `github-copilot` → `github_copilot`, `open_ai` → `openai`). - // `resolve_registry_provider` canonicalises these to the registry's - // `id` field, so a DB value of `claude` produces `cfg.backend == - // "anthropic"`. normalize_backend must delegate to the registry so - // this is recognised as alias drift, not a fallback. Otherwise the - // DB gets rewritten on every startup for users who happen to have - // the alias form saved. - assert!(!fallback_fired(Some("claude"), "anthropic")); - assert!(!fallback_fired(Some("bigmodel"), "zai")); - assert!(!fallback_fired(Some("github-copilot"), "github_copilot")); - assert!(!fallback_fired(Some("githubcopilot"), "github_copilot")); - assert!(!fallback_fired(Some("open_ai"), "openai")); - assert!(!fallback_fired( - Some("openai-compatible"), - "openai_compatible" - )); - assert!(!fallback_fired(Some("compatible"), "openai_compatible")); - assert!(!fallback_fired(Some("open_router"), "openrouter")); - } + // Seed the user's DB row with a properly-configured registry backend + // selection but without a hydratable API key, mirroring the #3229 + // reproduction (Gemini configured in onboarding, key not yet + // injected from the encrypted secrets store). + let store = FakeSettingsStore::new(); + store + .seed( + "owner-user", + "llm_backend", + serde_json::Value::String("openrouter".to_string()), + ) + .await; + store + .seed( + "owner-user", + "selected_model", + serde_json::Value::String("openai/gpt-4o-mini".to_string()), + ) + .await; - #[test] - fn fallback_fired_ignores_empty_or_unset_configured() { - // When the DB never had llm_backend set, the default of "nearai" - // resolves naturally — there is nothing to sync back. - assert!(!fallback_fired(None, "nearai")); - assert!(!fallback_fired(Some(""), "nearai")); - assert!(!fallback_fired(Some(" "), "nearai")); - } + let toml = empty_toml_path(); + let cfg = Config::resolve_llm_with_secrets( + Some(&store as &(dyn crate::db::SettingsStore + Sync)), + "owner-user", + Some(toml.path()), + None, // no secrets store: forces the unusable-config fallback path + true, + ) + .await + .expect("startup-path resolve should succeed via in-memory NearAI fallback"); - #[test] - fn fallback_fired_treats_case_insensitively() { - // DB values can be lowercase or mixed-case; don't treat case-only - // drift as a meaningful change. - assert!(!fallback_fired(Some("NearAI"), "nearai")); - assert!(!fallback_fired(Some("OPENAI"), "openai")); - } + // In-memory: the runtime is NearAI so the instance is usable + // (#2514 crash-loop prevention still works). + assert_eq!( + cfg.backend, "nearai", + "missing API key must trigger the in-memory NearAI fallback" + ); - #[test] - fn fallback_fired_same_backend_no_sync() { - // A properly-configured backend must not trigger a DB rewrite. - assert!(!fallback_fired(Some("nearai"), "nearai")); - assert!(!fallback_fired(Some("anthropic"), "anthropic")); - assert!(!fallback_fired(Some("openrouter"), "openrouter")); + // Critical invariant: the user's DB row is untouched. On the next + // restart, with secrets hydration succeeding, the user's original + // openrouter+model selection takes effect again. The pre-fix code + // overwrote llm_backend to "nearai" and deleted selected_model, + // permanently destroying the user's intent. + let backend = store + .get_setting("owner-user", "llm_backend") + .await + .expect("DB read"); + assert_eq!( + backend, + Some(serde_json::Value::String("openrouter".to_string())), + "startup fallback must preserve the user's persisted llm_backend (#3229)" + ); + let model = store + .get_setting("owner-user", "selected_model") + .await + .expect("DB read"); + assert_eq!( + model, + Some(serde_json::Value::String("openai/gpt-4o-mini".to_string())), + "startup fallback must preserve the user's persisted selected_model (#3229)" + ); } } diff --git a/src/config/transcription.rs b/src/config/transcription.rs index ed72832ab76..89ad1a6f2f3 100644 --- a/src/config/transcription.rs +++ b/src/config/transcription.rs @@ -92,7 +92,7 @@ impl TranscriptionConfig { /// Create the transcription provider if enabled and configured. pub fn create_provider( &self, - ) -> Option<Box<dyn crate::llm::transcription::TranscriptionProvider>> { + ) -> Option<Box<dyn ironclaw_llm::transcription::TranscriptionProvider>> { if !self.enabled { return None; } @@ -107,7 +107,7 @@ impl TranscriptionConfig { ); let mut provider = - crate::llm::transcription::ChatCompletionsTranscriptionProvider::new( + ironclaw_llm::transcription::ChatCompletionsTranscriptionProvider::new( api_key.clone(), ) .with_model(&self.model); @@ -125,7 +125,7 @@ impl TranscriptionConfig { ); let mut provider = - crate::llm::transcription::OpenAiWhisperProvider::new(api_key.clone()) + ironclaw_llm::transcription::OpenAiWhisperProvider::new(api_key.clone()) .with_model(&self.model); if let Some(ref base_url) = self.base_url { diff --git a/src/context/manager.rs b/src/context/manager.rs index b1259ed1745..f3d28583a0a 100644 --- a/src/context/manager.rs +++ b/src/context/manager.rs @@ -660,7 +660,7 @@ mod tests { // Update memory by adding a message manager .update_memory(job_id, |m| { - m.add_message(crate::llm::ChatMessage::user("hello from test")); + m.add_message(ironclaw_llm::ChatMessage::user("hello from test")); }) .await .unwrap(); diff --git a/src/context/memory.rs b/src/context/memory.rs index 05313e6787a..025b8bbf49d 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -7,7 +7,7 @@ use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::llm::ChatMessage; +use ironclaw_llm::ChatMessage; /// A record of an action taken during job execution. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -121,7 +121,7 @@ impl ConversationMemory { // Trim old messages if needed (keeping system message if present) while self.messages.len() > self.max_messages { // Don't remove system messages - if self.messages.first().map(|m| m.role) == Some(crate::llm::Role::System) { + if self.messages.first().map(|m| m.role) == Some(ironclaw_llm::Role::System) { if self.messages.len() > 1 { self.messages.remove(1); } else { @@ -416,7 +416,7 @@ mod tests { assert_eq!(mem.len(), 3); // safety: test // System message must survive - assert_eq!(mem.messages()[0].role, crate::llm::Role::System); // safety: test + assert_eq!(mem.messages()[0].role, ironclaw_llm::Role::System); // safety: test assert_eq!(mem.messages()[0].content, "You are helpful"); // safety: test // Oldest non-system message (msg1) should be gone assert_eq!(mem.messages()[1].content, "msg2"); // safety: test @@ -432,7 +432,7 @@ mod tests { mem.add(ChatMessage::user("b")); assert_eq!(mem.len(), 2); // safety: test - assert_eq!(mem.messages()[0].role, crate::llm::Role::System); // safety: test + assert_eq!(mem.messages()[0].role, ironclaw_llm::Role::System); // safety: test assert_eq!(mem.messages()[1].content, "b"); // safety: test } diff --git a/src/context/state.rs b/src/context/state.rs index 11aab7abe07..7c47abde881 100644 --- a/src/context/state.rs +++ b/src/context/state.rs @@ -9,8 +9,8 @@ use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::llm::recording::HttpInterceptor; use crate::tools::ApprovalContext; +use ironclaw_llm::recording::HttpInterceptor; /// Error returned when a job exceeds its token budget. #[derive(Debug, thiserror::Error)] diff --git a/src/db/libsql/pairing.rs b/src/db/libsql/pairing.rs index 7b114e8d58c..bf942b7dd02 100644 --- a/src/db/libsql/pairing.rs +++ b/src/db/libsql/pairing.rs @@ -462,6 +462,27 @@ impl ChannelPairingStore for LibSqlBackend { .map_err(|e| DatabaseError::Query(e.to_string()))?; Ok(()) } + + async fn create_channel_identity( + &self, + channel: &str, + external_id: &str, + owner_id: &str, + ) -> Result<(), DatabaseError> { + let channel = crate::pairing::normalize_channel_name(channel); + let id = uuid::Uuid::new_v4().to_string(); + let conn = self.connect().await?; + conn.execute( + "INSERT INTO channel_identities (id, owner_id, channel, external_id) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (channel, external_id) + DO UPDATE SET owner_id = ?2", + params![id, owner_id, channel, external_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } } #[cfg(test)] diff --git a/src/db/libsql/users.rs b/src/db/libsql/users.rs index 936c1c8d595..c306b2e5180 100644 --- a/src/db/libsql/users.rs +++ b/src/db/libsql/users.rs @@ -45,6 +45,17 @@ fn row_to_api_token(row: &libsql::Row) -> Result<ApiTokenRecord, DatabaseError> }) } +fn parse_libsql_decimal_text( + value: &str, + field_name: &str, +) -> Result<rust_decimal::Decimal, DatabaseError> { + rust_decimal::Decimal::from_str_exact(value) + .or_else(|_| rust_decimal::Decimal::from_scientific(value)) + .map_err(|e| { + DatabaseError::Query(format!("invalid {} value '{}': {}", field_name, value, e)) + }) +} + pub(crate) async fn seed_initial_assistant_thread( conn: &libsql::Connection, user_id: &str, @@ -658,9 +669,7 @@ impl UserStore for LibSqlBackend { .map_err(|e| DatabaseError::Query(e.to_string()))? { let cost_str = get_text(&row, 5); - let total_cost = rust_decimal::Decimal::from_str_exact(&cost_str).map_err(|e| { - DatabaseError::Query(format!("invalid cost value '{}': {}", cost_str, e)) - })?; + let total_cost = parse_libsql_decimal_text(&cost_str, "cost")?; stats.push(crate::db::UserUsageStats { user_id: get_text(&row, 0), model: get_text(&row, 1), @@ -836,9 +845,7 @@ impl UserStore for LibSqlBackend { .map_err(|e| DatabaseError::Query(e.to_string()))? { let cost_str = get_text(&row, 2); - let total_cost = rust_decimal::Decimal::from_str_exact(&cost_str).map_err(|e| { - DatabaseError::Query(format!("invalid cost value '{}': {}", cost_str, e)) - })?; + let total_cost = parse_libsql_decimal_text(&cost_str, "cost")?; stats.push(crate::db::UserSummaryStats { user_id: get_text(&row, 0), job_count: row @@ -896,12 +903,7 @@ impl UserStore for LibSqlBackend { })?; let usage_cost_str = get_text(&row, 8); - let usage_cost = rust_decimal::Decimal::from_str_exact(&usage_cost_str).map_err(|e| { - DatabaseError::Query(format!( - "invalid usage_cost value '{}': {}", - usage_cost_str, e - )) - })?; + let usage_cost = parse_libsql_decimal_text(&usage_cost_str, "usage_cost")?; Ok(AdminUsageSummary { total_users: row @@ -1237,6 +1239,22 @@ mod tests { .unwrap(); } + #[test] + fn test_parse_libsql_decimal_text_accepts_scientific_and_reports_invalid() { + let scientific = parse_libsql_decimal_text("7.5e-05", "cost").unwrap(); + assert_eq!( + scientific, + rust_decimal::Decimal::from_str_exact("0.000075").unwrap() + ); + + let err = parse_libsql_decimal_text("not-a-decimal", "cost").unwrap_err(); + assert!( + err.to_string() + .contains("invalid cost value 'not-a-decimal'"), + "unexpected error: {err}" + ); + } + #[tokio::test] async fn test_user_summary_stats_empty() { let (db, _dir) = setup().await; diff --git a/src/db/libsql/workspace.rs b/src/db/libsql/workspace.rs index 5ff50433bf1..ff4446fd2cc 100644 --- a/src/db/libsql/workspace.rs +++ b/src/db/libsql/workspace.rs @@ -112,9 +112,7 @@ pub(crate) fn resolve_embedding_dimension() -> Option<usize> { let model = std::env::var("EMBEDDING_MODEL").unwrap_or_else(|_| "text-embedding-3-small".to_string()); - Some(crate::config::embeddings::default_dimension_for_model( - &model, - )) + Some(ironclaw_embeddings::default_dimension_for_model(&model)) } impl LibSqlBackend { diff --git a/src/db/migration_fixup.rs b/src/db/migration_fixup.rs index ef76932fc0b..cc348786c69 100644 --- a/src/db/migration_fixup.rs +++ b/src/db/migration_fixup.rs @@ -301,6 +301,10 @@ mod tests { use std::collections::HashMap; use std::path::PathBuf; + #[cfg(feature = "integration")] + static POSTGRES_MIGRATION_FIXUP_TEST_LOCK: tokio::sync::Mutex<()> = + tokio::sync::Mutex::const_new(()); + fn migrations_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("migrations") } @@ -647,6 +651,7 @@ mod tests { return; } }; + let _guard = POSTGRES_MIGRATION_FIXUP_TEST_LOCK.lock().await; // Make sure refinery_schema_history exists. We can't rely on // the test DB having had migrations run, so create it on @@ -795,6 +800,7 @@ mod tests { return; } }; + let _guard = POSTGRES_MIGRATION_FIXUP_TEST_LOCK.lock().await; // Make sure refinery_schema_history exists so we exercise the // post-history-check branch (the early-return on missing table diff --git a/src/db/mod.rs b/src/db/mod.rs index 5c97069e169..cf5f55f2f76 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1198,6 +1198,15 @@ pub trait ChannelPairingStore: Send + Sync { channel: &str, external_id: &str, ) -> Result<(), DatabaseError>; + + /// Create or update a channel identity directly (trusted path, e.g. OAuth). + /// Inserts into channel_identities without requiring a pairing code. + async fn create_channel_identity( + &self, + channel: &str, + external_id: &str, + owner_id: &str, + ) -> Result<(), DatabaseError>; } /// Generates an 8-character pairing code from an unambiguous alphabet. diff --git a/src/db/postgres.rs b/src/db/postgres.rs index 0418d9cb46b..03acdd90dc7 100644 --- a/src/db/postgres.rs +++ b/src/db/postgres.rs @@ -1473,6 +1473,31 @@ impl ChannelPairingStore for PgBackend { .map_err(|e| DatabaseError::Query(e.to_string()))?; Ok(()) } + + async fn create_channel_identity( + &self, + channel: &str, + external_id: &str, + owner_id: &str, + ) -> Result<(), DatabaseError> { + let channel = crate::pairing::normalize_channel_name(channel); + let client = self + .pool() + .get() + .await + .map_err(|e| DatabaseError::Pool(e.to_string()))?; + client + .execute( + "INSERT INTO channel_identities (owner_id, channel, external_id) + VALUES ($1, $2, $3) + ON CONFLICT (channel, external_id) + DO UPDATE SET owner_id = $1", + &[&owner_id, &channel, &external_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } } // ==================== IdentityStore ==================== diff --git a/src/document_extraction/extractors.rs b/src/document_extraction/extractors.rs index 6ab1e208da9..a4a746e0614 100644 --- a/src/document_extraction/extractors.rs +++ b/src/document_extraction/extractors.rs @@ -884,7 +884,7 @@ mod tests { #[test] fn extract_pptx_rejects_oversized_slide() { use std::io::{Cursor, Write}; - let big_slide = "<a:t>".to_string() + &"x".repeat(60 * 1024 * 1024) + "</a:t>"; + let big_slide = format!("<a:t>{}</a:t>", "x".repeat(60 * 1024 * 1024)); let buf = Vec::new(); let cursor = Cursor::new(buf); let mut writer = zip::ZipWriter::new(cursor); diff --git a/src/error.rs b/src/error.rs index 65d9d30700b..dc5f31e5844 100644 --- a/src/error.rs +++ b/src/error.rs @@ -72,6 +72,19 @@ pub enum ConfigError { Io(#[from] std::io::Error), } +impl From<ironclaw_llm::LlmConfigError> for ConfigError { + fn from(err: ironclaw_llm::LlmConfigError) -> Self { + match err { + ironclaw_llm::LlmConfigError::MissingRequired { key, hint } => { + Self::MissingRequired { key, hint } + } + ironclaw_llm::LlmConfigError::InvalidValue { key, message } => { + Self::InvalidValue { key, message } + } + } + } +} + /// Database-related errors. #[derive(Debug, thiserror::Error)] pub enum DatabaseError { @@ -141,8 +154,10 @@ pub enum ChannelError { HealthCheckFailed { name: String }, } -// LlmError lives in src/llm/error.rs; re-exported here for backward compatibility. -pub use crate::llm::error::LlmError; +// LlmError lives in `ironclaw_llm`; re-exported here so existing +// `crate::error::LlmError` callers and the `Error::Llm(#[from] LlmError)` +// variant keep working without churn. +pub use ironclaw_llm::LlmError; /// Tool execution errors. #[derive(Debug, thiserror::Error)] @@ -425,7 +440,7 @@ pub enum RoutineError { partial_tokens: Option<i32>, /// Whether the underlying LLM error was classified as retryable. /// Set at the `LlmError` → `RoutineError` conversion site using - /// `crate::llm::retry::is_retryable()`, avoiding fragile substring + /// `ironclaw_llm::retry::is_retryable()`, avoiding fragile substring /// matching on the stringified reason. retryable: bool, }, @@ -450,7 +465,7 @@ impl RoutineError { /// Whether this error is transient and worth retrying with backoff. /// /// Retryable: LLM failures where the underlying `LlmError` was classified - /// as retryable by `crate::llm::retry::is_retryable()`, empty responses, + /// as retryable by `ironclaw_llm::retry::is_retryable()`, empty responses, /// and truncated responses. /// Non-retryable: configuration errors, authorization, resource limits, /// DB errors, and LLM failures caused by auth/content-policy/context-length. diff --git a/src/extensions/discovery.rs b/src/extensions/discovery.rs index 64cdf104a58..ce29bbcf67d 100644 --- a/src/extensions/discovery.rs +++ b/src/extensions/discovery.rs @@ -107,6 +107,7 @@ impl OnlineDiscovery { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }) } else { None @@ -183,6 +184,7 @@ impl OnlineDiscovery { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }) }) .collect() diff --git a/src/extensions/manager.rs b/src/extensions/manager.rs index 65e0a0334aa..372cfa22a94 100644 --- a/src/extensions/manager.rs +++ b/src/extensions/manager.rs @@ -23,10 +23,17 @@ use crate::channels::wasm::{ }; use crate::extensions::discovery::OnlineDiscovery; use crate::extensions::registry::ExtensionRegistry; +use crate::extensions::wechat_login::{ + PendingWechatLogin, WECHAT_BASE_URL_SETTING_PATH, WECHAT_BOUND_USER_SETTING_PATH, + WECHAT_CHANNEL_NAME, WECHAT_DEFAULT_BASE_URL, WECHAT_DEFAULT_BOT_TYPE, WechatLoginPollOutcome, + interactive_login_info as wechat_interactive_login_info, poll_login as poll_wechat_login, + purge_expired_logins as purge_expired_wechat_logins, start_login as start_wechat_login, +}; use crate::extensions::{ ActivateResult, AuthResult, ConfigureResult, EnsureReadyIntent, EnsureReadyOutcome, ExtensionError, ExtensionKind, ExtensionPhase, ExtensionSource, InstallResult, - InstalledExtension, LatentProviderAction, RegistryEntry, ResultSource, SearchResult, + InstalledExtension, InteractiveLoginInfo, InteractiveLoginPollResult, + InteractiveLoginStartResult, LatentProviderAction, RegistryEntry, ResultSource, SearchResult, ToolAuthState, UpgradeOutcome, UpgradeResult, naming::{ canonicalize_extension_name, extension_name_candidates, legacy_extension_alias, @@ -113,10 +120,156 @@ fn oauth_refresh_secret_name(secret_name: &str) -> String { format!("{}_refresh_token", secret_name.to_lowercase()) } +fn is_reserved_wasm_runtime_config_key(key: &str) -> bool { + matches!( + key, + crate::channels::wasm::RUNTIME_CONFIG_KEY_TUNNEL_URL + | crate::channels::wasm::RUNTIME_CONFIG_KEY_WEBHOOK_SECRET + | crate::channels::wasm::RUNTIME_CONFIG_KEY_OWNER_ID + ) +} + fn oauth_scopes_secret_name(secret_name: &str) -> String { format!("{}_scopes", secret_name.to_lowercase()) } +const SETUP_SECRET_VALIDATION_PATTERN_MAX_BYTES: usize = 16 * 1024; +const SETUP_SECRET_VALIDATION_REGEX_SIZE_LIMIT: usize = 1 << 20; +const SETUP_SECRET_VALIDATION_REGEX_DFA_SIZE_LIMIT: usize = 1 << 20; +const SETUP_VALIDATION_RESPONSE_BODY_MAX_BYTES: usize = 64 * 1024; + +fn validation_endpoint_placeholder_names(template: &str) -> std::collections::BTreeSet<String> { + let mut names = std::collections::BTreeSet::new(); + let mut offset = 0; + + while let Some(relative_start) = template[offset..].find('{') { + let start = offset + relative_start; + let value_start = start + 1; + let Some(relative_end) = template[value_start..].find('}') else { + break; + }; + let end = value_start + relative_end; + let name = &template[value_start..end]; + if !name.is_empty() && !name.contains(['{', '}']) { + names.insert(name.to_string()); + } + offset = end + 1; + } + + names +} + +fn validation_endpoint_disallowed_placeholder<'a>( + placeholder_names: &'a std::collections::BTreeSet<String>, + allowed_secrets: &HashSet<String>, +) -> Option<&'a str> { + placeholder_names + .iter() + .map(String::as_str) + .find(|name| !allowed_secrets.contains(*name)) +} + +fn validation_endpoint_body_error(body: &[u8]) -> Option<String> { + let parsed: serde_json::Value = serde_json::from_slice(body).ok()?; + let errcode = parsed.get("errcode")?.as_i64()?; + if errcode == 0 { + return None; + } + + let errmsg = parsed + .get("errmsg") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown error"); + Some(format!( + "Validation endpoint returned errcode {errcode}: {errmsg}" + )) +} + +fn validation_response_exceeds_limit(current_len: usize, chunk_len: usize, limit: usize) -> bool { + match current_len.checked_add(chunk_len) { + Some(total) => total > limit, + None => true, + } +} + +async fn read_setup_validation_response_body( + response: &mut reqwest::Response, +) -> Result<Vec<u8>, ExtensionError> { + if let Some(content_length) = response.content_length() + && content_length > SETUP_VALIDATION_RESPONSE_BODY_MAX_BYTES as u64 + { + return Err(ExtensionError::Other(format!( + "Validation response exceeded {} bytes", + SETUP_VALIDATION_RESPONSE_BODY_MAX_BYTES + ))); + } + + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| ExtensionError::Other(format!("Failed to read validation response: {}", e)))? + { + if validation_response_exceeds_limit( + body.len(), + chunk.len(), + SETUP_VALIDATION_RESPONSE_BODY_MAX_BYTES, + ) { + return Err(ExtensionError::Other(format!( + "Validation response exceeded {} bytes", + SETUP_VALIDATION_RESPONSE_BODY_MAX_BYTES + ))); + } + body.extend_from_slice(&chunk); + } + + Ok(body) +} + +fn validate_setup_secret_value( + secret_name: &str, + value: &str, + validation: Option<&str>, +) -> Result<(), ExtensionError> { + if value.chars().any(char::is_control) { + return Err(ExtensionError::ValidationFailed(format!( + "Secret '{}' contains disallowed control characters", + secret_name + ))); + } + + if let Some(pattern) = validation { + if pattern.len() > SETUP_SECRET_VALIDATION_PATTERN_MAX_BYTES { + return Err(ExtensionError::Config(format!( + "Validation pattern for secret '{}' is too large", + secret_name + ))); + } + + // Capabilities files may be installed from external packages. The + // regex crate matches in linear time, but compilation can still spend + // excessive memory on very large or accidentally complex patterns. + let re = regex::RegexBuilder::new(pattern) + .size_limit(SETUP_SECRET_VALIDATION_REGEX_SIZE_LIMIT) + .dfa_size_limit(SETUP_SECRET_VALIDATION_REGEX_DFA_SIZE_LIMIT) + .build() + .map_err(|e| { + ExtensionError::Config(format!( + "Invalid validation pattern for secret '{}': {}", + secret_name, e + )) + })?; + if !re.is_match(value) { + return Err(ExtensionError::ValidationFailed(format!( + "Secret '{}' does not match the expected format", + secret_name + ))); + } + } + + Ok(()) +} + fn normalize_oauth_callback_path(path: &str) -> String { let trimmed_path = path.trim_end_matches('/'); if trimmed_path.is_empty() { @@ -159,6 +312,7 @@ struct ChannelRuntimeState { pub struct ExtensionSetupSchema { pub secrets: Vec<crate::channels::web::types::SecretFieldInfo>, pub fields: Vec<crate::channels::web::types::SetupFieldInfo>, + pub interactive_login: Option<InteractiveLoginInfo>, } /// Only these global (non-namespaced) setting paths may be written by extension @@ -169,6 +323,21 @@ const ALLOWED_GLOBAL_SETUP_SETTING_PATHS: &[&str] = &["llm_backend", "selected_m type TestWasmChannelLoader = Arc<dyn Fn(&str) -> Result<LoadedChannel, ExtensionError> + Send + Sync>; +#[cfg(test)] +type TestWechatLoginStarter = Arc< + dyn Fn( + &str, + &str, + &str, + ) -> Result<(PendingWechatLogin, InteractiveLoginStartResult), ExtensionError> + + Send + + Sync, +>; +#[cfg(test)] +type TestWechatLoginPoller = Arc< + dyn Fn(&mut PendingWechatLogin) -> Result<WechatLoginPollOutcome, ExtensionError> + Send + Sync, +>; + #[cfg(test)] fn telegram_request_error(action: &'static str, error: &reqwest::Error) -> ExtensionError { tracing::warn!( @@ -441,6 +610,8 @@ pub struct ExtensionManager { /// Stored here so the web gateway can verify incoming callbacks without /// any env var or shared secret. relay_signing_secret_cache: Arc<std::sync::Mutex<Option<Vec<u8>>>>, + /// PairingStore for multi-tenant relay identity resolution. + pairing_store: Option<Arc<crate::pairing::PairingStore>>, /// When `true`, OAuth flows always return an auth URL to the caller /// instead of opening a browser on the server via `open::that()`. /// Set by the web gateway at startup via `enable_gateway_mode()`. @@ -448,9 +619,14 @@ pub struct ExtensionManager { /// The gateway's own base URL for building OAuth redirect URIs. /// Set by the web gateway at startup via `enable_gateway_mode()`. gateway_base_url: RwLock<Option<String>>, + pending_wechat_logins: RwLock<HashMap<String, PendingWechatLogin>>, channel_activation_locks: RwLock<HashMap<String, Arc<tokio::sync::Mutex<()>>>>, #[cfg(test)] test_wasm_channel_loader: RwLock<Option<TestWasmChannelLoader>>, + #[cfg(test)] + test_wechat_login_starter: RwLock<Option<TestWechatLoginStarter>>, + #[cfg(test)] + test_wechat_login_poller: RwLock<Option<TestWechatLoginPoller>>, } /// Sanitize a URL for logging by removing query parameters and credentials. @@ -670,19 +846,34 @@ impl ExtensionManager { relay_config: crate::config::RelayConfig::from_env(), relay_event_tx: Arc::new(tokio::sync::Mutex::new(None)), relay_signing_secret_cache: Arc::new(std::sync::Mutex::new(None)), + pairing_store: None, gateway_mode: std::sync::atomic::AtomicBool::new(false), gateway_base_url: RwLock::new(None), + pending_wechat_logins: RwLock::new(HashMap::new()), channel_activation_locks: RwLock::new(HashMap::new()), #[cfg(test)] test_wasm_channel_loader: RwLock::new(None), + #[cfg(test)] + test_wechat_login_starter: RwLock::new(None), + #[cfg(test)] + test_wechat_login_poller: RwLock::new(None), } } #[cfg(test)] - async fn set_test_wasm_channel_loader(&self, loader: TestWasmChannelLoader) { + pub(crate) async fn set_test_wasm_channel_loader(&self, loader: TestWasmChannelLoader) { *self.test_wasm_channel_loader.write().await = Some(loader); } + #[cfg(test)] + pub(crate) async fn set_test_wechat_login_starter(&self, starter: TestWechatLoginStarter) { + *self.test_wechat_login_starter.write().await = Some(starter); + } + + #[cfg(test)] + pub(crate) async fn set_test_wechat_login_poller(&self, poller: TestWechatLoginPoller) { + *self.test_wechat_login_poller.write().await = Some(poller); + } /// Enable gateway mode so OAuth flows return auth URLs to the frontend /// instead of calling `open::that()` on the server. /// @@ -750,7 +941,7 @@ impl ExtensionManager { } /// Get the relay config stored at startup. - fn relay_config(&self) -> Result<&crate::config::RelayConfig, ExtensionError> { + pub(crate) fn relay_config(&self) -> Result<&crate::config::RelayConfig, ExtensionError> { self.relay_config.as_ref().ok_or_else(|| { ExtensionError::Config( "CHANNEL_RELAY_URL and CHANNEL_RELAY_API_KEY must be set".to_string(), @@ -771,7 +962,7 @@ impl ExtensionManager { /// and the URL must not contain userinfo (embedded credentials). This /// prevents a malicious override from exfiltrating the instance-wide relay /// API key to an attacker-controlled host. - async fn effective_relay_url(&self, name: &str) -> Option<String> { + pub(crate) async fn effective_relay_url(&self, name: &str) -> Option<String> { if let Some(ref store) = self.store { let key = format!("extensions.{name}.relay_url"); if let Ok(Some(v)) = store.get_setting(&self.user_id, &key).await { @@ -926,9 +1117,41 @@ impl ExtensionManager { async fn load_channel_runtime_config_overrides( &self, name: &str, - ) -> HashMap<String, serde_json::Value> { + activation_user_id: &str, + ) -> Result<HashMap<String, serde_json::Value>, ExtensionError> { let mut overrides = HashMap::new(); + if let Some(store) = self.settings_store() { + let prefix = format!("channels.wasm_channel_runtime_overrides.{name}:"); + match store.get_all_settings(&self.user_id).await { + Ok(settings) => { + for (setting_key, value) in settings { + let Some(config_key) = setting_key.strip_prefix(&prefix) else { + continue; + }; + let config_key = config_key.trim(); + if config_key.is_empty() { + continue; + } + if is_reserved_wasm_runtime_config_key(config_key) { + tracing::warn!( + channel = %name, + key = %config_key, + "Ignoring reserved wasm runtime config override key" + ); + continue; + } + overrides.insert(config_key.to_string(), value); + } + } + Err(e) => { + return Err(ExtensionError::Config(format!( + "Failed to load persisted runtime config overrides for channel '{name}': {e}" + ))); + } + } + } + if name == TELEGRAM_CHANNEL_NAME && let Some(store) = self.settings_store() && let Ok(Some(serde_json::Value::String(username))) = store @@ -942,7 +1165,66 @@ impl ExtensionManager { ); } - overrides + if name == WECHAT_CHANNEL_NAME { + let bound_user_id = self + .load_wechat_bound_user_id() + .await + .unwrap_or_else(|| activation_user_id.to_string()); + if !bound_user_id.trim().is_empty() { + overrides.insert( + "bound_user_id".to_string(), + serde_json::json!(bound_user_id.clone()), + ); + } + + if let Some(store) = self.store.as_ref() + && let Ok(Some(serde_json::Value::String(base_url))) = store + .get_setting(&bound_user_id, WECHAT_BASE_URL_SETTING_PATH) + .await + && !base_url.trim().is_empty() + { + overrides.insert("base_url".to_string(), serde_json::json!(base_url)); + } + } + + Ok(overrides) + } + + async fn load_wechat_bound_user_id(&self) -> Option<String> { + let store = self.store.as_ref()?; + match store + .get_setting(&self.user_id, WECHAT_BOUND_USER_SETTING_PATH) + .await + { + Ok(Some(serde_json::Value::String(value))) => { + let trimmed = value.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + } + Ok(_) => None, + Err(error) => { + tracing::warn!( + error = %error, + "Failed to load WeChat bound user setting" + ); + None + } + } + } + + async fn persist_wechat_bound_user_id(&self, user_id: &str) -> Result<(), ExtensionError> { + let Some(store) = self.store.as_ref() else { + return Ok(()); + }; + store + .set_setting( + &self.user_id, + WECHAT_BOUND_USER_SETTING_PATH, + &serde_json::Value::String(user_id.to_string()), + ) + .await + .map_err(|error| { + ExtensionError::Other(format!("Failed to persist WeChat bound user: {error}")) + }) } pub async fn has_wasm_channel_owner_binding(&self, name: &str) -> bool { @@ -991,8 +1273,8 @@ impl ExtensionManager { let external_id = crate::pairing::ExternalId::from(external_id.to_string()); let config_overrides = self - .load_channel_runtime_config_overrides(channel_name) - .await; + .load_channel_runtime_config_overrides(channel_name, &self.user_id) + .await?; let deps = crate::pairing::approval::ApprovalDeps { tunnel_url: self.tunnel_url.as_deref(), store: self.store.as_ref(), @@ -1156,6 +1438,10 @@ impl ExtensionManager { &self.secrets } + pub fn pairing_store(&self) -> Option<&Arc<crate::pairing::PairingStore>> { + self.pairing_store.as_ref() + } + /// Expose the per-user MCP client store. Tool wrappers registered in /// the global `ToolRegistry` hold an `Arc<McpClientStore>` and resolve /// the caller's client at dispatch time via @@ -1333,28 +1619,28 @@ impl ExtensionManager { /// list after the user deactivates everything. The setup wizard's /// `channels.wasm_channels` list is only a first-run fallback before any /// runtime activation state has been persisted. + /// + /// Fails loud on settings-store errors: a DB outage or schema drift + /// returns `Err` rather than silently dropping the persisted state and + /// falling back to the configured list, which would mask the failure + /// and quietly re-activate channels the user had deactivated. pub async fn load_startup_active_channels( &self, user_id: &str, configured_names: Vec<String>, - ) -> Vec<String> { + ) -> Result<Vec<String>, crate::error::DatabaseError> { let Some(store) = self.settings_store() else { - return normalize_extension_names(configured_names); + return Ok(normalize_extension_names(configured_names)); }; - match store.get_setting(user_id, "activated_channels").await { - Ok(Some(value)) => match serde_json::from_value::<Vec<String>>(value) { - Ok(names) => normalize_extension_names(names), - Err(e) => { - tracing::warn!(error = %e, "Failed to deserialize activated_channels"); - Vec::new() - } - }, - Ok(None) => normalize_extension_names(configured_names), - Err(e) => { - tracing::warn!(error = %e, "Failed to load activated_channels setting"); - Vec::new() + match store.get_setting(user_id, "activated_channels").await? { + Some(value) => { + let names = serde_json::from_value::<Vec<String>>(value).map_err(|e| { + crate::error::DatabaseError::Serialization(format!("activated_channels: {e}")) + })?; + Ok(normalize_extension_names(names)) } + None => Ok(normalize_extension_names(configured_names)), } } @@ -1412,6 +1698,11 @@ impl ExtensionManager { self } + pub fn with_pairing_store(mut self, store: Arc<crate::pairing::PairingStore>) -> Self { + self.pairing_store = Some(store); + self + } + async fn clear_pending_extension_auth(&self, name: &str, user_id: &str) { { let mut pending = self.pending_auth.write().await; @@ -2090,6 +2381,7 @@ impl ExtensionManager { tools, needs_setup: false, has_auth, + requires_binding: false, installed: true, activation_error: None, version: None, @@ -2142,6 +2434,7 @@ impl ExtensionManager { tools: if active { vec![name] } else { Vec::new() }, needs_setup: auth_state == ToolAuthState::NeedsSetup, has_auth: auth_state != ToolAuthState::NoAuth, + requires_binding: false, installed: true, activation_error: None, version, @@ -2171,20 +2464,25 @@ impl ExtensionManager { .get_with_kind(&name, Some(ExtensionKind::WasmChannel)) .await; let display_name = registry_entry.as_ref().map(|e| e.display_name.clone()); - let version = if let Some(ref cap_path) = discovered.capabilities_path { - tokio::fs::read(cap_path) - .await - .ok() - .and_then(|bytes| { - crate::channels::wasm::ChannelCapabilitiesFile::from_bytes( - &bytes, - ) + let (version, requires_binding) = + if let Some(ref cap_path) = discovered.capabilities_path { + tokio::fs::read(cap_path) + .await .ok() - }) - .and_then(|cap| cap.version) - } else { - None - }; + .and_then(|bytes| { + crate::channels::wasm::ChannelCapabilitiesFile::from_bytes( + &bytes, + ) + .ok() + }) + .map(|cap| { + let requires_binding = cap.requires_binding(); + (cap.version, requires_binding) + }) + } else { + None + } + .unwrap_or((None, false)); let version = version.or_else(|| registry_entry.and_then(|e| e.version.clone())); extensions.push(InstalledExtension { @@ -2198,6 +2496,7 @@ impl ExtensionManager { tools: Vec::new(), needs_setup: auth_state == ToolAuthState::NeedsSetup, has_auth: auth_state != ToolAuthState::NoAuth, + requires_binding, installed: true, activation_error, version, @@ -2236,6 +2535,7 @@ impl ExtensionManager { tools: Vec::new(), needs_setup: false, has_auth: true, + requires_binding: false, installed: true, activation_error, version: None, @@ -2259,6 +2559,14 @@ impl ExtensionManager { if installed_names.contains(&(entry.name.clone(), entry.kind)) { continue; } + // Hidden registry entries (e.g. `telegram_mtproto` alongside the + // canonical `telegram` channel) are omitted from default-discovery + // surfaces so the agent doesn't enumerate them as competing + // options for "connect my X". They remain installable by explicit + // name via `tool_install`. Issue #3533. + if entry.hidden { + continue; + } extensions.push(InstalledExtension { name: entry.name, kind: entry.kind, @@ -2270,6 +2578,7 @@ impl ExtensionManager { tools: Vec::new(), needs_setup: false, has_auth: false, + requires_binding: false, installed: false, activation_error: None, version: entry.version, @@ -3280,10 +3589,56 @@ impl ExtensionManager { .await?; self.invalidate_latent_wasm_provider_actions_cache().await; + // Register the WASM tool with the engine's tool registry + // immediately so the model can call it without a separate + // enablement step. Auth is checked at execute time by + // `AuthManager::check_action_auth`, which raises an + // `Authentication` gate when the declared credential is + // missing — the inline-await machinery (#3133/#3166) parks + // the caller until OAuth completes, then retries the action. + // + // Best-effort: a registration failure here doesn't unwind the + // download. The user can retry via the existing /activate + // endpoint, or the next ensure_extension_ready cycle picks + // it up. We log so a CI failure isn't silent. The + // `InstallResult.message` reflects which arm we hit so the + // caller / UI can prompt for follow-up instead of optimistically + // claiming readiness when activation actually failed. + let activated = match self.activate_wasm_tool(name, &self.user_id).await { + Ok(_) => { + tracing::debug!( + extension = %name, + "Auto-registered WASM tool with registry on install" + ); + true + } + Err(e) => { + tracing::warn!( + extension = %name, + error = %e, + "Failed to auto-register WASM tool on install — \ + falling back to lazy activation. The tool will \ + not be callable until the user resolves the \ + activation error or completes setup." + ); + false + } + }; + + let message = if activated { + format!("WASM tool '{}' installed and ready.", name) + } else { + format!( + "WASM tool '{}' installed; activation failed — \ + retry via the activate endpoint or complete setup.", + name + ) + }; + Ok(InstallResult { name: name.to_string(), kind: ExtensionKind::WasmTool, - message: format!("WASM tool '{}' installed. Run activate to load it.", name), + message, }) } @@ -4216,14 +4571,21 @@ impl ExtensionManager { return ToolAuthState::NoAuth; } - let all_provided = futures::future::join_all( - required - .iter() - .map(|s| self.secrets.exists(user_id, &s.name)), - ) + let all_provided = futures::future::join_all(required.iter().map(|secret| async move { + let decrypted = match self.secrets.get_decrypted(user_id, &secret.name).await { + Ok(secret_value) => secret_value, + Err(_) => return false, + }; + validate_setup_secret_value( + &secret.name, + decrypted.expose(), + secret.validation.as_deref(), + ) + .is_ok() + })) .await .into_iter() - .all(|r| r.unwrap_or(false)); + .all(std::convert::identity); if all_provided { ToolAuthState::Ready @@ -5109,6 +5471,18 @@ impl ExtensionManager { .await .map_err(|e| e.to_string())?; + // Half-2 of #3133, two-pronged auto-resume. See + // `src/channels/web/features/oauth/mod.rs` for the + // matching wire on the gateway-OAuth path. + let _ = + crate::bridge::resolve_inline_gates_for_credential(&user_id, &secret_name) + .await; + let _ = crate::bridge::resume_paused_missions_for_credential( + &user_id, + &secret_name, + ) + .await; + Ok(()) } .await; @@ -5139,7 +5513,10 @@ impl ExtensionManager { } if let Some(ref sse) = sse_manager { - sse.broadcast(ironclaw_common::AppEvent::OnboardingState { + // Scope to the OAuth flow owner — a global broadcast + // would surface this onboarding state to every + // connected tenant tab. + let onboarding_event = ironclaw_common::AppEvent::OnboardingState { extension_name: ironclaw_common::ExtensionName::from_trusted(ext_name), state: if success { ironclaw_common::OnboardingStateDto::Ready @@ -5153,7 +5530,8 @@ impl ExtensionManager { setup_url: None, onboarding: None, thread_id: None, - }); + }; + sse.broadcast_for_user(&user_id, onboarding_event); // projection-exempt: channel-lifecycle, WASM extension OAuth completion } }); @@ -5478,6 +5856,15 @@ impl ExtensionManager { return Ok(AuthResult::authenticated(name, ExtensionKind::WasmChannel)); } + if name == WECHAT_CHANNEL_NAME { + return Ok(AuthResult::awaiting_token( + name, + ExtensionKind::WasmChannel, + "Open the WeChat channel setup to scan a QR code and connect it.".to_string(), + cap_file.setup.setup_url.clone(), + )); + } + // Prompt for the first missing secret let secret = &missing[0]; if let Some(auth_result) = self @@ -5925,6 +6312,7 @@ impl ExtensionManager { &channel_manager, &wasm_channel_router, wasm_channel_owner_ids.get(name).copied(), + user_id, ) .await } @@ -5936,6 +6324,7 @@ impl ExtensionManager { channel_manager: &Arc<ChannelManager>, wasm_channel_router: &Arc<WasmChannelRouter>, owner_id: Option<i64>, + activation_user_id: &str, ) -> Result<ActivateResult, ExtensionError> { let channel_name = loaded.name().to_string(); if is_reserved_wasm_channel_name(&channel_name) { @@ -5958,8 +6347,14 @@ impl ExtensionManager { let webhook_secret_name = loaded.webhook_secret_name(); let secret_header = loaded.webhook_secret_header().map(|s| s.to_string()); let webhook_secret_managed_by_host = loaded.webhook_secret_managed_by_host(); + let webhook_methods = loaded.webhook_methods(); let sig_key_secret_name = loaded.signature_key_secret_name(); let hmac_secret_name = loaded.hmac_secret_name(); + let channel_secret_scope_id = if channel_name == WECHAT_CHANNEL_NAME { + activation_user_id + } else { + &self.user_id + }; let secret_config_mappings = loaded .capabilities_file .as_ref() @@ -5969,7 +6364,7 @@ impl ExtensionManager { // Get webhook secret from secrets store let webhook_secret = self .secrets - .get_decrypted(&self.user_id, &webhook_secret_name) + .get_decrypted(channel_secret_scope_id, &webhook_secret_name) .await .ok() .map(|s| s.expose().to_string()); @@ -5984,12 +6379,12 @@ impl ExtensionManager { owner_actor_id.as_deref(), ); config_updates.extend( - self.load_channel_runtime_config_overrides(&channel_name) - .await, + self.load_channel_runtime_config_overrides(&channel_name, activation_user_id) + .await?, ); inject_wasm_channel_secret_config_mappings( &channel_name, - &self.user_id, + channel_secret_scope_id, self.secrets.as_ref(), &secret_config_mappings, &mut config_updates, @@ -6018,7 +6413,7 @@ impl ExtensionManager { let endpoints = vec![RegisteredEndpoint { channel_name: channel_name.clone(), path: webhook_path, - methods: vec!["POST".to_string()], + methods: webhook_methods, require_secret: host_webhook_secret.is_some(), }]; @@ -6036,7 +6431,7 @@ impl ExtensionManager { if let Some(ref sig_key_name) = sig_key_secret_name && let Ok(key_secret) = self .secrets - .get_decrypted(&self.user_id, sig_key_name) + .get_decrypted(channel_secret_scope_id, sig_key_name) .await { match wasm_channel_router @@ -6054,7 +6449,11 @@ impl ExtensionManager { // Register HMAC signing secret if declared in capabilities if let Some(hmac_name) = &hmac_secret_name { - match self.secrets.get_decrypted(&self.user_id, hmac_name).await { + match self + .secrets + .get_decrypted(channel_secret_scope_id, hmac_name) + .await + { Ok(secret) => { wasm_channel_router .register_hmac_secret(&channel_name, secret.expose()) @@ -6073,7 +6472,7 @@ impl ExtensionManager { &channel_arc, Some(self.secrets.as_ref()), &channel_name, - &self.user_id, + channel_secret_scope_id, ) .await { @@ -6146,6 +6545,10 @@ impl ExtensionManager { let webhook_path = format!("/webhook/{}", name); let existing_channel = match router.get_channel_for_path(&webhook_path).await { + Some(ch) => Some(ch), + None => router.get_channel_by_name(name).await, + }; + let existing_channel = match existing_channel { Some(ch) => ch, None => { return Ok(ActivateResult { @@ -6214,7 +6617,10 @@ impl ExtensionManager { None, owner_actor_id.as_deref(), ); - config_updates.extend(self.load_channel_runtime_config_overrides(name).await); + config_updates.extend( + self.load_channel_runtime_config_overrides(name, user_id) + .await?, + ); inject_wasm_channel_secret_config_mappings( name, &self.user_id, @@ -6462,6 +6868,15 @@ impl ExtensionManager { ExtensionError::AuthFailed(format!("Failed to store OAuth state: {e}")) })?; + // Store the initiating user_id so the OAuth callback knows which IronClaw + // user to pair with the Slack authed_user_id. + let user_key = format!("relay:{}:oauth_user", name); + let _ = self.secrets.delete(&self.user_id, &user_key).await; + self.secrets + .create(&self.user_id, CreateSecretParams::new(&user_key, user_id)) + .await + .map_err(|e| ExtensionError::AuthFailed(format!("Failed to store OAuth user: {e}")))?; + // Channel-relay derives all URLs from trusted instance_url in chat-api. // We only pass the nonce for CSRF validation on the callback. tracing::trace!( @@ -6612,7 +7027,7 @@ impl ExtensionManager { // Create the event channel for webhook callbacks let (event_tx, event_rx) = tokio::sync::mpsc::channel(64); - let channel = crate::channels::relay::RelayChannel::new_with_provider( + let mut channel = crate::channels::relay::RelayChannel::new_with_provider( client.clone(), crate::channels::relay::channel::RelayProvider::Slack, team_id.clone(), @@ -6620,6 +7035,9 @@ impl ExtensionManager { event_tx.clone(), event_rx, ); + if let Some(ref ps) = self.pairing_store { + channel = channel.with_pairing_store(Arc::clone(ps)); + } // Hot-add to channel manager let cm_guard = self.relay_channel_manager.read().await; @@ -6919,6 +7337,21 @@ impl ExtensionManager { } !expired }); + + let mut wechat_logins = self.pending_wechat_logins.write().await; + purge_expired_wechat_logins(&mut wechat_logins); + } + + fn interactive_login_info_for_extension( + name: &str, + kind: ExtensionKind, + ) -> Option<InteractiveLoginInfo> { + match (kind, name) { + (ExtensionKind::WasmChannel, WECHAT_CHANNEL_NAME) => { + Some(wechat_interactive_login_info()) + } + _ => None, + } } /// Get the setup schema for an extension (secret/text fields and their status). @@ -6940,10 +7373,22 @@ impl ExtensionManager { return Ok(ExtensionSetupSchema { secrets: Vec::new(), fields: Vec::new(), + interactive_login: Self::interactive_login_info_for_extension( + name, + ExtensionKind::WasmChannel, + ), }); } }; + if name == WECHAT_CHANNEL_NAME { + return Ok(ExtensionSetupSchema { + secrets: Vec::new(), + fields: Vec::new(), + interactive_login: Some(wechat_interactive_login_info()), + }); + } + let mut secrets = Vec::new(); for secret in &cap_file.setup.required_secrets { let provided = self @@ -6955,6 +7400,7 @@ impl ExtensionManager { name: secret.name.clone(), prompt: secret.prompt.clone(), optional: secret.optional, + validation: secret.validation.clone(), provided, auto_generate: secret.auto_generate.is_some(), }); @@ -6964,6 +7410,7 @@ impl ExtensionManager { Ok(ExtensionSetupSchema { secrets, fields: Vec::new(), + interactive_login: None, }) } ExtensionKind::WasmTool => { @@ -6971,6 +7418,7 @@ impl ExtensionManager { return Ok(ExtensionSetupSchema { secrets: Vec::new(), fields: Vec::new(), + interactive_login: None, }); }; @@ -6999,6 +7447,7 @@ impl ExtensionManager { name: secret.name.clone(), prompt: secret.prompt.clone(), optional: secret.optional, + validation: None, provided, auto_generate: false, }); @@ -7017,7 +7466,11 @@ impl ExtensionManager { }); } } - Ok(ExtensionSetupSchema { secrets, fields }) + Ok(ExtensionSetupSchema { + secrets, + fields, + interactive_login: None, + }) } ExtensionKind::ChannelRelay => { let relay_url_key = format!("extensions.{name}.relay_url"); @@ -7052,37 +7505,259 @@ impl ExtensionManager { provided: current_url.is_some(), input_type: crate::tools::wasm::ToolSetupFieldInputType::Text, }], + interactive_login: None, }) } _ => Ok(ExtensionSetupSchema { secrets: Vec::new(), fields: Vec::new(), + interactive_login: None, }), } } - /// Configure secrets and setup fields for an extension, then attempt activation. - /// - /// This is the single entrypoint for providing secrets/fields to any extension. - /// Both the chat auth flow and the Extensions tab setup form call this method. - /// - /// - Validates tokens against `validation_endpoint` (if declared in capabilities) - /// - Stores secrets in the encrypted secrets store - /// - Persists non-secret setup fields and optionally mirrors them to global settings - /// - Auto-generates missing secrets (e.g., webhook keys) - /// - Activates the extension after configuration - pub async fn configure( + async fn resolve_wechat_base_url(&self, user_id: &str) -> String { + if let Some(store) = &self.store + && let Ok(Some(serde_json::Value::String(value))) = store + .get_setting(user_id, WECHAT_BASE_URL_SETTING_PATH) + .await + { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + let cap_path = self + .wasm_channels_dir + .join(format!("{}.capabilities.json", WECHAT_CHANNEL_NAME)); + if let Ok(cap_bytes) = tokio::fs::read(&cap_path).await + && let Ok(cap_file) = + crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes) + && let Some(value) = cap_file + .config + .get("base_url") + .and_then(|value| value.as_str()) + { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + WECHAT_DEFAULT_BASE_URL.to_string() + } + + async fn resolve_wechat_bot_type(&self) -> String { + let cap_path = self + .wasm_channels_dir + .join(format!("{}.capabilities.json", WECHAT_CHANNEL_NAME)); + if let Ok(cap_bytes) = tokio::fs::read(&cap_path).await + && let Ok(cap_file) = + crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes) + && let Some(value) = cap_file + .config + .get("bot_type") + .and_then(|value| value.as_str()) + { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + WECHAT_DEFAULT_BOT_TYPE.to_string() + } + + pub async fn start_interactive_login( &self, name: &str, - secrets: &std::collections::HashMap<String, String>, - fields: &std::collections::HashMap<String, String>, + user_id: &str, + ) -> Result<InteractiveLoginStartResult, ExtensionError> { + Self::validate_extension_name(name)?; + let kind = self.determine_installed_kind(name, user_id).await?; + if Self::interactive_login_info_for_extension(name, kind).is_none() { + return Err(ExtensionError::AuthNotSupported(format!( + "Interactive login is not supported for '{}'", + name + ))); + } + + if name != WECHAT_CHANNEL_NAME { + return Err(ExtensionError::AuthNotSupported(format!( + "Interactive login is not implemented for '{}'", + name + ))); + } + + self.cleanup_expired_auths().await; + + let base_url = self.resolve_wechat_base_url(user_id).await; + let bot_type = self.resolve_wechat_bot_type().await; + #[cfg(test)] + let login_result = + if let Some(starter) = self.test_wechat_login_starter.read().await.as_ref() { + starter(user_id, &base_url, &bot_type) + } else { + start_wechat_login(user_id, &base_url, &bot_type).await + }; + #[cfg(not(test))] + let login_result = start_wechat_login(user_id, &base_url, &bot_type).await; + + let (session, result) = login_result?; + + self.pending_wechat_logins + .write() + .await + .insert(session.session_id.clone(), session); + + Ok(result) + } + + pub async fn poll_interactive_login( + &self, + name: &str, + session_id: &str, + user_id: &str, + ) -> Result<InteractiveLoginPollResult, ExtensionError> { + Self::validate_extension_name(name)?; + let kind = self.determine_installed_kind(name, user_id).await?; + if Self::interactive_login_info_for_extension(name, kind).is_none() { + return Err(ExtensionError::AuthNotSupported(format!( + "Interactive login is not supported for '{}'", + name + ))); + } + + if name != WECHAT_CHANNEL_NAME { + return Err(ExtensionError::AuthNotSupported(format!( + "Interactive login is not implemented for '{}'", + name + ))); + } + + self.cleanup_expired_auths().await; + + let mut session = { + let mut sessions = self.pending_wechat_logins.write().await; + let Some(existing) = sessions.get(session_id) else { + return Err(ExtensionError::Other( + "This WeChat login session no longer exists. Start again.".to_string(), + )); + }; + if existing.user_id != user_id { + return Err(ExtensionError::AuthFailed( + "This WeChat login session belongs to another user".to_string(), + )); + } + sessions.remove(session_id).ok_or_else(|| { + ExtensionError::Other( + "This WeChat login session no longer exists. Start again.".to_string(), + ) + })? + }; + + #[cfg(test)] + let outcome = if let Some(poller) = self.test_wechat_login_poller.read().await.clone() { + poller(&mut session) + } else { + poll_wechat_login(&mut session).await + }; + #[cfg(not(test))] + let outcome = poll_wechat_login(&mut session).await; + + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + if session.is_fresh() { + self.pending_wechat_logins + .write() + .await + .entry(session.session_id.clone()) + .or_insert(session); + } + return Err(error); + } + }; + + match outcome { + WechatLoginPollOutcome::Pending(result) => { + if !matches!(result.status.as_str(), "failed") && session.is_fresh() { + self.pending_wechat_logins + .write() + .await + .entry(session.session_id.clone()) + .or_insert(session); + } + Ok(result) + } + WechatLoginPollOutcome::Confirmed(confirmed) => { + if let Some(base_url) = confirmed.base_url.as_deref() + && let Some(store) = &self.store + { + let _ = store + .set_setting( + user_id, + WECHAT_BASE_URL_SETTING_PATH, + &serde_json::Value::String(base_url.to_string()), + ) + .await; + } + + let mut secrets = std::collections::HashMap::new(); + secrets.insert("wechat_bot_token".to_string(), confirmed.bot_token); + let configure = self + .configure(name, &secrets, &std::collections::HashMap::new(), user_id) + .await?; + + Ok(InteractiveLoginPollResult { + session_id: session_id.to_string(), + status: if configure.activated { + "succeeded".to_string() + } else { + "failed".to_string() + }, + message: if configure.activated { + format!( + "WeChat connected as {}. {}", + confirmed.ilink_bot_id, configure.message + ) + } else { + format!( + "WeChat login succeeded for {} but activation failed: {}", + confirmed.ilink_bot_id, configure.message + ) + }, + qr_code_url: None, + activated: Some(configure.activated), + }) + } + } + } + + /// Configure secrets and setup fields for an extension, then attempt activation. + /// + /// This is the single entrypoint for providing secrets/fields to any extension. + /// Both the chat auth flow and the Extensions tab setup form call this method. + /// + /// - Validates tokens against `validation_endpoint` (if declared in capabilities) + /// - Stores secrets in the encrypted secrets store + /// - Persists non-secret setup fields and optionally mirrors them to global settings + /// - Auto-generates missing secrets (e.g., webhook keys) + /// - Activates the extension after configuration + pub async fn configure( + &self, + name: &str, + secrets: &std::collections::HashMap<String, String>, + fields: &std::collections::HashMap<String, String>, user_id: &str, ) -> Result<ConfigureResult, ExtensionError> { let name = canonicalize_extension_name(name)?; let kind = self.determine_installed_kind(&name, user_id).await?; // Load allowed secret names and tool setup field definitions from capabilities. - let mut channel_cap_file: Option<crate::channels::wasm::ChannelCapabilitiesFile> = None; + let mut channel_secret_defs: Vec<crate::channels::wasm::SecretSetupSchema> = Vec::new(); + let mut channel_validation_endpoint: Option<String> = None; let (allowed_secrets, setup_fields): ( std::collections::HashSet<String>, Vec<crate::tools::wasm::ToolFieldSetupSchema>, @@ -7099,7 +7774,8 @@ impl ExtensionManager { .iter() .map(|s| s.name.clone()) .collect(); - channel_cap_file = Some(cap_file); + channel_secret_defs = cap_file.setup.required_secrets.clone(); + channel_validation_endpoint = cap_file.setup.validation_endpoint.clone(); (names, Vec::new()) } ExtensionKind::WasmTool => { @@ -7159,53 +7835,118 @@ impl ExtensionManager { .map(|f| (f.name.clone(), f)) .collect(); + let channel_secret_defs_by_name: std::collections::HashMap< + String, + crate::channels::wasm::SecretSetupSchema, + > = channel_secret_defs + .iter() + .cloned() + .map(|secret| (secret.name.clone(), secret)) + .collect(); + + for (secret_name, secret_value) in secrets { + let trimmed_value = secret_value.trim(); + if trimmed_value.is_empty() { + continue; + } + if let Some(secret_def) = channel_secret_defs_by_name.get(secret_name) { + validate_setup_secret_value( + secret_name, + trimmed_value, + secret_def.validation.as_deref(), + )?; + } + } + // Validate secrets against the validation_endpoint if declared in capabilities. - // The endpoint URL template uses {secret_name} placeholders that are - // substituted with the provided secret value before making the request. - // Skip for Telegram — validate_telegram_token() below does the same getMe - // call but also extracts bot_username, avoiding a redundant API round-trip. + // The endpoint URL template uses {secret_name} placeholders and resolves + // them from submitted values first, then falls back to stored secrets. + // Skip Telegram because validate_telegram_token() below performs getMe + // and extracts bot_username without a redundant round-trip. if name != TELEGRAM_CHANNEL_NAME - && let Some(ref cap_file) = channel_cap_file - && let Some(ref endpoint_template) = cap_file.setup.validation_endpoint - && let Some(secret_def) = cap_file - .setup - .required_secrets - .iter() - .find(|s| !s.optional && secrets.contains_key(&s.name)) - && let Some(token_value) = secrets.get(&secret_def.name) + && let Some(ref endpoint_template) = channel_validation_endpoint { - let token = token_value.trim(); - if !token.is_empty() { - let encoded = - url::form_urlencoded::byte_serialize(token.as_bytes()).collect::<String>(); - let url = endpoint_template.replace(&format!("{{{}}}", secret_def.name), &encoded); - // SSRF defense: block private IPs, localhost, cloud metadata endpoints - crate::tools::builtin::skill_tools::validate_fetch_url(&url) - .map_err(|e| ExtensionError::Other(format!("SSRF blocked: {}", e)))?; - let resp = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .build() - .map_err(|e| ExtensionError::Other(e.to_string()))? - .get(&url) - .send() - .await - .map_err(|e| { - // Log the raw error at debug level (may contain sensitive URL paths) - // but return a generic message to callers. - tracing::debug!( - is_timeout = e.is_timeout(), - is_connect = e.is_connect(), - status = e.status().map(|s| s.as_u16()), - "Token validation request failed" - ); - ExtensionError::Other("Token validation request failed".to_string()) - })?; - if !resp.status().is_success() { + let placeholder_names = validation_endpoint_placeholder_names(endpoint_template); + if let Some(secret_name) = + validation_endpoint_disallowed_placeholder(&placeholder_names, &allowed_secrets) + { + return Err(ExtensionError::Other(format!( + "Validation endpoint for extension '{name}' references undeclared secret placeholder '{secret_name}'" + ))); + } + + let mut validation_url = endpoint_template.to_string(); + let mut all_placeholders_resolved = true; + + for secret_name in &placeholder_names { + let resolved_value = if let Some(value) = secrets + .get(secret_name.as_str()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + { + Some(value) + } else { + self.secrets + .get_decrypted(user_id, secret_name) + .await + .ok() + .map(|secret| secret.expose().trim().to_string()) + .filter(|value| !value.is_empty()) + }; + + let Some(secret_value) = resolved_value else { + all_placeholders_resolved = false; + break; + }; + + let encoded = if name == TELEGRAM_CHANNEL_NAME + && secret_name.as_str() == "telegram_bot_token" + { + secret_value + } else { + url::form_urlencoded::byte_serialize(secret_value.as_bytes()).collect() + }; + validation_url = validation_url.replace(&format!("{{{secret_name}}}"), &encoded); + } + + if all_placeholders_resolved { + let parsed_validation_url = + crate::tools::builtin::skill_tools::validate_fetch_url(&validation_url) + .map_err(|e| ExtensionError::Other(format!("SSRF blocked: {}", e)))?; + let validation_target = + crate::tools::wasm::validate_and_resolve_http_target(&validation_url) + .await + .map_err(|e| ExtensionError::Other(format!("SSRF blocked: {}", e)))?; + let mut response = + crate::tools::wasm::ssrf_safe_client_builder_for_target(&validation_target) + .timeout(std::time::Duration::from_secs(10)) + .build() + .map_err(|e| ExtensionError::Other(e.to_string()))? + .get(parsed_validation_url) + .send() + .await + .map_err(|e| { + // Log the raw error at debug level (may contain sensitive URL paths) + // but return a generic message to callers. + tracing::debug!( + is_timeout = e.is_timeout(), + is_connect = e.is_connect(), + status = e.status().map(|s| s.as_u16()), + "Token validation request failed" + ); + ExtensionError::Other("Token validation request failed".to_string()) + })?; + let status = response.status(); + if !status.is_success() { return Err(ExtensionError::ValidationFailed(format!( "Invalid token (API returned {})", - resp.status() + status ))); } + let body = read_setup_validation_response_body(&mut response).await?; + if let Some(error) = validation_endpoint_body_error(&body) { + return Err(ExtensionError::ValidationFailed(error)); + } } } @@ -7305,8 +8046,8 @@ impl ExtensionManager { } // Auto-generate any missing secrets (channel-only feature) - if let Some(ref cap_file) = channel_cap_file { - for secret_def in &cap_file.setup.required_secrets { + if kind == ExtensionKind::WasmChannel { + for secret_def in &channel_secret_defs { if let Some(ref auto_gen) = secret_def.auto_generate { let already_provided = secrets .get(&secret_def.name) @@ -7336,6 +8077,30 @@ impl ExtensionManager { } } } + + for secret_def in &channel_secret_defs { + if secret_def.optional { + continue; + } + let submitted = secrets + .get(&secret_def.name) + .is_some_and(|v| !v.trim().is_empty()); + let stored = self + .secrets + .exists(user_id, &secret_def.name) + .await + .map_err(|e| ExtensionError::AuthFailed(e.to_string()))?; + if !submitted && !stored { + return Err(ExtensionError::ValidationFailed(format!( + "Required secret '{}' is missing for extension '{}'", + secret_def.name, name + ))); + } + } + } + + if kind == ExtensionKind::WasmChannel && name == WECHAT_CHANNEL_NAME { + self.persist_wechat_bound_user_id(user_id).await?; } // Validate Telegram bot token and persist bot_username for mention detection. @@ -7944,9 +8709,13 @@ mod tests { kind_allows_local_discovery, normalize_hosted_callback_url, read_crate_name_from_cargo_toml, send_telegram_text_message, telegram_bot_api_url, }; + use crate::extensions::wechat_login::{ + ConfirmedWechatLogin, PendingWechatLogin, WECHAT_BASE_URL_SETTING_PATH, + WECHAT_BOUND_USER_SETTING_PATH, WechatLoginPollOutcome, + }; use crate::extensions::{ - AuthHint, ExtensionError, ExtensionKind, ExtensionSource, InstallResult, RegistryEntry, - ToolAuthState, + AuthHint, ExtensionError, ExtensionKind, ExtensionSource, InstallResult, + InteractiveLoginPollResult, InteractiveLoginStartResult, RegistryEntry, ToolAuthState, }; use crate::pairing::PairingStore; use crate::secrets::CreateSecretParams; @@ -8020,6 +8789,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, } } @@ -8437,7 +9207,8 @@ mod tests { "test", vec!["telegram".to_string(), "discord-bot".to_string()], ) - .await; + .await + .expect("load"); assert_eq!(actual, vec!["telegram", "discord_bot"]); } @@ -8459,7 +9230,8 @@ mod tests { let actual = manager .load_startup_active_channels("test", vec!["telegram".to_string()]) - .await; + .await + .expect("load"); assert!( actual.is_empty(), @@ -8467,6 +9239,120 @@ mod tests { ); } + /// Headless server path: no DB / no settings store at all. The configured + /// list (from the setup wizard's `channels.wasm_channels`) is the only + /// source of truth, and channels listed there must be activated at + /// startup. This is the regression scenario from #3105. + #[tokio::test] + async fn load_startup_active_channels_falls_back_to_configured_when_no_settings_store() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let manager = make_test_manager_with_dirs( + None, + temp_dir.path().join("tools"), + temp_dir.path().join("channels"), + None, + ); + + let actual = manager + .load_startup_active_channels( + "test", + vec!["telegram".to_string(), "discord-bot".to_string()], + ) + .await + .expect("load"); + + assert_eq!(actual, vec!["telegram", "discord_bot"]); + } + + /// Settings-store errors must propagate. Silently returning the + /// configured list (or an empty list) on a DB outage would mask the + /// failure and quietly re-activate channels the user had deactivated. + #[tokio::test] + async fn load_startup_active_channels_propagates_settings_store_errors() { + use crate::db::SettingsStore; + use std::sync::Arc; + + struct FailingSettingsStore; + + #[async_trait::async_trait] + impl SettingsStore for FailingSettingsStore { + async fn get_setting( + &self, + _user_id: &str, + _key: &str, + ) -> Result<Option<serde_json::Value>, crate::error::DatabaseError> { + Err(crate::error::DatabaseError::Query("simulated".into())) + } + async fn get_setting_full( + &self, + _user_id: &str, + _key: &str, + ) -> Result<Option<crate::history::SettingRow>, crate::error::DatabaseError> + { + unreachable!() + } + async fn set_setting( + &self, + _user_id: &str, + _key: &str, + _value: &serde_json::Value, + ) -> Result<(), crate::error::DatabaseError> { + unreachable!() + } + async fn delete_setting( + &self, + _user_id: &str, + _key: &str, + ) -> Result<bool, crate::error::DatabaseError> { + unreachable!() + } + async fn list_settings( + &self, + _user_id: &str, + ) -> Result<Vec<crate::history::SettingRow>, crate::error::DatabaseError> { + unreachable!() + } + async fn get_all_settings( + &self, + _user_id: &str, + ) -> Result< + std::collections::HashMap<String, serde_json::Value>, + crate::error::DatabaseError, + > { + unreachable!() + } + async fn set_all_settings( + &self, + _user_id: &str, + _settings: &std::collections::HashMap<String, serde_json::Value>, + ) -> Result<(), crate::error::DatabaseError> { + unreachable!() + } + async fn has_settings( + &self, + _user_id: &str, + ) -> Result<bool, crate::error::DatabaseError> { + unreachable!() + } + } + + let temp_dir = tempfile::tempdir().expect("temp dir"); + let manager = make_test_manager_with_dirs( + None, + temp_dir.path().join("tools"), + temp_dir.path().join("channels"), + None, + ) + .with_settings_store(Arc::new(FailingSettingsStore)); + + let err = manager + .load_startup_active_channels("test", vec!["telegram".to_string()]) + .await + .expect_err("settings-store error must propagate"); + + assert!(matches!(err, crate::error::DatabaseError::Query(_))); + } + fn make_test_tar_gz(entries: &[(&str, &[u8])]) -> Vec<u8> { use flate2::Compression; use flate2::write::GzEncoder; @@ -8861,6 +9747,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }; let manager = make_test_manager_with_catalog( None, @@ -8888,6 +9775,81 @@ mod tests { ); } + /// Issue #3533 regression. Two telegram registry entries used to surface + /// to the agent as "Activatable Integrations" — the canonical `telegram` + /// channel and `telegram_mtproto` — and the model would correctly + /// enumerate them as competing options for "connect my telegram". Hidden + /// entries must be dropped from the available-but-not-installed appendix + /// of `list()`, so only the canonical channel reaches the prompt. + /// Hidden entries remain installable by explicit name via `tool_install`. + #[tokio::test] + async fn list_filters_hidden_registry_entries_from_available_set() { + let dir = tempfile::tempdir().expect("temp dir"); + let telegram = RegistryEntry { + name: "telegram".to_string(), + display_name: "Telegram Channel".to_string(), + kind: ExtensionKind::WasmChannel, + description: "Talk to your agent through a Telegram bot".to_string(), + keywords: vec!["telegram".into(), "messaging".into()], + source: ExtensionSource::WasmDownload { + wasm_url: "https://example.com/telegram.wasm".to_string(), + capabilities_url: None, + }, + fallback_source: None, + auth_hint: AuthHint::CapabilitiesAuth, + version: None, + hidden: false, + }; + let mtproto = RegistryEntry { + name: "telegram_mtproto".to_string(), + display_name: "Telegram Tool".to_string(), + kind: ExtensionKind::WasmTool, + description: "Direct MTProto integration".to_string(), + keywords: vec!["telegram".into(), "mtproto".into()], + source: ExtensionSource::WasmDownload { + wasm_url: "https://example.com/telegram_mtproto.wasm".to_string(), + capabilities_url: None, + }, + fallback_source: None, + auth_hint: AuthHint::CapabilitiesAuth, + version: None, + hidden: true, + }; + let manager = make_test_manager_with_catalog( + None, + dir.path().join("tools"), + dir.path().join("channels"), + None, + vec![telegram, mtproto], + ); + + let listed = manager.list(None, true, "test").await.expect("list"); + let entries: Vec<(&str, bool)> = listed + .iter() + .map(|e| (e.name.as_str(), e.installed)) + .collect(); + let names: Vec<&str> = entries.iter().map(|(name, _)| *name).collect(); + assert!( + names.contains(&"telegram"), + "canonical telegram channel must still surface: {entries:?}" + ); + assert!( + !names.contains(&"telegram_mtproto"), + "hidden registry entry must be filtered out of include_available list: {entries:?}" + ); + // Confirm the only telegram entry that surfaces is uninstalled — + // i.e. it came from the registry append path that the hidden filter + // governs, not from a real install discovered on disk. + let telegram_entry = entries + .iter() + .find(|(name, _)| *name == "telegram") + .expect("telegram entry"); + assert!( + !telegram_entry.1, + "test fixture: telegram must be uninstalled" + ); + } + #[tokio::test] async fn latent_provider_actions_include_cached_inactive_mcp_tools() { let dir = tempfile::tempdir().expect("temp dir"); @@ -9158,6 +10120,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }; let manager = make_test_manager_with_catalog( @@ -9241,6 +10204,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }; let manager = make_test_manager_with_catalog( @@ -9472,17 +10436,74 @@ mod tests { } #[tokio::test] - async fn test_configure_rejects_disallowed_setting_path() { + async fn test_configure_wasm_channel_validation_endpoint_rejects_loopback_url() { let dir = tempfile::tempdir().expect("temp dir"); - let (store, _db_dir) = make_test_store().await; - let tools_dir = write_test_tool( + let channels_dir = write_test_channel( dir.path(), - "evil-tool", + "loopback-channel", r#"{ + "version": "0.1.0", + "wit_version": "0.3.1", + "type": "channel", + "name": "loopback-channel", "setup": { - "required_fields": [ + "required_secrets": [ { - "name": "session", + "name": "sample_bot_token", + "prompt": "Sample bot token used only by this validation test.", + "optional": false + } + ], + "validation_endpoint": "http://127.0.0.1:9/validate?token={sample_bot_token}" + } + }"#, + ); + let tools_dir = dir.path().join("tools"); + let mgr = make_test_manager_with_dirs(None, tools_dir, channels_dir, None); + let secrets = std::collections::HashMap::from([( + "sample_bot_token".to_string(), + "should-not-persist".to_string(), + )]); + + let err = match mgr + .configure( + "loopback-channel", + &secrets, + &std::collections::HashMap::new(), + "test-user", + ) + .await + { + Ok(_) => panic!("loopback validation endpoint should be blocked"), + Err(err) => err, + }; + + let msg = err.to_string(); + assert!( + msg.contains("SSRF blocked"), + "unexpected error message: {msg}" + ); + assert!( + !mgr.secrets + .exists("test-user", "sample_bot_token") + .await + .expect("secret existence check"), + "validation failure must not persist the submitted token" + ); + } + + #[tokio::test] + async fn test_configure_rejects_disallowed_setting_path() { + let dir = tempfile::tempdir().expect("temp dir"); + let (store, _db_dir) = make_test_store().await; + let tools_dir = write_test_tool( + dir.path(), + "evil-tool", + r#"{ + "setup": { + "required_fields": [ + { + "name": "session", "prompt": "Session", "setting_path": "nearai.session_token" } @@ -10028,6 +11049,7 @@ mod tests { &channel_manager, &router, None, + "test", ) .await .map_err(|e| format!("activation failed: {e}"))?; @@ -10093,6 +11115,216 @@ mod tests { Ok(()) } + #[cfg(feature = "libsql")] + #[tokio::test] + async fn test_wechat_interactive_login_poll_persists_state_and_activates() -> Result<(), String> + { + let dir = tempfile::tempdir().map_err(|err| format!("temp dir: {err}"))?; + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).map_err(|err| format!("channels dir: {err}"))?; + std::fs::write(channels_dir.join("wechat.wasm"), b"mock") + .map_err(|err| format!("write wasm: {err}"))?; + std::fs::write( + channels_dir.join("wechat.capabilities.json"), + serde_json::to_vec(&serde_json::json!({ + "type": "channel", + "name": "wechat", + "setup": { + "required_secrets": [ + { + "name": "wechat_bot_token", + "prompt": "Connect WeChat", + "optional": false + } + ] + }, + "capabilities": { + "channel": { + "allowed_paths": ["/webhook/wechat"] + } + }, + "config": { + "base_url": "https://ilinkai.weixin.qq.com", + "bot_type": "3" + } + })) + .map_err(|err| format!("serialize capabilities: {err}"))?, + ) + .map_err(|err| format!("write capabilities: {err}"))?; + + let (db, _db_tmp) = crate::testing::test_db().await; + let manager = { + use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + use crate::testing::credentials::TEST_CRYPTO_KEY; + use crate::tools::ToolRegistry; + use crate::tools::mcp::process::McpProcessManager; + use crate::tools::mcp::session::McpSessionManager; + + let master_key = secrecy::SecretString::from(TEST_CRYPTO_KEY.to_string()); + let crypto = Arc::new( + SecretsCrypto::new(master_key) + .unwrap_or_else(|err| panic!("failed to construct test crypto: {err}")), + ); + + Arc::new(ExtensionManager::new( + Arc::new(McpSessionManager::new()), + Arc::new(McpProcessManager::new()), + Arc::new(InMemorySecretsStore::new(crypto)), + Arc::new(ToolRegistry::new()), + None, + None, + dir.path().join("tools"), + channels_dir.clone(), + None, + "test".to_string(), + Some(db.clone()), + Vec::new(), + )) + }; + + let channel_manager = Arc::new(ChannelManager::new()); + let runtime = Arc::new( + WasmChannelRuntime::new(WasmChannelRuntimeConfig::for_testing()) + .map_err(|err| format!("runtime: {err}"))?, + ); + let pairing_store = Arc::new(PairingStore::new( + Arc::clone(&db), + Arc::new(crate::ownership::OwnershipCache::new()), + )); + let router = Arc::new(WasmChannelRouter::new()); + manager + .set_channel_runtime( + Arc::clone(&channel_manager), + Arc::clone(&runtime), + Arc::clone(&pairing_store), + Arc::clone(&router), + std::collections::HashMap::new(), + ) + .await; + manager + .set_test_wasm_channel_loader(Arc::new({ + let runtime = Arc::clone(&runtime); + let pairing_store = Arc::clone(&pairing_store); + move |name| { + Ok(make_test_loaded_channel( + Arc::clone(&runtime), + name, + Arc::clone(&pairing_store), + )) + } + })) + .await; + manager + .set_test_wechat_login_starter(Arc::new(|user_id, base_url, bot_type| { + Ok(( + PendingWechatLogin { + user_id: user_id.to_string(), + session_id: "wechat-session-1".to_string(), + qrcode: "qr-123".to_string(), + qr_code_url: "https://qr.example/one".to_string(), + started_at: std::time::Instant::now(), + base_url: base_url.to_string(), + bot_type: bot_type.to_string(), + refresh_count: 0, + }, + InteractiveLoginStartResult { + session_id: "wechat-session-1".to_string(), + status: "pending".to_string(), + message: "Open the WeChat QR page to continue.".to_string(), + qr_code_url: Some("https://qr.example/one".to_string()), + instructions: Some( + "Keep this window open while you scan and confirm on your phone." + .to_string(), + ), + }, + )) + })) + .await; + manager + .set_test_wechat_login_poller(Arc::new(|session| { + if session.session_id != "wechat-session-1" { + return Err(ExtensionError::Other(format!( + "unexpected session id: {}", + session.session_id + ))); + } + Ok(WechatLoginPollOutcome::Confirmed(ConfirmedWechatLogin { + bot_token: "wechat-token-123".to_string(), + base_url: Some("https://wechat.example".to_string()), + ilink_bot_id: "wx-bot-1".to_string(), + })) + })) + .await; + + let start = manager + .start_interactive_login("wechat", "test") + .await + .map_err(|err| format!("start interactive login: {err}"))?; + require_eq( + start.session_id.clone(), + "wechat-session-1".to_string(), + "start session id", + )?; + require_eq(start.status, "pending".to_string(), "start status")?; + + let poll = manager + .poll_interactive_login("wechat", &start.session_id, "test") + .await + .map_err(|err| format!("poll interactive login: {err}"))?; + + require_eq(poll.status, "succeeded".to_string(), "poll status")?; + require_eq(poll.activated, Some(true), "poll activated")?; + require( + poll.message.contains("WeChat connected as wx-bot-1"), + format!("unexpected poll message: {}", poll.message), + )?; + require( + manager.active_channel_names.read().await.contains("wechat"), + "wechat should be marked active after successful login", + )?; + require( + channel_manager.get_channel("wechat").await.is_some(), + "wechat should be hot-added to the running channel manager", + )?; + require_eq( + manager.load_persisted_active_channels("test").await, + vec!["wechat".to_string()], + "persisted active channels", + )?; + require( + manager + .secrets + .exists("test", "wechat_bot_token") + .await + .map_err(|err| format!("check stored wechat token: {err}"))?, + "wechat bot token should be stored after successful login", + )?; + let persisted_base_url = manager + .store + .as_ref() + .ok_or_else(|| "db-backed manager missing".to_string())? + .get_setting("test", WECHAT_BASE_URL_SETTING_PATH) + .await + .map_err(|err| format!("wechat base_url setting query: {err}"))?; + require_eq( + persisted_base_url, + Some(serde_json::json!("https://wechat.example")), + "wechat base_url setting", + )?; + let persisted_bound_user = manager + .store + .as_ref() + .ok_or_else(|| "db-backed manager missing".to_string())? + .get_setting("test", WECHAT_BOUND_USER_SETTING_PATH) + .await + .map_err(|err| format!("wechat bound user setting query: {err}"))?; + require_eq( + persisted_bound_user, + Some(serde_json::json!("test")), + "wechat bound user setting", + ) + } + #[cfg(feature = "libsql")] #[tokio::test] async fn test_current_channel_owner_id_uses_store_fallback() -> Result<(), String> { @@ -10180,6 +11412,76 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_wechat_interactive_login_poll_releases_session_map_lock() -> Result<(), String> { + let dir = tempfile::tempdir().map_err(|err| format!("temp dir: {err}"))?; + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).map_err(|err| format!("channels dir: {err}"))?; + std::fs::write(channels_dir.join("wechat.wasm"), b"mock") + .map_err(|err| format!("write wasm: {err}"))?; + + let manager = Arc::new(make_manager_custom_dirs( + dir.path().join("tools"), + channels_dir.clone(), + )); + let session_id = "wechat-session-lock"; + manager.pending_wechat_logins.write().await.insert( + session_id.to_string(), + PendingWechatLogin { + user_id: "test".to_string(), + session_id: session_id.to_string(), + qrcode: "qr-123".to_string(), + qr_code_url: "https://qr.example/one".to_string(), + started_at: std::time::Instant::now(), + base_url: "https://ilinkai.weixin.qq.com".to_string(), + bot_type: "3".to_string(), + refresh_count: 0, + }, + ); + + let map_was_unlocked = Arc::new(std::sync::atomic::AtomicBool::new(false)); + manager + .set_test_wechat_login_poller(Arc::new({ + let manager = Arc::clone(&manager); + let map_was_unlocked = Arc::clone(&map_was_unlocked); + move |session| { + map_was_unlocked.store( + manager.pending_wechat_logins.try_write().is_ok(), + std::sync::atomic::Ordering::SeqCst, + ); + Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "pending".to_string(), + message: "Waiting for the QR code to be scanned.".to_string(), + qr_code_url: None, + activated: None, + }, + )) + } + })) + .await; + + let poll = manager + .poll_interactive_login("wechat", session_id, "test") + .await + .map_err(|err| format!("poll interactive login: {err}"))?; + + require_eq(poll.status, "pending".to_string(), "poll status")?; + require( + map_was_unlocked.load(std::sync::atomic::Ordering::SeqCst), + "pending_wechat_logins should not be locked while polling WeChat", + )?; + require( + manager + .pending_wechat_logins + .read() + .await + .contains_key(session_id), + "pending WeChat session should be written back after a pending poll", + ) + } + /// Regression for nearai/ironclaw#1921 — caller-level coverage. /// /// The web extensions list handler used to derive @@ -11962,6 +13264,184 @@ mod tests { ); } + #[tokio::test] + async fn test_configure_wasm_channel_rejects_invalid_secret_format() { + let dir = tempfile::tempdir().expect("temp dir"); + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).expect("channels dir"); + + std::fs::write(channels_dir.join("sample.wasm"), b"\0asm fake").expect("write wasm"); + let caps = serde_json::json!({ + "type": "channel", + "name": "sample", + "setup": { + "required_secrets": [ + { + "name": "sample_bot_token", + "prompt": "Enter a sample bot token", + "validation": "^tok_[A-Za-z0-9]{8}$" + } + ] + } + }); + std::fs::write( + channels_dir.join("sample.capabilities.json"), + serde_json::to_string(&caps).expect("serialize caps"), + ) + .expect("write capabilities"); + + let mgr = make_manager_custom_dirs(dir.path().join("tools"), channels_dir); + let err = mgr + .configure( + "sample", + &std::collections::HashMap::from([( + "sample_bot_token".to_string(), + "not-a-token".to_string(), + )]), + &std::collections::HashMap::new(), + "test", + ) + .await + .expect_err("invalid token should fail validation"); + + assert!( + matches!(err, ExtensionError::ValidationFailed(_)), + "expected ValidationFailed, got {err:?}" + ); + assert!( + !mgr.secrets + .exists("test", "sample_bot_token") + .await + .unwrap_or(true), + "invalid secret must not be persisted" + ); + } + + #[tokio::test] + async fn test_check_channel_auth_status_treats_invalid_stored_secret_as_needs_setup() { + let dir = tempfile::tempdir().expect("temp dir"); + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).expect("channels dir"); + + std::fs::write(channels_dir.join("sample.wasm"), b"\0asm fake").expect("write wasm"); + let caps = serde_json::json!({ + "type": "channel", + "name": "sample", + "setup": { + "required_secrets": [ + { + "name": "sample_signing_key", + "prompt": "Enter a sample signing key", + "validation": "^[A-Za-z0-9]{43}$" + } + ] + } + }); + std::fs::write( + channels_dir.join("sample.capabilities.json"), + serde_json::to_string(&caps).expect("serialize caps"), + ) + .expect("write capabilities"); + + let mgr = make_manager_custom_dirs(dir.path().join("tools"), channels_dir); + mgr.secrets + .create( + "test", + crate::secrets::CreateSecretParams::new( + "sample_signing_key", + "bad key with spaces", + ), + ) + .await + .expect("store invalid secret"); + + assert_eq!( + mgr.check_channel_auth_status("sample", "test").await, + ToolAuthState::NeedsSetup + ); + } + + #[test] + fn validation_endpoint_body_error_extracts_errcode_message() { + let error = + super::validation_endpoint_body_error(br#"{"errcode":40013,"errmsg":"invalid token"}"#) + .expect("non-zero errcode should be treated as failure"); + assert_eq!( + error, + "Validation endpoint returned errcode 40013: invalid token" + ); + assert!(super::validation_endpoint_body_error(br#"{"errcode":0,"errmsg":"ok"}"#).is_none()); + } + + #[test] + fn validation_response_exceeds_limit_detects_chunk_overflow() { + assert!(!super::validation_response_exceeds_limit(10, 20, 30)); + assert!(super::validation_response_exceeds_limit(10, 21, 30)); + assert!(super::validation_response_exceeds_limit(usize::MAX, 1, 30)); + } + + #[test] + fn validate_setup_secret_value_accepts_bounded_validation_pattern() { + assert!( + super::validate_setup_secret_value( + "sample_secret", + "tok_123abc", + Some(r"^tok_[a-z0-9]+$") + ) + .is_ok() + ); + } + + #[test] + fn validate_setup_secret_value_rejects_oversized_validation_pattern() { + let pattern = "a".repeat(super::SETUP_SECRET_VALIDATION_PATTERN_MAX_BYTES + 1); + let err = super::validate_setup_secret_value("sample_secret", "tok_123abc", Some(&pattern)) + .expect_err("oversized validation pattern should fail closed"); + + assert!( + matches!(&err, ExtensionError::Config(message) if message.contains("too large")), + "unexpected error: {err:?}" + ); + } + + #[test] + fn validation_endpoint_placeholder_names_extracts_unique_names_without_regex() { + let names = super::validation_endpoint_placeholder_names( + "https://api.example.com/validate?first={sample_secret}&second={sample_refresh_secret}&again={sample_secret}", + ); + + assert_eq!( + names.into_iter().collect::<Vec<_>>(), + vec![ + "sample_refresh_secret".to_string(), + "sample_secret".to_string() + ] + ); + } + + #[test] + fn validation_endpoint_disallowed_placeholder_rejects_undeclared_secret_names() { + let placeholders = std::collections::BTreeSet::from([ + "openai_api_key".to_string(), + "sample_secret".to_string(), + ]); + let allowed = std::collections::HashSet::from(["sample_secret".to_string()]); + + assert_eq!( + super::validation_endpoint_disallowed_placeholder(&placeholders, &allowed), + Some("openai_api_key") + ); + + let allowed = std::collections::HashSet::from([ + "openai_api_key".to_string(), + "sample_secret".to_string(), + ]); + assert_eq!( + super::validation_endpoint_disallowed_placeholder(&placeholders, &allowed), + None + ); + } + #[tokio::test] async fn test_auth_is_read_only_for_wasm_channel() { // Regression: auth() must be a pure status check — it must not store @@ -12108,6 +13588,47 @@ mod tests { ) } + #[tokio::test] + async fn test_configure_rejects_missing_required_channel_secret() { + let dir = tempfile::tempdir().expect("temp dir"); + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).unwrap(); + std::fs::write(channels_dir.join("test-channel.wasm"), b"\0asm fake").unwrap(); + std::fs::write( + channels_dir.join("test-channel.capabilities.json"), + serde_json::json!({ + "type": "channel", + "name": "test-channel", + "setup": { + "required_secrets": [ + { + "name": "test_channel_token", + "prompt": "Enter token", + "optional": false + } + ] + } + }) + .to_string(), + ) + .unwrap(); + + let mgr = make_manager_custom_dirs(dir.path().join("tools"), channels_dir); + let result = mgr + .configure( + "test-channel", + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + "test", + ) + .await; + + assert!( + matches!(result, Err(ExtensionError::ValidationFailed(_))), + "missing required channel secret should be a validation error: {result:?}" + ); + } + #[tokio::test] async fn test_configure_dispatches_activation_by_kind() { // Regression: configure() must dispatch to the correct activation method diff --git a/src/extensions/mod.rs b/src/extensions/mod.rs index dad5bb9a52f..1263798792a 100644 --- a/src/extensions/mod.rs +++ b/src/extensions/mod.rs @@ -8,19 +8,22 @@ //! - **MCP servers** — external API integrations via Model Context Protocol //! //! The agent can search a built-in registry (or discover online), install, -//! authenticate, and activate extensions at runtime without CLI commands. +//! and authenticate extensions at runtime without CLI commands. //! //! ```text //! User: "add telegram" //! -> tool_search("telegram") -> finds channel in registry -//! -> tool_install("telegram") -> installs and follows through setup/auth/activation when possible -//! -> tool_activate("telegram") -> used only if an installed extension still needs explicit activation +//! -> tool_install("telegram") -> installs and follows through setup/auth when possible +//! After install the new tools are direct-callable; missing OAuth tokens +//! raise an Authentication gate at execute time (see #3133 / #3166) and +//! resume automatically once the user completes the OAuth flow. //! ``` pub mod discovery; pub mod manager; pub mod naming; pub mod registry; +pub(crate) mod wechat_login; pub use discovery::OnlineDiscovery; pub use manager::ExtensionManager; @@ -74,13 +77,19 @@ pub struct RegistryEntry { /// Where to get this extension. pub source: ExtensionSource, /// Fallback source when the primary source fails (e.g., download 404 → build from source). - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub fallback_source: Option<Box<ExtensionSource>>, /// How authentication works. pub auth_hint: AuthHint, /// Extension version (semver), if known. - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub version: Option<String>, + /// When true, this entry is omitted from default-discovery surfaces + /// (agent prompt's `Activatable Integrations`, settings catalog). It is + /// still installable by explicit name. See `ExtensionManifest::hidden` + /// for the motivation (issue #3533). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub hidden: bool, } /// Where the extension binary or server lives. @@ -92,17 +101,14 @@ pub enum ExtensionSource { /// Downloadable WASM binary. WasmDownload { wasm_url: String, - #[serde(default)] capabilities_url: Option<String>, }, /// Build from local source directory. WasmBuildable { #[serde(alias = "repo_url")] source_dir: String, - #[serde(default)] build_dir: Option<String>, /// Crate name used to locate the build artifact binary. - #[serde(default)] crate_name: Option<String>, }, /// Discovered online (not yet validated for a specific source type). @@ -420,13 +426,9 @@ impl<'de> Deserialize<'de> for AuthResult { struct Raw { name: String, kind: ExtensionKind, - #[serde(default)] auth_url: Option<String>, - #[serde(default)] callback_type: Option<String>, - #[serde(default)] instructions: Option<String>, - #[serde(default)] setup_url: Option<String>, #[serde(default)] awaiting_token: bool, @@ -472,6 +474,52 @@ impl<'de> Deserialize<'de> for AuthResult { } } +/// Interactive login metadata surfaced to setup UIs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InteractiveLoginInfo { + /// Login method identifier (for example `qr_code`). + pub method: String, + /// User-facing button label. + pub button_label: String, + /// Optional short instructions shown above the login control. + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<String>, +} + +/// Result of starting an interactive extension login flow. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InteractiveLoginStartResult { + /// Opaque session identifier used by follow-up poll requests. + pub session_id: String, + /// Flow status (`pending`, `error`). + pub status: String, + /// Human-readable message for the UI. + pub message: String, + /// Optional QR/image URL for browser display. + #[serde(skip_serializing_if = "Option::is_none")] + pub qr_code_url: Option<String>, + /// Optional short instructions shown alongside the QR code. + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<String>, +} + +/// Result of polling an interactive extension login flow. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InteractiveLoginPollResult { + /// Session identifier associated with this poll result. + pub session_id: String, + /// Flow status (`pending`, `scanned`, `refreshed`, `succeeded`, `failed`). + pub status: String, + /// Human-readable message for the UI. + pub message: String, + /// Optional refreshed QR/image URL. + #[serde(skip_serializing_if = "Option::is_none")] + pub qr_code_url: Option<String>, + /// Whether the extension was successfully activated as part of login completion. + #[serde(skip_serializing_if = "Option::is_none")] + pub activated: Option<bool>, +} + /// Result of activating an extension. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ActivateResult { @@ -612,6 +660,10 @@ pub struct InstalledExtension { /// Whether this extension has an auth configuration (OAuth or manual token). #[serde(default)] pub has_auth: bool, + /// Whether this extension still needs owner binding / pairing before it should + /// be treated as fully active in the UI. + #[serde(default)] + pub requires_binding: bool, /// Whether this extension is installed locally (false = available in registry but not installed). #[serde(default = "default_true")] pub installed: bool, @@ -983,6 +1035,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; let sr = SearchResult { entry, @@ -1013,6 +1066,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }; let sr = SearchResult { entry, @@ -1098,6 +1152,7 @@ mod tests { tools: vec!["send_email".to_string(), "read_inbox".to_string()], needs_setup: true, has_auth: true, + requires_binding: false, installed: false, activation_error: Some("token expired".to_string()), version: None, diff --git a/src/extensions/registry.rs b/src/extensions/registry.rs index a7bd66978a1..77fe6d82031 100644 --- a/src/extensions/registry.rs +++ b/src/extensions/registry.rs @@ -50,7 +50,11 @@ impl ExtensionRegistry { /// Search the registry by query string. Returns results sorted by relevance. /// /// Splits the query into lowercase tokens and scores each entry by matches - /// in name, keywords, and description. + /// in name, keywords, and description. Entries marked `hidden: true` + /// (e.g. `telegram_mtproto` alongside the canonical `telegram` channel) + /// are omitted from search results so the agent doesn't enumerate them + /// as competing options. They remain installable by explicit name via + /// `tool_install` (`get`/`get_with_kind` do not filter). pub async fn search(&self, query: &str) -> Vec<SearchResult> { let tokens: Vec<String> = query .to_lowercase() @@ -63,6 +67,7 @@ impl ExtensionRegistry { return self .entries .iter() + .filter(|e| !e.hidden) .map(|e| SearchResult { entry: e.clone(), source: ResultSource::Registry, @@ -75,6 +80,9 @@ impl ExtensionRegistry { // Score built-in entries for entry in &self.entries { + if entry.hidden { + continue; + } let score = score_entry(entry, &tokens); if score > 0 { scored.push(( @@ -91,6 +99,9 @@ impl ExtensionRegistry { // Score cached discoveries let cache = self.discovery_cache.read().await; for entry in cache.iter() { + if entry.hidden { + continue; + } let score = score_entry(entry, &tokens); if score > 0 { scored.push(( @@ -270,6 +281,7 @@ pub fn builtin_entries_with_relay(relay_url: Option<String>) -> Vec<RegistryEntr fallback_source: None, auth_hint: AuthHint::ChannelRelayOAuth, version: None, + hidden: false, }); } @@ -295,6 +307,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; let score = score_entry(&entry, &["notion".to_string()]); @@ -319,6 +332,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; let score = score_entry(&entry, &["calendar".to_string()]); @@ -343,6 +357,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; let score = score_entry(&entry, &["wiki".to_string()]); @@ -367,6 +382,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; let score = score_entry(&entry, &["xyzfoobar".to_string()]); @@ -412,6 +428,37 @@ mod tests { assert!(linear_pos.is_some(), "Linear should appear in results"); } + /// Hidden registry entries (e.g. `telegram_mtproto`) must not surface + /// through `search()` — otherwise the agent re-introduces the "two + /// Telegram options" outcome that #3533 fixes for `list()`. Hidden + /// entries remain installable by exact name (via `get`/`get_with_kind`). + #[tokio::test] + async fn test_search_skips_hidden_entries() { + let registry = registry_with_catalog(); + + let results = registry.search("telegram").await; + assert!( + results.iter().all(|r| r.entry.name != "telegram_mtproto"), + "telegram_mtproto is hidden and must not surface in search; got: {:?}", + results.iter().map(|r| &r.entry.name).collect::<Vec<_>>() + ); + + let empty_results = registry.search("").await; + assert!( + empty_results + .iter() + .all(|r| r.entry.name != "telegram_mtproto"), + "empty-query search must also exclude hidden entries" + ); + + // Hidden entries remain installable by exact name. + let exact = registry.get("telegram_mtproto").await; + assert!( + exact.is_some(), + "hidden entries must still be retrievable by exact name" + ); + } + #[tokio::test] async fn test_search_gws_alias_finds_google_tools() { let registry = registry_with_catalog(); @@ -456,6 +503,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }; registry.cache_discovered(vec![discovered]).await; @@ -483,6 +531,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }; registry.cache_discovered(vec![entry.clone()]).await; @@ -509,6 +558,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, // Two entries with same name but different kinds should coexist RegistryEntry { @@ -523,6 +573,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }, RegistryEntry { name: "dual-ext".to_string(), @@ -538,6 +589,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, ]; @@ -576,6 +628,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }, RegistryEntry { name: "test-ext".to_string(), @@ -589,6 +642,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }, ]; @@ -618,6 +672,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, RegistryEntry { name: "telegram".to_string(), @@ -633,6 +688,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, ]; @@ -696,6 +752,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }; let channel_entry = RegistryEntry { name: "cached-ext".to_string(), @@ -711,6 +768,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }; registry @@ -755,6 +813,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, RegistryEntry { name: "telegram".to_string(), @@ -770,6 +829,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, version: None, + hidden: false, }, ]; @@ -819,6 +879,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }, RegistryEntry { name: "myext".to_string(), @@ -834,6 +895,7 @@ mod tests { fallback_source: None, auth_hint: AuthHint::None, version: None, + hidden: false, }, ]; diff --git a/src/extensions/wechat_login.rs b/src/extensions/wechat_login.rs new file mode 100644 index 00000000000..55b7f4157cf --- /dev/null +++ b/src/extensions/wechat_login.rs @@ -0,0 +1,588 @@ +use std::time::{Duration, Instant}; + +use reqwest::Client; +use serde::Deserialize; +use uuid::Uuid; + +use crate::extensions::{ + ExtensionError, InteractiveLoginInfo, InteractiveLoginPollResult, InteractiveLoginStartResult, +}; + +pub(crate) const WECHAT_CHANNEL_NAME: &str = "wechat"; +pub(crate) const WECHAT_BASE_URL_SETTING_PATH: &str = "extensions.wechat.base_url"; +pub(crate) const WECHAT_BOUND_USER_SETTING_PATH: &str = "extensions.wechat.bound_user_id"; +pub(crate) const WECHAT_DEFAULT_BASE_URL: &str = "https://ilinkai.weixin.qq.com"; +pub(crate) const WECHAT_DEFAULT_BOT_TYPE: &str = "3"; + +const LOGIN_SESSION_TTL: Duration = Duration::from_secs(5 * 60); +const QR_LONG_POLL_TIMEOUT: Duration = Duration::from_secs(35); +const QR_FETCH_TIMEOUT: Duration = Duration::from_secs(15); +const MAX_QR_REFRESH_COUNT: u8 = 3; +const WECHAT_ALLOWED_LOGIN_HOST: &str = "ilinkai.weixin.qq.com"; + +#[derive(Debug, Clone)] +pub(crate) struct PendingWechatLogin { + pub user_id: String, + pub session_id: String, + pub qrcode: String, + pub qr_code_url: String, + pub started_at: Instant, + pub base_url: String, + pub bot_type: String, + pub refresh_count: u8, +} + +impl PendingWechatLogin { + pub fn is_fresh(&self) -> bool { + self.started_at.elapsed() < LOGIN_SESSION_TTL + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ConfirmedWechatLogin { + pub bot_token: String, + pub base_url: Option<String>, + pub ilink_bot_id: String, +} + +pub(crate) enum WechatLoginPollOutcome { + Pending(InteractiveLoginPollResult), + Confirmed(ConfirmedWechatLogin), +} + +#[derive(Debug, Clone, Deserialize)] +struct QrCodeResponse { + qrcode: String, + qrcode_img_content: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct QrStatusResponse { + status: String, + bot_token: Option<String>, + ilink_bot_id: Option<String>, + baseurl: Option<String>, +} + +fn validate_wechat_login_base_url(raw: &str) -> Result<String, ExtensionError> { + // Trust model: WeChat QR login trusts the system CA store for HTTPS validation + // to allowed WeChat domains. We do not certificate-pin iLink endpoints. + let parsed = reqwest::Url::parse(raw).map_err(|e| { + ExtensionError::AuthFailed(format!("WeChat login returned an invalid base URL: {e}")) + })?; + + if parsed.scheme() != "https" { + return Err(ExtensionError::AuthFailed( + "WeChat login returned a non-HTTPS base URL".to_string(), + )); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err(ExtensionError::AuthFailed( + "WeChat login returned a base URL with embedded credentials".to_string(), + )); + } + if parsed.query().is_some() || parsed.fragment().is_some() { + return Err(ExtensionError::AuthFailed( + "WeChat login returned a base URL with an unexpected query or fragment".to_string(), + )); + } + if parsed.path() != "/" && !parsed.path().is_empty() { + return Err(ExtensionError::AuthFailed( + "WeChat login returned a base URL with an unexpected path".to_string(), + )); + } + + let Some(host) = parsed.host_str() else { + return Err(ExtensionError::AuthFailed( + "WeChat login returned a base URL without a host".to_string(), + )); + }; + if host != WECHAT_ALLOWED_LOGIN_HOST { + return Err(ExtensionError::AuthFailed(format!( + "WeChat login returned an untrusted base URL host: {host}" + ))); + } + + Ok(format!("https://{WECHAT_ALLOWED_LOGIN_HOST}")) +} + +pub(crate) fn interactive_login_info() -> InteractiveLoginInfo { + InteractiveLoginInfo { + method: "qr_code".to_string(), + button_label: "Connect WeChat".to_string(), + instructions: Some("Scan the QR code with WeChat to connect this channel.".to_string()), + } +} + +pub(crate) fn purge_expired_logins( + sessions: &mut std::collections::HashMap<String, PendingWechatLogin>, +) { + sessions.retain(|_, session| session.is_fresh()); +} + +pub(crate) async fn start_login( + user_id: &str, + base_url: &str, + bot_type: &str, +) -> Result<(PendingWechatLogin, InteractiveLoginStartResult), ExtensionError> { + let base_url = validate_wechat_login_base_url(base_url)?; + let qr = fetch_qr_code(&base_url, bot_type).await?; + Ok(build_pending_login(user_id, &base_url, bot_type, qr)) +} + +pub(crate) async fn poll_login( + session: &mut PendingWechatLogin, +) -> Result<WechatLoginPollOutcome, ExtensionError> { + if !session.is_fresh() { + return Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "failed".to_string(), + message: "The QR code expired. Start a new WeChat connection.".to_string(), + qr_code_url: None, + activated: Some(false), + }, + )); + } + + let status = poll_qr_status(&session.base_url, &session.qrcode).await?; + let refreshed_qr = if status.status == "expired" && session.refresh_count < MAX_QR_REFRESH_COUNT + { + Some(fetch_qr_code(&session.base_url, &session.bot_type).await?) + } else { + None + }; + + handle_poll_status(session, status, refreshed_qr) +} + +fn build_pending_login( + user_id: &str, + base_url: &str, + bot_type: &str, + qr: QrCodeResponse, +) -> (PendingWechatLogin, InteractiveLoginStartResult) { + let session_id = Uuid::new_v4().to_string(); + let session = PendingWechatLogin { + user_id: user_id.to_string(), + session_id: session_id.clone(), + qrcode: qr.qrcode, + qr_code_url: qr.qrcode_img_content.clone(), + started_at: Instant::now(), + base_url: base_url.to_string(), + bot_type: bot_type.to_string(), + refresh_count: 0, + }; + + let result = InteractiveLoginStartResult { + session_id, + status: "pending".to_string(), + message: "Open the WeChat QR page to continue.".to_string(), + qr_code_url: Some(qr.qrcode_img_content), + instructions: Some( + "Keep this window open while you scan and confirm on your phone.".to_string(), + ), + }; + + (session, result) +} + +fn handle_poll_status( + session: &mut PendingWechatLogin, + status: QrStatusResponse, + refreshed_qr: Option<QrCodeResponse>, +) -> Result<WechatLoginPollOutcome, ExtensionError> { + match status.status.as_str() { + "wait" => Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "pending".to_string(), + message: "Waiting for the QR code to be scanned.".to_string(), + qr_code_url: None, + activated: None, + }, + )), + "scaned" => Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "scanned".to_string(), + message: "QR code scanned. Confirm the login in WeChat.".to_string(), + qr_code_url: None, + activated: None, + }, + )), + "expired" => { + session.refresh_count = session.refresh_count.saturating_add(1); + if session.refresh_count > MAX_QR_REFRESH_COUNT { + return Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "failed".to_string(), + message: "The QR code expired too many times. Start again.".to_string(), + qr_code_url: None, + activated: Some(false), + }, + )); + } + + let refreshed = refreshed_qr.ok_or_else(|| { + ExtensionError::Other( + "WeChat QR status expired without a refreshed QR code".to_string(), + ) + })?; + session.qrcode = refreshed.qrcode; + session.qr_code_url = refreshed.qrcode_img_content.clone(); + session.started_at = Instant::now(); + + Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "refreshed".to_string(), + message: "The QR code expired, so a fresh one was generated.".to_string(), + qr_code_url: Some(refreshed.qrcode_img_content), + activated: None, + }, + )) + } + "confirmed" => { + let bot_token = status.bot_token.filter(|token| !token.trim().is_empty()); + let ilink_bot_id = status + .ilink_bot_id + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + ExtensionError::Other( + "WeChat login succeeded but no bot account id was returned".to_string(), + ) + })?; + + let bot_token = bot_token.ok_or_else(|| { + ExtensionError::Other( + "WeChat login succeeded but no bot token was returned".to_string(), + ) + })?; + let base_url = status + .baseurl + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(validate_wechat_login_base_url) + .transpose()?; + + Ok(WechatLoginPollOutcome::Confirmed(ConfirmedWechatLogin { + bot_token, + base_url, + ilink_bot_id, + })) + } + other => { + tracing::warn!(status = other, "Unexpected WeChat QR status"); + Ok(WechatLoginPollOutcome::Pending( + InteractiveLoginPollResult { + session_id: session.session_id.clone(), + status: "failed".to_string(), + message: format!("Unexpected WeChat login status: {other}"), + qr_code_url: None, + activated: Some(false), + }, + )) + } + } +} + +fn ensure_trailing_slash(base_url: &str) -> String { + if base_url.ends_with('/') { + base_url.to_string() + } else { + format!("{base_url}/") + } +} + +async fn fetch_qr_code(base_url: &str, bot_type: &str) -> Result<QrCodeResponse, ExtensionError> { + let base_url = validate_wechat_login_base_url(base_url)?; + let base = ensure_trailing_slash(&base_url); + let url = format!( + "{base}ilink/bot/get_bot_qrcode?bot_type={}", + urlencoding::encode(bot_type) + ); + let client = Client::builder() + .timeout(QR_FETCH_TIMEOUT) + .build() + .map_err(|e| ExtensionError::Other(format!("Failed to create WeChat login client: {e}")))?; + + let response = client + .get(&url) + .send() + .await + .map_err(|e| ExtensionError::Other(format!("Failed to fetch WeChat QR code: {e}")))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::warn!(status = %status, "WeChat QR code request failed"); + return Err(ExtensionError::Other(format!( + "WeChat QR code request failed with {status}: {body}" + ))); + } + + response + .json::<QrCodeResponse>() + .await + .map_err(|e| ExtensionError::Other(format!("Failed to parse WeChat QR code response: {e}"))) +} + +async fn poll_qr_status(base_url: &str, qrcode: &str) -> Result<QrStatusResponse, ExtensionError> { + let base_url = validate_wechat_login_base_url(base_url)?; + let base = ensure_trailing_slash(&base_url); + let url = format!( + "{base}ilink/bot/get_qrcode_status?qrcode={}", + urlencoding::encode(qrcode) + ); + let client = Client::builder() + .timeout(QR_LONG_POLL_TIMEOUT) + .build() + .map_err(|e| ExtensionError::Other(format!("Failed to create WeChat poll client: {e}")))?; + + let response = client + .get(&url) + .header("iLink-App-ClientVersion", "1") + .send() + .await; + + let response = match response { + Ok(response) => response, + Err(error) if error.is_timeout() => { + return Ok(QrStatusResponse { + status: "wait".to_string(), + bot_token: None, + ilink_bot_id: None, + baseurl: None, + }); + } + Err(error) => { + return Err(ExtensionError::Other(format!( + "Failed to poll WeChat QR status: {error}" + ))); + } + }; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::warn!(status = %status, "WeChat QR status poll failed"); + return Err(ExtensionError::Other(format!( + "WeChat QR status poll failed with {status}: {body}" + ))); + } + + response + .json::<QrStatusResponse>() + .await + .map_err(|e| ExtensionError::Other(format!("Failed to parse WeChat QR status: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::{ + MAX_QR_REFRESH_COUNT, QrCodeResponse, QrStatusResponse, WechatLoginPollOutcome, + build_pending_login, handle_poll_status, start_login, + }; + use crate::extensions::ExtensionError; + + #[test] + fn test_build_pending_login_returns_qr_state_and_result() { + let (session, start_result) = build_pending_login( + "owner", + "https://ilink.example", + "3", + QrCodeResponse { + qrcode: "qr-123".to_string(), + qrcode_img_content: "https://qr.example/one".to_string(), + }, + ); + + assert_eq!(session.user_id, "owner"); + assert_eq!(session.base_url, "https://ilink.example"); + assert_eq!(session.bot_type, "3"); + assert_eq!(session.qrcode, "qr-123"); + assert_eq!(session.qr_code_url, "https://qr.example/one"); + assert_eq!(start_result.status, "pending"); + assert_eq!( + start_result.qr_code_url.as_deref(), + Some("https://qr.example/one") + ); + assert_eq!(start_result.session_id, session.session_id); + } + + #[test] + fn test_handle_poll_status_confirms_login() -> Result<(), String> { + let (mut session, _) = build_pending_login( + "owner", + "https://ilink.example", + "3", + QrCodeResponse { + qrcode: "qr-123".to_string(), + qrcode_img_content: "https://qr.example/one".to_string(), + }, + ); + let outcome = handle_poll_status( + &mut session, + QrStatusResponse { + status: "confirmed".to_string(), + bot_token: Some("bot-token-123".to_string()), + ilink_bot_id: Some("wx-bot-1".to_string()), + baseurl: Some("https://ilinkai.weixin.qq.com".to_string()), + }, + None, + ) + .map_err(|e| e.to_string())?; + + match outcome { + WechatLoginPollOutcome::Confirmed(confirmed) => { + assert_eq!(confirmed.bot_token, "bot-token-123"); + assert_eq!(confirmed.ilink_bot_id, "wx-bot-1"); + assert_eq!( + confirmed.base_url.as_deref(), + Some("https://ilinkai.weixin.qq.com") + ); + Ok(()) + } + WechatLoginPollOutcome::Pending(result) => Err(format!( + "expected confirmed login, got pending status {}", + result.status + )), + } + } + + #[test] + fn test_handle_poll_status_refreshes_expired_qr() -> Result<(), String> { + let (mut session, _) = build_pending_login( + "owner", + "https://ilink.example", + "3", + QrCodeResponse { + qrcode: "qr-initial".to_string(), + qrcode_img_content: "https://qr.example/initial".to_string(), + }, + ); + let outcome = handle_poll_status( + &mut session, + QrStatusResponse { + status: "expired".to_string(), + bot_token: None, + ilink_bot_id: None, + baseurl: None, + }, + Some(QrCodeResponse { + qrcode: "qr-refreshed".to_string(), + qrcode_img_content: "https://qr.example/refreshed".to_string(), + }), + ) + .map_err(|e| e.to_string())?; + + match outcome { + WechatLoginPollOutcome::Pending(result) => { + assert_eq!(result.status, "refreshed"); + assert_eq!( + result.qr_code_url.as_deref(), + Some("https://qr.example/refreshed") + ); + assert_eq!(session.qrcode, "qr-refreshed"); + assert_eq!(session.refresh_count, 1); + Ok(()) + } + WechatLoginPollOutcome::Confirmed(_) => { + Err("expected QR refresh before confirmation".to_string()) + } + } + } + + #[test] + fn test_handle_poll_status_fails_after_qr_refresh_exhaustion() -> Result<(), String> { + let (mut session, _) = build_pending_login( + "owner", + "https://ilink.example", + "3", + QrCodeResponse { + qrcode: "qr-initial".to_string(), + qrcode_img_content: "https://qr.example/initial".to_string(), + }, + ); + session.refresh_count = MAX_QR_REFRESH_COUNT; + + let outcome = handle_poll_status( + &mut session, + QrStatusResponse { + status: "expired".to_string(), + bot_token: None, + ilink_bot_id: None, + baseurl: None, + }, + None, + ) + .map_err(|e| e.to_string())?; + + match outcome { + WechatLoginPollOutcome::Pending(result) => { + assert_eq!(result.status, "failed"); + assert_eq!(result.activated, Some(false)); + assert!( + result.message.contains("expired too many times"), + "unexpected message: {}", + result.message + ); + Ok(()) + } + WechatLoginPollOutcome::Confirmed(_) => { + Err("expected refresh exhaustion failure".to_string()) + } + } + } + + #[test] + fn test_handle_poll_status_rejects_untrusted_base_url() { + let (mut session, _) = build_pending_login( + "owner", + "https://ilink.example", + "3", + QrCodeResponse { + qrcode: "qr-123".to_string(), + qrcode_img_content: "https://qr.example/one".to_string(), + }, + ); + + let error = match handle_poll_status( + &mut session, + QrStatusResponse { + status: "confirmed".to_string(), + bot_token: Some("bot-token-123".to_string()), + ilink_bot_id: Some("wx-bot-1".to_string()), + baseurl: Some("https://evil.example".to_string()), + }, + None, + ) { + Ok(_) => panic!("untrusted host should fail"), + Err(error) => error, + }; + + match error { + ExtensionError::AuthFailed(message) => { + assert!(message.contains("untrusted base URL host")); + } + other => panic!("expected AuthFailed, got {other:?}"), + } + } + + #[tokio::test] + async fn test_start_login_rejects_untrusted_base_url_before_http() { + let error = start_login("owner", "https://evil.example", "3") + .await + .expect_err("untrusted host should fail before fetching QR"); + + match error { + ExtensionError::AuthFailed(message) => { + assert!(message.contains("untrusted base URL host")); + } + other => panic!("expected AuthFailed, got {other:?}"), + } + } +} diff --git a/src/gate/store.rs b/src/gate/store.rs index dbd4e248e0b..c5fabe631c1 100644 --- a/src/gate/store.rs +++ b/src/gate/store.rs @@ -53,6 +53,12 @@ pub enum GateStoreError { #[error("a gate is already pending for this thread")] AlreadyExists, + /// A gate matches the supplied request id but belongs to a different + /// user. Distinct from `NotFound` so callers (HTTP handlers) can + /// return 403 without leaking whether the gate exists at all. + #[error("not authorized to resolve this gate")] + Unauthorized, + #[error("persistence error: {reason}")] Persistence { reason: String }, } @@ -193,6 +199,32 @@ impl PendingGateStore { .map(PendingGateView::from) } + /// Read-only peek at a pending gate keyed by `request_id`, scoped to + /// the requesting user. Returns `None` if no gate matches, the gate + /// is owned by another user, or it has expired. + /// + /// Used by the foreground cancel path to recover the owning thread + /// when the client omits `thread_id` in the resolution payload — + /// without this, a foreground inline-await gate would be stranded + /// (gate marked cancelled, parked VM never unwound). See PR #3366 + /// review. + pub async fn peek_by_request_id( + &self, + request_id: Uuid, + expected_user_id: &str, + ) -> Option<PendingGateView> { + let inner = self.inner.lock().await; + let key = inner.by_request_id.get(&request_id)?; + if key.user_id != expected_user_id { + return None; + } + inner + .by_key + .get(key) + .filter(|g| !g.is_expired()) + .map(PendingGateView::from) + } + /// List all non-expired gates for a user. pub async fn list_for_user(&self, user_id: &str) -> Vec<PendingGate> { let inner = self.inner.lock().await; @@ -204,6 +236,85 @@ impl PendingGateStore { .collect() } + /// Atomically take a pending gate by `request_id`, verifying user + /// ownership, channel authorization, and expiry under a single lock. + /// + /// Mirrors [`take_verified`], but resolves the composite key from + /// the wire `request_id` first. Used by HTTP surfaces (the + /// inline-await fast path) where the caller has only the + /// channel-visible thread identifier — which for the web gateway is + /// recorded on the gate as `scope_thread_id`, not as the internal + /// engine `ThreadId`. Looking up by `request_id` (unique + /// system-wide) avoids the wire-vs.-engine identifier confusion + /// that would otherwise miss the gate entirely. + /// + /// Returns: + /// - `Ok(gate)` on success — the gate is removed from both indices. + /// - `Err(NotFound)` when no gate matches `request_id` (already + /// resolved, never existed, or unrecoverable after a restart). + /// - `Err(Unauthorized)` when a gate exists but `expected_user_id` + /// does not own it. This is intentionally distinct from + /// `NotFound` so callers can surface a 403 without leaking gate + /// existence across tenants. + /// - `Err(ChannelMismatch | Expired)` — same semantics as + /// [`take_verified`]. + /// + /// [`take_verified`]: PendingGateStore::take_verified + pub async fn take_verified_by_request_id( + &self, + request_id: Uuid, + expected_user_id: &str, + responding_channel: &str, + ) -> Result<PendingGate, GateStoreError> { + let (key, gate) = { + let mut inner = self.inner.lock().await; + + let key = inner + .by_request_id + .get(&request_id) + .cloned() + .ok_or(GateStoreError::NotFound)?; + + if key.user_id != expected_user_id { + return Err(GateStoreError::Unauthorized); + } + + let gate = inner.by_key.get(&key).ok_or(GateStoreError::NotFound)?; + + // Verify channel authorization + let channel_authorized = gate.source_channel == responding_channel + || TRUSTED_GATE_CHANNELS.contains(&responding_channel); + if !channel_authorized { + return Err(GateStoreError::ChannelMismatch { + expected: gate.source_channel.clone(), + actual: responding_channel.to_string(), + }); + } + + // Check expiry — clean up expired gate while we hold the lock. + if gate.is_expired() { + let removed = inner.by_key.remove(&key); + if let Some(ref g) = removed { + inner.by_request_id.remove(&g.request_id); + } + return Err(GateStoreError::Expired); + } + + // Atomically remove — no TOCTOU gap. + let gate = inner.by_key.remove(&key).ok_or(GateStoreError::NotFound)?; + inner.by_request_id.remove(&gate.request_id); + (key, gate) + }; + + // Persist removal after lock is released. + if let Some(ref persistence) = self.persistence + && let Err(e) = persistence.remove(&key).await + { + tracing::debug!(error = %e, "gate persistence removal failed (gate already taken from memory)"); + } + Ok(gate) + } + /// List all non-expired gates. pub async fn list_all(&self) -> Vec<PendingGate> { let inner = self.inner.lock().await; @@ -469,6 +580,61 @@ mod tests { assert_eq!(store.list_all().await.len(), 1); } + // ── peek_by_request_id (foreground cancel fallback) ────── + + #[tokio::test] + async fn test_peek_by_request_id_returns_view_for_owning_user() { + // Regression: PR #3366 review — chat_gate_resolve_handler's + // Cancelled arm uses this path to recover the owning thread when + // the client omits `thread_id`, otherwise the parked VM is + // stranded. Also asserts ownership scoping so a cross-user lookup + // doesn't leak the gate. + let store = PendingGateStore::in_memory(); + let gate = sample_gate_with("alice", ThreadId::new(), "web", 300); + let request_id = gate.request_id; + store.insert(gate).await.unwrap(); + + let view = store + .peek_by_request_id(request_id, "alice") + .await + .expect("owning user sees the gate"); + assert_eq!(view.request_id, request_id.to_string()); + + // Cross-user lookup yields None (do not leak gate existence). + assert!(store.peek_by_request_id(request_id, "bob").await.is_none()); + + // Unknown request_id yields None. + assert!( + store + .peek_by_request_id(Uuid::new_v4(), "alice") + .await + .is_none() + ); + + // Peek does not consume — second peek still works. + assert!( + store + .peek_by_request_id(request_id, "alice") + .await + .is_some() + ); + } + + #[tokio::test] + async fn test_peek_by_request_id_skips_expired() { + let store = PendingGateStore::in_memory(); + let gate = sample_gate_with("alice", ThreadId::new(), "web", -1); + let request_id = gate.request_id; + store.insert(gate).await.unwrap(); + + assert!( + store + .peek_by_request_id(request_id, "alice") + .await + .is_none() + ); + } + // ── Channel verification ───────────────────────────────── #[tokio::test] @@ -597,6 +763,78 @@ mod tests { assert_eq!(failures, 1, "Exactly one concurrent take must fail"); } + // ── take_verified_by_request_id ────────────────────────────────── + + #[tokio::test] + async fn test_take_verified_by_request_id_resolves_via_request_id() { + // The HTTP fast path looks gates up by request_id when the + // wire-supplied thread identifier (channel scope id) does not + // equal the engine `ThreadId`. Verify the lookup still reaches + // the gate and removes both indices on success. + let store = PendingGateStore::in_memory(); + let gate = sample_gate("web"); + let key = gate.key(); + let request_id = gate.request_id; + store.insert(gate).await.unwrap(); + + let taken = store + .take_verified_by_request_id(request_id, "user1", "web") + .await + .expect("take by request_id must succeed"); + assert_eq!(taken.action_name, "shell"); + + // Both indices must be cleared. + assert!(store.peek(&key).await.is_none()); + assert!(matches!( + store + .take_verified_by_request_id(request_id, "user1", "web") + .await, + Err(GateStoreError::NotFound) + )); + } + + #[tokio::test] + async fn test_take_verified_by_request_id_rejects_other_user() { + // Tenant isolation: a gate matching `request_id` but owned by + // a different user must surface `Unauthorized`, distinct from + // `NotFound`, so HTTP callers can return 403 without leaking + // gate existence across tenants. + let store = PendingGateStore::in_memory(); + let gate = sample_gate_with("alice", ThreadId::new(), "web", 300); + let key = gate.key(); + let request_id = gate.request_id; + store.insert(gate).await.unwrap(); + + assert!(matches!( + store + .take_verified_by_request_id(request_id, "mallory", "web") + .await, + Err(GateStoreError::Unauthorized) + )); + // Gate is left intact — the legitimate owner must still be able + // to resolve it. + assert!(store.peek(&key).await.is_some()); + } + + #[tokio::test] + async fn test_take_verified_by_request_id_channel_mismatch() { + let store = PendingGateStore::in_memory(); + let gate = sample_gate("telegram"); + let key = gate.key(); + let request_id = gate.request_id; + store.insert(gate).await.unwrap(); + + // Slack is not the source channel and not in the trusted set. + assert!(matches!( + store + .take_verified_by_request_id(request_id, "user1", "slack") + .await, + Err(GateStoreError::ChannelMismatch { .. }) + )); + // Channel-mismatch must NOT consume the gate. + assert!(store.peek(&key).await.is_some()); + } + // ── Peek ───────────────────────────────────────────────── #[tokio::test] diff --git a/src/generated_images.rs b/src/generated_images.rs index afa5d79cacc..e3f787ae049 100644 --- a/src/generated_images.rs +++ b/src/generated_images.rs @@ -2,7 +2,12 @@ use std::borrow::Cow; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use uuid::Uuid; + pub(crate) const MAX_RECORDED_IMAGE_SENTINEL_BYTES: usize = 512 * 1024; +pub(crate) const MAX_GENERATED_IMAGE_ATTACHMENT_BYTES: usize = 10 * 1024 * 1024; const MAX_EMBEDDED_JSON_STRING_LAYERS: usize = 3; #[derive(Clone, Debug, PartialEq, Eq)] @@ -53,11 +58,6 @@ impl GeneratedImageSentinel { pub(crate) fn summary_for_context(&self) -> String { let media_type = self.media_type().unwrap_or("image"); - if let Some(path) = self.path() - && !path.is_empty() - { - return format!("Generated image ({media_type}) at {path}"); - } format!("Generated image ({media_type})") } @@ -136,6 +136,75 @@ fn normalize_embedded_json(value: &serde_json::Value) -> Option<Cow<'_, serde_js Some(Cow::Owned(current)) } +pub(crate) fn generated_image_extension(media_type: &str) -> Result<&'static str, String> { + match media_type.trim().to_ascii_lowercase().as_str() { + "image/jpeg" | "image/jpg" => Ok("jpg"), + "image/png" | "" => Ok("png"), + "image/gif" => Ok("gif"), + "image/webp" => Ok("webp"), + other => Err(format!("unsupported generated image media type '{other}'")), + } +} + +pub(crate) fn stage_generated_image_data_url(data_url: &str) -> Result<String, String> { + let (header, encoded) = data_url + .split_once(',') + .ok_or_else(|| "generated image data URL is missing a comma separator".to_string())?; + let metadata = header + .strip_prefix("data:") + .ok_or_else(|| "generated image data URL is missing data: prefix".to_string())?; + let mut parts = metadata.split(';'); + let media_type = parts.next().unwrap_or(""); + if !parts.any(|part| part.eq_ignore_ascii_case("base64")) { + return Err("generated image data URL is not base64 encoded".to_string()); + } + + let extension = generated_image_extension(media_type)?; + let bytes = BASE64_STANDARD + .decode(encoded.as_bytes()) + .map_err(|e| format!("failed to decode generated image data URL: {e}"))?; + if bytes.len() > MAX_GENERATED_IMAGE_ATTACHMENT_BYTES { + return Err(format!( + "generated image exceeds {} MB channel delivery limit", + MAX_GENERATED_IMAGE_ATTACHMENT_BYTES / (1024 * 1024) + )); + } + + let path = std::path::Path::new("/tmp").join(format!( + "ironclaw-generated-image-{}.{}", + Uuid::new_v4(), + extension + )); + std::fs::write(&path, bytes) + .map_err(|e| format!("failed to stage generated image '{}': {e}", path.display()))?; + Ok(path.to_string_lossy().into_owned()) +} + +pub(crate) fn is_staged_generated_image_path(path: &str) -> bool { + let path = std::path::Path::new(path); + let is_tmp = path.parent() == Some(std::path::Path::new("/tmp")); + let is_generated = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("ironclaw-generated-image-")); + is_tmp && is_generated +} + +pub(crate) fn remove_staged_generated_image_attachments(paths: &[String]) { + for path in paths { + if !is_staged_generated_image_path(path) { + continue; + } + if let Err(e) = std::fs::remove_file(path) { + tracing::debug!( + path = %path, + error = %e, + "Failed to remove staged generated image" + ); + } + } +} + #[cfg(test)] mod tests { use super::{GeneratedImageSentinel, MAX_RECORDED_IMAGE_SENTINEL_BYTES}; @@ -207,7 +276,7 @@ mod tests { assert_eq!( sentinel.summary_for_context(), - "Generated image (image/png) at workspace/out.png" + "Generated image (image/png)" ); } diff --git a/src/hooks/session_summary.rs b/src/hooks/session_summary.rs index cf500faadac..38fadc7e618 100644 --- a/src/hooks/session_summary.rs +++ b/src/hooks/session_summary.rs @@ -12,8 +12,8 @@ use crate::db::ConversationStore; use crate::hooks::hook::{ Hook, HookContext, HookError, HookEvent, HookFailureMode, HookOutcome, HookPoint, }; -use crate::llm::{ChatMessage, CompletionRequest, LlmProvider}; use crate::tools::builtin::memory::WorkspaceResolver; +use ironclaw_llm::{ChatMessage, CompletionRequest, LlmProvider}; /// Maximum number of concurrent LLM summarization calls. /// Prevents thundering herd when many sessions expire at once (e.g. restart after idle). @@ -208,11 +208,11 @@ mod tests { use super::*; use crate::db::ConversationStore; use crate::history::{ConversationMessage, ConversationSummary}; - use crate::llm::{ - CompletionResponse, FinishReason, LlmError, ToolCompletionRequest, ToolCompletionResponse, - }; use crate::workspace::Workspace; use chrono::Utc; + use ironclaw_llm::{ + CompletionResponse, FinishReason, LlmError, ToolCompletionRequest, ToolCompletionResponse, + }; use rust_decimal::Decimal; use uuid::Uuid; diff --git a/src/http_intercept.rs b/src/http_intercept.rs index 3b2dd07f59d..af5fbbe8984 100644 --- a/src/http_intercept.rs +++ b/src/http_intercept.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; -use crate::llm::recording::{HttpExchangeRequest, HttpExchangeResponse, HttpInterceptor}; +use ironclaw_llm::recording::{HttpExchangeRequest, HttpExchangeResponse, HttpInterceptor}; #[derive(Debug)] pub struct CompositeHttpInterceptor { diff --git a/src/lib.rs b/src/lib.rs index 8821b37cedd..594a17a59b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -64,7 +64,7 @@ pub mod hooks; pub mod http_intercept; #[cfg(feature = "import")] pub mod import; -pub mod llm; +pub mod llm_host; pub mod observability; pub mod orchestrator; pub mod ownership; @@ -100,8 +100,8 @@ pub mod prelude { pub use crate::config::Config; pub use crate::context::{JobContext, JobState}; pub use crate::error::{Error, Result}; - pub use crate::llm::LlmProvider; pub use crate::tools::{Tool, ToolOutput, ToolRegistry}; pub use crate::workspace::{MemoryDocument, Workspace}; + pub use ironclaw_llm::LlmProvider; pub use ironclaw_safety::{SanitizedOutput, Sanitizer}; } diff --git a/src/llm_host.rs b/src/llm_host.rs new file mode 100644 index 00000000000..efb4d7a6307 --- /dev/null +++ b/src/llm_host.rs @@ -0,0 +1,112 @@ +//! Adapters that bridge main-crate types into the trait surface +//! `ironclaw_llm` exposes for session management. +//! +//! `ironclaw_llm::SessionManager` owns NEAR-AI session-token lifecycle and +//! authenticates API requests, but it deliberately does not depend on the +//! main crate's `Database`, `SecretsStore`, `setup` UI, or `bootstrap` env +//! file helpers. The adapters in this module wire those concrete impls into +//! the LLM-side traits without forcing `ironclaw_llm` to know about them. +//! +//! See `crates/ironclaw_llm/src/host.rs` for the trait definitions. + +use std::sync::Arc; + +use async_trait::async_trait; +use ironclaw_llm::host::{SessionDb, SessionKeyPersistor, SessionSecrets}; +use secrecy::SecretString; + +use crate::db::Database; +use crate::secrets::{CreateSecretParams, SecretsStore}; + +/// Adapter exposing the main-crate `Database` as the narrow `SessionDb` trait. +pub struct DatabaseSessionDb { + db: Arc<dyn Database>, +} + +impl DatabaseSessionDb { + pub fn new(db: Arc<dyn Database>) -> Self { + Self { db } + } +} + +#[async_trait] +impl SessionDb for DatabaseSessionDb { + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), String> { + self.db + .set_setting(user_id, key, value) + .await + .map_err(|e| e.to_string()) + } + + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result<Option<serde_json::Value>, String> { + self.db + .get_setting(user_id, key) + .await + .map_err(|e| e.to_string()) + } +} + +/// Adapter exposing the encrypted `SecretsStore` as the narrow `SessionSecrets` trait. +pub struct SecretsStoreSessionSecrets { + secrets: Arc<dyn SecretsStore + Send + Sync>, +} + +impl SecretsStoreSessionSecrets { + pub fn new(secrets: Arc<dyn SecretsStore + Send + Sync>) -> Self { + Self { secrets } + } +} + +#[async_trait] +impl SessionSecrets for SecretsStoreSessionSecrets { + async fn create( + &self, + user_id: &str, + name: &str, + value: String, + provider: Option<&str>, + ) -> Result<(), String> { + let mut params = CreateSecretParams::new(name, value); + if let Some(p) = provider { + params = params.with_provider(p); + } + self.secrets + .create(user_id, params) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + } + + async fn get_decrypted(&self, user_id: &str, name: &str) -> Result<SecretString, String> { + let plaintext = self + .secrets + .get_decrypted(user_id, name) + .await + .map_err(|e| e.to_string())?; + Ok(SecretString::from(plaintext.expose().to_string())) + } +} + +/// `SessionKeyPersistor` impl wired to the main crate's runtime env overlay +/// and bootstrap `.env` writer. Used when the interactive renewer collects a +/// fresh NEAR-AI Cloud API key. +pub struct BootstrapKeyPersistor; + +impl SessionKeyPersistor for BootstrapKeyPersistor { + fn set_runtime_env(&self, key: &str, value: &str) { + crate::config::helpers::set_runtime_env(key, value); + } + + fn upsert_bootstrap_var(&self, key: &str, value: &str) -> std::io::Result<()> { + crate::bootstrap::upsert_bootstrap_var(key, value) + } +} diff --git a/src/main.rs b/src/main.rs index 44e65022172..a9dc2cf5aae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,6 @@ use ironclaw::{ }, config::Config, hooks::bootstrap_hooks, - llm::create_session_manager, orchestrator::{ReaperConfig, SandboxReaper}, pairing::PairingStore, tracing_fmt::{init_cli_tracing, init_worker_tracing}, @@ -91,7 +90,7 @@ fn non_cli_channels_enabled(cli_only: bool) -> bool { !cli_only } -fn normalize_persisted_wasm_channel_names<I, S>(names: I) -> std::collections::HashSet<String> +fn normalize_startup_wasm_channel_names<I, S>(names: I) -> std::collections::HashSet<String> where I: IntoIterator<Item = S>, S: AsRef<str>, @@ -106,7 +105,7 @@ where tracing::warn!( channel = name.as_ref(), error = %e, - "Ignoring invalid persisted WASM channel name" + "Ignoring invalid startup WASM channel name" ); } } @@ -114,28 +113,28 @@ where normalized } -async fn persisted_active_wasm_channel_names( +async fn startup_active_wasm_channel_names( ext_mgr: &ironclaw::extensions::ExtensionManager, user_id: &str, - persisted_active_channels: &[String], + startup_active_channels: &[String], ) -> std::collections::HashSet<String> { let mut relay_channels = std::collections::HashSet::new(); - for name in persisted_active_channels { + for name in startup_active_channels { if ext_mgr.is_relay_channel(name, user_id).await { relay_channels.insert(name.clone()); } } - persisted_non_relay_wasm_channel_names(persisted_active_channels, &relay_channels) + startup_non_relay_wasm_channel_names(startup_active_channels, &relay_channels) } -fn persisted_non_relay_wasm_channel_names( - persisted_active_channels: &[String], - persisted_active_relay_channels: &std::collections::HashSet<String>, +fn startup_non_relay_wasm_channel_names( + startup_active_channels: &[String], + startup_active_relay_channels: &std::collections::HashSet<String>, ) -> std::collections::HashSet<String> { - normalize_persisted_wasm_channel_names( - persisted_active_channels + normalize_startup_wasm_channel_names( + startup_active_channels .iter() - .filter(|name| !persisted_active_relay_channels.contains(*name)), + .filter(|name| !startup_active_relay_channels.contains(*name)), ) } @@ -263,33 +262,36 @@ async fn async_main() -> anyhow::Result<()> { Some(Command::Login { openai_codex }) => { init_cli_tracing(); if *openai_codex { - // Resolve codex config so OPENAI_CODEX_* env overrides are - // honoured even when LLM_BACKEND isn't set to openai_codex. - let codex_config = { - let config = Config::from_env() - .await - .map_err(|e| anyhow::anyhow!("{}", e))?; - config.llm.openai_codex.unwrap_or_else(|| { - use ironclaw::llm::OpenAiCodexConfig; - let mut cfg = OpenAiCodexConfig::default(); - if let Ok(v) = std::env::var("OPENAI_CODEX_AUTH_URL") { - cfg.auth_endpoint = v; - } - if let Ok(v) = std::env::var("OPENAI_CODEX_API_URL") { - cfg.api_base_url = v; - } - if let Ok(v) = std::env::var("OPENAI_CODEX_CLIENT_ID") { - cfg.client_id = v; - } - if let Ok(v) = std::env::var("OPENAI_CODEX_SESSION_PATH") { - cfg.session_path = std::path::PathBuf::from(v); - } - cfg - }) + use ironclaw_llm::auth::{ + AuthPrompt, LoginRequest, OpenAiCodexLoginOptions, start_login, }; - let mgr = ironclaw::llm::OpenAiCodexSessionManager::new(codex_config) - .map_err(|e| anyhow::anyhow!("{}", e))?; - mgr.device_code_login() + + struct CliPrompt; + impl AuthPrompt for CliPrompt { + fn show_device_code(&self, verification_uri: &str, user_code: &str) { + println!("Visit {verification_uri} and enter the code: {user_code}"); + } + } + + // Resolve the full config so codex endpoints / client-id / + // session-path overrides committed to TOML or DB are + // honoured, not just env vars. `Config::from_env` runs the + // standard precedence pipeline (TOML < env < DB), and any + // resolved codex config wins over the env-only fallback. + let opts = match Config::from_env().await { + Ok(cfg) => cfg + .llm + .openai_codex + .as_ref() + .map(OpenAiCodexLoginOptions::from_resolved_config) + .unwrap_or_else(OpenAiCodexLoginOptions::from_env), + // Login should still work on a fresh machine where + // `Config::from_env` would fail (e.g. no DB). Fall + // back to env-only options in that case. + Err(_) => OpenAiCodexLoginOptions::from_env(), + }; + + start_login(LoginRequest::OpenAiCodex(opts), &CliPrompt) .await .map_err(|e| anyhow::anyhow!("{}", e))?; println!( @@ -378,7 +380,7 @@ async fn async_main() -> anyhow::Result<()> { } // Load initial config from env + disk + optional TOML (before DB is available). - // Credentials may be missing at this point — that's fine. LlmConfig::resolve() + // Credentials may be missing at this point — that's fine. crate::config::llm::resolve() // defers gracefully, and AppBuilder::build_all() re-resolves after loading // secrets from the encrypted DB. let toml_path = cli.config.as_deref(); @@ -396,7 +398,7 @@ async fn async_main() -> anyhow::Result<()> { }; // Initialize session manager before channel setup - let session = create_session_manager(config.llm.session.clone()).await; + let session = ironclaw_llm::create_session_manager(config.llm.session.clone()).await; // Create log broadcaster before tracing init so the WebLogLayer can capture all events. let log_broadcaster = Arc::new(LogBroadcaster::new()); @@ -484,20 +486,14 @@ async fn async_main() -> anyhow::Result<()> { // Default user ID for extension operations (single-user mode). let ext_user_id = config.owner_id.clone(); - let settings_persistence_available = components.db.is_some(); - let persisted_active_channels: Vec<String> = - if settings_persistence_available && let Some(ref ext_mgr) = components.extension_manager { - ext_mgr.load_persisted_active_channels(&ext_user_id).await - } else { - Vec::new() - }; - let persisted_active_wasm_channels: std::collections::HashSet<String> = - if settings_persistence_available && let Some(ref ext_mgr) = components.extension_manager { - persisted_active_wasm_channel_names(ext_mgr, &ext_user_id, &persisted_active_channels) - .await - } else { - std::collections::HashSet::new() - }; + // Startup-active WASM channels are resolved lazily inside the + // `enable_non_cli && wasm_channels_enabled` gate below. Defaulting to + // an empty set here keeps the later auto-activation block (gated on + // `wasm_channel_runtime_state`) compiling without computing — and + // potentially failing on — settings-store reads in `--cli-only` / + // `WASM_CHANNELS_ENABLED=false` runs. + let mut startup_active_wasm_channels: std::collections::HashSet<String> = + std::collections::HashSet::new(); let channels = ChannelManager::new(); let mut channel_names: Vec<String> = Vec::new(); @@ -509,7 +505,8 @@ async fn async_main() -> anyhow::Result<()> { Arc<WasmChannelRouter>, )> = None; - // Create CLI channel (REPL or TUI — mutually exclusive, both claim stdin) + // Create stdin channel (REPL or TUI — mutually exclusive, both claim stdin). + // TUI has its own config, so it must not depend on the CLI channel being enabled. let tui_mode = config.channels.tui.is_some(); #[cfg(feature = "tui")] @@ -620,7 +617,7 @@ async fn async_main() -> anyhow::Result<()> { #[cfg(not(feature = "tui"))] if tui_mode { tracing::warn!( - "CLI_MODE=tui requested but the 'tui' feature is not enabled. Falling back to REPL." + "TUI mode is configured but the 'tui' feature is not enabled. Falling back to REPL if CLI is enabled." ); } @@ -681,13 +678,40 @@ async fn async_main() -> anyhow::Result<()> { && config.channels.wasm_channels_enabled && config.channels.wasm_channels_dir.exists() { + // Resolve startup-active channels: persisted state is authoritative + // when present; otherwise fall back to the setup wizard's + // `channels.wasm_channels` so headless installs (no DB, no web UI) + // still auto-activate channels listed in the config. Settings-store + // errors propagate — masking them would silently re-activate channels + // the user had deactivated. Resolved here (not at outer scope) so a + // corrupt `activated_channels` row only fails startup when channels + // are actually about to be restored. + let startup_active_channels: Vec<String> = + if let Some(ref ext_mgr) = components.extension_manager { + ext_mgr + .load_startup_active_channels( + &ext_user_id, + config.channels.configured_wasm_channels.clone(), + ) + .await? + } else { + ironclaw::extensions::naming::normalize_extension_names( + config.channels.configured_wasm_channels.clone(), + ) + }; + startup_active_wasm_channels = if let Some(ref ext_mgr) = components.extension_manager { + startup_active_wasm_channel_names(ext_mgr, &ext_user_id, &startup_active_channels).await + } else { + startup_active_channels.iter().cloned().collect() + }; + let wasm_result = ironclaw::channels::wasm::setup_wasm_channels( &config, &components.secrets_store, components.extension_manager.as_ref(), components.db.as_ref(), &channel_names, - settings_persistence_available.then_some(&persisted_active_wasm_channels), + &startup_active_wasm_channels, Arc::clone(&components.ownership_cache), ) .await; @@ -1007,11 +1031,18 @@ async fn async_main() -> anyhow::Result<()> { let gw_state = Arc::clone(gw.state()); tokio::spawn(async move { while let Ok((_job_id, user_id, event)) = rx.recv().await { - if user_id.is_empty() { - gw_state.sse.broadcast(event); - } else { - gw_state.sse.broadcast_for_user(&user_id, event); - } + // Reuse the gateway's central status-event router so + // the sandbox dispatch path inherits the same drop / + // WARN / broadcast policy as `Channel::send_status`. + // Empty `user_id` collapses into the None arm via + // `dispatch_status_event`'s `!is_empty()` filter. + let user_id_opt = (!user_id.is_empty()).then_some(user_id.as_str()); + ironclaw::channels::web::dispatch_status_event( + &gw_state.sse, + gw_state.multi_tenant_mode, + user_id_opt, + event, + ); } }); } @@ -1150,9 +1181,11 @@ async fn async_main() -> anyhow::Result<()> { .await; tracing::debug!("Channel runtime wired into extension manager for hot-activation"); - // Auto-activate WASM channels that were active in a previous session. + // Auto-activate WASM channels resolved at startup — either persisted + // from a prior session or supplied by the setup wizard's + // `channels.wasm_channels` config when no settings store is available. // Relay channels are handled separately below via restore_relay_channels(). - for name in &persisted_active_wasm_channels { + for name in &startup_active_wasm_channels { if active_at_startup.contains(name) || ext_mgr.is_relay_channel(name, &ext_user_id).await { @@ -1173,14 +1206,14 @@ async fn async_main() -> anyhow::Result<()> { tracing::debug!( channel = %name, message = %message, - "Auto-activated persisted WASM channel" + "Auto-activated startup WASM channel" ); } Ok(ironclaw::extensions::EnsureReadyOutcome::NeedsAuth { auth, .. }) => { tracing::warn!( channel = %name, instructions = ?auth.instructions(), - "Persisted WASM channel still needs authentication" + "Startup WASM channel still needs authentication" ); } Ok(ironclaw::extensions::EnsureReadyOutcome::NeedsSetup { @@ -1189,14 +1222,14 @@ async fn async_main() -> anyhow::Result<()> { tracing::warn!( channel = %name, instructions = %instructions, - "Persisted WASM channel still needs setup" + "Startup WASM channel still needs setup" ); } Err(e) => { tracing::warn!( channel = %name, error = %e, - "Failed to auto-activate persisted WASM channel" + "Failed to auto-activate startup WASM channel" ); } } @@ -1224,11 +1257,32 @@ async fn async_main() -> anyhow::Result<()> { components.tools.register_plan_tools(Some(Arc::clone(sse))); } - // Snapshot memory for trace recording before the agent starts + // Snapshot memory for trace recording before the agent starts. + // The recorder lives in `ironclaw_llm` and must not depend on the + // host's `Workspace` type, so we materialise entries here. if let Some(ref recorder) = components.recording_handle && let Some(ref ws) = components.workspace { - recorder.snapshot_memory(ws).await; + let mut entries = Vec::new(); + match ws.list_all().await { + Ok(paths) => { + for path in paths { + match ws.read(&path).await { + Ok(doc) => entries.push(ironclaw_llm::MemorySnapshotEntry { + path: doc.path, + content: doc.content, + }), + Err(e) => { + tracing::debug!(path = %path, error = %e, "Skipped memory doc in snapshot") + } + } + } + } + Err(e) => { + tracing::warn!(error = %e, "Failed to list memory documents; trace will have empty memory snapshot"); + } + } + recorder.snapshot_memory(entries).await; } let http_interceptor = ironclaw::http_intercept::chain( @@ -1290,11 +1344,10 @@ async fn async_main() -> anyhow::Result<()> { cost_guard: components.cost_guard, sse_tx: sse_manager, http_interceptor, - transcription: config.transcription.create_provider().map(|p| { - Arc::new(ironclaw::llm::transcription::TranscriptionMiddleware::new( - p, - )) - }), + transcription: config + .transcription + .create_provider() + .map(|p| Arc::new(ironclaw_llm::transcription::TranscriptionMiddleware::new(p))), document_extraction: Some(Arc::new( ironclaw::document_extraction::DocumentExtractionMiddleware::new(), )), @@ -1547,6 +1600,7 @@ async fn async_main() -> anyhow::Result<()> { content: format!("Warning: {warning}"), thread_id: None, attachments: Vec::new(), + inline_attachments: Vec::new(), metadata: serde_json::json!({ "source": "system", "type": "warning", @@ -1770,9 +1824,9 @@ mod tests { } #[test] - fn normalize_persisted_wasm_channel_names_canonicalizes_and_dedupes() { + fn normalize_startup_wasm_channel_names_canonicalizes_and_dedupes() { let normalized = - normalize_persisted_wasm_channel_names(["slack-relay", "slack_relay", "telegram"]); + normalize_startup_wasm_channel_names(["slack-relay", "slack_relay", "telegram"]); assert_eq!(normalized.len(), 2); assert!(normalized.contains("slack_relay")); @@ -1780,8 +1834,8 @@ mod tests { } #[test] - fn normalize_persisted_wasm_channel_names_skips_invalid_entries() { - let normalized = normalize_persisted_wasm_channel_names(["../bad", "telegram"]); + fn normalize_startup_wasm_channel_names_skips_invalid_entries() { + let normalized = normalize_startup_wasm_channel_names(["../bad", "telegram"]); assert_eq!( normalized, @@ -1790,9 +1844,9 @@ mod tests { } #[test] - fn persisted_non_relay_wasm_channel_names_preserves_legacy_relay_entries() { + fn startup_non_relay_wasm_channel_names_preserves_legacy_relay_entries() { let relay_names = std::collections::HashSet::from(["slack-relay".to_string()]); - let names = persisted_non_relay_wasm_channel_names( + let names = startup_non_relay_wasm_channel_names( &["slack-relay".to_string(), "telegram".to_string()], &relay_names, ); @@ -1804,10 +1858,10 @@ mod tests { } #[test] - fn normalize_persisted_wasm_channel_names_rejects_invalid_extension_names() { + fn normalize_startup_wasm_channel_names_rejects_invalid_extension_names() { // ExtensionName rejects uppercase, dots, consecutive underscores let normalized = - normalize_persisted_wasm_channel_names(["My.Channel", "bad__name", "already_ok"]); + normalize_startup_wasm_channel_names(["My.Channel", "bad__name", "already_ok"]); assert_eq!( normalized, diff --git a/src/orchestrator/api.rs b/src/orchestrator/api.rs index ebafdb2cdd4..646c9cfeb90 100644 --- a/src/orchestrator/api.rs +++ b/src/orchestrator/api.rs @@ -16,7 +16,6 @@ use uuid::Uuid; use crate::channels::web::types::ToolDecisionDto; use crate::db::Database; -use crate::llm::{CompletionRequest, LlmProvider, ToolCompletionRequest}; use crate::orchestrator::auth::{TokenStore, worker_auth_middleware}; use crate::orchestrator::job_manager::ContainerJobManager; use crate::secrets::SecretsStore; @@ -26,6 +25,7 @@ use crate::worker::api::{ ProxyCompletionResponse, ProxyToolCompletionRequest, ProxyToolCompletionResponse, StatusUpdate, }; use ironclaw_common::{AppEvent, JobResultStatus}; +use ironclaw_llm::{CompletionRequest, LlmProvider, ToolCompletionRequest}; /// A follow-up prompt queued for a Claude Code bridge. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -256,6 +256,7 @@ async fn llm_complete_with_tools( finish_reason: format_finish_reason(resp.finish_reason), cache_read_input_tokens: resp.cache_read_input_tokens, cache_creation_input_tokens: resp.cache_creation_input_tokens, + reasoning: resp.reasoning, })) } @@ -576,13 +577,13 @@ async fn get_credentials_handler( )) } -fn format_finish_reason(reason: crate::llm::FinishReason) -> String { +fn format_finish_reason(reason: ironclaw_llm::FinishReason) -> String { match reason { - crate::llm::FinishReason::Stop => "stop".to_string(), - crate::llm::FinishReason::Length => "length".to_string(), - crate::llm::FinishReason::ToolUse => "tool_use".to_string(), - crate::llm::FinishReason::ContentFilter => "content_filter".to_string(), - crate::llm::FinishReason::Unknown => "unknown".to_string(), + ironclaw_llm::FinishReason::Stop => "stop".to_string(), + ironclaw_llm::FinishReason::Length => "length".to_string(), + ironclaw_llm::FinishReason::ToolUse => "tool_use".to_string(), + ironclaw_llm::FinishReason::ContentFilter => "content_filter".to_string(), + ironclaw_llm::FinishReason::Unknown => "unknown".to_string(), } } diff --git a/src/orchestrator/mod.rs b/src/orchestrator/mod.rs index fe1379ff068..a3a91e94d4a 100644 --- a/src/orchestrator/mod.rs +++ b/src/orchestrator/mod.rs @@ -47,9 +47,9 @@ use tokio::sync::{Mutex, broadcast}; use uuid::Uuid; use crate::db::Database; -use crate::llm::LlmProvider; use crate::secrets::SecretsStore; use ironclaw_common::AppEvent; +use ironclaw_llm::LlmProvider; /// Resolve the orchestrator port from the `ORCHESTRATOR_PORT` environment /// variable, falling back to 50051. diff --git a/src/pairing/store.rs b/src/pairing/store.rs index 77777f4335d..932fc129c4f 100644 --- a/src/pairing/store.rs +++ b/src/pairing/store.rs @@ -197,4 +197,22 @@ impl PairingStore { self.cache.evict(&channel, external_id); Ok(()) } + + /// Create a channel identity directly (trusted path, e.g. OAuth completion). + /// Inserts into channel_identities and populates the cache without requiring + /// a pairing code flow. + pub async fn create_identity( + &self, + channel: &str, + external_id: &str, + owner_id: &UserId, + ) -> Result<(), DatabaseError> { + let channel = crate::pairing::normalize_channel_name(channel); + if let Some(ref db) = self.db { + db.create_channel_identity(&channel, external_id, owner_id.as_str()) + .await?; + } + self.cache.insert(&channel, external_id, owner_id.clone()); + Ok(()) + } } diff --git a/src/registry/installer.rs b/src/registry/installer.rs index 4e3489c8ac0..68a664d8ae0 100644 --- a/src/registry/installer.rs +++ b/src/registry/installer.rs @@ -843,6 +843,7 @@ mod tests { tags: Vec::new(), url: None, auth: None, + hidden: false, } } diff --git a/src/registry/manifest.rs b/src/registry/manifest.rs index 031ac7e7d60..14a86aefbf1 100644 --- a/src/registry/manifest.rs +++ b/src/registry/manifest.rs @@ -54,6 +54,16 @@ pub struct ExtensionManifest { /// Only present for `McpServer` manifests. #[serde(default)] pub auth: Option<String>, + + /// When true, this entry is omitted from the default "available but not + /// installed" surfaces (the agent's `Activatable Integrations` prompt + /// section and the settings catalog). It remains buildable and + /// installable by explicit name. Used for alternate/legacy variants of + /// a canonical integration (e.g. `telegram_mtproto` vs the canonical + /// `telegram` channel) so the agent doesn't enumerate them as options. + /// Issue #3533. + #[serde(default)] + pub hidden: bool, } /// Extension kind as declared in manifests. @@ -217,6 +227,7 @@ impl ExtensionManifest { fallback_source: None, auth_hint, version: self.version.clone(), + hidden: self.hidden, }) } @@ -285,6 +296,7 @@ impl ExtensionManifest { fallback_source, auth_hint, version: self.version.clone(), + hidden: self.hidden, } } } @@ -325,9 +337,41 @@ mod tests { assert_eq!(manifest.kind, ManifestKind::Tool); assert_eq!(manifest.version.as_deref(), Some("0.1.0")); assert!(manifest.tags.contains(&"default".to_string())); + // Default: not hidden. + assert!(!manifest.hidden); let entry = manifest.to_registry_entry().unwrap(); assert_eq!(entry.kind, ExtensionKind::WasmTool); + assert!(!entry.hidden); + } + + /// Issue #3533 regression: a manifest can declare itself hidden, in which + /// case the `RegistryEntry` it produces carries that flag forward to + /// `ExtensionManager::list`. + #[test] + fn test_parse_hidden_manifest_propagates_to_registry_entry() { + let json = r#"{ + "name": "telegram_mtproto", + "display_name": "Telegram Tool", + "kind": "tool", + "version": "0.2.1", + "hidden": true, + "description": "Direct MTProto integration (alternate)", + "source": { + "dir": "tools-src/telegram", + "capabilities": "telegram-tool.capabilities.json", + "crate_name": "telegram-tool" + } + }"#; + + let manifest: ExtensionManifest = serde_json::from_str(json).expect("parse manifest"); + assert!(manifest.hidden); + + let entry = manifest.to_registry_entry().expect("entry"); + assert!( + entry.hidden, + "hidden flag must propagate from manifest to registry entry" + ); } #[test] diff --git a/src/settings.rs b/src/settings.rs index 8b793789b41..e023a7c8e37 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -54,7 +54,7 @@ impl std::fmt::Debug for CustomLlmProviderSettings { /// Per-provider overrides for built-in LLM providers (API key and/or model). /// /// Stored as `llm_builtin_overrides` in the settings store, keyed by provider ID -/// (e.g. `"openai"`, `"gemini"`). Resolved at startup during `LlmConfig::resolve()`. +/// (e.g. `"openai"`, `"gemini"`). Resolved at startup during `crate::config::llm::resolve()`. /// /// Note: The global `selected_model` (if set) takes precedence over these /// per-provider overrides, which in turn take precedence over environment variables. @@ -69,6 +69,40 @@ pub struct LlmBuiltinOverride { /// Base URL override. Takes precedence over environment variables. #[serde(default, skip_serializing_if = "Option::is_none")] pub base_url: Option<String>, + /// Per-provider settings bag for non-OpenAI-shape backends that need + /// extra fields beyond api_key / model / base_url. Example keys: + /// + /// - `bedrock`: `region`, `cross_region`, `profile` + /// - `gemini_oauth`: `credentials_path` + /// - `openai_codex`: (none today; reserved) + /// + /// Settings flow into this bag through the wizard's generic + /// `SetupHint` dispatch (Layer C); the binary-side resolver in + /// `crate::config::llm::resolve` reads them when assembling the + /// per-provider config struct. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub extras: HashMap<String, String>, +} + +impl LlmBuiltinOverride { + /// Look up an extras-bag field by key; returns `None` if absent or empty. + pub fn extra(&self, key: &str) -> Option<&str> { + self.extras + .get(key) + .map(|s| s.as_str()) + .filter(|s| !s.is_empty()) + } + + /// Set an extras-bag field; clears the entry when `value` is empty. + pub fn set_extra(&mut self, key: impl Into<String>, value: impl Into<String>) { + let key = key.into(); + let value = value.into(); + if value.is_empty() { + self.extras.remove(&key); + } else { + self.extras.insert(key, value); + } + } } impl std::fmt::Debug for LlmBuiltinOverride { @@ -77,6 +111,7 @@ impl std::fmt::Debug for LlmBuiltinOverride { .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]")) .field("model", &self.model) .field("base_url", &self.base_url) + .field("extras", &self.extras) .finish() } } @@ -157,16 +192,23 @@ pub struct Settings { #[serde(default)] pub openai_compatible_base_url: Option<String>, - /// Bedrock region (when llm_backend = "bedrock"). - #[serde(default)] + /// **Deprecated.** Bedrock region — moved to + /// `llm_builtin_overrides["bedrock"].extras["region"]` in Layer D. + /// Existing values are migrated on load via + /// [`Settings::migrate_legacy_provider_fields`]; new code must read + /// from / write to the extras bag instead. Kept for one release so + /// users upgrading from older settings.json files don't lose data. + #[serde(default, skip_serializing_if = "Option::is_none")] pub bedrock_region: Option<String>, - /// Bedrock cross-region inference prefix (when llm_backend = "bedrock"). - #[serde(default)] + /// **Deprecated.** Bedrock cross-region inference prefix — moved to + /// `llm_builtin_overrides["bedrock"].extras["cross_region"]`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub bedrock_cross_region: Option<String>, - /// AWS profile name for Bedrock (when llm_backend = "bedrock"). - #[serde(default)] + /// **Deprecated.** AWS profile name for Bedrock — moved to + /// `llm_builtin_overrides["bedrock"].extras["profile"]`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub bedrock_profile: Option<String>, // === Step 4: Model Selection === @@ -477,6 +519,14 @@ pub struct ChannelSettings { #[serde(default)] pub wasm_channel_owner_ids: std::collections::HashMap<String, i64>, + /// Runtime config overrides for WASM channels. + /// + /// Keys use `<channel>:<config_key>` format (for example, + /// `wecom:allow_from`), and values are passed to the channel config as + /// JSON values. + #[serde(default)] + pub wasm_channel_runtime_overrides: std::collections::HashMap<String, serde_json::Value>, + /// Enabled WASM channels by name. /// Primarily used by the setup wizard to track which channels were configured. /// @@ -527,6 +577,7 @@ impl Default for ChannelSettings { matrix_auto_join: None, matrix_display_name: None, wasm_channel_owner_ids: std::collections::HashMap::new(), + wasm_channel_runtime_overrides: std::collections::HashMap::new(), wasm_channels: Vec::new(), wasm_channels_enabled: true, wasm_channels_dir: None, @@ -1169,6 +1220,13 @@ impl Settings { } } + // Same migration that runs on JSON disk loads — DB rows written + // before Layer D still carry the named `bedrock_*` columns, and + // resolvers now read only from `llm_builtin_overrides["bedrock"] + // .extras`. Without this call, existing DB-backed operators + // silently lose their bedrock region/profile/cross-region after + // upgrading. + settings.migrate_legacy_provider_fields(); settings } @@ -1199,9 +1257,56 @@ impl Settings { /// Load settings from a specific path (used by bootstrap legacy migration). pub fn load_from(path: &std::path::Path) -> Self { - match std::fs::read_to_string(path) { + let mut settings: Self = match std::fs::read_to_string(path) { Ok(data) => serde_json::from_str(&data).unwrap_or_default(), Err(_) => Self::default(), + }; + settings.migrate_legacy_provider_fields(); + settings + } + + /// Move legacy named per-provider fields into the generic + /// `llm_builtin_overrides[<id>].extras` bag. + /// + /// Layer D moved bedrock-specific config out of named columns + /// (`bedrock_region`, `bedrock_cross_region`, `bedrock_profile`) and + /// into a per-provider settings bag so adding a new non-OpenAI-shape + /// backend doesn't require new `Settings` columns. Old persisted + /// settings.json / config.toml files still carry the named fields; + /// this helper folds them into `extras` once on load and clears the + /// originals so subsequent saves write only the new shape. + /// + /// Existing extras values win over the legacy fields. A file that + /// somehow carries both shapes (manual hand-edit, or a future writer + /// that emits both) keeps the new-shape value rather than getting + /// silently overwritten with the legacy one. + /// + /// Idempotent: re-runs are no-ops because the named fields are + /// `take()`-drained. + pub fn migrate_legacy_provider_fields(&mut self) { + let region = self.bedrock_region.take(); + let cross_region = self.bedrock_cross_region.take(); + let profile = self.bedrock_profile.take(); + if region.is_some() || cross_region.is_some() || profile.is_some() { + let entry = self + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); + if let Some(v) = region + && entry.extra("region").is_none() + { + entry.set_extra("region", v); + } + if let Some(v) = cross_region + && entry.extra("cross_region").is_none() + { + entry.set_extra("cross_region", v); + } + if let Some(v) = profile + && entry.extra("profile").is_none() + { + entry.set_extra("profile", v); + } } } @@ -1221,8 +1326,13 @@ impl Settings { Err(e) => return Err(format!("failed to read {}: {}", path.display(), e)), }; - let settings: Self = toml::from_str(&data) + let mut settings: Self = toml::from_str(&data) .map_err(|e| format!("invalid TOML in {}: {}", path.display(), e))?; + // Same migration that runs on JSON disk loads and DB rebuilds — + // TOML files written before Layer D still carry the named + // `bedrock_*` keys at the top level, and resolvers now read + // only from `llm_builtin_overrides["bedrock"].extras`. + settings.migrate_legacy_provider_fields(); Ok(Some(settings)) } @@ -1637,6 +1747,54 @@ mod tests { ); } + #[test] + fn test_wasm_channel_runtime_overrides_db_round_trip() { + let mut settings = Settings::default(); + settings.channels.wasm_channel_runtime_overrides.insert( + "wecom:dm_policy".to_string(), + serde_json::json!("allowlist"), + ); + settings.channels.wasm_channel_runtime_overrides.insert( + "wecom:allow_from".to_string(), + serde_json::json!(["zhangsan", "lisi"]), + ); + + let map = settings.to_db_map(); + let restored = Settings::from_db_map(&map); + assert_eq!( + restored + .channels + .wasm_channel_runtime_overrides + .get("wecom:dm_policy"), + Some(&serde_json::json!("allowlist")) + ); + assert_eq!( + restored + .channels + .wasm_channel_runtime_overrides + .get("wecom:allow_from"), + Some(&serde_json::json!(["zhangsan", "lisi"])) + ); + } + + #[test] + fn test_wasm_channel_runtime_overrides_via_set() { + let mut settings = Settings::default(); + settings + .set( + "channels.wasm_channel_runtime_overrides.wecom:allow_from", + "[\"zhangsan\"]", + ) + .unwrap(); + assert_eq!( + settings + .channels + .wasm_channel_runtime_overrides + .get("wecom:allow_from"), + Some(&serde_json::json!(["zhangsan"])) + ); + } + #[test] fn test_llm_backend_round_trip() { let dir = tempfile::tempdir().unwrap(); @@ -2910,6 +3068,7 @@ mod tests { api_key: Some("sk-secret-123".to_string()), model: Some("gpt-4".to_string()), base_url: None, + extras: Default::default(), }; let debug_output = format!("{:?}", override_val); assert!( @@ -2921,4 +3080,177 @@ mod tests { "Debug output must show [REDACTED] for api_key" ); } + + /// Upgrade-path: a legacy `settings.json` carrying named `bedrock_*` + /// columns must round-trip through `load_from` → `save` → `load_from` + /// without losing data or re-emitting the deprecated columns. + #[test] + fn legacy_bedrock_migration_round_trips_through_save() { + let tmpdir = tempfile::tempdir().expect("tempdir"); + let path = tmpdir.path().join("settings.json"); + std::fs::write( + &path, + r#"{ + "llm_backend": "bedrock", + "bedrock_region": "eu-west-1", + "bedrock_cross_region": "eu", + "bedrock_profile": "prod-bedrock" + }"#, + ) + .expect("write legacy fixture"); + + let settings = Settings::load_from(&path); + let serialized = serde_json::to_string(&settings).expect("serialize"); + std::fs::write(&path, &serialized).expect("write reloaded settings"); + + assert!( + !serialized.contains("\"bedrock_region\""), + "saved settings must not re-emit deprecated bedrock_region: {serialized}" + ); + assert!(!serialized.contains("\"bedrock_cross_region\"")); + assert!(!serialized.contains("\"bedrock_profile\"")); + + let reloaded = Settings::load_from(&path); + let bedrock = reloaded + .llm_builtin_overrides + .get("bedrock") + .expect("bedrock entry must survive round-trip"); + assert_eq!(bedrock.extra("region"), Some("eu-west-1")); + assert_eq!(bedrock.extra("cross_region"), Some("eu")); + assert_eq!(bedrock.extra("profile"), Some("prod-bedrock")); + } + + /// Defense-in-depth: a settings file that somehow carries BOTH a legacy + /// `bedrock_region` column AND a populated `extras["region"]` must keep + /// the new-shape value, not silently downgrade to the legacy one. + /// Reachable in practice only via manual hand-edit, but the docstring + /// promises this behaviour. + #[test] + fn legacy_bedrock_migration_preserves_existing_extras() { + let tmpdir = tempfile::tempdir().expect("tempdir"); + let path = tmpdir.path().join("settings.json"); + std::fs::write( + &path, + r#"{ + "llm_backend": "bedrock", + "bedrock_region": "eu-west-1", + "bedrock_profile": "prod-bedrock", + "llm_builtin_overrides": { + "bedrock": { + "extras": { + "region": "us-east-2" + } + } + } + }"#, + ) + .expect("write conflicting fixture"); + + let settings = Settings::load_from(&path); + let bedrock = settings + .llm_builtin_overrides + .get("bedrock") + .expect("bedrock entry"); + assert_eq!( + bedrock.extra("region"), + Some("us-east-2"), + "pre-existing extras value must win over legacy bedrock_region" + ); + assert_eq!( + bedrock.extra("profile"), + Some("prod-bedrock"), + "absent extras must still be backfilled from legacy field" + ); + assert!(settings.bedrock_region.is_none(), "legacy field is drained"); + assert!(settings.bedrock_profile.is_none()); + } + + /// Upgrade-path: a DB row carrying legacy `bedrock_*` settings + /// (written before Layer D) must reach the resolver via the new + /// `llm_builtin_overrides["bedrock"].extras` bag after + /// `Settings::from_db_map`. Without the migration call inside + /// `from_db_map`, the resolver would silently fall back to env vars + /// or defaults and route through the wrong AWS region/profile. + #[test] + fn legacy_bedrock_fields_migrate_into_extras_on_db_load() { + let mut map: std::collections::HashMap<String, serde_json::Value> = Default::default(); + map.insert( + "llm_backend".to_string(), + serde_json::Value::String("bedrock".to_string()), + ); + map.insert( + "bedrock_region".to_string(), + serde_json::Value::String("eu-west-1".to_string()), + ); + map.insert( + "bedrock_cross_region".to_string(), + serde_json::Value::String("eu".to_string()), + ); + map.insert( + "bedrock_profile".to_string(), + serde_json::Value::String("prod-bedrock".to_string()), + ); + + let settings = Settings::from_db_map(&map); + let bedrock = settings + .llm_builtin_overrides + .get("bedrock") + .expect("bedrock entry must exist after DB migration"); + assert_eq!(bedrock.extra("region"), Some("eu-west-1")); + assert_eq!(bedrock.extra("cross_region"), Some("eu")); + assert_eq!(bedrock.extra("profile"), Some("prod-bedrock")); + assert!( + settings.bedrock_region.is_none(), + "named bedrock_region must be drained by the migration" + ); + assert!(settings.bedrock_cross_region.is_none()); + assert!(settings.bedrock_profile.is_none()); + } + + /// Upgrade-path: a `config.toml` written before Layer D carries the + /// named `bedrock_*` keys at the top level. `Settings::load_toml` + /// must run the same migration as JSON and DB loads so the resolver + /// sees the values via `extras`. + #[test] + fn legacy_bedrock_fields_migrate_into_extras_on_toml_load() { + let tmpdir = tempfile::tempdir().expect("tempdir"); + let path = tmpdir.path().join("config.toml"); + std::fs::write( + &path, + r#"llm_backend = "bedrock" +bedrock_region = "eu-west-1" +bedrock_cross_region = "eu" +bedrock_profile = "prod-bedrock" +"#, + ) + .expect("write legacy toml fixture"); + + let settings = Settings::load_toml(&path).expect("load ok").expect("some"); + let bedrock = settings + .llm_builtin_overrides + .get("bedrock") + .expect("bedrock entry must exist after TOML migration"); + assert_eq!(bedrock.extra("region"), Some("eu-west-1")); + assert_eq!(bedrock.extra("cross_region"), Some("eu")); + assert_eq!(bedrock.extra("profile"), Some("prod-bedrock")); + assert!(settings.bedrock_region.is_none()); + assert!(settings.bedrock_cross_region.is_none()); + assert!(settings.bedrock_profile.is_none()); + } + + /// `migrate_legacy_provider_fields` is idempotent in-memory: once the + /// named fields are drained, repeated calls observe `None` and exit + /// without touching `extras`. + #[test] + fn legacy_bedrock_migration_is_idempotent_in_memory() { + let mut settings = Settings { + bedrock_region: Some("eu-west-1".to_string()), + ..Default::default() + }; + settings.migrate_legacy_provider_fields(); + let before = serde_json::to_value(&settings.llm_builtin_overrides).expect("serialize"); + settings.migrate_legacy_provider_fields(); + let after = serde_json::to_value(&settings.llm_builtin_overrides).expect("serialize"); + assert_eq!(after, before, "second migrate call must be a no-op"); + } } diff --git a/src/setup/README.md b/src/setup/README.md index b3c275d53f2..5db88f5066f 100644 --- a/src/setup/README.md +++ b/src/setup/README.md @@ -357,6 +357,11 @@ key first, then falls back to the standard env var. - Reads `capabilities.json` for `setup.required_secrets` - For each secret: check existing, prompt or auto-generate, validate regex - Save each secret via `SecretsContext` +- For WeCom, additionally prompts for non-secret runtime preferences such as + DM admission policy, optional sender allowlist, and inbound media merge + window. These values are persisted as + `settings.channels.wasm_channel_runtime_overrides.wecom:<key>` entries and + merged back into the channel runtime config during activation/reconfiguration. - Persist selected channel names in `settings.channels.wasm_channels` as a first-run startup fallback. Once the running app writes `activated_channels`, that runtime state becomes the authoritative restore diff --git a/src/setup/channels.rs b/src/setup/channels.rs index a0a0e33dc01..5adb7a55360 100644 --- a/src/setup/channels.rs +++ b/src/setup/channels.rs @@ -6,6 +6,7 @@ //! 3. Validates the configuration //! 4. Saves secrets to the database +use std::collections::HashMap; use std::sync::Arc; use base64::Engine; @@ -22,6 +23,8 @@ use crate::setup::prompts::{ secret_input, select_one, }; +const VALIDATION_RESPONSE_BODY_MAX_BYTES: usize = 64 * 1024; + /// Typed errors for channel setup flows. #[derive(Debug, thiserror::Error)] pub enum ChannelSetupError { @@ -1016,6 +1019,7 @@ pub async fn setup_signal(_settings: &Settings) -> Result<SignalSetupResult, Cha pub struct WasmChannelSetupResult { pub enabled: bool, pub channel_name: String, + pub config_overrides: HashMap<String, serde_json::Value>, } /// Set up a WASM channel using its capabilities file setup schema. @@ -1122,11 +1126,19 @@ pub async fn setup_wasm_channel( } } + let config_overrides = if channel_name.eq_ignore_ascii_case("wecom") { + println!(); + setup_wecom_channel_runtime_overrides()? + } else { + HashMap::new() + }; + print_success(&format!("{} channel configured", channel_name)); Ok(WasmChannelSetupResult { enabled: true, channel_name: channel_name.to_string(), + config_overrides, }) } @@ -1143,6 +1155,60 @@ fn setup_mode_note(channel_name: &str) -> Option<&'static str> { } } +fn parse_comma_separated_values(input: &str) -> Vec<String> { + let mut values = Vec::new(); + for value in input + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + if !values.iter().any(|existing| existing == value) { + values.push(value.to_string()); + } + } + values +} + +fn setup_wecom_channel_runtime_overrides() +-> Result<HashMap<String, serde_json::Value>, ChannelSetupError> { + print_info("WeCom access policy (applies to DM sender authorization):"); + let dm_policy = optional_input( + "DM policy (pairing/open/allowlist)", + Some("default: pairing"), + )? + .unwrap_or_else(|| "pairing".to_string()); + let dm_policy = dm_policy.trim().to_ascii_lowercase(); + + if !matches!(dm_policy.as_str(), "pairing" | "open" | "allowlist") { + return Err(ChannelSetupError::Validation(format!( + "Invalid DM policy '{}'; expected one of: pairing, open, allowlist", + dm_policy + ))); + } + + let allow_from_raw = optional_input( + "Allow from (comma-separated WeCom user IDs)", + Some("leave empty to require pairing when dm_policy=pairing"), + )? + .unwrap_or_default(); + let allow_from = parse_comma_separated_values(&allow_from_raw); + + print_info(&format!("DM policy: {}", dm_policy)); + if allow_from.is_empty() { + print_info("Allow from: (none)"); + } else { + print_info(&format!("Allow from: {}", allow_from.join(", "))); + } + + let mut overrides = HashMap::new(); + overrides.insert( + "dm_policy".to_string(), + serde_json::Value::String(dm_policy), + ); + overrides.insert("allow_from".to_string(), serde_json::json!(allow_from)); + Ok(overrides) +} + async fn validate_channel_credentials( secrets: &SecretsContext, validation_endpoint: &str, @@ -1164,7 +1230,7 @@ async fn validate_channel_credentials( .build() .map_err(|e| ChannelSetupError::Network(format!("Failed to build HTTP client: {}", e)))?; - let response = client.get(parsed.clone()).send().await.map_err(|e| { + let mut response = client.get(parsed.clone()).send().await.map_err(|e| { ChannelSetupError::Network(format!( "Validation request to {} failed: {}", target, @@ -1172,26 +1238,83 @@ async fn validate_channel_credentials( )) })?; - if response.status().is_success() { - Ok(()) + let status = response.status(); + if status.is_success() { + let body = read_validation_response_body(&mut response).await?; + if let Some(error) = validation_endpoint_body_error(&body) { + Err(ChannelSetupError::Validation(error)) + } else { + Ok(()) + } } else { Err(ChannelSetupError::Validation(format!( "Validation endpoint returned HTTP {} from {}", - response.status(), - target + status, target ))) } } +fn validation_response_exceeds_limit(current_len: usize, chunk_len: usize, limit: usize) -> bool { + match current_len.checked_add(chunk_len) { + Some(total) => total > limit, + None => true, + } +} + +async fn read_validation_response_body( + response: &mut reqwest::Response, +) -> Result<Vec<u8>, ChannelSetupError> { + if let Some(content_length) = response.content_length() + && content_length > VALIDATION_RESPONSE_BODY_MAX_BYTES as u64 + { + return Err(ChannelSetupError::Network(format!( + "Validation response exceeded {} bytes", + VALIDATION_RESPONSE_BODY_MAX_BYTES + ))); + } + + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|e| { + ChannelSetupError::Network(format!("Failed to read validation response: {}", e)) + })? { + if validation_response_exceeds_limit( + body.len(), + chunk.len(), + VALIDATION_RESPONSE_BODY_MAX_BYTES, + ) { + return Err(ChannelSetupError::Network(format!( + "Validation response exceeded {} bytes", + VALIDATION_RESPONSE_BODY_MAX_BYTES + ))); + } + body.extend_from_slice(&chunk); + } + + Ok(body) +} + +fn validation_endpoint_body_error(body: &[u8]) -> Option<String> { + let parsed: serde_json::Value = serde_json::from_slice(body).ok()?; + let errcode = parsed.get("errcode")?.as_i64()?; + if errcode == 0 { + return None; + } + + let errmsg = parsed + .get("errmsg") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown error"); + Some(format!( + "Validation endpoint returned errcode {errcode}: {errmsg}" + )) +} + async fn substitute_validation_placeholders( secrets: &SecretsContext, validation_endpoint: &str, ) -> Result<String, ChannelSetupError> { let mut resolved = validation_endpoint.to_string(); - let placeholder_names: std::collections::BTreeSet<String> = validation_placeholder_regex() - .captures_iter(validation_endpoint) - .filter_map(|caps| caps.get(1).map(|m| m.as_str().to_string())) - .collect(); + let placeholder_names = validation_placeholder_names(validation_endpoint); for secret_name in placeholder_names { let secret_value = secrets.get_secret(&secret_name).await?; @@ -1332,12 +1455,25 @@ fn normalize_validation_domain(host: &str) -> &str { host.trim_end_matches('.') } -fn validation_placeholder_regex() -> &'static regex::Regex { - static PLACEHOLDER_RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new(); - PLACEHOLDER_RE.get_or_init(|| { - regex::Regex::new(r"\{([A-Za-z0-9_]+)\}") - .expect("validation placeholder regex must compile") // safety: hardcoded literal - }) +fn validation_placeholder_names(template: &str) -> std::collections::BTreeSet<String> { + let mut names = std::collections::BTreeSet::new(); + let mut offset = 0; + + while let Some(relative_start) = template[offset..].find('{') { + let start = offset + relative_start; + let value_start = start + 1; + let Some(relative_end) = template[value_start..].find('}') else { + break; + }; + let end = value_start + relative_end; + let name = &template[value_start..end]; + if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + names.insert(name.to_string()); + } + offset = end + 1; + } + + names } fn validation_target_display(parsed: &Url) -> String { @@ -1533,6 +1669,13 @@ mod tests { assert!(!validate_cloudflare_token_format("")); } + #[test] + fn test_validation_response_exceeds_limit_detects_chunk_overflow() { + assert!(!super::validation_response_exceeds_limit(10, 20, 30)); + assert!(super::validation_response_exceeds_limit(10, 21, 30)); + assert!(super::validation_response_exceeds_limit(usize::MAX, 1, 30)); + } + #[tokio::test] async fn test_substitute_validation_placeholders() { let secrets = test_secrets_context(); @@ -1588,6 +1731,18 @@ mod tests { ); } + #[test] + fn test_validation_placeholder_names_extracts_unique_secret_names() { + let names = super::validation_placeholder_names( + "https://api.example.com/{workspace_id}/verify?token={telegram_bot_token}&again={workspace_id}", + ); + + assert_eq!( + names.into_iter().collect::<Vec<_>>(), + vec!["telegram_bot_token".to_string(), "workspace_id".to_string()] + ); + } + #[tokio::test] async fn test_substitute_validation_placeholders_missing_secret() { let secrets = test_secrets_context(); diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 0fcf4e32f0b..a959990c158 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -23,13 +23,6 @@ use crate::channels::wasm::{ ChannelCapabilitiesFile, available_channel_names, install_bundled_channel, }; use crate::config::OAUTH_PLACEHOLDER; -use crate::llm::models::{ - build_nearai_model_fetch_config, fetch_anthropic_models, fetch_ollama_models, - fetch_openai_compatible_models, fetch_openai_models, -}; -#[cfg(test)] -use crate::llm::models::{is_openai_chat_model, sort_openai_models}; -use crate::llm::{SessionConfig, SessionManager}; use crate::secrets::{SecretsCrypto, SecretsStore}; use crate::settings::{KeySource, Settings}; use crate::setup::channels::{ @@ -39,6 +32,31 @@ use crate::setup::prompts::{ confirm, input, optional_input, print_banner, print_error, print_header, print_info, print_step, print_success, secret_input, select_many, select_one, }; +use ironclaw_llm::auth::AuthPrompt; +use ironclaw_llm::models::{ModelFetchOptions, build_nearai_model_fetch_config, fetch_models_for}; +use ironclaw_llm::{SessionConfig, SessionManager}; + +/// `AuthPrompt` impl for the interactive setup wizard: prints the device +/// code to stdout and tries to open the verification URL in a browser. +struct WizardAuthPrompt; + +impl AuthPrompt for WizardAuthPrompt { + fn show_device_code(&self, verification_uri: &str, user_code: &str) { + print_info(&format!("Verification URL: {verification_uri}")); + print_info(&format!("One-time code: {user_code}")); + if let Err(e) = open::that(verification_uri) { + tracing::debug!( + url = %verification_uri, + error = %e, + "Failed to open device login URL in browser" + ); + print_info("Open the URL above manually if your browser did not launch."); + } else { + print_info("Opened your browser to complete the device login."); + } + print_info("Waiting for authorization..."); + } +} // unused const, keep commented for clarity / future use // const CHANNEL_INDEX_CLI: usize = 0; @@ -317,7 +335,7 @@ impl SetupWizard { } self.llm_api_key = Some(SecretString::from(api_key)); if self.settings.selected_model.is_none() { - let default = crate::llm::DEFAULT_MODEL; + let default = ironclaw_llm::DEFAULT_MODEL; self.settings.selected_model = Some(default.to_string()); print_info(&format!("Using default model: {default}")); } @@ -339,7 +357,7 @@ impl SetupWizard { } } self.llm_api_key = Some(SecretString::from(api_key)); - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); if self.settings.selected_model.is_none() { let default = registry .find("anthropic") @@ -362,7 +380,7 @@ impl SetupWizard { } } self.llm_api_key = Some(SecretString::from(api_key)); - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); if self.settings.selected_model.is_none() { let default = registry .find("openai") @@ -385,7 +403,7 @@ impl SetupWizard { } } self.llm_api_key = Some(SecretString::from(api_key)); - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); if self.settings.selected_model.is_none() { let default = registry .find("openrouter") @@ -1266,60 +1284,27 @@ impl SetupWizard { /// Step 3: Inference provider selection. /// - /// Uses the provider registry to dynamically build the selection menu. - /// NearAI is always first (special auth), then all registry providers - /// that have setup hints. + /// The menu is built entirely from the LLM crate's + /// [`ironclaw_llm::ProviderRegistry::selectable`] — every supported + /// backend including the dedicated-config ones (nearai, bedrock, + /// openai_codex, gemini_oauth) declares its `SetupHint` in + /// `providers.json`, and dispatch goes through that hint. No + /// per-backend `if id == "..."` branches live here. async fn step_inference_provider(&mut self) -> Result<(), SetupError> { - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); - // Show current provider if already configured + // Show current provider if already configured. if let Some(current) = self.settings.llm_backend.clone() { - let display = if current == "nearai" { - "NEAR AI".to_string() - } else if let Some(def) = registry.find(&current) { - def.setup - .as_ref() - .map(|s| s.display_name().to_string()) - .unwrap_or_else(|| def.id.clone()) - } else { - match current.as_str() { - "nearai" => "NEAR AI".to_string(), - "gemini_oauth" | "gemini-oauth" => "Gemini API (OAuth)".to_string(), - _ => { - if let Some(def) = registry.find(&current) { - def.setup - .as_ref() - .map(|s| s.display_name().to_string()) - .unwrap_or_else(|| def.id.clone()) - } else { - current.clone() - } - } - } - }; + let display = registry + .find(&current) + .and_then(|def| def.setup.as_ref().map(|s| s.display_name().to_string())) + .unwrap_or_else(|| current.clone()); print_info(&format!("Current provider: {}", display)); println!(); - let is_known = current == "nearai" - || current == "bedrock" - || current == "gemini_oauth" - || current == "gemini-oauth" - || current == "openai_codex" - || registry.is_known(&current); + let is_known = registry.is_known(&current); if is_known && confirm("Keep current provider?", true).map_err(SetupError::Io)? { - if current == "bedrock" { - print_info("Keeping existing AWS Bedrock configuration."); - return Ok(()); - } - if current == "gemini_oauth" || current == "gemini-oauth" { - print_info("Keeping existing Gemini CLI OAuth configuration."); - return Ok(()); - } - if current == "openai_codex" { - print_info("Keeping existing OpenAI Codex configuration."); - return Ok(()); - } return self.run_provider_setup(&current, &registry).await; } @@ -1334,171 +1319,115 @@ impl SetupWizard { print_info("Select your inference provider:"); println!(); - // Build menu: NearAI first, then Gemini OAuth, then OpenAI Codex, then registry providers, then Bedrock let selectable = registry.selectable(); - // Detect which providers have API keys already set in the environment. - let detected_env: HashMap<&str, bool> = [ - ("nearai", std::env::var("NEARAI_API_KEY").is_ok()), - ( - "anthropic", - std::env::var("ANTHROPIC_API_KEY").is_ok() - || std::env::var("ANTHROPIC_OAUTH_TOKEN").is_ok(), - ), - ("openai", std::env::var("OPENAI_API_KEY").is_ok()), - ("openrouter", std::env::var("OPENROUTER_API_KEY").is_ok()), - ] - .into_iter() - .collect(); - - // Helper: build a label for a provider entry, prepending a checkmark if detected. - let make_label = |id: &str, name: &str, desc: &str| -> String { - if detected_env.get(id).copied().unwrap_or(false) { - format!("\u{2713} {:<15}- {}", name, desc) - } else { - format!(" {:<15}- {}", name, desc) - } + // A provider is "detected" if its declared API key env var is + // populated, OR if it's Anthropic and the OAuth-token env var is + // set instead. Other backends (bedrock = AWS chain, codex = OAuth, + // nearai = session) have no env-detectable presence. + let is_detected = |def: &ironclaw_llm::ProviderDefinition| -> bool { + let primary = def + .api_key_env + .as_deref() + .is_some_and(|env| std::env::var(env).is_ok()); + let anthropic_oauth = + def.id == "anthropic" && std::env::var("ANTHROPIC_OAUTH_TOKEN").is_ok(); + primary || anthropic_oauth }; - // Collect all entries as (provider_id, label, is_detected). struct ProviderEntry { id: String, label: String, detected: bool, } - let mut entries: Vec<ProviderEntry> = Vec::with_capacity(2 + selectable.len()); - - entries.push(ProviderEntry { - id: "nearai".to_string(), - label: make_label("nearai", "NEAR AI", "multi-model access via NEAR account"), - detected: detected_env.get("nearai").copied().unwrap_or(false), - }); - - entries.push(ProviderEntry { - id: "gemini_oauth".to_string(), - label: make_label( - "gemini_oauth", - "Gemini CLI", - "Official Gemini API via Gemini CLI OAuth", - ), - detected: false, - }); - - entries.push(ProviderEntry { - id: "openai_codex".to_string(), - label: make_label( - "openai_codex", - "OpenAI Codex", - "ChatGPT subscription (Plus/Pro/Max)", - ), - detected: false, - }); - - for def in &selectable { - let display_name = def - .setup - .as_ref() - .map(|s| s.display_name()) - .unwrap_or(&def.id); - entries.push(ProviderEntry { - id: def.id.clone(), - label: make_label(&def.id, display_name, &def.description), - detected: detected_env.get(def.id.as_str()).copied().unwrap_or(false), - }); - } - - // Bedrock is a special case (native AWS SDK, not registry-based) - entries.push(ProviderEntry { - id: "bedrock".to_string(), - label: make_label( - "bedrock", - "AWS Bedrock", - "Claude & other models via AWS (IAM, SSO)", - ), - detected: false, - }); + let mut entries: Vec<ProviderEntry> = selectable + .iter() + .map(|def| { + let display_name = def + .setup + .as_ref() + .map(|s| s.display_name()) + .unwrap_or(&def.id); + let detected = is_detected(def); + let prefix = if detected { '\u{2713}' } else { ' ' }; + ProviderEntry { + id: def.id.clone(), + label: format!("{prefix} {:<15}- {}", display_name, def.description), + detected, + } + }) + .collect(); - // Sort: detected providers first, preserving relative order within each group. + // Detected providers first; relative order within each group is + // preserved, so providers.json controls the menu ordering for + // the no-detection case. entries.sort_by_key(|e| !e.detected); - let mut options: Vec<String> = Vec::with_capacity(entries.len()); - let mut provider_ids: Vec<String> = Vec::with_capacity(entries.len()); - for entry in &entries { - options.push(entry.label.clone()); - provider_ids.push(entry.id.clone()); - } - + let options: Vec<String> = entries.iter().map(|e| e.label.clone()).collect(); let option_refs: Vec<&str> = options.iter().map(|s| s.as_str()).collect(); let choice = select_one("Provider:", &option_refs).map_err(SetupError::Io)?; - let selected_id = &provider_ids[choice]; + let selected_id = entries[choice].id.clone(); - if selected_id == "bedrock" { - self.setup_bedrock().await?; - } else if selected_id == "gemini_oauth" { - self.setup_gemini_oauth().await?; - } else { - self.run_provider_setup(selected_id, &registry).await?; - } + self.run_provider_setup(&selected_id, &registry).await?; Ok(()) } - /// Run the setup flow for a specific provider. + /// Run the setup flow for a specific provider, dispatching purely + /// off the registry's [`ironclaw_llm::registry::SetupHint`]. /// - /// NearAI has its own special flow. Registry providers dispatch - /// based on their `SetupHint` kind. + /// The wizard does not know about per-backend strings — every supported + /// backend declares its credential collection style in `providers.json`, + /// and that style maps to one of the seven `setup_*` helpers below. + /// The two remaining `def.id == "..."` checks live inside the `ApiKey` + /// arm because Anthropic and GitHub Copilot present a hybrid choice + /// (API key OR OAuth) that the simple ApiKey hint does not capture. async fn run_provider_setup( &mut self, provider_id: &str, - registry: &crate::llm::ProviderRegistry, + registry: &ironclaw_llm::ProviderRegistry, ) -> Result<(), SetupError> { - if provider_id == "nearai" { - return self.setup_nearai().await; - } - - if provider_id == "openai_codex" { - return self.setup_openai_codex().await; - } - let def = registry .find(provider_id) .ok_or_else(|| SetupError::Config(format!("Unknown provider: {}", provider_id)))?; - // Providers without a setup hint (e.g., user-defined providers configured - // purely via env vars) skip credential setup and go to model selection. + // Providers without a setup hint (e.g., user-defined providers + // configured purely via env vars) skip credential setup and go + // straight to model selection. let Some(setup) = def.setup.as_ref() else { print_info(&format!( "Provider '{}' has no setup wizard. Configure via environment variables.", provider_id )); - self.set_llm_backend_preserving_model(provider_id); + self.set_llm_backend_preserving_model(&def.id); return Ok(()); }; - // Anthropic has a custom flow: API key or OAuth token from `claude login`. - if provider_id == "anthropic" { - return self.setup_anthropic().await; - } - - if provider_id == "github_copilot" { - return self.setup_github_copilot().await; - } - + use ironclaw_llm::registry::SetupHint; match setup { - crate::llm::registry::SetupHint::ApiKey { + SetupHint::ApiKey { secret_name, key_url, display_name, .. } => { + // Hybrid flows: Anthropic adds an "API key or `claude + // login` OAuth" choice; GitHub Copilot adds a paste-vs- + // device-code choice. Both still classify as ApiKey. + if def.id == "anthropic" { + return self.setup_anthropic().await; + } + if def.id == "github_copilot" { + return self.setup_github_copilot().await; + } + let env_var = def.api_key_env.as_deref().unwrap_or("LLM_API_KEY"); let url = key_url.as_deref().unwrap_or("the provider's website"); // Only store base URL for providers that resolve through - // LLM_BASE_URL (openai_compatible, openrouter). Other providers - // like groq/nvidia have their own base_url_env and don't need - // this backward-compat setting. + // LLM_BASE_URL (openai_compatible, openrouter). Other + // providers (groq/nvidia/etc.) have their own base_url_env. if def.base_url_env.as_deref() == Some("LLM_BASE_URL") && let Some(ref base_url) = def.default_base_url { @@ -1515,10 +1444,10 @@ impl SetupWizard { ) .await?; } - crate::llm::registry::SetupHint::Ollama { .. } => { + SetupHint::Ollama { .. } => { self.setup_ollama_generic(def)?; } - crate::llm::registry::SetupHint::OpenAiCompatible { + SetupHint::OpenAiCompatible { secret_name, display_name, .. @@ -1526,6 +1455,23 @@ impl SetupWizard { self.setup_openai_compatible_generic(&def.id, secret_name, display_name) .await?; } + SetupHint::AwsCredentials { .. } => { + self.setup_bedrock().await?; + } + SetupHint::OAuthDeviceCode { backend, .. } => match backend.as_str() { + "openai_codex" => self.setup_openai_codex().await?, + other => { + return Err(SetupError::Config(format!( + "OAuth device-code flow for backend '{other}' is not implemented" + ))); + } + }, + SetupHint::FileBasedCredentials { .. } => { + self.setup_gemini_oauth().await?; + } + SetupHint::SessionToken { .. } => { + self.setup_nearai().await?; + } } Ok(()) @@ -1690,58 +1636,31 @@ impl SetupWizard { return Err(SetupError::Auth("No token provided".to_string())); } - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(15)) - .build() - .map_err(|e| SetupError::Auth(format!("Failed to create HTTP client: {e}")))?; + ironclaw_llm::auth::validate_token(ironclaw_llm::auth::AuthBackend::GithubCopilot, &token) + .await + .map_err(|e| SetupError::Auth(e.to_string()))?; - self.save_github_copilot_token(&client, &token).await + self.save_github_copilot_token(&token).await } async fn setup_github_copilot_device_login(&mut self) -> Result<(), SetupError> { self.set_llm_backend_preserving_model("github_copilot"); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(15)) - .build() - .map_err(|e| SetupError::Auth(format!("Failed to create HTTP client: {e}")))?; - - let device = crate::llm::github_copilot_auth::request_device_code(&client) - .await - .map_err(|e| SetupError::Auth(e.to_string()))?; - print_info("Authorize IronClaw with GitHub Copilot in your browser."); - print_info(&format!("Verification URL: {}", device.verification_uri)); - print_info(&format!("One-time code: {}", device.user_code)); - - if let Err(e) = open::that(&device.verification_uri) { - tracing::debug!( - url = %device.verification_uri, - error = %e, - "Failed to open GitHub Copilot device login URL" - ); - print_info("Open the URL above manually if your browser did not launch."); - } else { - print_info("Opened your browser to GitHub device login."); - } - - print_info("Waiting for GitHub authorization..."); - let token = crate::llm::github_copilot_auth::wait_for_device_login(&client, &device) - .await - .map_err(|e| SetupError::Auth(e.to_string()))?; + let outcome = ironclaw_llm::auth::start_login( + ironclaw_llm::auth::LoginRequest::GithubCopilot, + &WizardAuthPrompt, + ) + .await + .map_err(|e| SetupError::Auth(e.to_string()))?; - self.save_github_copilot_token(&client, &token).await + let token = outcome.token_to_persist.ok_or_else(|| { + SetupError::Auth("github_copilot login returned no token".to_string()) + })?; + self.save_github_copilot_token(token.expose_secret()).await } - async fn save_github_copilot_token( - &mut self, - client: &reqwest::Client, - token: &str, - ) -> Result<(), SetupError> { - crate::llm::github_copilot_auth::validate_token(client, token) - .await - .map_err(|e| SetupError::Auth(e.to_string()))?; - + async fn save_github_copilot_token(&mut self, token: &str) -> Result<(), SetupError> { if let Ok(ctx) = self.init_secrets_context().await { let key = SecretString::from(token.to_string()); ctx.save_secret("llm_github_copilot_token", &key) @@ -1916,17 +1835,14 @@ impl SetupWizard { self.settings.selected_model = None; } - use crate::config::OpenAiCodexConfig; - use crate::llm::OpenAiCodexSessionManager; - - let config = OpenAiCodexConfig::default(); - - let mgr = OpenAiCodexSessionManager::new(config).map_err(|e| { - SetupError::Config(format!("OpenAI Codex session manager init failed: {}", e)) - })?; - mgr.device_code_login().await.map_err(|e| { - SetupError::Config(format!("OpenAI Codex authentication failed: {}", e)) - })?; + ironclaw_llm::auth::start_login( + ironclaw_llm::auth::LoginRequest::OpenAiCodex( + ironclaw_llm::auth::OpenAiCodexLoginOptions::default(), + ), + &WizardAuthPrompt, + ) + .await + .map_err(|e| SetupError::Config(format!("OpenAI Codex authentication failed: {e}")))?; print_success("OpenAI Codex configured (ChatGPT subscription)"); Ok(()) @@ -1935,7 +1851,7 @@ impl SetupWizard { /// Generic Ollama-style setup: just needs a base URL, no API key. fn setup_ollama_generic( &mut self, - def: &crate::llm::ProviderDefinition, + def: &ironclaw_llm::ProviderDefinition, ) -> Result<(), SetupError> { self.set_llm_backend_preserving_model(&def.id); @@ -1966,22 +1882,34 @@ impl SetupWizard { } /// AWS Bedrock provider setup: region, auth, and cross-region config. + /// + /// Writes through the generic `llm_builtin_overrides["bedrock"].extras` + /// bag introduced in Layer D — no top-level `bedrock_*` `Settings` + /// fields are touched. async fn setup_bedrock(&mut self) -> Result<(), SetupError> { self.set_llm_backend_preserving_model("bedrock"); // Region - let default_region = self + let bedrock = self .settings - .bedrock_region - .as_deref() - .unwrap_or("us-east-1"); + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); + let default_region = bedrock + .extra("region") + .unwrap_or(ironclaw_llm::BedrockConfig::DEFAULT_REGION) + .to_string(); let region_input = optional_input("AWS region", Some(&format!("default: {}", default_region))) .map_err(SetupError::Io)?; - let region = region_input.unwrap_or_else(|| default_region.to_string()); - self.settings.bedrock_region = Some(region.clone()); + let region = region_input.unwrap_or(default_region); + self.settings + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default() + .set_extra("region", region.clone()); // Auth method print_info("Select authentication method:"); @@ -1992,10 +1920,15 @@ impl SetupWizard { ]; let auth_choice = select_one("Auth:", auth_options).map_err(SetupError::Io)?; + let bedrock = self + .settings + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); match auth_choice { 0 => { // Default AWS credentials — clear any stale named profile - self.settings.bedrock_profile = None; + bedrock.set_extra("profile", ""); print_info( "Using default AWS credential chain (env vars, ~/.aws/credentials, IAM roles).", ); @@ -2005,11 +1938,10 @@ impl SetupWizard { let profile = input("AWS profile name (from ~/.aws/config)").map_err(SetupError::Io)?; if profile.trim().is_empty() { - // Empty input clears any previously configured profile - self.settings.bedrock_profile = None; + bedrock.set_extra("profile", ""); print_info("AWS profile cleared; using default AWS credential chain instead."); } else { - self.settings.bedrock_profile = Some(profile.clone()); + bedrock.set_extra("profile", profile.clone()); print_success(&format!("AWS profile '{}' saved", profile)); } } @@ -2032,21 +1964,22 @@ impl SetupWizard { ]; let cross_choice = select_one("Cross-region:", cross_options).map_err(SetupError::Io)?; - let cross_region = match cross_choice { - 0 => Some("us".to_string()), - 1 => Some("global".to_string()), - 2 => Some("eu".to_string()), - 3 => Some("apac".to_string()), - 4 => None, - _ => None, - }; - self.settings.bedrock_cross_region = cross_region; - - let region = self + let bedrock = self .settings - .bedrock_region - .as_deref() - .unwrap_or("us-east-1"); + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); + match cross_choice { + 0 => bedrock.set_extra("cross_region", "us"), + 1 => bedrock.set_extra("cross_region", "global"), + 2 => bedrock.set_extra("cross_region", "eu"), + 3 => bedrock.set_extra("cross_region", "apac"), + _ => bedrock.set_extra("cross_region", ""), + } + + let region = bedrock + .extra("region") + .unwrap_or(ironclaw_llm::BedrockConfig::DEFAULT_REGION); print_success(&format!("AWS Bedrock configured (region: {})", region)); Ok(()) } @@ -2108,27 +2041,22 @@ impl SetupWizard { print_info("Starting Gemini CLI OAuth authentication..."); println!(); - let creds_path = crate::config::GeminiOauthConfig::default_credentials_path(); - let cred_manager = - crate::llm::gemini_oauth::CredentialManager::new(&creds_path).map_err(|e| { - SetupError::Config(format!( - "Failed to initialize Gemini credential manager: {}", - e - )) - })?; + let credentials_path = crate::config::GeminiOauthConfig::default_credentials_path(); + let outcome = ironclaw_llm::auth::start_login( + ironclaw_llm::auth::LoginRequest::Gemini { credentials_path }, + &WizardAuthPrompt, + ) + .await + .map_err(|e| { + SetupError::Config(format!( + "Gemini CLI authentication failed: {e}. Please try again." + )) + })?; - match cred_manager.get_valid_credential().await { - Ok(cred) => { - print_success("Gemini CLI authentication successful!"); - if let Some(ref pid) = cred.project_id { - print_info(&format!("Cloud Code project: {}", pid)); - } - } - Err(e) => { - return Err(SetupError::Config(format!( - "Gemini CLI authentication failed: {}. Please try again.", - e - ))); + print_success("Gemini CLI authentication successful!"); + for (key, value) in &outcome.display { + if key == "project_id" { + print_info(&format!("Cloud Code project: {value}")); } } @@ -2158,14 +2086,14 @@ impl SetupWizard { } let backend = self.settings.llm_backend.as_deref().unwrap_or("nearai"); - let registry = crate::llm::ProviderRegistry::load(); + let registry = ironclaw_llm::ProviderRegistry::load(); match backend { "nearai" => { // NEAR AI: use existing provider list_models() let fetched = self.fetch_nearai_models().await; let models = if fetched.is_empty() { - crate::llm::default_models() + ironclaw_llm::default_models() } else { fetched.iter().map(|m| (m.clone(), m.clone())).collect() }; @@ -2233,32 +2161,30 @@ impl SetupWizard { .as_ref() .map(|k| k.expose_secret().to_string()); - let models = match backend { - "anthropic" => fetch_anthropic_models(cached_key.as_deref()).await, - "openai" => fetch_openai_models(cached_key.as_deref()).await, - "ollama" => { - let base_url = self - .settings - .ollama_base_url - .as_deref() - .or(def.default_base_url.as_deref()) - .unwrap_or("http://localhost:11434"); - let models = fetch_ollama_models(base_url).await; - if models.is_empty() { - print_info( - "No models found. Pull one first: ollama pull llama3", - ); - } - models - } - _ => { - // Generic OpenAI-compatible model listing - let base_url = def.default_base_url.as_deref().unwrap_or(""); - fetch_openai_compatible_models(base_url, cached_key.as_deref()) - .await - } + let ollama_base = self + .settings + .ollama_base_url + .as_deref() + .or(def.default_base_url.as_deref()) + .unwrap_or("http://localhost:11434"); + let base_url = if backend == "ollama" { + Some(ollama_base) + } else { + def.default_base_url.as_deref() }; + let models = fetch_models_for( + backend, + &ModelFetchOptions { + api_key: cached_key.as_deref(), + base_url, + }, + ) + .await; + if backend == "ollama" && models.is_empty() { + print_info("No models found. Pull one first: ollama pull llama3"); + } + // Apply models_filter from setup hint let models = if let Some(filter) = def.setup.as_ref().and_then(|s| s.models_filter()) @@ -2357,7 +2283,7 @@ impl SetupWizard { None => return vec![], }; - use crate::llm::create_llm_provider; + use ironclaw_llm::create_llm_provider; let config = build_nearai_model_fetch_config(); @@ -2814,6 +2740,12 @@ impl SetupWizard { // Process selected WASM channels let mut enabled_wasm_channels = Vec::new(); + let existing_runtime_overrides = self + .settings + .channels + .wasm_channel_runtime_overrides + .clone(); + let mut enabled_runtime_overrides: HashMap<String, serde_json::Value> = HashMap::new(); for channel_name in selected_wasm_channels { println!(); if let Some(ref ctx) = secrets { @@ -2828,6 +2760,7 @@ impl SetupWizard { crate::setup::channels::WasmChannelSetupResult { enabled: true, channel_name: channel_name.clone(), + config_overrides: HashMap::new(), } } } else { @@ -2839,7 +2772,19 @@ impl SetupWizard { }; if result.enabled { - enabled_wasm_channels.push(result.channel_name); + let channel_name = result.channel_name; + enabled_wasm_channels.push(channel_name.clone()); + let channel_overrides = merge_wasm_channel_runtime_overrides_for_channel( + &existing_runtime_overrides, + &channel_name, + result.config_overrides, + ); + for (config_key, value) in channel_overrides { + enabled_runtime_overrides.insert( + wasm_channel_runtime_override_key(&channel_name, &config_key), + value, + ); + } } } else { // No secrets context, just enable the channel @@ -2848,10 +2793,21 @@ impl SetupWizard { capitalize_first(&channel_name) )); enabled_wasm_channels.push(channel_name.clone()); + let channel_overrides = collect_wasm_channel_runtime_overrides_for_channel( + &existing_runtime_overrides, + &channel_name, + ); + for (config_key, value) in channel_overrides { + enabled_runtime_overrides.insert( + wasm_channel_runtime_override_key(&channel_name, &config_key), + value, + ); + } } } self.settings.channels.wasm_channels = enabled_wasm_channels; + self.settings.channels.wasm_channel_runtime_overrides = enabled_runtime_overrides; Ok(()) } @@ -3837,6 +3793,36 @@ fn capitalize_first(s: &str) -> String { } } +fn wasm_channel_runtime_override_key(channel_name: &str, config_key: &str) -> String { + format!("{}:{}", channel_name, config_key) +} + +fn collect_wasm_channel_runtime_overrides_for_channel( + stored: &HashMap<String, serde_json::Value>, + channel_name: &str, +) -> HashMap<String, serde_json::Value> { + let mut result = HashMap::new(); + let prefix = format!("{channel_name}:"); + for (key, value) in stored { + if let Some(config_key) = key.strip_prefix(&prefix) + && !config_key.trim().is_empty() + { + result.insert(config_key.to_string(), value.clone()); + } + } + result +} + +fn merge_wasm_channel_runtime_overrides_for_channel( + stored: &HashMap<String, serde_json::Value>, + channel_name: &str, + new_overrides: HashMap<String, serde_json::Value>, +) -> HashMap<String, serde_json::Value> { + let mut result = collect_wasm_channel_runtime_overrides_for_channel(stored, channel_name); + result.extend(new_overrides); + result +} + #[cfg(test)] async fn install_missing_bundled_channels( channels_dir: &std::path::Path, @@ -4031,7 +4017,7 @@ async fn install_selected_bundled_channels( #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; #[cfg(unix)] use std::ffi::OsString; @@ -4181,6 +4167,33 @@ mod tests { ); } + #[test] + fn wasm_channel_runtime_overrides_merge_existing_with_new_values() { + let stored = HashMap::from([ + ( + "wecom:allow_from".to_string(), + serde_json::json!(["ZhangSan"]), + ), + ("wecom:dm_policy".to_string(), serde_json::json!("pairing")), + ("telegram:dm_policy".to_string(), serde_json::json!("open")), + ]); + let new_overrides = + HashMap::from([("dm_policy".to_string(), serde_json::json!("allowlist"))]); + + let merged = + merge_wasm_channel_runtime_overrides_for_channel(&stored, "wecom", new_overrides); + + assert_eq!( + merged.get("allow_from"), + Some(&serde_json::json!(["ZhangSan"])) + ); + assert_eq!( + merged.get("dm_policy"), + Some(&serde_json::json!("allowlist")) + ); + assert!(!merged.contains_key("telegram:dm_policy")); + } + #[test] #[cfg(unix)] fn test_try_with_config_and_toml_propagates_invalid_owner_env() { @@ -4292,7 +4305,7 @@ mod tests { async fn test_fetch_anthropic_models_static_fallback() { // With no API key, should return static defaults let _guard = EnvGuard::clear("ANTHROPIC_API_KEY"); - let models = fetch_anthropic_models(None).await; + let models = fetch_models_for("anthropic", &ModelFetchOptions::default()).await; assert!(!models.is_empty()); assert!( models.iter().any(|(id, _)| id.contains("claude")), @@ -4303,7 +4316,7 @@ mod tests { #[tokio::test] async fn test_fetch_openai_models_static_fallback() { let _guard = EnvGuard::clear("OPENAI_API_KEY"); - let models = fetch_openai_models(None).await; + let models = fetch_models_for("openai", &ModelFetchOptions::default()).await; assert!(!models.is_empty()); assert_eq!(models[0].0, "gpt-5.3-codex"); assert!( @@ -4342,49 +4355,6 @@ mod tests { ); } - #[test] - fn test_is_openai_chat_model_includes_gpt5_and_filters_non_chat_variants() { - assert!(is_openai_chat_model("gpt-5")); - assert!(is_openai_chat_model("gpt-5-mini-2026-01-01")); - assert!(is_openai_chat_model("o3-2025-04-16")); - assert!(!is_openai_chat_model("chatgpt-image-latest")); - assert!(!is_openai_chat_model("gpt-4o-realtime-preview")); - assert!(!is_openai_chat_model("gpt-4o-mini-transcribe")); - assert!(!is_openai_chat_model("text-embedding-3-large")); - } - - #[test] - fn test_sort_openai_models_prioritizes_best_models_first() { - let mut models = vec![ - ("gpt-4o-mini".to_string(), "gpt-4o-mini".to_string()), - ("gpt-5-mini".to_string(), "gpt-5-mini".to_string()), - ("o3".to_string(), "o3".to_string()), - ("gpt-4.1".to_string(), "gpt-4.1".to_string()), - ("gpt-5".to_string(), "gpt-5".to_string()), - ]; - - sort_openai_models(&mut models); - - let ordered: Vec<String> = models.into_iter().map(|(id, _)| id).collect(); - assert_eq!( - ordered, - vec![ - "gpt-5".to_string(), - "gpt-5-mini".to_string(), - "o3".to_string(), - "gpt-4.1".to_string(), - "gpt-4o-mini".to_string(), - ] - ); - } - - #[tokio::test] - async fn test_fetch_ollama_models_unreachable_fallback() { - // Point at a port nothing listens on - let models = fetch_ollama_models("http://127.0.0.1:1").await; - assert!(!models.is_empty(), "should fall back to static defaults"); - } - #[tokio::test] async fn test_discover_wasm_channels_empty_dir() { let dir = tempdir().unwrap(); // safety: test-only tempdir setup @@ -4536,20 +4506,30 @@ mod tests { } /// Regression: switching from another provider to bedrock must clear - /// selected_model, and choosing "default credentials" must clear - /// bedrock_profile. + /// selected_model, and choosing "default credentials" must clear the + /// bedrock profile setting. #[test] fn test_bedrock_clears_stale_profile_on_default_creds() { let mut wizard = SetupWizard::new(); wizard.settings.llm_backend = Some("bedrock".to_string()); - wizard.settings.bedrock_profile = Some("old-sso-profile".to_string()); + let bedrock = wizard + .settings + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); + bedrock.set_extra("profile", "old-sso-profile"); // Simulate auth_choice == 0 (default credentials) clearing the profile - wizard.settings.bedrock_profile = None; + let bedrock = wizard + .settings + .llm_builtin_overrides + .get_mut("bedrock") + .expect("bedrock entry"); + bedrock.set_extra("profile", ""); assert!( - wizard.settings.bedrock_profile.is_none(), - "bedrock_profile should be cleared when selecting default credentials" + bedrock.extra("profile").is_none(), + "bedrock profile should be cleared when selecting default credentials" ); } @@ -4558,20 +4538,65 @@ mod tests { #[test] fn test_bedrock_empty_profile_clears_existing() { let mut wizard = SetupWizard::new(); - wizard.settings.bedrock_profile = Some("old-profile".to_string()); + let bedrock = wizard + .settings + .llm_builtin_overrides + .entry("bedrock".to_string()) + .or_default(); + bedrock.set_extra("profile", "old-profile"); // Simulate auth_choice == 1 with empty input let profile = "".to_string(); + let bedrock = wizard + .settings + .llm_builtin_overrides + .get_mut("bedrock") + .expect("bedrock entry"); if profile.trim().is_empty() { - wizard.settings.bedrock_profile = None; + bedrock.set_extra("profile", ""); } else { - wizard.settings.bedrock_profile = Some(profile); + bedrock.set_extra("profile", profile); } assert!( - wizard.settings.bedrock_profile.is_none(), - "empty profile input should clear existing bedrock_profile" + bedrock.extra("profile").is_none(), + "empty profile input should clear existing bedrock profile" + ); + } + + /// Regression: legacy `bedrock_*` fields on disk must migrate into + /// `llm_builtin_overrides["bedrock"].extras` on `Settings::load_from`, + /// so existing users don't lose configuration when upgrading past + /// the Layer D refactor. + #[test] + fn legacy_bedrock_fields_migrate_into_extras_on_load() { + let tmpdir = tempfile::tempdir().expect("tempdir"); + let path = tmpdir.path().join("settings.json"); + std::fs::write( + &path, + r#"{ + "llm_backend": "bedrock", + "bedrock_region": "eu-west-1", + "bedrock_cross_region": "eu", + "bedrock_profile": "prod-bedrock" + }"#, + ) + .expect("write fixture"); + + let settings = crate::settings::Settings::load_from(&path); + let bedrock = settings + .llm_builtin_overrides + .get("bedrock") + .expect("bedrock entry must exist after migration"); + assert_eq!(bedrock.extra("region"), Some("eu-west-1")); + assert_eq!(bedrock.extra("cross_region"), Some("eu")); + assert_eq!(bedrock.extra("profile"), Some("prod-bedrock")); + assert!( + settings.bedrock_region.is_none(), + "named bedrock_region must be drained by the migration" ); + assert!(settings.bedrock_cross_region.is_none()); + assert!(settings.bedrock_profile.is_none()); } #[tokio::test] @@ -4581,13 +4606,13 @@ mod tests { // to be kept during re-onboarding. let mut wizard = SetupWizard::new(); - let mut providers: Vec<crate::llm::registry::ProviderDefinition> = + let mut providers: Vec<ironclaw_llm::registry::ProviderDefinition> = serde_json::from_str(include_str!("../../providers.json")).unwrap(); // Add a provider with no setup hint - providers.push(crate::llm::registry::ProviderDefinition { + providers.push(ironclaw_llm::registry::ProviderDefinition { id: "custom_no_setup".to_string(), aliases: vec![], - protocol: crate::llm::registry::ProviderProtocol::OpenAiCompletions, + protocol: ironclaw_llm::registry::ProviderProtocol::OpenAiCompletions, default_base_url: Some("http://localhost:9999/v1".to_string()), base_url_env: None, base_url_required: false, @@ -4600,7 +4625,7 @@ mod tests { setup: None, unsupported_params: vec![], }); - let registry = crate::llm::ProviderRegistry::new(providers); + let registry = ironclaw_llm::ProviderRegistry::new(providers); let result = wizard .run_provider_setup("custom_no_setup", &registry) diff --git a/src/skills/attenuation.rs b/src/skills/attenuation.rs index 8e6fab6392e..8efe59d8296 100644 --- a/src/skills/attenuation.rs +++ b/src/skills/attenuation.rs @@ -17,7 +17,7 @@ //! | Trusted only | All tools (user placed these, full trust) | //! | Installed present | Read-only tools ONLY | -use crate::llm::ToolDefinition; +use ironclaw_llm::ToolDefinition; use ironclaw_skills::{LoadedSkill, SkillTrust}; /// Tools that are always safe -- read-only, no side effects. diff --git a/src/skills/mod.rs b/src/skills/mod.rs index f34b3ee10e6..fa75f86558f 100644 --- a/src/skills/mod.rs +++ b/src/skills/mod.rs @@ -1,11 +1,13 @@ //! Skills system for IronClaw. //! -//! This module contains main-crate skill logic that depends on types from -//! other `src/` modules (e.g. `crate::llm::ToolDefinition`, `crate::secrets`). -//! For core skill types, parsing, and registry, import from `ironclaw_skills` directly. +//! This module contains main-crate skill logic that depends on types from the +//! extracted `ironclaw_llm` crate (e.g. `ironclaw_llm::ToolDefinition`) and +//! other `src/` modules (e.g. `crate::secrets`). For core skill types, +//! parsing, and registry, import from `ironclaw_skills` directly. //! -//! The `attenuation` submodule remains here because it depends on -//! `crate::llm::ToolDefinition` which is a main-crate type. +//! The `attenuation` submodule lives here because it operates on +//! `ironclaw_llm::ToolDefinition` together with main-crate trust state, so it +//! sits at the seam between the two. //! //! # V1 migration notes //! diff --git a/src/testing/mod.rs b/src/testing/mod.rs index 0c14be46af0..ba3b2867e91 100644 --- a/src/testing/mod.rs +++ b/src/testing/mod.rs @@ -19,12 +19,16 @@ //! ``` pub mod credentials; -pub mod fault_injection; + +// `StubLlm`, `StubErrorKind`, and `fault_injection` live in `ironclaw_llm` +// (the natural home for the trait they implement). Re-exported under the +// existing `crate::testing::*` paths so existing test imports keep working. +pub use ironclaw_llm::testing::{StubErrorKind, StubLlm, fault_injection}; use std::sync::Arc; use std::sync::Mutex; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use rust_decimal::Decimal; @@ -36,11 +40,8 @@ use crate::channels::{ }; use crate::db::Database; use crate::error::{ChannelError, LlmError}; -use crate::llm::{ - CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, - ToolCompletionResponse, -}; use crate::tools::ToolRegistry; +use ironclaw_llm::{CompletionRequest, FinishReason, LlmProvider}; /// Create a libSQL-backed test database in a temporary directory. /// @@ -62,181 +63,6 @@ pub async fn test_db() -> (Arc<dyn Database>, tempfile::TempDir) { (Arc::new(backend) as Arc<dyn Database>, dir) } -/// What kind of error the stub should produce when failing. -#[derive(Clone, Copy, Debug)] -pub enum StubErrorKind { - /// Transient/retryable error (`LlmError::RequestFailed`). - Transient, - /// Non-transient error (`LlmError::ContextLengthExceeded`). - NonTransient, -} - -/// A configurable LLM provider stub for tests. -/// -/// Supports: -/// - Fixed response content -/// - Call counting via [`calls()`](Self::calls) -/// - Runtime failure toggling via [`set_failing()`](Self::set_failing) -/// - Configurable error kinds (transient vs non-transient) -/// -/// Use this in tests instead of creating ad-hoc stub implementations. -pub struct StubLlm { - model_name: String, - response: String, - call_count: AtomicU32, - should_fail: AtomicBool, - error_kind: StubErrorKind, - /// Optional fault injector for fine-grained failure control. - /// When set, takes precedence over the `should_fail` / `error_kind` fields. - fault_injector: Option<Arc<fault_injection::FaultInjector>>, -} - -impl StubLlm { - /// Create a new stub that returns the given response. - pub fn new(response: impl Into<String>) -> Self { - Self { - model_name: "stub-model".to_string(), - response: response.into(), - call_count: AtomicU32::new(0), - should_fail: AtomicBool::new(false), - error_kind: StubErrorKind::Transient, - fault_injector: None, - } - } - - /// Create a stub that always fails with a transient error. - pub fn failing(name: impl Into<String>) -> Self { - Self { - model_name: name.into(), - response: String::new(), - call_count: AtomicU32::new(0), - should_fail: AtomicBool::new(true), - error_kind: StubErrorKind::Transient, - fault_injector: None, - } - } - - /// Create a stub that always fails with a non-transient error. - pub fn failing_non_transient(name: impl Into<String>) -> Self { - Self { - model_name: name.into(), - response: String::new(), - call_count: AtomicU32::new(0), - should_fail: AtomicBool::new(true), - error_kind: StubErrorKind::NonTransient, - fault_injector: None, - } - } - - /// Set the model name. - pub fn with_model_name(mut self, name: impl Into<String>) -> Self { - self.model_name = name.into(); - self - } - - /// Get the number of times `complete` or `complete_with_tools` was called. - pub fn calls(&self) -> u32 { - self.call_count.load(Ordering::Relaxed) - } - - /// Attach a fault injector for fine-grained failure control. - /// - /// When set, the injector's `next_action()` is consulted on every call, - /// taking precedence over the `should_fail` / `error_kind` fields. - pub fn with_fault_injector(mut self, injector: Arc<fault_injection::FaultInjector>) -> Self { - self.fault_injector = Some(injector); - self - } - - /// Toggle whether calls should fail at runtime. - pub fn set_failing(&self, fail: bool) { - self.should_fail.store(fail, Ordering::Relaxed); - } - - /// Check the fault injector or should_fail flag, returning an error if - /// the call should fail, or None if it should succeed. - async fn check_faults(&self) -> Option<LlmError> { - if let Some(ref injector) = self.fault_injector { - match injector.next_action() { - fault_injection::FaultAction::Fail(fault) => { - return Some(fault.to_llm_error(&self.model_name)); - } - fault_injection::FaultAction::Delay(duration) => { - tokio::time::sleep(duration).await; - } - fault_injection::FaultAction::Succeed => {} - } - } else if self.should_fail.load(Ordering::Relaxed) { - return Some(self.make_error()); - } - None - } - - fn make_error(&self) -> LlmError { - match self.error_kind { - StubErrorKind::Transient => LlmError::RequestFailed { - provider: self.model_name.clone(), - reason: "server error".to_string(), - }, - StubErrorKind::NonTransient => LlmError::ContextLengthExceeded { - used: 100_000, - limit: 50_000, - }, - } - } -} - -impl Default for StubLlm { - fn default() -> Self { - Self::new("OK") - } -} - -#[async_trait] -impl LlmProvider for StubLlm { - fn model_name(&self) -> &str { - &self.model_name - } - - fn cost_per_token(&self) -> (Decimal, Decimal) { - (Decimal::ZERO, Decimal::ZERO) - } - - async fn complete(&self, _request: CompletionRequest) -> Result<CompletionResponse, LlmError> { - self.call_count.fetch_add(1, Ordering::Relaxed); - if let Some(err) = self.check_faults().await { - return Err(err); - } - Ok(CompletionResponse { - content: self.response.clone(), - input_tokens: 10, - output_tokens: 5, - finish_reason: FinishReason::Stop, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }) - } - - async fn complete_with_tools( - &self, - _request: ToolCompletionRequest, - ) -> Result<ToolCompletionResponse, LlmError> { - self.call_count.fetch_add(1, Ordering::Relaxed); - if let Some(err) = self.check_faults().await { - return Err(err); - } - Ok(ToolCompletionResponse { - content: Some(self.response.clone()), - tool_calls: Vec::new(), - input_tokens: 10, - output_tokens: 5, - finish_reason: FinishReason::Stop, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }) - } -} - /// A configurable channel stub for tests. /// /// Supports: @@ -1671,8 +1497,8 @@ mod tests { #[tokio::test] async fn stub_llm_fault_injector_sequence() { - use crate::llm::LlmProvider; use crate::testing::fault_injection::{FaultAction, FaultInjector, FaultType}; + use ironclaw_llm::LlmProvider; let injector = Arc::new(FaultInjector::sequence([ FaultAction::Fail(FaultType::RateLimited { retry_after: None }), @@ -1681,7 +1507,7 @@ mod tests { let stub = StubLlm::new("hello").with_fault_injector(injector); - let req = crate::llm::CompletionRequest::new(vec![crate::llm::ChatMessage::user("hi")]); + let req = ironclaw_llm::CompletionRequest::new(vec![ironclaw_llm::ChatMessage::user("hi")]); // First call should fail with RateLimited let result = stub.complete(req.clone()).await; diff --git a/src/tools/autonomy.rs b/src/tools/autonomy.rs index ab3e502942a..ea754f303aa 100644 --- a/src/tools/autonomy.rs +++ b/src/tools/autonomy.rs @@ -16,7 +16,6 @@ pub const AUTONOMOUS_TOOL_DENYLIST: &[&str] = &[ "restart", "tool_install", "tool_auth", - "tool_activate", "tool_remove", "tool_upgrade", "skill_install", diff --git a/src/tools/builder/core.rs b/src/tools/builder/core.rs index 0d729ec2dcd..44c9379a6e2 100644 --- a/src/tools/builder/core.rs +++ b/src/tools/builder/core.rs @@ -40,14 +40,14 @@ use uuid::Uuid; use crate::context::JobContext; use crate::error::ToolError as AgentToolError; -use crate::llm::{ - ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolDefinition, -}; use crate::tools::tool::{ ApprovalContext, ApprovalRequirement, EngineCompatibility, Tool, ToolError, ToolOutput, check_approval_in_context, }; use crate::tools::{ToolRegistry, prepare_tool_params}; +use ironclaw_llm::{ + ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolDefinition, +}; /// Deserialize `dependencies` from either a list of strings, a list of objects, /// or a flat object map. LLMs often produce TOML-style inline tables @@ -770,6 +770,7 @@ Create alongside the .wasm file to grant capabilities: RespondResult::ToolCalls { tool_calls, content, + reasoning: _, } => { tools_executed = true; diff --git a/src/tools/builtin/extension_tools.rs b/src/tools/builtin/extension_tools.rs index d9ebc32578b..aff9f327b8c 100644 --- a/src/tools/builtin/extension_tools.rs +++ b/src/tools/builtin/extension_tools.rs @@ -1,12 +1,12 @@ //! Agent-callable tools for managing extensions (MCP servers and WASM tools). //! //! These built-ins manage extension discovery and lifecycle from conversation. -//! In engine v2, the normal model-facing enablement path is -//! `tool_activate(name=...)`: blocked integrations surface in capability -//! background, and `tool_activate` internally handles install/auth/activation -//! as needed. `tool_search`, `tool_list`, and `tool_info` support discovery; -//! `tool_install` / `tool_auth` remain available as narrower runtime/compat -//! surfaces rather than the primary v2 prompt contract. +//! In engine v2, installed-but-unauthed tools are callable directly: the +//! engine's auth preflight raises an `Authentication` gate at execute time, +//! the inline-await machinery parks the VM, and the OAuth callback delivers +//! the resolved credential to retry the action. `tool_search`, `tool_list`, +//! and `tool_info` support discovery; `tool_install` / `tool_auth` cover the +//! narrower install + manual-auth flows. use std::sync::Arc; @@ -113,11 +113,11 @@ impl Tool for ToolSearchTool { fn description(&self) -> &str { "Search for available extensions to add new capabilities. Extensions include \ channels (Telegram, Slack, Discord — connect messaging platforms so IronClaw can \ - receive and reply there), tools, and MCP servers. Use `tool_install` for explicit \ - installation, then call `tool_activate(name=\"...\")` to make a discovered \ - integration usable. Use the `message` tool for \ - proactive outbound sends. Use discover:true to search online if the built-in registry \ - has no results." + receive and reply there), tools, and MCP servers. Use `tool_install` to install \ + a discovered integration; once installed, its tools become directly callable \ + (the engine's auth preflight raises an authentication gate at execute time when \ + credentials are missing). Use the `message` tool for proactive outbound sends. \ + Use discover:true to search online if the built-in registry has no results." } fn parameters_schema(&self) -> serde_json::Value { @@ -329,64 +329,6 @@ impl Tool for ToolAuthTool { } } -// ── tool_activate ──────────────────────────────────────────────────────── - -pub struct ToolActivateTool { - manager: Arc<ExtensionManager>, -} - -impl ToolActivateTool { - pub fn new(manager: Arc<ExtensionManager>) -> Self { - Self { manager } - } -} - -#[async_trait] -impl Tool for ToolActivateTool { - fn name(&self) -> &str { - "tool_activate" - } - - fn description(&self) -> &str { - "Make an integration usable. This may install, authenticate, and activate it as needed \ - before loading tools, starting channels, or connecting to MCP servers." - } - - fn parameters_schema(&self) -> serde_json::Value { - serde_json::json!({ - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Extension name to activate" - } - }, - "required": ["name"] - }) - } - - async fn execute( - &self, - params: serde_json::Value, - ctx: &JobContext, - ) -> Result<ToolOutput, ToolError> { - let start = std::time::Instant::now(); - - let name = require_str(&params, "name")?; - - let result = self - .manager - .ensure_extension_ready(name, &ctx.user_id, EnsureReadyIntent::ExplicitActivate) - .await - .map_err(|e| ToolError::ExecutionFailed(e.to_string()))?; - - Ok(ToolOutput::success( - output_from_ensure_ready(result), - start.elapsed(), - )) - } -} - // ── tool_list ──────────────────────────────────────────────────────────── pub struct ToolListTool { @@ -905,7 +847,7 @@ mod tests { }; let description = tool.description(); - assert!(description.contains("call `tool_activate(name=\"...\")`")); + assert!(description.contains("Use `tool_install` to install")); assert!(description.contains("Use the `message` tool for proactive outbound sends")); } @@ -945,23 +887,6 @@ mod tests { ); } - #[test] - fn test_tool_activate_schema() { - use crate::tools::tool::ApprovalRequirement; - let tool = ToolActivateTool { - manager: test_manager_stub(), - }; - assert_eq!(tool.name(), "tool_activate"); - assert!( - tool.description() - .contains("install, authenticate, and activate") - ); - assert_eq!( - tool.requires_approval(&serde_json::json!({})), - ApprovalRequirement::Never - ); - } - #[test] fn activation_error_requires_auth_detects_auth_required_variants() { assert!(activation_error_requires_auth("Authentication required")); diff --git a/src/tools/builtin/http.rs b/src/tools/builtin/http.rs index 910c8a83d4a..a7cfb413ccf 100644 --- a/src/tools/builtin/http.rs +++ b/src/tools/builtin/http.rs @@ -759,13 +759,13 @@ impl Tool for HttpTool { // API keys, and injected query-param/URL-path credentials never // reach the recorder. Replay matching uses `method` + `url` only, // so omitting the injected values is safe for determinism. - let intercept_req = crate::llm::recording::HttpExchangeRequest { + let intercept_req = ironclaw_llm::recording::HttpExchangeRequest { method: method_upper, url: caller_url.to_string(), headers: caller_headers, body: body_bytes .as_ref() - .map(|b| crate::llm::recording::redact_body(&String::from_utf8_lossy(b))), + .map(|b| ironclaw_llm::recording::redact_body(&String::from_utf8_lossy(b))), }; // Check HTTP interceptor (replay mode returns pre-recorded response) @@ -1059,7 +1059,7 @@ impl Tool for HttpTool { interceptor .after_response( &intercept_req, - &crate::llm::recording::HttpExchangeResponse { + &ironclaw_llm::recording::HttpExchangeResponse { status, headers: resp_headers, body: body_text.clone(), @@ -1880,7 +1880,7 @@ mod tests { /// no real HTTP call is made. #[derive(Debug)] struct SpyInterceptor { - captured: tokio::sync::Mutex<Option<crate::llm::recording::HttpExchangeRequest>>, + captured: tokio::sync::Mutex<Option<ironclaw_llm::recording::HttpExchangeRequest>>, } impl SpyInterceptor { @@ -1890,19 +1890,19 @@ mod tests { } } - async fn captured_request(&self) -> Option<crate::llm::recording::HttpExchangeRequest> { + async fn captured_request(&self) -> Option<ironclaw_llm::recording::HttpExchangeRequest> { self.captured.lock().await.clone() } } #[async_trait::async_trait] - impl crate::llm::recording::HttpInterceptor for SpyInterceptor { + impl ironclaw_llm::recording::HttpInterceptor for SpyInterceptor { async fn before_request( &self, - request: &crate::llm::recording::HttpExchangeRequest, - ) -> Option<crate::llm::recording::HttpExchangeResponse> { + request: &ironclaw_llm::recording::HttpExchangeRequest, + ) -> Option<ironclaw_llm::recording::HttpExchangeResponse> { *self.captured.lock().await = Some(request.clone()); - Some(crate::llm::recording::HttpExchangeResponse { + Some(ironclaw_llm::recording::HttpExchangeResponse { status: 200, headers: vec![], body: r#"{"ok":true}"#.to_string(), @@ -1911,8 +1911,8 @@ mod tests { async fn after_response( &self, - _request: &crate::llm::recording::HttpExchangeRequest, - _response: &crate::llm::recording::HttpExchangeResponse, + _request: &ironclaw_llm::recording::HttpExchangeRequest, + _response: &ironclaw_llm::recording::HttpExchangeResponse, ) { } } @@ -1962,7 +1962,8 @@ mod tests { let spy = Arc::new(SpyInterceptor::new()); let mut ctx = crate::context::JobContext::new("test", "test"); - ctx.http_interceptor = Some(spy.clone() as Arc<dyn crate::llm::recording::HttpInterceptor>); + ctx.http_interceptor = + Some(spy.clone() as Arc<dyn ironclaw_llm::recording::HttpInterceptor>); // `api.github.com` chosen because DNS resolution runs before the // interceptor short-circuits (see `validate_and_resolve_url` in @@ -2037,7 +2038,8 @@ mod tests { let spy = Arc::new(SpyInterceptor::new()); let mut ctx = crate::context::JobContext::new("test", "test"); - ctx.http_interceptor = Some(spy.clone() as Arc<dyn crate::llm::recording::HttpInterceptor>); + ctx.http_interceptor = + Some(spy.clone() as Arc<dyn ironclaw_llm::recording::HttpInterceptor>); let params = serde_json::json!({ "method": "GET", diff --git a/src/tools/builtin/image_gen.rs b/src/tools/builtin/image_gen.rs index 29a10eeac96..dd9f085ffe6 100644 --- a/src/tools/builtin/image_gen.rs +++ b/src/tools/builtin/image_gen.rs @@ -195,7 +195,7 @@ impl Tool for ImageGenerateTool { let media_type = infer_generated_image_media_type(image_data); - // Return sentinel JSON for image display + // Return sentinel JSON for image display. let sentinel = serde_json::json!({ "type": "image_generated", "data": format!("data:{media_type};base64,{}", image_data), diff --git a/src/tools/builtin/memory.rs b/src/tools/builtin/memory.rs index f23ced44f80..0f7df3ce3d6 100644 --- a/src/tools/builtin/memory.rs +++ b/src/tools/builtin/memory.rs @@ -192,7 +192,7 @@ const REASONING_LLM_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// prior work, decisions, preferences, or any historical context. pub struct MemorySearchTool { resolver: Arc<dyn WorkspaceResolver>, - llm: Option<Arc<dyn crate::llm::LlmProvider>>, + llm: Option<Arc<dyn ironclaw_llm::LlmProvider>>, reasoning_enabled: bool, /// Per-user rate limiter for reasoning LLM calls. reasoning_limiter: Arc<crate::tools::rate_limiter::RateLimiter>, @@ -212,7 +212,7 @@ impl MemorySearchTool { /// Create a memory search tool with optional reasoning-augmented recall. pub fn with_reasoning( resolver: Arc<dyn WorkspaceResolver>, - llm: Option<Arc<dyn crate::llm::LlmProvider>>, + llm: Option<Arc<dyn ironclaw_llm::LlmProvider>>, reasoning_enabled: bool, ) -> Self { Self { @@ -334,15 +334,16 @@ impl Tool for MemorySearchTool { .join("\n\n"); let llm_messages = vec![ - crate::llm::ChatMessage::system(include_str!( + ironclaw_llm::ChatMessage::system(include_str!( "../../../crates/ironclaw_engine/prompts/memory_reasoning_synthesis.md" )), - crate::llm::ChatMessage::user(format!( + ironclaw_llm::ChatMessage::user(format!( "Query: {query}\n\nMemory fragments:\n{fragments}" )), ]; - let request = crate::llm::CompletionRequest::new(llm_messages).with_max_tokens(500); + let request = + ironclaw_llm::CompletionRequest::new(llm_messages).with_max_tokens(500); match tokio::time::timeout(REASONING_LLM_TIMEOUT, llm.complete(request)).await { Ok(Ok(response)) => { @@ -1682,7 +1683,7 @@ mod tests { let pool = crate::channels::web::platform::state::WorkspacePool::new( db, None, - crate::workspace::EmbeddingCacheConfig::default(), + ironclaw_embeddings::EmbeddingCacheConfig::default(), crate::config::WorkspaceSearchConfig::default(), crate::config::WorkspaceConfig::default(), ); @@ -1703,7 +1704,7 @@ mod tests { let pool = crate::channels::web::platform::state::WorkspacePool::new( db, None, - crate::workspace::EmbeddingCacheConfig::default(), + ironclaw_embeddings::EmbeddingCacheConfig::default(), crate::config::WorkspaceSearchConfig::default(), crate::config::WorkspaceConfig::default(), ); @@ -1723,7 +1724,7 @@ mod tests { #[cfg(feature = "libsql")] mod reasoning_recall_tests { use super::*; - use crate::llm::{ + use ironclaw_llm::{ CompletionRequest, CompletionResponse, FinishReason, LlmError, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, }; diff --git a/src/tools/builtin/mod.rs b/src/tools/builtin/mod.rs index 992f2a2e966..666e5094817 100644 --- a/src/tools/builtin/mod.rs +++ b/src/tools/builtin/mod.rs @@ -25,8 +25,8 @@ mod tool_info; pub use echo::EchoTool; pub use extension_tools::{ - ExtensionInfoTool, ToolActivateTool, ToolAuthTool, ToolInstallTool, ToolListTool, - ToolPermissionSetTool, ToolRemoveTool, ToolSearchTool, ToolUpgradeTool, + ExtensionInfoTool, ToolAuthTool, ToolInstallTool, ToolListTool, ToolPermissionSetTool, + ToolRemoveTool, ToolSearchTool, ToolUpgradeTool, }; pub use file::{ApplyPatchTool, ListDirTool, ReadFileTool, WriteFileTool}; pub use file_edit_guard::{SharedReadFileState, shared_read_file_state}; @@ -57,11 +57,13 @@ mod html_converter; pub mod image_analyze; pub mod image_edit; pub mod image_gen; +mod pairing; pub use html_converter::convert_html_to_markdown; pub use image_analyze::ImageAnalyzeTool; pub use image_edit::ImageEditTool; pub use image_gen::ImageGenerateTool; +pub use pairing::PairingApproveTool; /// Detect image media type from file extension via `mime_guess`. /// Falls back to `image/jpeg` for unrecognized or non-image extensions. diff --git a/src/tools/builtin/pairing.rs b/src/tools/builtin/pairing.rs new file mode 100644 index 00000000000..f0121ae34c2 --- /dev/null +++ b/src/tools/builtin/pairing.rs @@ -0,0 +1,118 @@ +use async_trait::async_trait; +use std::sync::Arc; + +use crate::context::JobContext; +use crate::pairing::PairingStore; +use crate::tools::tool::{ApprovalRequirement, Tool, ToolError, ToolOutput, require_str}; + +const CHANNEL: &str = "slack-relay"; + +pub struct PairingApproveTool { + store: Arc<PairingStore>, +} + +impl PairingApproveTool { + pub fn new(store: Arc<PairingStore>) -> Self { + Self { store } + } +} + +#[async_trait] +impl Tool for PairingApproveTool { + fn name(&self) -> &str { + "pairing_approve" + } + + fn description(&self) -> &str { + "Approve a Slack pairing code to bind the user's Slack account to their IronClaw user. The user receives the code in Slack and provides it here." + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The pairing code received in Slack (e.g. WZG8LQAB)" + } + }, + "required": ["code"] + }) + } + + async fn execute( + &self, + params: serde_json::Value, + ctx: &JobContext, + ) -> Result<ToolOutput, ToolError> { + let start = std::time::Instant::now(); + + let code = require_str(&params, "code")?; + let channel = CHANNEL; + + let user_id = + crate::ownership::UserId::new(&ctx.user_id, crate::ownership::UserRole::Regular) + .map_err(|e| ToolError::ExecutionFailed(format!("invalid user_id: {e}")))?; + + match self.store.approve(channel, code, &user_id).await { + Ok(approval) => { + let msg = format!( + "Pairing approved! Your {} account (external ID: {}) is now linked to your IronClaw user.", + approval.channel, approval.external_id + ); + Ok(ToolOutput::text(&msg, start.elapsed())) + } + Err(e) => { + let msg = format!( + "Pairing failed: {e}. Make sure the code is correct and hasn't expired." + ); + Ok(ToolOutput::text(&msg, start.elapsed())) + } + } + } + + fn requires_sanitization(&self) -> bool { + false + } + + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + ApprovalRequirement::Always + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::registry::is_protected_tool_name; + + #[test] + fn tool_metadata() { + let store = Arc::new(PairingStore::new_noop()); + let tool = PairingApproveTool::new(store); + assert_eq!(tool.name(), "pairing_approve"); + assert!(tool.description().contains("Slack")); + let schema = tool.parameters_schema(); + assert!(schema["properties"]["code"].is_object()); + assert!(schema["properties"]["channel"].is_null()); + } + + #[test] + fn always_requires_approval() { + let store = Arc::new(PairingStore::new_noop()); + let tool = PairingApproveTool::new(store); + assert_eq!( + tool.requires_approval(&serde_json::json!({})), + ApprovalRequirement::Always + ); + } + + #[test] + fn is_protected_builtin() { + assert!(is_protected_tool_name("pairing_approve")); + } + + #[test] + fn channel_is_slack_relay() { + assert_eq!(CHANNEL, "slack-relay"); + } +} diff --git a/src/tools/builtin/skill_tools.rs b/src/tools/builtin/skill_tools.rs index b3e875e35cb..b6bdf1598a4 100644 --- a/src/tools/builtin/skill_tools.rs +++ b/src/tools/builtin/skill_tools.rs @@ -32,6 +32,7 @@ const MAX_TOTAL_UNZIPPED_BYTES: u64 = 20 * 1024 * 1024; /// case a future refactor (parallel fetching, retries) changes that /// invariant. const MAX_CHAIN_QUEUE: usize = MAX_CHAIN_DEPS * 10; +const INSTALL_METADATA_FILE_NAME: &str = ".ironclaw-install.json"; #[derive(Debug, Clone, Error)] #[error("{message}")] @@ -464,6 +465,76 @@ fn append_chain_install_report_fields(output: &mut serde_json::Value, report: &C } } +fn normalize_install_source_url(url: &str) -> String { + url.trim().trim_end_matches('/').to_string() +} + +fn loaded_skill_name_for_source_url( + registry: &SkillRegistry, + requested_url: &str, +) -> Option<String> { + let requested_url = normalize_install_source_url(requested_url); + if requested_url.is_empty() { + return None; + } + + registry + .skills() + .iter() + .find_map(|skill| { + let skill_dir = match &skill.source { + ironclaw_skills::SkillSource::Installed(path) => path, + _ => return None, + }; + installed_source_url_matches(skill_dir, &requested_url) + .then(|| skill.name().to_string()) + }) + .or_else(|| installed_skill_name_for_source_url_on_disk(registry, &requested_url)) +} + +fn installed_source_url_matches(skill_dir: &Path, requested_url: &str) -> bool { + let metadata_path = skill_dir.join(INSTALL_METADATA_FILE_NAME); + let Some(metadata) = std::fs::read(metadata_path).ok().and_then(|bytes| { + serde_json::from_slice::<ironclaw_skills::registry::InstalledSkillMetadata>(&bytes).ok() + }) else { + return false; + }; + metadata + .source_url + .as_deref() + .is_some_and(|source_url| normalize_install_source_url(source_url) == requested_url) +} + +fn installed_skill_name_for_source_url_on_disk( + registry: &SkillRegistry, + requested_url: &str, +) -> Option<String> { + let entries = std::fs::read_dir(registry.install_target_dir()).ok()?; + for entry in entries.flatten() { + if !entry.file_type().ok()?.is_dir() { + continue; + } + let path = entry.path(); + if installed_source_url_matches(&path, requested_url) + && let Some(name) = entry.file_name().to_str().filter(|name| !name.is_empty()) + { + return Some(name.to_string()); + } + } + None +} + +fn installed_skill_dir_by_name(registry: &SkillRegistry, name: &str) -> Option<PathBuf> { + let mut components = Path::new(name).components(); + match (components.next(), components.next()) { + (Some(Component::Normal(_)), None) => { + let dir = registry.install_target_dir().join(name); + dir.join("SKILL.md").is_file().then_some(dir) + } + _ => None, + } +} + // ── skill_list ────────────────────────────────────────────────────────── pub struct SkillListTool { @@ -795,16 +866,38 @@ impl Tool for SkillInstallTool { .filter(|s| !s.is_empty()) .map(str::to_string); - // Idempotent: if a skill with this name is already loaded (from any + // Idempotent: if the requested skill is already loaded (from any // source — local SKILL.md, bundled, previously installed), avoid - // reinstalling the top-level skill. Dependency installs are not a - // no-op: when explicitly requested, walk the loaded skill's companion - // list instead of returning early. + // reinstalling the top-level skill. URL installs can have a suggested + // `name` derived from the repository slug instead of the final parsed + // skill name, so also match against persisted install metadata. + // Dependency installs are not a no-op: when explicitly requested, + // walk the loaded skill's companion list instead of returning early. let loaded_required_skills = { let guard = self .registry .read() .map_err(|e| ToolError::ExecutionFailed(format!("Lock poisoned: {}", e)))?; + + if !install_dependencies { + if let Some(url) = params.get("url").and_then(|v| v.as_str()) + && let Some(installed_name) = loaded_skill_name_for_source_url(&guard, url) + { + let report = ChainInstallReport::default(); + return Ok(ToolOutput::success( + build_already_installed_output(&installed_name, &report), + start.elapsed(), + )); + } + if installed_skill_dir_by_name(&guard, name).is_some() { + let report = ChainInstallReport::default(); + return Ok(ToolOutput::success( + build_already_installed_output(name, &report), + start.elapsed(), + )); + } + } + if let Some(loaded_skill) = guard.find_by_name(name) { let required_skills = loaded_skill.manifest.requires.skills.clone(); if install_dependencies && !required_skills.is_empty() { @@ -883,11 +976,13 @@ impl Tool for SkillInstallTool { ) .map_err(|e| ToolError::ExecutionFailed(e.to_string()))?; - if guard.has(&skill_name) { - return Err(ToolError::ExecutionFailed(format!( - "Skill '{}' already exists", - skill_name - ))); + if guard.has(&skill_name) || installed_skill_dir_by_name(&guard, &skill_name).is_some() + { + let report = ChainInstallReport::default(); + return Ok(ToolOutput::success( + build_already_installed_output(&skill_name, &report), + start.elapsed(), + )); } ( @@ -1001,19 +1096,29 @@ impl Tool for SkillInstallTool { .and_then(|v| v.as_bool()) .unwrap_or(false); - // No-op shortcut: if a skill with this name is already loaded (bundled, - // user, workspace, or previously installed), `execute` will return + // No-op shortcut: if a skill is already loaded (bundled, user, + // workspace, or previously installed), `execute` will return // `already_installed` without touching the catalog. Asking for approval // on a guaranteed no-op is pure friction, so we mirror the idempotent - // path here. Dependency-chain installs may still fetch companion - // skills, so they must go through the approval path below. - if !install_deps - && let Some(name) = params.get("name").and_then(|v| v.as_str()) - && !name.is_empty() - && let Ok(guard) = self.registry.read() - && guard.has(name) - { - return ApprovalRequirement::Never; + // path here. URL installs may use a repository-derived `name`, so also + // match the request URL against persisted install metadata. + // Dependency-chain installs may still fetch companion skills, so they + // must go through the approval path below. + if !install_deps && let Ok(guard) = self.registry.read() { + let name_is_loaded = params + .get("name") + .and_then(|v| v.as_str()) + .filter(|name| !name.is_empty()) + .is_some_and(|name| { + guard.has(name) || installed_skill_dir_by_name(&guard, name).is_some() + }); + let source_url_is_loaded = params + .get("url") + .and_then(|v| v.as_str()) + .is_some_and(|url| loaded_skill_name_for_source_url(&guard, url).is_some()); + if name_is_loaded || source_url_is_loaded { + return ApprovalRequirement::Never; + } } // Chain installs pull up to MAX_CHAIN_DEPS additional skills, each @@ -2036,6 +2141,90 @@ mod tests { ); } + #[tokio::test] + async fn skill_install_url_duplicate_is_idempotent_without_approval() { + use crate::tools::tool::ApprovalRequirement; + + let registry = test_registry(); + let source_url = "https://github.com/Pika-Labs/Pika-Skills"; + let (name, loaded) = { + let dir = registry.read().unwrap().install_target_dir().to_path_buf(); + SkillRegistry::prepare_install_bundle_to_disk( + &dir, + "pikastream-video-meeting", + &skill_content("pikastream-video-meeting", &[]), + &[], + Some(&ironclaw_skills::registry::InstalledSkillMetadata { + source_url: Some(source_url.to_string()), + source_subdir: None, + }), + ) + .await + .expect("prepare should succeed") + }; + registry + .write() + .unwrap() + .commit_install(&name, loaded) + .expect("commit should succeed"); + let tool = SkillInstallTool::new(Arc::clone(&registry), test_catalog()); + + let params = serde_json::json!({ + "name": "pika-skills", + "url": source_url, + }); + assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never); + + let output = tool + .execute(params, &JobContext::default()) + .await + .expect("duplicate URL install should be a no-op"); + assert_eq!(output.result["status"], "already_installed"); + assert_eq!(output.result["name"], "pikastream-video-meeting"); + assert!( + output.result["message"] + .as_str() + .unwrap() + .contains("no install needed") + ); + } + + #[tokio::test] + async fn skill_install_disk_duplicate_is_idempotent_without_loaded_registry() { + use crate::tools::tool::ApprovalRequirement; + + let registry = test_registry(); + let source_url = "https://github.com/Pika-Labs/Pika-Skills"; + { + let dir = registry.read().unwrap().install_target_dir().to_path_buf(); + SkillRegistry::prepare_install_bundle_to_disk( + &dir, + "pikastream-video-meeting", + &skill_content("pikastream-video-meeting", &[]), + &[], + Some(&ironclaw_skills::registry::InstalledSkillMetadata { + source_url: Some(source_url.to_string()), + source_subdir: None, + }), + ) + .await + .expect("prepare should succeed"); + } + let tool = SkillInstallTool::new(Arc::clone(&registry), test_catalog()); + let params = serde_json::json!({ + "name": "pikastream-video-meeting", + "url": source_url, + }); + + assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never); + let output = tool + .execute(params, &JobContext::default()) + .await + .expect("disk duplicate install should be a no-op"); + assert_eq!(output.result["status"], "already_installed"); + assert_eq!(output.result["name"], "pikastream-video-meeting"); + } + #[tokio::test] async fn skill_install_execute_honors_dependencies_when_already_loaded() { let registry = test_registry(); diff --git a/src/tools/execute.rs b/src/tools/execute.rs index d6003e98d1b..a540bfd29bc 100644 --- a/src/tools/execute.rs +++ b/src/tools/execute.rs @@ -8,8 +8,8 @@ use std::borrow::Cow; use crate::context::JobContext; use crate::error::Error; -use crate::llm::ChatMessage; use crate::tools::{ToolRegistry, prepare_tool_params, redact_params}; +use ironclaw_llm::ChatMessage; use ironclaw_safety::SafetyLayer; /// Execute a tool with safety checks: lookup → validate → timeout → execute → serialize. @@ -451,7 +451,7 @@ mod tests { "Content should contain the output: {}", content ); - assert_eq!(message.role, crate::llm::Role::Tool); + assert_eq!(message.role, ironclaw_llm::Role::Tool); assert_eq!(message.name.as_deref(), Some("echo")); } @@ -477,7 +477,7 @@ mod tests { "Error content should contain the message: {}", content ); - assert_eq!(message.role, crate::llm::Role::Tool); + assert_eq!(message.role, ironclaw_llm::Role::Tool); assert_eq!(message.name.as_deref(), Some("echo")); } diff --git a/src/tools/mod.rs b/src/tools/mod.rs index d8027512099..a50c6c0e0a1 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -33,7 +33,7 @@ pub use builder::{ LlmSoftwareBuilder, SoftwareBuilder, SoftwareType, Template, TemplateEngine, TemplateType, TestCase, TestHarness, TestResult, TestSuite, ValidationError, ValidationResult, WasmValidator, }; -pub(crate) use coercion::prepare_tool_params; +pub(crate) use coercion::{prepare_params_for_schema, prepare_tool_params}; pub use rate_limiter::RateLimiter; pub use registry::{ToolRegistry, is_protected_tool_name}; pub use tool::{ diff --git a/src/tools/permissions.rs b/src/tools/permissions.rs index 8911cd12c23..bf2fea3cc53 100644 --- a/src/tools/permissions.rs +++ b/src/tools/permissions.rs @@ -42,9 +42,9 @@ pub fn seeded_default_permission(tool_name: &str) -> Option<PermissionState> { fn seeded_default_permission_canonical(canonical_tool_name: &str) -> Option<PermissionState> { match canonical_tool_name { "echo" | "time" | "json" | "memory_search" | "memory_read" | "memory_write" - | "memory_tree" | "tool_list" | "tool_info" | "tool_search" | "tool_activate" - | "skill_list" | "skill_search" | "http" | "list_jobs" | "job_status" | "job_events" - | "image_analyze" | "message" => Some(PermissionState::AlwaysAllow), + | "memory_tree" | "tool_list" | "tool_info" | "tool_search" | "skill_list" + | "skill_search" | "http" | "list_jobs" | "job_status" | "job_events" | "image_analyze" + | "message" => Some(PermissionState::AlwaysAllow), "shell" | "read_file" | "write_file" @@ -322,12 +322,12 @@ pub async fn load_cached_admin_tool_policy<'a>( /// When the policy failed to load or parse, returns an empty tool list /// (fail-closed) to preserve admin restrictions. pub fn filter_admin_disabled_tools( - tool_defs: Vec<crate::llm::ToolDefinition>, + tool_defs: Vec<ironclaw_llm::ToolDefinition>, multi_tenant: bool, is_admin: bool, user_id: &str, policy_state: &AdminToolPolicyState, -) -> Vec<crate::llm::ToolDefinition> { +) -> Vec<ironclaw_llm::ToolDefinition> { if !multi_tenant || is_admin { return tool_defs; } @@ -421,19 +421,19 @@ mod tests { #[test] fn test_effective_permission_checks_tool_name_aliases() { let mut overrides = HashMap::new(); - overrides.insert("tool-activate".to_string(), PermissionState::Disabled); + overrides.insert("tool-install".to_string(), PermissionState::Disabled); assert_eq!( - effective_permission("tool_activate", &overrides), + effective_permission("tool_install", &overrides), PermissionState::Disabled, "hyphenated saved override should apply to underscore runtime lookup" ); overrides.clear(); - overrides.insert("tool_activate".to_string(), PermissionState::AskEachTime); + overrides.insert("tool_install".to_string(), PermissionState::AskEachTime); assert_eq!( - effective_permission("tool-activate", &overrides), + effective_permission("tool-install", &overrides), PermissionState::AskEachTime, "underscore saved override should apply to hyphenated runtime lookup" ); diff --git a/src/tools/registry.rs b/src/tools/registry.rs index 356911aee6e..a3db7bd36b0 100644 --- a/src/tools/registry.rs +++ b/src/tools/registry.rs @@ -8,8 +8,6 @@ use tokio::sync::RwLock; use crate::context::ContextManager; use crate::db::{Database, UserStore}; use crate::extensions::ExtensionManager; -use crate::llm::recording::HttpInterceptor; -use crate::llm::{LlmProvider, ToolDefinition}; use crate::orchestrator::job_manager::ContainerJobManager; use crate::secrets::SecretsStore; use crate::tools::builder::{ @@ -20,9 +18,9 @@ use crate::tools::builtin::{ GlobTool, GrepTool, HttpTool, JobEventsTool, JobPromptTool, JobStatusTool, JsonTool, ListDirTool, ListJobsTool, MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTool, PlanUpdateTool, PromptQueue, ReadFileTool, ShellTool, SkillInstallTool, SkillListTool, - SkillRemoveTool, SkillSearchTool, TimeTool, ToolActivateTool, ToolAuthTool, ToolInstallTool, - ToolListTool, ToolPermissionSetTool, ToolRemoveTool, ToolSearchTool, ToolUpgradeTool, - WriteFileTool, shared_file_history, shared_read_file_state, + SkillRemoveTool, SkillSearchTool, TimeTool, ToolAuthTool, ToolInstallTool, ToolListTool, + ToolPermissionSetTool, ToolRemoveTool, ToolSearchTool, ToolUpgradeTool, WriteFileTool, + shared_file_history, shared_read_file_state, }; use crate::tools::rate_limiter::RateLimiter; use crate::tools::tool::{ @@ -33,6 +31,8 @@ use crate::tools::wasm::{ WasmStorageError, WasmToolRuntime, WasmToolStore, WasmToolWrapper, }; use crate::workspace::Workspace; +use ironclaw_llm::recording::HttpInterceptor; +use ironclaw_llm::{LlmProvider, ToolDefinition}; use ironclaw_skills::catalog::SkillCatalog; use ironclaw_skills::registry::SkillRegistry; @@ -79,7 +79,6 @@ const PROTECTED_TOOL_NAMES: &[&str] = &[ "tool_search", "tool_install", "tool_auth", - "tool_activate", "tool_list", "tool_remove", "tool_upgrade", @@ -109,6 +108,8 @@ const PROTECTED_TOOL_NAMES: &[&str] = &[ "plan_update", // Permission tools "tool_permission_set", + // Pairing tools + "pairing_approve", // Aliases (web_fetch is an alias for http in some contexts) "web_fetch", ]; @@ -586,7 +587,7 @@ impl ToolRegistry { pub fn register_memory_tools_with_resolver( &self, resolver: Arc<dyn crate::tools::builtin::memory::WorkspaceResolver>, - reasoning_llm: Option<Arc<dyn crate::llm::LlmProvider>>, + reasoning_llm: Option<Arc<dyn ironclaw_llm::LlmProvider>>, reasoning_enabled: bool, ) { self.register_sync(Arc::new(MemorySearchTool::with_reasoning( @@ -703,19 +704,18 @@ impl ToolRegistry { tracing::debug!("Registered 2 secret management tools (list, delete)"); } - /// Register extension management tools (search, install, auth, activate, list, remove). + /// Register extension management tools (search, install, auth, list, remove). /// /// These allow the LLM to manage MCP servers and WASM tools through conversation. pub fn register_extension_tools(&self, manager: Arc<ExtensionManager>) { self.register_sync(Arc::new(ToolSearchTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolInstallTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolAuthTool::new(Arc::clone(&manager)))); - self.register_sync(Arc::new(ToolActivateTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolListTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolRemoveTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolUpgradeTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ExtensionInfoTool::new(manager))); - tracing::debug!("Registered 8 extension management tools"); + tracing::debug!("Registered 7 extension management tools"); } /// Register the permission management tool (`tool_permission_set`). diff --git a/src/tools/schema_validator.rs b/src/tools/schema_validator.rs index 6fe067bc626..a7290578a01 100644 --- a/src/tools/schema_validator.rs +++ b/src/tools/schema_validator.rs @@ -596,16 +596,6 @@ mod tests { "required": ["name"] }), ), - ( - "tool_activate", - serde_json::json!({ - "type": "object", - "properties": { - "name": { "type": "string", "description": "Extension name" } - }, - "required": ["name"] - }), - ), ( "tool_list", serde_json::json!({ diff --git a/src/tools/tool.rs b/src/tools/tool.rs index ca9c31660bd..f9aedca6e7d 100644 --- a/src/tools/tool.rs +++ b/src/tools/tool.rs @@ -472,6 +472,23 @@ pub trait Tool: Send + Sync { None } + /// Names of the secrets store credentials this tool needs to function. + /// + /// Returns the `secret_name` for every non-optional credential the + /// tool declares (e.g. WASM tools' `capabilities.http.credentials`). + /// The engine's auth preflight (`AuthManager::check_action_auth`) + /// consults this list and raises an `Authentication` gate if any + /// declared credential is missing from the secrets store, so the + /// model can call the tool directly — no separate enablement step + /// is required. + /// + /// Default returns empty — built-in tools that don't need a + /// credential, or that handle missing credentials internally, + /// override only when relevant. + fn required_credentials(&self) -> Vec<String> { + Vec::new() + } + /// Get the tool schema for LLM function calling. fn schema(&self) -> ToolSchema { let parameters = self.parameters_schema(); diff --git a/src/tools/wasm/http_security.rs b/src/tools/wasm/http_security.rs index 4844f9df7e3..487c74b9fe2 100644 --- a/src/tools/wasm/http_security.rs +++ b/src/tools/wasm/http_security.rs @@ -61,10 +61,36 @@ pub(crate) async fn validate_and_resolve_http_target( if let Ok(ip) = host.parse::<IpAddr>() { return if is_private_ip(ip) { - Err(format!( - "HTTP request to private/internal IP {} is not allowed", - ip - )) + // Test escape hatch: when `IRONCLAW_TEST_HTTP_REWRITE_MAP` + // is set, the URL has been rewritten by + // `rewrite_http_url_for_tool_testing` to a loopback target + // (the rewrite itself enforces loopback-only). Mirror that + // invariant here — only loopback IPs (127.0.0.0/8, ::1) are + // allowed; the unspecified address (0.0.0.0, ::) is never + // produced by the rewrite and remains rejected. Only + // active in debug/test builds (the env var is checked by + // `rewrite_http_url_for_tool_testing`'s `cfg`). + if cfg!(any(test, debug_assertions)) + && std::env::var("IRONCLAW_TEST_HTTP_REWRITE_MAP") + .map(|v| !v.trim().is_empty()) + .unwrap_or(false) + && ip.is_loopback() + { + tracing::debug!( + %ip, + "Allowing loopback target — IRONCLAW_TEST_HTTP_REWRITE_MAP is set" + ); + Ok(ValidatedHttpTarget { + host, + resolved_addrs: vec![SocketAddr::new(ip, port)], + pin_host_resolution: false, + }) + } else { + Err(format!( + "HTTP request to private/internal IP {} is not allowed", + ip + )) + } } else { Ok(ValidatedHttpTarget { host, diff --git a/src/tools/wasm/mod.rs b/src/tools/wasm/mod.rs index 09c196e03f5..aebd47a2ae5 100644 --- a/src/tools/wasm/mod.rs +++ b/src/tools/wasm/mod.rs @@ -80,7 +80,7 @@ pub const WIT_TOOL_VERSION: &str = "0.3.0"; /// Host WIT version for channel extensions. -pub const WIT_CHANNEL_VERSION: &str = "0.3.0"; +pub const WIT_CHANNEL_VERSION: &str = "0.3.1"; mod allowlist; mod capabilities; diff --git a/src/tools/wasm/wrapper.rs b/src/tools/wasm/wrapper.rs index 26bef557feb..af472e17aa7 100644 --- a/src/tools/wasm/wrapper.rs +++ b/src/tools/wasm/wrapper.rs @@ -19,7 +19,6 @@ use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiVie use crate::auth::resolve_secret_for_runtime; use crate::context::JobContext; use crate::db::UserStore; -use crate::llm::recording::{HttpExchangeRequest, HttpExchangeResponse, HttpInterceptor}; use crate::secrets::SecretsStore; use crate::secrets::host_matches_pattern; use crate::tools::tool::{Tool, ToolDiscoverySummary, ToolError, ToolOutput}; @@ -30,6 +29,7 @@ use crate::tools::wasm::host::{HostState, LogLevel}; use crate::tools::wasm::limits::{ResourceLimits, WasmResourceLimiter}; use crate::tools::wasm::runtime::{EPOCH_TICK_INTERVAL, PreparedModule, WasmToolRuntime}; use crate::tools::wasm::{ssrf_safe_client_builder_for_target, validate_and_resolve_http_target}; +use ironclaw_llm::recording::{HttpExchangeRequest, HttpExchangeResponse, HttpInterceptor}; use ironclaw_safety::LeakDetector; // Generate component model bindings from the WIT file. @@ -392,6 +392,22 @@ impl near::agent::host::Host for StoreData { self.inject_host_credentials(&host, &mut headers, &mut url); } + // Apply IRONCLAW_TEST_HTTP_REWRITE_MAP for tests (parallel to + // the WASM-channel path in `src/channels/wasm/wrapper.rs`). + // Without this, a test fixture's rewrite of e.g. + // gmail.googleapis.com → mock_llm doesn't apply to WASM tools' + // HTTP calls, and the tool reaches the real upstream — the + // exact bug uncovered while debugging the #3133 live test. + #[cfg(any(test, debug_assertions))] + if let Some(rewritten) = rewrite_http_url_for_tool_testing(&url) { + tracing::debug!( + original = %url, + rewritten = %rewritten, + "WASM tool HTTP: applying TEST_HTTP_REWRITE_MAP" + ); + url = rewritten; + } + // Get the max response size from capabilities (default 10MB). let max_response_bytes = self .host_state @@ -1229,6 +1245,26 @@ impl Tool for WasmToolWrapper { Some(&self.prepared.name) } + /// Walk the WASM tool's declared HTTP credentials and return every + /// non-optional `secret_name`. The engine's auth preflight + /// (`AuthManager::check_action_auth`) uses this to raise an + /// `Authentication` gate when the tool is called and any required + /// credential is missing from the secrets store — so the model can + /// call e.g. `gmail` directly with no separate enablement step. + fn required_credentials(&self) -> Vec<String> { + let mut out = Vec::new(); + if let Some(http) = &self.capabilities.http { + for mapping in http.credentials.values() { + if !mapping.optional { + out.push(mapping.secret_name.clone()); + } + } + } + out.sort(); + out.dedup(); + out + } + /// Compose the tool schema for LLM function calling. /// /// When the advertised schema is permissive (no typed properties), appends @@ -1535,6 +1571,85 @@ async fn resolve_host_credentials( } } +/// Apply `IRONCLAW_TEST_HTTP_REWRITE_MAP` to a WASM-tool HTTP request. +/// +/// Mirrors the WASM-channel rewrite at +/// `src/channels/wasm/wrapper.rs::rewrite_http_url_for_testing`. The +/// env var is a JSON object `{ host: replacement_base }` (e.g. +/// `{"gmail.googleapis.com":"http://127.0.0.1:8080"}`); a matching +/// host substitutes the host's scheme/authority while preserving path +/// and query. Replacement bases must be loopback URLs. +/// +/// Without this, WASM tools' HTTP calls (e.g. `gmail` hitting +/// gmail.googleapis.com) ignored the test rewrite and reached the +/// real upstream — the bug uncovered while debugging the #3133 live +/// test. +#[cfg(any(test, debug_assertions))] +pub(crate) fn rewrite_http_url_for_tool_testing(url: &str) -> Option<String> { + let raw = std::env::var("IRONCLAW_TEST_HTTP_REWRITE_MAP").ok()?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let map: HashMap<String, String> = serde_json::from_str(trimmed).ok()?; + let parsed = url::Url::parse(url).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + let host = parsed.host_str()?.to_lowercase(); + let base = map.get(&host)?.trim().trim_end_matches('/').to_string(); + // Match `channels::wasm::wrapper::is_loopback_test_rewrite_base`: + // accept any http/https loopback URL (`localhost`, `127.0.0.1/8`, + // `[::1]`) via URL parsing rather than hardcoded `http://...` prefix + // matching, which silently dropped valid forms like + // `https://localhost:8443` that the channel-side helper accepts. + if !is_loopback_test_rewrite_base(&base) { + tracing::warn!( + host = %host, + base = %base, + "IRONCLAW_TEST_HTTP_REWRITE_MAP: ignoring non-loopback target" + ); + return None; + } + let path = parsed.path().trim_start_matches('/'); + let mut rewritten = format!("{base}/{path}"); + if let Some(query) = parsed.query() { + rewritten.push('?'); + rewritten.push_str(query); + } + Some(rewritten) +} + +/// Validate that a test-rewrite target is an http/https loopback URL. +/// +/// Accepts `localhost`, IPv4 loopback (`127.0.0.0/8`), and IPv6 `::1` +/// (the `url` crate keeps brackets on IPv6 hosts, so we strip them +/// before `IpAddr::parse`). Mirrors +/// `channels::wasm::wrapper::is_loopback_test_rewrite_base` — single +/// source of truth would be nicer, but the two `wrapper.rs` files sit +/// in sibling modules with independent `cfg(any(test, +/// debug_assertions))` gates. +#[cfg(any(test, debug_assertions))] +fn is_loopback_test_rewrite_base(base: &str) -> bool { + let Ok(parsed) = url::Url::parse(base) else { + return false; + }; + if !matches!(parsed.scheme(), "http" | "https") { + return false; + } + let Some(host) = parsed.host_str() else { + return false; + }; + let host = host + .strip_prefix('[') + .and_then(|v| v.strip_suffix(']')) + .unwrap_or(host); + host.eq_ignore_ascii_case("localhost") + || host + .parse::<std::net::IpAddr>() + .is_ok_and(|ip| ip.is_loopback()) +} + /// Extract the hostname from a URL string. /// /// Handles `https://host:port/path`, stripping scheme, port, and path. @@ -4109,4 +4224,84 @@ mod tests { "secret_value leaked: {debug_output}" ); } + + #[test] + fn is_loopback_test_rewrite_base_accepts_http_and_https_loopback() { + // Mirror the channel-side helper: accept localhost / 127.0.0.0/8 / + // [::1] under both http and https, reject anything else. Locks in + // the contract that previously diverged from the channel-side + // (PR #3366 review). + assert!(super::is_loopback_test_rewrite_base("http://localhost")); + assert!(super::is_loopback_test_rewrite_base( + "http://localhost:9999" + )); + assert!(super::is_loopback_test_rewrite_base("http://127.0.0.1")); + assert!(super::is_loopback_test_rewrite_base( + "http://127.0.0.1:8080" + )); + assert!(super::is_loopback_test_rewrite_base("http://[::1]")); + assert!(super::is_loopback_test_rewrite_base("http://[::1]:8443")); + assert!(super::is_loopback_test_rewrite_base("https://localhost")); + assert!(super::is_loopback_test_rewrite_base( + "https://localhost:8443" + )); + assert!(super::is_loopback_test_rewrite_base("https://127.0.0.1")); + assert!(super::is_loopback_test_rewrite_base("https://[::1]")); + + // Non-loopback / wrong scheme / malformed → reject. + assert!(!super::is_loopback_test_rewrite_base("https://example.com")); + assert!(!super::is_loopback_test_rewrite_base("http://10.0.0.1")); + assert!(!super::is_loopback_test_rewrite_base("http://0.0.0.0")); + assert!(!super::is_loopback_test_rewrite_base("ftp://localhost")); + assert!(!super::is_loopback_test_rewrite_base("not a url")); + } + + #[test] + fn rewrite_http_url_for_tool_testing_accepts_https_loopback() { + // Regression for PR #3366 review: the tool-side rewrite previously + // hardcoded `http://...` prefix matches and silently rejected valid + // https-loopback bases that the channel-side helper accepts. + let _guard = crate::config::helpers::lock_env(); + let original = std::env::var("IRONCLAW_TEST_HTTP_REWRITE_MAP").ok(); + + // SAFETY: guarded by lock_env() — no concurrent env access. + unsafe { + std::env::set_var( + "IRONCLAW_TEST_HTTP_REWRITE_MAP", + r#"{"gmail.googleapis.com":"https://localhost:8443"}"#, + ); + } + let rewritten = super::rewrite_http_url_for_tool_testing( + "https://gmail.googleapis.com/gmail/v1/users/me/profile", + ); + assert_eq!( + rewritten.as_deref(), + Some("https://localhost:8443/gmail/v1/users/me/profile"), + "https-loopback base must be accepted (parity with channel-side helper)" + ); + + // Non-loopback target is still rejected. + unsafe { + std::env::set_var( + "IRONCLAW_TEST_HTTP_REWRITE_MAP", + r#"{"gmail.googleapis.com":"https://attacker.example.com"}"#, + ); + } + let rejected = super::rewrite_http_url_for_tool_testing( + "https://gmail.googleapis.com/gmail/v1/users/me/profile", + ); + assert!( + rejected.is_none(), + "non-loopback rewrite targets must remain rejected" + ); + + // SAFETY: restore original env var under the same lock. + unsafe { + if let Some(ref val) = original { + std::env::set_var("IRONCLAW_TEST_HTTP_REWRITE_MAP", val); + } else { + std::env::remove_var("IRONCLAW_TEST_HTTP_REWRITE_MAP"); + } + } + } } diff --git a/src/tunnel/mod.rs b/src/tunnel/mod.rs index 6823862a4f4..91c552f3917 100644 --- a/src/tunnel/mod.rs +++ b/src/tunnel/mod.rs @@ -417,6 +417,7 @@ mod tests { wasm_channels_enabled: false, configured_wasm_channels: Vec::new(), wasm_channel_owner_ids: std::collections::HashMap::new(), + wasm_channel_runtime_overrides: std::collections::HashMap::new(), } } diff --git a/src/util.rs b/src/util.rs index 784cdfb14a9..215bcadc8e9 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,6 +1,6 @@ //! Shared utility functions used across the codebase. -use crate::llm::{ChatMessage, Role}; +use ironclaw_llm::{ChatMessage, Role}; use serde_json::{Map, Value}; /// Find the largest valid UTF-8 char boundary at or before `pos`. @@ -109,11 +109,11 @@ pub fn canonicalize_json_value(value: Value) -> Value { #[cfg(test)] mod tests { - use crate::llm::ChatMessage; use crate::util::{ canonicalize_json_value, ensure_ends_with_user_message, floor_char_boundary, llm_signals_completion, }; + use ironclaw_llm::ChatMessage; // ── canonicalize_json_value ── @@ -192,7 +192,7 @@ mod tests { let mut msgs: Vec<ChatMessage> = vec![]; ensure_ends_with_user_message(&mut msgs); assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0].role, crate::llm::Role::User); + assert_eq!(msgs[0].role, ironclaw_llm::Role::User); } #[test] @@ -200,7 +200,7 @@ mod tests { let mut msgs = vec![ChatMessage::user("hi"), ChatMessage::assistant("hello")]; ensure_ends_with_user_message(&mut msgs); assert_eq!(msgs.len(), 3); - assert_eq!(msgs[2].role, crate::llm::Role::User); + assert_eq!(msgs[2].role, ironclaw_llm::Role::User); } #[test] @@ -211,7 +211,7 @@ mod tests { ]; ensure_ends_with_user_message(&mut msgs); assert_eq!(msgs.len(), 3); - assert_eq!(msgs[2].role, crate::llm::Role::User); + assert_eq!(msgs[2].role, ironclaw_llm::Role::User); } #[test] diff --git a/src/worker/api.rs b/src/worker/api.rs index 9bfda52cd97..f0ceb632065 100644 --- a/src/worker/api.rs +++ b/src/worker/api.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::error::WorkerError; -use crate::llm::{ +use ironclaw_llm::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, }; @@ -80,6 +80,12 @@ pub struct ProxyToolCompletionResponse { pub cache_read_input_tokens: u32, #[serde(default)] pub cache_creation_input_tokens: u32, + /// Provider-emitted reasoning content that must be echoed on the next + /// turn (#3201, #3225). The orchestrator forwards it back to the + /// container worker, which attaches it to the assistant `ChatMessage` + /// before the next LLM call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning: Option<String>, } /// Completion result for the worker to report when done. @@ -268,6 +274,7 @@ impl WorkerHttpClient { finish_reason: parse_finish_reason(&proxy_resp.finish_reason), cache_read_input_tokens: proxy_resp.cache_read_input_tokens, cache_creation_input_tokens: proxy_resp.cache_creation_input_tokens, + reasoning: proxy_resp.reasoning, }) } diff --git a/src/worker/autonomous_recovery.rs b/src/worker/autonomous_recovery.rs index 4b4b4cc1ec8..df104e2b57a 100644 --- a/src/worker/autonomous_recovery.rs +++ b/src/worker/autonomous_recovery.rs @@ -1,4 +1,4 @@ -use crate::llm::{ResponseAnomaly, ResponseMetadata}; +use ironclaw_llm::{ResponseAnomaly, ResponseMetadata}; pub(crate) const EMPTY_TOOL_COMPLETION_NUDGE: &str = "\ Your previous tool-enabled response was empty or malformed.\n\ diff --git a/src/worker/container.rs b/src/worker/container.rs index 31e9a29ef6d..4cd415c4801 100644 --- a/src/worker/container.rs +++ b/src/worker/container.rs @@ -21,7 +21,6 @@ use crate::agent::agentic_loop::{ use crate::config::SafetyConfig; use crate::context::JobContext; use crate::error::WorkerError; -use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, ResponseMetadata}; use crate::tools::ToolRegistry; use crate::tools::execute::{execute_tool_simple, process_tool_result}; use crate::worker::api::{CompletionReport, JobEventPayload, StatusUpdate, WorkerHttpClient}; @@ -30,6 +29,7 @@ use crate::worker::autonomous_recovery::{ EMPTY_TOOL_COMPLETION_NUDGE, FORCE_TEXT_RECOVERY_PROMPT, }; use crate::worker::proxy_llm::ProxyLlmProvider; +use ironclaw_llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, ResponseMetadata}; use ironclaw_safety::SafetyLayer; /// Configuration for the worker runtime. @@ -423,7 +423,7 @@ impl LoopDelegate for ContainerDelegate { reasoning: &Reasoning, reason_ctx: &mut ReasoningContext, _iteration: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { // Container uses respond_with_tools (which may return either text or tool calls) reasoning .respond_with_tools(reason_ctx) @@ -507,9 +507,10 @@ impl LoopDelegate for ContainerDelegate { async fn execute_tool_calls( &self, - tool_calls: Vec<crate::llm::ToolCall>, + tool_calls: Vec<ironclaw_llm::ToolCall>, content: Option<String>, reason_ctx: &mut ReasoningContext, + reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { { let mut recovery = self.recovery_state.lock().await; @@ -527,13 +528,12 @@ impl LoopDelegate for ContainerDelegate { .await; } - // Add assistant message with tool_calls (OpenAI protocol) - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); + // Add assistant message with tool_calls (OpenAI protocol). + // Carry reasoning for the next turn — see #3201, #3225. + reason_ctx.messages.push( + ChatMessage::assistant_with_tool_calls(content, tool_calls.clone()) + .with_reasoning(reasoning), + ); // Execute tools sequentially (container context — no parallel execution) let mut tool_failure_count: usize = 0; diff --git a/src/worker/job.rs b/src/worker/job.rs index caf9434d605..f99335d51ed 100644 --- a/src/worker/job.rs +++ b/src/worker/job.rs @@ -22,10 +22,6 @@ use crate::channels::web::types::ToolDecisionDto; use crate::context::{ContextManager, JobState}; use crate::error::Error; use crate::hooks::HookRegistry; -use crate::llm::{ - ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, - ResponseMetadata, ToolCall, ToolSelection, -}; use crate::tenant::SystemScope; use crate::tools::execute::process_tool_result; use crate::tools::rate_limiter::RateLimitResult; @@ -35,6 +31,10 @@ use crate::worker::autonomous_recovery::{ EMPTY_TOOL_COMPLETION_NUDGE, FORCE_TEXT_RECOVERY_PROMPT, }; use ironclaw_common::{AppEvent, JobResultStatus}; +use ironclaw_llm::{ + ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, + ResponseMetadata, ToolCall, ToolSelection, +}; use ironclaw_safety::SafetyLayer; /// Shared dependencies for worker execution. @@ -58,7 +58,7 @@ pub struct WorkerDeps { /// are pre-approved for autonomous execution. pub approval_context: Option<ApprovalContext>, /// HTTP interceptor for trace recording/replay (propagated to JobContext). - pub http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>, + pub http_interceptor: Option<Arc<dyn ironclaw_llm::recording::HttpInterceptor>>, /// Whether the deployment is multi-tenant (used for admin tool policy filtering). pub multi_tenant: bool, } @@ -967,6 +967,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } else { Some(action.reasoning.clone()) }, + signature: None, }], )); @@ -1244,7 +1245,7 @@ impl<'a> JobDelegate<'a> { &self, retry_after: Option<Duration>, context: &str, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { use std::sync::atomic::Ordering::Relaxed; let count = self.consecutive_rate_limits.fetch_add(1, Relaxed) + 1; @@ -1279,10 +1280,10 @@ impl<'a> JobDelegate<'a> { ); tokio::time::sleep(wait).await; - Ok(crate::llm::RespondOutput { + Ok(ironclaw_llm::RespondOutput { result: RespondResult::Text(String::new()), - usage: crate::llm::TokenUsage::default(), - finish_reason: crate::llm::FinishReason::Stop, + usage: ironclaw_llm::TokenUsage::default(), + finish_reason: ironclaw_llm::FinishReason::Stop, metadata: ResponseMetadata::default(), }) } @@ -1311,7 +1312,7 @@ impl<'a> JobDelegate<'a> { &self, context: &str, error: &crate::error::LlmError, - ) -> Option<crate::llm::RespondOutput> { + ) -> Option<ironclaw_llm::RespondOutput> { if !is_completion_eligible_error(error) { return None; } @@ -1327,10 +1328,10 @@ impl<'a> JobDelegate<'a> { "{context} empty response after text output — treating as completion" ); self.mark_completed_or_warn(context).await; - Some(crate::llm::RespondOutput { + Some(ironclaw_llm::RespondOutput { result: RespondResult::Text(String::new()), - usage: crate::llm::TokenUsage::default(), - finish_reason: crate::llm::FinishReason::Stop, + usage: ironclaw_llm::TokenUsage::default(), + finish_reason: ironclaw_llm::FinishReason::Stop, metadata: ResponseMetadata::default(), }) } @@ -1486,7 +1487,7 @@ impl<'a> LoopDelegate for JobDelegate<'a> { reasoning: &Reasoning, reason_ctx: &mut ReasoningContext, _iteration: usize, - ) -> Result<crate::llm::RespondOutput, crate::error::Error> { + ) -> Result<ironclaw_llm::RespondOutput, crate::error::Error> { // Try select_tools first, fall back to respond_with_tools match reasoning.select_tools(reason_ctx).await { Ok(s) if !s.is_empty() => { @@ -1499,13 +1500,14 @@ impl<'a> LoopDelegate for JobDelegate<'a> { .iter() .find_map(|sel| (!sel.reasoning.is_empty()).then_some(sel.reasoning.clone())); let tool_calls: Vec<ToolCall> = selections_to_tool_calls(&s); - return Ok(crate::llm::RespondOutput { + return Ok(ironclaw_llm::RespondOutput { result: RespondResult::ToolCalls { tool_calls, content: reasoning_text, + reasoning: None, }, - usage: crate::llm::TokenUsage::default(), - finish_reason: crate::llm::FinishReason::ToolUse, + usage: ironclaw_llm::TokenUsage::default(), + finish_reason: ironclaw_llm::FinishReason::ToolUse, metadata: ResponseMetadata::default(), }); } @@ -1668,9 +1670,10 @@ impl<'a> LoopDelegate for JobDelegate<'a> { async fn execute_tool_calls( &self, - tool_calls: Vec<crate::llm::ToolCall>, + tool_calls: Vec<ironclaw_llm::ToolCall>, content: Option<String>, reason_ctx: &mut ReasoningContext, + reasoning: Option<String>, ) -> Result<Option<LoopOutcome>, crate::error::Error> { { let mut recovery = self.recovery_state.lock().await; @@ -1732,13 +1735,13 @@ impl<'a> LoopDelegate for JobDelegate<'a> { ); } - // Add assistant message with tool_calls (OpenAI protocol) - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); + // Add assistant message with tool_calls (OpenAI protocol). + // Carry reasoning for the next turn — DeepSeek thinking-mode and + // Gemini 2.5+ reject the follow-up with HTTP 400 otherwise (#3201, #3225). + reason_ctx.messages.push( + ChatMessage::assistant_with_tool_calls(content, tool_calls.clone()) + .with_reasoning(reasoning), + ); // Convert to ToolSelections let selections: Vec<ToolSelection> = tool_calls @@ -1816,6 +1819,7 @@ fn selections_to_tool_calls(selections: &[ToolSelection]) -> Vec<ToolCall> { } else { Some(s.reasoning.clone()) }, + signature: None, }) .collect() } @@ -1838,18 +1842,18 @@ mod tests { use std::sync::Arc; use crate::channels::ChannelManager; - use crate::llm::ToolSelection; + use ironclaw_llm::ToolSelection; use super::*; use crate::config::SafetyConfig; use crate::context::JobContext; - use crate::llm::{ - CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, - ToolCompletionResponse, - }; use crate::testing::{BroadcastCapture, RecordingBroadcastChannel}; use crate::tools::builtin::MessageTool; use crate::tools::{Tool, ToolError as ToolExecError, ToolOutput}; + use ironclaw_llm::{ + CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, + }; use ironclaw_safety::SafetyLayer; /// A test tool that sleeps for a configurable duration before returning. diff --git a/src/worker/proxy_llm.rs b/src/worker/proxy_llm.rs index 95dc38af284..f92bc947a98 100644 --- a/src/worker/proxy_llm.rs +++ b/src/worker/proxy_llm.rs @@ -9,11 +9,11 @@ use async_trait::async_trait; use rust_decimal::Decimal; use crate::error::LlmError; -use crate::llm::{ +use crate::worker::api::WorkerHttpClient; +use ironclaw_llm::{ CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, }; -use crate::worker::api::WorkerHttpClient; /// An LLM provider that routes all calls through the orchestrator's HTTP API. /// diff --git a/src/workspace/README.md b/src/workspace/README.md index 88cc2566a4d..52b1609f81d 100644 --- a/src/workspace/README.md +++ b/src/workspace/README.md @@ -39,13 +39,32 @@ workspace/ ```rust use std::sync::Arc; -use crate::workspace::{Workspace, OpenAiEmbeddings, paths}; - -// Create workspace for a user (wraps embeddings in a default LRU cache) -let workspace = Workspace::new("user_123", pool) - .with_embeddings(Arc::new(OpenAiEmbeddings::new(api_key))); - -// For tests: skip the cache layer (avoids unnecessary overhead with mocks) +use crate::workspace::{Workspace, paths}; +use ironclaw_embeddings::{create_provider, EmbeddingCacheConfig, ProviderDeps}; + +// Construct an embedding provider through the factory — concrete provider +// types (OpenAI, NEAR AI, Ollama, Bedrock) are crate-private and must be +// reached via `create_provider`. The factory applies a baseline defense- +// in-depth URL check (rejects cloud-metadata IPs and non-http(s) schemes) +// and returns `None` when embeddings are disabled or misconfigured. The +// full operator-tunable SSRF policy lives in the binary's resolver at +// `src/config/embeddings.rs::resolve_embeddings_config`, which calls +// `validate_operator_base_url` on env-driven URLs before populating +// `EmbeddingsConfig`. +let embeddings = create_provider( + &config.embeddings, + ProviderDeps { session, bedrock_setup: None }, +).await; + +let mut workspace = Workspace::new("user_123", pool); +if let Some(emb) = embeddings { + let cache = EmbeddingCacheConfig { max_entries: 1024 }; + workspace = workspace.with_embeddings_cached(emb, cache); +} + +// For tests: skip the cache layer and use the deterministic mock +// (gated behind the `testing` feature on `ironclaw_embeddings`). +// use ironclaw_embeddings::MockEmbeddings; // let workspace = Workspace::new("user_123", pool) // .with_embeddings_uncached(Arc::new(MockEmbeddings::new(1536))); diff --git a/src/workspace/embeddings.rs b/src/workspace/embeddings.rs deleted file mode 100644 index f138a8bf5f5..00000000000 --- a/src/workspace/embeddings.rs +++ /dev/null @@ -1,783 +0,0 @@ -//! Embedding providers for semantic search. -//! -//! Embeddings convert text into dense vectors that capture semantic meaning. -//! Similar concepts have similar vectors, enabling semantic search. - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -/// Error type for embedding operations. -#[derive(Debug, thiserror::Error)] -pub enum EmbeddingError { - #[error("HTTP request failed: {0}")] - HttpError(String), - - #[error("Invalid response: {0}")] - InvalidResponse(String), - - #[error("Rate limited, retry after {retry_after:?}")] - RateLimited { - retry_after: Option<std::time::Duration>, - }, - - #[error("Authentication failed")] - AuthFailed, - - #[error("Text too long: {length} > {max}")] - TextTooLong { length: usize, max: usize }, -} - -impl From<reqwest::Error> for EmbeddingError { - fn from(e: reqwest::Error) -> Self { - EmbeddingError::HttpError(e.to_string()) - } -} - -/// Trait for embedding providers. -#[async_trait] -pub trait EmbeddingProvider: Send + Sync { - /// Get the embedding dimension. - fn dimension(&self) -> usize; - - /// Get the model name. - fn model_name(&self) -> &str; - - /// Maximum input length in characters. - fn max_input_length(&self) -> usize; - - /// Generate an embedding for a single text. - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError>; - - /// Generate embeddings for multiple texts (batched). - /// - /// Default implementation calls embed() for each text. - async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> { - let mut embeddings = Vec::with_capacity(texts.len()); - for text in texts { - embeddings.push(self.embed(text).await?); - } - Ok(embeddings) - } -} - -/// Default base URL for the OpenAI API. -const OPENAI_API_BASE_URL: &str = "https://api.openai.com"; - -/// OpenAI embedding provider using text-embedding-ada-002 or text-embedding-3-small. -/// -/// Supports any OpenAI-compatible embedding endpoint via [`with_base_url`](Self::with_base_url). -pub struct OpenAiEmbeddings { - client: reqwest::Client, - api_key: String, - model: String, - dimension: usize, - base_url: String, -} - -impl OpenAiEmbeddings { - /// Create a new OpenAI embedding provider with the default model. - /// - /// Uses text-embedding-3-small which has 1536 dimensions. - pub fn new(api_key: impl Into<String>) -> Self { - Self { - client: reqwest::Client::new(), - api_key: api_key.into(), - model: "text-embedding-3-small".to_string(), - dimension: 1536, - base_url: OPENAI_API_BASE_URL.to_string(), - } - } - - /// Use text-embedding-ada-002 model. - pub fn ada_002(api_key: impl Into<String>) -> Self { - Self { - client: reqwest::Client::new(), - api_key: api_key.into(), - model: "text-embedding-ada-002".to_string(), - dimension: 1536, - base_url: OPENAI_API_BASE_URL.to_string(), - } - } - - /// Use text-embedding-3-large model. - pub fn large(api_key: impl Into<String>) -> Self { - Self { - client: reqwest::Client::new(), - api_key: api_key.into(), - model: "text-embedding-3-large".to_string(), - dimension: 3072, - base_url: OPENAI_API_BASE_URL.to_string(), - } - } - - /// Use a custom model with specified dimension. - pub fn with_model( - api_key: impl Into<String>, - model: impl Into<String>, - dimension: usize, - ) -> Self { - Self { - client: reqwest::Client::new(), - api_key: api_key.into(), - model: model.into(), - dimension, - base_url: OPENAI_API_BASE_URL.to_string(), - } - } - - /// Set a custom base URL for OpenAI-compatible embedding providers. - /// - /// The URL must use `http://` or `https://` scheme. If no scheme is present, - /// `https://` is prepended automatically. Trailing slashes are stripped. - pub fn with_base_url(mut self, base_url: &str) -> Self { - let url = base_url.trim(); - - // Auto-prepend https:// if no scheme is present. - let mut url = if !url.starts_with("http://") && !url.starts_with("https://") { - tracing::debug!( - "No scheme in embedding base URL '{}', prepending https://", - url - ); - format!("https://{url}") - } else { - url.to_string() - }; - - while url.ends_with('/') { - url.pop(); - } - - self.base_url = url; - self - } -} - -#[derive(Debug, Serialize)] -struct OpenAiEmbeddingRequest<'a> { - model: &'a str, - input: &'a [String], -} - -#[derive(Debug, Deserialize)] -struct OpenAiEmbeddingResponse { - data: Vec<OpenAiEmbeddingData>, -} - -#[derive(Debug, Deserialize)] -struct OpenAiEmbeddingData { - embedding: Vec<f32>, -} - -#[async_trait] -impl EmbeddingProvider for OpenAiEmbeddings { - fn dimension(&self) -> usize { - self.dimension - } - - fn model_name(&self) -> &str { - &self.model - } - - fn max_input_length(&self) -> usize { - // text-embedding-3-small/large: 8191 tokens (~32k chars) - // text-embedding-ada-002: 8191 tokens - 32_000 - } - - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> { - if text.len() > self.max_input_length() { - return Err(EmbeddingError::TextTooLong { - length: text.len(), - max: self.max_input_length(), - }); - } - - let embeddings = self.embed_batch(&[text.to_string()]).await?; - embeddings - .into_iter() - .next() - .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) - } - - async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> { - if texts.is_empty() { - return Ok(Vec::new()); - } - - let request = OpenAiEmbeddingRequest { - model: &self.model, - input: texts, - }; - - let url = format!("{}/v1/embeddings", self.base_url); - - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", self.api_key)) - .json(&request) - .send() - .await?; - - let status = response.status(); - - if status == reqwest::StatusCode::UNAUTHORIZED { - return Err(EmbeddingError::AuthFailed); - } - - if status == reqwest::StatusCode::TOO_MANY_REQUESTS { - let retry_after = Some(crate::llm::retry::parse_retry_after( - response.headers().get("retry-after"), - )); - return Err(EmbeddingError::RateLimited { retry_after }); - } - - if !status.is_success() { - let error_text = response.text().await.unwrap_or_default(); - return Err(EmbeddingError::HttpError(format!( - "Status {}: {}", - status, error_text - ))); - } - - let result: OpenAiEmbeddingResponse = response.json().await.map_err(|e| { - EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) - })?; - - Ok(result.data.into_iter().map(|d| d.embedding).collect()) - } -} - -/// NEAR AI embedding provider using the NEAR AI API. -/// -/// Uses the same session-based auth as the LLM provider. -pub struct NearAiEmbeddings { - client: reqwest::Client, - base_url: String, - session: std::sync::Arc<crate::llm::SessionManager>, - model: String, - dimension: usize, -} - -impl NearAiEmbeddings { - /// Create a new NEAR AI embedding provider. - /// - /// Uses the same session manager as the LLM provider for auth. - pub fn new( - base_url: impl Into<String>, - session: std::sync::Arc<crate::llm::SessionManager>, - ) -> Self { - Self { - client: reqwest::Client::new(), - base_url: base_url.into(), - session, - model: "text-embedding-3-small".to_string(), - dimension: 1536, - } - } - - /// Use a specific model. - pub fn with_model(mut self, model: impl Into<String>, dimension: usize) -> Self { - self.model = model.into(); - self.dimension = dimension; - self - } -} - -#[derive(Debug, Serialize)] -struct NearAiEmbeddingRequest<'a> { - model: &'a str, - input: &'a [String], -} - -#[derive(Debug, Deserialize)] -struct NearAiEmbeddingResponse { - data: Vec<NearAiEmbeddingData>, -} - -#[derive(Debug, Deserialize)] -struct NearAiEmbeddingData { - embedding: Vec<f32>, -} - -#[async_trait] -impl EmbeddingProvider for NearAiEmbeddings { - fn dimension(&self) -> usize { - self.dimension - } - - fn model_name(&self) -> &str { - &self.model - } - - fn max_input_length(&self) -> usize { - 32_000 - } - - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> { - if text.len() > self.max_input_length() { - return Err(EmbeddingError::TextTooLong { - length: text.len(), - max: self.max_input_length(), - }); - } - - let embeddings = self.embed_batch(&[text.to_string()]).await?; - embeddings - .into_iter() - .next() - .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) - } - - async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> { - use secrecy::ExposeSecret; - - if texts.is_empty() { - return Ok(Vec::new()); - } - - let request = NearAiEmbeddingRequest { - model: &self.model, - input: texts, - }; - - let token = self - .session - .get_token() - .await - .map_err(|_| EmbeddingError::AuthFailed)?; - - let url = format!("{}/v1/embeddings", self.base_url); - - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", token.expose_secret())) - .json(&request) - .send() - .await?; - - let status = response.status(); - - if status == reqwest::StatusCode::UNAUTHORIZED { - return Err(EmbeddingError::AuthFailed); - } - - if status == reqwest::StatusCode::TOO_MANY_REQUESTS { - let retry_after = Some(crate::llm::retry::parse_retry_after( - response.headers().get("retry-after"), - )); - return Err(EmbeddingError::RateLimited { retry_after }); - } - - if !status.is_success() { - let error_text = response.text().await.unwrap_or_default(); - return Err(EmbeddingError::HttpError(format!( - "Status {}: {}", - status, error_text - ))); - } - - let result: NearAiEmbeddingResponse = response.json().await.map_err(|e| { - EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) - })?; - - Ok(result.data.into_iter().map(|d| d.embedding).collect()) - } -} - -/// AWS Bedrock embedding provider using Titan Text Embeddings V2. -#[cfg(feature = "bedrock")] -pub struct BedrockEmbeddings { - client: aws_sdk_bedrockruntime::Client, - model: String, - dimension: usize, -} - -#[cfg(feature = "bedrock")] -impl BedrockEmbeddings { - /// Create a new Bedrock embedding provider. - pub async fn new( - config: &crate::llm::BedrockConfig, - model: impl Into<String>, - dimension: usize, - ) -> Result<Self, EmbeddingError> { - let mut builder = aws_config::defaults(aws_config::BehaviorVersion::latest()) - .region(aws_config::Region::new(config.region.clone())); - if let Some(ref profile) = config.profile { - builder = builder.profile_name(profile); - } - - let sdk_config = builder.load().await; - Ok(Self { - client: aws_sdk_bedrockruntime::Client::new(&sdk_config), - model: model.into(), - dimension, - }) - } -} - -#[cfg(feature = "bedrock")] -#[derive(Debug, Serialize)] -struct BedrockTitanEmbeddingRequest<'a> { - #[serde(rename = "inputText")] - input_text: &'a str, - dimensions: usize, - normalize: bool, -} - -#[cfg(feature = "bedrock")] -#[derive(Debug, Deserialize)] -struct BedrockTitanEmbeddingResponse { - embedding: Vec<f32>, -} - -#[cfg(feature = "bedrock")] -fn map_bedrock_invoke_model_error<R: std::fmt::Debug>( - error: &aws_sdk_bedrockruntime::error::SdkError< - aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError, - R, - >, -) -> EmbeddingError { - use aws_sdk_bedrockruntime::error::SdkError; - use aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError; - - match error { - SdkError::ServiceError(service_err) => match service_err.err() { - InvokeModelError::ThrottlingException(_) => { - EmbeddingError::RateLimited { retry_after: None } - } - InvokeModelError::AccessDeniedException(_) => EmbeddingError::AuthFailed, - InvokeModelError::ValidationException(e) => EmbeddingError::InvalidResponse(format!( - "Bedrock validation error: {}", - e.message().unwrap_or("unknown") - )), - InvokeModelError::ModelNotReadyException(e) => EmbeddingError::HttpError(format!( - "Bedrock model not ready: {}", - e.message().unwrap_or("unknown") - )), - other => EmbeddingError::HttpError(format!("Bedrock service error: {other:?}")), - }, - SdkError::TimeoutError(_) => { - EmbeddingError::HttpError("Bedrock request timed out".to_string()) - } - other => EmbeddingError::HttpError(format!("Bedrock request failed: {other:?}")), - } -} - -#[cfg(feature = "bedrock")] -#[async_trait] -impl EmbeddingProvider for BedrockEmbeddings { - fn dimension(&self) -> usize { - self.dimension - } - - fn model_name(&self) -> &str { - &self.model - } - - fn max_input_length(&self) -> usize { - 32_000 - } - - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> { - if text.len() > self.max_input_length() { - return Err(EmbeddingError::TextTooLong { - length: text.len(), - max: self.max_input_length(), - }); - } - - let request = BedrockTitanEmbeddingRequest { - input_text: text, - dimensions: self.dimension, - normalize: true, - }; - - let body = serde_json::to_vec(&request).map_err(|e| { - EmbeddingError::InvalidResponse(format!("Failed to serialize request: {}", e)) - })?; - - let response = self - .client - .invoke_model() - .model_id(&self.model) - .content_type("application/json") - .accept("application/json") - .body(aws_smithy_types::Blob::new(body)) - .send() - .await - .map_err(|e| map_bedrock_invoke_model_error(&e))?; - - let result: BedrockTitanEmbeddingResponse = serde_json::from_slice(response.body.as_ref()) - .map_err(|e| { - EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e)) - })?; - - if result.embedding.len() != self.dimension { - return Err(EmbeddingError::InvalidResponse(format!( - "Bedrock returned embedding of dimension {}, expected {}", - result.embedding.len(), - self.dimension, - ))); - } - - Ok(result.embedding) - } -} - -/// Ollama embedding provider using a local Ollama instance. -/// -/// Ollama serves embedding models (e.g. `nomic-embed-text`, `mxbai-embed-large`) -/// via a REST API, typically at `http://localhost:11434`. -pub struct OllamaEmbeddings { - client: reqwest::Client, - base_url: String, - model: String, - dimension: usize, -} - -impl OllamaEmbeddings { - /// Create a new Ollama embedding provider. - /// - /// Defaults to `nomic-embed-text` (768 dimensions). - pub fn new(base_url: impl Into<String>) -> Self { - Self { - client: reqwest::Client::new(), - base_url: base_url.into(), - model: "nomic-embed-text".to_string(), - dimension: 768, - } - } - - /// Use a specific model with a given dimension. - pub fn with_model(mut self, model: impl Into<String>, dimension: usize) -> Self { - self.model = model.into(); - self.dimension = dimension; - self - } -} - -#[derive(Debug, Serialize)] -struct OllamaEmbedRequest<'a> { - model: &'a str, - input: &'a [String], -} - -#[derive(Debug, Deserialize)] -struct OllamaEmbedResponse { - embeddings: Vec<Vec<f32>>, -} - -#[async_trait] -impl EmbeddingProvider for OllamaEmbeddings { - fn dimension(&self) -> usize { - self.dimension - } - - fn model_name(&self) -> &str { - &self.model - } - - fn max_input_length(&self) -> usize { - // Most Ollama embedding models support 8192 tokens (~32k chars) - 32_000 - } - - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> { - if text.len() > self.max_input_length() { - return Err(EmbeddingError::TextTooLong { - length: text.len(), - max: self.max_input_length(), - }); - } - - let embeddings = self.embed_batch(&[text.to_string()]).await?; - embeddings - .into_iter() - .next() - .ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string())) - } - - async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> { - if texts.is_empty() { - return Ok(Vec::new()); - } - - let request = OllamaEmbedRequest { - model: &self.model, - input: texts, - }; - - let url = format!("{}/api/embed", self.base_url); - - let response = self.client.post(&url).json(&request).send().await?; - - let status = response.status(); - - if !status.is_success() { - let error_text = response.text().await.unwrap_or_default(); - return Err(EmbeddingError::HttpError(format!( - "Ollama returned HTTP {}: {}", - status, error_text - ))); - } - - let result: OllamaEmbedResponse = response.json().await.map_err(|e| { - EmbeddingError::InvalidResponse(format!("Failed to parse Ollama response: {}", e)) - })?; - - // Validate that returned embeddings match the configured dimension. - for (i, emb) in result.embeddings.iter().enumerate() { - if emb.len() != self.dimension { - return Err(EmbeddingError::InvalidResponse(format!( - "Ollama returned embedding of dimension {}, expected {} at index {}", - emb.len(), - self.dimension, - i - ))); - } - } - - Ok(result.embeddings) - } -} - -/// A mock embedding provider for testing. -/// -/// Generates deterministic embeddings based on text hash. -/// Useful for unit and integration tests. -pub struct MockEmbeddings { - dimension: usize, -} - -impl MockEmbeddings { - /// Create a new mock embeddings provider with the given dimension. - pub fn new(dimension: usize) -> Self { - Self { dimension } - } -} - -#[async_trait] -impl EmbeddingProvider for MockEmbeddings { - fn dimension(&self) -> usize { - self.dimension - } - - fn model_name(&self) -> &str { - "mock-embedding" - } - - fn max_input_length(&self) -> usize { - 10_000 - } - - async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> { - // Generate a deterministic embedding based on text hash - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - text.hash(&mut hasher); - let hash = hasher.finish(); - - let mut embedding = Vec::with_capacity(self.dimension); - let mut seed = hash; - for _ in 0..self.dimension { - // Simple LCG for deterministic random values - seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); - let value = (seed as f32 / u64::MAX as f32) * 2.0 - 1.0; - embedding.push(value); - } - - // Normalize to unit length - let magnitude: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt(); - if magnitude > 0.0 { - for x in &mut embedding { - *x /= magnitude; - } - } - - Ok(embedding) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_mock_embeddings() { - let provider = MockEmbeddings::new(128); - - let embedding = provider.embed("hello world").await.unwrap(); - assert_eq!(embedding.len(), 128); - - // Check normalization (should be unit vector) - let magnitude: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt(); - assert!((magnitude - 1.0).abs() < 0.001); - } - - #[tokio::test] - async fn test_mock_embeddings_deterministic() { - let provider = MockEmbeddings::new(64); - - let emb1 = provider.embed("test").await.unwrap(); - let emb2 = provider.embed("test").await.unwrap(); - - // Same input should produce same embedding - assert_eq!(emb1, emb2); - } - - #[tokio::test] - async fn test_mock_embeddings_batch() { - let provider = MockEmbeddings::new(64); - - let texts = vec!["hello".to_string(), "world".to_string()]; - let embeddings = provider.embed_batch(&texts).await.unwrap(); - - assert_eq!(embeddings.len(), 2); - assert_eq!(embeddings[0].len(), 64); - assert_eq!(embeddings[1].len(), 64); - - // Different texts should produce different embeddings - assert_ne!(embeddings[0], embeddings[1]); - } - - #[test] - fn test_openai_embeddings_config() { - let provider = OpenAiEmbeddings::new("test-key"); - assert_eq!(provider.dimension(), 1536); - assert_eq!(provider.model_name(), "text-embedding-3-small"); - assert_eq!(provider.base_url, OPENAI_API_BASE_URL); - - let provider = OpenAiEmbeddings::large("test-key"); - assert_eq!(provider.dimension(), 3072); - assert_eq!(provider.model_name(), "text-embedding-3-large"); - assert_eq!(provider.base_url, OPENAI_API_BASE_URL); - } - - #[test] - fn test_openai_with_base_url_valid() { - let provider = - OpenAiEmbeddings::new("test-key").with_base_url("https://custom.example.com"); - assert_eq!(provider.base_url, "https://custom.example.com"); - } - - #[test] - fn test_openai_with_base_url_strips_trailing_slashes() { - let provider = - OpenAiEmbeddings::new("test-key").with_base_url("https://custom.example.com///"); - assert_eq!(provider.base_url, "https://custom.example.com"); - } - - #[test] - fn test_openai_with_base_url_http_scheme() { - let provider = OpenAiEmbeddings::new("test-key").with_base_url("http://localhost:8080"); - assert_eq!(provider.base_url, "http://localhost:8080"); - } - - #[test] - fn test_openai_with_base_url_schemeless_prepends_https() { - let provider = OpenAiEmbeddings::new("test-key").with_base_url("custom.example.com/v1"); - assert_eq!(provider.base_url, "https://custom.example.com/v1"); - } -} diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index 6efc5c5b25b..f93ce75793e 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -42,8 +42,6 @@ mod chunker; mod document; -mod embedding_cache; -mod embeddings; pub mod extension_state; pub mod hygiene; pub mod layer; @@ -63,12 +61,6 @@ pub use document::{ content_sha256, is_config_path, is_identity_path, is_reserved_scope, merge_workspace_entries, paths, }; -pub use embedding_cache::{CachedEmbeddingProvider, EmbeddingCacheConfig}; -#[cfg(feature = "bedrock")] -pub use embeddings::BedrockEmbeddings; -pub use embeddings::{ - EmbeddingProvider, MockEmbeddings, NearAiEmbeddings, OllamaEmbeddings, OpenAiEmbeddings, -}; #[cfg(feature = "postgres")] pub use repository::Repository; pub use search::{ @@ -94,6 +86,9 @@ use deadpool_postgres::Pool; use uuid::Uuid; use crate::error::WorkspaceError; +use ironclaw_embeddings::{ + CachedEmbeddingProvider, EmbeddingCacheConfig, EmbeddingError, EmbeddingProvider, +}; use ironclaw_safety::{Sanitizer, Severity}; /// Files injected into the system prompt. Writes to these are scanned for @@ -751,18 +746,39 @@ impl Workspace { pub fn scoped_to_user(&self, user_id: impl Into<String>) -> Self { let user_id = user_id.into(); + let private_source_scopes: Vec<String> = self + .memory_layers + .iter() + .filter(|layer| layer.sensitivity == crate::workspace::layer::LayerSensitivity::Private) + .map(|layer| layer.scope.clone()) + .collect(); + let non_private_source_scopes: Vec<String> = self + .memory_layers + .iter() + .filter(|layer| layer.sensitivity != crate::workspace::layer::LayerSensitivity::Private) + .map(|layer| layer.scope.clone()) + .collect(); + let mut memory_layers = self.memory_layers.clone(); for layer in &mut memory_layers { - if layer.sensitivity == crate::workspace::layer::LayerSensitivity::Private - && layer.scope == self.user_id - { + if layer.sensitivity == crate::workspace::layer::LayerSensitivity::Private { layer.scope = user_id.clone(); } } let mut read_user_ids = vec![user_id.clone()]; for scope in &self.read_user_ids { - if scope != &self.user_id && !read_user_ids.contains(scope) { + let used_by_non_private_layer = non_private_source_scopes.contains(scope); + let old_primary_private_scope = scope == &self.user_id && !used_by_non_private_layer; + let private_layer_source_scope = + private_source_scopes.contains(scope) && !used_by_non_private_layer; + + // Drop scopes that came only from the source private identity, but preserve + // any scope string that a non-private layer intentionally shares. + if !old_primary_private_scope + && !private_layer_source_scope + && !read_user_ids.contains(scope) + { read_user_ids.push(scope.clone()); } } @@ -2467,7 +2483,7 @@ impl Workspace { "Failed to embed chunk {}: {}{}", chunk.id, e, - if matches!(e, embeddings::EmbeddingError::AuthFailed) { + if matches!(e, EmbeddingError::AuthFailed) { ". Check OPENAI_API_KEY or set EMBEDDING_PROVIDER=ollama for local embeddings" } else { "" diff --git a/tests/admin_tool_policy_e2e.rs b/tests/admin_tool_policy_e2e.rs index dbfc8dd6882..743bd64c328 100644 --- a/tests/admin_tool_policy_e2e.rs +++ b/tests/admin_tool_policy_e2e.rs @@ -15,11 +15,11 @@ mod tests { use async_trait::async_trait; use ironclaw::channels::IncomingMessage; use ironclaw::config::Config; - use ironclaw::llm::{ + use ironclaw::tools::permissions::{ADMIN_SETTINGS_USER_ID, ADMIN_TOOL_POLICY_KEY}; + use ironclaw_llm::{ CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, }; - use ironclaw::tools::permissions::{ADMIN_SETTINGS_USER_ID, ADMIN_TOOL_POLICY_KEY}; use rust_decimal::Decimal; use crate::support::test_rig::TestRigBuilder; @@ -70,6 +70,7 @@ mod tests { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } diff --git a/tests/cross_tenant_resource_isolation.rs b/tests/cross_tenant_resource_isolation.rs new file mode 100644 index 00000000000..020953cd0f5 --- /dev/null +++ b/tests/cross_tenant_resource_isolation.rs @@ -0,0 +1,444 @@ +//! Cross-tenant access regression tests for non-thread resources. +//! +//! Companion to `thread_isolation_integration.rs`: drives the running +//! gateway over real HTTP and asserts that Bob cannot reach Alice's +//! sandbox-job artefacts (events, file list, file content) or routine +//! run history by guessing/stealing an id. Each handler that takes an +//! id from the request gets its own negative test plus a positive +//! "Alice can reach her own" pin so a future refactor that flips the +//! ownership predicate fails BOTH directions. +//! +//! Path-traversal pin lives in `alice_job_file_read_rejects_dotdot`: +//! even Alice's own `?path=../../etc/passwd` request must not escape +//! the project directory. The handler relies on `canonicalize()` + +//! `starts_with(base_canonical)`; if that guard regresses, the test +//! catches it before shipping. +//! +//! Gated on `feature = "libsql"` so the suite has a real DB to seed +//! sandbox jobs and routines into. + +#![cfg(feature = "libsql")] + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use chrono::Utc; +use ironclaw::agent::SessionManager; +use ironclaw::agent::routine::{ + Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger, +}; +use ironclaw::channels::web::auth::{MultiAuthState, UserIdentity}; +use ironclaw::channels::web::platform::router::start_server; +use ironclaw::channels::web::platform::state::{GatewayState, PerUserRateLimiter, RateLimiter}; +use ironclaw::channels::web::sse::SseManager; +use ironclaw::channels::web::ws::WsConnectionTracker; +use ironclaw::db::Database; +use ironclaw::history::SandboxJobRecord; +use uuid::Uuid; + +const ALICE_TOKEN: &str = "tok-alice-resource-isolation"; +const BOB_TOKEN: &str = "tok-bob-resource-isolation"; +const ALICE_USER_ID: &str = "alice"; +const BOB_USER_ID: &str = "bob"; + +fn two_user_auth() -> MultiAuthState { + let mut tokens = HashMap::new(); + tokens.insert( + ALICE_TOKEN.to_string(), + UserIdentity { + user_id: ALICE_USER_ID.to_string(), + role: "admin".to_string(), + workspace_read_scopes: Vec::new(), + }, + ); + tokens.insert( + BOB_TOKEN.to_string(), + UserIdentity { + user_id: BOB_USER_ID.to_string(), + role: "admin".to_string(), + workspace_read_scopes: Vec::new(), + }, + ); + MultiAuthState::multi(tokens) +} + +async fn start_server_with_db() -> ( + SocketAddr, + Arc<GatewayState>, + Arc<dyn Database>, + tempfile::TempDir, +) { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("test.db"); + let backend = ironclaw::db::libsql::LibSqlBackend::new_local(&path) + .await + .expect("backend"); + backend.run_migrations().await.expect("migrations"); + let db: Arc<dyn Database> = Arc::new(backend); + + let (agent_tx, _agent_rx) = tokio::sync::mpsc::channel(64); + let auth = two_user_auth(); + let session_manager = Arc::new(SessionManager::new()); + + let state = Arc::new(GatewayState { + msg_tx: tokio::sync::RwLock::new(Some(agent_tx)), + sse: Arc::new(SseManager::new()), + workspace: None, + workspace_pool: None, + multi_tenant_mode: true, + session_manager: Some(session_manager), + log_broadcaster: None, + log_level_handle: None, + extension_manager: None, + tool_registry: None, + store: Some(Arc::clone(&db)), + settings_cache: None, + job_manager: None, + prompt_queue: None, + scheduler: None, + owner_id: ALICE_USER_ID.to_string(), + shutdown_tx: tokio::sync::RwLock::new(None), + ws_tracker: Some(Arc::new(WsConnectionTracker::new())), + llm_provider: None, + llm_reload: None, + llm_session_manager: None, + config_toml_path: None, + skill_registry: None, + skill_catalog: None, + auth_manager: None, + chat_rate_limiter: PerUserRateLimiter::new(30, 60), + oauth_rate_limiter: PerUserRateLimiter::new(20, 60), + registry_entries: Vec::new(), + cost_guard: None, + routine_engine: Arc::new(tokio::sync::RwLock::new(None)), + startup_time: std::time::Instant::now(), + webhook_rate_limiter: RateLimiter::new(10, 60), + active_config: Arc::new(tokio::sync::RwLock::new(Default::default())), + secrets_store: None, + db_auth: None, + pairing_store: None, + oauth_providers: None, + oauth_state_store: None, + oauth_base_url: None, + oauth_allowed_domains: Vec::new(), + near_nonce_store: None, + near_rpc_url: None, + near_network: None, + oauth_sweep_shutdown: None, + frontend_html_cache: std::sync::Arc::new(tokio::sync::RwLock::new(None)), + tool_dispatcher: None, + }); + + let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + let bound = start_server(addr, state.clone(), auth.into()) + .await + .expect("start_server"); + + (bound, state, db, temp_dir) +} + +/// Seed a sandbox job owned by `user_id` whose `project_dir` is a fresh +/// temp directory the test can also write files into. Returns the job +/// id and the project root so callers can place fixtures and probe the +/// path-traversal guard against a real on-disk layout. +async fn seed_sandbox_job(db: &Arc<dyn Database>, user_id: &str) -> (Uuid, tempfile::TempDir) { + let project_dir = tempfile::tempdir().expect("project tempdir"); + let job = SandboxJobRecord { + id: Uuid::new_v4(), + task: format!("{user_id} task"), + status: "running".to_string(), + user_id: user_id.to_string(), + project_dir: project_dir.path().to_string_lossy().into_owned(), + success: None, + failure_reason: None, + created_at: Utc::now(), + started_at: Some(Utc::now()), + completed_at: None, + credential_grants_json: "[]".to_string(), + mcp_servers: None, + max_iterations: None, + }; + let id = job.id; + db.save_sandbox_job(&job).await.expect("save sandbox job"); + (id, project_dir) +} + +async fn seed_routine(db: &Arc<dyn Database>, user_id: &str) -> Uuid { + let id = Uuid::new_v4(); + let routine = Routine { + id, + name: format!("test-routine-{id}"), + description: format!("{user_id}'s routine"), + user_id: user_id.to_string(), + enabled: true, + trigger: Trigger::Manual, + action: RoutineAction::FullJob { + title: "task".to_string(), + description: "desc".to_string(), + max_iterations: 5, + }, + guardrails: RoutineGuardrails { + cooldown: std::time::Duration::from_secs(0), + max_concurrent: 1, + dedup_window: None, + }, + notify: Default::default(), + last_run_at: None, + next_fire_at: None, + run_count: 0, + consecutive_failures: 0, + state: serde_json::json!({}), + created_at: Utc::now(), + updated_at: Utc::now(), + }; + db.create_routine(&routine).await.expect("create routine"); + let run = RoutineRun { + id: Uuid::new_v4(), + routine_id: id, + trigger_type: "manual".to_string(), + trigger_detail: None, + started_at: Utc::now(), + completed_at: None, + status: RunStatus::Running, + result_summary: Some(format!("{user_id}'s secret run summary")), + tokens_used: None, + job_id: None, + created_at: Utc::now(), + }; + db.create_routine_run(&run).await.expect("seed run"); + id +} + +// --------------------------------------------------------------------------- +// Sandbox jobs — persisted events history (`GET /api/jobs/{id}/events`) +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_job_events_for_alice_job_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let (alice_job_id, _proj) = seed_sandbox_job(&db, ALICE_USER_ID).await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{addr}/api/jobs/{alice_job_id}/events")) + .header("Authorization", format!("Bearer {BOB_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Bob requesting Alice's job events must get 404 (not 403, to prevent enumeration); body {}", + resp.text().await.unwrap_or_default() + ); +} + +#[tokio::test] +async fn alice_job_events_for_own_job_succeeds() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let (alice_job_id, _proj) = seed_sandbox_job(&db, ALICE_USER_ID).await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{addr}/api/jobs/{alice_job_id}/events")) + .header("Authorization", format!("Bearer {ALICE_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 200, "Alice must reach her own job events"); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["job_id"], alice_job_id.to_string()); +} + +// --------------------------------------------------------------------------- +// Sandbox jobs — workspace file list / read +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_job_files_list_for_alice_job_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let (alice_job_id, alice_proj) = seed_sandbox_job(&db, ALICE_USER_ID).await; + std::fs::write(alice_proj.path().join("secret.txt"), "alice's data") + .expect("seed file in alice's project"); + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{addr}/api/jobs/{alice_job_id}/files/list")) + .header("Authorization", format!("Bearer {BOB_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Bob listing files in Alice's job workspace must get 404; body {}", + resp.text().await.unwrap_or_default() + ); +} + +#[tokio::test] +async fn bob_job_file_read_for_alice_job_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let (alice_job_id, alice_proj) = seed_sandbox_job(&db, ALICE_USER_ID).await; + std::fs::write(alice_proj.path().join("secret.txt"), "alice's data") + .expect("seed file in alice's project"); + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{addr}/api/jobs/{alice_job_id}/files/read?path=secret.txt" + )) + .header("Authorization", format!("Bearer {BOB_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Bob reading a file from Alice's job workspace must get 404; body {}", + resp.text().await.unwrap_or_default() + ); +} + +#[tokio::test] +async fn alice_job_file_read_for_own_job_succeeds() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let (alice_job_id, alice_proj) = seed_sandbox_job(&db, ALICE_USER_ID).await; + let payload = "hello from alice"; + std::fs::write(alice_proj.path().join("note.txt"), payload).expect("seed note"); + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{addr}/api/jobs/{alice_job_id}/files/read?path=note.txt" + )) + .header("Authorization", format!("Bearer {ALICE_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 200, "Alice must reach her own job file"); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["content"], payload); +} + +/// Even the resource owner must not be able to escape the project +/// directory via `..`. The handler relies on `canonicalize()` + +/// `starts_with(base_canonical)` to enforce containment; this test +/// pins that guard so a refactor (e.g. switching to `clean()` or a +/// raw `join` without the starts_with check) regresses loudly. +/// +/// The directory tree is built by hand instead of via `seed_sandbox_job` +/// so the planted file lives at exactly `<project_dir>/../outside.txt`. +/// A regression that allows `..` traversal then deterministically reads +/// the planted bytes and returns 200, rather than a 404 because the +/// probe happened to point at empty space. +#[tokio::test] +async fn alice_job_file_read_rejects_dotdot_traversal() { + let (addr, _state, db, _outer) = start_server_with_db().await; + + let parent = tempfile::tempdir().expect("traversal-parent tempdir"); + let alice_proj_path = parent.path().join("alice_proj"); + std::fs::create_dir(&alice_proj_path).expect("create project dir"); + // Plant the file at exactly `<project_dir>/../outside.txt`. If + // canonicalize+starts_with regresses, the handler resolves the + // probe to this exact file and returns its content with a 200. + let outside = parent.path().join("outside.txt"); + std::fs::write(&outside, "should not be reachable").expect("plant outside file"); + + let alice_job_id = Uuid::new_v4(); + let job = SandboxJobRecord { + id: alice_job_id, + task: format!("{ALICE_USER_ID} task"), + status: "running".to_string(), + user_id: ALICE_USER_ID.to_string(), + project_dir: alice_proj_path.to_string_lossy().into_owned(), + success: None, + failure_reason: None, + created_at: Utc::now(), + started_at: Some(Utc::now()), + completed_at: None, + credential_grants_json: "[]".to_string(), + mcp_servers: None, + max_iterations: None, + }; + db.save_sandbox_job(&job).await.expect("save sandbox job"); + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{addr}/api/jobs/{alice_job_id}/files/read")) + .query(&[("path", "../outside.txt")]) + .header("Authorization", format!("Bearer {ALICE_TOKEN}")) + .send() + .await + .unwrap(); + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + assert!( + status == 403 || status == 404, + "`..` traversal must be rejected; got {status}, body {body}" + ); + assert!( + !body.contains("should not be reachable"), + "planted outside-project bytes leaked through `..` probe; body {body}" + ); +} + +// --------------------------------------------------------------------------- +// Routines — runs list (`GET /api/routines/{id}/runs`) +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_routine_runs_for_alice_routine_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_routine_id = seed_routine(&db, ALICE_USER_ID).await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{addr}/api/routines/{alice_routine_id}/runs" + )) + .header("Authorization", format!("Bearer {BOB_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Bob listing Alice's routine runs must get 404; body {}", + resp.text().await.unwrap_or_default() + ); +} + +#[tokio::test] +async fn alice_routine_runs_for_own_routine_succeeds() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_routine_id = seed_routine(&db, ALICE_USER_ID).await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{addr}/api/routines/{alice_routine_id}/runs" + )) + .header("Authorization", format!("Bearer {ALICE_TOKEN}")) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 200, + "Alice must reach her own routine runs; body {}", + resp.text().await.unwrap_or_default() + ); + let body: serde_json::Value = resp.json().await.unwrap(); + let runs = body["runs"].as_array().expect("runs array"); + assert!(!runs.is_empty(), "Alice's routine had a run seeded"); +} diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index dd2f731d031..a1fbc64fd11 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -55,7 +55,6 @@ HEADED=1 pytest scenarios/ | `test_tool_approval.py` | Approval card appears, buttons disable on approve/deny, parameters toggle via `page.evaluate("showApproval(...)")`; the waiting-approval regression uses a real HTTP tool call | | `test_extension_uninstall_cleanup.py` | Real install/setup/remove coverage for WASM tools, WASM channels, OAuth-backed shared Google tools, and MCP servers; verifies uninstall deletes stored secrets from the libSQL `secrets` table while preserving shared credentials until the last referencing extension is removed | | `test_oauth_refresh.py` | Hosted Gmail OAuth regression: complete setup via `/oauth/callback`, expire the stored access token in libSQL, trigger a real `gmail` tool call through `/api/chat/send`, and verify refresh goes through the mock `/oauth/refresh` proxy without forwarding `client_secret` | -| `test_v2_tool_activate_surface.py` | Engine-v2 prompt contract: `tool_activate` is the visible enablement tool, `tool_auth`/`tool_install` stay off the normal surfaced prompt, and blocked integrations appear under `Activatable Integrations` | | `test_dom_resource_limits.py` | DOM pruning at MAX_DOM_MESSAGES cap, no setInterval timer leaks across SSE reconnect cycles, streaming message preservation during pruning | ## `helpers.py` diff --git a/tests/e2e/E2E_DEBT.md b/tests/e2e/E2E_DEBT.md new file mode 100644 index 00000000000..a49036a5132 --- /dev/null +++ b/tests/e2e/E2E_DEBT.md @@ -0,0 +1,29 @@ +# E2E skip/xfail debt inventory + +This inventory tracks skip/xfail debt found in `tests/e2e/scenarios/` while auditing from `origin/main` for the first E2E debt campaign. + +## Summary + +| Cluster | Files | Debt type | Determinism | Recommended action | +| --- | --- | --- | --- | --- | +| ClawHub skills search/install | `test_skills.py` | Runtime `pytest.skip` when registry/search/install is unavailable or slow | Deterministic with Playwright route mocks | Fix in this campaign: mock the skills API for frontend lifecycle tests so CI never depends on ClawHub availability. | +| Gmail extension OAuth legacy flow | `test_extension_oauth.py`, `test_routine_oauth_credential_injection.py` | Runtime skips when Gmail install/auth prerequisites are absent | Mostly deterministic if migrated to existing isolated OAuth fixtures | Follow-up: consolidate with hosted OAuth fixtures used by newer v2 OAuth tests; avoid module-order globals. | +| MCP auth flow legacy flow | `test_mcp_auth_flow.py` | Runtime skips when mock MCP install/auth URL prerequisites are absent | Mostly deterministic with fixture refactor | Follow-up: convert install/auth state to fixtures instead of module-order state and make failure explicit. | +| Telegram OAuth URL placeholders | `test_oauth_url_parameters.py` | Static skip placeholders with `pass` bodies | Blocked on Telegram channel E2E fixture/product setup | Follow-up: either implement a fake Telegram channel fixture or move placeholders to issue/docs and remove dead skipped tests. | +| Portfolio widget availability | `test_portfolio.py` | Runtime skip when portfolio widget is not registered in this build | Product/build dependent | Follow-up: decide whether portfolio is a required test fixture or optional extension; if optional, keep documented skip. | +| v2 auth/OAuth matrix xfails | `test_v2_auth_oauth_matrix.py` | Static xfails for known engine-v2 contract/product gaps | Requires product behavior changes or deeper engine debug | Follow-up: split into product issues; do not silently un-xfail without matching contract changes. | +| v2 auth approval/cancel fallback | `test_v2_engine_auth_cancel.py`, `test_v2_engine_auth_flow.py` | Runtime skips when dedicated fixtures remain in approval gating rather than auth gating | Requires fixture/model prompt control | Follow-up: pin mock LLM/tool prompt path so tests reach intended auth state deterministically. | +| v2 Google OAuth binary/refresh prerequisites | `test_v2_engine_oauth_google.py` | Runtime skips when google-drive WASM binary or refresh-token prerequisite is missing | Deterministic if fixture builds/provides WASM and OAuth callback state | Follow-up: prebuild or fixture-install the WASM artifact, then make refresh tests independent. | +| Skill OAuth guided auth fallback | `test_skill_oauth_flow.py` | Runtime skip if auth flow does not trigger under current engine mode | Requires fixture/model control | Follow-up: pin engine mode and mock LLM response to force the guided auth branch. | + +## Selected cluster for this campaign + +**ClawHub skills search/install** is selected because it is the smallest high-value deterministic cluster: the tests are intended to validate the browser skills UI lifecycle, but currently depend on live ClawHub search results and network timing. The E2E README already recommends `page.route()` for tabs that depend on external data. Mocking `/api/skills`, `/api/skills/search`, `/api/skills/install`, and `DELETE /api/skills/{name}` keeps the test end-to-end at the browser/API contract layer without requiring live external services. + +Live ClawHub contract coverage belongs below the browser E2E tier: gateway/API integration tests should validate request/response shape, authentication, error handling, and registry availability separately from deterministic UI lifecycle tests. + +## Remaining debt policy + +- Runtime skips are acceptable only when the prerequisite is genuinely outside the deterministic E2E harness and the reason names that prerequisite. +- UI lifecycle tests should prefer local route mocks when they are not validating the backend integration itself. +- Placeholder skipped tests with `pass` bodies should either become real tests or move to tracked follow-up documentation/issues. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 708b608be48..cb37964be93 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -69,6 +69,11 @@ Then Playwright drives a headless Chromium browser against the gateway, making D For the live 20+ turn persona workflows and recurring tool-misuse patterns seen there, see [`LIVE_TOOL_FAILURES.md`](./LIVE_TOOL_FAILURES.md). +## Skip/xfail debt + +For the current inventory of E2E skips/xfails and the policy for keeping browser +lifecycle tests deterministic, see [`E2E_DEBT.md`](./E2E_DEBT.md). + ## Mocking API responses with `page.route()` For tabs that depend on external data (extensions, jobs, memory, routines), use diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 367dac9b64a..3694a92eda0 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -13,6 +13,7 @@ import sys import tempfile from pathlib import Path +from typing import Any import httpx import pytest @@ -505,6 +506,9 @@ async def ironclaw_server( "IRONCLAW_OAUTH_EXCHANGE_URL": mock_llm_server, } _forward_coverage_env(env) + env["SECRETS_MASTER_KEY"] = ( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ) proc = await asyncio.create_subprocess_exec( ironclaw_binary, "--no-onboard", stdin=asyncio.subprocess.DEVNULL, @@ -1549,3 +1553,168 @@ async def telegram_e2e_server_with_routines( routines_enabled=True, ): yield server + + +# ── Mission auto-resume (#3133 / #3166) ────────────────────────────────── + + +@pytest.fixture +async def mission_gmail_live_server( + ironclaw_binary, + mock_llm_server, + wasm_tools_dir, + request, +): + """Isolated ironclaw instance for the mission auto-resume live test. + + Wires together three runtime pieces: + + 1. The `live_llm_proxy.py` record/replay proxy as ironclaw's + LLM backend. The proxy is launched with a per-test fixture + file in `tests/e2e/fixtures/live/<test_name>.json`. In record + mode (`IRONCLAW_LIVE_TEST=1`) it forwards to the upstream + LLM and writes the trace; in replay mode it reads the + committed trace. + 2. The existing mock_llm.py for the Gmail HTTP API mocks. The + `IRONCLAW_TEST_HTTP_REWRITE_MAP` env var routes + `gmail.googleapis.com` at mock_llm.py so the gmail WASM + tool's HTTP calls land on deterministic responses (the + `/__mock/gmail/state` endpoint exposes counters for + assertions). OAuth exchange (`/oauth/exchange`) also lives + on mock_llm.py. + 3. A function-scoped ironclaw process with engine v2 enabled + and admin tools auto-approved so the chat-driven + `routine_create` + `mission_fire` flow runs without a + human in the loop. The *authentication* gate raised by + the auth preflight on a direct `gmail` call is still active + even with auto-approve — that's the gate the #3133 + auto-resume path exercises. + """ + from live_harness import start_live_proxy + + proxy_iter = start_live_proxy(request.node.name) + proxy = await proxy_iter.__anext__() + + reserved = _reserve_loopback_sockets(2) + db_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-e2e-mission-gmail-db-") + home_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-e2e-mission-gmail-home-") + channels_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-e2e-mission-gmail-channels-") + tools_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-e2e-mission-gmail-tools-") + + try: + gateway_port = reserved[0].getsockname()[1] + http_port = reserved[1].getsockname()[1] + for sock in reserved: + if sock.fileno() != -1: + sock.close() + + # Override LLM base URL to the live proxy. _build_gateway_env + # bakes in the LLM_BASE_URL from `mock_llm_server`; we override + # via extra_env after building the rest of the env block. + env = _build_gateway_env( + mock_llm_server=mock_llm_server, + wasm_tools_dir=tools_tmpdir.name, + home_dir=home_tmpdir.name, + gateway_port=gateway_port, + http_port=http_port, + db_path=os.path.join(db_tmpdir.name, "mission-gmail-live.db"), + extra_env={ + "SECRETS_MASTER_KEY": ( + "0123456789abcdef0123456789abcdef" + "0123456789abcdef0123456789abcdef" + ), + "WASM_CHANNELS_DIR": channels_tmpdir.name, + # Route LLM through the live proxy. + "LLM_BASE_URL": proxy["url"], + # Route gmail.googleapis.com at mock_llm so the + # gmail WASM tool's HTTP calls land deterministically. + "IRONCLAW_TEST_HTTP_REWRITE_MAP": json.dumps( + {"gmail.googleapis.com": mock_llm_server} + ), + "IRONCLAW_OAUTH_PROXY_ALLOW_LOOPBACK": "1", + "GOOGLE_OAUTH_CLIENT_ID": "hosted-google-client-id", + # Auto-approve administrative tools so the chat-driven + # mission_create + fire sequence runs without a human + # approval prompt. Authentication gates remain active. + "AGENT_AUTO_APPROVE_TOOLS": "true", + # Engine v2 is required: only v2 missions carry the + # `paused_gate` field half-2 keys off of. + "ENGINE_V2": "true", + }, + ) + + # Tee ironclaw stderr to a debug log so live-test failures are + # diagnosable without re-running. Only used during /tests/e2e + # live recordings — production paths don't see this. + stderr_log_path = os.environ.get("IRONCLAW_E2E_STDERR_LOG") + stderr_dest: Any = asyncio.subprocess.PIPE + if stderr_log_path: + stderr_dest = open(stderr_log_path, "w") # noqa: SIM115 + proc = await asyncio.create_subprocess_exec( + ironclaw_binary, "--no-onboard", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=stderr_dest, + env=env, + ) + startup_kill_attempted = False + base_url = f"http://127.0.0.1:{gateway_port}" + try: + await wait_for_ready(f"{base_url}/api/health", timeout=60) + yield { + "base_url": base_url, + "mock_llm_url": mock_llm_server, + "live_proxy_url": proxy["url"], + "fixture": str(proxy["fixture"]), + "mode": proxy["mode"], + } + except TimeoutError: + if proc.returncode is None: + startup_kill_attempted = True + await _stop_process(proc, timeout=2) + returncode = proc.returncode + stderr_bytes = b"" + if proc.stderr: + try: + stderr_bytes = await asyncio.wait_for( + proc.stderr.read(8192), timeout=2 + ) + except asyncio.TimeoutError: + pass + stderr_text = stderr_bytes.decode("utf-8", errors="replace") + pytest.fail( + f"mission_gmail_live_server failed to start on port " + f"{gateway_port} (returncode={returncode}).\n" + f"stderr:\n{stderr_text}" + ) + finally: + if proc.returncode is None: + if startup_kill_attempted: + await _stop_process(proc, timeout=2) + else: + await _stop_process(proc, sig=signal.SIGINT, timeout=10) + if proc.returncode is None: + await _stop_process(proc, timeout=2) + finally: + # Tear the proxy down too. + try: + await proxy_iter.__anext__() + except StopAsyncIteration: + pass + for sock in reserved: + if sock.fileno() != -1: + sock.close() + db_tmpdir.cleanup() + home_tmpdir.cleanup() + channels_tmpdir.cleanup() + tools_tmpdir.cleanup() + + +@pytest.fixture +async def mission_gmail_live_page(mission_gmail_live_server, browser): + """Browser page bound to the mission_gmail_live_server fixture.""" + context = await browser.new_context(viewport={"width": 1280, "height": 720}) + pg = await context.new_page() + await _open_authed_gateway_page(pg, mission_gmail_live_server["base_url"]) + yield pg + await context.close() diff --git a/tests/e2e/fixtures/live/.gitkeep b/tests/e2e/fixtures/live/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/fixtures/live/test_mission_gmail_draft_3133.json b/tests/e2e/fixtures/live/test_mission_gmail_draft_3133.json new file mode 100644 index 00000000000..78b124ee970 --- /dev/null +++ b/tests/e2e/fixtures/live/test_mission_gmail_draft_3133.json @@ -0,0 +1,558 @@ +{ + "model": "mock-model", + "schema_version": 1, + "entries": [ + { + "request_hash": "8b0594268d8163bcd1adf3f5268990be03bbde10e953aff35a6f58a709737b43", + "request_summary": { + "model": "mock-model", + "n_messages": 2, + "last_user_content": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name = 'gmail-draft-3133'\n goal = 'Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.'\n cadence = cron expression '*/3 * * * *'\nThen immediately use mission_fire to trigger it once. Do not call tool_list, tool_info, tool_activate, or any other tool before mission_create. Just create the mission and fire it." + } + ] + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"name\": \"gmail-draft-3133\", \"goal\": \"Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \\\"Test mission #3133\\\" and body \\\"Mock draft from the IronClaw e2e test.\\\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\", \"cadence\": \"*/3 * * * *\"}", + "name": "mission_create" + }, + "id": "call_529804722287442abf4af8c4", + "type": "function" + }, + { + "function": { + "arguments": "{\"name\": \"gmail-draft-3133\"}", + "name": "mission_fire" + }, + "id": "call_0c3ac26a8b324127a2cee4a5", + "type": "function" + } + ] + } + } + ], + "created": 1778219609, + "id": "chatcmpl-9d694b797f3acd72", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 332, + "prompt_tokens": 15249, + "total_tokens": 15581 + } + } + }, + { + "request_hash": "17d5a286b3e7533ecdd4209841290909381960a576304ad294d6bc8a596a4bd2", + "request_summary": { + "model": "mock-model", + "n_messages": 5, + "last_user_content": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name = 'gmail-draft-3133'\n goal = 'Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.'\n cadence = cron expression '*/3 * * * *'\nThen immediately use mission_fire to trigger it once. Do not call tool_list, tool_info, tool_activate, or any other tool before mission_create. Just create the mission and fire it." + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "\n\n" + } + ], + "tool_calls": [ + { + "type": "function", + "function": { + "name": "mission_create", + "arguments": "{\"cadence\":\"*/3 * * * *\",\"goal\":\"Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \\\"Test mission #3133\\\" and body \\\"Mock draft from the IronClaw e2e test.\\\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\",\"name\":\"gmail-draft-3133\"}" + } + }, + { + "type": "function", + "function": { + "name": "mission_fire", + "arguments": "{\"name\":\"gmail-draft-3133\"}" + } + } + ] + }, + { + "role": "tool", + "content": "{'mission_id': '<uuid>', 'name': 'gmail-draft-3133', 'status': 'created'}" + }, + { + "role": "tool", + "content": "[ACTION FAILED] mission_fire: {'error': 'effect execution error: mission not found by name: tried [\"gmail-draft-3133\"]. Use mission_list to see available missions.'}" + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\nThe mission was created successfully with ID `f19887ba-34a6-4e93-98da-6d8d3394112f`, but the `mission_fire` call failed because it couldn't find the mission by name immediately after creation. Let me try firing it using the mission ID instead:\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"id\": \"f19887ba-34a6-4e93-98da-6d8d3394112f\"}", + "name": "mission_fire" + }, + "id": "call_fb552114d76c407581db9baf", + "type": "function" + } + ] + } + } + ], + "created": 1778219616, + "id": "chatcmpl-ac1de833db95dc10", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 137, + "prompt_tokens": 15536, + "prompt_tokens_details": { + "cached_tokens": 14672 + }, + "total_tokens": 15673 + } + } + }, + { + "request_hash": "dc61373fb9af3b80914e85077785cd49e9de6c5dd150721ec4d00401e1e672e1", + "request_summary": { + "model": "mock-model", + "n_messages": 7, + "last_user_content": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Use the mission_create tool to create a mission with these exact parameters and DO NOT run any other tools first:\n name = 'gmail-draft-3133'\n goal = 'Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.'\n cadence = cron expression '*/3 * * * *'\nThen immediately use mission_fire to trigger it once. Do not call tool_list, tool_info, tool_activate, or any other tool before mission_create. Just create the mission and fire it." + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "\n\n" + } + ], + "tool_calls": [ + { + "type": "function", + "function": { + "name": "mission_create", + "arguments": "{\"cadence\":\"*/3 * * * *\",\"goal\":\"Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \\\"Test mission #3133\\\" and body \\\"Mock draft from the IronClaw e2e test.\\\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\",\"name\":\"gmail-draft-3133\"}" + } + }, + { + "type": "function", + "function": { + "name": "mission_fire", + "arguments": "{\"name\":\"gmail-draft-3133\"}" + } + } + ] + }, + { + "role": "tool", + "content": "{'mission_id': '<uuid>', 'name': 'gmail-draft-3133', 'status': 'created'}" + }, + { + "role": "tool", + "content": "[ACTION FAILED] mission_fire: {'error': 'effect execution error: mission not found by name: tried [\"gmail-draft-3133\"]. Use mission_list to see available missions.'}" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "\n\nThe mission was created successfully with ID `<uuid>`, but the `mission_fire` call failed because it couldn't find the mission by name immediately after creation. Let me try firing it using the mission ID instead:\n\n" + } + ], + "tool_calls": [ + { + "type": "function", + "function": { + "name": "mission_fire", + "arguments": "{\"id\":\"<uuid>\"}" + } + } + ] + }, + { + "role": "tool", + "content": "{'status': 'fired', 'thread_id': '<uuid>'}" + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\nThe mission has been successfully created and fired:\n\n- **Mission created**: `gmail-draft-3133` with ID `f19887ba-34a6-4e93-98da-6d8d3394112f`\n- **Cadence**: `*/3 * * * *` (runs every 3 minutes)\n- **Mission fired**: Triggered a one-time execution with thread ID `9c67e951-a968-4b50-9d64-30f6062f279f`\n\nThe mission will now execute the Gmail draft creation as specified in its goal. The initial `mission_fire` by name failed because the mission wasn't immediately indexed by name, but firing by ID worked successfully.", + "role": "assistant" + } + } + ], + "created": 1778219618, + "id": "chatcmpl-b0f987f5b7c39231", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 181, + "prompt_tokens": 15736, + "total_tokens": 15917 + } + } + }, + { + "request_hash": "9bd54aee4f8ff02b0e33fe5a818197126723960e05667ff742506ddd676a81a4", + "request_summary": { + "model": "mock-model", + "n_messages": 2, + "last_user_content": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\n\n## Current Focus\nThis is the first run. Start by understanding the goal and determining the first step.\n\n## Knowledge from Prior Threads\n[SKILLS]\n\nThis is thread #1 for this mission.\n\n## Instructions\nBased on the above context, take the next step toward the goal. Use tools to gather information, analyze data, or take actions. When done, call FINAL() with your response. Include:\n1. What you accomplished in this step\n2. What the next focus should be (for the next thread)\n3. Whether the goal has been achieved (yes/no)" + } + ] + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"detail\": \"schema\", \"include_schema\": true, \"name\": \"gmail\"}", + "name": "tool_info" + }, + "id": "call_1b59317beb8a4f6c948d48eb", + "type": "function" + } + ] + } + } + ], + "created": 1778219620, + "id": "chatcmpl-87b2f730c41a85dd", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 91, + "prompt_tokens": 17722, + "prompt_tokens_details": { + "cached_tokens": 6288 + }, + "total_tokens": 17813 + } + } + }, + { + "request_hash": "c60635dffe9950e95fdc7b7e7d9f446954a0a804b0a2a65d0edda2d41235b27f", + "request_summary": { + "model": "mock-model", + "n_messages": 4, + "last_user_content": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\n\n## Current Focus\nThis is the first run. Start by understanding the goal and determining the first step.\n\n## Knowledge from Prior Threads\n[SKILLS]\n\nThis is thread #1 for this mission.\n\n## Instructions\nBased on the above context, take the next step toward the goal. Use tools to gather information, analyze data, or take actions. When done, call FINAL() with your response. Include:\n1. What you accomplished in this step\n2. What the next focus should be (for the next thread)\n3. Whether the goal has been achieved (yes/no)" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "\n\n" + } + ], + "tool_calls": [ + { + "type": "function", + "function": { + "name": "tool_info", + "arguments": "{\"detail\":\"schema\",\"include_schema\":true,\"name\":\"gmail\"}" + } + } + ] + }, + { + "role": "tool", + "content": "{'description': 'Read, search, send, draft, and reply to emails via Gmail. Supports Gmail search query syntax (is:unread, from:, subject:, after:, etc.).', 'name': 'gmail', 'parameters': ['action', 'bcc', 'body', 'cc', 'label_ids', 'max_results', 'message_id', 'query', 'reply_all', 'subject', 'to'], 'schema': {'properties': {'action': {'description': 'The Gmail operation to perform', 'enum': ['list_messages', 'get_message', 'send_message', 'create_draft', 'reply_to_message', 'trash_message'], 'type': 'string'}, 'bcc': {'description': 'BCC recipients, comma-separated. Used by: send_message, create_draft', 'type': 'string'}, 'body': {'description': 'Email body (plain text). Required for: send_message, create_draft, reply_to_message', 'type': 'string'}, 'cc': {'description': 'CC recipients, comma-separated. Used by: send_message, create_draft', 'type': 'string'}, 'label_ids': {'description': \"Label IDs to filter by (e.g., 'INBOX', 'SENT', 'DRAFT'). Used by: list_messages\", 'items': {'type': 'string'}, 'type': 'array'}, 'max_results': {'default': 20, 'description': 'Maximum number of messages to return (default: 20). Used by: list_messages', 'type': 'integer'}, 'message_id': {'description': 'Message ID. Required for: get_message, reply_to_message, trash_message', 'type': 'string'}, 'query': {'description': \"Gmail search query (same syntax as Gmail search box, e.g., 'is:unread', 'from:alice@example.com'). Used by: list_messages\", 'type': 'string'}, 'reply_all': {'default': False, 'description': 'If true, reply to all recipients (default: false). Used by: reply_to_message', 'type': 'boolean'}, 'subject': {'description': 'Email subject. Required for: send_message, create_draft', 'type': 'string'}, 'to': {'description': 'Recipient email address(es), comma-separated. Required for: send_message, create_draft', 'type': 'string'}}, 'required': ['action'], 'type': 'object'}}" + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"action\": \"create_draft\", \"body\": \"Mock draft from the IronClaw e2e test.\", \"subject\": \"Test mission #3133\", \"to\": \"owner@example.com\"}", + "name": "gmail" + }, + "id": "call_0c6030b6b0cb41ea8e25174b", + "type": "function" + } + ] + } + } + ], + "created": 1778219624, + "id": "chatcmpl-91b03ec249e4389d", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 86, + "prompt_tokens": 18287, + "prompt_tokens_details": { + "cached_tokens": 16768 + }, + "total_tokens": 18373 + } + } + }, + { + "request_hash": "96f7b9aeb1e15f3d3d61f58782be9030c7470b2c727bd76f2e16b3f7250c0d8d", + "request_summary": { + "model": "mock-model", + "n_messages": 2, + "last_user_content": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with", + "tool_count": 16 + }, + "request_canonical": { + "model": "mock-model", + "tail": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "# Mission: gmail-draft-3133\n\nGoal: Use the gmail tool with action=create_draft to send a draft to owner@example.com with subject \"Test mission #3133\" and body \"Mock draft from the IronClaw e2e test.\". Just call the gmail tool directly — the runtime handles authentication. Do NOT call tool_activate, tool_install, or any other setup tool first.\n\n## Previous Approaches\n1. PAUSED: gate 'authentication' on action 'gmail' awaiting authentication\n\n## Knowledge from Prior Threads\n[SKILLS]\n\nThis is thread #2 for this mission.\n\n## Instructions\nBased on the above context, take the next step toward the goal. Use tools to gather information, analyze data, or take actions. When done, call FINAL() with your response. Include:\n1. What you accomplished in this step\n2. What the next focus should be (for the next thread)\n3. Whether the goal has been achieved (yes/no)" + } + ] + } + ], + "tools": [ + "echo", + "http", + "json", + "memory_read", + "memory_search", + "memory_tree", + "memory_write", + "skill_install", + "skill_list", + "skill_search", + "time", + "tool_info", + "tool_list", + "tool_search", + "tool_upgrade" + ] + }, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "content": "\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"action\": \"create_draft\", \"to\": \"owner@example.com\", \"subject\": \"Test mission #3133\", \"body\": \"Mock draft from the IronClaw e2e test.\"}", + "name": "gmail" + }, + "id": "call_66394a8fadd141c59b23016f", + "type": "function" + } + ] + } + } + ], + "created": 1778219626, + "id": "chatcmpl-8e9d08c27ca19920", + "model": "Qwen/Qwen3.5-122B-A10B", + "object": "chat.completion", + "usage": { + "completion_tokens": 114, + "prompt_tokens": 17722, + "prompt_tokens_details": { + "cached_tokens": 6288 + }, + "total_tokens": 17836 + } + } + } + ] +} diff --git a/tests/e2e/helpers.py b/tests/e2e/helpers.py index 2ffa0dfc9f3..3e6f2a5cedf 100644 --- a/tests/e2e/helpers.py +++ b/tests/e2e/helpers.py @@ -382,17 +382,36 @@ async def send_chat_and_wait_for_terminal_message( - ``text``: rendered text of the newest terminal message The default predicate waits for the assistant message to fully settle — - ``data-streaming`` attribute cleared AND input re-enabled. On slow CI - runners under heavy parallelism that compound condition can race with - SSE reconnects (chunks arrive during the reconnect window, the - attribute-clearing delta is lost, predicate never flips). Callers that - already assert on specific response text can pass - ``expected_text_contains=`` to short-circuit as soon as that substring - appears in the assistant bubble. The test's own content assertion is - the correctness gate, not the streaming-attribute flag. + ``data-streaming`` attribute cleared, input re-enabled, and this send's + optimistic pending marker removed. On slow CI runners under heavy + parallelism the streaming condition can race with SSE reconnects (chunks + arrive during the reconnect window, the attribute-clearing delta is lost, + predicate never flips). Callers that already assert on specific response + text can pass ``expected_text_contains=`` to short-circuit as soon as that + substring appears in the assistant bubble, but the helper still waits for + this send's pending marker to clear so late history renders from previous + turns cannot be mistaken for the new response. """ chat_input = await ensure_writable_chat_input(page) + # The page fixture waits for auth/SSE readiness, but the first + # loadHistory() call may still be replacing the skeleton with older + # persisted messages. Count terminal bubbles only after that settles; + # otherwise a late history-rendered assistant from a previous turn can + # satisfy the "new assistant" predicate for the message we are about to + # send. + await page.wait_for_function( + """() => !document.querySelector('#chat-messages .skeleton-container')""", + timeout=timeout, + ) + + pending_marker = await page.evaluate( + """() => ({ + threadId: typeof currentThreadId !== 'undefined' ? currentThreadId : null, + pendingId: typeof _nextPendingId !== 'undefined' ? _nextPendingId : null, + })""" + ) + assistant_sel = SEL["message_assistant"] system_sel = SEL["message_system"] before_assistant = await page.locator(assistant_sel).count() @@ -409,7 +428,16 @@ async def send_chat_and_wait_for_terminal_message( assistantCount, systemCount, expectedContains, + pendingThreadId, + pendingId, }) => { + const pendingForThisSendCleared = () => { + if (!pendingThreadId || pendingId === null || pendingId === undefined) return true; + if (typeof _pendingUserMessages === 'undefined') return true; + const pending = _pendingUserMessages.get(pendingThreadId); + return !pending || !pending.some((p) => p.id === pendingId); + }; + const input = document.querySelector(chatInputSelector); const systems = document.querySelectorAll(systemSelector); if (systems.length > systemCount) { @@ -426,7 +454,7 @@ async def send_chat_and_wait_for_terminal_message( const last = assistants[assistants.length - 1]; const content = last.querySelector('.message-content'); const text = ((content && content.innerText) || last.innerText || '').trim(); - if (text.length > 0) { + if (text.length > 0 && pendingForThisSendCleared()) { if (expectedContains && text.includes(expectedContains)) { return { role: 'assistant', text }; } @@ -447,6 +475,8 @@ async def send_chat_and_wait_for_terminal_message( "assistantCount": before_assistant, "systemCount": before_system, "expectedContains": expected_text_contains, + "pendingThreadId": pending_marker["threadId"], + "pendingId": pending_marker["pendingId"], }, timeout=timeout, ) diff --git a/tests/e2e/live_harness.py b/tests/e2e/live_harness.py new file mode 100644 index 00000000000..080a85d0278 --- /dev/null +++ b/tests/e2e/live_harness.py @@ -0,0 +1,169 @@ +"""Helpers for live-LLM Playwright tests. + +Mirrors the Rust `LiveTestHarnessBuilder` pattern (see +`tests/support/live_harness.rs`). Spins up the `live_llm_proxy.py` +record/replay proxy in front of a real or recorded LLM, points an +ironclaw instance at it, and lets a Playwright test drive the chat +flow against deterministic LLM output. + +Modes +----- + +- **Record** (`IRONCLAW_LIVE_TEST=1`): the proxy forwards + `/v1/chat/completions` to the upstream LLM whose URL/key/model are + configured via `IRONCLAW_LIVE_LLM_BASE_URL`, `IRONCLAW_LIVE_LLM_API_KEY`, + `IRONCLAW_LIVE_LLM_MODEL`. Each prompt+response pair is appended to + the test's fixture file. + +- **Replay** (default): the proxy reads the committed fixture and + serves recorded responses by canonical-request hash. Tests are + skipped (with a clear message) when the fixture is missing so a + fresh checkout doesn't hard-fail before someone has recorded one. + +Fixture path convention +----------------------- + +Per-test fixtures live at:: + + tests/e2e/fixtures/live/<test_name>.json + +where ``<test_name>`` is the bare test function name (no module +prefix). The convention matches the Rust live harness which keys on +the `#[tokio::test]` function name. +""" + +from __future__ import annotations + +import asyncio +import os +import re +import signal +import sys +from pathlib import Path +from typing import Any, AsyncIterator + +import httpx +import pytest + + +HERE = Path(__file__).resolve().parent +FIXTURE_DIR = HERE / "fixtures" / "live" +PROXY_SCRIPT = HERE / "live_llm_proxy.py" + + +def is_live_mode() -> bool: + """True when the test should record a fresh trace from a real LLM.""" + return os.environ.get("IRONCLAW_LIVE_TEST", "").strip() in ("1", "true") + + +def fixture_path_for(test_name: str) -> Path: + """Return the JSON fixture path for a given test.""" + return FIXTURE_DIR / f"{test_name}.json" + + +async def _wait_for_port_line( + process: asyncio.subprocess.Process, pattern: str, *, timeout: float = 10.0 +) -> int: + """Read the proxy's stdout until `pattern` matches (and capture group 1).""" + deadline = asyncio.get_event_loop().time() + timeout + rx = re.compile(pattern) + assert process.stdout is not None + while asyncio.get_event_loop().time() < deadline: + line = await asyncio.wait_for(process.stdout.readline(), timeout=timeout) + if not line: + raise RuntimeError( + f"live_llm_proxy exited before emitting {pattern!r}; " + f"stderr (truncated): {(await process.stderr.read(2000)).decode()}" + ) + decoded = line.decode("utf-8", errors="replace").strip() + m = rx.search(decoded) + if m: + return int(m.group(1)) + raise asyncio.TimeoutError(f"live_llm_proxy did not emit {pattern!r} in {timeout}s") + + +async def start_live_proxy( + test_name: str, + *, + record_required: bool = False, +) -> AsyncIterator[dict[str, Any]]: + """Async generator: spin up the proxy and yield ``{"url": ..., "fixture": + ..., "mode": ...}``. The caller supplies ``test_name`` from + ``request.node.name``. + + In replay mode with no committed fixture, raises ``pytest.skip`` so + a fresh checkout does not hard-fail. + """ + fixture = fixture_path_for(test_name) + mode = "record" if is_live_mode() else "replay" + + if mode == "replay" and not fixture.exists(): + pytest.skip( + f"no live-LLM trace fixture at {fixture.relative_to(HERE.parent.parent)}. " + f"To record one, set IRONCLAW_LIVE_TEST=1 plus IRONCLAW_LIVE_LLM_BASE_URL / " + f"IRONCLAW_LIVE_LLM_API_KEY / IRONCLAW_LIVE_LLM_MODEL and re-run." + ) + + if record_required and mode != "record": + pytest.skip( + "this test must run in record mode (IRONCLAW_LIVE_TEST=1)" + ) + + if mode == "record": + if not os.environ.get("IRONCLAW_LIVE_LLM_BASE_URL"): + pytest.skip( + "record mode requires IRONCLAW_LIVE_LLM_BASE_URL " + "(and usually IRONCLAW_LIVE_LLM_API_KEY / IRONCLAW_LIVE_LLM_MODEL)" + ) + + proxy_stderr_log = os.environ.get("IRONCLAW_LIVE_PROXY_STDERR_LOG") + proxy_stderr: Any = asyncio.subprocess.PIPE + if proxy_stderr_log: + proxy_stderr = open(proxy_stderr_log, "w") # noqa: SIM115 + proc = await asyncio.create_subprocess_exec( + sys.executable, + str(PROXY_SCRIPT), + "--port", + "0", + "--fixture", + str(fixture), + "--mode", + mode, + stdout=asyncio.subprocess.PIPE, + stderr=proxy_stderr, + env={**os.environ}, + ) + try: + port = await _wait_for_port_line(proc, r"LIVE_LLM_PROXY_PORT=(\d+)", timeout=15) + url = f"http://127.0.0.1:{port}" + # Wait for the /v1/models endpoint to come up. + deadline = asyncio.get_event_loop().time() + 10 + while asyncio.get_event_loop().time() < deadline: + try: + async with httpx.AsyncClient() as client: + r = await client.get(f"{url}/v1/models", timeout=2) + if r.status_code == 200: + break + except Exception: + pass + await asyncio.sleep(0.1) + yield { + "url": url, + "fixture": fixture, + "mode": mode, + } + finally: + if proc.returncode is None: + try: + proc.send_signal(signal.SIGINT) + await asyncio.wait_for(proc.wait(), timeout=5) + except (asyncio.TimeoutError, ProcessLookupError): + proc.kill() + + +async def proxy_state(url: str) -> dict[str, Any]: + """Read the proxy's runtime state (entry counts, mode, miss count).""" + async with httpx.AsyncClient() as client: + response = await client.get(f"{url}/__live/state", timeout=10) + response.raise_for_status() + return response.json() diff --git a/tests/e2e/live_llm_proxy.py b/tests/e2e/live_llm_proxy.py new file mode 100644 index 00000000000..693e494a992 --- /dev/null +++ b/tests/e2e/live_llm_proxy.py @@ -0,0 +1,639 @@ +"""Record/replay HTTP proxy for live-LLM Playwright tests. + +This is the Python tier's analogue to the Rust `LiveTestHarnessBuilder` +trace-recording infrastructure (`tests/support/live_harness.rs`). It +sits between ironclaw and a real LLM (NearAI / OpenAI / Anthropic) and: + +- In **record** mode, forwards each `/v1/chat/completions` request to + the upstream LLM, captures the request + response pair, and appends + it to a JSON fixture file. The committed fixture lets later runs + replay the conversation deterministically without an LLM API key. + +- In **replay** mode, reads the fixture and returns recorded responses + by matching the canonical request shape (model + tools + message + sequence). Matching is structural: it ignores non-deterministic + fields like tool-call IDs, request IDs, and timestamps. + +Usage in a test: + + # tests/e2e/conftest.py + @pytest.fixture + async def live_llm_proxy(request): + from live_harness import live_proxy_for + async for url in live_proxy_for(request.node.name): + yield url + + # ironclaw_server fixture sets: + # LLM_BASE_URL = url + # The proxy auto-detects record vs replay based on + # IRONCLAW_LIVE_TEST and the fixture file's existence. + +Environment variables: + +- ``IRONCLAW_LIVE_TEST=1`` — record mode. Requires upstream LLM + credentials (``IRONCLAW_LIVE_LLM_BASE_URL``, ``IRONCLAW_LIVE_LLM_API_KEY``, + ``IRONCLAW_LIVE_LLM_MODEL``). Writes / overwrites the fixture file. +- (unset) — replay mode. Reads the committed fixture file. Skips + the test (with ``pytest.skip``) when the fixture is missing so a + fresh checkout doesn't fail before someone has recorded one. + +Fixture file shape (JSON): + + { + "model": "<recorded model id>", + "entries": [ + { + "request_hash": "<sha256 of canonicalized request>", + "request_summary": { + "model": "...", + "n_messages": <int>, + "last_user_content": "<truncated>", + "tool_count": <int> + }, + "response": { ... full /v1/chat/completions JSON ... } + }, + ... + ] + } + +Matching uses request_hash. Multiple identical requests produce +multiple entries (each with the same hash); replay consumes them in +order. +""" + +import argparse +import asyncio +import hashlib +import json +import os +import re +import sys +import time +import uuid +from pathlib import Path +from typing import Any + +import aiohttp +from aiohttp import web + + +# ── Canonicalization ──────────────────────────────────────────────────── + + +_TOOL_CALL_ID_RE = re.compile(r"call_[A-Za-z0-9_-]{8,}") +# UUIDs (project_id, thread_id, mission_id, etc.) are dynamic per run. +# Strip them so hashing is stable across recordings. +_UUID_RE = re.compile( + r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", + re.IGNORECASE, +) +# RFC 3339 timestamps embedded in system prompts / tool results. +_TS_RE = re.compile( + r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?" +) + + +def _strip_dynamic(text: str) -> str: + """Strip run-to-run non-determinism from a string for hashing. + + Replaces UUIDs, tool-call ids, and timestamps with placeholders + and elides the skills section (whose order varies because the + skill registry iterates a HashMap, and whose content varies as + skills are added/removed from the registry) so two semantically- + identical requests produce the same hash regardless of run-to-run + variation. + """ + text = _TOOL_CALL_ID_RE.sub("call_<id>", text) + text = _UUID_RE.sub("<uuid>", text) + text = _TS_RE.sub("<ts>", text) + text = _normalize_skills_block(text) + text = _normalize_mission_list_result(text) + return text + + +_SKILL_MARKER_RE = re.compile(r"(?:^|\n)(?:### )?\[SKILL\] skill:([A-Za-z0-9_\-]+)") +# End-of-skills boundaries that appear in real prompts. We can't use a +# generic `## ` regex because skill bodies frequently contain markdown +# `## ` headers — those would falsely terminate the skills section. +_SKILLS_END_BOUNDARIES = ( + "\nThis is thread #", + "\n## Instructions\n", + "\n## Available Actions\n", + "\n## Tools Available\n", +) + + +def _normalize_skills_block(text: str) -> str: + """Drop the entire `[SKILL] skill:NAME ...` block, replacing it + with a single `[SKILLS]` placeholder. + + Skills appear in two forms: prefixed with `### ` (system prompt + style) or bare (rendered into the user-facing mission goal). The + full body of each skill varies as the registry adds/edits/removes + entries between recordings. The local skill registry is also + machine-specific, so even the *set* of active skill names cannot + be assumed stable across record/replay machines. + + For canonicalization we therefore drop the whole block — the + deterministic test prompt is engineered so the LLM's response + does not branch on which skills are present. The resulting + placeholder is intentionally opaque (no embedded names). + + Strategy: locate the first `[SKILL] skill:NAME` marker, find the + end of the skills section (next known top-level boundary or end + of string), and replace the entire range with `[SKILLS]`. + """ + matches = list(_SKILL_MARKER_RE.finditer(text)) + if not matches: + return text + head_end = matches[0].start() + # Find where the skills section ends. Search after the last match + # for a known top-level boundary; if none, the section runs to EOF. + last_match_end = matches[-1].end() + tail_start = len(text) + for boundary in _SKILLS_END_BOUNDARIES: + idx = text.find(boundary, last_match_end) + if idx != -1 and idx < tail_start: + tail_start = idx + head = text[:head_end] + tail = text[tail_start:] if tail_start < len(text) else "" + # Drop the entire skills block from the canonical form. The skill + # registry is local-machine state (skills can be installed/removed + # at any time) so the *set* of active skills cannot be assumed + # stable across record/replay machines. The deterministic test + # prompt is engineered so the LLM's response does not branch on + # which skills are present. + return f"{head}\n[SKILLS]\n{tail}" + + +_MISSION_LIST_RE = re.compile(r"\[\{'cadence':.*?\}\](?=\n|$|]|,)", re.DOTALL) + + +def _normalize_mission_list_result(text: str) -> str: + """Collapse mission_list tool results to a stable shape. + + The mission_list tool returns full mission rows with descriptions + that contain non-deterministic content (system seed missions can + be added/reordered between runs). For canonicalization we only + care about the names. With the deterministic sort applied in + `list_missions_with_shared`, a stable repr appears in the + fixture; this helper protects against past recordings whose + capture predates the sort. + """ + return text # No-op; sort in store_adapter.rs handles ordering now. + + +def _canonicalize_request(body: dict[str, Any]) -> dict[str, Any]: + """Build a stable hash key for a chat-completions request. + + The full system prompt isn't hashed because it varies run-to-run + (skills loaded in HashMap order, embedded UUIDs/timestamps, etc.) + while the LLM's response selection is driven by a much smaller + set of stable inputs: + + - The model id (selects the response shape). + - The conversation tail: roles + payloads of the last few + non-system messages, with UUIDs/timestamps/tool-call-ids + stripped. This captures "what is the LLM being asked, given + what it just did". + - The set of tool names exposed (function calls fall through to + hash-based dispatch). + + Two semantically-identical requests (same conversation tail, + same tool surface) produce the same hash regardless of system- + prompt drift. + """ + canon: dict[str, Any] = { + "model": body.get("model"), + "tail": [], + } + # Walk all non-system messages; stable tail captures the + # conversation state. System prompts vary too much to hash. + for msg in body.get("messages", []) or []: + role = msg.get("role") + if role == "system": + continue + content = msg.get("content") + if isinstance(content, str): + content = _strip_dynamic(content) + elif isinstance(content, list): + new_parts = [] + for part in content: + if not isinstance(part, dict): + new_parts.append(part) + continue + p = dict(part) + if "text" in p and isinstance(p["text"], str): + p["text"] = _strip_dynamic(p["text"]) + new_parts.append(p) + content = new_parts + norm = {"role": role, "content": content} + if "name" in msg: + norm["name"] = msg["name"] + if "tool_calls" in msg: + calls = [] + for tc in msg.get("tool_calls", []) or []: + args = (tc.get("function") or {}).get("arguments") + if isinstance(args, str): + args = _strip_dynamic(args) + calls.append({ + "type": tc.get("type"), + "function": { + "name": (tc.get("function") or {}).get("name"), + "arguments": args, + }, + }) + norm["tool_calls"] = calls + canon["tail"].append(norm) + + if body.get("tools"): + # Tool *names* drive response selection; full schemas don't. + # Sort so reordering doesn't break replay. + canon["tools"] = sorted( + (tool.get("function", {}) or {}).get("name") or "" + for tool in body["tools"] + ) + + return canon + + +def _hash_request(body: dict[str, Any]) -> str: + canon = _canonicalize_request(body) + blob = json.dumps(canon, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(blob).hexdigest() + + +def _summarize_request(body: dict[str, Any]) -> dict[str, Any]: + last_user = "" + for msg in body.get("messages", []) or []: + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, str): + last_user = content + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + last_user = part.get("text", "") + break + return { + "model": body.get("model"), + "n_messages": len(body.get("messages") or []), + "last_user_content": last_user[:120], + "tool_count": len(body.get("tools") or []), + } + + +# ── Fixture I/O ───────────────────────────────────────────────────────── + + +def _empty_fixture(model: str | None) -> dict[str, Any]: + return { + "model": model, + "schema_version": 1, + "entries": [], + } + + +def _load_fixture(path: Path) -> dict[str, Any]: + if not path.exists(): + return _empty_fixture(None) + with path.open("r", encoding="utf-8") as fp: + return json.load(fp) + + +def _save_fixture(path: Path, fixture: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fp: + json.dump(fixture, fp, indent=2, ensure_ascii=False) + fp.write("\n") + + +# ── Proxy app ─────────────────────────────────────────────────────────── + + +def _new_state( + *, + mode: str, + fixture_path: Path, + upstream_url: str | None, + upstream_key: str | None, + upstream_model: str | None, +) -> dict[str, Any]: + fixture = _load_fixture(fixture_path) + # Track per-hash replay cursor so multiple identical requests in a + # single run consume distinct recorded entries (e.g. retries). + cursors: dict[str, int] = {} + return { + "mode": mode, + "fixture_path": fixture_path, + "fixture": fixture, + "cursors": cursors, + "upstream_url": upstream_url, + "upstream_key": upstream_key, + "upstream_model": upstream_model, + "record_count": 0, + "replay_count": 0, + "miss_count": 0, + } + + +async def chat_completions(request: web.Request) -> web.Response: + state = request.app["state"] + body = await request.json() + request_hash = _hash_request(body) + print( + f"live_llm_proxy: chat_completions mode={state['mode']} " + f"hash={request_hash[:16]} n_msg={len(body.get('messages') or [])} " + f"tools={len(body.get('tools') or [])}", + file=sys.stderr, + flush=True, + ) + + if state["mode"] == "replay": + return await _replay(state, body, request_hash) + return await _record(state, body, request_hash) + + +async def _replay( + state: dict[str, Any], body: dict[str, Any], request_hash: str +) -> web.Response: + entries = state["fixture"].get("entries", []) or [] + matching = [e for e in entries if e["request_hash"] == request_hash] + cursor = state["cursors"].setdefault(request_hash, 0) + if cursor >= len(matching): + state["miss_count"] += 1 + # On miss, dump the canonical blob to a debug file so the + # test author can diff it against fixture entries to find + # what's different between record and replay. + canon = _canonicalize_request(body) + debug_dir = state["fixture_path"].parent + debug_path = debug_dir / f"{state['fixture_path'].stem}.miss_{state['miss_count']:03d}.json" + debug_path.write_text( + json.dumps({"request_hash": request_hash, "canonical": canon}, indent=2) + ) + # Build a diagnostic so the test sees exactly which prompt + # missed when it inevitably fails to drive the next step. + summary = _summarize_request(body) + return web.json_response( + { + "error": "live_llm_proxy: no recorded response for this request", + "request_hash": request_hash, + "request_summary": summary, + "fixture_path": str(state["fixture_path"]), + "miss_dump": str(debug_path), + "available_hashes": [ + { + "hash": e["request_hash"], + "summary": e.get("request_summary", {}), + } + for e in entries + ], + }, + status=500, + ) + entry = matching[cursor] + state["cursors"][request_hash] = cursor + 1 + state["replay_count"] += 1 + + response_body = entry["response"] + streaming = bool(body.get("stream")) + if streaming: + return await _emit_streamed_response(response_body) + return web.json_response(response_body) + + +async def _record( + state: dict[str, Any], body: dict[str, Any], request_hash: str +) -> web.Response: + upstream_url = state["upstream_url"] + upstream_key = state["upstream_key"] + if not upstream_url: + return web.json_response( + {"error": "live_llm_proxy: record mode requires IRONCLAW_LIVE_LLM_BASE_URL"}, + status=500, + ) + + # Override the model with the upstream model when configured. This + # lets ironclaw send the literal "mock-model" string while the + # proxy sends a real model name to the upstream. + forwarded_body = dict(body) + if state.get("upstream_model"): + forwarded_body["model"] = state["upstream_model"] + # Force non-streaming upstream so we capture a deterministic JSON + # body. We can re-emit as streaming on replay if the original + # request asked for it. + forwarded_body["stream"] = False + + headers = {"Content-Type": "application/json"} + if upstream_key: + headers["Authorization"] = f"Bearer {upstream_key}" + + timeout = aiohttp.ClientTimeout(total=120) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + f"{upstream_url.rstrip('/')}/v1/chat/completions", + json=forwarded_body, + headers=headers, + ) as response: + response_body = await response.json() + if response.status >= 400: + print( + f"live_llm_proxy: upstream {response.status} body={json.dumps(response_body)[:1500]}", + file=sys.stderr, + flush=True, + ) + return web.json_response( + { + "error": "live_llm_proxy: upstream returned error", + "upstream_status": response.status, + "upstream_body": response_body, + }, + status=response.status, + ) + + # Persist the new entry. + entry = { + "request_hash": request_hash, + "request_summary": _summarize_request(body), + # Keep the canonical blob alongside the entry so a future + # miss can diff against it without re-recording. The blob + # is what the hash is computed over. + "request_canonical": _canonicalize_request(body), + "response": response_body, + } + state["fixture"].setdefault("entries", []).append(entry) + if state["fixture"].get("model") is None and body.get("model"): + state["fixture"]["model"] = body["model"] + _save_fixture(state["fixture_path"], state["fixture"]) + state["record_count"] += 1 + + streaming = bool(body.get("stream")) + if streaming: + return await _emit_streamed_response(response_body) + return web.json_response(response_body) + + +async def _emit_streamed_response(body: dict[str, Any]) -> web.Response: + """Re-emit a non-streaming chat-completions JSON body as a single + SSE chunk plus the [DONE] sentinel. Good enough for ironclaw's + streaming consumer — every test we run here uses the chunk-or-text + accumulator, not delta-by-delta token rendering. + + Returns a one-shot `web.Response` with `text/event-stream` content + type rather than a true `web.StreamResponse`; the underlying + `_send_sse` helper buffers the payload and returns a + `web.Response` because we don't have a request-scoped `prepare()` + handle here. + """ + response = web.StreamResponse( + status=200, + headers={"Content-Type": "text/event-stream"}, + ) + # Build a single-chunk delta from the choice's message. + choice = (body.get("choices") or [{}])[0] + message = choice.get("message", {}) + delta = { + "id": body.get("id", f"chatcmpl-{uuid.uuid4().hex[:24]}"), + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": body.get("model", "live-replay"), + "choices": [ + { + "index": 0, + "delta": { + "role": message.get("role", "assistant"), + "content": message.get("content"), + "tool_calls": message.get("tool_calls"), + }, + "finish_reason": choice.get("finish_reason", "stop"), + } + ], + } + return await _send_sse_payload(response, delta) + + +async def _send_sse_payload( + response: web.StreamResponse, delta: dict[str, Any] +) -> web.Response: + return await _send_sse_lines(response, [json.dumps(delta), "[DONE]"]) + + +async def _send_sse_lines( + response: web.StreamResponse, payloads: list[str] +) -> web.Response: + return await _send_sse(response, payloads) + + +async def _send_sse(_response: web.StreamResponse, payloads: list[str]) -> web.Response: + # aiohttp StreamResponse needs a request-scoped prepare. We don't + # have direct access to the original request here; instead, use a + # trick: build the payload as a single bytes blob and return it as + # a regular Response with text/event-stream content type. SSE + # consumers tolerate a complete-on-arrival event stream. The + # `_response` argument is kept for signature symmetry with the + # streaming variant we may swap in later. + body_bytes = b"" + for payload in payloads: + body_bytes += b"data: " + payload.encode("utf-8") + b"\n\n" + return web.Response( + body=body_bytes, + headers={"Content-Type": "text/event-stream"}, + ) + + +async def models(request: web.Request) -> web.Response: + state = request.app["state"] + model_id = state["fixture"].get("model") or "live-replay" + return web.json_response( + { + "object": "list", + "data": [{"id": model_id, "object": "model", "owned_by": "ironclaw-test"}], + } + ) + + +async def state_handler(request: web.Request) -> web.Response: + state = request.app["state"] + return web.json_response( + { + "mode": state["mode"], + "fixture_path": str(state["fixture_path"]), + "n_entries": len(state["fixture"].get("entries", []) or []), + "record_count": state["record_count"], + "replay_count": state["replay_count"], + "miss_count": state["miss_count"], + } + ) + + +# ── Entry point ───────────────────────────────────────────────────────── + + +def _resolve_mode(args: argparse.Namespace) -> str: + if args.mode: + return args.mode + if os.environ.get("IRONCLAW_LIVE_TEST", "").strip() in ("1", "true"): + return "record" + return "replay" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--port", type=int, default=0) + parser.add_argument("--fixture", required=True, help="Path to the JSON trace fixture.") + parser.add_argument( + "--mode", + choices=("record", "replay"), + help="Override the IRONCLAW_LIVE_TEST-derived default.", + ) + args = parser.parse_args() + + mode = _resolve_mode(args) + upstream_url = os.environ.get("IRONCLAW_LIVE_LLM_BASE_URL") + upstream_key = os.environ.get("IRONCLAW_LIVE_LLM_API_KEY") + upstream_model = os.environ.get("IRONCLAW_LIVE_LLM_MODEL") + + if mode == "record" and not upstream_url: + print( + "live_llm_proxy: record mode requires " + "IRONCLAW_LIVE_LLM_BASE_URL (and usually IRONCLAW_LIVE_LLM_API_KEY).", + file=sys.stderr, + ) + sys.exit(2) + + fixture_path = Path(args.fixture) + state = _new_state( + mode=mode, + fixture_path=fixture_path, + upstream_url=upstream_url, + upstream_key=upstream_key, + upstream_model=upstream_model, + ) + + app = web.Application() + app["state"] = state + app.router.add_post("/v1/chat/completions", chat_completions) + app.router.add_post("/chat/completions", chat_completions) + app.router.add_get("/v1/models", models) + app.router.add_get("/models", models) + app.router.add_get("/__live/state", state_handler) + + async def start() -> None: + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", args.port) + await site.start() + port = site._server.sockets[0].getsockname()[1] + print(f"LIVE_LLM_PROXY_PORT={port}", flush=True) + print( + f"live_llm_proxy: mode={mode} fixture={fixture_path} " + f"entries={len(state['fixture'].get('entries', []) or [])}", + flush=True, + ) + await asyncio.Event().wait() + + asyncio.run(start()) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index f1cd6c21454..d084de47b9f 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -14,6 +14,11 @@ import uuid from aiohttp import web +DENIAL_PATTERN = re.compile( + r"user denied action|user denied tool|denied:\s*", + re.IGNORECASE, +) + CANNED_RESPONSES = [ (re.compile(r"empty routine response", re.IGNORECASE), ""), (re.compile(r"\bhello\b|\bhi\b|\bhey\b", re.IGNORECASE), "Hello! How can I help you today?"), @@ -866,6 +871,24 @@ def _conversation_has_active_skill(messages: list[dict], skill_name: str) -> boo return False +def _conversation_uses_codeact(messages: list[dict]) -> bool: + for msg in messages: + if msg.get("role") != "system": + continue + text = _message_text(msg) + if "Python REPL environment" in text and "```repl" in text: + return True + return False + + +def _conversation_includes_denial(messages: list[dict]) -> bool: + for msg in messages: + text = f"{_message_text(msg)}\n{_message_payload_text(msg)}" + if DENIAL_PATTERN.search(text): + return True + return False + + def _active_skill_names(messages: list[dict]) -> set[str]: names = set() for msg in messages: @@ -1036,9 +1059,20 @@ def match_response(messages: list[dict]) -> str: resumed = _resumed_action_summary(messages) if resumed: return resumed - if "user denied action" in content.lower(): - action_match = re.search(r"User denied action '([^']+)'", content) - action_name = action_match.group(1) if action_match else "that action" + denial_text = f"{content}\n{payload_text}" + if DENIAL_PATTERN.search(denial_text): + action_match = re.search( + r"User denied action '([^']+)'", denial_text, re.IGNORECASE + ) + tool_match = re.search( + r"user denied tool '([^']+)'", denial_text, re.IGNORECASE + ) + if action_match: + action_name = action_match.group(1) + elif tool_match: + action_name = tool_match.group(1) + else: + action_name = "that action" return ( f"The request for {action_name} was denied. " "No installation or setup was performed." @@ -1205,18 +1239,69 @@ def match_tool_call(messages: list[dict], has_tools: bool) -> list[dict] | None: return None lower = content.lower() recent_tool_results = _find_tool_results(messages) - if ( - ("check gmail unread" in lower or "gmail unread" in lower) - and any( - tr["name"] == "gmail" - and "Extension not installed:" in tr["content"] + # #3533: gmail-install-then-retry sequence. + # + # Turn 1: user says "check gmail unread" → match_tool_call below dispatches + # a direct `gmail` call. Engine rejects with either "Extension not + # installed:" (pre-#3533 wording, from the bridge-side + # not-installed reject — the chat-driven install path was wired up + # here in mock_llm but non-functional because `tool_install` was + # hidden from the agent surface) or "is not callable in this + # execution context" (post-#3533, engine-side preflight rejection, + # tool_install restored on the agent surface). + # Turn 2: this branch fires — call `tool_install("gmail")`. + # Turn 3: install succeeded → call `gmail` again, this time the engine's + # auth preflight raises an Authentication gate. + # Turn 4 (after OAuth completes): mock LLM falls through to the + # tool-result-summary path returning the "Quarterly update" text. + if "check gmail unread" in lower or "gmail unread" in lower: + # Three engine paths can surface gmail-unavailable depending on + # whether gmail is in the registry, installed-but-blocked, or + # entirely unknown: + # * "Extension not installed: gmail" — registry has it, not installed + # * "is not callable in this execution context" — installed but engine-v2 blocked + # * "Tool gmail not found" — not even in the dispatcher (workflow-canary stack) + # A real LLM would treat all three the same way and reach for + # `tool_install`. Mirror that — restricting to the first two + # made the workflow-canary `tool_install_chat` probe fall + # through to text and never recover. + gmail_error = next( + ( + tr + for tr in recent_tool_results + if tr["name"] == "gmail" + and ( + "Extension not installed:" in tr["content"] + or "is not callable in this execution context" in tr["content"] + or "Tool gmail not found" in tr["content"] + ) + ), + None, + ) + install_done = any( + tr["name"] == "tool_install" + and "error" not in tr["content"].lower() for tr in recent_tool_results ) - ): - return [{ - "tool_name": "tool_install", - "arguments": {"name": "gmail"}, - }] + if gmail_error and not install_done: + return [{ + "tool_name": "tool_install", + "arguments": {"name": "gmail"}, + }] + if install_done and not any( + tr["name"] == "gmail" and "is not callable" not in tr["content"] + and "Extension not installed" not in tr["content"] + and "Tool gmail not found" not in tr["content"] + for tr in recent_tool_results + ): + # Retry gmail after install — the engine's auth preflight will + # raise an Authentication gate, which surfaces the auth card. + # After OAuth completes, this re-fires and reaches the actual + # gmail tool with the correct `list_messages` action. + return [{ + "tool_name": "gmail", + "arguments": {"action": "list_messages"}, + }] if _conversation_has_active_skill(messages, "pikastream-video-meeting"): bundle_path = _active_skill_bundle_path(messages, "pikastream-video-meeting") if ( @@ -1312,6 +1397,10 @@ def _find_tool_result(messages: list[dict]) -> dict | None: return results[0] if results else None +def _tool_results_include_denial(tool_results: list[dict]) -> bool: + return any(DENIAL_PATTERN.search(tr.get("content", "")) for tr in tool_results) + + def _recent_tool_names(messages: list[dict]) -> set[str]: """Collect tool names referenced after the most recent user turn.""" last_user_idx = -1 @@ -1801,9 +1890,47 @@ async def chat_completions(request: web.Request) -> web.StreamResponse: if special and _conversation_has_user_trigger(messages, lifecycle_trigger): return await _dispatch_special_response(request, cid, stream, special) - # Tool result(s) in messages -> text summary covering every fresh result tool_results = _find_tool_results(messages) + # #3533: when a multi-step recovery is in progress (e.g. gmail not + # installed → `tool_install` → retry gmail), the next move is another + # tool call, not a text summary of the failure. Let `match_tool_call` + # take precedence over the tool-result-summary fallback whenever it + # has a follow-up call to emit. + if tool_results: + followup = match_tool_call(messages, has_tools) + if followup: + if not stream: + return _tool_call_response(cid, followup) + return await _stream_tool_call(request, cid, followup) + if ( + not tool_results + and _conversation_uses_codeact(messages) + and re.search( + r"list.*(?:google|drive).*files|show.*drive", + _last_user_content(messages), + re.IGNORECASE, + ) + ): + text = ( + "```repl\n" + f"result = await http(method=\"GET\", url=\"{_github_api_url}/drive/v3/files\")\n" + "FINAL(str(result))\n" + "```" + ) + if not stream: + return _text_response(cid, text) + return await _stream_text(request, cid, text) + + # Tool result(s) in messages -> text summary covering every fresh result if _conversation_has_active_skill(messages, "pikastream-video-meeting"): + if _conversation_includes_denial(messages) or _tool_results_include_denial(tool_results): + text = ( + "The request for shell was denied. " + "No installation or setup was performed." + ) + if not stream: + return _text_response(cid, text) + return await _stream_text(request, cid, text) recent_tool_names = _recent_tool_names(messages) if "shell" in recent_tool_names: text = ( @@ -1816,6 +1943,14 @@ async def chat_completions(request: web.Request) -> web.StreamResponse: return await _stream_text(request, cid, text) if tool_results: if _conversation_has_active_skill(messages, "pikastream-video-meeting"): + if _tool_results_include_denial(tool_results): + text = ( + "The request for shell was denied. " + "No installation or setup was performed." + ) + if not stream: + return _text_response(cid, text) + return await _stream_text(request, cid, text) if any(tr["name"] == "shell" for tr in tool_results): text = ( "Python dependencies are prepared for the Pika video-meeting skill. " @@ -2337,6 +2472,111 @@ async def mcp_oauth_token(request: web.Request) -> web.Response: }) +# ── Gmail API mocks (#3133 / #3166) ────────────────────────────────────── +# +# Per-app counters that the e2e test for the mission auto-resume path +# inspects to confirm the gmail WASM tool actually fired against this +# mock (rather than hitting the real Gmail API or silently no-oping). +# Stored in `app["gmail_state"]` so each mock_llm instance owns its own +# counters. + + +def _new_gmail_state() -> dict: + return { + "drafts_created": 0, + "messages_sent": 0, + "messages_listed": 0, + "last_draft": None, + "last_send": None, + } + + +async def gmail_create_draft(request: web.Request) -> web.Response: + """POST /gmail/v1/users/me/drafts — minimal create-draft mock. + + Maps from the gmail WASM tool's `create_draft` action. The request + body shape mirrors the real Gmail API: `{"message": {"raw": "..."}}`. + Returns a deterministic draft id so the agent can quote it back. + """ + body = await request.json() + state = request.app["gmail_state"] + state["drafts_created"] += 1 + state["last_draft"] = body + draft_id = f"mock-draft-{state['drafts_created']}" + return web.json_response({ + "id": draft_id, + "message": { + "id": f"mock-msg-{state['drafts_created']}", + "threadId": f"mock-thread-{state['drafts_created']}", + "labelIds": ["DRAFT"], + }, + }) + + +async def gmail_send_message(request: web.Request) -> web.Response: + """POST /gmail/v1/users/me/messages/send — minimal send mock. + + Maps from the gmail WASM tool's `send_message` and `reply_to_message` + actions. Returns the same shape Google does: `{id, threadId, labelIds}`. + """ + body = await request.json() + state = request.app["gmail_state"] + state["messages_sent"] += 1 + state["last_send"] = body + msg_id = f"mock-msg-{state['messages_sent']}" + return web.json_response({ + "id": msg_id, + "threadId": body.get("threadId") or f"mock-thread-{state['messages_sent']}", + "labelIds": ["SENT"], + }) + + +async def gmail_list_messages(request: web.Request) -> web.Response: + """GET /gmail/v1/users/me/messages — minimal list mock. + + Returns one canned message id so the agent has something to quote + back. The list endpoint only returns ids; the gmail WASM tool then + fetches metadata for each, which we serve from `gmail_get_message`. + """ + state = request.app["gmail_state"] + state["messages_listed"] += 1 + return web.json_response({ + "messages": [{"id": "mock-canned-msg-1", "threadId": "mock-canned-thread-1"}], + "resultSizeEstimate": 1, + }) + + +async def gmail_get_message(request: web.Request) -> web.Response: + """GET /gmail/v1/users/me/messages/{id} — minimal get-message mock.""" + msg_id = request.match_info["id"] + return web.json_response({ + "id": msg_id, + "threadId": "mock-canned-thread-1", + "labelIds": ["INBOX", "UNREAD"], + "snippet": "Mock canned snippet for the e2e test", + "payload": { + "headers": [ + {"name": "From", "value": "test@example.com"}, + {"name": "To", "value": "owner@example.com"}, + {"name": "Subject", "value": "Mock canned subject"}, + {"name": "Date", "value": "Mon, 1 Jan 2026 00:00:00 +0000"}, + ], + "body": {"data": ""}, + }, + }) + + +async def gmail_state_handler(request: web.Request) -> web.Response: + """GET /__mock/gmail/state — read the gmail counters in tests.""" + return web.json_response(request.app["gmail_state"]) + + +async def gmail_state_reset(request: web.Request) -> web.Response: + """POST /__mock/gmail/reset — clear counters between tests.""" + request.app["gmail_state"] = _new_gmail_state() + return web.json_response({"ok": True}) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--port", type=int, default=0) @@ -2344,6 +2584,7 @@ def main(): app = web.Application() app["oauth_state"] = _new_oauth_state() app["mcp_state"] = _new_mcp_state() + app["gmail_state"] = _new_gmail_state() # Register both /v1/ and non-/v1/ paths (rig-core omits the /v1/ prefix) app.router.add_post("/v1/chat/completions", chat_completions) app.router.add_post("/chat/completions", chat_completions) @@ -2380,6 +2621,14 @@ async def get_last_chat_request(request: web.Request) -> web.Response: app.router.add_get("/.well-known/oauth-authorization-server/{tail:.*}", mcp_auth_server_metadata) app.router.add_post("/oauth/register", mcp_oauth_register) app.router.add_post("/oauth/token", mcp_oauth_token) + # Gmail API mocks (consumed by the WASM gmail tool when + # IRONCLAW_TEST_HTTP_REWRITE_MAP routes gmail.googleapis.com here). + app.router.add_post("/gmail/v1/users/me/drafts", gmail_create_draft) + app.router.add_post("/gmail/v1/users/me/messages/send", gmail_send_message) + app.router.add_get("/gmail/v1/users/me/messages", gmail_list_messages) + app.router.add_get("/gmail/v1/users/me/messages/{id}", gmail_get_message) + app.router.add_get("/__mock/gmail/state", gmail_state_handler) + app.router.add_post("/__mock/gmail/reset", gmail_state_reset) async def start(): runner = web.AppRunner(app) diff --git a/tests/e2e/pyproject.toml b/tests/e2e/pyproject.toml index 531cd57f8b8..934ff61c73b 100644 --- a/tests/e2e/pyproject.toml +++ b/tests/e2e/pyproject.toml @@ -20,8 +20,27 @@ telegram = [ "telethon>=1.40", ] +# `fixtures/` (live-LLM record/replay JSON, no Python) sits alongside +# `scenarios/` after #3133 landed. Setuptools auto-discovery refuses a +# flat layout with multiple top-level packages, so scope discovery to +# the actual Python package and exclude data-only directories. +[tool.setuptools.packages.find] +include = ["scenarios*"] +exclude = ["fixtures*", "screenshots*", "test-pages*"] + [tool.pytest.ini_options] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" timeout = 360 +# Strict-by-default xfail: an XPASS (test starts passing despite the +# xfail mark) is a real failure, forcing whoever fixed the underlying +# contract to either un-xfail the test or update the reason. Two stale +# xfail(strict=False) markers in test_v2_auth_oauth_matrix.py masked +# the chat-driven tool_install regression shipped in #3366 for five +# days — neither XFAIL nor XPASS would have alerted anyone with the +# default (strict=False). Un-xfailed in #3589. Explicit +# `strict=False` is still allowed for the rare genuinely-flaky case +# but must be deliberate, not the default. See `.claude/rules/testing.md` +# for the policy. +xfail_strict = true diff --git a/tests/e2e/scenarios/test_admin_api.py b/tests/e2e/scenarios/test_admin_api.py new file mode 100644 index 00000000000..52b1fd33ff5 --- /dev/null +++ b/tests/e2e/scenarios/test_admin_api.py @@ -0,0 +1,153 @@ +"""Admin API integration tests — user CRUD, secrets, suspend/activate.""" + +import uuid + +import httpx +import pytest + +from helpers import AUTH_TOKEN + + +@pytest.fixture() +async def admin_client(ironclaw_server): + """Async HTTP client with admin auth headers.""" + async with httpx.AsyncClient( + base_url=ironclaw_server, + headers={ + "Authorization": f"Bearer {AUTH_TOKEN}", + "Content-Type": "application/json", + }, + timeout=10, + ) as client: + yield client + + +@pytest.fixture() +async def test_user(admin_client): + """Create a test user and clean up after the test.""" + email = f"test-{uuid.uuid4().hex[:8]}@example.com" + r = await admin_client.post("/api/admin/users", json={ + "display_name": "E2E Test User", + "email": email, + "role": "member", + }) + assert r.status_code == 200 + data = r.json() + yield data + # Cleanup + await admin_client.delete(f"/api/admin/users/{data['id']}") + + +# --------------------------------------------------------------- +# User CRUD +# --------------------------------------------------------------- + + +async def test_create_user(admin_client): + email = f"test-{uuid.uuid4().hex[:8]}@example.com" + r = await admin_client.post("/api/admin/users", json={ + "display_name": "Create Test", + "email": email, + "role": "member", + }) + assert r.status_code == 200 + data = r.json() + assert "id" in data + assert "token" in data + assert data["status"] == "active" + assert data["role"] == "member" + # Cleanup + await admin_client.delete(f"/api/admin/users/{data['id']}") + + +async def test_list_users_contains_new_user(admin_client, test_user): + r = await admin_client.get("/api/admin/users") + assert r.status_code == 200 + ids = [u["id"] for u in r.json()["users"]] + assert test_user["id"] in ids + + +async def test_get_user_detail(admin_client, test_user): + r = await admin_client.get(f"/api/admin/users/{test_user['id']}") + assert r.status_code == 200 + data = r.json() + assert data["display_name"] == "E2E Test User" + assert data["id"] == test_user["id"] + + +async def test_update_user(admin_client, test_user): + r = await admin_client.patch(f"/api/admin/users/{test_user['id']}", json={ + "display_name": "Updated Name", + "metadata": {"ref": "abound-123"}, + }) + assert r.status_code == 200 + data = r.json() + assert data["display_name"] == "Updated Name" + assert data["metadata"]["ref"] == "abound-123" + + +# --------------------------------------------------------------- +# Suspend / Activate +# --------------------------------------------------------------- + + +async def test_suspend_and_activate(admin_client, test_user): + uid = test_user["id"] + + r = await admin_client.post(f"/api/admin/users/{uid}/suspend") + assert r.status_code == 200 + assert r.json()["status"] == "suspended" + + r = await admin_client.post(f"/api/admin/users/{uid}/activate") + assert r.status_code == 200 + assert r.json()["status"] == "active" + + +# --------------------------------------------------------------- +# Secrets +# --------------------------------------------------------------- + + +async def test_secret_lifecycle(admin_client, test_user): + uid = test_user["id"] + + # Create + r = await admin_client.put( + f"/api/admin/users/{uid}/secrets/abound_token", + json={"value": "secret-value", "provider": "abound"}, + ) + assert r.status_code == 200 + assert r.json()["name"] == "abound_token" + + # List + r = await admin_client.get(f"/api/admin/users/{uid}/secrets") + assert r.status_code == 200 + names = [s["name"] for s in r.json()["secrets"]] + assert "abound_token" in names + + # Delete + r = await admin_client.delete(f"/api/admin/users/{uid}/secrets/abound_token") + assert r.status_code == 200 + assert r.json()["deleted"] is True + + +# --------------------------------------------------------------- +# Delete +# --------------------------------------------------------------- + + +async def test_delete_user_and_verify_gone(admin_client): + email = f"test-{uuid.uuid4().hex[:8]}@example.com" + r = await admin_client.post("/api/admin/users", json={ + "display_name": "Delete Me", + "email": email, + "role": "member", + }) + uid = r.json()["id"] + + r = await admin_client.delete(f"/api/admin/users/{uid}") + assert r.status_code == 200 + assert r.json()["deleted"] is True + + r = await admin_client.get(f"/api/admin/users/{uid}") + assert r.status_code == 404 diff --git a/tests/e2e/scenarios/test_auth_no_duplicate_response.py b/tests/e2e/scenarios/test_auth_no_duplicate_response.py index 34133374e4a..2ae9872844d 100644 --- a/tests/e2e/scenarios/test_auth_no_duplicate_response.py +++ b/tests/e2e/scenarios/test_auth_no_duplicate_response.py @@ -11,7 +11,8 @@ 2. Connects to the SSE stream 3. Sends a chat message that triggers the GitHub skill → HTTP 401 → auth onboarding 4. Collects SSE events and asserts: - - onboarding_state/auth_required event IS present + - an auth prompt event is present (gate_required/Authentication or + onboarding_state/auth_required) - No response event contains auth instruction text (the regression) """ @@ -94,11 +95,12 @@ def _write_skill(skills_dir: str, mock_api_host: str): f.write(f"""--- name: github version: "1.0.0" -keywords: - - github - - issues -tags: - - github +activation: + keywords: + - github + - issues + tags: + - github credentials: - name: github_token provider: github @@ -170,6 +172,7 @@ async def auth_sse_server(ironclaw_binary, mock_llm_server, mock_api): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_DB_TMPDIR.name, "auth-sse-e2e.db"), @@ -230,8 +233,22 @@ async def _pin_mock_github_api_url(mock_llm_server, mock_api): # Test # --------------------------------------------------------------------------- +def _is_auth_prompt_event(event: dict) -> bool: + if ( + event.get("type") == "onboarding_state" + and event.get("state") == "auth_required" + ): + return True + if event.get("type") != "gate_required": + return False + resume = event.get("resume_kind") or {} + return (event.get("gate_name") or "").lower() == "authentication" or ( + isinstance(resume, dict) and isinstance(resume.get("Authentication"), dict) + ) + + async def test_auth_required_sse_without_duplicate_response(auth_sse_server): - """When auth is triggered, SSE emits onboarding_state but NOT a response with instructions.""" + """Auth emits an auth gate but NOT a response with instructions.""" base_url = auth_sse_server # Create thread @@ -274,15 +291,14 @@ async def collect_sse(): ) assert send_r.status_code == 202 - # Wait for onboarding_state/auth_required, then collect for a grace period to catch any - # trailing duplicate response events that might arrive shortly after. + # Wait for an auth prompt event, then collect for a grace period to catch any + # trailing duplicate response events that might arrive shortly after. Engine-v2 + # auth gates are surfaced as gate_required/Authentication; legacy paths may + # still emit onboarding_state/auth_required. deadline = asyncio.get_running_loop().time() + 45 auth_seen_at = None while asyncio.get_running_loop().time() < deadline: - has_auth_event = any( - e.get("type") == "onboarding_state" and e.get("state") == "auth_required" - for e in collected_events - ) + has_auth_event = any(_is_auth_prompt_event(e) for e in collected_events) if has_auth_event and auth_seen_at is None: auth_seen_at = asyncio.get_running_loop().time() if auth_seen_at and (asyncio.get_running_loop().time() - auth_seen_at) > 3: @@ -295,13 +311,10 @@ async def collect_sse(): except asyncio.CancelledError: pass - # Assert onboarding_state/auth_required event was emitted - has_auth_event = any( - e.get("type") == "onboarding_state" and e.get("state") == "auth_required" - for e in collected_events - ) + # Assert an auth prompt event was emitted. + has_auth_event = any(_is_auth_prompt_event(e) for e in collected_events) assert has_auth_event, ( - f"Expected onboarding_state/auth_required in SSE events, got: {collected_events}" + f"Expected auth prompt SSE event, got: {collected_events}" ) # Assert NO response event contains auth instruction text. diff --git a/tests/e2e/scenarios/test_channel_approval_gates.py b/tests/e2e/scenarios/test_channel_approval_gates.py new file mode 100644 index 00000000000..ef657f3daa1 --- /dev/null +++ b/tests/e2e/scenarios/test_channel_approval_gates.py @@ -0,0 +1,548 @@ +"""End-to-end approval-gate tests for non-web channels. + +Inline mid-turn approval gates (PR #3157) pause the engine inside the +running CodeAct script and surface the prompt back to the originating +channel via ``BridgeGateController::emit_gate_prompt``. These tests +verify the flow for the two bundled WASM channels that the e2e harness +already wires up — Telegram and Slack — covering: + +- DM ``yes`` / ``no`` / ``always`` reply resolves the gate. +- ``always`` persists ``always_allow`` to the DB so a second tool call + to the same tool auto-approves without a prompt. +- Cross-channel resolution: a gate raised from a Telegram or Slack + message can be approved from the web ``/api/chat/approval`` endpoint. + +The non-web web-equivalent file is ``test_v2_engine_approval_flow.py``; +this file deliberately reuses the ``"make approval post <label>"`` mock +LLM pattern so the same shape (``http`` POST → approval) exercises +non-web channel surface. +""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import httpx +import pytest + +from helpers import api_get, api_post, AUTH_TOKEN, auth_headers + +# Reuse helpers from the per-channel scenarios so this file stays focused +# on the gate-resolution assertions and doesn't reimplement webhook +# plumbing or pairing. +from .test_telegram_e2e import ( + OWNER_USER_ID as TG_OWNER_USER_ID, + WEBHOOK_SECRET as TG_WEBHOOK_SECRET, + _next_test_update_id, + activate_telegram, + post_telegram_webhook, + reset_fake_tg, +) +from .test_slack_e2e import ( + OWNER_USER_ID as SL_OWNER_USER_ID, + activate_slack, + build_slack_dm_event, + build_slack_mention_event, + post_slack_webhook, + reset_fake_slack, +) + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +async def _set_tool_permission(base_url: str, tool_name: str, state: str) -> None: + """Pin the given tool to ``state`` so approval prompts are deterministic. + + The seeded default for ``http`` is ``always_allow``. Channel approval + tests need the explicit ``ask_each_time`` path, just like the web-only + flow in ``test_v2_engine_approval_flow.py``. + """ + async with httpx.AsyncClient() as client: + response = await client.put( + f"{base_url}/api/settings/tools/{tool_name}", + json={"state": state}, + headers=auth_headers(), + timeout=15, + ) + assert response.status_code == 200, ( + f"Failed to set {tool_name} permission to {state}: " + f"{response.status_code} {response.text}" + ) + + +async def _find_channel_thread( + base_url: str, + *, + channel: str, + timeout: float = 30, +) -> str: + """Return the most recently active thread id whose channel matches. + + Telegram/Slack open one thread per chat under the same user. Tests + don't know the engine-assigned UUID up-front, so look it up by the + ``channel`` field that ``/api/chat/threads`` already exposes. + """ + deadline = time.monotonic() + timeout + last_payload: Any = None + async with httpx.AsyncClient() as client: + while time.monotonic() < deadline: + r = await client.get( + f"{base_url}/api/chat/threads", + headers=auth_headers(), + timeout=10, + ) + if r.status_code == 200: + data = r.json() + last_payload = data + threads = data.get("threads", []) + matching = [t for t in threads if t.get("channel") == channel] + if matching: + matching.sort( + key=lambda t: t.get("updated_at") or "", reverse=True + ) + return matching[0]["id"] + await asyncio.sleep(0.5) + raise AssertionError( + f"No thread with channel='{channel}' appeared within {timeout}s. " + f"Last /api/chat/threads payload: {last_payload!r}" + ) + + +async def _wait_for_pending_gate( + base_url: str, + thread_id: str, + *, + timeout: float = 60, +) -> dict: + """Poll history until ``pending_gate`` is set; return the gate dict.""" + deadline = time.monotonic() + timeout + last_history: Any = None + while time.monotonic() < deadline: + r = await api_get( + base_url, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + r.raise_for_status() + last_history = r.json() + pending = last_history.get("pending_gate") + if pending and pending.get("request_id"): + return pending + await asyncio.sleep(0.5) + raise AssertionError( + f"Timed out waiting for pending_gate on thread {thread_id}. " + f"Last history: {last_history!r}" + ) + + +async def _wait_for_no_pending_gate( + base_url: str, + thread_id: str, + *, + timeout: float = 60, +) -> dict: + """Poll history until ``pending_gate`` clears; return the final history.""" + deadline = time.monotonic() + timeout + last_history: Any = None + while time.monotonic() < deadline: + r = await api_get( + base_url, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + r.raise_for_status() + last_history = r.json() + if not last_history.get("pending_gate"): + return last_history + await asyncio.sleep(0.5) + raise AssertionError( + f"Timed out waiting for pending_gate to clear on thread {thread_id}. " + f"Last history: {last_history!r}" + ) + + +async def _read_tool_permission(base_url: str, tool_name: str) -> str | None: + """Return the persisted ``tool_permissions.<tool>`` value or None.""" + r = await api_get( + base_url, f"/api/settings/tool_permissions.{tool_name}", timeout=15 + ) + if r.status_code != 200: + return None + return r.json().get("value") + + +# --------------------------------------------------------------------------- +# Telegram-specific helpers +# --------------------------------------------------------------------------- + + +def _telegram_message_update(text: str, *, message_id: int) -> dict: + """Build a Telegram webhook update for the test owner DM.""" + return { + "update_id": _next_test_update_id(), + "message": { + "message_id": message_id, + "from": { + "id": TG_OWNER_USER_ID, + "is_bot": False, + "first_name": "E2E Tester", + }, + "chat": {"id": TG_OWNER_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": text, + }, + } + + +async def _send_tg(http_url: str, text: str, *, message_id: int) -> None: + resp = await post_telegram_webhook( + http_url, + _telegram_message_update(text, message_id=message_id), + secret=TG_WEBHOOK_SECRET, + ) + assert resp.status_code == 200, ( + f"Telegram webhook returned {resp.status_code}: {resp.text}" + ) + + +# --------------------------------------------------------------------------- +# Slack-specific helpers +# --------------------------------------------------------------------------- + + +async def _send_slack_dm(http_url: str, text: str) -> None: + resp = await post_slack_webhook( + http_url, build_slack_dm_event(SL_OWNER_USER_ID, text) + ) + assert resp.status_code == 200, ( + f"Slack webhook returned {resp.status_code}: {resp.text}" + ) + + +async def _send_slack_mention(http_url: str, text: str, *, channel: str = "C0001") -> str: + ts = f"{time.time():.6f}" + resp = await post_slack_webhook( + http_url, + build_slack_mention_event( + SL_OWNER_USER_ID, text, channel=channel, ts=ts + ), + ) + assert resp.status_code == 200, ( + f"Slack mention webhook returned {resp.status_code}: {resp.text}" + ) + return ts + + +# --------------------------------------------------------------------------- +# Telegram fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def telegram_with_ask_each_time(telegram_e2e_server): + """Activate Telegram and pin the http tool to ask_each_time.""" + base_url = telegram_e2e_server["base_url"] + http_url = telegram_e2e_server["http_url"] + fake_tg_url = telegram_e2e_server["fake_tg_url"] + channels_dir = telegram_e2e_server["channels_dir"] + + await activate_telegram(base_url, http_url, fake_tg_url, channels_dir) + await reset_fake_tg(fake_tg_url) + # The seeded default for `http` is `always_allow`; force the + # ask-each-time path so these tests deterministically exercise the + # inline-await gate prompt instead of the auto-approve fast path. + await _set_tool_permission(base_url, "http", "ask_each_time") + return telegram_e2e_server + + +# --------------------------------------------------------------------------- +# Slack fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def slack_with_ask_each_time(slack_e2e_server): + """Activate Slack and pin the http tool to ask_each_time.""" + base_url = slack_e2e_server["base_url"] + http_url = slack_e2e_server["http_url"] + fake_slack_url = slack_e2e_server["fake_slack_url"] + channels_dir = slack_e2e_server["channels_dir"] + + await activate_slack(base_url, http_url, fake_slack_url, channels_dir) + await reset_fake_slack(fake_slack_url) + await _set_tool_permission(base_url, "http", "ask_each_time") + return slack_e2e_server + + +# --------------------------------------------------------------------------- +# Telegram approval-gate tests +# --------------------------------------------------------------------------- + + +async def test_telegram_dm_approval_yes_resolves_inline_gate( + telegram_with_ask_each_time, +): + """Telegram DM 'yes' reply approves a paused inline gate.""" + base_url = telegram_with_ask_each_time["base_url"] + http_url = telegram_with_ask_each_time["http_url"] + + await _send_tg(http_url, "make approval post tg-alpha", message_id=100) + thread_id = await _find_channel_thread(base_url, channel="telegram") + pending = await _wait_for_pending_gate(base_url, thread_id) + assert pending["tool_name"] == "http", pending + + await _send_tg(http_url, "yes", message_id=101) + history = await _wait_for_no_pending_gate(base_url, thread_id) + # Sanity: a turn record exists for the original prompt. + assert history.get("turns"), history + + +async def test_telegram_dm_approval_no_denies_inline_gate( + telegram_with_ask_each_time, +): + """Telegram DM 'no' reply denies the gate; pending_gate clears.""" + base_url = telegram_with_ask_each_time["base_url"] + http_url = telegram_with_ask_each_time["http_url"] + + await _send_tg(http_url, "make approval post tg-deny", message_id=110) + thread_id = await _find_channel_thread(base_url, channel="telegram") + await _wait_for_pending_gate(base_url, thread_id) + + await _send_tg(http_url, "no", message_id=111) + await _wait_for_no_pending_gate(base_url, thread_id) + + # `no` must NOT install always_allow. + perm = await _read_tool_permission(base_url, "http") + assert perm in (None, "ask_each_time"), ( + f"Deny must not persist always_allow, got {perm!r}" + ) + + +async def test_telegram_dm_approval_always_persists_to_db( + telegram_with_ask_each_time, +): + """Telegram DM 'always' clears the gate AND persists always_allow.""" + base_url = telegram_with_ask_each_time["base_url"] + http_url = telegram_with_ask_each_time["http_url"] + + await _send_tg(http_url, "make approval post tg-always", message_id=120) + thread_id = await _find_channel_thread(base_url, channel="telegram") + await _wait_for_pending_gate(base_url, thread_id) + + await _send_tg(http_url, "always", message_id=121) + await _wait_for_no_pending_gate(base_url, thread_id) + + perm = await _read_tool_permission(base_url, "http") + assert perm == "always_allow", ( + f"'always' approval must persist always_allow, got {perm!r}" + ) + + # Reset back to ask_each_time so subsequent tests sharing the + # session-scoped server start from a clean slate. + await _set_tool_permission(base_url, "http", "ask_each_time") + + +async def test_telegram_gate_can_be_resolved_via_web_api( + telegram_with_ask_each_time, +): + """A gate fired from Telegram is also resolvable via /api/chat/approval. + + This covers the cross-channel resolve path: prompt was surfaced on + Telegram, but the human chose to approve from the web dashboard. + """ + base_url = telegram_with_ask_each_time["base_url"] + http_url = telegram_with_ask_each_time["http_url"] + + await _send_tg(http_url, "make approval post tg-cross", message_id=130) + thread_id = await _find_channel_thread(base_url, channel="telegram") + pending = await _wait_for_pending_gate(base_url, thread_id) + + approval = await api_post( + base_url, + "/api/chat/approval", + json={ + "request_id": pending["request_id"], + "action": "approve", + "thread_id": thread_id, + }, + timeout=15, + ) + assert approval.status_code == 202, ( + f"Cross-channel approve failed: {approval.status_code} {approval.text}" + ) + await _wait_for_no_pending_gate(base_url, thread_id) + + +# --------------------------------------------------------------------------- +# Slack approval-gate tests +# --------------------------------------------------------------------------- + + +@pytest.mark.xfail( + reason=( + "Slack WASM bundle does not currently propagate same-channel " + "'yes/no/always' text replies into the agent loop while a gate is " + "parked on a sibling engine thread (each Slack event ts opens its " + "own engine thread, and the parked thread blocks dispatch of the " + "reply). Cross-channel resolution via /api/chat/approval works — " + "see test_slack_gate_can_be_resolved_via_web_api. Track as a " + "channel-native approval gap." + ), + strict=True, +) +async def test_slack_dm_approval_yes_resolves_inline_gate( + slack_with_ask_each_time, +): + """Slack DM 'yes' reply approves a paused inline gate.""" + base_url = slack_with_ask_each_time["base_url"] + http_url = slack_with_ask_each_time["http_url"] + + await _send_slack_dm(http_url, "make approval post sl-alpha") + thread_id = await _find_channel_thread(base_url, channel="slack") + pending = await _wait_for_pending_gate(base_url, thread_id) + assert pending["tool_name"] == "http", pending + + await _send_slack_dm(http_url, "yes") + history = await _wait_for_no_pending_gate(base_url, thread_id) + assert history.get("turns"), history + + +@pytest.mark.xfail( + reason=( + "Same channel-native gap as the matching 'yes' test — see the " + "xfail note on test_slack_dm_approval_yes_resolves_inline_gate." + ), + strict=True, +) +async def test_slack_dm_approval_no_denies_inline_gate( + slack_with_ask_each_time, +): + """Slack DM 'no' reply denies the gate; pending_gate clears.""" + base_url = slack_with_ask_each_time["base_url"] + http_url = slack_with_ask_each_time["http_url"] + + await _send_slack_dm(http_url, "make approval post sl-deny") + thread_id = await _find_channel_thread(base_url, channel="slack") + await _wait_for_pending_gate(base_url, thread_id) + + await _send_slack_dm(http_url, "no") + await _wait_for_no_pending_gate(base_url, thread_id) + + perm = await _read_tool_permission(base_url, "http") + assert perm in (None, "ask_each_time"), ( + f"Deny must not persist always_allow, got {perm!r}" + ) + + +@pytest.mark.xfail( + reason=( + "Same channel-native gap as the matching 'yes' test — see the " + "xfail note on test_slack_dm_approval_yes_resolves_inline_gate." + ), + strict=True, +) +async def test_slack_dm_approval_always_persists_to_db( + slack_with_ask_each_time, +): + """Slack DM 'always' clears the gate AND persists always_allow.""" + base_url = slack_with_ask_each_time["base_url"] + http_url = slack_with_ask_each_time["http_url"] + + await _send_slack_dm(http_url, "make approval post sl-always") + thread_id = await _find_channel_thread(base_url, channel="slack") + await _wait_for_pending_gate(base_url, thread_id) + + await _send_slack_dm(http_url, "always") + await _wait_for_no_pending_gate(base_url, thread_id) + + perm = await _read_tool_permission(base_url, "http") + assert perm == "always_allow", ( + f"'always' approval must persist always_allow, got {perm!r}" + ) + + await _set_tool_permission(base_url, "http", "ask_each_time") + + +async def test_slack_gate_can_be_resolved_via_web_api( + slack_with_ask_each_time, +): + """Cross-channel: Slack gate resolved via /api/chat/approval.""" + base_url = slack_with_ask_each_time["base_url"] + http_url = slack_with_ask_each_time["http_url"] + + await _send_slack_dm(http_url, "make approval post sl-cross") + thread_id = await _find_channel_thread(base_url, channel="slack") + pending = await _wait_for_pending_gate(base_url, thread_id) + + approval = await api_post( + base_url, + "/api/chat/approval", + json={ + "request_id": pending["request_id"], + "action": "approve", + "thread_id": thread_id, + }, + timeout=15, + ) + assert approval.status_code == 202, ( + f"Cross-channel approve failed: {approval.status_code} {approval.text}" + ) + await _wait_for_no_pending_gate(base_url, thread_id) + + +async def test_slack_app_mention_approval_does_not_post_to_dm( + slack_with_ask_each_time, +): + """app_mention firing a gate must not post an approval card to a DM. + + Engine v2 + RelayChannel skips approval rendering for non-DM events + (`relay/channel.rs:435-447`). The WASM Slack channel + fake API used + here does not post a Block Kit approval card either; the closest + observable signal is "no chat.postMessage to D<owner> within a few + seconds" while the gate sits parked in `pending_gates`. + + We don't wait for the 30-min `expires_at` to fire — we just verify + the prompt didn't leak to the owner DM, then clean up by approving + via the web API so the parked engine task gets unstuck before the + next test runs. + """ + base_url = slack_with_ask_each_time["base_url"] + http_url = slack_with_ask_each_time["http_url"] + fake_slack_url = slack_with_ask_each_time["fake_slack_url"] + + await reset_fake_slack(fake_slack_url) + await _send_slack_mention(http_url, "make approval post sl-mention") + + thread_id = await _find_channel_thread(base_url, channel="slack") + pending = await _wait_for_pending_gate(base_url, thread_id) + assert pending["tool_name"] == "http", pending + + # Give the engine a few seconds to (incorrectly) push to the owner DM. + await asyncio.sleep(3) + async with httpx.AsyncClient() as c: + r = await c.get(f"{fake_slack_url}/__mock/sent_messages", timeout=5) + messages = r.json().get("messages", []) + dm_replies = [m for m in messages if m.get("channel") == f"D{SL_OWNER_USER_ID}"] + assert not dm_replies, ( + f"app_mention-fired gate must not post an approval card to the owner DM, " + f"got: {dm_replies}" + ) + + # Clean up so the parked engine task doesn't bleed into later tests. + approval = await api_post( + base_url, + "/api/chat/approval", + json={ + "request_id": pending["request_id"], + "action": "deny", + "thread_id": thread_id, + }, + timeout=15, + ) + assert approval.status_code == 202, approval.text + await _wait_for_no_pending_gate(base_url, thread_id) diff --git a/tests/e2e/scenarios/test_message_persistence.py b/tests/e2e/scenarios/test_message_persistence.py index 41adecb7a4b..36f894259aa 100644 --- a/tests/e2e/scenarios/test_message_persistence.py +++ b/tests/e2e/scenarios/test_message_persistence.py @@ -433,17 +433,29 @@ async def test_processing_indicator_shows_for_incomplete_turn(page, ironclaw_ser "(id) => currentThreadId === id", arg=thread_a, timeout=10000, ) - # The "Processing..." thinking indicator should be visible + # The "Processing..." thinking indicator should be visible — but + # there is a window between the API-side state check above and the + # Playwright switchThread + wait_for where the turn can complete and + # the indicator gets cleared by the response render. Race the + # indicator's visibility against the assistant message landing; if + # the response arrived first, the turn was healthy and the + # rendering happened correctly without us catching the indicator. thinking = page.locator(SEL["activity_thinking"]) - await thinking.wait_for(state="visible", timeout=10000) + assistant = page.locator(SEL["message_assistant"]) + try: + await thinking.wait_for(state="visible", timeout=10000) + except Exception: + if await assistant.count() == 0: + raise + # Response landed before the indicator could render — the + # in-progress UX path is still valid; fall through to the + # completion + response assertions. # Wait for the turn to complete — indicator should disappear await _wait_for_completed_turn(ironclaw_server, thread_a, timeout=30) # The assistant response should appear (live SSE renders it) - await page.locator(SEL["message_assistant"]).wait_for( - state="visible", timeout=15000, - ) + await assistant.wait_for(state="visible", timeout=15000) async def test_refresh_preserves_in_progress_turn(page, ironclaw_server): @@ -470,6 +482,37 @@ async def test_refresh_preserves_in_progress_turn(page, ironclaw_server): ).wait_for(state="visible", timeout=15000) +async def test_response_event_does_not_duplicate_history_rendered_response(page): + """A late response SSE must not duplicate a response already rendered from history.""" + thread_id = "history-rendered-thread" + content = "The answer is 4." + + await page.wait_for_function( + "() => typeof eventSource !== 'undefined' && eventSource && sseHasConnectedBefore === true", + timeout=10000, + ) + await page.evaluate( + """({ threadId, content }) => { + currentThreadId = threadId; + const container = document.getElementById('chat-messages'); + container.innerHTML = ''; + addMessage('user', 'What is 2+2?'); + addMessage('assistant', content); + eventSource.dispatchEvent(new MessageEvent('response', { + data: JSON.stringify({ + type: 'response', + thread_id: threadId, + content, + }), + })); + }""", + {"threadId": thread_id, "content": content}, + ) + + assistant_messages = page.locator(SEL["message_assistant"]).filter(has_text=content) + assert await assistant_messages.count() == 1 + + async def test_switching_back_preserves_in_progress_turn(page, ironclaw_server): """Switching away and back mid-turn should rehydrate the running thread.""" thread_a, _payload = await _start_thread_and_wait_for_in_progress( diff --git a/tests/e2e/scenarios/test_mission_gmail_3133.py b/tests/e2e/scenarios/test_mission_gmail_3133.py new file mode 100644 index 00000000000..038c2352d83 --- /dev/null +++ b/tests/e2e/scenarios/test_mission_gmail_3133.py @@ -0,0 +1,399 @@ +"""Live-LLM Playwright regression for issue #3133 + half-2 (#3166). + +Issue #3133: a Gmail-sending mission firing every 3 minutes whose +child thread bailed with the LLM-rendered "Failed to send email. +Status: None Error: None" pattern. Half-1 (PR #3155) added a typed +`GatePaused` outcome that pauses the mission and surfaces an +`AuthRequired` status update on the user's auth tray. Half-2 (this +PR) auto-resumes the mission once the user completes OAuth on the +matching credential. + +Post-#3133, installed-but-unauthed provider tools are direct-callable; +the engine's auth preflight raises an `Authentication` gate at +execute time and the inline-await machinery parks the mission until +OAuth completes. The previous `tool_activate` enablement step has +been removed — the model calls the tool directly in both Tier 0 and +Tier 1 (CodeAct / Python via Monty), and the gate path fires from +either tier. + +This test drives the full chat-driven mission lifecycle through a +real LLM (or a recorded trace from one) and asserts the auto-resume +mechanism end-to-end: + + 1. Browser opens the chat tab against an isolated gateway. + 2. User types: "Create a mission to send a Gmail draft every 3 + minutes. Trigger it once now." + 3. The LLM's response (live or replayed) emits a `routine_create` + tool call; the bridge alias translates that to a `mission_create` + in engine v2. + 4. The LLM's next response emits `mission_fire` (or the test calls + `/api/engine/missions/{id}/fire` directly when the LLM fails to + fire it) and the child thread's first turn emits a direct + `gmail(action="create_draft", ...)` call. + 5. Gmail is installed but not authenticated, so the auth preflight + raises an Authentication gate and the mission transitions to + **Paused** with `paused_gate.resume_kind = + Authentication { credential = google_oauth_token }`. + 6. The test polls `/api/engine/missions` until the mission flips + to `Paused`. + 7. The test completes OAuth via `/oauth/callback?code=mock_auth_code`. + The credential write fires `bridge::resume_paused_missions_for_credential`, + which transitions the mission Paused → Active and immediately + re-fires it. + 8. The re-fired child thread sees gmail authenticated, so the + `gmail` tool succeeds. The HTTP rewrite map routes + `gmail.googleapis.com` at mock_llm.py, whose + `/gmail/v1/users/me/drafts` endpoint returns a deterministic + draft id so the next turn can quote it. + 9. The test polls mock_llm's `/__mock/gmail/state` until + `drafts_created >= 1`, proving the auto-resumed mission + actually completed the work the original gate had blocked. + 10. The test asserts no chat thread carries the `Status: None` + + `Error: None` dual-marker (the #3133 fingerprint). + +Live infrastructure (see `tests/e2e/live_harness.py` and +`tests/e2e/live_llm_proxy.py`): + + * Run with `IRONCLAW_LIVE_TEST=1` plus `IRONCLAW_LIVE_LLM_BASE_URL` / + `IRONCLAW_LIVE_LLM_API_KEY` / `IRONCLAW_LIVE_LLM_MODEL` to record + a fresh trace into `tests/e2e/fixtures/live/<test_name>.json`. + Commit the resulting JSON so CI can replay deterministically. + * Without `IRONCLAW_LIVE_TEST`, the test runs in replay mode against + the committed fixture. If the fixture is missing the test is + skipped (not failed) so a fresh checkout doesn't bog down on + missing recordings. +""" + +import asyncio +import os +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest + +from helpers import SEL, api_get, api_post + + +# ── Regression markers ─────────────────────────────────────────────────── + +STATUS_NONE_MARKER = "Status: None" +ERROR_NONE_MARKER = "Error: None" +CONSECUTIVE_ERRORS_MARKER = "consecutive code errors" + + +CHAT_PROMPT = ( + "Use the mission_create tool to create a mission with these exact " + "parameters and DO NOT run any other tools first:\n" + " name = 'gmail-draft-3133'\n" + " goal = 'Use the gmail tool with action=create_draft to send a " + "draft to owner@example.com with subject \"Test mission #3133\" and " + "body \"Mock draft from the IronClaw e2e test.\". Just call the " + "gmail tool directly — the runtime handles authentication. Do NOT " + "call tool_activate, tool_install, or any other setup tool first.'\n" + " cadence = cron expression '*/3 * * * *'\n" + "Then immediately use mission_fire to trigger it once. " + "Do not call tool_list, tool_info, tool_activate, or any other tool " + "before mission_create. Just create the mission and fire it." +) + + +def _extract_state(auth_url: str) -> str: + parsed = urlparse(auth_url) + state = parse_qs(parsed.query).get("state", [None])[0] + assert state, f"auth_url should include state: {auth_url}" + return state + + +async def _install_gmail(server: str) -> None: + response = await api_post( + server, "/api/extensions/install", json={"name": "gmail"}, timeout=180 + ) + assert response.status_code == 200, response.text + assert response.json().get("success") is True, response.text + + +async def _start_oauth_flow(server: str) -> str: + response = await api_post( + server, "/api/extensions/gmail/setup", json={"secrets": {}}, timeout=30 + ) + assert response.status_code == 200, response.text + auth_url = response.json().get("auth_url") + assert auth_url, response.json() + return _extract_state(auth_url) + + +async def _complete_oauth(server: str, state: str) -> None: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{server}/oauth/callback", + params={"code": "mock_auth_code", "state": state}, + timeout=30, + follow_redirects=True, + ) + assert response.status_code == 200, response.text[:400] + assert "connected" in response.text.lower() or "success" in response.text.lower() + + +async def _list_engine_missions(server: str) -> list[dict]: + response = await api_get(server, "/api/engine/missions", timeout=15) + response.raise_for_status() + return response.json().get("missions", []) or [] + + +async def _wait_for_engine_mission( + server: str, name_substr: str, *, timeout: float = 60.0 +) -> dict: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + for m in await _list_engine_missions(server): + mname = (m.get("name") or "").lower() + if name_substr.lower() in mname: + return m + await asyncio.sleep(0.5) + raise AssertionError( + f"engine mission containing {name_substr!r} never appeared within {timeout}s. " + f"Have: {[m.get('name') for m in await _list_engine_missions(server)]}" + ) + + +async def _wait_for_mission_status( + server: str, + name_substr: str, + statuses: tuple[str, ...], + *, + timeout: float, +) -> dict: + deadline = asyncio.get_event_loop().time() + timeout + last_seen: dict | None = None + while asyncio.get_event_loop().time() < deadline: + for m in await _list_engine_missions(server): + mname = (m.get("name") or "").lower() + if name_substr.lower() in mname: + last_seen = m + if m.get("status") in statuses: + return m + await asyncio.sleep(0.5) + raise AssertionError( + f"mission matching {name_substr!r} never reached one of " + f"{statuses} within {timeout}s. Last seen: {last_seen}" + ) + + +async def _fire_engine_mission(server: str, mission_id: str) -> dict: + response = await api_post( + server, f"/api/engine/missions/{mission_id}/fire", timeout=15 + ) + assert response.status_code in (200, 202), response.text + return response.json() + + +async def _gmail_mock_state(mock_llm_url: str) -> dict: + async with httpx.AsyncClient() as client: + response = await client.get(f"{mock_llm_url}/__mock/gmail/state", timeout=10) + response.raise_for_status() + return response.json() + + +async def _reset_gmail_mock_state(mock_llm_url: str) -> None: + async with httpx.AsyncClient() as client: + response = await client.post( + f"{mock_llm_url}/__mock/gmail/reset", timeout=10 + ) + response.raise_for_status() + + +async def _wait_for_gmail_drafts( + mock_llm_url: str, *, target: int = 1, timeout: float = 120.0 +) -> dict: + deadline = asyncio.get_event_loop().time() + timeout + last_seen: dict | None = None + while asyncio.get_event_loop().time() < deadline: + last_seen = await _gmail_mock_state(mock_llm_url) + if last_seen.get("drafts_created", 0) >= target: + return last_seen + await asyncio.sleep(0.5) + raise AssertionError( + f"gmail mock never recorded {target} draft(s) within {timeout}s. " + f"Last seen: {last_seen}" + ) + + +async def _send_chat(page, text: str, *, timeout_ms: int = 5000) -> None: + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=timeout_ms) + if await chat_input.evaluate("el => !!el.disabled"): + await page.keyboard.press("Control+n") + await page.wait_for_function( + """selector => { + const input = document.querySelector(selector); + return !!input && !input.disabled; + }""", + arg=SEL["chat_input"], + timeout=10000, + ) + await chat_input.fill(text) + await chat_input.press("Enter") + await page.wait_for_selector( + SEL["message_user"], state="visible", timeout=10000 + ) + + +# ── Tests ──────────────────────────────────────────────────────────────── + + +async def test_mission_gmail_draft_3133( + mission_gmail_live_page, mission_gmail_live_server +): + """Issue #3133 / #3166: full chat-driven mission lifecycle. + + In replay mode this runs deterministically against the committed + LLM trace. In record mode (`IRONCLAW_LIVE_TEST=1`) it forwards to + a real LLM, captures the trace into the fixture file, and asserts + the same end-state — so a re-recording can't accidentally bake in + a regression. + + Failing this test means one of: + + - half-1 #3133 regressed (mission stays Active on the gate path) + - the persistent `paused_gate` field stopped being written + - `bridge::resume_paused_missions_for_credential` stopped firing + from `/oauth/callback` + - the Paused → Active + immediate-fire transition is broken + """ + server = mission_gmail_live_server["base_url"] + mock_llm = mission_gmail_live_server["mock_llm_url"] + mode = mission_gmail_live_server["mode"] + print(f"[#3133] running in {mode} mode against {server}") + + # 1. Reset mock gmail state and install Gmail (NOT authenticated). + await _reset_gmail_mock_state(mock_llm) + await _install_gmail(server) + extensions = await api_get(server, "/api/extensions", timeout=15) + gmail = next( + e for e in extensions.json()["extensions"] if e["name"] == "gmail" + ) + assert not gmail["authenticated"], ( + "Gmail must start unauthenticated for the gate path to fire" + ) + + # 2. Drive chat → real LLM → routine_create / mission_create. The + # bridge's `routine_to_mission_alias` lands the result in + # engine v2's mission store, so we poll /api/engine/missions. + await _send_chat(mission_gmail_live_page, CHAT_PROMPT) + mission = await _wait_for_engine_mission(server, "gmail", timeout=120.0) + mission_id = mission["id"] + mission_name = mission["name"] + print(f"[#3133] created mission {mission_name} ({mission_id})") + + # 3. The LLM's "trigger it now" response should emit `mission_fire`, + # but if the recording elided that for any reason we fall back + # to a direct fire so the test stays robust to LLM phrasing + # variation. Either way the child thread runs. + pre_fire_status = mission.get("status") + if pre_fire_status not in ("Paused", "Failed"): + # Wait briefly for the LLM's mission_fire to fire it. + try: + await _wait_for_mission_status( + server, "gmail", ("Paused",), timeout=20.0 + ) + except AssertionError: + print("[#3133] LLM did not auto-fire the mission; firing directly") + await _fire_engine_mission(server, mission_id) + + # 4. Wait until the mission lands in Paused after the gate fires. + # In record mode this can fail if the LLM's child thread runs + # in Tier 1 (CodeAct) — see the module docstring's known + # limitation. Surface a clear skip-or-fail diagnostic instead + # of timing out silently. + try: + paused = await _wait_for_mission_status( + server, "gmail", ("Paused",), timeout=120.0 + ) + except AssertionError as e: + if mode == "record": + from live_harness import proxy_state + st = await proxy_state(mission_gmail_live_server["live_proxy_url"]) + pytest.skip( + f"record mode: mission never reached Paused. The live " + f"LLM's child thread did not call gmail (or the auth " + f"preflight didn't fire). Trace recorded " + f"{st['record_count']} entries to " + f"{mission_gmail_live_server['fixture']}. Inspect that " + f"file to see what the LLM did. Original error: {e}" + ) + raise + print(f"[#3133] mission paused: {paused.get('status')}") + + # 5. Mission must NOT have created any drafts yet — the auth + # preflight gate paused execution before the gmail tool's + # HTTP call could complete. + pre_oauth = await _gmail_mock_state(mock_llm) + assert pre_oauth["drafts_created"] == 0, ( + f"no draft should be created before OAuth: {pre_oauth}" + ) + + # 6. Complete OAuth. This is the half-2 trigger: + # /oauth/callback → bridge::resume_paused_missions_for_credential + # → MissionManager::resume_paused_for_credential → mission + # Paused → Active and immediate fire. + state = await _start_oauth_flow(server) + await _complete_oauth(server, state) + print("[#3133] OAuth callback completed") + + # 7. Mission must auto-resume to Active or Completed. + final = await _wait_for_mission_status( + server, "gmail", ("Active", "Completed"), timeout=120.0 + ) + print(f"[#3133] mission auto-resumed: status={final.get('status')}") + + # 8. The auto-resumed child thread must complete the gmail draft. + # create_draft lands on mock_llm's /gmail/v1/users/me/drafts. + gmail_state = await _wait_for_gmail_drafts(mock_llm, target=1, timeout=180.0) + assert gmail_state["drafts_created"] >= 1, gmail_state + assert gmail_state["last_draft"] is not None + print(f"[#3133] gmail mock recorded {gmail_state['drafts_created']} draft(s)") + + # 9. Regression marker for #3133: no chat thread should carry + # the dual `Status: None` + `Error: None` fingerprint. + threads = ( + await api_get(server, "/api/chat/threads", timeout=15) + ).json().get("threads", []) or [] + for thread in threads: + thread_id = thread.get("id") + if not thread_id: + continue + history = ( + await api_get( + server, f"/api/chat/history?thread_id={thread_id}", timeout=15 + ) + ).json() + for turn in history.get("turns", []) or []: + for message in turn.get("messages", []) or []: + body = message.get("content") or message.get("text") or "" + if not isinstance(body, str): + continue + has_status = STATUS_NONE_MARKER in body + has_error = ERROR_NONE_MARKER in body + assert not (has_status and has_error), ( + f"regression: chat history carried both " + f"'{STATUS_NONE_MARKER}' and '{ERROR_NONE_MARKER}' — " + f"the #3133 fingerprint. Thread {thread_id}, " + f"body: {body[:400]}" + ) + assert CONSECUTIVE_ERRORS_MARKER not in body.lower(), ( + f"regression: chat history carried " + f"'{CONSECUTIVE_ERRORS_MARKER}' from #2583. " + f"Thread {thread_id}, body: {body[:400]}" + ) + + # 10. Live mode sanity: the proxy logged actual upstream calls. + if mode == "record": + from live_harness import proxy_state + st = await proxy_state(mission_gmail_live_server["live_proxy_url"]) + assert st["record_count"] > 0, ( + f"record mode should have captured at least one LLM call: {st}" + ) + print( + f"[#3133] recorded {st['record_count']} LLM call(s) into " + f"{mission_gmail_live_server['fixture']}" + ) diff --git a/tests/e2e/scenarios/test_reborn_gateway_smoke.py b/tests/e2e/scenarios/test_reborn_gateway_smoke.py new file mode 100644 index 00000000000..3359541e063 --- /dev/null +++ b/tests/e2e/scenarios/test_reborn_gateway_smoke.py @@ -0,0 +1,261 @@ +"""Dedicated Reborn gateway smoke E2E. + +This is intentionally small. The Rust Reborn gate proves the host/runtime +architecture. This Playwright/API smoke test proves the reborn-main branch still +boots an isolated ENGINE_V2 gateway, serves the browser shell, persists a normal +chat turn, and completes a simple tool-capable turn without duplicate terminal +assistant responses. +""" + +import asyncio +import os +import signal +import socket +from pathlib import Path + +import pytest +from playwright.async_api import expect + +from helpers import AUTH_TOKEN, SEL, api_get, api_post, wait_for_ready + +def _find_free_port() -> int: + """Ask the OS for an available loopback port. + + The returned port is only a startup hint; the gateway fixture retries with a + fresh port pair if another process wins the bind race before ironclaw starts. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _read_log(path: Path, limit: int = 8192) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace")[-limit:] + except OSError: + return "" + + +def _forward_coverage_env(env: dict[str, str]) -> None: + for key, value in os.environ.items(): + if key.startswith(("CARGO_LLVM_COV", "LLVM_")) or key in { + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_INCREMENTAL", + }: + env[key] = value + + +async def _stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None: + """Signal a subprocess and wait for exit without re-reading stdio pipes.""" + if proc.returncode is not None: + return + + try: + proc.send_signal(sig) + except ProcessLookupError: + return + + try: + await asyncio.wait_for(proc.wait(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + await asyncio.wait_for(proc.wait(), timeout=5) + + +@pytest.fixture(scope="module") +async def reborn_gateway_server(ironclaw_binary, mock_llm_server, tmp_path_factory): + """Start an isolated gateway configured for the Reborn/V2 product shell.""" + home_dir = tmp_path_factory.mktemp("ironclaw-reborn-gateway-home") + db_dir = tmp_path_factory.mktemp("ironclaw-reborn-gateway-db") + base_dir = home_dir / ".ironclaw" + base_dir.mkdir(parents=True, exist_ok=True) + + proc = None + base_url = None + last_stderr = "" + last_gateway_port = None + + for attempt in range(1, 4): + gateway_port = _find_free_port() + http_port = _find_free_port() + last_gateway_port = gateway_port + stdout_path = home_dir / f"reborn-gateway-attempt-{attempt}.stdout.log" + stderr_path = home_dir / f"reborn-gateway-attempt-{attempt}.stderr.log" + + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(home_dir), + "IRONCLAW_BASE_DIR": str(base_dir), + "RUST_LOG": "ironclaw=info", + "RUST_BACKTRACE": "1", + "ENGINE_V2": "true", + "AGENT_AUTO_APPROVE_TOOLS": "true", + "GATEWAY_ENABLED": "true", + "GATEWAY_HOST": "127.0.0.1", + "GATEWAY_PORT": str(gateway_port), + "GATEWAY_AUTH_TOKEN": AUTH_TOKEN, + "GATEWAY_USER_ID": "reborn-gateway-e2e-user", + "HTTP_HOST": "127.0.0.1", + "HTTP_PORT": str(http_port), + "CLI_ENABLED": "false", + "LLM_BACKEND": "openai_compatible", + "LLM_BASE_URL": mock_llm_server, + "LLM_API_KEY": "mock-api-key", + "LLM_MODEL": "mock-model", + "DATABASE_BACKEND": "libsql", + "LIBSQL_PATH": str(db_dir / "reborn-gateway-e2e.db"), + "SANDBOX_ENABLED": "false", + "SKILLS_ENABLED": "false", + "ROUTINES_ENABLED": "false", + "HEARTBEAT_ENABLED": "false", + "EMBEDDING_ENABLED": "false", + "WASM_ENABLED": "false", + "ONBOARD_COMPLETED": "true", + } + _forward_coverage_env(env) + + with stdout_path.open("wb") as stdout_file, stderr_path.open("wb") as stderr_file: + proc = await asyncio.create_subprocess_exec( + ironclaw_binary, + "--no-onboard", + stdin=asyncio.subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + env=env, + ) + base_url = f"http://127.0.0.1:{gateway_port}" + + try: + await wait_for_ready(f"{base_url}/api/health", timeout=60) + break + except TimeoutError: + if proc.returncode is None: + await _stop_process(proc, timeout=2) + last_stderr = _read_log(stderr_path) + proc = None + else: + pytest.fail( + "Reborn gateway smoke server failed to start after 3 attempts.\n" + f"Last attempted port: {last_gateway_port}\n" + f"stderr:\n{last_stderr}" + ) + + try: + yield base_url + finally: + if proc is not None and proc.returncode is None: + await _stop_process(proc, sig=signal.SIGINT, timeout=10) + if proc.returncode is None: + await _stop_process(proc, sig=signal.SIGTERM, timeout=5) + + +@pytest.fixture +async def reborn_gateway_page(reborn_gateway_server, browser): + context = await browser.new_context(viewport={"width": 1280, "height": 720}) + page = await context.new_page() + await page.goto(f"{reborn_gateway_server}/?token={AUTH_TOKEN}") + await page.wait_for_selector(SEL["auth_screen"], state="hidden", timeout=15000) + await page.wait_for_function( + "() => typeof sseHasConnectedBefore !== 'undefined' && sseHasConnectedBefore === true", + timeout=10000, + ) + yield page + await context.close() + + +async def _create_thread(base_url: str) -> str: + response = await api_post(base_url, "/api/chat/thread/new", timeout=15) + response.raise_for_status() + return response.json()["id"] + + +async def _send_message(base_url: str, thread_id: str, content: str) -> None: + response = await api_post( + base_url, + "/api/chat/send", + json={"content": content, "thread_id": thread_id}, + timeout=30, + ) + assert response.status_code in (200, 202), response.text + + +async def _wait_for_terminal_turn( + base_url: str, + thread_id: str, + expected_user_input: str, + *, + timeout: float = 45.0, +) -> dict: + last_history = {} + for _ in range(int(timeout * 2)): + response = await api_get( + base_url, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + response.raise_for_status() + history = response.json() + last_history = history + turns = history.get("turns", []) + matching_turns = [ + turn + for turn in turns + if expected_user_input in (turn.get("user_input") or "") + ] + if matching_turns and (matching_turns[-1].get("response") or "").strip(): + return matching_turns[-1] + await asyncio.sleep(0.5) + + raise AssertionError( + f"Timed out waiting for terminal turn containing {expected_user_input!r}. " + f"Last history: {last_history}" + ) + + +async def test_reborn_gateway_loads_engine_v2_shell(reborn_gateway_page): + """The isolated Reborn smoke gateway should boot the ENGINE_V2 shell.""" + chat_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="chat")) + missions_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="missions")) + routines_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="routines")) + + await expect(chat_tab).to_be_visible() + await expect(missions_tab).to_be_visible() + await expect(routines_tab).to_be_hidden() + + +async def test_reborn_gateway_persists_text_and_tool_turns_without_duplicate_response( + reborn_gateway_server, +): + """A text turn and an auto-approved tool turn should each produce one terminal response.""" + thread_id = await _create_thread(reborn_gateway_server) + + text_prompt = "reborn gateway smoke: what is 2+2?" + await _send_message(reborn_gateway_server, thread_id, text_prompt) + text_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, text_prompt) + assert "4" in text_turn.get("response", "") + + tool_prompt = "echo reborn gateway smoke tool result" + await _send_message(reborn_gateway_server, thread_id, tool_prompt) + tool_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, tool_prompt) + assert "reborn gateway smoke tool result" in tool_turn.get("response", "").lower() + + tool_calls = tool_turn.get("tool_calls", []) + assert tool_calls, f"Expected persisted tool call metadata, got: {tool_turn}" + assert any(call.get("name") == "echo" and call.get("has_result") for call in tool_calls) + + history_response = await api_get( + reborn_gateway_server, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + history_response.raise_for_status() + matching_tool_turns = [ + turn + for turn in history_response.json().get("turns", []) + if tool_prompt in (turn.get("user_input") or "") + and (turn.get("response") or "").strip() + ] + assert len(matching_tool_turns) == 1, ( + "Expected one terminal assistant response for the tool prompt, got " + f"{len(matching_tool_turns)} turns: {matching_tool_turns}" + ) diff --git a/tests/e2e/scenarios/test_responses_api.py b/tests/e2e/scenarios/test_responses_api.py new file mode 100644 index 00000000000..841acadbe2b --- /dev/null +++ b/tests/e2e/scenarios/test_responses_api.py @@ -0,0 +1,188 @@ +"""Responses API integration tests — HTTP, streaming, context injection.""" + +import uuid + +import httpx +import pytest + +from helpers import AUTH_TOKEN + + +@pytest.fixture() +async def responses_user(ironclaw_server): + """Create a test user for Responses API tests, yield (base_url, user_token), clean up.""" + email = f"resp-{uuid.uuid4().hex[:8]}@example.com" + async with httpx.AsyncClient( + base_url=ironclaw_server, + headers={"Authorization": f"Bearer {AUTH_TOKEN}", "Content-Type": "application/json"}, + timeout=10, + ) as admin: + r = await admin.post( + "/api/admin/users", + json={ + "display_name": "Responses Test User", + "email": email, + "role": "member", + }, + ) + assert r.status_code == 200 + data = r.json() + user_id = data["id"] + user_token = data["token"] + + yield ironclaw_server, user_token + + await admin.delete(f"/api/admin/users/{user_id}") + + +@pytest.fixture() +async def responses_client(responses_user): + """HTTP client pointed at the test IronClaw Responses API.""" + base_url, user_token = responses_user + async with httpx.AsyncClient( + base_url=base_url, + headers={"Authorization": f"Bearer {user_token}", "Content-Type": "application/json"}, + timeout=120, + ) as client: + yield client + + +async def create_response(client: httpx.AsyncClient, **payload): + """Create a response through the compatibility alias used by existing clients.""" + body = {"model": "default", **payload} + r = await client.post("/v1/responses", json=body) + assert r.status_code == 200, r.text + return r.json() + + +# --------------------------------------------------------------- +# Non-streaming +# --------------------------------------------------------------- + + +async def test_non_streaming_text_input(responses_client): + response = await create_response( + responses_client, + input="Say hello in exactly 3 words", + ) + assert response["id"].startswith("resp_") + assert response["status"] == "completed" + assert len(response["output"]) > 0 + + +async def test_non_streaming_messages_input(responses_client): + response = await create_response( + responses_client, + input=[{"role": "user", "content": "What is 2+2? Reply with just the number."}], + ) + assert response["status"] == "completed" + assert len(response["output"]) > 0 + + +# --------------------------------------------------------------- +# Multi-turn +# --------------------------------------------------------------- + + +async def test_continue_conversation(responses_client): + r1 = await create_response(responses_client, input="Say hello") + assert r1["status"] == "completed" + + r2 = await create_response( + responses_client, + input="Now say goodbye", + previous_response_id=r1["id"], + ) + assert r2["status"] == "completed" + assert r2["id"] != r1["id"] + + +# --------------------------------------------------------------- +# GET by ID +# --------------------------------------------------------------- + + +async def test_get_response_by_id(responses_client): + response = await create_response(responses_client, input="Remember this: the sky is blue") + retrieved = await responses_client.get(f"/v1/responses/{response['id']}") + assert retrieved.status_code == 200, retrieved.text + data = retrieved.json() + assert data["id"] == response["id"] + assert len(data["output"]) > 0 + + +# --------------------------------------------------------------- +# Streaming +# --------------------------------------------------------------- + + +async def test_streaming_raw_sse(responses_client): + async with responses_client.stream( + "POST", + "/v1/responses", + json={"model": "default", "input": "Say hi", "stream": True}, + ) as resp: + assert resp.status_code == 200 + event_count = 0 + async for line in resp.aiter_lines(): + if line.startswith("event:"): + event_count += 1 + assert event_count > 0 + + +# --------------------------------------------------------------- +# Context injection +# --------------------------------------------------------------- + + +async def test_context_injection_approval(responses_client): + data = await create_response( + responses_client, + input="Go ahead with the transfer", + x_context={ + "notification_response": { + "notification_id": "msg_456", + "action": "approved", + "original_signal": "convert_now", + "score": 72, + } + }, + stream=False, + ) + assert data["status"] == "completed" + assert len(data["output"]) > 0 + + +async def test_context_injection_rejection(responses_client): + data = await create_response( + responses_client, + input="Cancel it", + x_context={ + "notification_response": { + "notification_id": "msg_789", + "action": "rejected", + } + }, + stream=False, + ) + assert data["status"] == "completed" + + +# --------------------------------------------------------------- +# Error cases +# --------------------------------------------------------------- + + +async def test_error_no_auth(ironclaw_server): + async with httpx.AsyncClient(timeout=10) as client: + r = await client.post( + f"{ironclaw_server}/v1/responses", + headers={"Content-Type": "application/json"}, + json={"input": "hello"}, + ) + assert r.status_code == 401 + + +async def test_error_empty_input(responses_client): + r = await responses_client.post("/v1/responses", json={"model": "default", "input": ""}) + assert r.status_code == 400 diff --git a/tests/e2e/scenarios/test_settings_search.py b/tests/e2e/scenarios/test_settings_search.py index fa3b57ee4d4..2fdedf693ea 100644 --- a/tests/e2e/scenarios/test_settings_search.py +++ b/tests/e2e/scenarios/test_settings_search.py @@ -8,9 +8,11 @@ - Clearing search restores all items """ +import asyncio import json +import uuid -from helpers import SEL, api_post +from helpers import SEL, api_get, api_post # ─── Helpers ────────────────────────────────────────────────────────────────── @@ -180,21 +182,35 @@ async def test_search_filters_extension_cards(page): async def test_search_filters_user_rows(page, ironclaw_server): """Search filters user table rows in the Users subtab.""" # Seed two users via the admin API - await api_post(ironclaw_server, "/api/admin/users", json={ - "display_name": "Alice Searchtest", - "email": "alice-searchtest@example.test", + suffix = uuid.uuid4().hex[:8] + alice = await api_post(ironclaw_server, "/api/admin/users", json={ + "display_name": f"Alice Searchtest {suffix}", + "email": f"alice-searchtest-{suffix}@example.test", "role": "member", }) - await api_post(ironclaw_server, "/api/admin/users", json={ - "display_name": "Bob Searchtest", - "email": "bob-searchtest@example.test", + assert alice.status_code == 200, alice.text + bob = await api_post(ironclaw_server, "/api/admin/users", json={ + "display_name": f"Bob Searchtest {suffix}", + "email": f"bob-searchtest-{suffix}@example.test", "role": "member", }) + assert bob.status_code == 200, bob.text + + expected_ids = {alice.json()["id"], bob.json()["id"]} + for _ in range(20): + listed = await api_get(ironclaw_server, "/api/admin/users") + assert listed.status_code == 200, listed.text + listed_ids = {u["id"] for u in listed.json().get("users", [])} + if expected_ids <= listed_ids: + break + await asyncio.sleep(0.25) + else: + raise AssertionError("Seeded users did not appear in admin API list") await _open_settings_subtab(page, "users") rows = page.locator(SEL["users_tbody_row"]) - await rows.first.wait_for(state="visible", timeout=5000) + await rows.first.wait_for(state="visible", timeout=15000) total = await rows.count() assert total >= 2, f"Need at least 2 user rows, got {total}" diff --git a/tests/e2e/scenarios/test_skill_oauth_flow.py b/tests/e2e/scenarios/test_skill_oauth_flow.py index 148f42d494e..59478557657 100644 --- a/tests/e2e/scenarios/test_skill_oauth_flow.py +++ b/tests/e2e/scenarios/test_skill_oauth_flow.py @@ -28,6 +28,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from helpers import api_get, api_post, auth_headers, AUTH_TOKEN +from scenarios.test_auth_no_duplicate_response import auth_sse_server # --------------------------------------------------------------------------- @@ -362,11 +363,16 @@ class TestSSEAuthEvents: """Test that auth events are emitted via SSE for web gateway.""" @pytest.mark.asyncio - async def test_auth_required_sse_event(self, ironclaw_server): + async def test_auth_required_sse_event(self, auth_sse_server): """Auth onboarding SSE event should be emitted when credential is missing.""" + # Use the isolated auth SSE fixture: it starts ironclaw with a mock + # GitHub skill whose credential mapping points at the loopback mock API. + # The shared ironclaw_server cannot add that host mapping after startup. + base_url = auth_sse_server + # Connect to SSE stream thread_r = await api_post( - ironclaw_server, "/api/chat/thread/new", timeout=15 + base_url, "/api/chat/thread/new", timeout=15 ) thread_id = thread_r.json()["id"] @@ -374,19 +380,22 @@ async def test_auth_required_sse_event(self, ironclaw_server): async def collect_sse_events(): """Collect SSE events in the background.""" - url = f"{ironclaw_server}/api/chat/events?token={AUTH_TOKEN}" - async with httpx.AsyncClient() as client: - async with client.stream("GET", url, timeout=30) as resp: - async for line in resp.aiter_lines(): - if line.startswith("data:"): - try: - data = json.loads(line[5:].strip()) - events_received.append(data) - except json.JSONDecodeError: - pass - # Stop after getting enough events - if len(events_received) > 20: - break + url = f"{base_url}/api/chat/events?token={AUTH_TOKEN}" + try: + async with httpx.AsyncClient() as client: + async with client.stream("GET", url, timeout=30) as resp: + async for line in resp.aiter_lines(): + if line.startswith("data:"): + try: + data = json.loads(line[5:].strip()) + events_received.append(data) + except json.JSONDecodeError: + pass + # Stop after getting enough events + if len(events_received) > 20: + break + except (asyncio.CancelledError, httpx.TimeoutException): + pass # Start collecting events sse_task = asyncio.create_task(collect_sse_events()) @@ -396,7 +405,7 @@ async def collect_sse_events(): # Send a message that triggers auth await api_post( - ironclaw_server, + base_url, "/api/chat/send", json={ "content": "show github issues for nearai/ironclaw", @@ -405,8 +414,16 @@ async def collect_sse_events(): timeout=30, ) - # Wait for events to arrive - await asyncio.sleep(10) + # Wait for auth events to arrive. Coverage-instrumented CI can be + # slow, so poll the collected stream instead of sleeping a fixed 10s. + deadline = asyncio.get_running_loop().time() + 45 + while asyncio.get_running_loop().time() < deadline: + if any( + e.get("type") == "onboarding_state" and e.get("state") == "auth_required" + for e in events_received + ) or "approval_needed" in [e.get("type", "") for e in events_received]: + break + await asyncio.sleep(0.5) sse_task.cancel() try: await sse_task diff --git a/tests/e2e/scenarios/test_skills.py b/tests/e2e/scenarios/test_skills.py index 50f5b6be812..2e45516ec16 100644 --- a/tests/e2e/scenarios/test_skills.py +++ b/tests/e2e/scenarios/test_skills.py @@ -1,9 +1,37 @@ """Scenario 3: Skills search, install, and remove lifecycle.""" -import pytest +import json +from urllib.parse import unquote, urlparse + from helpers import SEL +MOCK_CATALOG_SKILL = { + "slug": "e2e/markdown-helper", + "name": "Markdown Helper", + "description": "Deterministic E2E skill for markdown workflows.", + "version": "1.0.0", + "score": 1.0, + "updatedAt": 1778000000000, + "stars": 12, + "downloads": 3456, + "owner": "e2e", + "installed": False, +} + +MOCK_INSTALLED_SKILL = { + "name": "markdown-helper", + "description": "Deterministic E2E skill for markdown workflows.", + "version": "1.0.0", + "trust": "Installed", + "source": "Installed", + "keywords": ["markdown", "e2e"], + "usage_hint": "Type `/markdown-helper` in chat to force-activate this skill.", + "has_requirements": False, + "has_scripts": False, +} + + async def go_to_skills(page): """Navigate to Settings > Skills subtab.""" await page.locator(SEL["tab_button"].format(tab="settings")).click() @@ -13,6 +41,74 @@ async def go_to_skills(page): ) +async def mock_skills_api(page): + """Mock skills API endpoints used by the browser lifecycle tests. + + These tests validate the Settings > Skills UI contract, not live ClawHub + availability. Keeping the API local avoids skip-on-network behavior while + still exercising the real browser code paths. + """ + installed = [] + install_requests = [] + + async def fulfill_json(route, payload): + await route.fulfill( + json=payload, + headers={"Cache-Control": "no-store"}, + ) + + async def handle(route): + nonlocal installed + request = route.request + path = urlparse(request.url).path + + if path == "/api/skills" and request.method == "GET": + await fulfill_json(route, {"skills": installed, "count": len(installed)}) + return + + if path == "/api/skills/search" and request.method == "POST": + catalog_skill = dict(MOCK_CATALOG_SKILL) + catalog_skill["installed"] = any( + skill["name"] == MOCK_INSTALLED_SKILL["name"] for skill in installed + ) + await fulfill_json( + route, + { + "catalog": [catalog_skill], + "installed": installed, + "registry_url": "https://clawhub.example.test", + }, + ) + return + + if path == "/api/skills/install" and request.method == "POST": + install_requests.append(json.loads(request.post_data or "{}")) + if not any(skill["name"] == MOCK_INSTALLED_SKILL["name"] for skill in installed): + installed = [dict(MOCK_INSTALLED_SKILL)] + await fulfill_json( + route, + { + "success": True, + "message": "Skill 'markdown-helper' installed", + }, + ) + return + + if path.startswith("/api/skills/") and request.method == "DELETE": + name = unquote(path.removeprefix("/api/skills/")) + installed = [skill for skill in installed if skill["name"] != name] + await fulfill_json( + route, + {"success": True, "message": f"Skill '{name}' removed"}, + ) + return + + await route.continue_() + + await page.route("**/api/skills**", handle) + return {"install_requests": install_requests} + + async def test_skills_tab_visible(page): """Skills subtab shows the search interface.""" await go_to_skills(page) @@ -22,68 +118,63 @@ async def test_skills_tab_visible(page): async def test_skills_search(page): - """Search ClawHub for skills and verify results appear.""" + """Search renders deterministic catalog results without live ClawHub.""" + await mock_skills_api(page) await go_to_skills(page) search_input = page.locator(SEL["skill_search_input"]) await search_input.fill("markdown") await search_input.press("Enter") - # Wait for results (ClawHub may be slow) - try: - results = page.locator(SEL["skill_search_result"]) - await results.first.wait_for(state="visible", timeout=20000) - except Exception: - pytest.skip("ClawHub registry unreachable or returned no results") + results = page.locator(SEL["skill_search_result"]) + await results.first.wait_for(state="visible", timeout=5000) count = await results.count() assert count >= 1, "Expected at least 1 search result" + assert "Markdown Helper" in await results.first.inner_text() async def test_skills_install_and_remove(page): - """Install a skill from search results, then remove it.""" + """Install a mocked catalog skill from search results, then remove it.""" + mock_api = await mock_skills_api(page) await go_to_skills(page) - # Search search_input = page.locator(SEL["skill_search_input"]) await search_input.fill("markdown") await search_input.press("Enter") - try: - results = page.locator(SEL["skill_search_result"]) - await results.first.wait_for(state="visible", timeout=20000) - except Exception: - pytest.skip("ClawHub registry unreachable or returned no results") + results = page.locator(SEL["skill_search_result"]) + await results.first.wait_for(state="visible", timeout=5000) - # Auto-accept confirm dialogs - await page.evaluate("window.confirm = () => true") - - # Install first result install_btn = results.first.locator("button", has_text="Install") - if await install_btn.count() == 0: - pytest.skip("No installable skills found in results") - await install_btn.click() - - # Wait for install to complete -- the UI calls loadSkills() after install, - # which populates #skills-list with .ext-card elements + assert await install_btn.count() == 1, "Expected mocked catalog skill to be installable" + async with page.expect_response(lambda r: "/api/skills/install" in r.url) as install_response: + await install_btn.click() + response = await install_response.value + assert response.ok + assert mock_api["install_requests"] == [ + {"name": MOCK_CATALOG_SKILL["name"], "slug": MOCK_CATALOG_SKILL["slug"]} + ], "Install request should use the catalog skill name and slug" + + # The app refreshes the installed-skills list after a successful install; + # waiting on the DOM keeps this as a black-box UI contract. installed = page.locator(SEL["skill_installed"]) - try: - await installed.first.wait_for(state="visible", timeout=15000) - except Exception: - pytest.skip("Skill install did not update the installed list in time") + await installed.first.wait_for(state="visible", timeout=5000) installed_count = await installed.count() assert installed_count >= 1, "Skill should appear in installed list after install" + assert "markdown-helper" in await installed.first.inner_text() - # Remove the skill via confirm modal remove_btn = installed.first.locator("button", has_text="Remove") - if await remove_btn.count() > 0: - await remove_btn.click() - # Confirm in the modal - confirm_btn = page.locator(SEL["confirm_modal_btn"]) - await confirm_btn.wait_for(state="visible", timeout=5000) - await confirm_btn.click() - # Wait for the card to disappear or list to shrink - await page.wait_for_timeout(3000) - new_count = await page.locator(SEL["skill_installed"]).count() - assert new_count < installed_count, "Skill should be removed from installed list" + assert await remove_btn.count() == 1, "Installed mocked skill should be removable" + await remove_btn.click() + + confirm_btn = page.locator(SEL["confirm_modal_btn"]) + await confirm_btn.wait_for(state="visible", timeout=5000) + await confirm_btn.click() + + await page.wait_for_function( + """(selector) => document.querySelectorAll(selector).length === 0""", + arg=SEL["skill_installed"], + timeout=5000, + ) diff --git a/tests/e2e/scenarios/test_sse_reconnect.py b/tests/e2e/scenarios/test_sse_reconnect.py index c65bda0ab2a..54d7b0804a1 100644 --- a/tests/e2e/scenarios/test_sse_reconnect.py +++ b/tests/e2e/scenarios/test_sse_reconnect.py @@ -292,33 +292,38 @@ async def patch_threads_response(route): await page.route("**/api/chat/threads", patch_threads_response) - # 4. Reload — loadThreads() should skip the "http" active thread - await page.reload() - await page.wait_for_selector("#auth-screen", state="hidden", timeout=15000) - await _wait_for_connected(page, timeout=15000) + try: + # 4. Reload — loadThreads() should skip the "http" active thread + await page.reload() + await page.wait_for_selector("#auth-screen", state="hidden", timeout=15000) + await _wait_for_connected(page, timeout=15000) - # 5. Assert we landed on the writable gateway conversation, not the - # patched read-only external thread, and that the legacy pinned assistant - # chrome is gone from the DOM. - assert fallback_gateway_thread_id, "expected a gateway fallback thread id" - await page.wait_for_function( - "(expected) => currentThreadId === expected", - arg=fallback_gateway_thread_id, - timeout=15000, - ) - assert await page.locator("#assistant-thread").count() == 0 - - # 6. Chat input should be enabled (not disabled by read-only state) - chat_input = page.locator(SEL["chat_input"]) - await chat_input.wait_for(state="visible", timeout=5000) - is_disabled = await chat_input.is_disabled() - assert not is_disabled, "Chat input should be enabled on the fallback gateway thread" - - # Drain in-flight route callbacks before the `page` fixture closes the - # context (see the setup comment at step 3 for the root cause). The - # `ignoreErrors` behavior swallows the cancellation of any mid-flight - # `route.fetch()` so it cannot surface on an unrelated later test. - await page.unroute_all(behavior="ignoreErrors") + # 5. Assert we landed on a writable gateway conversation, not the + # patched read-only external thread. In a full-suite run there may be + # an existing gateway conversation newer than the assistant fallback, + # so the invariant is "not the read-only active thread" rather than a + # specific fallback id. + assert fallback_gateway_thread_id, "expected a gateway fallback thread id" + await page.wait_for_function( + "(external) => !!currentThreadId && currentThreadId !== external", + arg=ext_thread_id, + timeout=15000, + ) + current_thread = await page.evaluate("() => currentThreadId") + assert current_thread != ext_thread_id + assert await page.locator("#assistant-thread").count() == 0 + + # 6. Chat input should be enabled (not disabled by read-only state) + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + is_disabled = await chat_input.is_disabled() + assert not is_disabled, "Chat input should be enabled on the fallback gateway thread" + finally: + # Drain in-flight route callbacks before the `page` fixture closes the + # context (see the setup comment at step 3 for the root cause). The + # `ignoreErrors` behavior swallows the cancellation of any mid-flight + # `route.fetch()` so it cannot surface on an unrelated later test. + await page.unroute_all(behavior="ignoreErrors") async def test_sse_keepalive_comments_arrive(managed_gateway_server): diff --git a/tests/e2e/scenarios/test_telegram_e2e.py b/tests/e2e/scenarios/test_telegram_e2e.py index 4098a0eb4b9..112cac8ff17 100644 --- a/tests/e2e/scenarios/test_telegram_e2e.py +++ b/tests/e2e/scenarios/test_telegram_e2e.py @@ -8,14 +8,29 @@ import json import os import re +import shutil import time from itertools import count +from pathlib import Path import httpx import pytest from helpers import api_post, auth_headers +# Local WASM build (relative to repo root) — when present, overlays the +# release artifact downloaded by `/api/extensions/install` so tests run +# against the source tree's pairing-reply text and other recent +# behaviour rather than the last-published binary. +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_LOCAL_TELEGRAM_WASM = ( + _REPO_ROOT + / "channels-src/telegram/target/wasm32-wasip2/release/telegram_channel.wasm" +) +_LOCAL_TELEGRAM_CAPS = ( + _REPO_ROOT / "channels-src/telegram/telegram.capabilities.json" +) + # Bot token used throughout these tests. BOT_TOKEN = "111222333:FAKE_E2E_TOKEN" # Owner user id used in subsequent Telegram messages. @@ -46,8 +61,15 @@ async def reset_fake_tg(fake_tg_url: str): await c.post(f"{fake_tg_url}/__mock/reset") -async def install_telegram(base_url: str): - """Install the bundled Telegram WASM channel if not already installed.""" +async def install_telegram(base_url: str, channels_dir: str | None = None): + """Install the bundled Telegram WASM channel if not already installed. + + The install API downloads the published release artifact, which lags + behind the source tree until the next release is cut. When a locally- + built WASM is available, overlay it on top of the downloaded file so + tests exercise the source-tree behaviour (e.g. the pairing-reply + wording in #3317) instead of the previous release's bytes. + """ r = await api_post( base_url, "/api/extensions/install", @@ -59,6 +81,21 @@ async def install_telegram(base_url: str): f"Telegram install failed ({r.status_code}): {r.text}" ) + if channels_dir and _LOCAL_TELEGRAM_WASM.exists(): + try: + shutil.copy(_LOCAL_TELEGRAM_WASM, Path(channels_dir) / "telegram.wasm") + if _LOCAL_TELEGRAM_CAPS.exists(): + shutil.copy( + _LOCAL_TELEGRAM_CAPS, + Path(channels_dir) / "telegram.capabilities.json", + ) + except OSError as e: + # Don't hard-fail if the overlay can't land (read-only FS in some + # CI sandboxes). The downloaded artifact still works for tests + # that don't rely on source-tree-only behaviour; assertions that + # do will fail loudly with a clear diff. + print(f"[install_telegram] local WASM overlay skipped: {e}") + def _patch_capabilities_for_testing(channels_dir: str): """Patch the installed capabilities file for E2E testing. @@ -124,7 +161,7 @@ async def activate_telegram( return await reset_fake_tg(fake_tg_url) - await install_telegram(base_url) + await install_telegram(base_url, channels_dir) # Patch capabilities for testing (remove validation_endpoint, ensure # webhook secret is declared in required_secrets). diff --git a/tests/e2e/scenarios/test_telegram_pairing_chat_claim.py b/tests/e2e/scenarios/test_telegram_pairing_chat_claim.py new file mode 100644 index 00000000000..a54bea1d8b0 --- /dev/null +++ b/tests/e2e/scenarios/test_telegram_pairing_chat_claim.py @@ -0,0 +1,473 @@ +"""Regression coverage for #3317 — Telegram pairing chat-claim flow. + +The user-visible bug: the Telegram bot's pairing reply said +"Enter this code in IronClaw to pair your Telegram account: <code>" without +naming a specific surface, so users naturally pasted the code into their +TUI/CLI chat. The agent rejected it ("wrong place; send it in Telegram"), +leaving them stuck. + +This scenario asserts both halves of the fix: + +1. The pairing reply now lists every IronClaw surface explicitly + (Settings → Channels, agent chat, terminal CLI), so the user knows + exactly where to type the code. + +2. Typing ``approve telegram <code>`` in any chat surface — including + the gateway's `/api/chat/send` — actually completes the pairing, + matching the bot reply's instructions. + +Without this coverage, the surface-explicit reply could quietly regress +to a generic "Enter this code in IronClaw" wording, or the chat-claim +parser could be unhooked from the bridge handler, and #3317 would +silently come back. +""" + +import asyncio +import json +import time + +import pytest + +from helpers import api_post, sse_stream + +from .test_telegram_e2e import ( + _LOCAL_TELEGRAM_WASM, + PAIRED_USER_ID, + WEBHOOK_SECRET, + _next_test_update_id, + activate_telegram, + extract_pairing_code, + post_telegram_webhook, + reset_fake_tg, + wait_for_sent_messages, +) + + +async def _send_and_collect_response( + base_url: str, + *, + thread_id: str, + content: str, + predicate, + timeout: float = 30.0, +) -> str: + """Send a chat message and return the matching `response` SSE event. + + `Submission::PairingClaim` is handled by the bridge layer and the reply + is delivered through `WebChannel::respond` → `AppEvent::Response` over + SSE only — no `Turn` is persisted, so polling `/api/chat/history` + cannot see it. The chat-surface tests therefore have to listen on the + same event stream the browser/TUI does, and the SSE stream must be + open *before* the send so the response event isn't missed in the + fan-out window. + """ + matched: list[str] = [] + + async def collect(): + async with sse_stream(base_url, timeout=timeout + 5) as resp: + # Note we're connected; the bridge response is broadcast after + # this point. + collect_started.set() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + line_bytes = await asyncio.wait_for( + resp.content.readline(), timeout=remaining + ) + except asyncio.TimeoutError: + break + if not line_bytes: + break + line = line_bytes.decode("utf-8", errors="replace").strip() + if not line.startswith("data:"): + continue + try: + event = json.loads(line[5:].strip()) + except json.JSONDecodeError: + continue + if event.get("type") != "response": + continue + if event.get("thread_id") != thread_id: + continue + content_field = event.get("content", "") + if predicate(content_field): + matched.append(content_field) + return + + collect_started = asyncio.Event() + collector = asyncio.create_task(collect()) + try: + # Wait for the SSE stream to attach so the broadcast doesn't fan + # out to zero subscribers before the send completes. + await asyncio.wait_for(collect_started.wait(), timeout=10) + + send_r = await api_post( + base_url, + "/api/chat/send", + json={"content": content, "thread_id": thread_id}, + timeout=30, + ) + assert send_r.status_code in (200, 202), ( + f"chat send failed ({send_r.status_code}): {send_r.text}" + ) + + await asyncio.wait_for(collector, timeout=timeout + 5) + finally: + if not collector.done(): + collector.cancel() + try: + await collector + except (asyncio.CancelledError, Exception): + pass + + assert matched, ( + f"No matching `response` SSE event arrived within {timeout}s " + f"for thread {thread_id}" + ) + return matched[0] + + +async def test_telegram_pairing_reply_names_every_surface( + isolated_telegram_e2e_server, +): + """The bot's pairing reply must name web Settings, agent chat, and CLI. + + A regression that drops any of those three surfaces would re-create + the ambiguity that #3317 surfaced (user pastes code into TUI, no + handler matches, agent improvises an unhelpful reply). + + The reply text lives inside the Telegram WASM channel binary, so this + assertion only runs when a locally-built WASM is available to overlay + onto the registry-downloaded artifact. CI workflows that don't build + `channels-src/telegram/` skip this scenario; the canary lane in + `scripts/live_canary/auth_registry.py` covers the same wording + against the deployed binary. + """ + if not _LOCAL_TELEGRAM_WASM.exists(): + pytest.skip( + "Locally-built Telegram WASM not present at " + f"{_LOCAL_TELEGRAM_WASM} — pairing-reply wording asserts " + "source-tree text and can't be exercised against the registry " + "artifact." + ) + base_url = isolated_telegram_e2e_server["base_url"] + http_url = isolated_telegram_e2e_server["http_url"] + fake_tg_url = isolated_telegram_e2e_server["fake_tg_url"] + channels_dir = isolated_telegram_e2e_server["channels_dir"] + + await activate_telegram(base_url, http_url, fake_tg_url, channels_dir) + await reset_fake_tg(fake_tg_url) + + # Trigger the pairing reply by DM'ing the bot from an unknown user. + pairing_resp = await post_telegram_webhook( + http_url, + { + "update_id": _next_test_update_id(), + "message": { + "message_id": 5001, + "from": { + "id": PAIRED_USER_ID, + "is_bot": False, + "first_name": "Pairing Tester", + }, + "chat": {"id": PAIRED_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": "hello", + }, + }, + secret=WEBHOOK_SECRET, + ) + assert pairing_resp.status_code == 200 + + messages = await wait_for_sent_messages(fake_tg_url, min_count=1, timeout=60) + pairing_text = next( + (m["text"] for m in reversed(messages) if "pair" in m.get("text", "").lower()), + None, + ) + assert pairing_text, f"No pairing-reply text found in: {messages}" + + # Every surface must be named so users know where the code is valid. + assert "Settings" in pairing_text and "Channels" in pairing_text, ( + f"pairing reply must mention Settings → Channels: {pairing_text}" + ) + assert "approve telegram" in pairing_text, ( + f"pairing reply must mention chat-surface command 'approve telegram': " + f"{pairing_text}" + ) + assert "ironclaw pairing approve telegram" in pairing_text, ( + f"pairing reply must mention CLI fallback: {pairing_text}" + ) + + # Telegram itself must NOT be advertised as a chat surface for the + # `approve telegram CODE` command. The recipient is by definition + # unpaired, so their DMs are intercepted by the allowlist gate in + # the WASM channel before the agent parser ever sees the command — + # they'd just get another pairing reply. Pairing approval requires + # an already-authenticated IronClaw surface (web / TUI / CLI). + # Reference: review on PR #3381. + assert "TUI / web / Telegram" not in pairing_text, ( + "pairing reply must not list Telegram itself as a chat surface " + f"for the approve command: {pairing_text}" + ) + + code = extract_pairing_code(messages) + assert code, f"Expected pairing code in reply, got: {pairing_text}" + assert code.isalnum() and code.isupper(), ( + f"pairing code must be alphanumeric uppercase, got: {code!r}" + ) + + +async def test_chat_surface_approves_pairing_code( + isolated_telegram_e2e_server, +): + """Typing `approve telegram CODE` in chat completes pairing end-to-end. + + The chat surface here is the web gateway's `/api/chat/send`, but the + same parser runs for TUI/CLI/Telegram-itself. We then verify the + paired user can actually exchange messages — proving the pairing + propagated to the running WASM channel via + `complete_pairing_approval`, not just the DB row. + """ + base_url = isolated_telegram_e2e_server["base_url"] + http_url = isolated_telegram_e2e_server["http_url"] + fake_tg_url = isolated_telegram_e2e_server["fake_tg_url"] + channels_dir = isolated_telegram_e2e_server["channels_dir"] + + await activate_telegram(base_url, http_url, fake_tg_url, channels_dir) + await reset_fake_tg(fake_tg_url) + + # Step 1 — DM the bot from an unknown user to mint a pairing code. + pairing_resp = await post_telegram_webhook( + http_url, + { + "update_id": _next_test_update_id(), + "message": { + "message_id": 6001, + "from": { + "id": PAIRED_USER_ID, + "is_bot": False, + "first_name": "Chat-Claim Tester", + }, + "chat": {"id": PAIRED_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": "hello", + }, + }, + secret=WEBHOOK_SECRET, + ) + assert pairing_resp.status_code == 200 + + pairing_messages = await wait_for_sent_messages( + fake_tg_url, min_count=1, timeout=60 + ) + code = extract_pairing_code(pairing_messages) + assert code, f"Expected pairing code, got messages: {pairing_messages}" + await reset_fake_tg(fake_tg_url) + + # Step 2 — Submit the pairing claim through the chat surface that + # users naturally try first. This is the exact path #3317 said was + # rejected before the fix. + thread_r = await api_post(base_url, "/api/chat/thread/new", timeout=15) + thread_r.raise_for_status() + thread_id = thread_r.json()["id"] + + pairing_response = await _send_and_collect_response( + base_url, + thread_id=thread_id, + content=f"approve telegram {code}", + predicate=lambda c: ( + "Pairing approved" in c + or "Pairing was approved" in c + or "Invalid or expired pairing code" in c + ), + timeout=30, + ) + + assert "Pairing approved" in pairing_response, ( + f"Expected successful pairing, got: {pairing_response}" + ) + assert "telegram" in pairing_response, ( + f"Pairing response must name the channel: {pairing_response}" + ) + + # Step 3 — Prove the pairing actually propagated: the previously- + # unknown PAIRED_USER_ID should now exchange messages without + # triggering another pairing reply. + await reset_fake_tg(fake_tg_url) + paired_resp = await post_telegram_webhook( + http_url, + { + "update_id": _next_test_update_id(), + "message": { + "message_id": 6002, + "from": { + "id": PAIRED_USER_ID, + "is_bot": False, + "first_name": "Chat-Claim Tester", + }, + "chat": {"id": PAIRED_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": "hello again", + }, + }, + secret=WEBHOOK_SECRET, + ) + assert paired_resp.status_code == 200 + + follow_up_messages = await wait_for_sent_messages( + fake_tg_url, min_count=1, timeout=60 + ) + follow_up_text = "\n".join(m.get("text", "") for m in follow_up_messages) + assert "approve telegram" not in follow_up_text, ( + f"Paired user must not receive another pairing reply, got: {follow_up_text}" + ) + assert any( + m.get("chat_id") == PAIRED_USER_ID for m in follow_up_messages + ), ( + f"Expected at least one reply addressed to PAIRED_USER_ID after pairing, " + f"got: {follow_up_messages}" + ) + + +async def test_chat_surface_rejects_invalid_pairing_code( + isolated_telegram_e2e_server, +): + """Garbage codes get a clear 'invalid or expired' response, not a stuck thread. + + A second regression class #3317 hinted at: silent rejection. If the + chat handler just routed bad codes back to the LLM, the user would + again see an improvised "wrong place" reply. The handler must + distinguish "valid syntax, unknown code" from "garbage input" and + surface that as a normal turn response. + """ + base_url = isolated_telegram_e2e_server["base_url"] + http_url = isolated_telegram_e2e_server["http_url"] + fake_tg_url = isolated_telegram_e2e_server["fake_tg_url"] + channels_dir = isolated_telegram_e2e_server["channels_dir"] + + await activate_telegram(base_url, http_url, fake_tg_url, channels_dir) + await reset_fake_tg(fake_tg_url) + + thread_r = await api_post(base_url, "/api/chat/thread/new", timeout=15) + thread_r.raise_for_status() + thread_id = thread_r.json()["id"] + + invalid_response = await _send_and_collect_response( + base_url, + thread_id=thread_id, + content="approve telegram NOSUCHCODE99", + predicate=lambda c: "Invalid or expired pairing code" in c, + timeout=30, + ) + + assert "Invalid or expired pairing code" in invalid_response, ( + f"Invalid pairing claim must surface a clear rejection, got: {invalid_response}" + ) + + +async def test_telegram_dm_approve_command_is_intercepted_by_allowlist_gate( + isolated_telegram_e2e_server, +): + """An unpaired Telegram user cannot complete pairing from Telegram itself. + + Reviewer concern on PR #3381: if the bot's pairing reply tells users + they can type `approve telegram CODE` "in any IronClaw chat + (TUI / web / Telegram)", a user following that instruction *back into + Telegram* gets the message intercepted by `handle_message`'s + allowlist gate before the agent parser ever sees it — they just get + another pairing reply. + + The fix is to remove Telegram from the surfaces the bot promises. + This test locks in the channel-layer behavior that motivates the + fix: an unpaired DM containing `approve telegram CODE` must NOT + complete pairing — it must be re-issued as a pairing reply (or, at + worst, ignored). It exercises the Telegram webhook path, not + `/api/chat/send`, so it covers exactly the layer the reviewer + flagged. + """ + if not _LOCAL_TELEGRAM_WASM.exists(): + pytest.skip( + "Locally-built Telegram WASM not present at " + f"{_LOCAL_TELEGRAM_WASM} — channel-layer interception is in " + "the WASM binary and can't be exercised against the registry " + "artifact." + ) + base_url = isolated_telegram_e2e_server["base_url"] + http_url = isolated_telegram_e2e_server["http_url"] + fake_tg_url = isolated_telegram_e2e_server["fake_tg_url"] + channels_dir = isolated_telegram_e2e_server["channels_dir"] + + await activate_telegram(base_url, http_url, fake_tg_url, channels_dir) + await reset_fake_tg(fake_tg_url) + + # Step 1 — DM the bot to mint a pairing code. + pairing_resp = await post_telegram_webhook( + http_url, + { + "update_id": _next_test_update_id(), + "message": { + "message_id": 7001, + "from": { + "id": PAIRED_USER_ID, + "is_bot": False, + "first_name": "Allowlist Gate Tester", + }, + "chat": {"id": PAIRED_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": "hello", + }, + }, + secret=WEBHOOK_SECRET, + ) + assert pairing_resp.status_code == 200 + + pairing_messages = await wait_for_sent_messages( + fake_tg_url, min_count=1, timeout=60 + ) + code = extract_pairing_code(pairing_messages) + assert code, f"Expected pairing code, got messages: {pairing_messages}" + await reset_fake_tg(fake_tg_url) + + # Step 2 — Same unpaired user types `approve telegram CODE` back + # into the Telegram DM. The allowlist gate in `handle_message` + # must intercept this before the agent parser sees it. + claim_resp = await post_telegram_webhook( + http_url, + { + "update_id": _next_test_update_id(), + "message": { + "message_id": 7002, + "from": { + "id": PAIRED_USER_ID, + "is_bot": False, + "first_name": "Allowlist Gate Tester", + }, + "chat": {"id": PAIRED_USER_ID, "type": "private"}, + "date": int(time.time()), + "text": f"approve telegram {code}", + }, + }, + secret=WEBHOOK_SECRET, + ) + assert claim_resp.status_code == 200 + + follow_up_messages = await wait_for_sent_messages( + fake_tg_url, min_count=1, timeout=60 + ) + follow_up_text = "\n".join(m.get("text", "") for m in follow_up_messages) + + # The user must NOT see "Pairing approved" — they're still unpaired, + # the channel layer correctly didn't route their DM to the agent. + assert "Pairing approved" not in follow_up_text, ( + "An unpaired Telegram DM must not complete pairing approval; " + f"got: {follow_up_text}" + ) + # And they should see *another* pairing reply (the channel layer + # treats every unauthorized DM as a fresh pairing request). + assert "Pair this Telegram account" in follow_up_text, ( + "Unpaired Telegram DM must produce another pairing reply, not " + f"silent intake: {follow_up_text}" + ) diff --git a/tests/e2e/scenarios/test_tool_approval.py b/tests/e2e/scenarios/test_tool_approval.py index fd70cd6d75e..9c27426e28e 100644 --- a/tests/e2e/scenarios/test_tool_approval.py +++ b/tests/e2e/scenarios/test_tool_approval.py @@ -113,7 +113,7 @@ async def test_approval_card_appears(page): # Check card contents header = card.locator(SEL["approval_header"].replace(".approval-card ", "")) - assert await header.text_content() == "Tool requires approval" + assert await header.text_content() == "Approve tool call" tool_name = card.locator(".approval-tool-name") assert await tool_name.text_content() == "shell" diff --git a/tests/e2e/scenarios/test_v2_auth_oauth_matrix.py b/tests/e2e/scenarios/test_v2_auth_oauth_matrix.py index d662898dd9d..8349b707cec 100644 --- a/tests/e2e/scenarios/test_v2_auth_oauth_matrix.py +++ b/tests/e2e/scenarios/test_v2_auth_oauth_matrix.py @@ -37,7 +37,6 @@ api_post, create_member_user, open_authed_page, - send_chat_and_wait_for_terminal_message, sse_stream, wait_for_ready, ) @@ -213,10 +212,11 @@ def _write_google_skill(skills_dir: str, mock_api_host: str) -> None: f"""--- name: google_auth_matrix version: "1.0.0" -keywords: - - google - - drive - - gmail +activation: + keywords: + - google + - drive + - gmail credentials: - name: google_oauth_token provider: google @@ -303,6 +303,35 @@ async def _pin_mock_llm_settings(base_url: str, mock_llm_server: str) -> None: ) +async def _set_tool_permission( + base_url: str, tool_name: str, state: str +) -> None: + """Override a tool's permission state via the settings API. + + Post-#3559 (security-review follow-up to #3533): no DB row exists + for tools the user hasn't explicitly customized — the seeder was + removed and a one-shot startup migration deletes ghost-seeded rows. + Any value written through this helper is therefore a true user + override and `AGENT_AUTO_APPROVE_TOOLS=true` will NOT bypass it. + Use this helper to: (a) pre-approve a tool with no seeded default + so a post-install retry doesn't gate, or (b) force a specific + permission for tests that intentionally exercise the gate path + (typically combined with `AGENT_AUTO_APPROVE_TOOLS=false`). + """ + headers = {"Authorization": f"Bearer {AUTH_TOKEN}"} + async with httpx.AsyncClient() as client: + response = await client.put( + f"{base_url}/api/settings/tool_permissions.{tool_name}", + headers=headers, + json={"value": state}, + timeout=15, + ) + assert response.status_code in (200, 201, 204), ( + f"failed to set tool_permissions.{tool_name}={state}: " + f"{response.status_code} {response.text[:300]}" + ) + + async def _start_auth_matrix_server( ironclaw_binary: str, mock_llm_server: str, @@ -310,6 +339,7 @@ async def _start_auth_matrix_server( *, exchange_url: str, existing_paths: dict | None = None, + auto_approve_tools: bool = True, ): reserved = [] for _ in range(2): @@ -381,6 +411,12 @@ async def _start_auth_matrix_server( "WASM_TOOLS_DIR": tools_dir, "WASM_CHANNELS_DIR": channels_dir, "ONBOARD_COMPLETED": "true", + # Auto-approve administrative tools so the chat-driven install + # path (`tool_install` from chat in #3533) runs without a human + # approval prompt. Authentication gates remain active. Tests + # that exercise the explicit approval path opt out of this via + # the `auto_approve_tools=False` fixture parameter. + "AGENT_AUTO_APPROVE_TOOLS": "true" if auto_approve_tools else "false", "IRONCLAW_OAUTH_CALLBACK_URL": "https://oauth.test.example/oauth/callback", "IRONCLAW_OAUTH_EXCHANGE_URL": exchange_url, # The exchange proxy runs on 127.0.0.1 in tests; the SSRF guard @@ -657,6 +693,41 @@ async def auth_matrix_repl(ironclaw_binary, mock_llm_server): await mock_api["runner"].cleanup() +@pytest.fixture +async def auth_matrix_server_no_auto_approve(ironclaw_binary, mock_llm_server): + """Sibling of `auth_matrix_server` that disables tool auto-approve. + + Used by `test_chat_install_approval_then_auth_card` so the explicit + approval gate for `tool_install` actually fires through to the + user-facing approval card. + """ + mock_api = await _start_mock_google_api() + server = await _start_auth_matrix_server( + ironclaw_binary, + mock_llm_server, + mock_api["base_url"], + exchange_url=mock_llm_server, + auto_approve_tools=False, + ) + try: + yield server + finally: + await _shutdown_auth_matrix_server(server) + await mock_api["runner"].cleanup() + + +@pytest.fixture +async def auth_matrix_page_no_auto_approve(browser, auth_matrix_server_no_auto_approve): + context = await browser.new_context(viewport={"width": 1280, "height": 720}) + page = await context.new_page() + await page.goto(f"{auth_matrix_server_no_auto_approve['base_url']}/?token={AUTH_TOKEN}") + await page.wait_for_selector("#auth-screen", state="hidden", timeout=15000) + try: + yield page + finally: + await context.close() + + @pytest.fixture async def auth_matrix_page(browser, auth_matrix_server): context = await browser.new_context(viewport={"width": 1280, "height": 720}) @@ -802,6 +873,18 @@ async def _read_repl_until_any( raise AssertionError(f"Matched union {union!r} but no individual pattern matched") +async def _try_read_repl_until_any( + repl: dict, + patterns: list[str], + *, + timeout: float = 30.0, +) -> tuple[str, str] | None: + try: + return await _read_repl_until_any(repl, patterns, timeout=timeout) + except AssertionError: + return None + + async def _drain_repl_output(repl: dict, *, idle_secs: float = 0.4) -> str: chunks: list[str] = [] while True: @@ -1074,12 +1157,15 @@ async def _wait_for_tool_call( thread_id: str, tool_name: str, timeout: float = 30.0, + *, + token: str = AUTH_TOKEN, ) -> dict: approved_request_ids = set() for _ in range(int(timeout * 2)): response = await api_get( base_url, f"/api/chat/history?thread_id={thread_id}", + token=token, timeout=15, ) response.raise_for_status() @@ -1090,6 +1176,7 @@ async def _wait_for_tool_call( approve = await api_post( base_url, "/api/chat/approval", + token=token, json={ "request_id": pending["request_id"], "action": "approve", @@ -1550,22 +1637,33 @@ async def test_mcp_same_server_multi_user_via_browser(browser, auth_matrix_serve browser, server["base_url"], token=member["token"] ) try: - owner_result = await send_chat_and_wait_for_terminal_message( - owner_page, - "check mock mcp search", - timeout=300000, - expected_text_contains="Mock MCP search result", + # Send the chat through each user's browser session. Engine v2 opens + # an `approval` pending_gate on the first MCP tool call; the browser + # has no auto-approve UI in this fixture, so drive approval through + # the per-user API while polling for the tool call to land. Without + # this, both pages hang in the streaming predicate forever. + await owner_page.locator(SEL["chat_input"]).fill("check mock mcp search") + await owner_page.locator(SEL["chat_input"]).press("Enter") + await member_page.locator(SEL["chat_input"]).fill("check mock mcp search") + await member_page.locator(SEL["chat_input"]).press("Enter") + + owner_thread = await _current_thread_id(owner_page) + member_thread = await _current_thread_id(member_page) + + await _wait_for_tool_call( + server["base_url"], + owner_thread, + "mock_mcp_mock_search", + timeout=60.0, + token=AUTH_TOKEN, ) - member_result = await send_chat_and_wait_for_terminal_message( - member_page, - "check mock mcp search", - timeout=300000, - expected_text_contains="Mock MCP search result", + await _wait_for_tool_call( + server["base_url"], + member_thread, + "mock_mcp_mock_search", + timeout=60.0, + token=member["token"], ) - assert owner_result["role"] == "assistant", owner_result - assert member_result["role"] == "assistant", member_result - assert "Mock MCP search result" in owner_result["text"], owner_result - assert "Mock MCP search result" in member_result["text"], member_result mcp_state = await _get_mock_mcp_state(server["mock_llm_url"]) tool_call_auths = { @@ -1580,25 +1678,22 @@ async def test_mcp_same_server_multi_user_via_browser(browser, auth_matrix_serve await member_context.close() -@pytest.mark.xfail( - strict=False, - reason=( - "Engine does not yet auto-install registry extensions on LLM latent " - "action invocation. ensure_extension_ready(UseCapability) surfaces " - "NotInstalled intentionally (see src/extensions/manager.rs ~L1680 " - "comment: 'path must surface as NotInstalled so the bridge can route " - "it through the approval/install gate'), but the bridge-side install/" - "approval gate that would turn that into an auth card is not " - "implemented in src/bridge/effect_adapter.rs. The chat simply fails " - "with 'Extension not installed'. Tracked as a follow-up." - ), -) async def test_chat_first_gmail_installs_prompts_and_retries( auth_matrix_server, auth_matrix_page ): server = auth_matrix_server page = auth_matrix_page await _remove_extension_if_present(server["base_url"], "gmail") + # The fixture sets `AGENT_AUTO_APPROVE_TOOLS=true`. Post-#3559 + # (security-review follow-up to #3533), the boot-time seeder that + # wrote ghost `tool_install = AskEachTime` rows has been removed + # and a startup migration cleans up any pre-existing ghosts. With + # no DB row, `effective_permission` falls back to the code-level + # `AskEachTime` baseline, which is implicit — so the env knob + # bypasses the gate without `_set_tool_permission` having to force + # `always_allow`. A user who deliberately picks `AskEachTime` + # through the settings UI WOULD have it respected (regression + # covered by `bridge::tool_permissions::tests`). chat_input = page.locator(SEL["chat_input"]) await chat_input.fill("check gmail unread") @@ -1633,6 +1728,90 @@ async def test_chat_first_gmail_installs_prompts_and_retries( assert extension["active"] is True, extension +async def test_chat_install_approval_then_auth_card( + auth_matrix_server_no_auto_approve, auth_matrix_page_no_auto_approve +): + """#3533: chat-driven `tool_install` raises an approval gate. + + Sibling to `test_chat_first_gmail_installs_prompts_and_retries`. The + other test pre-approves `tool_install` so install completes silently + and only the auth card surfaces. This one keeps the seeded + `AskEachTime` default so the explicit approval flow is exercised: + + 1. User types "check gmail unread". + 2. Mock LLM dispatches `gmail()` → engine rejects (not installed). + 3. Mock LLM dispatches `tool_install("gmail")` → engine raises an + **Approval gate**, surfacing the `.approval-card`. + 4. Test clicks the card's Approve button. + 5. Install completes, gmail registers; the engine retries the next + turn with the **Authentication gate** that surfaces the + `.auth-card`. + 6. Test completes OAuth via `/oauth/callback`. + 7. Gmail tool runs against the mock Google API and the final + response contains the canned subject line. + """ + server = auth_matrix_server_no_auto_approve + page = auth_matrix_page_no_auto_approve + await _remove_extension_if_present(server["base_url"], "gmail") + # Note: deliberately NOT pre-approving `tool_install` here so the + # approval gate fires and the approval card surfaces in the UI. + # The dedicated `_no_auto_approve` fixture passes + # `AGENT_AUTO_APPROVE_TOOLS=false` so the env knob doesn't bypass + # the code-level `AskEachTime` baseline for `tool_install`. + # Pre-approve `gmail` so the post-install retry doesn't *also* gate + # — this test isolates the explicit-approval path for `tool_install` + # specifically. (Gmail has no seeded permission default, so without + # `AGENT_AUTO_APPROVE_TOOLS=true` it would otherwise gate too.) + await _set_tool_permission(server["base_url"], "gmail", "always_allow") + + chat_input = page.locator(SEL["chat_input"]) + await chat_input.fill("check gmail unread") + await chat_input.press("Enter") + + approval_card = page.locator(".approval-card").first + await approval_card.wait_for(state="visible", timeout=20000) + assert await approval_card.get_attribute("data-request-id"), ( + "expected approval gate request id on the approval card" + ) + tool_name_text = await approval_card.locator(".approval-tool-name").text_content() + assert tool_name_text and "install" in tool_name_text.lower(), ( + f"approval card should be for tool_install, got: {tool_name_text!r}" + ) + + # Single "Approve" click is sufficient. The stack of #3533 fixes + # (`resume_output` on InlineGate so inline-await doesn't re-execute + # `tool_install`, OAuth callback skipping `ExternalCallback` when an + # inline waiter is already in flight, and discarding the matching + # Authentication pending-gate row when the inline path delivers + # Approved) means no second `tool_install` dispatch fires, so a + # plain "Approve" suffices — no "Always" workaround needed. + await approval_card.locator("button.approve").click() + await approval_card.locator(".approval-resolved").wait_for( + state="visible", timeout=10000 + ) + + auth_card = await _wait_for_auth_card(page) + assert await auth_card.get_attribute("data-extension-name") in { + "gmail", + "google_oauth_token", + } + auth_url = await _auth_oauth_url_from_card(page) + assert auth_url, "Expected auth card to expose an OAuth URL" + response = await _complete_callback( + server["base_url"], auth_url, code="mock_auth_code" + ) + assert response.status_code == 200, response.text[:400] + await auth_card.wait_for(state="hidden", timeout=20000) + + thread_id = await _current_thread_id(page) + tokens = await _wait_for_mock_google_tokens(server["mock_api_url"], timeout=60.0) + assert tokens, "expected Gmail to hit the mock Google API after OAuth replay" + history = await _wait_for_response_contains( + server["base_url"], thread_id, "Quarterly update", timeout=60.0 + ) + assert history.get("pending_gate") is None, history + + async def test_settings_first_gmail_auth_then_chat_runs( auth_matrix_server, auth_matrix_page ): @@ -1681,18 +1860,6 @@ async def test_settings_first_gmail_auth_then_chat_runs( ) -@pytest.mark.xfail( - strict=False, - reason=( - "After settings-first MCP install + OAuth + chat, the mock LLM never " - "observes a follow-up request containing 'Tool `mock_mcp_mock_search` " - "returned', meaning the MCP tool output isn't feeding back to the LLM. " - "test_mcp_oauth_roundtrip proves the MCP OAuth flow itself works, and " - "test_mcp_oauth_refresh_on_demand proves chat-driven MCP invocation " - "does reach the server; the gap is specific to post-auth tool-output " - "propagation through the settings-first UI path. Needs deeper debug." - ), -) async def test_settings_first_custom_mcp_auth_then_chat_runs( auth_matrix_server, auth_matrix_page ): @@ -1734,6 +1901,14 @@ async def test_settings_first_custom_mcp_auth_then_chat_runs( await chat_input.press("Enter") thread_id = await _current_thread_id(page) + # Engine v2 gates the first MCP tool call on `approval` before it runs; + # the browser fixture has no auto-approve UI, so drive approval through + # the API while polling for the tool to land. Same pattern as + # test_wasm_tool_oauth_refresh_on_demand and + # test_mcp_same_server_multi_user_via_browser (#3235). + await _wait_for_tool_call( + server["base_url"], thread_id, "mock_mcp_mock_search", timeout=60.0 + ) history = await _wait_for_response_contains( server["base_url"], thread_id, "Mock MCP search result", timeout=60.0 ) @@ -1863,6 +2038,15 @@ async def test_wasm_tool_oauth_refresh_on_demand(auth_matrix_server): thread_id = await _create_thread(server["base_url"]) await _send_chat(server["base_url"], thread_id, "check gmail unread") + # Engine v2 gates the gmail call on `approval` before reaching the + # http credential-injection layer that performs the refresh. Without + # approving, the chat sits in pending_gate forever and the refresh + # endpoint is never hit. Drive approval through the API while waiting + # for the tool to land. + await _wait_for_tool_call( + server["base_url"], thread_id, "gmail", timeout=30.0 + ) + oauth_state = await _wait_for_refresh_request(server["mock_llm_url"]) assert oauth_state["refresh_count"] >= 1, oauth_state @@ -1982,16 +2166,29 @@ async def test_repl_http_auth_prompt_accepts_token_and_retries(auth_matrix_repl) "OAuth callback paths are covered by other auth-matrix tests." ) - await _drain_repl_output(repl) - await _send_repl_line(repl, prompt) - output, matched = await _read_repl_until_any( - repl, - [ - r"The http tool returned:|Budget Q1\.xlsx|Roadmap\.md", - r"requires approval|Reply .*yes.*approve", - ], - timeout=60.0, - ) + result_patterns = [ + r"The http tool returned:|Budget Q1\.xlsx|Roadmap\.md", + r"requires approval|Reply .*yes.*approve", + ] + + # Token entry resolves the inline auth gate and the suspended CodeAct turn + # resumes asynchronously. Under coverage CI that resume can still be + # processing after the secret row appears; sending a duplicate prompt at + # that point races the active REPL turn and can leave the test waiting on + # the duplicate while the original turn owns the spinner. Prefer the + # resumed original output, and only fall back to a manual retry if no + # output appears. + resumed = await _try_read_repl_until_any(repl, result_patterns, timeout=60.0) + if resumed is None: + await _drain_repl_output(repl) + await _send_repl_line(repl, prompt) + output, matched = await _read_repl_until_any( + repl, + result_patterns, + timeout=60.0, + ) + else: + output, matched = resumed if "requires approval" in matched.lower() or "reply" in matched.lower(): output += await _drain_repl_output(repl) await _send_repl_line(repl, "yes") diff --git a/tests/e2e/scenarios/test_v2_engine_approval_flow.py b/tests/e2e/scenarios/test_v2_engine_approval_flow.py index 6ae25875ebd..0b21a586b49 100644 --- a/tests/e2e/scenarios/test_v2_engine_approval_flow.py +++ b/tests/e2e/scenarios/test_v2_engine_approval_flow.py @@ -65,6 +65,21 @@ async def _drain_pipes(): await _drain_pipes() +async def _set_tool_permission(base_url: str, tool_name: str, state: str): + """Persist an explicit tool permission override for deterministic approval tests.""" + async with httpx.AsyncClient() as client: + response = await client.put( + f"{base_url}/api/settings/tools/{tool_name}", + json={"state": state}, + headers={"Authorization": f"Bearer {AUTH_TOKEN}"}, + timeout=15, + ) + assert response.status_code == 200, ( + f"Failed to set {tool_name} permission to {state}: " + f"{response.status_code} {response.text}" + ) + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -109,6 +124,8 @@ async def v2_approval_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_V2_APPROVAL_DB_TMPDIR.name, "v2-approval-e2e.db"), @@ -133,6 +150,11 @@ async def v2_approval_server(ironclaw_binary, mock_llm_server): base_url = f"http://127.0.0.1:{gateway_port}" try: await wait_for_ready(f"{base_url}/api/health", timeout=60) + # The production seeded default for the read-only `http` tool is + # `always_allow`. These approval tests intentionally exercise the + # explicit ask-each-time path, so pin the fixture user to that policy + # after startup seeding completes. + await _set_tool_permission(base_url, "http", "ask_each_time") yield base_url except TimeoutError: if proc.returncode is None: @@ -278,25 +300,44 @@ async def _approve( class TestV2EngineApprovalFlow: async def test_same_user_approvals_are_thread_scoped(self, v2_approval_server): + """Two threads for the same user can each park at their own + approval gate concurrently; resolving one does not affect the + other. + + Regression for the inline-gate-park dispatch interaction (#3157 + follow-up). Without the gate-park handoff, thread A's parked + engine call holds the per-user agent loop's `handle_message` + for up to 5 minutes, and thread B's send sits in `msg_tx` + behind it — pending_b never surfaces in this window. The + bridge now spawns a post-park continuation when it detects a + pending gate row for the (user, thread), which lets the agent + loop unblock immediately and dispatch thread B. + """ base_url = v2_approval_server thread_a = (await api_post(base_url, "/api/chat/thread/new", timeout=15)).json()["id"] thread_b = (await api_post(base_url, "/api/chat/thread/new", timeout=15)).json()["id"] + # Sequence the two sends so the second one is dispatched only after + # the first thread's gate has surfaced. With the inline-gate-await + # path (PR #3157) the engine parks the live execution at the gate; + # firing both sends back-to-back races the per-user dispatch with + # the first thread's parked state and previously left both gates + # un-surfaced (see history of this test for the racy variant). await api_post( base_url, "/api/chat/send", json={"content": "make approval post alpha", "thread_id": thread_a}, timeout=30, ) + pending_a = await _wait_for_approval(base_url, thread_a, timeout=60) + await api_post( base_url, "/api/chat/send", json={"content": "make approval post beta", "thread_id": thread_b}, timeout=30, ) - - pending_a = await _wait_for_approval(base_url, thread_a, timeout=60) pending_b = await _wait_for_approval(base_url, thread_b, timeout=60) assert pending_a["request_id"] != pending_b["request_id"] @@ -316,14 +357,18 @@ async def test_same_user_approvals_are_thread_scoped(self, v2_approval_server): """Test the v2 engine tool approval lifecycle. - Uses text-based approval ("yes"/"no"/"always" as chat messages) rather - than the /api/chat/approval endpoint, since the v2 engine's pending_approval - metadata uses engine thread IDs that differ from the v1 session thread IDs - shown in the history API. + Resolves gates via ``POST /api/chat/approval`` because PR #3157 made + Approval gates pause inline inside the running CodeAct script via + ``BridgeGateController::pause``. The legacy "reply with text + 'yes'/'no'/'always' through /api/chat/send" path no longer unsticks + the parked engine — only ``/api/chat/approval`` (which routes + through the controller's in-memory delivery channel) does. The + request_id needed for approval is taken straight from the + ``pending_gate`` payload returned by ``/api/chat/history``. """ async def test_approval_yes(self, v2_approval_server): - """Approve a pending http POST tool call by replying 'yes'.""" + """Approve a pending http POST tool call via /api/chat/approval.""" base = v2_approval_server thread_r = await api_post(base, "/api/chat/thread/new", timeout=15) @@ -337,39 +382,15 @@ async def test_approval_yes(self, v2_approval_server): timeout=30, ) - # Wait for the approval prompt (delivered via pending_gate, not response text) - await _wait_for_approval(base, thread_id, timeout=60) + pending = await _wait_for_approval(base, thread_id, timeout=60) + approve = await _approve(base, thread_id, pending["request_id"], "approve") + assert approve.status_code == 202, approve.text - # Reply "yes" to approve — goes through SubmissionParser as ApprovalResponse - await api_post( - base, "/api/chat/send", - json={"content": "yes", "thread_id": thread_id}, - timeout=30, - ) - - # Wait for the approval to be processed — poll until the response - # changes from the approval prompt (tool executes after approval) - for _ in range(120): - await asyncio.sleep(0.5) - r = await api_get(base, f"/api/chat/history?thread_id={thread_id}", timeout=15) - history = r.json() - turns = history.get("turns", []) - if turns: - last = (turns[-1].get("response") or "").lower() - if last and "requires approval" not in last: - break - # Also check if pending_gate is cleared (approval processed) - if not history.get("pending_gate"): - break - - # After approval, pending_gate should be cleared - assert history.get("pending_gate") is None, ( - f"After approval, pending_gate should be cleared. " - f"Got: {history.get('pending_gate')}" - ) + history = await _wait_for_no_pending_gate(base, thread_id, timeout=60) + assert history.get("pending_gate") is None, history async def test_approval_no(self, v2_approval_server): - """Deny a pending tool call by replying 'no'.""" + """Deny a pending tool call via /api/chat/approval action='deny'.""" base = v2_approval_server thread_r = await api_post(base, "/api/chat/thread/new", timeout=15) @@ -382,47 +403,17 @@ async def test_approval_no(self, v2_approval_server): timeout=30, ) - # Wait for the approval prompt (delivered via pending_gate) - await _wait_for_approval(base, thread_id, timeout=60) - - # Deny - await api_post( - base, "/api/chat/send", - json={"content": "no", "thread_id": thread_id}, - timeout=30, - ) - - # Wait for the denial response — poll until the approval prompt is - # no longer the latest response (meaning the denial was processed) - for _ in range(120): - await asyncio.sleep(0.5) - r = await api_get(base, f"/api/chat/history?thread_id={thread_id}", timeout=15) - history = r.json() - turns = history.get("turns", []) - if turns: - last = (turns[-1].get("response") or "").lower() - # The denial is processed when the last response changes from - # the approval prompt or mentions denial - if last and "requires approval" not in last: - break - if "denied" in last or "rejected" in last: - break - - all_responses = " ".join( - (t.get("response") or "") for t in history.get("turns", []) - ).lower() + pending = await _wait_for_approval(base, thread_id, timeout=60) + deny = await _approve(base, thread_id, pending["request_id"], "deny") + assert deny.status_code == 202, deny.text - # After denial, approval prompt should no longer be pending - assert history.get("pending_gate") is None, ( - f"After denial, pending_gate should be cleared. " - f"Got: {history.get('pending_gate')}" - ) + history = await _wait_for_no_pending_gate(base, thread_id, timeout=60) + assert history.get("pending_gate") is None, history async def test_approval_always(self, v2_approval_server): - """Approve with 'always' — second request auto-approves.""" + """'always' approval clears the gate and auto-approves the next call.""" base = v2_approval_server - # First thread: trigger approval and reply "always" thread_r = await api_post(base, "/api/chat/thread/new", timeout=15) thread_id_1 = thread_r.json()["id"] @@ -432,17 +423,12 @@ async def test_approval_always(self, v2_approval_server): timeout=30, ) - await _wait_for_approval(base, thread_id_1, timeout=60) - - await api_post( - base, "/api/chat/send", - json={"content": "always", "thread_id": thread_id_1}, - timeout=30, - ) - - await _wait_for_response(base, thread_id_1, timeout=60) + pending = await _wait_for_approval(base, thread_id_1, timeout=60) + always = await _approve(base, thread_id_1, pending["request_id"], "always") + assert always.status_code == 202, always.text + await _wait_for_no_pending_gate(base, thread_id_1, timeout=60) - # Second thread: same tool should auto-approve (no pause) + # Second thread: same tool must auto-approve (no pause). thread_r2 = await api_post(base, "/api/chat/thread/new", timeout=15) thread_id_2 = thread_r2.json()["id"] @@ -452,7 +438,6 @@ async def test_approval_always(self, v2_approval_server): timeout=30, ) - # Should complete directly without approval prompt history = await _wait_for_response(base, thread_id_2, timeout=60) all_responses = " ".join( (t.get("response") or "") for t in history.get("turns", []) @@ -461,7 +446,6 @@ async def test_approval_always(self, v2_approval_server): assert "requires approval" not in all_responses, ( f"Second thread should auto-approve. Got: {all_responses[:500]}" ) - # Verify the tool actually ran (not just that approval was skipped) turns = history.get("turns", []) assert len(turns) >= 1, ( f"Expected at least 1 turn with tool execution after auto-approve. " @@ -571,6 +555,8 @@ async def restartable_v2_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_RESTART_DB_TMPDIR.name, "v2-restart-e2e.db"), @@ -655,17 +641,27 @@ async def stop(): await start() try: + # First boot must require approval so the test can persist an + # `always_allow` decision, then verify a later restart honors it. + # Do not reset inside start(), or the restart would erase the state + # this test is specifically validating. + await _set_tool_permission(base_url, "http", "ask_each_time") yield {"base_url": base_url, "start": start, "stop": stop} finally: await stop() async def test_always_approve_survives_restart(restartable_v2_server): - """After 'always' approval and process restart, the tool auto-approves without a gate.""" + """After 'always' approval and process restart, the tool auto-approves without a gate. + + Resolves the gate via ``POST /api/chat/approval`` (action='always') so + the inline-await ``BridgeGateController`` is unparked; the legacy + text-replay path was removed by PR #3157. + """ server = restartable_v2_server base = server["base_url"] - # 1. Trigger approval and reply "always" + # 1. Trigger approval and resolve with action='always' thread_r = await api_post(base, "/api/chat/thread/new", timeout=15) thread_id = thread_r.json()["id"] @@ -675,13 +671,9 @@ async def test_always_approve_survives_restart(restartable_v2_server): timeout=30, ) - await _wait_for_approval(base, thread_id, timeout=60) - - await api_post( - base, "/api/chat/send", - json={"content": "always", "thread_id": thread_id}, - timeout=30, - ) + pending = await _wait_for_approval(base, thread_id, timeout=60) + always = await _approve(base, thread_id, pending["request_id"], "always") + assert always.status_code == 202, always.text await _wait_for_no_pending_gate(base, thread_id, timeout=60) diff --git a/tests/e2e/scenarios/test_v2_engine_auth_cancel.py b/tests/e2e/scenarios/test_v2_engine_auth_cancel.py index 8bf67a3cd0d..ce451de765d 100644 --- a/tests/e2e/scenarios/test_v2_engine_auth_cancel.py +++ b/tests/e2e/scenarios/test_v2_engine_auth_cancel.py @@ -89,7 +89,8 @@ def _write_skill(skills_dir, mock_api_host): f.write(f"""--- name: github version: "1.0.0" -keywords: [github, issues] +activation: + keywords: [github, issues] credentials: - name: github_token provider: github @@ -158,6 +159,8 @@ async def cancel_server(ironclaw_binary, mock_llm_server, cancel_mock_api): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_CANCEL_DB_TMPDIR.name, "cancel-e2e.db"), diff --git a/tests/e2e/scenarios/test_v2_engine_auth_flow.py b/tests/e2e/scenarios/test_v2_engine_auth_flow.py index f86ef73dd01..6b7f340ab09 100644 --- a/tests/e2e/scenarios/test_v2_engine_auth_flow.py +++ b/tests/e2e/scenarios/test_v2_engine_auth_flow.py @@ -176,15 +176,16 @@ def _write_test_skill(skills_dir: str, mock_api_host: str): skill_content = f"""--- name: github version: "1.0.0" -keywords: - - github - - issues - - pull request - - repo - - repository -tags: - - github - - api +activation: + keywords: + - github + - issues + - pull request + - repo + - repository + tags: + - github + - api credentials: - name: github_token provider: github @@ -266,6 +267,8 @@ async def v2_server(ironclaw_binary, mock_llm_server, mock_api): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_V2_DB_TMPDIR.name, "v2-e2e.db"), @@ -361,6 +364,8 @@ async def v2_skill_install_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(db_tmpdir.name, "v2-skill-install.db"), @@ -449,6 +454,8 @@ async def v2_skill_install_server_isolated(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(db_tmpdir.name, "v2-skill-install.db"), @@ -1407,22 +1414,26 @@ async def test_duplicate_install_is_idempotent_and_keeps_single_card( # fire a fresh approval gate. `SkillInstallTool::requires_approval` # short-circuits to `ApprovalRequirement::Never` when the skill # is already loaded — asking the user to approve a guaranteed - # no-op is pure friction. The test used to wait for an approval - # card here and time out; now it skips the approval step and - # goes straight to the terminal message, which must carry the - # idempotent "already installed / no install needed" wording. + # no-op is pure friction. The observable contract here is that the + # repeated request does not open an approval gate and does not create + # duplicate UI affordances, even if the final assistant wording is + # delivered through history/SSE on a different cadence. await _open_chat_tab(v2_skill_page) - baseline = await _message_counts(v2_skill_page) + thread_id = await _wait_for_current_thread_id(v2_skill_page) await _send_chat_message( v2_skill_page, "install https://github.com/Pika-Labs/Pika-Skills", ) - installed = await _wait_for_terminal_message( - v2_skill_page, - timeout=90000, - baseline=baseline, - ) - assert "already" in installed["text"].lower() or "no install needed" in installed["text"].lower(), installed + for _ in range(10): + history_response = await api_get( + base_url, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + history_response.raise_for_status() + history = history_response.json() + assert history.get("pending_gate") is None, history + await asyncio.sleep(0.5) await _open_skills_settings(v2_skill_page) assert await v2_skill_page.locator(SEL["skill_installed"]).filter( diff --git a/tests/e2e/scenarios/test_v2_engine_error_handling.py b/tests/e2e/scenarios/test_v2_engine_error_handling.py index 215ee3a8783..85028ce7ed2 100644 --- a/tests/e2e/scenarios/test_v2_engine_error_handling.py +++ b/tests/e2e/scenarios/test_v2_engine_error_handling.py @@ -121,6 +121,8 @@ async def v2_error_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_V2_ERR_DB_TMPDIR.name, "v2-err-e2e.db"), diff --git a/tests/e2e/scenarios/test_v2_engine_oauth_google.py b/tests/e2e/scenarios/test_v2_engine_oauth_google.py index 3780fdf79a4..b12cf74544d 100644 --- a/tests/e2e/scenarios/test_v2_engine_oauth_google.py +++ b/tests/e2e/scenarios/test_v2_engine_oauth_google.py @@ -180,14 +180,15 @@ def _write_google_skill(skills_dir: str, mock_api_host: str): skill_content = f"""--- name: google_drive version: "1.0.0" -keywords: - - google - - drive - - files - - docs -tags: - - google - - api +activation: + keywords: + - google + - drive + - files + - docs + tags: + - google + - api credentials: - name: google_drive_token provider: google @@ -405,6 +406,8 @@ async def v2_google_server(ironclaw_binary, mock_llm_server, mock_google_api): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": db_path, diff --git a/tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py b/tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py index be1298bda8b..82b7c8333c6 100644 --- a/tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py +++ b/tests/e2e/scenarios/test_v2_engine_tool_lifecycle.py @@ -99,6 +99,8 @@ async def v2_tool_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_V2_TOOL_DB_TMPDIR.name, "v2-tool-e2e.db"), diff --git a/tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py b/tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py index 63b23eff28b..e376a6524b4 100644 --- a/tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py +++ b/tests/e2e/scenarios/test_v2_kernel_auth_gateway_flow.py @@ -112,12 +112,13 @@ def _write_github_skill(skills_dir: str, mock_api_host: str): skill_content = f"""--- name: github version: "1.0.0" -keywords: - - github - - issues - - repo -tags: - - github +activation: + keywords: + - github + - issues + - repo + tags: + - github credentials: - name: github_token provider: github @@ -192,6 +193,8 @@ async def v2_server(ironclaw_binary, mock_llm_server, mock_api): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(_DB_TMPDIR.name, "gw-auth-e2e.db"), diff --git a/tests/e2e/scenarios/test_v2_thread_visibility.py b/tests/e2e/scenarios/test_v2_thread_visibility.py index 80fdd370f49..136796e9e37 100644 --- a/tests/e2e/scenarios/test_v2_thread_visibility.py +++ b/tests/e2e/scenarios/test_v2_thread_visibility.py @@ -87,6 +87,8 @@ async def v2_visibility_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join( diff --git a/tests/e2e/scenarios/test_v2_tool_activate_surface.py b/tests/e2e/scenarios/test_v2_tool_activate_surface.py deleted file mode 100644 index a119ef97fed..00000000000 --- a/tests/e2e/scenarios/test_v2_tool_activate_surface.py +++ /dev/null @@ -1,600 +0,0 @@ -"""E2E test: v2 tool activation surface. - -Verifies the simplified model-facing contract: -- `tool_activate` is the surfaced enablement tool in engine v2 -- `tool_auth` and `tool_install` are not part of the normal surfaced prompt -- blocked managed integrations appear in `Activatable Integrations` -""" - -import asyncio -import json -import os -import signal -import socket -import tempfile -from urllib.parse import parse_qs, urlparse - -import httpx -import pytest - -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from helpers import api_get, api_post, AUTH_TOKEN, wait_for_ready - - -_DB_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-v2-activate-surface-db-") -_HOME_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-v2-activate-surface-home-") - - -def _forward_coverage_env(env: dict): - for key in os.environ: - if key.startswith( - ("CARGO_LLVM_COV", "LLVM_", "CARGO_ENCODED_RUSTFLAGS", "CARGO_INCREMENTAL") - ): - env[key] = os.environ[key] - - -async def _pin_mock_llm_settings(base_url: str, mock_llm_server: str) -> None: - headers = {"Authorization": f"Bearer {AUTH_TOKEN}"} - writes = [ - ("llm_backend", "openai_compatible"), - ("openai_compatible_base_url", mock_llm_server), - ("selected_model", "mock-model"), - ] - async with httpx.AsyncClient() as client: - for key, value in writes: - response = await client.put( - f"{base_url}/api/settings/{key}", - headers=headers, - json={"value": value}, - timeout=15, - ) - assert response.status_code in (200, 201, 204), ( - f"failed to pin {key}: {response.status_code} {response.text[:300]}" - ) - - -async def _stop_process(proc, sig=signal.SIGINT, timeout=5): - async def _drain_pipes(): - try: - await asyncio.wait_for(proc.communicate(), timeout=1) - except (asyncio.TimeoutError, ValueError): - pass - - try: - proc.send_signal(sig) - except ProcessLookupError: - await _drain_pipes() - return - try: - await asyncio.wait_for(proc.wait(), timeout=timeout) - except asyncio.TimeoutError: - proc.kill() - await proc.wait() - await _drain_pipes() - - -async def _start_mock_google_api(): - from aiohttp import web - - received_tokens: list[str] = [] - received_requests: list[str] = [] - messages = [ - { - "id": "msg-1", - "threadId": "thread-1", - "labelIds": ["INBOX", "UNREAD"], - "snippet": "Quarterly update is ready", - "payload": { - "headers": [ - {"name": "Subject", "value": "Quarterly update"}, - {"name": "From", "value": "ceo@example.com"}, - {"name": "To", "value": "surface@example.com"}, - ], - "body": {}, - }, - } - ] - - def _authorized(request: web.Request) -> str | None: - auth = request.headers.get("Authorization", "") - if not auth.startswith("Bearer "): - return None - token = auth.split(" ", 1)[1] - received_tokens.append(token) - return token - - def _record_request(request: web.Request) -> None: - received_requests.append(f"{request.method} {request.path}") - - async def handle_userinfo(request: web.Request) -> web.Response: - _record_request(request) - return web.json_response({"email": "surface@example.com", "name": "Surface User"}) - - async def handle_gmail_messages(request: web.Request) -> web.Response: - _record_request(request) - if _authorized(request) is None: - return web.json_response({"error": "missing_auth"}, status=401) - return web.json_response( - { - "messages": [ - {"id": message["id"], "threadId": message["threadId"]} - for message in messages - ], - "resultSizeEstimate": len(messages), - } - ) - - async def handle_gmail_message(request: web.Request) -> web.Response: - _record_request(request) - if _authorized(request) is None: - return web.json_response({"error": "missing_auth"}, status=401) - message_id = request.match_info["message_id"] - message = next((item for item in messages if item["id"] == message_id), None) - if message is None: - return web.json_response({"error": "not_found"}, status=404) - return web.json_response(message) - - async def handle_received_tokens(request: web.Request) -> web.Response: - return web.json_response({"tokens": received_tokens}) - - async def handle_received_requests(request: web.Request) -> web.Response: - return web.json_response({"requests": received_requests}) - - async def handle_reset(request: web.Request) -> web.Response: - received_tokens.clear() - received_requests.clear() - return web.json_response({"ok": True}) - - app = web.Application() - app.router.add_get("/oauth2/v1/userinfo", handle_userinfo) - app.router.add_get("/oauth2/v2/userinfo", handle_userinfo) - app.router.add_get("/gmail/v1/users/me/messages", handle_gmail_messages) - app.router.add_get("/gmail/v1/users/me/messages/{message_id}", handle_gmail_message) - app.router.add_get("/__mock/received-tokens", handle_received_tokens) - app.router.add_get("/__mock/received-requests", handle_received_requests) - app.router.add_post("/__mock/reset", handle_reset) - - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, "127.0.0.1", 0) - await site.start() - port = site._server.sockets[0].getsockname()[1] - return { - "base_url": f"http://127.0.0.1:{port}", - "runner": runner, - } - - -def _extract_state(auth_url: str) -> str: - parsed = urlparse(auth_url) - state = parse_qs(parsed.query).get("state", [None])[0] - assert state, f"auth_url missing state: {auth_url}" - return state - - -async def _wait_for_response_contains( - base_url: str, - thread_id: str, - needle: str, - *, - timeout: float = 45.0, -) -> dict: - for _ in range(int(timeout * 2)): - response = await api_get(base_url, f"/api/chat/history?thread_id={thread_id}", timeout=15) - response.raise_for_status() - history = response.json() - all_text = " ".join((turn.get("response") or "") for turn in history.get("turns", [])) - if needle.lower() in all_text.lower(): - return history - await asyncio.sleep(0.5) - raise AssertionError(f"Timed out waiting for response containing {needle!r}") - - -async def _wait_for_mock_google_tokens(mock_api_url: str, *, timeout: float = 30.0) -> list[str]: - async with httpx.AsyncClient() as client: - for _ in range(int(timeout * 2)): - response = await client.get(f"{mock_api_url}/__mock/received-tokens", timeout=15) - response.raise_for_status() - tokens = response.json().get("tokens", []) - if tokens: - return tokens - await asyncio.sleep(0.5) - raise AssertionError("Timed out waiting for Gmail HTTP execution against the mock API") - - -async def _get_mock_google_requests(mock_api_url: str) -> list[str]: - async with httpx.AsyncClient() as client: - response = await client.get(f"{mock_api_url}/__mock/received-requests", timeout=15) - response.raise_for_status() - return response.json().get("requests", []) - - -async def _reset_mock_google_state(mock_api_url: str) -> None: - async with httpx.AsyncClient() as client: - response = await client.post(f"{mock_api_url}/__mock/reset", timeout=10) - response.raise_for_status() - - -async def _wait_for_tool_call( - base_url: str, - thread_id: str, - tool_name: str, - timeout: float = 30.0, -) -> dict: - approved_request_ids = set() - for _ in range(int(timeout * 2)): - response = await api_get(base_url, f"/api/chat/history?thread_id={thread_id}", timeout=15) - response.raise_for_status() - history = response.json() - - pending = history.get("pending_gate") or history.get("pending_approval") - if pending and pending["request_id"] not in approved_request_ids: - approve = await api_post( - base_url, - "/api/chat/approval", - json={ - "request_id": pending["request_id"], - "action": "approve", - "thread_id": thread_id, - }, - timeout=15, - ) - assert approve.status_code == 202, approve.text - approved_request_ids.add(pending["request_id"]) - - for turn in history.get("turns", []): - for tool_call in turn.get("tool_calls", []): - if tool_call.get("name") == tool_name: - return history - - await asyncio.sleep(0.5) - - raise AssertionError(f"Timed out waiting for {tool_name} tool call in thread {thread_id}") - - -async def _complete_callback( - base_url: str, - auth_url: str, - *, - code: str, -) -> httpx.Response: - async with httpx.AsyncClient() as client: - response = await client.get( - f"{base_url}/oauth/callback", - params={"code": code, "state": _extract_state(auth_url)}, - timeout=30, - follow_redirects=True, - ) - return response - - -async def _gmail_setup_auth_url(base_url: str) -> str: - response = await api_post(base_url, "/api/extensions/gmail/setup", json={}, timeout=30) - assert response.status_code == 200, response.text - auth_url = response.json().get("auth_url") - assert auth_url, response.text - return auth_url - - -@pytest.fixture(scope="module") -async def v2_activate_surface_server(ironclaw_binary, mock_llm_server, wasm_tools_dir): - socks = [] - for _ in range(2): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(("127.0.0.1", 0)) - socks.append(sock) - gateway_port = socks[0].getsockname()[1] - http_port = socks[1].getsockname()[1] - for sock in socks: - sock.close() - - home_dir = _HOME_TMPDIR.name - env = { - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "HOME": home_dir, - "IRONCLAW_BASE_DIR": os.path.join(home_dir, ".ironclaw"), - "RUST_LOG": "ironclaw=debug", - "RUST_BACKTRACE": "1", - "ENGINE_V2": "true", - "GATEWAY_ENABLED": "true", - "GATEWAY_HOST": "127.0.0.1", - "GATEWAY_PORT": str(gateway_port), - "GATEWAY_AUTH_TOKEN": AUTH_TOKEN, - "GATEWAY_USER_ID": "e2e-v2-activate-surface", - "IRONCLAW_OWNER_ID": "e2e-v2-activate-surface", - "HTTP_HOST": "127.0.0.1", - "HTTP_PORT": str(http_port), - "CLI_ENABLED": "false", - "LLM_BACKEND": "openai_compatible", - "LLM_BASE_URL": mock_llm_server, - "LLM_API_KEY": "mock-api-key", - "LLM_MODEL": "mock-model", - "DATABASE_BACKEND": "libsql", - "LIBSQL_PATH": os.path.join(_DB_TMPDIR.name, "v2-activate-surface.db"), - "SANDBOX_ENABLED": "false", - "SKILLS_ENABLED": "true", - "ROUTINES_ENABLED": "false", - "HEARTBEAT_ENABLED": "false", - "EMBEDDING_ENABLED": "false", - "WASM_ENABLED": "true", - "WASM_TOOLS_DIR": wasm_tools_dir, - "ONBOARD_COMPLETED": "true", - "SECRETS_MASTER_KEY": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - } - _forward_coverage_env(env) - - proc = await asyncio.create_subprocess_exec( - ironclaw_binary, - "--no-onboard", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - - base_url = f"http://127.0.0.1:{gateway_port}" - try: - await wait_for_ready(f"{base_url}/api/health", timeout=60) - await _pin_mock_llm_settings(base_url, mock_llm_server) - yield base_url - finally: - if proc.returncode is None: - await _stop_process(proc, sig=signal.SIGINT, timeout=10) - if proc.returncode is None: - await _stop_process(proc, sig=signal.SIGTERM, timeout=5) - - -@pytest.fixture(scope="module") -async def v2_activate_surface_auth_server(ironclaw_binary, mock_llm_server, wasm_tools_dir): - mock_api = await _start_mock_google_api() - - socks = [] - for _ in range(2): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(("127.0.0.1", 0)) - socks.append(sock) - gateway_port = socks[0].getsockname()[1] - http_port = socks[1].getsockname()[1] - for sock in socks: - sock.close() - - db_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-v2-activate-auth-db-") - home_tmpdir = tempfile.TemporaryDirectory(prefix="ironclaw-v2-activate-auth-home-") - env = { - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "HOME": home_tmpdir.name, - "IRONCLAW_BASE_DIR": os.path.join(home_tmpdir.name, ".ironclaw"), - "RUST_LOG": "ironclaw=info", - "RUST_BACKTRACE": "1", - "ENGINE_V2": "true", - "HTTP_ALLOW_LOCALHOST": "true", - "GATEWAY_ENABLED": "true", - "GATEWAY_HOST": "127.0.0.1", - "GATEWAY_PORT": str(gateway_port), - "GATEWAY_AUTH_TOKEN": AUTH_TOKEN, - "GATEWAY_USER_ID": "e2e-v2-activate-auth-surface", - "IRONCLAW_OWNER_ID": "e2e-v2-activate-auth-surface", - "HTTP_HOST": "127.0.0.1", - "HTTP_PORT": str(http_port), - "CLI_ENABLED": "false", - "LLM_BACKEND": "openai_compatible", - "LLM_BASE_URL": mock_llm_server, - "LLM_API_KEY": "mock-api-key", - "LLM_MODEL": "mock-model", - "DATABASE_BACKEND": "libsql", - "LIBSQL_PATH": os.path.join(db_tmpdir.name, "v2-activate-auth-surface.db"), - "SANDBOX_ENABLED": "false", - "SKILLS_ENABLED": "true", - "ROUTINES_ENABLED": "false", - "HEARTBEAT_ENABLED": "false", - "EMBEDDING_ENABLED": "false", - "WASM_ENABLED": "true", - "WASM_TOOLS_DIR": wasm_tools_dir, - "ONBOARD_COMPLETED": "true", - "SECRETS_MASTER_KEY": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "IRONCLAW_OAUTH_CALLBACK_URL": "https://oauth.test.example/oauth/callback", - "IRONCLAW_OAUTH_EXCHANGE_URL": mock_llm_server, - "IRONCLAW_OAUTH_PROXY_ALLOW_LOOPBACK": "1", - "GOOGLE_OAUTH_CLIENT_ID": "hosted-google-client-id", - "IRONCLAW_TEST_HTTP_REMAP": ( - f"gmail.googleapis.com={mock_api['base_url']}," - f"www.googleapis.com={mock_api['base_url']}" - ), - } - _forward_coverage_env(env) - - proc = await asyncio.create_subprocess_exec( - ironclaw_binary, - "--no-onboard", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - - base_url = f"http://127.0.0.1:{gateway_port}" - try: - await wait_for_ready(f"{base_url}/api/health", timeout=60) - await _pin_mock_llm_settings(base_url, mock_llm_server) - yield { - "base_url": base_url, - "mock_api_url": mock_api["base_url"], - } - finally: - if proc.returncode is None: - await _stop_process(proc, sig=signal.SIGINT, timeout=10) - if proc.returncode is None: - await _stop_process(proc, sig=signal.SIGTERM, timeout=5) - await mock_api["runner"].cleanup() - db_tmpdir.cleanup() - home_tmpdir.cleanup() - - -async def _wait_for_engine_system_prompt( - base_url: str, - *, - goal_substring: str, - timeout: float = 45.0, -): - last_threads = [] - last_detail = {} - for _ in range(int(timeout * 2)): - threads_response = await api_get(base_url, "/api/engine/threads", timeout=15) - threads_response.raise_for_status() - threads = threads_response.json().get("threads", []) - last_threads = threads - matches = [ - thread - for thread in threads - if goal_substring.lower() in (thread.get("goal") or "").lower() - ] - matches.sort(key=lambda thread: thread.get("updated_at") or "") - - for match in reversed(matches): - detail_response = await api_get( - base_url, - f"/api/engine/threads/{match['id']}", - timeout=15, - ) - detail_response.raise_for_status() - detail = detail_response.json().get("thread", {}) - last_detail = detail - for message in detail.get("messages", []): - if message.get("role") == "System" and message.get("content"): - return message["content"] - - await asyncio.sleep(0.5) - - raise AssertionError( - f"Timed out waiting for engine system prompt for goal containing {goal_substring!r}. " - f"Last threads: {json.dumps(last_threads)[:1200]}; " - f"Last detail: {json.dumps(last_detail)[:1200]}" - ) - - -async def _get_extension(base_url: str, name: str): - response = await api_get(base_url, "/api/extensions", timeout=30) - response.raise_for_status() - payload = response.json() - for extension in payload.get("extensions", []): - if extension.get("name") == name: - return extension - return None - - -async def _ensure_removed(base_url: str, name: str): - extension = await _get_extension(base_url, name) - if extension: - await api_post(base_url, f"/api/extensions/{name}/remove", timeout=30) - - -async def test_tool_activate_is_the_visible_enablement_tool(v2_activate_surface_server): - goal = "baseline activate surface e2e" - thread_response = await api_post(v2_activate_surface_server, "/api/chat/thread/new", timeout=15) - thread_id = thread_response.json()["id"] - - await api_post( - v2_activate_surface_server, - "/api/chat/send", - json={"content": goal, "thread_id": thread_id}, - timeout=30, - ) - - system_prompt = await _wait_for_engine_system_prompt( - v2_activate_surface_server, - goal_substring=goal, - timeout=45, - ) - assert 'tool_activate(name="<integration>")' in system_prompt - assert 'tool_info(name="<tool>", detail="summary")' in system_prompt - assert "tool_auth" not in system_prompt - assert "tool_install" not in system_prompt - - -async def test_blocked_integration_surfaces_in_activatable_section(v2_activate_surface_server): - goal = "gmail activate surface e2e" - await _ensure_removed(v2_activate_surface_server, "gmail") - - try: - install = await api_post( - v2_activate_surface_server, - "/api/extensions/install", - json={"name": "gmail"}, - timeout=180, - ) - assert install.status_code == 200, install.text - payload = install.json() - assert payload.get("success") is True, payload - - thread_response = await api_post( - v2_activate_surface_server, - "/api/chat/thread/new", - timeout=15, - ) - thread_id = thread_response.json()["id"] - - await api_post( - v2_activate_surface_server, - "/api/chat/send", - json={"content": goal, "thread_id": thread_id}, - timeout=30, - ) - - system_prompt = await _wait_for_engine_system_prompt( - v2_activate_surface_server, - goal_substring=goal, - timeout=45, - ) - assert "## Activatable Integrations" in system_prompt - assert 'tool_activate(name="<integration>")' in system_prompt - assert 'tool_info(name="<tool>", detail="summary")' in system_prompt - assert "`gmail` [provider]" in system_prompt - assert "tool_install" not in system_prompt - finally: - await _ensure_removed(v2_activate_surface_server, "gmail") - - -async def test_blocked_gmail_auth_blocks_upstream_requests_before_auth( - v2_activate_surface_auth_server, -): - server = v2_activate_surface_auth_server - await _ensure_removed(server["base_url"], "gmail") - - try: - await _reset_mock_google_state(server["mock_api_url"]) - - install = await api_post( - server["base_url"], - "/api/extensions/install", - json={"name": "gmail"}, - timeout=180, - ) - assert install.status_code == 200, install.text - assert install.json().get("success") is True, install.text - - assert await _get_mock_google_requests(server["mock_api_url"]) == [] - auth_url = await _gmail_setup_auth_url(server["base_url"]) - response = await _complete_callback(server["base_url"], auth_url, code="mock_auth_code") - assert response.status_code == 200, response.text[:400] - - thread_response = await api_post(server["base_url"], "/api/chat/thread/new", timeout=15) - thread_id = thread_response.json()["id"] - send = await api_post( - server["base_url"], - "/api/chat/send", - json={"content": "check gmail unread", "thread_id": thread_id}, - timeout=30, - ) - assert send.status_code == 202, send.text - - await _wait_for_tool_call(server["base_url"], thread_id, "gmail", timeout=60.0) - tokens = await _wait_for_mock_google_tokens(server["mock_api_url"], timeout=60.0) - assert tokens, "expected Gmail to hit the mock Google API after auth replay" - history = await _wait_for_response_contains( - server["base_url"], thread_id, "Quarterly update", timeout=60.0 - ) - assert history.get("pending_gate") is None, history - finally: - await _ensure_removed(server["base_url"], "gmail") diff --git a/tests/e2e/scenarios/test_widget_customization.py b/tests/e2e/scenarios/test_widget_customization.py index e7f306db10e..065a3d2238d 100644 --- a/tests/e2e/scenarios/test_widget_customization.py +++ b/tests/e2e/scenarios/test_widget_customization.py @@ -152,6 +152,8 @@ async def single_tenant_gateway_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "SANDBOX_ENABLED": "false", "SKILLS_ENABLED": "true", @@ -234,6 +236,8 @@ async def multi_tenant_gateway_server(ironclaw_binary, mock_llm_server): "CLI_ENABLED": "false", "LLM_BACKEND": "openai_compatible", "LLM_BASE_URL": mock_llm_server, + # Dummy key: mock LLM ignores it, but openai_compatible config requires auth. + "LLM_API_KEY": "mock-api-key", "LLM_MODEL": "mock-model", "DATABASE_BACKEND": "libsql", "LIBSQL_PATH": os.path.join(db_tmpdir.name, "multi-tenant.db"), diff --git a/tests/e2e_advanced_traces.rs b/tests/e2e_advanced_traces.rs index b83bdd19493..7e4cd438da3 100644 --- a/tests/e2e_advanced_traces.rs +++ b/tests/e2e_advanced_traces.rs @@ -338,7 +338,7 @@ mod advanced { #[tokio::test] async fn routine_news_digest() { - use ironclaw::llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; + use ironclaw_llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; let trace = LlmTrace::from_file(format!("{FIXTURES}/routine_news_digest.json")).unwrap(); @@ -686,6 +686,7 @@ mod advanced { fallback_source: None, auth_hint: AuthHint::Dcr, version: None, + hidden: false, }) .await; diff --git a/tests/e2e_approval_traces.rs b/tests/e2e_approval_traces.rs index 396ab2a2653..9c9cd9af54e 100644 --- a/tests/e2e_approval_traces.rs +++ b/tests/e2e_approval_traces.rs @@ -424,7 +424,7 @@ mod approval_trace_tests { ); let last_user_yes = captured.iter().any(|msgs| { msgs.iter().any(|m| { - matches!(m.role, ironclaw::llm::Role::User) + matches!(m.role, ironclaw_llm::Role::User) && m.content.trim().eq_ignore_ascii_case("yes") }) }); diff --git a/tests/e2e_attachments.rs b/tests/e2e_attachments.rs index d90e26f532b..16c3815cef6 100644 --- a/tests/e2e_attachments.rs +++ b/tests/e2e_attachments.rs @@ -17,7 +17,7 @@ mod attachment_tests { use crate::support::trace_llm::LlmTrace; use ironclaw::channels::{AttachmentKind, IncomingAttachment, IncomingMessage}; - use ironclaw::llm::ContentPart; + use ironclaw_llm::ContentPart; const FIXTURES: &str = concat!( env!("CARGO_MANIFEST_DIR"), @@ -87,7 +87,7 @@ mod attachment_tests { let last_user_msg = last_request .iter() .rev() - .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .find(|m| matches!(m.role, ironclaw_llm::Role::User)) .expect("should have a user message"); // The augmented text should contain the attachment tags and transcript @@ -153,7 +153,7 @@ mod attachment_tests { let last_user_msg = last_request .iter() .rev() - .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .find(|m| matches!(m.role, ironclaw_llm::Role::User)) .expect("should have a user message"); // Should have image content parts @@ -204,7 +204,7 @@ mod attachment_tests { let last_user_msg = last_request .iter() .rev() - .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .find(|m| matches!(m.role, ironclaw_llm::Role::User)) .expect("should have a user message"); // No attachments → no augmentation tags, no content parts @@ -279,7 +279,7 @@ mod attachment_tests { let last_user_msg = last_request .iter() .rev() - .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .find(|m| matches!(m.role, ironclaw_llm::Role::User)) .expect("user message"); let expected_suffix = format!("{channel}-notes.txt"); diff --git a/tests/e2e_auth_gate_traces.rs b/tests/e2e_auth_gate_traces.rs deleted file mode 100644 index ea84c34c3ca..00000000000 --- a/tests/e2e_auth_gate_traces.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! Replay coverage for the engine-v2 authentication gate round-trip. -//! -//! Phase 2 of #2828 — extends the approval fixtures with the typed -//! `Submission::GateAuthResolution` / `Submission::ExternalCallback` -//! path that only engine v2 recognizes. The tests drive the full -//! cycle: LLM → `tool_activate` → GatePaused(Authentication) → -//! typed resolution → re-run → final response. -//! -//! Scenarios: -//! - `auth_credential_provided_resumes_action` — happy path -//! - `auth_cancelled_stops_thread` — cancel path -//! - `auth_retry_after_invalid_credential` — second pause gets a new request_id -//! - `auth_external_callback_resolves_oauth_gate` — OAuth callback path -//! - `auth_gate_emits_request_id_for_v2` — invariant: gate carries a Uuid - -#[cfg(feature = "libsql")] -mod support; - -#[cfg(feature = "libsql")] -mod auth_gate_trace_tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, OnceLock}; - use std::time::Duration; - - use async_trait::async_trait; - use tokio::sync::Mutex; - - use crate::support::test_rig::{TestRig, TestRigBuilder}; - use crate::support::trace_llm::LlmTrace; - use ironclaw::agent::submission::AuthGateResolution; - use ironclaw::channels::StatusUpdate; - use ironclaw::context::JobContext; - use ironclaw::tools::{Tool, ToolError, ToolOutput}; - - const TIMEOUT: Duration = Duration::from_secs(15); - - /// Serialize all tests in this file because engine v2 stores its state - /// in a process-global `OnceLock<RwLock<Option<EngineState>>>`. - /// Running these tests in parallel would let one test's gate_paused - /// state leak into the next test's engine instance. - fn engine_v2_test_lock() -> &'static Mutex<()> { - static LOCK: OnceLock<Mutex<()>> = OnceLock::new(); - LOCK.get_or_init(|| Mutex::new(())) - } - - /// Test stub for the `tool_activate` built-in. Returns a pre-configured - /// sequence of JSON outputs so the test can shape the GatePaused path. - /// - /// Each call pops the next output off the queue; after the queue is - /// drained the last output is reused (so resumes after the final - /// gate always see the "ready" state). - struct MockActivateTool { - outputs: Arc<std::sync::Mutex<Vec<serde_json::Value>>>, - executions: Arc<AtomicUsize>, - } - - impl MockActivateTool { - fn new(outputs: Vec<serde_json::Value>) -> (Arc<Self>, Arc<AtomicUsize>) { - let executions = Arc::new(AtomicUsize::new(0)); - let tool = Arc::new(Self { - outputs: Arc::new(std::sync::Mutex::new(outputs)), - executions: executions.clone(), - }); - (tool, executions) - } - } - - #[async_trait] - impl Tool for MockActivateTool { - fn name(&self) -> &str { - // Must match the protected built-in name so the effect adapter's - // `auth_gate_from_extension_result` path inspects our output. - "tool_activate" - } - - fn description(&self) -> &str { - "Test stub for tool_activate that emits scripted auth-gate outputs" - } - - fn parameters_schema(&self) -> serde_json::Value { - serde_json::json!({ - "type": "object", - "properties": { - "name": { "type": "string" }, - "token": { "type": "string" } - }, - "required": ["name"] - }) - } - - async fn execute( - &self, - _params: serde_json::Value, - _ctx: &JobContext, - ) -> Result<ToolOutput, ToolError> { - self.executions.fetch_add(1, Ordering::SeqCst); - let mut q = self.outputs.lock().expect("outputs lock poisoned"); - let next = if q.len() > 1 { - q.remove(0) - } else { - // Last output — reuse on every subsequent call so the - // post-credential re-run returns the "ready" state. - q.first().cloned().unwrap_or(serde_json::json!({})) - }; - Ok(ToolOutput::success(next, Duration::from_millis(1))) - } - } - - /// Poll `captured_status_events` until an `AuthRequired` with a - /// `request_id` is observed (past `initial_count`). Returns the first - /// new `request_id` as a `Uuid`, or a descriptive error when the - /// router emitted a malformed ID or the status never arrived. - async fn wait_for_auth_required( - rig: &TestRig, - initial_count: usize, - timeout: Duration, - ) -> Result<uuid::Uuid, String> { - let deadline = tokio::time::Instant::now() + timeout; - loop { - let events = rig.captured_status_events(); - let with_id: Vec<_> = events - .iter() - .filter_map(|s| match s { - StatusUpdate::AuthRequired { - request_id: Some(id), - .. - } => Some(id.clone()), - _ => None, - }) - .collect(); - if with_id.len() > initial_count { - let raw = with_id - .get(initial_count) - .expect("length checked above; request_id must exist"); - return uuid::Uuid::parse_str(raw) - .map_err(|e| format!("malformed AuthRequired.request_id {raw:?}: {e}")); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "timed out waiting for AuthRequired #{initial_count}; saw events: {events:?}" - )); - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - } - - fn fixture_path(name: &str) -> String { - format!( - "{}/tests/fixtures/llm_traces/coverage/{}", - env!("CARGO_MANIFEST_DIR"), - name - ) - } - - /// Write a minimal SKILL.md that declares a credential spec for - /// `credential_name` into `skills_dir`. This is the bare-minimum fixture - /// that makes `AuthManager::submit_auth_token` accept the credential - /// (it requires a matching skill credential spec when no extension is - /// installed). The skill body is intentionally trivial — the test does - /// not exercise prompt/activation behavior. - fn plant_probe_skill(skills_dir: &std::path::Path, credential_name: &str) { - let skill_dir = skills_dir.join(format!("probe_{credential_name}")); - std::fs::create_dir_all(&skill_dir).expect("create probe skill dir"); - let manifest = format!( - r#"--- -name: probe_{credential_name} -version: "0.0.0" -description: Probe skill for auth-gate replay coverage. -activation: - keywords: - - "probe-skill-should-never-activate-in-this-test" -credentials: - - name: {credential_name} - provider: test - location: - type: bearer - hosts: - - "example.com" ---- - -Probe skill. -"# - ); - std::fs::write(skill_dir.join("SKILL.md"), manifest).expect("write probe SKILL.md"); - } - - /// Common rig setup: engine v2 + a `MockActivateTool` override keyed by - /// the caller's scripted output sequence. Also plants a probe skill for - /// each credential name referenced in `outputs` so the auth-manager - /// credential-store path accepts the token during `CredentialProvided`. - async fn auth_rig( - trace: LlmTrace, - outputs: Vec<serde_json::Value>, - ) -> (TestRig, Arc<AtomicUsize>) { - // Extract credential names so we can plant matching probe skills. - let credential_names: std::collections::HashSet<String> = outputs - .iter() - .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) - .collect(); - - // Use `.into_path()` to release the tempdir from auto-cleanup so - // skill discovery can read from it after the builder finishes. OS - // cleanup handles the directory between runs. - let skills_tempdir = tempfile::tempdir().expect("probe skills tempdir"); - let skills_path = skills_tempdir.keep(); - for name in &credential_names { - plant_probe_skill(&skills_path, name); - } - - let (tool, executions) = MockActivateTool::new(outputs); - let rig = TestRigBuilder::new() - .with_trace(trace) - .with_engine_v2() - .with_test_tool_override(tool as Arc<dyn Tool>) - .with_skills_dir(skills_path) - .build() - .await; - rig.clear().await; - (rig, executions) - } - - #[tokio::test] - async fn auth_credential_provided_resumes_action() { - let _guard = engine_v2_test_lock().lock().await; - let trace = LlmTrace::from_file(fixture_path("auth_credential_provided.json")) - .expect("failed to load auth_credential_provided.json"); - let outputs = vec![ - serde_json::json!({ - "status": "awaiting_token", - "name": "test_credential", - "instructions": "Provide test token" - }), - serde_json::json!({ - "status": "ready", - "name": "test_credential", - "message": "Credential configured" - }), - ]; - let (rig, executions) = auth_rig(trace.clone(), outputs).await; - - rig.send_message("Set up the test credential").await; - - let request_id = wait_for_auth_required(&rig, 0, TIMEOUT) - .await - .expect("expected AuthRequired with request_id before token was provided"); - - rig.send_gate_auth_resolution( - request_id, - AuthGateResolution::CredentialProvided { - token: "test-token-value".to_string(), - }, - ) - .await; - - let responses = rig.wait_for_responses(1, TIMEOUT).await; - assert!( - executions.load(Ordering::SeqCst) >= 2, - "tool must re-run after credential submission (executions={})", - executions.load(Ordering::SeqCst) - ); - rig.verify_trace_expects(&trace, &responses); - rig.shutdown(); - } - - #[tokio::test] - async fn auth_cancelled_stops_thread() { - let _guard = engine_v2_test_lock().lock().await; - let trace = LlmTrace::from_file(fixture_path("auth_cancelled.json")) - .expect("failed to load auth_cancelled.json"); - let outputs = vec![serde_json::json!({ - "status": "awaiting_token", - "name": "test_credential", - "instructions": "Provide test token" - })]; - let (rig, executions) = auth_rig(trace.clone(), outputs).await; - - rig.send_message("Set up the credential I'll cancel").await; - - let request_id = wait_for_auth_required(&rig, 0, TIMEOUT) - .await - .expect("expected AuthRequired before cancel"); - - rig.send_gate_auth_resolution(request_id, AuthGateResolution::Cancelled) - .await; - - let responses = rig.wait_for_responses(1, TIMEOUT).await; - assert_eq!( - executions.load(Ordering::SeqCst), - 1, - "tool must run exactly once (cancel must not trigger a re-run)" - ); - assert!( - responses.iter().any(|r| r.content.contains("Cancelled")), - "expected 'Cancelled.' response after auth-gate cancel, got: {:?}", - responses.iter().map(|r| &r.content).collect::<Vec<_>>() - ); - rig.verify_trace_expects(&trace, &responses); - rig.shutdown(); - } - - #[tokio::test] - async fn auth_retry_after_invalid_credential() { - let _guard = engine_v2_test_lock().lock().await; - let trace = LlmTrace::from_file(fixture_path("auth_retry_invalid_then_valid.json")) - .expect("failed to load auth_retry_invalid_then_valid.json"); - // First two calls pause the gate (first token is "invalid"), third - // call returns ready. The mock doesn't actually validate tokens — - // it just emits the scripted output sequence. - let outputs = vec![ - serde_json::json!({ - "status": "awaiting_token", - "name": "test_credential", - "instructions": "Provide test token" - }), - serde_json::json!({ - "status": "awaiting_token", - "name": "test_credential", - "instructions": "That token was invalid; try again" - }), - serde_json::json!({ - "status": "ready", - "name": "test_credential" - }), - ]; - let (rig, executions) = auth_rig(trace.clone(), outputs).await; - - rig.send_message("Set up the credential").await; - - let first_id = wait_for_auth_required(&rig, 0, TIMEOUT) - .await - .expect("expected first AuthRequired"); - rig.send_gate_auth_resolution( - first_id, - AuthGateResolution::CredentialProvided { - token: "invalid".to_string(), - }, - ) - .await; - - let second_id = wait_for_auth_required(&rig, 1, TIMEOUT) - .await - .expect("expected second AuthRequired after invalid token"); - assert_ne!( - first_id, second_id, - "re-pause must emit a fresh request_id, not reuse the first" - ); - - rig.send_gate_auth_resolution( - second_id, - AuthGateResolution::CredentialProvided { - token: "valid".to_string(), - }, - ) - .await; - - let responses = rig.wait_for_responses(1, TIMEOUT).await; - assert!( - executions.load(Ordering::SeqCst) >= 3, - "tool must run at least three times (initial + two resumes), got {}", - executions.load(Ordering::SeqCst) - ); - rig.verify_trace_expects(&trace, &responses); - rig.shutdown(); - } - - #[tokio::test] - async fn auth_external_callback_resolves_oauth_gate() { - let _guard = engine_v2_test_lock().lock().await; - let trace = LlmTrace::from_file(fixture_path("auth_external_callback.json")) - .expect("failed to load auth_external_callback.json"); - let outputs = vec![ - serde_json::json!({ - "status": "awaiting_authorization", - "name": "oauth_service", - "auth_url": "https://example.com/oauth/start", - "instructions": "Complete the OAuth flow in your browser" - }), - serde_json::json!({ - "status": "ready", - "name": "oauth_service" - }), - ]; - let (rig, executions) = auth_rig(trace.clone(), outputs).await; - - rig.send_message("Connect the OAuth service").await; - - let request_id = wait_for_auth_required(&rig, 0, TIMEOUT) - .await - .expect("expected AuthRequired with request_id for OAuth"); - - // Confirm the gate carried an auth_url (OAuth shape, not bare token). - assert!( - rig.captured_status_events().iter().any(|s| matches!( - s, - StatusUpdate::AuthRequired { auth_url: Some(url), .. } if url.starts_with("https://example.com/oauth") - )), - "AuthRequired for OAuth must surface the auth_url" - ); - - rig.send_external_callback(request_id).await; - - let responses = rig.wait_for_responses(1, TIMEOUT).await; - assert!( - executions.load(Ordering::SeqCst) >= 2, - "tool must re-run after external callback (executions={})", - executions.load(Ordering::SeqCst) - ); - rig.verify_trace_expects(&trace, &responses); - rig.shutdown(); - } - - #[tokio::test] - async fn auth_gate_emits_request_id_for_v2() { - let _guard = engine_v2_test_lock().lock().await; - let trace = LlmTrace::from_file(fixture_path("auth_gate_request_id.json")) - .expect("failed to load auth_gate_request_id.json"); - let outputs = vec![serde_json::json!({ - "status": "awaiting_token", - "name": "probe_credential", - "instructions": "Provide" - })]; - let (rig, _executions) = auth_rig(trace, outputs).await; - - rig.send_message("Trigger the auth gate").await; - - let request_id = wait_for_auth_required(&rig, 0, TIMEOUT) - .await - .expect("engine v2 must always emit a request_id on AuthRequired"); - // Invariant: the gate's request_id must round-trip through Uuid parsing. - // (`wait_for_auth_required` already parsed it; this asserts non-nil.) - assert_ne!( - request_id, - uuid::Uuid::nil(), - "request_id on AuthRequired must be a real Uuid, not nil" - ); - - // Clean up the pending gate so the test doesn't leak an active thread. - rig.send_gate_auth_resolution(request_id, AuthGateResolution::Cancelled) - .await; - rig.shutdown(); - } -} diff --git a/tests/e2e_bug_bash_snapshots.rs b/tests/e2e_bug_bash_snapshots.rs new file mode 100644 index 00000000000..2bf203ba539 --- /dev/null +++ b/tests/e2e_bug_bash_snapshots.rs @@ -0,0 +1,47 @@ +//! Bug-bash regression snapshots. +//! +//! Each test replays a fixture from `tests/fixtures/llm_traces/bug_bash/` +//! and pins the `ReplayOutcome` as a snapshot. The snapshot encodes the +//! specific regression property the bug report is about — reintroducing +//! the bug causes the snapshot to drift. +//! +//! See `tests/fixtures/llm_traces/bug_bash/README.md` for the bug ↔ fixture +//! map and for instructions on recording new fixtures against staging. + +#[cfg(feature = "libsql")] +mod support; + +#[cfg(feature = "libsql")] +mod bug_bash_tests { + use std::time::Duration; + + use crate::assert_replay_snapshot; + use crate::support::replay_outcome::ReplayOutcome; + use crate::support::test_rig::TestRigBuilder; + use crate::support::trace_llm::LlmTrace; + + const FIXTURES: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/llm_traces/bug_bash" + ); + + /// Regression for [#2541](https://github.com/nearai/ironclaw/issues/2541): + /// agent must invoke a tool (not answer from training data) when the user + /// asks it to do something. The snapshot pins `tool_calls` to a non-empty + /// list with the `echo` tool. If the agent regresses to text-only + /// responses, the snapshot drifts to `tool_calls: []`. + #[tokio::test] + async fn snapshot_summarization_uses_tools() { + let trace = + LlmTrace::from_file(format!("{FIXTURES}/summarization_uses_tools.json")).unwrap(); + let rig = TestRigBuilder::new().with_trace(trace).build().await; + + rig.send_message("Echo 'status ok' and then tell me what you heard") + .await; + let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await; + + let outcome = ReplayOutcome::capture(&rig, &responses).await; + assert_replay_snapshot!("bug_bash_summarization_uses_tools", outcome); + rig.shutdown(); + } +} diff --git a/tests/e2e_builtin_tool_coverage.rs b/tests/e2e_builtin_tool_coverage.rs index 7263b5e3072..808223217de 100644 --- a/tests/e2e_builtin_tool_coverage.rs +++ b/tests/e2e_builtin_tool_coverage.rs @@ -105,7 +105,7 @@ mod tests { } } - fn requests_contain(requests: &[Vec<ironclaw::llm::ChatMessage>], needle: &str) -> bool { + fn requests_contain(requests: &[Vec<ironclaw_llm::ChatMessage>], needle: &str) -> bool { requests .iter() .flatten() @@ -1222,9 +1222,9 @@ mod tests { .expect("tool_search description should be a string"); assert!( tool_search_description.contains("`tool_install`") - && tool_search_description.contains("`tool_activate(name=\"...\")`"), - "tool_search description should describe setup/activation via tool_install and \ - tool_activate: {tool_search_description}" + && tool_search_description.contains("its tools become directly callable"), + "tool_search description should describe install + direct-callable post-#3133 \ + contract: {tool_search_description}" ); assert!( tool_search_description diff --git a/tests/e2e_live.rs b/tests/e2e_live.rs index 172f82ecdfd..9c32067812d 100644 --- a/tests/e2e_live.rs +++ b/tests/e2e_live.rs @@ -447,20 +447,18 @@ mod live_tests { gate_auth_url.is_some() ); - // The agent must NOT have run a tool_install / tool_activate - // recovery loop — that's the bad behaviour the post-flight - // detector eliminates. Match both bare names and the - // argument-prefixed `"<name>(args)"` form emitted by - // `format_action_display_name`; an exact-match check would silently - // miss `"tool_install(foo)"` and turn this into a false negative. - let bad_recovery = phase_a_tools.iter().any(|t| { - tool_name_matches(t, "tool_install") - || tool_name_matches(t, "tool_activate") - || tool_name_matches(t, "tool-install") - }); + // The agent must NOT have run a tool_install recovery loop — + // that's the bad behaviour the post-flight detector eliminates. + // Match both bare names and the argument-prefixed `"<name>(args)"` + // form emitted by `format_action_display_name`; an exact-match + // check would silently miss `"tool_install(foo)"` and turn this + // into a false negative. + let bad_recovery = phase_a_tools + .iter() + .any(|t| tool_name_matches(t, "tool_install") || tool_name_matches(t, "tool-install")); assert!( !bad_recovery, - "Phase A: agent ran a tool_install/tool_activate recovery loop instead \ + "Phase A: agent ran a tool_install recovery loop instead \ of pausing for auth on the first iteration. Tools attempted: {phase_a_tools:?}" ); @@ -545,9 +543,9 @@ mod live_tests { // // We assert two things: // - // 1. No `tool_install` / `tool_activate` / `secret_list` - // tool ever appears in Phase B's tool activity (the - // pre-fix recovery loop's smoking gun). + // 1. No `tool_install` / `secret_list` tool ever appears in + // Phase B's tool activity (the pre-fix recovery loop's + // smoking gun). // // 2. The first tool that does run in Phase B is one of // `google_drive_tool` or its action variants (so we know @@ -566,7 +564,6 @@ mod live_tests { let phase_b_recovery = phase_b_tools.iter().any(|t| { tool_name_matches(t, "tool_install") || tool_name_matches(t, "tool-install") - || tool_name_matches(t, "tool_activate") || tool_name_matches(t, "secret_list") || tool_name_matches(t, "tool_search") }); diff --git a/tests/e2e_live_mission_gmail.rs b/tests/e2e_live_mission_gmail.rs deleted file mode 100644 index d74a694211e..00000000000 --- a/tests/e2e_live_mission_gmail.rs +++ /dev/null @@ -1,324 +0,0 @@ -//! Live end-to-end test for issue #3133 — "Mission to send an email failed". -//! -//! The bug report shows a mission firing every 3 minutes whose child -//! thread bails with the FINAL() body: -//! -//! "Failed to send email. Status: None Error: None -//! Next focus: Debug Gmail authentication or try alternative email method." -//! -//! The "Status: None Error: None" wording is not from any of our Rust -//! code — it's the LLM looking at a tool response (probably the `http` -//! tool's reply shape, possibly a Gmail tool error path) where it -//! expected populated `status` and `error` fields and found neither. -//! Three plausible origins: -//! -//! 1. The agent fell back to an `http` POST to Gmail's REST API after -//! the WASM `gmail` tool call did not resolve cleanly. -//! 2. The Gmail OAuth token was not seeded for the mission's child -//! thread, so the tool errored "Google OAuth token not configured…" -//! and the model paraphrased it. -//! 3. The mission's lease did not include `gmail`, so the call never -//! reached the WASM tool at all. -//! -//! This test drives a verbatim-flavoured prompt through engine v2 with -//! Gmail OAuth credentials seeded, and asserts that: -//! -//! 1. The agent invoked `mission_create` and `mission_fire` -//! (or the routine_* aliases that resolve to the same handlers). -//! 2. The mission's child thread invoked the `gmail` tool — proving -//! the tool was discoverable and the lease covered it. -//! 3. The notification carries a `**[<mission-name>]**` marker, the -//! same structural signal `e2e_live_routine.rs` uses. -//! 4. The notification body does NOT contain the -//! "Status: None Error: None" pattern that #3133 reports — that's -//! the regression marker. -//! 5. The orchestrator never emits "<N> consecutive code errors" -//! (sanity check shared with #2583). -//! -//! Why drafts, not sends: the bug reproduces equally on `gmail.create_draft` -//! (which lands in the OAuth account's Drafts folder, no email actually -//! delivered) as on `gmail.send_message`. Drafts have minimal blast -//! radius — no real email goes out, and the resulting draft is easy to -//! delete by id from the test owner's Gmail. Switching the prompt to -//! "create a draft" instead of "send an email" preserves the auth / -//! tool-discovery / lease paths the bug actually exercises. -//! -//! Run live (records a trace fixture): -//! ```bash -//! IRONCLAW_LIVE_TEST=1 cargo test --features libsql --test e2e_live_mission_gmail -- --ignored -//! ``` -//! -//! Replay (deterministic, after a fixture has been recorded): -//! ```bash -//! cargo test --features libsql --test e2e_live_mission_gmail -- --ignored -//! ``` -//! -//! Live mode requires Gmail OAuth credentials in the developer's -//! `~/.ironclaw/ironclaw.db` under the names declared in `with_secrets` -//! below. The test rig seeds *only* those rows into its temporary -//! database; nothing else (workspace memory, conversation history, other -//! secrets) crosses the boundary. See `tests/support/LIVE_TESTING.md`. - -#[cfg(feature = "libsql")] -mod support; - -#[cfg(feature = "libsql")] -mod live_mission_gmail_tests { - use std::time::{Duration, Instant}; - - use crate::support::live_harness::{LiveTestHarnessBuilder, TestMode}; - use crate::support::live_mission_helpers::{ - ApprovalAutoResponder, looks_like_routine_notification, tool_is, wait_for_response_matching, - }; - use crate::support::test_rig::TestRig; - - /// Channel name to use for the rig — mirrors the real "gateway" channel - /// so mission notifications route back the same way they do in production. - const CHANNEL: &str = "gateway"; - - /// User prompt that reproduces the #3133 flow. We deliberately ask for - /// drafts (not sends) so the test has minimal external blast radius. - /// "Trigger it once right now" is required to get a synchronous fire - /// the test can observe — the `every three minutes` cron alone wouldn't - /// fire within the test window. - const USER_PROMPT: &str = "Create a mission that creates a Gmail draft \ - every three minutes. The draft should go to the user's own Gmail \ - address (whichever account the OAuth token belongs to), with subject \ - \"Test mission #3133\" and body \"This is a test draft from the \ - IronClaw mission system.\" Trigger it once right now and report \ - whether the draft was created."; - - /// The orchestrator's "consecutive code errors" failure surface — same - /// regression marker the routine test guards against. Source: - /// `crates/ironclaw_engine/orchestrator/default.py:1003`. - const CONSECUTIVE_ERRORS_MARKER: &str = "consecutive code errors"; - - /// The exact #3133 regression marker. The model wrote "Status: None - /// Error: None" verbatim into FINAL() because it inspected a tool - /// response shape with `status` and `error` keys that were both - /// `None`. If that pattern shows up in the captured notification, - /// the bug is back. - const STATUS_NONE_MARKER: &str = "Status: None"; - const ERROR_NONE_MARKER: &str = "Error: None"; - - fn init_tracing() { - let _ = tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), - ) - .with_test_writer() - .try_init(); - } - - /// Engine v2's mission_create matches; the bridge alias path also - /// translates `routine_create`. Accept either. - fn used_create(tools: &[String]) -> bool { - tools - .iter() - .any(|t| tool_is(t, "mission_create") || tool_is(t, "routine_create")) - } - - /// Same dual-name acceptance for fire. - fn used_fire(tools: &[String]) -> bool { - tools - .iter() - .any(|t| tool_is(t, "mission_fire") || tool_is(t, "routine_fire")) - } - - /// Did the mission's child thread actually invoke the Gmail WASM tool? - /// Tool name on the wire is `gmail`; arguments may be folded in as - /// `gmail(create_draft)` via `format_action_display_name`. - fn used_gmail(tools: &[String]) -> bool { - tools.iter().any(|t| tool_is(t, "gmail")) - } - - /// Assert no captured response contains the #3133 regression marker. - /// The check is structural — we look for both "Status: None" AND - /// "Error: None" in the same response, since either alone could appear - /// in unrelated narration; the dual presence is the bug fingerprint. - async fn assert_no_status_none_failure(rig: &TestRig, where_in_test: &str) { - let responses = rig.wait_for_responses(0, Duration::from_millis(0)).await; - for r in &responses { - let has_status = r.content.contains(STATUS_NONE_MARKER); - let has_error = r.content.contains(ERROR_NONE_MARKER); - assert!( - !(has_status && has_error), - "[{where_in_test}] regression: response carried both \ - '{STATUS_NONE_MARKER}' and '{ERROR_NONE_MARKER}' — the \ - #3133 fingerprint of a Gmail mission FINAL() that bailed \ - with no useful diagnosis. Full response: {}", - r.content - ); - } - } - - /// Assert no captured response contains the orchestrator's consecutive- - /// errors failure surface (mirrors the #2583 sanity check). - async fn assert_no_consecutive_errors(rig: &TestRig, where_in_test: &str) { - let responses = rig.wait_for_responses(0, Duration::from_millis(0)).await; - for r in &responses { - assert!( - !r.content.to_lowercase().contains(CONSECUTIVE_ERRORS_MARKER), - "[{where_in_test}] regression: response carried the \ - '{CONSECUTIVE_ERRORS_MARKER}' failure surface from #2583. \ - Full response: {}", - r.content - ); - } - } - - #[test] - fn assert_no_status_none_failure_helper_recognises_pattern() { - // Dual-marker presence is the regression fingerprint. Single-marker - // presence in narration is fine and shouldn't trip the assertion. - let bug_text = "Failed to send email. Status: None Error: None\n\ - Next focus: Debug Gmail authentication."; - assert!( - bug_text.contains("Status: None") && bug_text.contains("Error: None"), - "the regression fingerprint must be both substrings present in \ - the same response (the helper wires this together with an &&)" - ); - } - - #[tokio::test] - #[ignore] // Live tier: requires Gmail OAuth in ~/.ironclaw/ironclaw.db. - // Records/replays from - // tests/fixtures/llm_traces/live/mission_gmail_draft_3133.json. - async fn mission_gmail_draft_3133() { - init_tracing(); - - // auto_approve_tools is intentionally OFF for the same reason as - // the routine test: administrative actions (mission_create / - // mission_fire) and write-effect actions (gmail.create_draft) are - // expected to surface ApprovalNeeded gates, and we want the - // responder to round-trip them. If the gates never fire, that's - // logged as a separate concern (parked under #2583's open items). - let harness = LiveTestHarnessBuilder::new("mission_gmail_draft_3133") - .with_engine_v2(true) - .with_max_tool_iterations(40) - .with_auto_approve_tools(false) - .with_channel_name(CHANNEL) - .with_secrets([ - "google_oauth_token", - "google_oauth_token_refresh_token", - "google_oauth_token_scopes", - ]) - .build() - .await; - - let rig = harness.rig(); - let approver = ApprovalAutoResponder::spawn(rig.channel_handle()); - - rig.send_message(USER_PROMPT).await; - - // Wait for the mission's child thread to deliver a notification. - // Same `**[name]**` anchor the routine test uses — proves the - // mission_fire path itself works end-to-end. - let setup_deadline = Instant::now() + Duration::from_secs(900); - let notification_text = - match wait_for_response_matching(rig, looks_like_routine_notification, setup_deadline) - .await - { - Some(text) => text, - None => { - let captured: Vec<String> = rig - .wait_for_responses(0, Duration::from_millis(0)) - .await - .iter() - .map(|r| r.content.clone()) - .collect(); - let tools = rig.tool_calls_started(); - approver.shutdown(); - panic!( - "no mission notification (with **[name]** marker) arrived \ - within 15 minutes — the mission did not deliver output \ - via the channel. Tool calls observed: {tools:?}. \ - Captured responses: {captured:#?}" - ); - } - }; - eprintln!( - "[GmailMissionTest] Notification preview: {}", - notification_text.chars().take(400).collect::<String>() - ); - - // The agent must have invoked mission/routine creation + fire. - let tools = rig.tool_calls_started(); - eprintln!("[GmailMissionTest] Tools observed: {tools:?}"); - assert!( - used_create(&tools), - "expected agent to call mission_create or routine_create after \ - prompt; got tools: {tools:?}" - ); - assert!( - used_fire(&tools), - "expected agent to fire the newly-created mission for the \ - 'trigger it once right now' clause; got tools: {tools:?}" - ); - - // Did the Gmail tool actually run? If not, either: - // - the lease/capability path didn't include gmail (real bug), or - // - the agent "tried" via some other route (http fallback). - // In either case the FINAL() narration is suspect — emit a - // diagnostic warning, but don't fail the test on this alone since - // the structural notification check + Status:None check below are - // the actual regression guards. - if !used_gmail(&tools) { - eprintln!( - "[GmailMissionTest] WARNING: the gmail WASM tool was not \ - invoked even though the prompt asked for a Gmail draft. \ - The mission's child thread likely fell back to `http` or \ - narrated 'I would call gmail' without actually doing so. \ - This is one of the candidate root causes for #3133." - ); - } - - // The hard regression guard for #3133. - assert_no_status_none_failure(rig, "after mission notification").await; - - // Sanity: the #2583 fix must still hold — no `consecutive code - // errors` should surface. If a fresh path triggers them, that's a - // signal the mission_* alias regression is back. - assert_no_consecutive_errors(rig, "after mission notification").await; - - // ── Approval-gate observation (warning-only, parked under #2583) ── - let approved = approver.approved_tools().await; - eprintln!( - "[GmailMissionTest] Approvals captured ({}): {approved:?}", - approved.len() - ); - if approved.is_empty() { - eprintln!( - "[GmailMissionTest] WARNING: auto_approve was OFF and zero \ - ApprovalNeeded gates were observed. Administrative tools \ - (mission_create / mission_fire) and the write-effect \ - gmail tool ran without prompting. This is the same parked \ - concern from the #2583 PR — track separately." - ); - } - - // Live mode only: surface unexpected tool failures in the captured - // trace so a maintainer reviewing a fixture diff sees them. - if harness.mode() == TestMode::Live { - let trace_errors = harness.collect_trace_errors(); - if !trace_errors.is_empty() { - eprintln!( - "[GmailMissionTest] WARNING: trace contained tool errors \ - (not failing the test, but worth investigating): \ - {trace_errors:?}" - ); - } - } - - approver.shutdown(); - - let all_text: Vec<String> = rig - .wait_for_responses(0, Duration::from_millis(0)) - .await - .iter() - .map(|r| r.content.clone()) - .collect(); - harness.finish(USER_PROMPT, &all_text).await; - } -} diff --git a/tests/e2e_live_personas.rs b/tests/e2e_live_personas.rs index 9c2e4afe6cf..99149d89c67 100644 --- a/tests/e2e_live_personas.rs +++ b/tests/e2e_live_personas.rs @@ -70,15 +70,83 @@ mod persona_tests { /// calls. Loads the real `./skills/` directory so persona skills /// (ceo-setup, content-creator-setup, etc.) are available. async fn build_persona_harness(test_name: &str) -> LiveTestHarness { - LiveTestHarnessBuilder::new(test_name) + // Persona scenarios activate skills (developer-setup, ceo-setup, + // creator-setup, trader-setup) whose tool sets include + // credentialed integrations: github_tool, gmail_tool, + // google_calendar_tool, slack_tool, telegram_tool, composio_tool, + // google_docs/sheets/slides_tool. Engine v2's auth pre-flight + // raises AuthRequired the moment any of those tools is dispatched + // without a matching credential in the secrets store, which parks + // the thread and starves the LLM of a response — the + // `developer_full_workflow` first turn was failing on this in + // canary runs because no `github_token` was seeded. + // + // Each credential is read from a `LIVE_CANARY_<NAME>` env var + // when present, falling back to a dummy value otherwise. Two + // execution modes: + // + // - Local / canary CI with the secret wired through (preferred): + // real credential reaches the WASM tool, which hits the real + // external API end-to-end. This is what we actually want the + // persona-rotating lane to verify. + // + // - Local without env, or canary without the secret set: dummy + // value satisfies the auth pre-flight, the tool's API call + // fails with 401, the agent recovers via workspace tools, and + // the workspace-content assertions still pass (every needle + // in DEV_*_CHECKS comes from the user's own message, not from + // API data). + let mut builder = LiveTestHarnessBuilder::new(test_name) .with_engine_v2(true) .with_auto_approve_tools(true) .with_max_tool_iterations(60) - .with_skills_dir(repo_skills_dir()) - .build() - .await + .with_skills_dir(repo_skills_dir()); + for (name, env_var, fallback) in PERSONA_CREDENTIALS { + let value = std::env::var(env_var).unwrap_or_else(|_| (*fallback).to_string()); + builder = builder.with_secret(*name, value); + } + builder.build().await } + /// Credentials referenced by the persona-rotating skill set. + /// + /// Each tuple is `(secrets_store_name, env_var, dummy_fallback)`: + /// + /// - `secrets_store_name`: the name the WASM tool's + /// `capabilities.json` looks up in the secrets store. + /// - `env_var`: the env var the test harness reads first; set in + /// `.github/workflows/live-canary.yml` from a GH Actions secret + /// for the canary lane, or exported locally for repro runs. + /// - `dummy_fallback`: used when the env var is unset. Satisfies + /// the auth pre-flight; tools will 401 against the real API. + const PERSONA_CREDENTIALS: &[(&str, &str, &str)] = &[ + ( + "github_token", + "LIVE_CANARY_GITHUB_TOKEN", + "ghp_mock_persona_canary_token", + ), + ( + "google_oauth_token", + "LIVE_CANARY_GOOGLE_OAUTH_TOKEN", + "ya29.mock_persona_canary", + ), + ( + "slack_bot_token", + "LIVE_CANARY_SLACK_BOT_TOKEN", + "xoxb-mock-persona-canary", + ), + ( + "telegram_bot_token", + "LIVE_CANARY_TELEGRAM_BOT_TOKEN", + "111222333:MOCK_PERSONA_CANARY", + ), + ( + "composio_api_key", + "LIVE_CANARY_COMPOSIO_API_KEY", + "mock-composio-persona-canary", + ), + ]; + struct PersonaCheck { needles: &'static [&'static str], context: &'static str, diff --git a/tests/e2e_responses_api_external_tools.rs b/tests/e2e_responses_api_external_tools.rs new file mode 100644 index 00000000000..1e38b2e5252 --- /dev/null +++ b/tests/e2e_responses_api_external_tools.rs @@ -0,0 +1,518 @@ +//! End-to-end integration tests for the Responses API external-tools +//! path through engine v2 with replay (trace) LLMs. +//! +//! These tests exercise the engine-native flow added in PR #3122 +//! (commit `44135ca5a` and follow-ups): +//! - `ExternalToolCatalog` registers caller-supplied actions per-thread +//! - `EffectBridgeAdapter::execute_action` short-circuits on catalog hits +//! to a `GatePaused { ResumeKind::External { ext_tool:<call_id> } }` +//! - bridge router projects the pause to `AppEvent::ExternalToolCall` +//! - the resume goes through `Submission::ExternalCallback { payload }` +//! and `bridge::handle_external_callback` +//! +//! The tests deliberately drive engine v2 directly via `TestRigBuilder` +//! (rather than the HTTP `/v1/responses` endpoint) so we can use the +//! existing `TraceLlm` replay infrastructure without spinning up a full +//! HTTP gateway. Wire-shape coverage stays in +//! `tests/responses_api_path_prefix.rs`. +//! +//! ## Regression coverage +//! +//! These tests guard the four bugs surfaced during the test-driven +//! review of the engine-native external-tools path and fixed in the +//! follow-up commit: +//! +//! 1. **Thread-id mismatch (Bug 1)** — `engine_pauses_when_llm_calls_registered_external_tool`. +//! Catalog is keyed by engine `ThreadId`; the responses_api handler +//! registers under the conversation_scope UUID it generates. The +//! bridge `transfer` hook in `bridge::handle_with_engine_inner` +//! re-keys onto the actual ThreadId after `handle_user_message` +//! returns. If that hook regresses, this test panics on +//! "engine never paused on external tool". +//! 2. **Resume payload materialisation (Bug 2)** — `round_trip_resume_payload_reaches_llm`. +//! The bridge's `resolve_gate` path for `GateResolution::ExternalCallback` +//! used to consult `pending.resume_output` only — which is `None` +//! for tool-flavoured pauses, so it would re-run the action and +//! re-pause forever. The fix special-cases `ext_tool:` callback +//! ids and synthesises an `ActionResult` ThreadMessage from the +//! resolution payload, which the LLM sees on its next call. +//! 3. **Collision rejection (Bug 3)** — covered at the HTTP boundary +//! in `tests/responses_api_path_prefix.rs::external_tool_name_shadowing_registered_action_is_rejected`. +//! Caller-supplied tool names that shadow registered actions are +//! rejected up-front so a confused LLM can't be tricked into +//! running caller code while believing it ran the internal tool. +//! 4. **Catalog cleanup on terminal state (Bug 4)** — +//! `catalog_cleared_on_terminal_completed_outcome`. After +//! `await_thread_outcome` joins on a non-`GatePaused` outcome, +//! the catalog entry for the thread is dropped so it can't leak +//! monotonically. + +#[cfg(feature = "libsql")] +mod support; + +#[cfg(feature = "libsql")] +mod tests { + use std::sync::Arc; + use std::sync::OnceLock; + use std::time::Duration; + + use crate::support::test_rig::TestRigBuilder; + use crate::support::trace_llm::LlmTrace; + use ironclaw::bridge::ExternalToolCatalog; + use ironclaw_engine::{ActionDef, EffectType, ModelToolSurface, ThreadId}; + use tokio::sync::Mutex; + + const TIMEOUT: Duration = Duration::from_secs(10); + + /// Serializes the engine-touching tests in this module. The + /// engine v2 bridge holds its state in a process-global + /// `OnceLock<RwLock<Option<EngineState>>>` (see + /// `src/bridge/router.rs::ENGINE_STATE`), so two parallel tests + /// would each spin up a `TestRig` whose messages are routed + /// through whichever rig won the `init_engine` race — one test's + /// messages disappear into the other's engine. Per-test state + /// isolation would need to land in the bridge itself; until then, + /// any test that drives the engine end-to-end via `TestRigBuilder` + /// must hold this mutex for its duration. + fn engine_state_lock() -> &'static Mutex<()> { + static LOCK: OnceLock<Mutex<()>> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + /// Helper: build an `ActionDef` for a caller-supplied function tool. + fn caller_action(name: &str, description: &str) -> ActionDef { + ActionDef { + name: name.to_string(), + description: description.to_string(), + parameters_schema: serde_json::json!({ + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"] + }), + effects: vec![EffectType::Compute], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + } + } + + /// Load the prepared 2-turn trace from disk. The fixture script + /// has the LLM emit a tool_call for `lookup_weather` and then + /// (on the second LLM call) produce text quoting "sunny and 72F" + /// — that text only appears in the LLM's context if the resume + /// payload was materialised back as a tool result. + fn round_trip_trace() -> LlmTrace { + let path = format!( + "{}/tests/fixtures/llm_traces/engine_v2/external_tool_round_trip.json", + env!("CARGO_MANIFEST_DIR") + ); + LlmTrace::from_file(&path).expect("load round-trip trace fixture") + } + + /// Wait until the engine state has registered a pending external + /// tool gate for the given user, or the timeout expires. Returns + /// the request_id of the pending gate. + async fn wait_for_external_pending_gate( + user_id: &str, + timeout: Duration, + ) -> Option<(uuid::Uuid, String)> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Ok(Some(view)) = ironclaw::bridge::get_engine_pending_gate(user_id, None).await + && matches!( + view.resume_kind, + ironclaw_engine::ResumeKind::External { .. } + ) + { + let request_id = uuid::Uuid::parse_str(&view.request_id).ok()?; + return Some((request_id, view.tool_name.clone())); + } + if tokio::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + // ------------------------------------------------------------------- + // Round-trip tests (the load-bearing showstopper coverage) + // ------------------------------------------------------------------- + + /// **The load-bearing test**: full pause + resume cycle. After the + /// caller supplies a tool result via `Submission::ExternalCallback` + /// payload, the engine must surface that result back to the LLM as + /// a tool result message. The LLM's second turn (per the trace) + /// produces text that quotes a phrase only present in the supplied + /// tool output — so if the payload never reaches the LLM, this + /// test fails. + /// + /// Pins the resume materialisation contract: `resolve_gate`'s + /// `ExternalCallback` arm consumes the payload via + /// `extract_external_tool_output` and synthesises an + /// `ActionResult` `ThreadMessage` for the LLM's next turn. + #[tokio::test] + async fn round_trip_resume_payload_reaches_llm() { + let _engine_guard = engine_state_lock().lock().await; + // Drop any engine state left over from a prior test before + // building this rig so the global ENGINE_STATE is bound to + // *this* TestRig's components, not a previous one's. + let trace = round_trip_trace(); + let rig = TestRigBuilder::new() + .with_engine_v2() + .with_trace(trace.clone()) + .build() + .await; + + let scope_uuid = uuid::Uuid::new_v4(); + let scope_str = scope_uuid.to_string(); + + // Register tools under the conversation_scope UUID — the + // bridge transfer hook re-keys to the engine's actual + // ThreadId once `handle_user_message` returns (Bug 1 fix). + register_under_scope(scope_uuid, "lookup_weather").await; + + let msg = ironclaw::channels::IncomingMessage::new( + "gateway", + "test-user", + "Look up the weather in NYC.", + ) + .with_thread(scope_str); + rig.send_incoming(msg).await; + + let (request_id, action_name) = wait_for_external_pending_gate("test-user", TIMEOUT) + .await + .expect( + "engine never paused on external tool — Bug 1 \ + (thread-id transfer) may have regressed", + ); + assert_eq!(action_name, "lookup_weather"); + + // Resume with the OpenAI-shaped output payload the + // responses_api handler builds out of `function_call_output` + // items. Bug 2 fix synthesizes an ActionResult ThreadMessage + // from the matching entry; without that fix, the LLM's + // second-turn context wouldn't see the output and + // `verify_trace_expects` would fail on `response_contains: + // ["sunny", "72F"]`. + rig.send_external_callback_with_payload( + request_id, + serde_json::json!({ + "outputs": [{ + "call_id": "call_lookup_weather_1", + "output": "sunny and 72F" + }] + }), + ) + .await; + + let responses = rig.wait_for_responses(1, TIMEOUT).await; + assert!(!responses.is_empty(), "no final response after resume"); + rig.verify_trace_expects(&trace, &responses); + rig.shutdown(); + } + + /// **Bug 4 regression**: the bridge clears catalog entries when + /// a thread reaches a terminal `Completed` outcome. Without the + /// `await_thread_outcome` cleanup hook, this catalog entry would + /// leak forever. + #[tokio::test] + async fn catalog_cleared_on_terminal_completed_outcome() { + let _engine_guard = engine_state_lock().lock().await; + // Drop any engine state left over from a prior test before + // building this rig so the global ENGINE_STATE is bound to + // *this* TestRig's components, not a previous one's. + // A simple text-only trace so the thread completes immediately + // (no gate, no pause). We register a catalog entry under the + // request's conversation_scope, verify it gets transferred + // onto the engine ThreadId, then verify it's gone after the + // thread completes. + let trace = LlmTrace::from_file(format!( + "{}/tests/fixtures/llm_traces/engine_v2/smoke_text.json", + env!("CARGO_MANIFEST_DIR") + )) + .expect("smoke text trace"); + let rig = TestRigBuilder::new() + .with_engine_v2() + .with_trace(trace.clone()) + .build() + .await; + + let scope_uuid = uuid::Uuid::new_v4(); + let scope_str = scope_uuid.to_string(); + + // Use a unique action name so the cleanup-poll check can + // verify "no entry anywhere has this action" regardless of + // what other concurrent tests have registered. Keying off + // `catalog.len()` was racy under parallel runs because the + // engine_external_tool_catalog is process-global. + let unique_action = format!("cleanup_marker_{}", uuid::Uuid::new_v4().simple()); + register_under_scope(scope_uuid, &unique_action).await; + + let catalog = ironclaw::bridge::engine_external_tool_catalog() + .await + .expect("catalog"); + assert!( + catalog.contains_action_anywhere(&unique_action).await, + "marker action must be present immediately after registration" + ); + + let msg = ironclaw::channels::IncomingMessage::new( + "gateway", + "test-user", + "Hello! Introduce yourself briefly.", + ) + .with_thread(scope_str); + rig.send_incoming(msg).await; + + // Wait for the thread to complete. We don't strictly assert + // on the response count: under engine v2, completed threads + // primarily broadcast the response over SSE (which the test + // channel doesn't subscribe to) and only return through + // `Channel::respond` for some outcomes. The catalog-cleanup + // poll below is the substantive assertion. + let _ = rig.wait_for_responses(1, TIMEOUT).await; + + // Cleanup runs on the same task that does the join. Poll for + // up to TIMEOUT — under parallel load the join task can be + // scheduled later than the response broadcast, so we mirror + // the response timeout rather than picking a tighter bound. + let deadline = tokio::time::Instant::now() + TIMEOUT; + loop { + if !catalog.contains_action_anywhere(&unique_action).await { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "catalog entry for marker action {unique_action:?} was not cleared \ + after thread completion — terminal cleanup hook may have regressed" + ); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + rig.shutdown(); + } + + /// Smoke variant of the round-trip: register a caller tool under + /// a `conversation_scope` UUID, send a message that carries that + /// UUID as its scope, and verify the engine pauses on the + /// resulting tool_call. This exercises the `transfer` hook in + /// `bridge::handle_with_engine_inner` that re-keys the catalog + /// from the handler-supplied UUID onto the engine's actual + /// `ThreadId` before the LLM call lands. + #[tokio::test] + async fn engine_pauses_when_llm_calls_registered_external_tool() { + let _engine_guard = engine_state_lock().lock().await; + // Drop any engine state left over from a prior test before + // building this rig so the global ENGINE_STATE is bound to + // *this* TestRig's components, not a previous one's. + let trace = round_trip_trace(); + let rig = TestRigBuilder::new() + .with_engine_v2() + .with_trace(trace.clone()) + .build() + .await; + + let scope_uuid = uuid::Uuid::new_v4(); + let scope_str = scope_uuid.to_string(); + + // Register tools under the conversation_scope UUID. The bridge + // transfer hook will rebind onto the actual engine ThreadId + // once `handle_user_message` returns. Using + // `register_under_scope` rather than `engine_external_tool_catalog` + // directly is the resilient way to bootstrap from a clean + // rig — see the helper for the polling rationale. + register_under_scope(scope_uuid, "lookup_weather").await; + + // Send the user message with the matching conversation_scope. + let msg = ironclaw::channels::IncomingMessage::new( + "gateway", + "test-user", + "Look up the weather in NYC.", + ) + .with_thread(scope_str); + rig.send_incoming(msg).await; + + let pending = wait_for_external_pending_gate("test-user", TIMEOUT).await; + assert!( + pending.is_some(), + "expected an external-tool pending gate to fire after the LLM emitted \ + tool_calls for the registered name; found none" + ); + rig.shutdown(); + } + + /// Lazily register a caller tool under a `scope_uuid` ThreadId. + /// The engine catalog only exists after `init_engine` runs; that + /// happens on the first message routed through the bridge. To + /// bootstrap from a clean rig, we poll until the catalog is + /// available, then register. In practice the responses_api + /// handler depends on the same lazy bootstrap — so this mirrors + /// production behaviour. + async fn register_under_scope(scope_uuid: uuid::Uuid, action_name: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_millis(500); + loop { + if let Some(catalog) = ironclaw::bridge::engine_external_tool_catalog().await { + catalog + .register( + ThreadId(scope_uuid), + vec![caller_action(action_name, "caller-supplied test tool")], + ) + .await; + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "engine catalog never initialised; the bridge may not have \ + bootstrapped engine v2 — verify TestRigBuilder.with_engine_v2() ran" + ); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + // ------------------------------------------------------------------- + // Catalog lifecycle and isolation + // ------------------------------------------------------------------- + + /// Catalog isolation by ThreadId — registering under one thread + /// id does not leak into another. Direct catalog test, no engine + /// roundtrip. + #[tokio::test] + async fn catalog_isolates_by_thread_id() { + let catalog = ExternalToolCatalog::new(); + let thread_a = ThreadId::new(); + let thread_b = ThreadId::new(); + catalog + .register(thread_a, vec![caller_action("a_only", "tool A")]) + .await; + + assert!(catalog.contains(thread_a, "a_only").await); + assert!(!catalog.contains(thread_b, "a_only").await); + assert!(catalog.list(thread_b).await.is_empty()); + } + + /// Re-registering replaces, not merges. The Responses API contract + /// is that each request restates the full `tools[]` list — so + /// dropping one tool from a follow-up request must remove it from + /// the catalog rather than leave it lurking from the prior request. + #[tokio::test] + async fn catalog_register_overwrites_not_merges() { + let catalog = ExternalToolCatalog::new(); + let thread = ThreadId::new(); + catalog + .register( + thread, + vec![ + caller_action("first", "first"), + caller_action("second", "second"), + ], + ) + .await; + // Second request restates only one tool. + catalog + .register(thread, vec![caller_action("second", "second")]) + .await; + let listed = catalog.list(thread).await; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].name, "second"); + } + + /// TTL sweep evicts stale entries while leaving fresh ones. + /// Defensive coverage for the backstop against callers that + /// register tools and then abandon the conversation. + #[tokio::test] + async fn catalog_sweep_evicts_only_stale_entries() { + let catalog = ExternalToolCatalog::new(); + let fresh = ThreadId::new(); + catalog.register(fresh, vec![caller_action("a", "a")]).await; + // Sweep with an absurdly long max-age — nothing should evict. + let evicted = catalog.sweep_older_than(chrono::Duration::days(1)).await; + assert!(evicted.is_empty()); + // Sweep with a zero/negative max-age — everything evicts. + let evicted = catalog + .sweep_older_than(chrono::Duration::seconds(-1)) + .await; + assert_eq!(evicted, vec![fresh]); + assert!(catalog.is_empty().await); + } + + /// Two concurrent registrations against different thread ids do + /// not interfere. Documents the per-thread isolation invariant + /// even under contention — protects against a future change that + /// might accidentally collapse all entries into a single map. + #[tokio::test] + async fn catalog_handles_concurrent_registrations() { + let catalog = Arc::new(ExternalToolCatalog::new()); + let mut handles = Vec::new(); + for _ in 0..32 { + let catalog = Arc::clone(&catalog); + handles.push(tokio::spawn(async move { + let tid = ThreadId::new(); + catalog + .register(tid, vec![caller_action("concurrent", "concurrent test")]) + .await; + assert!(catalog.contains(tid, "concurrent").await); + })); + } + for h in handles { + h.await.expect("concurrent task did not panic"); + } + assert_eq!(catalog.len().await, 32); + } + + /// Callback-id helpers round-trip cleanly. This is the single + /// disambiguator between caller-tool pauses and OAuth/pairing + /// pauses in `bridge::router::notify_pending_gate` — if the + /// `ext_tool:` prefix invariant is broken, OAuth pauses would be + /// projected as `AppEvent::ExternalToolCall` (or vice versa) and + /// the wrong UI would render. + #[tokio::test] + async fn callback_id_disambiguates_external_from_oauth() { + use ironclaw::bridge::{ + call_id_from_external_callback, external_tool_callback_id, is_external_tool_callback_id, + }; + let cb = external_tool_callback_id("call_42"); + assert!(cb.starts_with("ext_tool:")); + assert!(is_external_tool_callback_id(&cb)); + assert_eq!(call_id_from_external_callback(&cb), Some("call_42")); + + // OAuth/pairing flows generate "pairing:<extension>" callback + // ids — must NOT match the external-tool prefix. + assert!(!is_external_tool_callback_id("pairing:telegram")); + assert_eq!( + call_id_from_external_callback("pairing:telegram"), + None, + "OAuth callback must not strip clean as a tool call_id" + ); + } + + /// Direct unit-level proof that `clear()` removes a registered + /// entry. The production cleanup (Bug 4 fix) is wired in + /// `await_thread_outcome` and exercised end-to-end by + /// `catalog_cleared_on_terminal_completed_outcome` above; this + /// test is the narrow primitive coverage for the helper itself. + #[tokio::test] + async fn catalog_clear_removes_entry() { + let catalog = ExternalToolCatalog::new(); + let thread = ThreadId::new(); + catalog + .register(thread, vec![caller_action("a", "a")]) + .await; + assert!(!catalog.is_empty().await); + + catalog.clear(thread).await; + assert!(catalog.is_empty().await); + } + + // Note on collision rejection: the request-validation rejection + // for caller-supplied tool names that shadow registered actions + // is exercised at the HTTP level in + // `tests/responses_api_path_prefix.rs::external_tool_name_shadowing_registered_action_is_rejected`. + // That test asserts the handler returns 400 mentioning the + // colliding name. A cargo-feature-gated registry isn't reachable + // from this engine-tier file, so the check belongs at the wire + // boundary. +} diff --git a/tests/e2e_routine_heartbeat.rs b/tests/e2e_routine_heartbeat.rs index 8c110721ff7..13544769c32 100644 --- a/tests/e2e_routine_heartbeat.rs +++ b/tests/e2e_routine_heartbeat.rs @@ -28,13 +28,13 @@ mod tests { use ironclaw::db::{Database, libsql::LibSqlBackend}; use ironclaw::extensions::ExtensionManager; use ironclaw::hooks::HookRegistry; - use ironclaw::llm::LlmProvider; use ironclaw::secrets::{InMemorySecretsStore, SecretsCrypto, SecretsStore}; use ironclaw::tools::builtin::routine::RoutineUpdateTool; use ironclaw::tools::mcp::{McpProcessManager, McpSessionManager}; use ironclaw::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput, ToolRegistry}; use ironclaw::workspace::Workspace; use ironclaw::workspace::hygiene::HygieneConfig; + use ironclaw_llm::LlmProvider; use ironclaw_safety::SafetyLayer; use crate::support::trace_llm::{LlmTrace, TraceLlm, TraceResponse, TraceStep, TraceToolCall}; diff --git a/tests/e2e_telegram_message_routing.rs b/tests/e2e_telegram_message_routing.rs index a460c32e55e..5e9ce7b98fe 100644 --- a/tests/e2e_telegram_message_routing.rs +++ b/tests/e2e_telegram_message_routing.rs @@ -19,7 +19,7 @@ mod tests { use ironclaw::config::Config; use ironclaw::db::{Database, libsql::LibSqlBackend}; use ironclaw::error::ChannelError; - use ironclaw::llm::{LlmProvider, SessionConfig, SessionManager}; + use ironclaw_llm::{LlmProvider, SessionConfig, SessionManager}; use tokio::sync::{Mutex, mpsc}; use tokio_stream::wrappers::ReceiverStream; diff --git a/tests/e2e_wasm_github_coercion.rs b/tests/e2e_wasm_github_coercion.rs index 04e54ff1cf8..f875b3ec2ce 100644 --- a/tests/e2e_wasm_github_coercion.rs +++ b/tests/e2e_wasm_github_coercion.rs @@ -25,7 +25,7 @@ mod tests { use serde_json::json; - use ironclaw::llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; + use ironclaw_llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; use crate::support::test_rig::TestRigBuilder; use crate::support::trace_llm::{ diff --git a/tests/e2e_wasm_portfolio.rs b/tests/e2e_wasm_portfolio.rs index 0e4d7f666a6..8a13f2d338a 100644 --- a/tests/e2e_wasm_portfolio.rs +++ b/tests/e2e_wasm_portfolio.rs @@ -19,7 +19,7 @@ mod tests { use serde_json::json; - use ironclaw::llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; + use ironclaw_llm::recording::{HttpExchange, HttpExchangeRequest, HttpExchangeResponse}; use crate::support::test_rig::TestRigBuilder; use crate::support::trace_llm::{ diff --git a/tests/e2e_workspace_coverage.rs b/tests/e2e_workspace_coverage.rs index f13c3e4753e..ab53563dc5c 100644 --- a/tests/e2e_workspace_coverage.rs +++ b/tests/e2e_workspace_coverage.rs @@ -308,7 +308,7 @@ mod tests { let first_request = &captured[0]; let system_msg = first_request .iter() - .find(|msg| matches!(msg.role, ironclaw::llm::Role::System)); + .find(|msg| matches!(msg.role, ironclaw_llm::Role::System)); assert!( system_msg.is_some(), "Expected a system message in the first request" diff --git a/tests/engine_v2_gate_integration.rs b/tests/engine_v2_gate_integration.rs index 137333d2a32..4113d86f3be 100644 --- a/tests/engine_v2_gate_integration.rs +++ b/tests/engine_v2_gate_integration.rs @@ -14,7 +14,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use async_trait::async_trait; use chrono::Utc; use tokio::sync::RwLock; @@ -27,13 +26,8 @@ use ironclaw_engine::{ ThreadMessage, ThreadOutcome, ThreadState, ThreadType, TokenUsage, }; -use ironclaw::bridge::EffectBridgeAdapter; -use ironclaw::context::JobContext; use ironclaw::gate::pending::{PendingGate, PendingGateKey}; use ironclaw::gate::store::{GateStoreError, PendingGateStore, TRUSTED_GATE_CHANNELS}; -use ironclaw::hooks::HookRegistry; -use ironclaw::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput, ToolRegistry}; -use ironclaw_safety::{SafetyConfig, SafetyLayer}; // ── Scripted LLM ───────────────────────────────────────────── @@ -139,43 +133,6 @@ struct GateMockEffects { authenticated: RwLock<std::collections::HashSet<String>>, } -struct ApprovalTool; - -#[async_trait] -impl Tool for ApprovalTool { - fn name(&self) -> &str { - "approval_test" - } - - fn description(&self) -> &str { - "Integration test approval tool" - } - - fn parameters_schema(&self) -> serde_json::Value { - serde_json::json!({ - "type": "object", - "properties": { - "value": { "type": "string" } - } - }) - } - - async fn execute( - &self, - params: serde_json::Value, - _ctx: &JobContext, - ) -> Result<ToolOutput, ToolError> { - Ok(ToolOutput::success( - serde_json::json!({"ok": true, "params": params}), - Duration::from_millis(1), - )) - } - - fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { - ApprovalRequirement::UnlessAutoApproved - } -} - impl GateMockEffects { fn new(gate_approval_tools: Vec<String>, gate_auth_tools: Vec<String>) -> Arc<Self> { Self::new_with_chain(gate_approval_tools, gate_auth_tools, Vec::new()) @@ -786,26 +743,6 @@ fn make_caps(require_approval: bool) -> CapabilityRegistry { caps } -fn make_caps_with_approval_tool() -> CapabilityRegistry { - let mut caps = CapabilityRegistry::new(); - caps.register(Capability { - name: "tools".into(), - description: "test tools".into(), - actions: vec![ActionDef { - name: "approval_test".into(), - description: "Approval test tool".into(), - parameters_schema: serde_json::json!({"type": "object"}), - effects: vec![EffectType::WriteExternal], - requires_approval: false, - model_tool_surface: ModelToolSurface::FullSchema, - discovery: None, - }], - knowledge: vec![], - policies: vec![], - }); - caps -} - fn make_caps_with_install_and_alias_followup() -> CapabilityRegistry { let mut caps = CapabilityRegistry::new(); caps.register(Capability { @@ -875,39 +812,106 @@ fn resumed_action_result_message( ThreadMessage::action_result(call_id, action_name, rendered) } +/// Test gate controller that approves every Approval gate inline: +/// records the request, marks the action approved on the underlying +/// `GateMockEffects`, and returns `Approved`. The engine's inline-retry +/// then re-executes the gated tool — which, with the action now in the +/// `approved` set, succeeds on the second call. +/// +/// Replaces the legacy `mgr.join_thread() → ThreadOutcome::GatePaused → +/// resume_thread` dance that this PR's inline-await design replaces for +/// `Approval` resume kinds. Authentication/External resume kinds still +/// take the legacy path, so tests asserting Auth gate semantics +/// continue to work without this controller. +struct AutoApprovingGateController { + effects: Arc<GateMockEffects>, + pauses: tokio::sync::Mutex<Vec<ironclaw_engine::GatePauseRequest>>, +} + +impl AutoApprovingGateController { + fn new(effects: Arc<GateMockEffects>) -> Arc<Self> { + Arc::new(Self { + effects, + pauses: tokio::sync::Mutex::new(Vec::new()), + }) + } + + async fn pauses_seen(&self) -> Vec<ironclaw_engine::GatePauseRequest> { + self.pauses.lock().await.clone() + } +} + +#[async_trait::async_trait] +impl ironclaw_engine::GateController for AutoApprovingGateController { + async fn pause( + &self, + request: ironclaw_engine::GatePauseRequest, + ) -> ironclaw_engine::GateResolution { + // Only auto-approve Approval gates. Authentication gates need + // an actual credential write — returning Cancelled here makes + // the engine fall through to the legacy `ThreadOutcome::GatePaused` + // unwind so legacy-path tests (auth resume via thread re-entry) + // continue to work alongside the new inline-await Authentication + // path covered by `authentication_gate_resolves_inline_via_controller`. + if matches!( + request.resume_kind, + ironclaw_engine::ResumeKind::Authentication { .. } + ) { + self.pauses.lock().await.push(request); + return ironclaw_engine::GateResolution::Cancelled; + } + self.effects.mark_approved(&request.action_name).await; + self.pauses.lock().await.push(request); + ironclaw_engine::GateResolution::Approved { always: true } + } +} + // ── Tests: GatePaused ThreadOutcome ────────────────────────── /// When effect executor returns GatePaused, the thread transitions to /// Waiting and the outcome carries the gate info. #[tokio::test] -async fn gate_paused_transitions_thread_to_waiting() { +async fn approval_gate_resolves_inline_via_controller() { + // Post-PR semantics: Approval gates raised by `EffectExecutor::execute_action` + // are caught inline by the engine's `GateController`, not bubbled up + // as `ThreadOutcome::GatePaused`. With an auto-approving controller + // wired, the gated action retries inline and the thread runs to + // completion in a single `join_thread`. Pre-PR this same fixture + // would surface `ThreadOutcome::GatePaused` and require an explicit + // `resume_thread`; that path is now reserved for Auth/External. let project_id = ProjectId::new(); let effects = GateMockEffects::new(vec!["http".into()], vec![]); - // LLM returns a structured tool call for http - let llm = ScriptedLlm::new(vec![LlmOutput { - response: LlmResponse::ActionCalls { - calls: vec![ironclaw_engine::ActionCall { - id: "call_1".into(), - action_name: "http".into(), - parameters: serde_json::json!({"url": "https://example.com"}), - }], - content: None, + let llm = ScriptedLlm::new(vec![ + LlmOutput { + response: LlmResponse::ActionCalls { + calls: vec![ironclaw_engine::ActionCall { + id: "call_1".into(), + action_name: "http".into(), + parameters: serde_json::json!({"url": "https://example.com"}), + }], + content: None, + }, + usage: TokenUsage::default(), }, - usage: TokenUsage::default(), - }]); + LlmOutput { + response: LlmResponse::Text("done".into()), + usage: TokenUsage::default(), + }, + ]); let store = TestStore::new(); - // Use requires_approval=false so PolicyEngine doesn't intercept before - // EffectExecutor — the mock returns GatePaused from execute_action(). - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm, - effects, + effects.clone(), store.clone() as Arc<dyn Store>, Arc::new(make_caps(false)), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + let controller = AutoApprovingGateController::new(effects.clone()); + mgr.set_gate_controller(controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; let tid = mgr .spawn_thread( @@ -922,28 +926,177 @@ async fn gate_paused_transitions_thread_to_waiting() { .expect("spawn_thread"); let outcome = mgr.join_thread(tid).await.expect("join_thread"); + assert!( + matches!(outcome, ThreadOutcome::Completed { .. }), + "expected Completed after inline approval, got {outcome:?}" + ); - // Thread should have paused with GatePaused outcome - match &outcome { - ThreadOutcome::GatePaused { - gate_name, - action_name, - resume_kind, - .. - } => { - assert_eq!(gate_name, "approval"); - assert_eq!(action_name, "http"); - assert!(matches!(resume_kind, ResumeKind::Approval { .. })); + // The controller observed exactly one Approval pause for the http call. + let pauses = controller.pauses_seen().await; + assert_eq!(pauses.len(), 1, "expected one inline pause"); + assert_eq!(pauses[0].gate_name, "approval"); + assert_eq!(pauses[0].action_name, "http"); + assert!(matches!(pauses[0].resume_kind, ResumeKind::Approval { .. })); + + // The thread reaches Done after the inline retry succeeds. + let saved = store.load_thread(tid).await.unwrap().unwrap(); + assert_eq!(saved.state, ThreadState::Done); + + // Both an ApprovalRequested (gate fired) and an ActionExecuted + // (post-approval retry) event are recorded. + let approval_events: Vec<_> = saved + .events + .iter() + .filter(|e| { + matches!( + e.kind, + ironclaw_engine::types::event::EventKind::ApprovalRequested { .. } + ) + }) + .collect(); + assert_eq!(approval_events.len(), 1, "exactly one approval requested"); + let executed = saved.events.iter().any(|e| { + matches!( + &e.kind, + ironclaw_engine::types::event::EventKind::ActionExecuted { action_name, .. } + if action_name == "http" + ) + }); + assert!(executed, "http should have executed after approval"); +} + +/// Issue #3133 / #3166: Tier 0 inline-await for Authentication gates. +/// +/// Companion to `approval_gate_resolves_inline_via_controller` — same +/// shape but with `ResumeKind::Authentication` instead of Approval. +/// Pre-fix the engine's Tier 0 retry loop bailed for Authentication +/// (returning `Err(GatePaused)` unmodified, which surfaced as +/// `ThreadOutcome::GatePaused` and required a thread re-entry to +/// resume). Post-fix Authentication flows through the same inline- +/// await path: the host controller delivers `Approved` once the +/// credential is registered (in production this happens via the +/// OAuth-callback hook in `bridge::resolve_inline_gates_for_credential`), +/// the action retries inline, and the thread runs to completion in a +/// single `join_thread`. +#[tokio::test] +async fn authentication_gate_resolves_inline_via_controller() { + let project_id = ProjectId::new(); + // `gate_auth_tools = ["http"]` makes the mock return GatePaused + // with Authentication resume kind on the first call. The + // `AutoApprovingGateController::pause` hook calls + // `mark_authenticated` before returning Approved, so the inline + // retry sees the credential as present and the action succeeds. + let effects = GateMockEffects::new(vec![], vec!["http".into()]); + + let llm = ScriptedLlm::new(vec![ + LlmOutput { + response: LlmResponse::ActionCalls { + calls: vec![ironclaw_engine::ActionCall { + id: "call_1".into(), + action_name: "http".into(), + parameters: serde_json::json!({"url": "https://api.example.com"}), + }], + content: None, + }, + usage: TokenUsage::default(), + }, + LlmOutput { + response: LlmResponse::Text("done".into()), + usage: TokenUsage::default(), + }, + ]); + + let store = TestStore::new(); + let mgr = Arc::new(ThreadManager::new( + llm, + effects.clone(), + store.clone() as Arc<dyn Store>, + Arc::new(make_caps(false)), + Arc::new(LeaseManager::new()), + Arc::new(PolicyEngine::new()), + )); + // Custom auto-approving controller that marks the action as + // *authenticated* (not just approved) before returning Approved. + // Mirrors the production path where the OAuth callback writes the + // credential to the secrets store and then delivers Approved to + // the parked waiter — the retry sees the credential as present. + struct AuthAutoApprover { + effects: Arc<GateMockEffects>, + pauses: tokio::sync::Mutex<Vec<ironclaw_engine::GatePauseRequest>>, + } + #[async_trait::async_trait] + impl ironclaw_engine::GateController for AuthAutoApprover { + async fn pause( + &self, + request: ironclaw_engine::GatePauseRequest, + ) -> ironclaw_engine::GateResolution { + self.effects.mark_authenticated(&request.action_name).await; + self.pauses.lock().await.push(request); + ironclaw_engine::GateResolution::Approved { always: false } } - other => panic!("Expected GatePaused, got: {other:?}"), } + let controller = Arc::new(AuthAutoApprover { + effects: effects.clone(), + pauses: tokio::sync::Mutex::new(Vec::new()), + }); + mgr.set_gate_controller(controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; - // Thread state should be Waiting (safety net in loop_engine.rs) - let thread = store.load_thread(tid).await.unwrap().unwrap(); - assert_eq!( - thread.state, - ThreadState::Waiting, - "Thread should be in Waiting state after GatePaused" + let tid = mgr + .spawn_thread( + "fetch from authenticated endpoint", + ThreadType::Foreground, + project_id, + ThreadConfig::default(), + None, + "test-user", + ) + .await + .expect("spawn_thread"); + + let outcome = mgr.join_thread(tid).await.expect("join_thread"); + assert!( + matches!(outcome, ThreadOutcome::Completed { .. }), + "expected Completed after inline auth resolution, got {outcome:?}" + ); + + // Controller observed exactly one Authentication pause. + let pauses = controller.pauses.lock().await; + assert_eq!(pauses.len(), 1, "expected one inline pause"); + assert_eq!(pauses[0].gate_name, "authentication"); + assert_eq!(pauses[0].action_name, "http"); + assert!( + matches!(pauses[0].resume_kind, ResumeKind::Authentication { .. }), + "pause should carry Authentication resume_kind" + ); + + // Thread reached Done after the inline retry. + let saved = store.load_thread(tid).await.unwrap().unwrap(); + assert_eq!(saved.state, ThreadState::Done); + + // Both the gate-fired event and the post-resolution retry are + // recorded — same audit shape as the Approval inline-await test. + let approval_events: Vec<_> = saved + .events + .iter() + .filter(|e| { + matches!( + e.kind, + ironclaw_engine::types::event::EventKind::ApprovalRequested { .. } + ) + }) + .collect(); + assert_eq!(approval_events.len(), 1, "exactly one gate raised"); + let executed = saved.events.iter().any(|e| { + matches!( + &e.kind, + ironclaw_engine::types::event::EventKind::ActionExecuted { action_name, .. } + if action_name == "http" + ) + }); + assert!( + executed, + "http should have executed after the credential was registered" ); } @@ -1011,7 +1164,13 @@ async fn gate_paused_authentication_carries_credential_name() { /// A paused thread remains resumable and completes after approval. #[tokio::test] -async fn gate_paused_thread_resumes_to_completion() { +async fn approval_denied_inline_completes_thread_with_failed_action() { + // Post-PR semantics: when the inline-await controller denies a + // gate, the gated tool call fails (typed denial, not the legacy + // pre-fix "execution paused by gate" RuntimeError) and the thread + // continues to completion. Pre-PR this fixture would have paused + // the thread; the inline-await design covers this with a single + // controller round-trip. let project_id = ProjectId::new(); let effects = GateMockEffects::new(vec!["http".into()], vec![]); @@ -1028,21 +1187,24 @@ async fn gate_paused_thread_resumes_to_completion() { usage: TokenUsage::default(), }, LlmOutput { - response: LlmResponse::Text("done".into()), + response: LlmResponse::Text("denied — moving on".into()), usage: TokenUsage::default(), }, ]); let store = TestStore::new(); - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm, effects.clone(), store.clone() as Arc<dyn Store>, Arc::new(make_caps(false)), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + // `CancellingGateController` is the engine default; equivalent to + // a controller that always denies. No explicit `set_gate_controller` + // needed. let tid = mgr .spawn_thread( "make an http post", @@ -1055,37 +1217,25 @@ async fn gate_paused_thread_resumes_to_completion() { .await .expect("spawn_thread"); - let first = mgr.join_thread(tid).await.expect("first join"); - assert!(matches!(first, ThreadOutcome::GatePaused { .. })); - assert_eq!( - store.load_thread(tid).await.unwrap().unwrap().state, - ThreadState::Waiting + let outcome = mgr.join_thread(tid).await.expect("join_thread"); + assert!( + matches!(outcome, ThreadOutcome::Completed { .. }), + "expected Completed after inline denial, got {outcome:?}" ); - effects.mark_approved("http").await; - mgr.resume_thread( - tid, - "test-user", - Some(ThreadMessage::user("approved")), - Some(("call_gate_1".into(), true)), - None, - ) - .await - .expect("resume_thread"); - - let resumed = mgr.join_thread(tid).await.expect("second join"); - if !matches!(resumed, ThreadOutcome::Completed { .. }) { - panic!("expected Completed after approved retry, got {:?}", resumed); - } let saved = store.load_thread(tid).await.unwrap().unwrap(); assert_eq!(saved.state, ThreadState::Done); - assert!( - saved.events.iter().any(|event| matches!( - event.kind, - ironclaw_engine::types::event::EventKind::ApprovalReceived { .. } - )), - "resume should record ApprovalReceived" - ); + + // The denied call surfaces as ActionFailed, not as a stranded + // pending gate. + let failed = saved.events.iter().any(|e| { + matches!( + &e.kind, + ironclaw_engine::types::event::EventKind::ActionFailed { action_name, .. } + if action_name == "http" + ) + }); + assert!(failed, "http should have failed after denial"); } #[tokio::test] @@ -1161,27 +1311,28 @@ async fn tool_info_does_not_gate_callable_tool_into_next_llm_callable_set() { } #[tokio::test] -async fn approval_resolution_executes_pending_call_directly() { +async fn auth_resolution_retries_same_pending_action_without_second_pause() { let project_id = ProjectId::new(); - let tools = Arc::new(ToolRegistry::new()); - tools.register(Arc::new(ApprovalTool)).await; - - let effects = Arc::new(EffectBridgeAdapter::new( - tools, - Arc::new(SafetyLayer::new(&SafetyConfig { - max_output_length: 10_000, - injection_check_enabled: false, - })), - Arc::new(HookRegistry::default()), - )); + let effects = GateMockEffects::new(vec![], vec!["http".into()]); let llm = ScriptedLlm::new(vec![ LlmOutput { response: LlmResponse::ActionCalls { calls: vec![ironclaw_engine::ActionCall { - id: "call_approval_1".into(), - action_name: "approval_test".into(), - parameters: serde_json::json!({"value": "hello"}), + id: "call_auth_1".into(), + action_name: "http".into(), + parameters: serde_json::json!({"url": "https://example.com/private"}), + }], + content: None, + }, + usage: TokenUsage::default(), + }, + LlmOutput { + response: LlmResponse::ActionCalls { + calls: vec![ironclaw_engine::ActionCall { + id: "call_auth_2".into(), + action_name: "http".into(), + parameters: serde_json::json!({"url": "https://example.com/private"}), }], content: None, }, @@ -1198,14 +1349,14 @@ async fn approval_resolution_executes_pending_call_directly() { llm, effects.clone(), store.clone() as Arc<dyn Store>, - Arc::new(make_caps_with_approval_tool()), + Arc::new(make_caps(false)), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), ); let tid = mgr .spawn_thread( - "run the approval tool", + "call the authenticated endpoint", ThreadType::Foreground, project_id, ThreadConfig::default(), @@ -1216,23 +1367,7 @@ async fn approval_resolution_executes_pending_call_directly() { .expect("spawn_thread"); let first = mgr.join_thread(tid).await.expect("first join"); - match first { - ThreadOutcome::GatePaused { - gate_name, - action_name, - call_id, - parameters, - resume_kind, - .. - } => { - assert_eq!(gate_name, "approval"); - assert_eq!(action_name, "approval_test"); - assert_eq!(call_id, "call_approval_1"); - assert_eq!(parameters["value"], "hello"); - assert!(matches!(resume_kind, ResumeKind::Approval { .. })); - } - other => panic!("expected GatePaused approval, got {other:?}"), - } + assert!(matches!(first, ThreadOutcome::GatePaused { .. })); assert_eq!( store.load_thread(tid).await.unwrap().unwrap().state, ThreadState::Waiting @@ -1241,43 +1376,47 @@ async fn approval_resolution_executes_pending_call_directly() { let thread = store.load_thread(tid).await.unwrap().unwrap(); let lease = mgr .leases - .find_lease_for_action(tid, "approval_test") + .find_lease_for_action(tid, "http") .await - .expect("lease for approval_test"); + .expect("lease for http"); let exec_ctx = ironclaw_engine::ThreadExecutionContext { thread_id: tid, thread_type: thread.thread_type, project_id: thread.project_id, user_id: "test-user".into(), step_id: ironclaw_engine::StepId::new(), - current_call_id: Some("call_approval_1".into()), + current_call_id: Some("call_auth_1".into()), source_channel: None, user_timezone: None, thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; - let tool_result = effects - .execute_resolved_pending_action( - "approval_test", - serde_json::json!({"value": "hello"}), + effects.mark_authenticated("http").await; + let result = effects + .execute_action( + "http", + serde_json::json!({"url": "https://example.com/private"}), &lease, &exec_ctx, - true, ) .await - .expect("approved pending call should execute directly"); + .expect("authenticated pending action should execute directly"); mgr.resume_thread( tid, "test-user", Some(resumed_action_result_message( - "call_approval_1", - "approval_test", - &tool_result.output, + "call_auth_1", + "http", + &result.output, )), - Some(("call_approval_1".into(), true)), - Some("call_approval_1".into()), + None, + Some("call_auth_1".into()), ) .await .expect("resume_thread"); @@ -1285,149 +1424,11 @@ async fn approval_resolution_executes_pending_call_directly() { let resumed = mgr.join_thread(tid).await.expect("second join"); assert!( matches!(resumed, ThreadOutcome::Completed { .. }), - "expected Completed after approval retry, got {resumed:?}" + "expected Completed after auth retry, got {resumed:?}" ); let saved = store.load_thread(tid).await.unwrap().unwrap(); - assert_eq!(saved.state, ThreadState::Done); - let approval_requests = saved - .events - .iter() - .filter(|event| { - matches!( - event.kind, - ironclaw_engine::types::event::EventKind::ApprovalRequested { .. } - ) - }) - .count(); - assert_eq!( - approval_requests, 1, - "resumed execution should not prompt for approval again" - ); - assert!( - saved.events.iter().any(|event| matches!( - event.kind, - ironclaw_engine::types::event::EventKind::ApprovalReceived { .. } - )), - "resume should record ApprovalReceived" - ); -} - -#[tokio::test] -async fn auth_resolution_retries_same_pending_action_without_second_pause() { - let project_id = ProjectId::new(); - let effects = GateMockEffects::new(vec![], vec!["http".into()]); - - let llm = ScriptedLlm::new(vec![ - LlmOutput { - response: LlmResponse::ActionCalls { - calls: vec![ironclaw_engine::ActionCall { - id: "call_auth_1".into(), - action_name: "http".into(), - parameters: serde_json::json!({"url": "https://example.com/private"}), - }], - content: None, - }, - usage: TokenUsage::default(), - }, - LlmOutput { - response: LlmResponse::ActionCalls { - calls: vec![ironclaw_engine::ActionCall { - id: "call_auth_2".into(), - action_name: "http".into(), - parameters: serde_json::json!({"url": "https://example.com/private"}), - }], - content: None, - }, - usage: TokenUsage::default(), - }, - LlmOutput { - response: LlmResponse::Text("done".into()), - usage: TokenUsage::default(), - }, - ]); - - let store = TestStore::new(); - let mgr = ThreadManager::new( - llm, - effects.clone(), - store.clone() as Arc<dyn Store>, - Arc::new(make_caps(false)), - Arc::new(LeaseManager::new()), - Arc::new(PolicyEngine::new()), - ); - - let tid = mgr - .spawn_thread( - "call the authenticated endpoint", - ThreadType::Foreground, - project_id, - ThreadConfig::default(), - None, - "test-user", - ) - .await - .expect("spawn_thread"); - - let first = mgr.join_thread(tid).await.expect("first join"); - assert!(matches!(first, ThreadOutcome::GatePaused { .. })); - assert_eq!( - store.load_thread(tid).await.unwrap().unwrap().state, - ThreadState::Waiting - ); - - let thread = store.load_thread(tid).await.unwrap().unwrap(); - let lease = mgr - .leases - .find_lease_for_action(tid, "http") - .await - .expect("lease for http"); - let exec_ctx = ironclaw_engine::ThreadExecutionContext { - thread_id: tid, - thread_type: thread.thread_type, - project_id: thread.project_id, - user_id: "test-user".into(), - step_id: ironclaw_engine::StepId::new(), - current_call_id: Some("call_auth_1".into()), - source_channel: None, - user_timezone: None, - thread_goal: Some(thread.goal.clone()), - available_actions_snapshot: None, - available_action_inventory_snapshot: None, - }; - - effects.mark_authenticated("http").await; - let result = effects - .execute_action( - "http", - serde_json::json!({"url": "https://example.com/private"}), - &lease, - &exec_ctx, - ) - .await - .expect("authenticated pending action should execute directly"); - mgr.resume_thread( - tid, - "test-user", - Some(resumed_action_result_message( - "call_auth_1", - "http", - &result.output, - )), - None, - Some("call_auth_1".into()), - ) - .await - .expect("resume_thread"); - - let resumed = mgr.join_thread(tid).await.expect("second join"); - assert!( - matches!(resumed, ThreadOutcome::Completed { .. }), - "expected Completed after auth retry, got {resumed:?}" - ); - - let saved = store.load_thread(tid).await.unwrap().unwrap(); - let auth_pauses = saved + let auth_pauses = saved .events .iter() .filter(|event| { @@ -1442,6 +1443,15 @@ async fn auth_resolution_retries_same_pending_action_without_second_pause() { #[tokio::test] async fn approval_chains_directly_into_auth_for_install_flow() { + // Approval (inline) → tool retries → Auth gate (legacy path). + // + // `tool_install` first surfaces an `Approval` gate from the + // `EffectExecutor`, which is caught inline by the auto-approving + // controller; the retry then surfaces an `Authentication` gate, + // which is NOT caught by the controller (Auth/External keep the + // legacy re-entry path) and bubbles up as `ThreadOutcome::GatePaused`. + // From there, the test follows the legacy auth-resume flow that + // remains intact post-PR. let project_id = ProjectId::new(); let effects = GateMockEffects::new_with_chain(vec![], vec![], vec!["tool_install".into()]); let install_params = serde_json::json!({"kind": "mcp_server", "name": "notion"}); @@ -1465,14 +1475,17 @@ async fn approval_chains_directly_into_auth_for_install_flow() { ]); let store = TestStore::new(); - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm, effects.clone(), store.clone() as Arc<dyn Store>, Arc::new(make_caps(false)), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + let controller = AutoApprovingGateController::new(effects.clone()); + mgr.set_gate_controller(controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; let tid = mgr .spawn_thread( @@ -1487,8 +1500,45 @@ async fn approval_chains_directly_into_auth_for_install_flow() { .expect("spawn_thread"); let first = mgr.join_thread(tid).await.expect("first join"); - assert!(matches!(first, ThreadOutcome::GatePaused { .. })); + match &first { + ThreadOutcome::GatePaused { + gate_name, + action_name, + resume_kind, + .. + } => { + assert_eq!(gate_name, "authentication"); + assert_eq!(action_name, "tool_install"); + match resume_kind { + ResumeKind::Authentication { + credential_name, .. + } => assert_eq!(credential_name.as_str(), "notion"), + other => panic!("expected auth gate after inline approval, got {other:?}"), + } + } + other => panic!("expected auth gate after inline approval, got {other:?}"), + } + // Both gates went through the controller post-#3133-half-2. The + // first is the Approval (inline-handled by AutoApprover), the + // second is the Authentication (Cancelled by AutoApprover so the + // engine falls through to legacy `ThreadOutcome::GatePaused`). + let pauses = controller.pauses_seen().await; + assert_eq!( + pauses.len(), + 2, + "expected approval + auth pauses, got {pauses:?}" + ); + assert_eq!(pauses[0].action_name, "tool_install"); + assert!(matches!(pauses[0].resume_kind, ResumeKind::Approval { .. })); + assert_eq!(pauses[1].action_name, "tool_install"); + assert!(matches!( + pauses[1].resume_kind, + ResumeKind::Authentication { .. } + )); + + // Now drive the legacy auth-resume path (unchanged by this PR). + effects.mark_authenticated("tool_install").await; let thread = store.load_thread(tid).await.unwrap().unwrap(); let lease = mgr .leases @@ -1507,33 +1557,11 @@ async fn approval_chains_directly_into_auth_for_install_flow() { thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; - - effects.mark_approved("tool_install").await; - let auth_pause = effects - .execute_action("tool_install", install_params.clone(), &lease, &exec_ctx) - .await - .expect_err("approved install should chain directly into auth"); - match auth_pause { - EngineError::GatePaused { - gate_name, - action_name, - resume_kind, - .. - } => { - assert_eq!(gate_name, "authentication"); - assert_eq!(action_name, "tool_install"); - match *resume_kind { - ResumeKind::Authentication { - credential_name, .. - } => assert_eq!(credential_name, "notion"), - other => panic!("expected auth gate after install approval, got {other:?}"), - } - } - other => panic!("expected auth gate immediately after install approval, got {other:?}"), - } - - effects.mark_authenticated("tool_install").await; let install_result = effects .execute_action("tool_install", install_params, &lease, &exec_ctx) .await @@ -1552,21 +1580,11 @@ async fn approval_chains_directly_into_auth_for_install_flow() { .await .expect("resume after auth"); - let final_outcome = mgr.join_thread(tid).await.expect("third join"); + let final_outcome = mgr.join_thread(tid).await.expect("second join"); assert!( matches!(final_outcome, ThreadOutcome::Completed { .. }), "expected completion after auth, got {final_outcome:?}" ); - - let calls = effects.recorded_calls().await; - let install_calls = calls - .iter() - .filter(|(name, _)| name == "tool_install") - .count(); - assert_eq!( - install_calls, 3, - "install flow should retry once for approval and once for auth" - ); } #[tokio::test] @@ -1650,6 +1668,10 @@ async fn install_auth_resume_followed_by_aliased_tool_call_completes_without_han thread_goal: Some(thread.goal.clone()), available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, }; effects.mark_authenticated("tool_install").await; @@ -2395,127 +2417,461 @@ async fn auto_approve_mode_still_pauses_always_tools() { assert_eq!(ctx.execution_mode, ExecutionMode::InteractiveAutoApprove); } -/// Execution obligation fires on the resume path: a thread hits a gate, -/// is resumed with a user message that signals execution intent ("run the -/// echo tool"), and the orchestrator's per-message intent detection enables -/// the obligation nudge even though the thread config did not have -/// `require_action_attempt` set at spawn time. +// ── Inline gate-await regression (CodeAct + Tier 0 mid-execution) ─ + +/// Effects mock for the inline gate-await tests. Exposes a single +/// `github_tool` action; on the first invocation returns +/// `EngineError::GatePaused` (mid-execution gate, mirroring the +/// user's reported bug where the github_tool gates inside CodeAct); +/// after `mark_approved` is called returns a success result. +struct InlineGateGithubEffects { + calls: tokio::sync::Mutex<Vec<serde_json::Value>>, + approved: tokio::sync::Mutex<bool>, +} + +impl InlineGateGithubEffects { + fn new() -> Arc<Self> { + Arc::new(Self { + calls: tokio::sync::Mutex::new(Vec::new()), + approved: tokio::sync::Mutex::new(false), + }) + } + + async fn mark_approved(&self) { + *self.approved.lock().await = true; + } + + async fn call_count(&self) -> usize { + self.calls.lock().await.len() + } +} + +#[async_trait::async_trait] +impl EffectExecutor for InlineGateGithubEffects { + async fn execute_action( + &self, + action_name: &str, + parameters: serde_json::Value, + _lease: &CapabilityLease, + _context: &ironclaw_engine::ThreadExecutionContext, + ) -> Result<ActionResult, EngineError> { + self.calls.lock().await.push(parameters.clone()); + let approved = *self.approved.lock().await; + if action_name == "github_tool" && !approved { + return Err(EngineError::GatePaused { + gate_name: "approval".into(), + action_name: action_name.into(), + call_id: "github_tool_call".into(), + parameters: Box::new(parameters), + resume_kind: Box::new(ResumeKind::Approval { allow_always: true }), + paused_lease: None, + resume_output: None, + }); + } + // Realistic mock payload — at the time of writing, nearai/ironclaw + // has multiple open P1 issues (e.g. #2818, #2997). Returning a + // non-empty fixture keeps the script's `len(items)` assertion + // grounded in reality rather than the misleading "Found 0". + Ok(ActionResult { + call_id: String::new(), + action_name: action_name.into(), + output: serde_json::json!({ + "items": [ + {"number": 2818, "title": "[P1] mock fixture title", "html_url": "https://github.com/nearai/ironclaw/issues/2818"}, + {"number": 2997, "title": "[P1] another mock fixture", "html_url": "https://github.com/nearai/ironclaw/issues/2997"} + ] + }), + is_error: false, + duration: Duration::from_millis(1), + }) + } + + async fn available_actions( + &self, + _leases: &[CapabilityLease], + _context: &ironclaw_engine::ThreadExecutionContext, + ) -> Result<Vec<ActionDef>, EngineError> { + Ok(vec![ActionDef { + name: "github_tool".into(), + description: "GitHub interactions".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::ReadExternal], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }]) + } + + async fn available_capabilities( + &self, + _leases: &[CapabilityLease], + _context: &ironclaw_engine::ThreadExecutionContext, + ) -> Result<Vec<ironclaw_engine::CapabilitySummary>, EngineError> { + Ok(vec![]) + } +} + +/// Test `GateController` that approves the first request it sees, +/// holding a reference to the effects mock so it can mark the action +/// approved BEFORE returning the resolution. The retry happens after +/// pause() returns, so atomicity here matters — using a detached +/// `tokio::spawn` to mark approval would race the retry. +struct OneShotApprovingGateController { + requests: tokio::sync::Mutex<Vec<ironclaw_engine::GatePauseRequest>>, + effects: Arc<InlineGateGithubEffects>, +} + +impl OneShotApprovingGateController { + fn new(effects: Arc<InlineGateGithubEffects>) -> Arc<Self> { + Arc::new(Self { + requests: tokio::sync::Mutex::new(Vec::new()), + effects, + }) + } + + async fn requests_seen(&self) -> Vec<ironclaw_engine::GatePauseRequest> { + self.requests.lock().await.clone() + } +} + +#[async_trait::async_trait] +impl ironclaw_engine::GateController for OneShotApprovingGateController { + async fn pause( + &self, + request: ironclaw_engine::GatePauseRequest, + ) -> ironclaw_engine::GateResolution { + let mut requests = self.requests.lock().await; + let first = requests.is_empty(); + requests.push(request); + drop(requests); + if first { + // Mark approved synchronously (await before returning). + // The engine's retry call to execute_action happens AFTER + // this future resolves, so the approval lands first. + self.effects.mark_approved().await; + ironclaw_engine::GateResolution::Approved { always: false } + } else { + ironclaw_engine::GateResolution::Cancelled + } + } +} + +/// Live regression for the user-reported CodeAct bug: +/// +/// > "what are p1 bugs in nearai/ironclaw filed in last 7 days" — the +/// > `github_tool` call inside the CodeAct script returned mid-execution +/// > with `EngineError::GatePaused`, and the script aborted with +/// > `RuntimeError: execution paused by gate 'approval'` instead of +/// > pausing for the user. +/// +/// With the inline-await wiring (`GateController` on +/// `ThreadExecutionContext`), the Monty VM stays alive across the gate, +/// the controller observes the pause request, and on `Approved` the +/// script's `await github_tool(...)` resolves to the tool's result — +/// no re-entry, no replay, no double execution. #[tokio::test] -async fn gate_resume_with_execution_obligation() { +async fn codeact_inline_gate_await_resumes_user_reproducer() { let project_id = ProjectId::new(); - let effects = GateMockEffects::new(vec!["http".into()], vec![]); - // LLM responses for both phases: - // Phase 1 (initial): tool_call(http) → gate fires - // Phase 2 (resume): text refusal → obligation nudge → tool_call(echo) → text done - let llm = ScriptedLlm::new(vec![ - // Phase 1: triggers the http gate - LlmOutput { - response: LlmResponse::ActionCalls { - calls: vec![ironclaw_engine::ActionCall { - id: "call_http_1".into(), - action_name: "http".into(), - parameters: serde_json::json!({"url": "https://example.com"}), - }], - content: None, - }, - usage: TokenUsage::default(), - }, - // Phase 2 after resume: http succeeds (approved), then LLM returns text - // refusal. The orchestrator should detect "run the echo tool" intent from - // the injected resume message and fire the obligation nudge. - LlmOutput { - response: LlmResponse::Text("I cannot execute tools from this reply path.".into()), - usage: TokenUsage::default(), - }, - // After obligation nudge: echo tool call - LlmOutput { - response: LlmResponse::ActionCalls { - calls: vec![ironclaw_engine::ActionCall { - id: "call_echo_1".into(), - action_name: "echo".into(), - parameters: serde_json::json!({"message": "obligation resume test"}), - }], - content: None, - }, - usage: TokenUsage::default(), - }, - // Final text - LlmOutput { - response: LlmResponse::Text("Echo returned: obligation resume test".into()), - usage: TokenUsage::default(), + // Effects: github_tool gates on the first invocation, succeeds on + // the second (after the controller marks it approved). + let effects = InlineGateGithubEffects::new(); + + // CodeAct script that mirrors the user's exact reproducer. + // FINAL() materializes the result so the engine can complete. + let codeact_script = r#" +result = await github_tool(action="search_issues_pull_requests", + query="repo:nearai/ironclaw is:issue is:open label:P1", + per_page=50) +items = result.get("items", []) if isinstance(result, dict) else [] +FINAL(f"Found {len(items)} P1 bugs in nearai/ironclaw.") +"#; + + let llm = ScriptedLlm::new(vec![LlmOutput { + response: LlmResponse::Code { + code: codeact_script.to_string(), + content: None, }, - ]); + usage: TokenUsage::default(), + }]); + + // Capabilities: register github_tool. `requires_approval=false` + // means the gate fires from the EffectExecutor (mid-execution), + // not from preflight policy — exactly the user's reported shape. + let mut caps = CapabilityRegistry::new(); + caps.register(Capability { + name: "tools".into(), + description: "test tools".into(), + actions: vec![ActionDef { + name: "github_tool".into(), + description: "GitHub interactions".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::ReadExternal], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }], + knowledge: vec![], + policies: vec![], + }); let store = TestStore::new(); - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm, effects.clone(), store.clone() as Arc<dyn Store>, - Arc::new(make_caps(false)), + Arc::new(caps), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + + // Wire the inline-await controller. It marks `github_tool` as + // approved on the effects mock BEFORE returning the resolution, + // so the retry-execute returns success rather than another gate. + let controller = OneShotApprovingGateController::new(effects.clone()); + mgr.set_gate_controller(controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; - // Phase 1: spawn with NO execution intent in the goal let tid = mgr .spawn_thread( - "check the api status", + "what are p1 bugs in nearai/ironclaw filed in last 7 days", ThreadType::Foreground, project_id, - ThreadConfig::default(), // require_action_attempt = false + ThreadConfig::default(), None, "test-user", ) .await .expect("spawn_thread"); - let first = mgr.join_thread(tid).await.expect("first join"); - assert!( - matches!(first, ThreadOutcome::GatePaused { .. }), - "expected GatePaused, got: {first:?}" - ); + let outcome = mgr.join_thread(tid).await.expect("join_thread"); + + // The thread completes — does NOT come back as `GatePaused` (the + // pre-fix unwind path) or `Failed` (the pre-fix RuntimeError leak). + match &outcome { + ThreadOutcome::Completed { .. } => {} + other => panic!( + "expected Completed after inline gate await, got: {:?}", + other + ), + } + + // Controller saw exactly one pause request — the github_tool call. + let requests = controller.requests_seen().await; assert_eq!( - store.load_thread(tid).await.unwrap().unwrap().state, - ThreadState::Waiting, + requests.len(), + 1, + "controller should receive exactly one pause request, got: {requests:?}" ); + assert_eq!(requests[0].gate_name, "approval"); + assert_eq!(requests[0].action_name, "github_tool"); + assert_eq!(requests[0].user_id, "test-user"); + assert!(matches!( + requests[0].resume_kind, + ResumeKind::Approval { .. } + )); - // Phase 2: approve the gate, resume with execution intent message. - // "run the echo tool" triggers signals_execution_intent() in the Python - // orchestrator's context check on run_loop startup. - effects.mark_approved("http").await; - mgr.resume_thread( - tid, - "test-user", - Some(ThreadMessage::user( - "run the echo tool with 'obligation resume test'", - )), - Some(("call_gate_1".into(), true)), - None, - ) - .await - .expect("resume_thread"); - - let resumed = mgr.join_thread(tid).await.expect("second join"); - assert!( - matches!(resumed, ThreadOutcome::Completed { .. }), - "expected Completed, got: {resumed:?}" + // Effects saw github_tool called twice — once gated, once approved + // and succeeded. Pre-fix, the retry never happened (the script + // aborted with RuntimeError on the first call's gate). + let call_count = effects.call_count().await; + assert_eq!( + call_count, 2, + "github_tool should be called twice (gated + approved retry), got: {call_count}" ); - // Verify the echo tool was called — proves obligation worked. - // Without the per-message intent detection fix, the LLM's text refusal - // would have been accepted as the final response (no nudge), and the - // echo tool would never execute. + // The thread's events include both `ApprovalRequested` (from the + // gate firing) and `ActionExecuted` (from the approved retry). let thread = store.load_thread(tid).await.unwrap().unwrap(); - let echo_executed = thread.events.iter().any(|e| { + let approval_requested = thread.events.iter().any(|e| { + matches!( + &e.kind, + ironclaw_engine::types::event::EventKind::ApprovalRequested { action_name, .. } + if action_name == "github_tool" + ) + }); + assert!( + approval_requested, + "ApprovalRequested event must be emitted for the gated call" + ); + let action_executed = thread.events.iter().any(|e| { matches!( &e.kind, ironclaw_engine::types::event::EventKind::ActionExecuted { action_name, .. } - if action_name == "echo" + if action_name == "github_tool" ) }); assert!( - echo_executed, - "echo tool should have been called after obligation nudge on resume" + action_executed, + "ActionExecuted event must be emitted for the post-approval retry" + ); + + // Pre-fix bug message must NOT appear anywhere — script did not + // abort with a leaked RuntimeError. + let final_response = match outcome { + ThreadOutcome::Completed { response, .. } => response.unwrap_or_default(), + _ => unreachable!(), + }; + assert!( + !final_response.contains("execution paused by gate"), + "final response must not surface the pre-fix bug string; got: {final_response}" + ); + // The mock fixture returns 2 items, so the script's `len(items)` + // produces "Found 2 P1 bugs ...". (At least one of those issue + // numbers — #2818, #2997 — is actually open in nearai/ironclaw at + // the time of writing.) + assert!( + final_response.contains("Found 2 P1 bugs in nearai/ironclaw"), + "FINAL() must reflect the post-approval tool result; got: {final_response}" ); } + +/// Companion test: when the controller denies the gate, the script +/// raises a typed `RuntimeError` inside Python (catchable by +/// `try/except`), and the gated tool runs exactly once (no retry). +#[tokio::test] +async fn codeact_inline_gate_await_denial_does_not_retry() { + let project_id = ProjectId::new(); + let effects = InlineGateGithubEffects::new(); + + // Script makes the gated call without try/except — denial raises + // a `RuntimeError` that aborts the script. The engine surfaces the + // failure on the thread events (no FINAL fires). The thread itself + // completes (the LLM gets a chance to respond after the failed + // step) — what we assert is that github_tool was called exactly + // once (no retry on denial) and the failure was recorded with a + // clear "user denied" message identifying the tool. + let codeact_script = r#" +result = await github_tool(action="search") +FINAL("should not reach here") +"#; + + let llm = ScriptedLlm::new(vec![LlmOutput { + response: LlmResponse::Code { + code: codeact_script.to_string(), + content: None, + }, + usage: TokenUsage::default(), + }]); + + let mut caps = CapabilityRegistry::new(); + caps.register(Capability { + name: "tools".into(), + description: "test tools".into(), + actions: vec![ActionDef { + name: "github_tool".into(), + description: "GitHub interactions".into(), + parameters_schema: serde_json::json!({"type": "object"}), + effects: vec![EffectType::ReadExternal], + requires_approval: false, + model_tool_surface: ModelToolSurface::FullSchema, + discovery: None, + }], + knowledge: vec![], + policies: vec![], + }); + + let store = TestStore::new(); + let mgr = Arc::new(ThreadManager::new( + llm, + effects.clone(), + store.clone() as Arc<dyn Store>, + Arc::new(caps), + Arc::new(LeaseManager::new()), + Arc::new(PolicyEngine::new()), + )); + + // Denying controller — always returns Denied. + struct DenyingGateController { + requests: tokio::sync::Mutex<u32>, + } + #[async_trait::async_trait] + impl ironclaw_engine::GateController for DenyingGateController { + async fn pause( + &self, + _request: ironclaw_engine::GatePauseRequest, + ) -> ironclaw_engine::GateResolution { + *self.requests.lock().await += 1; + ironclaw_engine::GateResolution::Denied { + reason: Some("not now".into()), + } + } + } + let controller = Arc::new(DenyingGateController { + requests: tokio::sync::Mutex::new(0), + }); + mgr.set_gate_controller(controller.clone() as Arc<dyn ironclaw_engine::GateController>) + .await; + + let tid = mgr + .spawn_thread( + "denial test", + ThreadType::Foreground, + project_id, + ThreadConfig::default(), + None, + "test-user", + ) + .await + .expect("spawn_thread"); + + let outcome = mgr.join_thread(tid).await.expect("join_thread"); + + // The thread completes (the orchestrator runs the LLM again + // after CodeAct fails — ScriptedLlm has no further responses, + // so it falls through to its default "done" text). The bug-fix + // assertion isn't on the final response — it's on what the + // engine recorded mid-step: + // + // 1. Exactly one `github_tool` execution (no retry on denial). + // 2. The step failed via a typed `user denied tool 'X': reason` + // error, NOT the pre-fix `execution paused by gate 'approval'`. + // 3. FINAL() never fired. + let _ = outcome; + + // (1) github_tool called exactly once — denial does NOT retry. + let call_count = effects.call_count().await; + assert_eq!( + call_count, 1, + "denial must not retry the gated tool; got: {call_count} calls" + ); + + // (2) Look for the typed denial message on a CodeExecuted / + // CodeExecutionFailed event. Pre-fix, this would say "execution + // paused by gate 'approval'"; post-fix, it says "user denied tool + // 'github_tool': not now". + let thread = store.load_thread(tid).await.unwrap().unwrap(); + let stdout_or_error_blobs: Vec<String> = thread + .events + .iter() + .filter_map(|e| match &e.kind { + ironclaw_engine::types::event::EventKind::CodeExecuted { stdout, .. } => { + Some(stdout.clone()) + } + ironclaw_engine::types::event::EventKind::CodeExecutionFailed { error, .. } => { + Some(error.clone()) + } + ironclaw_engine::types::event::EventKind::ActionFailed { error, .. } => { + Some(error.clone()) + } + _ => None, + }) + .collect(); + let combined = stdout_or_error_blobs.join("\n"); + assert!( + combined.contains("user denied tool 'github_tool'"), + "expected typed denial message identifying the tool; got: {combined}" + ); + assert!( + combined.contains("not now"), + "expected user-supplied reason in denial message; got: {combined}" + ); + assert!( + !combined.contains("execution paused by gate"), + "denial must not surface the pre-fix bug message; got: {combined}" + ); + + // (3) Controller was invoked exactly once. + let request_count = *controller.requests.lock().await; + assert_eq!(request_count, 1); +} diff --git a/tests/engine_v2_sandbox_integration.rs b/tests/engine_v2_sandbox_integration.rs index c04fd0e83ae..cd0b0a25d6b 100644 --- a/tests/engine_v2_sandbox_integration.rs +++ b/tests/engine_v2_sandbox_integration.rs @@ -94,6 +94,10 @@ fn make_context(project_id: ProjectId) -> ThreadExecutionContext { thread_goal: None, available_actions_snapshot: None, available_action_inventory_snapshot: None, + conversation_scope: None, + gate_controller: ironclaw_engine::CancellingGateController::arc(), + call_approval_granted: false, + conversation_id: None, } } diff --git a/tests/engine_v2_skill_codeact.rs b/tests/engine_v2_skill_codeact.rs index efc85d52809..659e75afd1c 100644 --- a/tests/engine_v2_skill_codeact.rs +++ b/tests/engine_v2_skill_codeact.rs @@ -157,6 +157,35 @@ impl PausingHttpMockEffects { } } +/// Test gate controller that approves Approval gates inline by +/// marking the underlying `PausingHttpMockEffects` approved and +/// returning `Approved`. The engine's inline-retry then re-runs the +/// gated action, which now returns the canned success response. +/// +/// Replaces the legacy `mgr.join_thread()` → `ThreadOutcome::GatePaused` +/// → `resume_thread` dance for `Approval` resume kinds; the new +/// inline-await design (PR #3157) catches the gate inside the engine. +struct AutoApprovingHttpController { + effects: Arc<PausingHttpMockEffects>, +} + +impl AutoApprovingHttpController { + fn new(effects: Arc<PausingHttpMockEffects>) -> Arc<Self> { + Arc::new(Self { effects }) + } +} + +#[async_trait::async_trait] +impl ironclaw_engine::GateController for AutoApprovingHttpController { + async fn pause( + &self, + _request: ironclaw_engine::GatePauseRequest, + ) -> ironclaw_engine::GateResolution { + self.effects.mark_approved().await; + ironclaw_engine::GateResolution::Approved { always: false } + } +} + #[async_trait::async_trait] impl EffectExecutor for HttpMockEffects { async fn execute_action( @@ -866,14 +895,17 @@ async fn skill_prompt_context_survives_pause_and_resume() { policies: vec![], }); - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm.clone(), effects.clone(), store.clone() as Arc<dyn Store>, Arc::new(caps), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + let controller = AutoApprovingHttpController::new(effects.clone()); + mgr.set_gate_controller(controller as Arc<dyn ironclaw_engine::GateController>) + .await; let tid = mgr .spawn_thread( @@ -887,30 +919,20 @@ async fn skill_prompt_context_survives_pause_and_resume() { .await .expect("spawn_thread"); - let first = mgr.join_thread(tid).await.expect("first join"); + // Inline-await: the controller approves the gate, the engine + // retries the http call inline, and the thread runs to completion + // in a single `join_thread` (no resume_thread needed for Approval + // gates post-PR #3157). + let outcome = mgr.join_thread(tid).await.expect("join_thread"); assert!( - matches!(first, ThreadOutcome::GatePaused { .. }), - "unexpected first outcome: {first:?}" + matches!(outcome, ThreadOutcome::Completed { .. }), + "unexpected outcome: {outcome:?}" ); - effects.mark_approved().await; - mgr.resume_thread( - tid, - "test-user", - Some(ThreadMessage::user("approved")), - Some(("call_http_gate_1".into(), true)), - None, - ) - .await - .expect("resume_thread"); - - let resumed = mgr.join_thread(tid).await.expect("second join"); - assert!(matches!(resumed, ThreadOutcome::Completed { .. })); - let seen = llm.seen_messages.lock().unwrap(); assert!( seen.len() >= 2, - "expected at least one LLM call before and after resume" + "expected at least one LLM call before and after the inline-approval retry" ); let resumed_system_prompt = &seen.last().unwrap()[0].content; assert!(resumed_system_prompt.contains("GitHub API Skill")); @@ -963,7 +985,10 @@ async fn skill_prompt_context_survives_compaction_and_resume() { name: "slack".into(), display_name: Some("Slack".into()), kind: ironclaw_engine::CapabilitySummaryKind::Provider, - status: ironclaw_engine::CapabilityStatus::NeedsAuth, + // NeedsSetup keeps slack visible in the Activatable + // Integrations prompt section. NeedsAuth is direct-callable + // post-#3133 and lives in the regular action inventory. + status: ironclaw_engine::CapabilityStatus::NeedsSetup, description: Some("Slack workspace integration".into()), action_preview: vec!["slack_send".into()], routing_hint: None, @@ -990,14 +1015,17 @@ async fn skill_prompt_context_survives_compaction_and_resume() { policies: vec![], }); - let mgr = ThreadManager::new( + let mgr = Arc::new(ThreadManager::new( llm.clone(), effects.clone(), store.clone() as Arc<dyn Store>, Arc::new(caps), Arc::new(LeaseManager::new()), Arc::new(PolicyEngine::new()), - ); + )); + let controller = AutoApprovingHttpController::new(effects.clone()); + mgr.set_gate_controller(controller as Arc<dyn ironclaw_engine::GateController>) + .await; let tid = mgr .spawn_thread( @@ -1016,51 +1044,24 @@ async fn skill_prompt_context_survives_compaction_and_resume() { .await .expect("spawn_thread"); - let first = mgr.join_thread(tid).await.expect("first join"); - assert!( - matches!(first, ThreadOutcome::GatePaused { .. }), - "unexpected first outcome: {first:?}" - ); - - let paused_thread = store.load_thread(tid).await.unwrap().unwrap(); - assert!( - paused_thread - .internal_messages - .iter() - .any(|message| message.content == "Compaction summary text"), - "compaction summary should be persisted into the active transcript" - ); + // Inline-await: the controller approves the http gate, the engine + // retries the call inline, and the thread runs to completion in a + // single `join_thread`. Compaction still happens during the + // pre-gate run; the post-approval retry continues on the compacted + // transcript. + let outcome = mgr.join_thread(tid).await.expect("join_thread"); assert!( - paused_thread.internal_messages.iter().any(|message| message - .content - .contains("Your conversation has been compacted.")), - "compaction notice should be persisted into the active transcript" - ); - - assert_eq!( - paused_thread - .internal_messages - .iter() - .filter(|message| message.role == ironclaw_engine::types::message::MessageRole::System) - .count(), - 1, - "compacted paused transcript should preserve exactly one system message" + matches!(outcome, ThreadOutcome::Completed { .. }), + "unexpected outcome: {outcome:?}" ); - effects.mark_approved().await; - mgr.resume_thread( - tid, - "test-user", - Some(ThreadMessage::user("approved")), - Some(("call_http_gate_1".into(), true)), - None, - ) - .await - .expect("resume_thread"); - - let resumed = mgr.join_thread(tid).await.expect("second join"); - assert!(matches!(resumed, ThreadOutcome::Completed { .. })); - + // Pre-PR this asserted on the persisted transcript at the *pause + // point* (before resume). With inline-await there is no externally + // observable pause point — the thread completes after the + // controller approves. The remaining assertions on the LLM call + // sequence below are the load-bearing check: they verify the + // post-compaction system prompt and message sequence reached the + // model both pre-gate and post-approval. let seen = llm.seen_messages.lock().unwrap(); let summary_prompt = "Summarize progress so far in a concise but complete way."; let non_summary_calls: Vec<&Vec<ThreadMessage>> = seen @@ -1071,21 +1072,23 @@ async fn skill_prompt_context_survives_compaction_and_resume() { .is_some_and(|message| !message.content.contains(summary_prompt)) }) .collect(); - assert_eq!( - non_summary_calls.len(), - 2, - "expected exactly one post-compaction call before pause and one resumed call" + assert!( + non_summary_calls.len() >= 2, + "expected at least one pre-gate post-compaction call and one post-approval call" ); let post_compaction_call = non_summary_calls[0]; - let resumed_call = non_summary_calls[1]; + let resumed_call = non_summary_calls.last().unwrap(); let post_compaction_system_prompt = &post_compaction_call[0].content; assert!(post_compaction_system_prompt.contains("GitHub API Skill")); assert!(post_compaction_system_prompt.contains("Active Skills")); assert!(post_compaction_system_prompt.contains("/missing")); assert!(post_compaction_system_prompt.contains("`slack` [provider]")); - assert!(post_compaction_system_prompt.contains("tool_activate(name=\"<integration>\")")); + assert!( + post_compaction_system_prompt + .contains("need user setup before their tools become callable") + ); assert_eq!( post_compaction_system_prompt .matches("## Activatable Integrations") @@ -1113,7 +1116,7 @@ async fn skill_prompt_context_survives_compaction_and_resume() { assert!(resumed_system_prompt.contains("Active Skills")); assert!(resumed_system_prompt.contains("/missing")); assert!(resumed_system_prompt.contains("`slack` [provider]")); - assert!(resumed_system_prompt.contains("tool_activate(name=\"<integration>\")")); + assert!(resumed_system_prompt.contains("need user setup before their tools become callable")); assert_eq!( resumed_system_prompt .matches("## Activatable Integrations") diff --git a/tests/fixtures/llm_traces/bug_bash/README.md b/tests/fixtures/llm_traces/bug_bash/README.md new file mode 100644 index 00000000000..5f86fe7820c --- /dev/null +++ b/tests/fixtures/llm_traces/bug_bash/README.md @@ -0,0 +1,55 @@ +# Bug-bash regression fixtures + +Each file in this directory is a replay fixture that pins a **specific open +bug** to a deterministic snapshot. When the bug is fixed, the snapshot diff +in the PR is the reviewable proof that the fix changed the observed +behaviour. When someone reintroduces the bug, the snapshot drifts and CI +blocks the merge. + +## Recording a new fixture + +Fixtures should be **recorded from the real agent**, not hand-written. +Hand-written JSON is fine for unit-style coverage, but it can't replicate +the prompt/context shape that triggered a production bug. + +```bash +# 1. Reproduce the bug live against staging. +IRONCLAW_RECORD_TRACE=1 \ +IRONCLAW_TRACE_OUTPUT=tests/fixtures/llm_traces/bug_bash/<name>.json \ +IRONCLAW_TRACE_MODEL_NAME=bug-bash-<issue>-<slug> \ +cargo run + +# 2. Interact with the agent until you observe the bug. + +# 3. Exit — the fixture is written on shutdown. + +# 4. Add a snapshot test that replays the fixture and asserts the +# ReplayOutcome shape. Example pattern in tests/e2e_engine_v2.rs: +# snapshot_single_tool_echo (engine v2), +# or in tests/e2e_bug_bash_snapshots.rs: +# snapshot_summarization_uses_tools (engine v1). +``` + +## Coverage map + +| Issue | Fixture | Regression assertion encoded in the snapshot | +|-------|---------|-----------------------------------------------| +| [#2540](https://github.com/nearai/ironclaw/issues/2540) | `routine_timeout_regression.json` (TODO — record) | `final_state == Done`, total wall time under 300 s | +| [#2541](https://github.com/nearai/ironclaw/issues/2541) | `summarization_uses_tools.json` | at least one `echo` tool call on a "do X" prompt | +| [#2542](https://github.com/nearai/ironclaw/issues/2542) | `routine_setup_has_terminal.json` (TODO — record) | conversation ends with `Done` or `Failed`, non-empty surface | +| [#2543](https://github.com/nearai/ironclaw/issues/2543) | `linear_oauth_recognized.json` (TODO — record) | no retrospective `tool_error` with `authorization` category | +| [#2544](https://github.com/nearai/ironclaw/issues/2544) | `plan_followed_by_execution.json` (TODO — record) | at least one `ActionExecuted` event after the plan step | +| [#2545](https://github.com/nearai/ironclaw/issues/2545) | `tool_result_non_empty.json` (TODO — record) | `ToolResult` preview length bucket > 0 | +| [#2546](https://github.com/nearai/ironclaw/issues/2546) | `orchestrator_error_wrapped.json` (TODO — record) | no raw "HTTP 502" in conversation surface | + +The TODO entries are placeholders until we have a staging environment that +can reproduce the bug. Each line names the fixture file the snapshot test +expects and the specific property the snapshot will pin. + +## Why hand-written is not enough + +Hand-written fixtures encode **our mental model** of the bug. Real +recordings capture the exact LLM reasoning and tool-sequence that produced +it. When the fix lands, the recorded fixture drifts in a way the +hand-written stub wouldn't — which is exactly what we want the snapshot +gate to catch. diff --git a/tests/fixtures/llm_traces/bug_bash/summarization_uses_tools.json b/tests/fixtures/llm_traces/bug_bash/summarization_uses_tools.json new file mode 100644 index 00000000000..3ca92821b0a --- /dev/null +++ b/tests/fixtures/llm_traces/bug_bash/summarization_uses_tools.json @@ -0,0 +1,38 @@ +{ + "model_name": "bug-bash-2541-summarization-uses-tools", + "turns": [ + { + "user_input": "Echo 'status ok' and then tell me what you heard", + "steps": [ + { + "request_hint": { "last_user_message_contains": "Echo" }, + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_echo_status_1", + "name": "echo", + "arguments": { "message": "status ok" } + } + ], + "input_tokens": 120, + "output_tokens": 20 + } + }, + { + "response": { + "type": "text", + "content": "The echo tool returned: status ok. That's the message I heard.", + "input_tokens": 180, + "output_tokens": 40 + } + } + ], + "expects": { + "tools_used": ["echo"], + "all_tools_succeeded": true, + "min_responses": 1 + } + } + ] +} diff --git a/tests/fixtures/llm_traces/coverage/auth_cancelled.json b/tests/fixtures/llm_traces/coverage/auth_cancelled.json deleted file mode 100644 index 9abe8c7a081..00000000000 --- a/tests/fixtures/llm_traces/coverage/auth_cancelled.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "model_name": "coverage-auth-cancelled", - "expects": { - "min_responses": 1, - "response_contains": ["Cancelled"] - }, - "turns": [ - { - "user_input": "Set up the credential I'll cancel", - "steps": [ - { - "request_hint": { "last_user_message_contains": "cancel" }, - "response": { - "type": "tool_calls", - "tool_calls": [ - { - "id": "call_auth_cancel", - "name": "tool_activate", - "arguments": { "name": "test_credential" } - } - ], - "input_tokens": 60, - "output_tokens": 20 - } - } - ] - } - ] -} diff --git a/tests/fixtures/llm_traces/coverage/auth_credential_provided.json b/tests/fixtures/llm_traces/coverage/auth_credential_provided.json deleted file mode 100644 index e1439b2e847..00000000000 --- a/tests/fixtures/llm_traces/coverage/auth_credential_provided.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "model_name": "coverage-auth-credential-provided", - "expects": { - "min_responses": 1, - "response_contains": ["configured"] - }, - "turns": [ - { - "user_input": "Set up the test credential", - "steps": [ - { - "request_hint": { "last_user_message_contains": "test credential" }, - "response": { - "type": "tool_calls", - "tool_calls": [ - { - "id": "call_auth_ok", - "name": "tool_activate", - "arguments": { "name": "test_credential" } - } - ], - "input_tokens": 60, - "output_tokens": 20 - } - }, - { - "response": { - "type": "text", - "content": "Credential 'test_credential' is now configured.", - "input_tokens": 90, - "output_tokens": 15 - } - } - ] - } - ] -} diff --git a/tests/fixtures/llm_traces/coverage/auth_external_callback.json b/tests/fixtures/llm_traces/coverage/auth_external_callback.json deleted file mode 100644 index eec333d60c0..00000000000 --- a/tests/fixtures/llm_traces/coverage/auth_external_callback.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "model_name": "coverage-auth-external-callback", - "expects": { - "min_responses": 1, - "response_contains": ["connected"] - }, - "turns": [ - { - "user_input": "Connect the OAuth service", - "steps": [ - { - "request_hint": { "last_user_message_contains": "OAuth" }, - "response": { - "type": "tool_calls", - "tool_calls": [ - { - "id": "call_oauth_callback", - "name": "tool_activate", - "arguments": { "name": "oauth_service" } - } - ], - "input_tokens": 60, - "output_tokens": 20 - } - }, - { - "response": { - "type": "text", - "content": "OAuth service is now connected.", - "input_tokens": 90, - "output_tokens": 15 - } - } - ] - } - ] -} diff --git a/tests/fixtures/llm_traces/coverage/auth_gate_request_id.json b/tests/fixtures/llm_traces/coverage/auth_gate_request_id.json deleted file mode 100644 index dc0ce5e44e4..00000000000 --- a/tests/fixtures/llm_traces/coverage/auth_gate_request_id.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "model_name": "coverage-auth-gate-request-id", - "expects": { - "tools_used": ["tool_activate"], - "min_responses": 0 - }, - "turns": [ - { - "user_input": "Trigger the auth gate", - "steps": [ - { - "request_hint": { "last_user_message_contains": "auth gate" }, - "response": { - "type": "tool_calls", - "tool_calls": [ - { - "id": "call_auth_requestid", - "name": "tool_activate", - "arguments": { "name": "probe_credential" } - } - ], - "input_tokens": 60, - "output_tokens": 20 - } - } - ] - } - ] -} diff --git a/tests/fixtures/llm_traces/coverage/auth_retry_invalid_then_valid.json b/tests/fixtures/llm_traces/coverage/auth_retry_invalid_then_valid.json deleted file mode 100644 index 5fb2fd6170e..00000000000 --- a/tests/fixtures/llm_traces/coverage/auth_retry_invalid_then_valid.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "model_name": "coverage-auth-retry", - "expects": { - "min_responses": 1, - "response_contains": ["configured"] - }, - "turns": [ - { - "user_input": "Set up the credential", - "steps": [ - { - "request_hint": { "last_user_message_contains": "credential" }, - "response": { - "type": "tool_calls", - "tool_calls": [ - { - "id": "call_auth_retry", - "name": "tool_activate", - "arguments": { "name": "test_credential" } - } - ], - "input_tokens": 60, - "output_tokens": 20 - } - }, - { - "response": { - "type": "text", - "content": "Credential 'test_credential' is now configured.", - "input_tokens": 90, - "output_tokens": 15 - } - } - ] - } - ] -} diff --git a/tests/fixtures/llm_traces/engine_v2/external_tool_round_trip.json b/tests/fixtures/llm_traces/engine_v2/external_tool_round_trip.json new file mode 100644 index 00000000000..b2c2ef44cd5 --- /dev/null +++ b/tests/fixtures/llm_traces/engine_v2/external_tool_round_trip.json @@ -0,0 +1,42 @@ +{ + "model_name": "v2-external-tool-roundtrip", + "turns": [ + { + "user_input": "Look up the weather in NYC.", + "steps": [ + { + "request_hint": { + "last_user_message_contains": "weather" + }, + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_ext_lookup_1", + "name": "lookup_weather", + "arguments": { "city": "NYC" } + } + ], + "input_tokens": 100, + "output_tokens": 30 + } + }, + { + "request_hint": { + "min_messages": 3 + }, + "response": { + "type": "text", + "content": "Per the lookup, the weather in NYC is sunny and 72F.", + "input_tokens": 150, + "output_tokens": 25 + } + } + ], + "expects": { + "response_contains": ["sunny", "72F"], + "min_responses": 1 + } + } + ] +} diff --git a/tests/fixtures/llm_traces/tools/job_list_cancel.json b/tests/fixtures/llm_traces/tools/job_list_cancel.json index 7b3ae55d213..0deeacebfea 100644 --- a/tests/fixtures/llm_traces/tools/job_list_cancel.json +++ b/tests/fixtures/llm_traces/tools/job_list_cancel.json @@ -5,8 +5,10 @@ "all_tools_succeeded": true, "min_responses": 1 }, + "_comment": "Steps 1-4 are pinned to the parent chat turn via the hint \"list jobs, then cancel\". The agent's create_job spawns a real background sub-job that runs its own LoopDelegate against the same TraceLlm queue; without hint scoping, sub-job + parent race for steps (depending on scheduler/await timing) and the parent can lose its third step. Step 5 catches the sub-job's first LLM call via the \"Continue.\" hint (the sub-job worker auto-injects a `Continue.` prompt as its first user turn) and returns immediate text, so the sub-job exits its loop without dipping back into parent's queue. Reference: ironclaw#3641 review thread.", "steps": [ { + "request_hint": { "last_user_message_contains": "list jobs, then cancel" }, "response": { "type": "tool_calls", "tool_calls": [ @@ -24,6 +26,7 @@ } }, { + "request_hint": { "last_user_message_contains": "list jobs, then cancel" }, "response": { "type": "tool_calls", "tool_calls": [ @@ -38,6 +41,7 @@ } }, { + "request_hint": { "last_user_message_contains": "list jobs, then cancel" }, "response": { "type": "tool_calls", "tool_calls": [ @@ -52,12 +56,58 @@ } }, { + "request_hint": { "last_user_message_contains": "list jobs, then cancel" }, "response": { "type": "text", "content": "Created a job, verified it appeared in the list, then cancelled it successfully.", "input_tokens": 400, "output_tokens": 20 } + }, + { + "request_hint": { "last_user_message_contains": "Continue." }, + "response": { + "type": "text", + "content": "Sub-job acknowledged; nothing to do.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "Continue." }, + "response": { + "type": "text", + "content": "Sub-job idle.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "Continue." }, + "response": { + "type": "text", + "content": "Sub-job idle.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "Continue." }, + "response": { + "type": "text", + "content": "Sub-job idle.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "Continue." }, + "response": { + "type": "text", + "content": "Sub-job idle.", + "input_tokens": 50, + "output_tokens": 10 + } } ] } diff --git a/tests/gemini_oauth_regression.rs b/tests/gemini_oauth_regression.rs deleted file mode 100644 index d1b40f71236..00000000000 --- a/tests/gemini_oauth_regression.rs +++ /dev/null @@ -1,99 +0,0 @@ -use ironclaw::llm::ChatMessage; -use ironclaw::llm::gemini_oauth::GeminiOauthProvider; - -/// Regression: Cloud Code API routing for Gemini 2.0+ models. -/// Gemini 1.x → legacy generativelanguage.googleapis.com -/// Gemini 2.0+ → Cloud Code API (cloudcode-pa.googleapis.com) -#[test] -fn test_regression_cloud_code_api_routing() { - // Legacy models (1.x) → false - assert!(!GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-1.5-pro" - )); - assert!(!GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-1.5-flash" - )); - - // 2.0+ models → true - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-2.0-flash" - )); - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-2.5-pro" - )); - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-2.5-flash" - )); - - // Preview models with hyphen → true - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-3.1-pro-preview" - )); - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-3-flash-preview" - )); - - // Gemini 3 family → true - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "gemini-3-pro" - )); -} - -/// Regression: "preview" false-positive fix. -/// `model.contains("-preview")` (with hyphen) prevents models whose name -/// happens to include "preview" without a hyphen prefix from being -/// mis-routed to Cloud Code API. -#[test] -fn test_regression_preview_false_positive_fix() { - // "my-preview-custom" still matches (contains "-preview") - assert!(GeminiOauthProvider::model_uses_cloud_code_api( - "my-preview-custom" - )); - - // "mypreviewcustom" does NOT match (no hyphen before "preview") - assert!(!GeminiOauthProvider::model_uses_cloud_code_api( - "mypreviewcustom" - )); - - // Non-Gemini models without "-preview" → false - assert!(!GeminiOauthProvider::model_uses_cloud_code_api( - "not-a-gemini-model" - )); -} - -/// Regression: model list consistency. -/// Wizard, list_models(), and LLM_PROVIDERS.md all return the same 8 models. -#[test] -fn test_regression_standardized_model_list() { - let expected_models = [ - "gemini-3.1-pro-preview", - "gemini-3.1-pro-preview-customtools", - "gemini-3-pro-preview", - "gemini-3-flash-preview", - "gemini-3.1-flash-lite-preview", - "gemini-2.5-pro", - "gemini-2.5-flash", - "gemini-2.5-flash-lite", - ]; - - // All standardized models must route to Cloud Code API (all are >= 2.0) - for model in &expected_models { - assert!( - GeminiOauthProvider::model_uses_cloud_code_api(model), - "Standardized model '{}' should route to Cloud Code API", - model - ); - } -} - -/// Regression: ChatMessage helper constructors. -#[test] -fn test_regression_chat_message_helpers() { - let user_msg = ChatMessage::user("hello"); - assert_eq!(user_msg.role, ironclaw::llm::Role::User); - assert_eq!(user_msg.content, "hello"); - - let system_msg = ChatMessage::system("you are helpful"); - assert_eq!(system_msg.role, ironclaw::llm::Role::System); - assert_eq!(system_msg.content, "you are helpful"); -} diff --git a/tests/heartbeat_integration.rs b/tests/heartbeat_integration.rs index eb06a8f94f1..4ac8a340d5b 100644 --- a/tests/heartbeat_integration.rs +++ b/tests/heartbeat_integration.rs @@ -10,13 +10,8 @@ use std::sync::Arc; -use ironclaw::{ - agent::HeartbeatRunner, - config::Config, - history::Store, - llm::{create_llm_provider, create_session_manager}, - workspace::Workspace, -}; +use ironclaw::{agent::HeartbeatRunner, config::Config, history::Store, workspace::Workspace}; +use ironclaw_llm::{create_llm_provider, create_session_manager}; #[tokio::test] #[ignore] // Requires running database and LLM credentials diff --git a/tests/multi_tenant_system_prompt.rs b/tests/multi_tenant_system_prompt.rs index 0f6aa697141..a4919a54828 100644 --- a/tests/multi_tenant_system_prompt.rs +++ b/tests/multi_tenant_system_prompt.rs @@ -24,8 +24,8 @@ mod tests { use std::time::Duration; use ironclaw::channels::IncomingMessage; - use ironclaw::llm::Role; use ironclaw::workspace::Workspace; + use ironclaw_llm::Role; use crate::support::test_rig::TestRigBuilder; use crate::support::trace_llm::{LlmTrace, TraceResponse, TraceStep}; @@ -82,7 +82,7 @@ mod tests { /// /// The system prompt is the first message with role=System in the first /// LLM request for a given turn. - fn extract_system_prompt(requests: &[Vec<ironclaw::llm::ChatMessage>]) -> Option<String> { + fn extract_system_prompt(requests: &[Vec<ironclaw_llm::ChatMessage>]) -> Option<String> { requests.last().and_then(|msgs| { msgs.iter() .find(|m| matches!(m.role, Role::System)) diff --git a/tests/openai_compat_integration.rs b/tests/openai_compat_integration.rs index 69f91b0fc32..e2ca3613792 100644 --- a/tests/openai_compat_integration.rs +++ b/tests/openai_compat_integration.rs @@ -15,7 +15,7 @@ use ironclaw::channels::web::platform::state::GatewayState; use ironclaw::channels::web::sse::SseManager; use ironclaw::channels::web::ws::WsConnectionTracker; use ironclaw::error::LlmError; -use ironclaw::llm::{ +use ironclaw_llm::{ CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest, ToolCompletionResponse, }; @@ -64,7 +64,7 @@ impl LlmProvider for MockLlmProvider { .messages .iter() .rev() - .find(|m| m.role == ironclaw::llm::Role::User) + .find(|m| m.role == ironclaw_llm::Role::User) .map(|m| m.content.clone()) .unwrap_or_else(|| "no user message".to_string()); @@ -92,17 +92,19 @@ impl LlmProvider for MockLlmProvider { if let Some(tool) = req.tools.first() { Ok(ToolCompletionResponse { content: None, - tool_calls: vec![ironclaw::llm::ToolCall { + tool_calls: vec![ironclaw_llm::ToolCall { id: "call_mock_001".to_string(), name: tool.name.clone(), arguments: serde_json::json!({"test": true}), reasoning: None, + signature: None, }], input_tokens: 15, output_tokens: 8, finish_reason: FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } else { Ok(ToolCompletionResponse { @@ -113,6 +115,7 @@ impl LlmProvider for MockLlmProvider { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -168,6 +171,7 @@ impl LlmProvider for FixedModelProvider { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } diff --git a/tests/provider_chaos.rs b/tests/provider_chaos.rs index b6fa4c4136b..5d748763a11 100644 --- a/tests/provider_chaos.rs +++ b/tests/provider_chaos.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use rust_decimal::Decimal; use ironclaw::error::LlmError; -use ironclaw::llm::{ +use ironclaw_llm::{ ChatMessage, CircuitBreakerConfig, CircuitBreakerProvider, CompletionRequest, CompletionResponse, CooldownConfig, FailoverProvider, FinishReason, LlmProvider, RetryConfig, RetryProvider, ToolCompletionRequest, ToolCompletionResponse, @@ -119,6 +119,7 @@ impl LlmProvider for FlakeyProvider { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -214,6 +215,7 @@ impl LlmProvider for GarbageProvider { finish_reason: FinishReason::Unknown, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -274,6 +276,7 @@ impl LlmProvider for ReliableProvider { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } } @@ -466,7 +469,7 @@ async fn test_circuit_breaker_trips_and_recovers() { let state = cb.circuit_state().await; assert_eq!( state, - ironclaw::llm::circuit_breaker::CircuitState::Open, + ironclaw_llm::circuit_breaker::CircuitState::Open, "circuit should be open after 3 failures" ); @@ -491,7 +494,7 @@ async fn test_circuit_breaker_trips_and_recovers() { let _ = cb.complete(make_request()).await; assert_eq!( cb.circuit_state().await, - ironclaw::llm::circuit_breaker::CircuitState::Open, + ironclaw_llm::circuit_breaker::CircuitState::Open, "probe failed, should reopen" ); @@ -502,7 +505,7 @@ async fn test_circuit_breaker_trips_and_recovers() { let _ = cb.complete(make_request()).await; assert_eq!( cb.circuit_state().await, - ironclaw::llm::circuit_breaker::CircuitState::Open, + ironclaw_llm::circuit_breaker::CircuitState::Open, "still one failure left, should reopen again" ); @@ -515,7 +518,7 @@ async fn test_circuit_breaker_trips_and_recovers() { assert_eq!(result.unwrap().content, "recovered"); assert_eq!( cb.circuit_state().await, - ironclaw::llm::circuit_breaker::CircuitState::Closed, + ironclaw_llm::circuit_breaker::CircuitState::Closed, "circuit should close after successful probe" ); } @@ -713,7 +716,7 @@ async fn test_retry_plus_circuit_breaker_integration() { assert_eq!(response.content, "stack success"); assert_eq!( cb.circuit_state().await, - ironclaw::llm::circuit_breaker::CircuitState::Closed, + ironclaw_llm::circuit_breaker::CircuitState::Closed, "circuit should remain closed" ); } @@ -784,7 +787,7 @@ async fn test_garbage_through_full_chain() { ); assert_eq!( cb.circuit_state().await, - ironclaw::llm::circuit_breaker::CircuitState::Closed, + ironclaw_llm::circuit_breaker::CircuitState::Closed, "Ok responses should not trip the breaker" ); } diff --git a/tests/responses_api_path_prefix.rs b/tests/responses_api_path_prefix.rs index 7abbc8ba948..8ade557315c 100644 --- a/tests/responses_api_path_prefix.rs +++ b/tests/responses_api_path_prefix.rs @@ -193,6 +193,307 @@ async fn both_paths_require_auth() { } } +/// `tools: [{type: "function", ...}]` is the externally-provided-tools +/// surface. POST handler must accept the field instead of rejecting with +/// 400 — the agent's reply is delivered asynchronously, but the request +/// validation has to clear. We use an obviously bad tool definition +/// (missing `name`) to assert the dedicated 400 path: this proves both +/// "the tools field is parsed" and "validation kicks in". +#[tokio::test] +async fn missing_tool_name_returns_validation_error() { + let (addr, _state, _guard) = start_test_server().await; + let url = format!("http://{}/api/v1/responses", addr); + + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hi", + "tools": [ + {"type": "function", "description": "nameless"} + ] + })) + .send() + .await + .expect("POST /api/v1/responses with malformed tool"); + + assert_eq!( + resp.status(), + 400, + "expected 400 from external-tool validator, got {}", + resp.status() + ); + let body = resp.text().await.unwrap_or_default(); + assert!( + body.contains("name"), + "validation error should mention the missing 'name' field, got: {body}" + ); +} + +/// Unsupported tool types (e.g. `web_search`) must be rejected by the +/// validator with a clear 400 — not silently accepted, since the engine +/// doesn't honour them. +#[tokio::test] +async fn unsupported_tool_type_returns_validation_error() { + let (addr, _state, _guard) = start_test_server().await; + let url = format!("http://{}/api/v1/responses", addr); + + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hi", + "tools": [ + {"type": "web_search", "name": "search"} + ] + })) + .send() + .await + .expect("POST /api/v1/responses with unsupported tool type"); + + assert_eq!(resp.status(), 400); + let body = resp.text().await.unwrap_or_default(); + assert!( + body.contains("web_search"), + "validation error should mention the unsupported tool type, got: {body}" + ); +} + +/// `instructions` is a per-request system/developer message (OpenAI Responses +/// API spec). The handler used to reject it with 400; it must now accept it +/// and route the request into the agent loop. We assert the request clears +/// the synchronous validation gate by asking for a malformed `model` so the +/// handler short-circuits with a 400 whose message is about `model`, not +/// about `instructions`. A 400 mentioning `instructions` would mean the +/// rejection regressed. +#[tokio::test] +async fn instructions_field_is_accepted() { + let (addr, _state, _guard) = start_test_server().await; + let url = format!("http://{}/api/v1/responses", addr); + + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "not-a-real-model", + "input": "hi", + "instructions": "You are a terse assistant. Always reply in one sentence.", + })) + .send() + .await + .expect("POST /api/v1/responses with instructions"); + + assert_eq!(resp.status(), 400); + let body = resp.text().await.unwrap_or_default(); + assert!( + !body.contains("instructions"), + "instructions must no longer be rejected, got: {body}" + ); + assert!( + body.contains("Model selection"), + "expected the model rejection to be the reason for 400, got: {body}" + ); +} + +/// External tools require engine v2: when the engine is not initialized +/// the handler must reject the request with a clear 4xx instead of +/// silently degrading. The path-prefix gateway boots without engine v2 +/// (no `init_engine` call, so the `ExternalToolCatalog` is absent), and +/// the handler keys off catalog presence rather than reading env vars +/// directly — so this test exercises the no-engine branch by virtue of +/// what `TestGatewayBuilder` actually wires up. No process-global env +/// mutation required. +#[tokio::test] +async fn external_tools_rejected_when_engine_v2_disabled() { + let (addr, _state, _guard) = start_test_server().await; + let url = format!("http://{}/api/v1/responses", addr); + + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hello", + "tools": [ + {"type": "function", "name": "lookup", "parameters": {"type": "object"}} + ] + })) + .send() + .await + .expect("POST /api/v1/responses with tools and engine v2 unavailable"); + + assert_eq!( + resp.status(), + 400, + "expected 400 when engine v2 is unavailable, got {}", + resp.status() + ); + let body = resp.text().await.unwrap_or_default(); + assert!( + body.to_ascii_lowercase().contains("engine v2"), + "rejection should mention engine v2, got: {body}" + ); +} + +/// `function_call_output` items are a resume signal: they must be +/// matched against a pending external-tool gate for the resolved +/// thread. Without one (e.g. because the caller fabricates a +/// `previous_response_id` or the gate already expired), the handler +/// must reject with 400 instead of silently sending the resume into a +/// fresh thread. +#[tokio::test] +async fn resume_without_pending_gate_returns_400() { + let (addr, _state, _guard) = start_test_server().await; + let url = format!("http://{}/api/v1/responses", addr); + + // Synthesize a wire-valid previous_response_id (resp_<32hex><32hex>) + // that names a thread the gateway has never seen. The handler + // accepts the format, looks for a pending gate, finds none, and + // must respond 400 — not silently drop the function_call_output + // and start a fresh turn against the thread. + let fake_prev = format!("resp_{}{}", "0".repeat(32), "1".repeat(32)); + + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "previous_response_id": fake_prev, + "input": [ + { + "type": "function_call_output", + "call_id": "call_made_up", + "output": "irrelevant" + } + ] + })) + .send() + .await + .expect("POST /api/v1/responses resume w/o pending gate"); + + assert_eq!( + resp.status(), + 400, + "expected 400 for resume without pending gate, got {}", + resp.status() + ); + let body = resp.text().await.unwrap_or_default(); + assert!( + body.contains("pending"), + "rejection should mention the missing pending gate, got: {body}" + ); +} + +/// A caller-supplied tool name that shadows a registered (built-in +/// or extension) action must be rejected at request validation with +/// 400. Without this check, the catalog short-circuit in +/// `EffectBridgeAdapter::execute_action` would silently route the +/// LLM's call to caller-side execution — even though the LLM saw +/// the *internal* tool's description in its action surface, since +/// `available_action_inventory` dedupes the opposite way (internal +/// wins). That's a confused-deputy surface where the caller can +/// craft any output and the LLM treats it as the trusted internal +/// tool's reply. +#[tokio::test] +async fn external_tool_name_shadowing_registered_action_is_rejected() { + use std::sync::Arc; + + use async_trait::async_trait; + use ironclaw::context::JobContext; + use ironclaw::tools::{Tool, ToolError, ToolOutput, ToolRegistry}; + + /// Stand-in for a registered built-in. Production tools have + /// the same name shape (`shell`, `memory_write`, etc.); this + /// tool deliberately uses a unique name so the test isn't + /// affected by which built-ins the harness happens to register. + struct StandInTool; + + #[async_trait] + impl Tool for StandInTool { + fn name(&self) -> &str { + "stand_in_tool" + } + fn description(&self) -> &str { + "stand-in for a registered tool — used for collision tests" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object"}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &JobContext, + ) -> Result<ToolOutput, ToolError> { + Ok(ToolOutput::success( + serde_json::json!({"ok": true}), + std::time::Duration::from_millis(0), + )) + } + } + + // Spin up a gateway with a registry that contains a known tool + // name, so the validation check has something to collide with. + let registry = Arc::new(ToolRegistry::new()); + registry.register(Arc::new(StandInTool)).await; + + let state = ironclaw::channels::web::test_helpers::TestGatewayBuilder::new() + .user_id("test-user") + .tool_registry(registry) + .build(); + let auth = ironclaw::channels::web::auth::MultiAuthState::single( + AUTH_TOKEN.to_string(), + "test-user".to_string(), + ); + let addr: SocketAddr = "127.0.0.1:0" + .parse() + .expect("hard-coded address must parse"); + let bound = + ironclaw::channels::web::platform::router::start_server(addr, state.clone(), auth.into()) + .await + .expect("start gateway test server"); + let shutdown = state.shutdown_tx.write().await.take(); + let _guard = ServerGuard { shutdown }; + + let url = format!("http://{}/api/v1/responses", bound); + let resp = client() + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hi", + "tools": [ + { + "type": "function", + "name": "stand_in_tool", + "description": "shadow attempt", + "parameters": {"type": "object"} + } + ] + })) + .send() + .await + .expect("POST /api/v1/responses with shadowing tool name"); + + assert_eq!( + resp.status(), + 400, + "shadowing tool name must be rejected, got {}", + resp.status() + ); + let body = resp.text().await.unwrap_or_default(); + assert!( + body.contains("stand_in_tool"), + "rejection should name the colliding tool, got: {body}" + ); + assert!( + body.to_ascii_lowercase().contains("shadow") + || body.to_ascii_lowercase().contains("built-in"), + "rejection should explain why (shadow / built-in), got: {body}" + ); +} + /// Both GET item paths (`/api/v1/responses/{id}` and `/v1/responses/{id}`) /// must also enforce bearer-token auth. A missing token should return 401, /// not 404 — the auth middleware has to apply to legacy aliases as well. diff --git a/tests/responses_api_temperature.rs b/tests/responses_api_temperature.rs new file mode 100644 index 00000000000..02a4b149299 --- /dev/null +++ b/tests/responses_api_temperature.rs @@ -0,0 +1,215 @@ +//! Caller-level regression tests for per-request `temperature` on the +//! Responses API (PR #3641, serrrfirat's Medium-severity follow-up). +//! +//! The handler used to reject any `temperature` field with 400; PR #3641 +//! removed that rejection and instead stamps the value into the outgoing +//! `IncomingMessage.metadata` so the agent dispatcher can apply it as a +//! per-request override before consulting user/admin settings. +//! +//! Per `.claude/rules/testing.md` ("Test Through the Caller, Not Just the +//! Helper"), exercising only the dispatcher's `resolve_settings_temperature` +//! helper is not enough — the endpoint→metadata wiring sits between the +//! POST body and the helper, and a future refactor that drops the field +//! on the floor would not break a helper-level test. These tests drive the +//! full router with a captured `msg_tx` and assert that the +//! `IncomingMessage` the agent loop would receive carries the value. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use ironclaw::channels::IncomingMessage; +use ironclaw::channels::web::auth::MultiAuthState; +use ironclaw::channels::web::platform::router::start_server; +use ironclaw::channels::web::platform::state::GatewayState; +use ironclaw::channels::web::test_helpers::TestGatewayBuilder; +use tokio::sync::{mpsc, oneshot}; + +const AUTH_TOKEN: &str = "test-responses-api-temperature-token"; +const USER_ID: &str = "test-user"; + +/// RAII guard that shuts the gateway test server down when dropped. +struct ServerGuard { + shutdown: Option<oneshot::Sender<()>>, +} + +impl Drop for ServerGuard { + fn drop(&mut self) { + if let Some(tx) = self.shutdown.take() { + let _ = tx.send(()); + } + } +} + +async fn start_test_server_with_capture() -> ( + SocketAddr, + Arc<GatewayState>, + mpsc::Receiver<IncomingMessage>, + ServerGuard, +) { + let (tx, rx) = mpsc::channel::<IncomingMessage>(8); + let state = TestGatewayBuilder::new() + .user_id(USER_ID) + .msg_tx(tx) + .build(); + let auth = MultiAuthState::single(AUTH_TOKEN.to_string(), USER_ID.to_string()); + let addr: SocketAddr = "127.0.0.1:0" + .parse() + .expect("hard-coded address must parse"); + let bound = start_server(addr, state.clone(), auth.into()) + .await + .expect("start gateway test server"); + let shutdown = state.shutdown_tx.write().await.take(); + (bound, state, rx, ServerGuard { shutdown }) +} + +fn client() -> reqwest::Client { + // Short timeout — the handler waits for SSE events that never arrive + // in this test fixture, so the HTTP call always times out from the + // client side. We only care about the IncomingMessage that lands on + // `rx` the moment the handler calls `send_to_agent`, which happens + // long before the SSE wait. The handler task is torn down when the + // gateway server is dropped at the end of the test. + reqwest::Client::builder() + .timeout(Duration::from_millis(500)) + .build() + .expect("build test http client") +} + +/// POST `/v1/responses` with `temperature` set must land an +/// `IncomingMessage` on the agent channel whose `metadata["temperature"]` +/// matches the request body. Regression: pre-#3641 the handler 400'd; if +/// a future change drops the `metadata["temperature"]` write, the agent +/// dispatcher's per-request override would never see it and the +/// `resolve_settings_temperature` helper test alone would not notice. +#[tokio::test] +async fn responses_request_temperature_lands_in_incoming_metadata() { + let (addr, _state, mut rx, _guard) = start_test_server_with_capture().await; + let url = format!("http://{}/v1/responses", addr); + + let http = client(); + let request = async move { + // The handler will block waiting for SSE events that never come; + // we don't care about the response, only the captured message. + let _ = http + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hello", + "temperature": 0.42, + })) + .send() + .await; + }; + let captured = async { + tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("agent channel must receive a message within 2s") + .expect("agent channel must not be closed") + }; + + let (_, msg) = tokio::join!(request, captured); + + let metadata = &msg.metadata; + let t = metadata + .get("temperature") + .unwrap_or_else(|| panic!("metadata missing 'temperature': {metadata}")); + let t = t + .as_f64() + .unwrap_or_else(|| panic!("metadata 'temperature' not a number: {t}")); + assert!( + (t - 0.42).abs() < 1e-6, + "expected metadata['temperature']=0.42, got {t}" + ); +} + +/// POST `/v1/responses` with `temperature` outside the OpenAI-compatible +/// `[0, 2]` range must reject with a 400 `invalid_request_error` at the +/// API boundary and must NOT enqueue an `IncomingMessage` on the agent +/// channel. The provider-side `Reasoning::respond_with_tools` path +/// clamps later, but callers expect the request boundary to fail loudly +/// rather than silently turn `temperature: 9.0` into `2.0`. +#[tokio::test] +async fn responses_request_temperature_out_of_range_rejects_and_does_not_enqueue() { + let (addr, _state, mut rx, _guard) = start_test_server_with_capture().await; + let url = format!("http://{}/v1/responses", addr); + let http = client(); + + for bad_temperature in [-0.5_f32, 2.5_f32] { + let resp = http + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hello", + "temperature": bad_temperature, + })) + .send() + .await + .expect("send /v1/responses request"); + assert_eq!( + resp.status().as_u16(), + 400, + "temperature {bad_temperature} must be rejected with 400", + ); + let body: serde_json::Value = resp.json().await.expect("parse JSON error body"); + let kind = body + .get("error") + .and_then(|e| e.get("type")) + .and_then(|t| t.as_str()) + .unwrap_or(""); + assert_eq!( + kind, "invalid_request_error", + "error.type for bad temperature should be invalid_request_error, body={body}", + ); + } + + // No `IncomingMessage` may have been enqueued by either rejected request. + match tokio::time::timeout(Duration::from_millis(200), rx.recv()).await { + Ok(Some(msg)) => panic!( + "no IncomingMessage should be enqueued for rejected temperatures, got: {:?}", + msg.metadata + ), + Ok(None) => panic!("agent channel must not be closed"), + Err(_) => {} // timeout = nothing enqueued, expected + } +} + +/// POST `/v1/responses` *without* a `temperature` field must not +/// fabricate one in metadata. The dispatcher uses +/// `metadata.get("temperature").is_some()` as the per-request signal — +/// an unconditional default here would override every user's settings +/// value silently. +#[tokio::test] +async fn responses_request_without_temperature_omits_metadata_field() { + let (addr, _state, mut rx, _guard) = start_test_server_with_capture().await; + let url = format!("http://{}/v1/responses", addr); + + let http = client(); + let request = async move { + let _ = http + .post(&url) + .bearer_auth(AUTH_TOKEN) + .json(&serde_json::json!({ + "model": "default", + "input": "hello", + })) + .send() + .await; + }; + let captured = async { + tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("agent channel must receive a message within 2s") + .expect("agent channel must not be closed") + }; + + let (_, msg) = tokio::join!(request, captured); + assert!( + msg.metadata.get("temperature").is_none(), + "metadata must not carry a fabricated temperature when the request \ + body has none — got {}", + msg.metadata + ); +} diff --git a/tests/skill_chain_load_lifecycle.rs b/tests/skill_chain_load_lifecycle.rs index fdac792eb45..3ed3825ae10 100644 --- a/tests/skill_chain_load_lifecycle.rs +++ b/tests/skill_chain_load_lifecycle.rs @@ -138,10 +138,7 @@ absent, it wasn't. /// Count occurrences of `needle` across every captured LLM /// request's messages (system + user + assistant). A positive /// count means the string was injected into at least one prompt. - fn occurrences_in_requests( - requests: &[Vec<ironclaw::llm::ChatMessage>], - needle: &str, - ) -> usize { + fn occurrences_in_requests(requests: &[Vec<ironclaw_llm::ChatMessage>], needle: &str) -> usize { let mut n = 0; for req in requests { for msg in req { diff --git a/tests/skill_setup_marker_lifecycle.rs b/tests/skill_setup_marker_lifecycle.rs index 92db2d1fdd9..3838ba69420 100644 --- a/tests/skill_setup_marker_lifecycle.rs +++ b/tests/skill_setup_marker_lifecycle.rs @@ -97,7 +97,7 @@ is selected. /// Each selected skill injects its body into the system prompt, /// so presence of the marker string means "the skill was /// selected for at least one turn". - fn marker_occurrences(requests: &[Vec<ironclaw::llm::ChatMessage>]) -> usize { + fn marker_occurrences(requests: &[Vec<ironclaw_llm::ChatMessage>]) -> usize { let mut count = 0; for request in requests { for msg in request { diff --git a/tests/snapshots/replay__bug_bash_summarization_uses_tools.snap b/tests/snapshots/replay__bug_bash_summarization_uses_tools.snap new file mode 100644 index 00000000000..2fafe5c21fc --- /dev/null +++ b/tests/snapshots/replay__bug_bash_summarization_uses_tools.snap @@ -0,0 +1,47 @@ +--- +source: tests/e2e_bug_bash_snapshots.rs +assertion_line: 44 +--- +response_count: 1 +has_final_response: true +tool_calls: + - name: echo + success: true +events: + - kind: thinking + message: processing... + - kind: thinking + message: thinking (step 1)... + - kind: other + variant: Other + - kind: thinking + message: running echo... + - kind: tool_started + name: echo + - kind: tool_completed + name: echo + success: true + error: ~ + - kind: tool_result_preview + name: echo + preview_len_bucket: 0 + - kind: other + variant: Other + - kind: thinking + message: thinking (step 2)... + - kind: other + variant: Other + - kind: other + variant: Other + - kind: status + message: done +event_kind_counts: + Other: 4 + Status: 1 + Thinking: 4 + ToolCompleted: 1 + ToolResult: 1 + ToolStarted: 1 +llm_call_count: 2 +safety_warning_count: 0 +engine_threads: [] diff --git a/tests/support/gateway_workflow_harness.rs b/tests/support/gateway_workflow_harness.rs index 7224f0434b1..71e35f6f5ed 100644 --- a/tests/support/gateway_workflow_harness.rs +++ b/tests/support/gateway_workflow_harness.rs @@ -22,12 +22,12 @@ use ironclaw::channels::web::ws::WsConnectionTracker; use ironclaw::config::{Config, RegistryProviderConfig, RoutineConfig}; use ironclaw::db::Database; use ironclaw::db::libsql::LibSqlBackend; -use ironclaw::llm::registry::ProviderProtocol; -use ironclaw::llm::{ - SessionConfig as LlmSessionConfig, SessionManager as LlmSessionManager, create_llm_provider, -}; use ironclaw::secrets::SecretsStore; use ironclaw::tools::{Tool, ToolError, ToolOutput}; +use ironclaw_llm::registry::ProviderProtocol; +use ironclaw_llm::{ + SessionConfig as LlmSessionConfig, SessionManager as LlmSessionManager, create_llm_provider, +}; use crate::support::test_channel::{TestChannel, TestChannelHandle}; diff --git a/tests/support/instrumented_llm.rs b/tests/support/instrumented_llm.rs index da24220577a..d64e3edcd64 100644 --- a/tests/support/instrumented_llm.rs +++ b/tests/support/instrumented_llm.rs @@ -13,7 +13,7 @@ use rust_decimal::Decimal; use tokio::sync::Mutex; use ironclaw::error::LlmError; -use ironclaw::llm::{ +use ironclaw_llm::{ CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, ToolCompletionRequest, ToolCompletionResponse, }; diff --git a/tests/support/live_harness.rs b/tests/support/live_harness.rs index e1f187dc981..e61d3d25850 100644 --- a/tests/support/live_harness.rs +++ b/tests/support/live_harness.rs @@ -30,8 +30,8 @@ use std::path::PathBuf; use std::sync::Arc; -use ironclaw::llm::recording::RecordingLlm; -use ironclaw::llm::{ChatMessage, CompletionRequest, LlmProvider, SessionConfig, SessionManager}; +use ironclaw_llm::recording::RecordingLlm; +use ironclaw_llm::{ChatMessage, CompletionRequest, LlmProvider, SessionConfig, SessionManager}; use crate::support::test_rig::{TestRig, TestRigBuilder}; use crate::support::trace_llm::LlmTrace; @@ -706,7 +706,7 @@ impl LiveTestHarnessBuilder { let source_user_id = config.owner_id.clone(); let session = Arc::new(SessionManager::new(SessionConfig::default())); - let (provider, cheap_llm, _, _) = ironclaw::llm::build_provider_chain(&config.llm, session) + let (provider, cheap_llm, _, _) = ironclaw_llm::build_provider_chain(&config.llm, session) .await .expect("Failed to build LLM provider chain for live test"); diff --git a/tests/support/live_mission_helpers.rs b/tests/support/live_mission_helpers.rs index ca338827734..7c3277c8981 100644 --- a/tests/support/live_mission_helpers.rs +++ b/tests/support/live_mission_helpers.rs @@ -1,10 +1,15 @@ //! Shared helpers for live tests that drive engine v2 missions/routines. //! //! Extracted from `tests/e2e_live_routine.rs` so additional scenarios -//! (`tests/e2e_live_mission_gmail.rs`, etc.) can reuse the same approval -//! responder and notification heuristics without copy-paste drift. +//! that drive engine v2 missions through a live LLM can reuse the same +//! approval responder and notification heuristics without copy-paste +//! drift. +//! +//! Note: the matching Playwright/HTTP coverage for #3133 lives at +//! `tests/e2e/scenarios/test_mission_gmail_3133.py` and uses the mock +//! LLM. The Rust live equivalent was removed when the auto-resume +//! coverage moved fully to the Python tier. -#![cfg(feature = "libsql")] #![allow(dead_code)] // shared API; not every test uses every helper use std::collections::HashSet; diff --git a/tests/support/test_rig.rs b/tests/support/test_rig.rs index 69b1d67c4ad..bf9887c8366 100644 --- a/tests/support/test_rig.rs +++ b/tests/support/test_rig.rs @@ -15,15 +15,15 @@ use ironclaw::channels::web::log_layer::LogBroadcaster; use ironclaw::channels::{OutgoingResponse, StatusUpdate}; use ironclaw::config::Config; use ironclaw::db::Database; -use ironclaw::llm::{LlmProvider, SessionConfig, SessionManager}; use ironclaw::tools::Tool; +use ironclaw_llm::{LlmProvider, SessionConfig, SessionManager}; use crate::support::instrumented_llm::InstrumentedLlm; use crate::support::metrics::{ToolInvocation, TraceMetrics}; use crate::support::test_channel::{CapturedEvent, TestChannel, TestChannelHandle}; use crate::support::trace_llm::{LlmTrace, TraceLlm}; -use ironclaw::llm::recording::{HttpExchange, HttpInterceptor, ReplayingHttpInterceptor}; +use ironclaw_llm::recording::{HttpExchange, HttpInterceptor, ReplayingHttpInterceptor}; // --------------------------------------------------------------------------- // TestRig @@ -299,7 +299,33 @@ impl TestRig { /// Resolve an OAuth-style gate by submitting a typed /// `Submission::ExternalCallback`. pub async fn send_external_callback(&self, request_id: uuid::Uuid) { - let submission = ironclaw::agent::submission::Submission::ExternalCallback { request_id }; + let submission = ironclaw::agent::submission::Submission::ExternalCallback { + request_id, + payload: None, + }; + let msg = ironclaw::channels::IncomingMessage::new( + self.channel.channel_name(), + self.channel.user_id(), + "", + ) + .with_structured_submission(submission); + self.channel.send_incoming(msg).await; + } + + /// Resolve a caller-tool external gate (Responses API path) with a + /// JSON resolution payload. The payload becomes + /// `GateResolution::ExternalCallback { payload }` after submission; + /// the engine then has to materialise it back into an `ActionResult` + /// the LLM can see. + pub async fn send_external_callback_with_payload( + &self, + request_id: uuid::Uuid, + payload: serde_json::Value, + ) { + let submission = ironclaw::agent::submission::Submission::ExternalCallback { + request_id, + payload: Some(payload), + }; let msg = ironclaw::channels::IncomingMessage::new( self.channel.channel_name(), self.channel.user_id(), @@ -312,7 +338,7 @@ impl TestRig { /// Return all message lists that were sent to the LLM provider. /// /// Only available when the rig was built with a `TraceLlm` (i.e., via `.with_trace()`). - pub fn captured_llm_requests(&self) -> Vec<Vec<ironclaw::llm::ChatMessage>> { + pub fn captured_llm_requests(&self) -> Vec<Vec<ironclaw_llm::ChatMessage>> { self.trace_llm .as_ref() .map(|t| t.captured_requests()) @@ -896,7 +922,7 @@ impl TestRigBuilder { /// Unlike `with_extra_tools`, these overrides are applied at the end of /// `build()` via `ToolRegistry::register_sync`, so a probe stub can /// intentionally replace an earlier built-in registration (e.g. - /// `tool_activate`, `tool_auth`) for gate testing. + /// `tool_install`, `tool_auth`) for gate testing. pub fn with_test_tool_override(mut self, tool: Arc<dyn Tool>) -> Self { self.test_tool_overrides.push(tool); self diff --git a/tests/support/trace_llm.rs b/tests/support/trace_llm.rs index 34514b654de..6d2b75f7481 100644 --- a/tests/support/trace_llm.rs +++ b/tests/support/trace_llm.rs @@ -13,7 +13,7 @@ use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use ironclaw::error::LlmError; -use ironclaw::llm::{ +use ironclaw_llm::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, }; @@ -22,7 +22,7 @@ use ironclaw::llm::{ // still import them from here. // Re-export all shared types so downstream test files can import from here. #[allow(unused_imports)] -pub use ironclaw::llm::recording::{ +pub use ironclaw_llm::recording::{ ExpectedToolResult, HttpExchange, HttpExchangeRequest, HttpExchangeResponse, MemorySnapshotEntry, RequestHint, TraceResponse, TraceStep, TraceToolCall, }; @@ -739,6 +739,7 @@ impl LlmProvider for TraceLlm { finish_reason: FinishReason::Stop, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }), TraceResponse::ToolCalls { tool_calls, @@ -752,6 +753,7 @@ impl LlmProvider for TraceLlm { name: tc.name, arguments: tc.arguments, reasoning: None, + signature: None, }) .collect(); Ok(ToolCompletionResponse { @@ -762,6 +764,7 @@ impl LlmProvider for TraceLlm { finish_reason: FinishReason::ToolUse, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + reasoning: None, }) } TraceResponse::UserInput { .. } => Err(LlmError::RequestFailed { diff --git a/tests/support_unit_tests.rs b/tests/support_unit_tests.rs index 87432999e2a..96d4f29fc10 100644 --- a/tests/support_unit_tests.rs +++ b/tests/support_unit_tests.rs @@ -354,7 +354,7 @@ mod test_channel_tests { mod trace_llm_tests { use crate::support::trace_llm::*; - use ironclaw::llm::{ + use ironclaw_llm::{ ChatMessage, CompletionRequest, FinishReason, LlmProvider, ToolCompletionRequest, }; diff --git a/tests/telegram_pairing_chat_claim_integration.rs b/tests/telegram_pairing_chat_claim_integration.rs new file mode 100644 index 00000000000..0ae2af2eafc --- /dev/null +++ b/tests/telegram_pairing_chat_claim_integration.rs @@ -0,0 +1,144 @@ +//! Regression coverage for #3317 — chat-surface pairing claim. +//! +//! Drives the full chain: submission parser → agent loop dispatch → +//! `bridge::handle_pairing_claim` → `PairingStore::approve`. The unit +//! tests in `src/bridge/router.rs` cover the no-extension-manager and +//! invalid-channel branches; this integration test exercises the +//! happy path through a real `Agent` + `ExtensionManager` + +//! `PairingStore`. +//! +//! Why this lives at the integration tier (per +//! `.claude/rules/testing.md` "Test Through the Caller"): the parser +//! and handler are correct individually, but the wiring between them +//! — `agent_loop.rs` calling `crate::bridge::handle_pairing_claim` — +//! is exactly where #3317 would silently regress if the new arm is +//! ever dropped from the dispatch match. + +#[cfg(feature = "libsql")] +mod support; + +#[cfg(feature = "libsql")] +mod pairing_chat_claim_tests { + use std::sync::OnceLock; + use std::time::Duration; + + use tokio::sync::Mutex; + + use crate::support::test_rig::TestRigBuilder; + use ironclaw::db::UserRecord; + + /// Seed the test-channel user into the users table so the FK on + /// `channel_identities.owner_id` is satisfied during pairing approval. + /// The TestRig defaults its channel `user_id` to `"test-user"`, but + /// only the owner row is created automatically — pairing-claim flows + /// reach the DB constraint that other rig consumers don't. + async fn seed_test_user(rig: &crate::support::test_rig::TestRig, user_id: &str) { + rig.database() + .get_or_create_user(UserRecord { + id: user_id.to_string(), + role: "member".to_string(), + display_name: user_id.to_string(), + status: "active".to_string(), + email: None, + last_login_at: None, + created_by: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + metadata: serde_json::Value::Null, + }) + .await + .expect("seed test user must succeed"); + } + + const TIMEOUT: Duration = Duration::from_secs(15); + + /// Engine v2 stores its state in a process-global `OnceLock`. + /// Serialize tests in this file so one test's state doesn't bleed + /// into the next instance. + fn engine_v2_test_lock() -> &'static Mutex<()> { + static LOCK: OnceLock<Mutex<()>> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + #[tokio::test] + async fn chat_approve_telegram_code_completes_pairing() { + let _guard = engine_v2_test_lock().lock().await; + + let rig = TestRigBuilder::new().with_engine_v2().build().await; + rig.clear().await; + + // Pairing approval writes `channel_identities.owner_id` which + // FKs to `users.id`. Seed the channel's user row so the + // constraint is satisfied. (The owner row is auto-seeded but + // the channel's user_id "test-user" is not.) + seed_test_user(&rig, "test-user").await; + + let ext_mgr = rig + .extension_manager() + .cloned() + .expect("test rig must wire an ExtensionManager"); + let pairing_store = ext_mgr + .pairing_store() + .cloned() + .expect("ExtensionManager must wire a PairingStore for the chat-claim flow"); + + // Mint a pairing code the user can claim. The bot's pairing + // reply is what tells real users this code in the matching + // production path; here we shortcut by inserting the request + // directly so the handler under test exercises the full + // approve → propagate → respond chain. + let pairing = pairing_store + .upsert_request("telegram", "tg-test-user-9001", None) + .await + .expect("pairing request upsert must succeed"); + assert!( + !pairing.code.is_empty(), + "pairing store must mint a non-empty code" + ); + + // Type the pairing claim into the chat surface — this is the + // same path users naturally try after seeing the bot's reply + // ("type `approve telegram CODE` in any IronClaw chat"). + rig.send_message(&format!("approve telegram {}", pairing.code)) + .await; + + let responses = rig.wait_for_responses(1, TIMEOUT).await; + assert!( + !responses.is_empty(), + "agent must respond to the pairing claim within {TIMEOUT:?}" + ); + let response_text = responses[0].content.clone(); + assert!( + response_text.contains("Pairing approved") && response_text.contains("telegram"), + "expected 'Pairing approved … telegram' response, got: {response_text}" + ); + + rig.shutdown(); + } + + #[tokio::test] + async fn chat_approve_invalid_code_responds_clearly() { + let _guard = engine_v2_test_lock().lock().await; + + let rig = TestRigBuilder::new().with_engine_v2().build().await; + rig.clear().await; + + // No pairing request was minted, so any code is "invalid". The + // user must see a clear rejection — not have the LLM improvise + // an unhelpful answer like the original #3317 report. + rig.send_message("approve telegram NEVERMINTED99").await; + + let responses = rig.wait_for_responses(1, TIMEOUT).await; + assert!( + !responses.is_empty(), + "agent must respond to invalid pairing claim within {TIMEOUT:?}" + ); + let response_text = responses[0].content.clone(); + assert!( + response_text.contains("Invalid or expired pairing code"), + "expected explicit invalid-code rejection, got: {response_text}" + ); + + rig.shutdown(); + } +} diff --git a/tests/thread_isolation_integration.rs b/tests/thread_isolation_integration.rs new file mode 100644 index 00000000000..820d3dc9010 --- /dev/null +++ b/tests/thread_isolation_integration.rs @@ -0,0 +1,422 @@ +//! Cross-tenant thread access regression tests. +//! +//! Drives the running gateway over real HTTP and asserts that Bob cannot +//! read Alice's threads, messages, or engine-thread metadata. Each +//! handler that takes a thread-id from the request gets its own negative +//! test: success means a foreign-id request is rejected at the boundary +//! (typically 404 to prevent enumeration, never 200). +//! +//! Modeled on the job-isolation tests in +//! `tests/multi_tenant_integration.rs` (`full_server_*_jobs_*`). +//! +//! Gated on `feature = "libsql"` so the suite has a real DB to seed +//! conversations into. + +#![cfg(feature = "libsql")] + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use ironclaw::agent::SessionManager; +use ironclaw::channels::web::auth::{MultiAuthState, UserIdentity}; +use ironclaw::channels::web::platform::router::start_server; +use ironclaw::channels::web::platform::state::{GatewayState, PerUserRateLimiter, RateLimiter}; +use ironclaw::channels::web::sse::SseManager; +use ironclaw::channels::web::ws::WsConnectionTracker; +use ironclaw::db::Database; + +const ALICE_TOKEN: &str = "tok-alice-thread-isolation"; +const BOB_TOKEN: &str = "tok-bob-thread-isolation"; +const ALICE_USER_ID: &str = "alice"; +const BOB_USER_ID: &str = "bob"; + +fn two_user_auth() -> MultiAuthState { + let mut tokens = HashMap::new(); + tokens.insert( + ALICE_TOKEN.to_string(), + UserIdentity { + user_id: ALICE_USER_ID.to_string(), + role: "admin".to_string(), + workspace_read_scopes: Vec::new(), + }, + ); + tokens.insert( + BOB_TOKEN.to_string(), + UserIdentity { + user_id: BOB_USER_ID.to_string(), + role: "admin".to_string(), + workspace_read_scopes: Vec::new(), + }, + ); + MultiAuthState::multi(tokens) +} + +/// Spin up a real Axum server backed by an in-memory libSQL database +/// with a `SessionManager` attached. The chat history / threads +/// endpoints require both, and the Responses-API GET requires the DB. +async fn start_server_with_db() -> ( + SocketAddr, + Arc<GatewayState>, + Arc<dyn Database>, + tempfile::TempDir, +) { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("test.db"); + let backend = ironclaw::db::libsql::LibSqlBackend::new_local(&path) + .await + .expect("backend"); + backend.run_migrations().await.expect("migrations"); + let db: Arc<dyn Database> = Arc::new(backend); + + let (agent_tx, _agent_rx) = tokio::sync::mpsc::channel(64); + let auth = two_user_auth(); + let session_manager = Arc::new(SessionManager::new()); + + let state = Arc::new(GatewayState { + msg_tx: tokio::sync::RwLock::new(Some(agent_tx)), + sse: Arc::new(SseManager::new()), + workspace: None, + workspace_pool: None, + multi_tenant_mode: true, + session_manager: Some(session_manager), + log_broadcaster: None, + log_level_handle: None, + extension_manager: None, + tool_registry: None, + store: Some(Arc::clone(&db)), + settings_cache: None, + job_manager: None, + prompt_queue: None, + scheduler: None, + owner_id: ALICE_USER_ID.to_string(), + shutdown_tx: tokio::sync::RwLock::new(None), + ws_tracker: Some(Arc::new(WsConnectionTracker::new())), + llm_provider: None, + llm_reload: None, + llm_session_manager: None, + config_toml_path: None, + skill_registry: None, + skill_catalog: None, + auth_manager: None, + chat_rate_limiter: PerUserRateLimiter::new(30, 60), + oauth_rate_limiter: PerUserRateLimiter::new(20, 60), + registry_entries: Vec::new(), + cost_guard: None, + routine_engine: Arc::new(tokio::sync::RwLock::new(None)), + startup_time: std::time::Instant::now(), + webhook_rate_limiter: RateLimiter::new(10, 60), + active_config: Arc::new(tokio::sync::RwLock::new(Default::default())), + secrets_store: None, + db_auth: None, + pairing_store: None, + oauth_providers: None, + oauth_state_store: None, + oauth_base_url: None, + oauth_allowed_domains: Vec::new(), + near_nonce_store: None, + near_rpc_url: None, + near_network: None, + oauth_sweep_shutdown: None, + frontend_html_cache: std::sync::Arc::new(tokio::sync::RwLock::new(None)), + tool_dispatcher: None, + }); + + let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + let bound = start_server(addr, state.clone(), auth.into()) + .await + .expect("start_server"); + + (bound, state, db, temp_dir) +} + +/// Seed a conversation with a single message, owned by `user_id`. +/// Returns the conversation id so the caller can probe access by id. +async fn seed_conversation(db: &Arc<dyn Database>, user_id: &str, content: &str) -> uuid::Uuid { + let id = db + .create_conversation_with_metadata( + "gateway", + user_id, + &serde_json::json!({"title": format!("{user_id}'s conversation")}), + ) + .await + .expect("create conversation"); + db.add_conversation_message(id, "user", content) + .await + .expect("add message"); + id +} + +// --------------------------------------------------------------------------- +// Chat history — paginated read by thread_id +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_history_for_alice_thread_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_thread = seed_conversation(&db, ALICE_USER_ID, "alice secret").await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{}/api/chat/history?thread_id={}", + addr, alice_thread + )) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Bob requesting Alice's thread_id must get 404; got status {} body {}", + resp.status(), + resp.text().await.unwrap_or_default() + ); +} + +#[tokio::test] +async fn bob_history_paginated_for_alice_thread_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_thread = seed_conversation(&db, ALICE_USER_ID, "alice secret").await; + + let client = reqwest::Client::new(); + // The handler's paginated branch parses `before` as RFC3339 then runs + // an unfiltered `list_conversation_messages_paginated` against the DB. + // Pre-fix that branch skipped the ownership check; this test pins the + // post-fix behavior. Use `.query()` so reqwest URL-encodes `+`/`:`. + let before = chrono::Utc::now().to_rfc3339(); + let resp = client + .get(format!("http://{}/api/chat/history", addr)) + .query(&[ + ("thread_id", alice_thread.to_string()), + ("before", before), + ("limit", "10".to_string()), + ]) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + // The paginated branch went straight to the unscoped DB query before + // F4. With ownership pre-checked at the handler boundary, Bob's + // request is rejected before any messages load. + assert_eq!( + resp.status(), + 404, + "Bob's paginated history request for Alice's thread must not return messages" + ); +} + +#[tokio::test] +async fn alice_history_for_own_thread_succeeds() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_thread = seed_conversation(&db, ALICE_USER_ID, "alice writes this").await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{}/api/chat/history?thread_id={}", + addr, alice_thread + )) + .header("Authorization", format!("Bearer {}", ALICE_TOKEN)) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 200, "Alice must reach her own thread"); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["thread_id"], alice_thread.to_string()); +} + +// --------------------------------------------------------------------------- +// Threads list — should never enumerate another user's conversations +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_threads_list_excludes_alice_threads() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + + // Seed multiple conversations for each user. + let alice_a = seed_conversation(&db, ALICE_USER_ID, "alice 1").await; + let alice_b = seed_conversation(&db, ALICE_USER_ID, "alice 2").await; + let bob_a = seed_conversation(&db, BOB_USER_ID, "bob 1").await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{}/api/chat/threads", addr)) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body: serde_json::Value = resp.json().await.unwrap(); + + let visible_ids: Vec<String> = body["threads"] + .as_array() + .unwrap_or(&Vec::new()) + .iter() + .filter_map(|t| t["id"].as_str().map(String::from)) + .collect(); + + let alice_a_str = alice_a.to_string(); + let alice_b_str = alice_b.to_string(); + let bob_a_str = bob_a.to_string(); + + assert!( + !visible_ids.contains(&alice_a_str), + "Bob's threads list must not include Alice's conversations: {visible_ids:?}" + ); + assert!( + !visible_ids.contains(&alice_b_str), + "Bob's threads list must not include Alice's conversations: {visible_ids:?}" + ); + // Bob's own conversation may appear; the assistant_thread is also + // his own (auto-created by the handler). The point is the absence + // of alice_*. + let _ = bob_a_str; +} + +// --------------------------------------------------------------------------- +// Engine v2 — detail / steps / events +// +// Scope: these tests pin the *handler shape* — an unknown thread id +// returns 404 / empty rather than 500ing or leaking. They do NOT +// exercise the cross-tenant ownership branch (Alice's actual id with +// Bob's token), because seeding an engine v2 thread requires +// `ENGINE_STATE` (a process-wide `OnceCell` in `bridge/router.rs`) +// to be initialized with a backing `Store`. That fixture doesn't +// exist for the integration test surface today; building it without +// breaking the singleton's invariants for parallel test runs is its +// own change. +// +// The ownership gate itself lives in +// `src/bridge/router.rs::{get_engine_thread, list_engine_thread_steps, +// list_engine_thread_events}` and uses `thread.is_owned_by(user_id)`. +// That predicate is unit-tested upstream; the missing piece here is +// the call-site test through the handler. +// +// Follow-up: introduce a per-test `ENGINE_STATE` injection seam (or a +// memory-backed `Store` test fixture) and add the Alice-seed/Bob-probe +// variant of each test below. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_engine_thread_detail_for_unknown_returns_404() { + let (addr, _state, _db, _tmp) = start_server_with_db().await; + let foreign_id = uuid::Uuid::new_v4(); + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{}/api/engine/threads/{}", addr, foreign_id)) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status(), + 404, + "Engine thread detail for an id Bob doesn't own must be 404" + ); +} + +#[tokio::test] +async fn bob_engine_thread_steps_for_unknown_returns_empty() { + let (addr, _state, _db, _tmp) = start_server_with_db().await; + let foreign_id = uuid::Uuid::new_v4(); + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{}/api/engine/threads/{}/steps", + addr, foreign_id + )) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 200); + let body: serde_json::Value = resp.json().await.unwrap(); + let steps = body["steps"].as_array().expect("steps array"); + assert!( + steps.is_empty(), + "Engine steps must be empty for an unowned thread id; got {body:?}" + ); +} + +#[tokio::test] +async fn bob_engine_thread_events_for_unknown_returns_empty() { + let (addr, _state, _db, _tmp) = start_server_with_db().await; + let foreign_id = uuid::Uuid::new_v4(); + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{}/api/engine/threads/{}/events", + addr, foreign_id + )) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 200); + let body: serde_json::Value = resp.json().await.unwrap(); + let events = body["events"].as_array().expect("events array"); + assert!( + events.is_empty(), + "Engine events must be empty for unowned thread id" + ); +} + +// --------------------------------------------------------------------------- +// Responses API — GET /api/v1/responses/{id} +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn bob_responses_get_for_alice_response_returns_404() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + // Seed alice's conversation to act as a response_id surrogate. The + // Responses API's GET path resolves a UUID to a conversation owner + // before reading messages, so any alice-owned conversation id is a + // valid attack surface. + let alice_thread = seed_conversation(&db, ALICE_USER_ID, "alice").await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!("http://{}/api/v1/responses/{}", addr, alice_thread)) + .header("Authorization", format!("Bearer {}", BOB_TOKEN)) + .send() + .await + .unwrap(); + + assert!( + resp.status() == 404 || resp.status() == 400, + "Bob requesting Alice's response id must not get 200; got {}", + resp.status() + ); +} + +// --------------------------------------------------------------------------- +// Unauthenticated access never reaches any of the above +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn unauthenticated_history_request_is_rejected() { + let (addr, _state, db, _tmp) = start_server_with_db().await; + let alice_thread = seed_conversation(&db, ALICE_USER_ID, "alice").await; + + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://{}/api/chat/history?thread_id={}", + addr, alice_thread + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401); +} diff --git a/tests/wit_compat.rs b/tests/wit_compat.rs index 597253b3138..9598ff3ea4c 100644 --- a/tests/wit_compat.rs +++ b/tests/wit_compat.rs @@ -275,6 +275,12 @@ fn instantiate_channel_component( }) .map_err(|e| format!("stub 'workspace-write': {e}"))?; + host.func_new("websocket-send-text", |_ctx, _ty, _args, results| { + results[0] = wasmtime::component::Val::Result(Ok(None)); + Ok(()) + }) + .map_err(|e| format!("stub 'websocket-send-text': {e}"))?; + host.func_new("pairing-upsert-request", |_ctx, _ty, _args, results| { results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( wasmtime::component::Val::String("stub".into()), @@ -303,18 +309,22 @@ fn instantiate_channel_component( Ok(()) } - { - let mut root = linker.root(); - let mut host = root - .instance("near:agent/channel-host") - .map_err(|e| format!("failed to create unversioned channel-host: {e}"))?; - stub_channel_host(&mut host)?; - } - { + let mut channel_host_interfaces = vec![ + "near:agent/channel-host".to_string(), + "near:agent/channel-host@0.3.0".to_string(), + format!( + "near:agent/channel-host@{}", + ironclaw::tools::wasm::WIT_CHANNEL_VERSION + ), + ]; + channel_host_interfaces.sort(); + channel_host_interfaces.dedup(); + + for interface in channel_host_interfaces { let mut root = linker.root(); let mut host = root - .instance("near:agent/channel-host@0.3.0") - .map_err(|e| format!("failed to create versioned channel-host@0.3.0: {e}"))?; + .instance(&interface) + .map_err(|e| format!("failed to create {interface}: {e}"))?; stub_channel_host(&mut host)?; } diff --git a/tests/workspace_integration.rs b/tests/workspace_integration.rs index 832635c1a4e..9e28ec81103 100644 --- a/tests/workspace_integration.rs +++ b/tests/workspace_integration.rs @@ -6,7 +6,8 @@ use std::sync::Arc; -use ironclaw::workspace::{MockEmbeddings, SearchConfig, Workspace, paths}; +use ironclaw::workspace::{SearchConfig, Workspace, paths}; +use ironclaw_embeddings::MockEmbeddings; fn get_pool() -> deadpool_postgres::Pool { let database_url = std::env::var("DATABASE_URL") diff --git a/tests/workspace_scoped_rebind.rs b/tests/workspace_scoped_rebind.rs index 2a9fe03da8b..9f50aa85e30 100644 --- a/tests/workspace_scoped_rebind.rs +++ b/tests/workspace_scoped_rebind.rs @@ -3,7 +3,7 @@ //! Issue: <https://github.com/nearai/ironclaw/issues/1652> //! //! `scoped_to_user()` rebinds a workspace to a different primary user while -//! preserving shared read scopes. These 7 tests verify the rebinding contracts +//! preserving shared read scopes. This suite verifies the rebinding contracts //! directly, rather than relying on high-level system prompt tests. //! //! Related indirect coverage: @@ -122,6 +122,74 @@ async fn rebind_rescopes_private_layers_without_mutating_shared_layers() { assert_no_duplicates(ids); } +#[tokio::test] +async fn rebind_preserves_non_private_layer_when_scope_matches_private_source() { + let (db, _dir) = setup().await; + + let colliding_scope = "resolved-owner-scope"; + let ws = Workspace::new_with_db("startup-owner", db).with_memory_layers(vec![ + MemoryLayer { + name: "private".to_string(), + scope: colliding_scope.to_string(), + writable: true, + sensitivity: LayerSensitivity::Private, + }, + MemoryLayer { + name: "team".to_string(), + scope: colliding_scope.to_string(), + writable: false, + sensitivity: LayerSensitivity::Shared, + }, + ]); + + let rebound = ws.scoped_to_user("alice"); + + let private = + MemoryLayer::find(rebound.memory_layers(), "private").expect("private layer must exist"); + assert_eq!(private.scope, "alice"); + + let team = MemoryLayer::find(rebound.memory_layers(), "team").expect("team layer must exist"); + assert_eq!(team.scope, colliding_scope); + + let ids = rebound.read_user_ids(); + assert_eq!(ids[0], "alice", "new primary must be first"); + assert!( + !ids.contains(&"startup-owner".to_string()), + "old primary must not remain in read_user_ids" + ); + assert!( + ids.contains(&colliding_scope.to_string()), + "shared colliding scope must remain readable" + ); + assert_no_duplicates(ids); +} + +#[tokio::test] +async fn rebind_rescopes_private_layers_even_when_source_scope_differs_from_primary() { + let (db, _dir) = setup().await; + + let ws = Workspace::new_with_db("startup-owner", db).with_memory_layers(vec![MemoryLayer { + name: "private".to_string(), + scope: "resolved-owner-scope".to_string(), + writable: true, + sensitivity: LayerSensitivity::Private, + }]); + + let rebound = ws.scoped_to_user("alice"); + + let private = + MemoryLayer::find(rebound.memory_layers(), "private").expect("private layer must exist"); + assert_eq!(private.scope, "alice"); + assert_eq!(rebound.read_user_ids(), &["alice".to_string()]); + assert!( + !rebound + .read_user_ids() + .contains(&"resolved-owner-scope".to_string()), + "private source scope must not remain readable after rebinding: {:?}", + rebound.read_user_ids() + ); +} + // ─── Test 3: Secondary read scopes preserved, old primary removed ──────── #[tokio::test] diff --git a/wit/channel.wit b/wit/channel.wit index 62421cdc543..f8dcd1def39 100644 --- a/wit/channel.wit +++ b/wit/channel.wit @@ -1,4 +1,4 @@ -package near:agent@0.3.0; +package near:agent@0.3.1; // WASM Channel Sandbox Interface // @@ -194,6 +194,16 @@ interface channel-host { /// - Write operation fails workspace-write: func(path: string, content: string) -> result<_, string>; + /// Send a raw websocket text frame through the host-managed websocket runtime. + /// + /// This is intentionally transport-agnostic: the channel is responsible for + /// constructing any protocol-specific payload. + /// + /// Returns Err if: + /// - the channel has no active websocket runtime + /// - the websocket runtime is shutting down + websocket-send-text: func(payload: string) -> result<_, string>; + // ==================== DM Pairing ==================== /// Result of upserting a pairing request.