diff --git a/services/kiloclaw/DEVELOPMENT.md b/services/kiloclaw/DEVELOPMENT.md index 3d35fcb096..5bcbae2c5d 100644 --- a/services/kiloclaw/DEVELOPMENT.md +++ b/services/kiloclaw/DEVELOPMENT.md @@ -591,7 +591,7 @@ Then run one of: no token -> `401`, correct proxy token -> pass-through. - Use this when changing proxy token logic or route/auth ordering. - `bash scripts/tests/smoke-live-provider.sh` - - Runs the packaged image against the real Kilo Gateway with `kilocode/kilo-auto/free`, verifying `openclaw config validate --json`, Control UI proxying, packaged Kilo Chat loading, and one live agent turn. + - Runs the packaged image against the real Kilo Gateway with a paid route (`kilocode/kilo-auto/balanced` by default; free routes are rejected), verifying `openclaw config validate --json`, Control UI proxying, packaged Kilo Chat loading, and one live agent turn per leg. In `--upgrade` mode the candidate is asserted both on the baseline's persisted root and on a fresh root. - Reads `KILOCODE_API_KEY` from the environment, or falls back locally to the active `kilocodeToken` and matching organization scope in `~/.kilocode/cli/config.json`. The credential is passed to the temporary container as an environment variable; the script does not print it or dump potentially sensitive controller logs on startup failure. - Publishes the temporary controller only on loopback and generates a random controller/proxy token unless `TOKEN` is explicitly set. - Uses a generated non-sensitive nonce prompt because Auto Free can route to upstream providers that log prompts. diff --git a/services/kiloclaw/scripts/tests/README.md b/services/kiloclaw/scripts/tests/README.md index c922f69ac0..34083d85fb 100644 --- a/services/kiloclaw/scripts/tests/README.md +++ b/services/kiloclaw/scripts/tests/README.md @@ -11,7 +11,7 @@ Validate an OpenClaw version bump before merging the bump PR. **Run this one:** ```bash -export KILOCODE_API_KEY= # for the live smoke; from app.kilo.ai/profile +export KILOCODE_API_KEY= # for the live smoke; from app.kilo.ai/profile export KILOCODE_ORGANIZATION_ID= # REQUIRED if the key is a personal key in an org bash services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh ``` diff --git a/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh b/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh index 84c9b7561d..f5a2581d07 100755 --- a/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh +++ b/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh @@ -433,17 +433,18 @@ This run covered only the checks that need NO Kilo API key. Before merge, run the credentialed live smoke locally too (it loads a real key into the freshly released OpenClaw, which is why nothing here runs in CI): - export KILOCODE_API_KEY= # not your personal key + export KILOCODE_API_KEY= # not your personal key export KILOCODE_ORGANIZATION_ID= # required to spend ORG credits bash services/kiloclaw/scripts/tests/openclaw-upgrade-smoke.sh That covers what CI cannot without a credential: - persisted-root upgrade boot (baseline -> candidate on the same /root) + - the candidate on a FRESH root too (the shape a new signup gets) - gateway readiness + proxied Control UI - kilo-chat plugin load, diagnostics, and webhook route - app config-write routes (/_kilo/config/patch, agent-defaults, agents CRUD) - exec-approvals seeding - - a real Auto Free agent turn through the live Kilo Gateway + - a real agent turn on a paid route through the live Kilo Gateway ---------------------------------------------------------------------- EOF diff --git a/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh b/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh index a7adc13d12..73ed224d41 100755 --- a/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh +++ b/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh @@ -12,8 +12,9 @@ set -uo pipefail # # Phase 2 credentialed live smoke — builds the before/after images, performs # the persisted-root upgrade (boots baseline, then candidate on the -# same /root), and runs every assertion incl. a real Auto Free gateway -# turn. Needs a dedicated free-model Kilo API key. +# same /root), then asserts the candidate BOTH as an upgraded instance +# and on a fresh root, each with a real gateway turn on a paid route. +# Needs a Kilo API key on an account with credits. # # OpenClaw is never built or run in CI (it is a security-sensitive upstream), so # this is the gate a human runs locally before marking the bump PR ready. @@ -150,7 +151,7 @@ elif [ "$IS_BUMP" -eq 1 ]; then echo " • Phase 2 — credentialed live smoke: WILL BE SKIPPED (no Kilo API key is set)" echo echo "Phase 2 (the live smoke) is half the coverage and needs a Kilo API key." - echo "For the full validation, set a dedicated free-model key and re-run:" + echo "For the full validation, set a key on an account WITH CREDITS and re-run:" echo " export KILOCODE_API_KEY= # from https://app.kilo.ai/profile (bottom)" echo " export KILOCODE_ORGANIZATION_ID= # REQUIRED to spend ORG credits;" echo " # a personal token alone spends only" diff --git a/services/kiloclaw/scripts/tests/smoke-live-provider.sh b/services/kiloclaw/scripts/tests/smoke-live-provider.sh index e1e1207e60..02e8ed00f9 100644 --- a/services/kiloclaw/scripts/tests/smoke-live-provider.sh +++ b/services/kiloclaw/scripts/tests/smoke-live-provider.sh @@ -2,8 +2,8 @@ set -euo pipefail # Live packaged-image smoke for KiloClaw + real Kilo Gateway routing. -# This script intentionally uses Auto Free and sends only a generated nonce prompt. -# It is opt-in/manual because it requires live credentials and free-model availability. +# This script uses a PAID auto route and sends only a generated nonce prompt. +# It is opt-in/manual because it requires live credentials that can pay for a turn. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" IMAGE="${IMAGE:-kiloclaw:controller}" @@ -20,7 +20,13 @@ print(s.getsockname()[1]) s.close()')}" TOKEN="${TOKEN:-$(python3 -c 'import secrets; print(secrets.token_hex(32))')}" KILOCODE_CONFIG_PATH="${KILOCODE_CONFIG_PATH:-$HOME/.kilocode/cli/config.json}" -KILOCODE_SMOKE_MODEL="${KILOCODE_SMOKE_MODEL:-kilocode/kilo-auto/free}" +# Deliberately a PAID auto route, matching what production instances default to. +# A free route is not a valid gate: openclaw 2026.6.11 could not complete a turn on +# kilocode/kilo-auto/free against the live gateway ("provider rejected the request +# schema or tool payload") while the same image succeeded on a paid route, so a +# free-route run conflates provider-side free-tier behaviour with the image under +# test. Overridable, but a free route is rejected below unless explicitly allowed. +KILOCODE_SMOKE_MODEL="${KILOCODE_SMOKE_MODEL:-kilocode/kilo-auto/balanced}" EXPECTED_VERSION_BEFORE="${EXPECTED_VERSION_BEFORE:-}" EXPECTED_VERSION_AFTER="${EXPECTED_VERSION_AFTER:-}" MODE="fresh" @@ -32,9 +38,12 @@ usage() { cat <<'EOF' Usage: bash scripts/tests/smoke-live-provider.sh [--upgrade] -Runs a packaged KiloClaw image against the real Kilo Gateway using the Auto Free -model by default. Provide KILOCODE_API_KEY explicitly or authenticate with the -Kilo CLI locally so ~/.kilocode/cli/config.json contains an active token. +Runs a packaged KiloClaw image against the real Kilo Gateway using a PAID auto +route by default (kilocode/kilo-auto/balanced, what production instances use). +Provide KILOCODE_API_KEY explicitly or authenticate with the Kilo CLI locally so +~/.kilocode/cli/config.json contains an active token. The account must have +credits: a free route is rejected, because free-route failures reflect provider +free-tier behaviour rather than the image under test. Options: --upgrade Boot IMAGE_BEFORE, then IMAGE_AFTER on the same temporary /root. @@ -131,6 +140,27 @@ for o in orgs: fi fi +# Free routes are not a valid gate. Confirmed against a live instance: openclaw +# 2026.6.11 on kilocode/kilo-auto/free fails every turn with "provider rejected +# the request schema or tool payload", while the same image on a paid route +# reaches the provider normally. Running the gate on a free route therefore tests +# free-tier provider behaviour, not the image, and produces failures that look +# like image regressions. +case "$KILOCODE_SMOKE_MODEL" in + *"/free"|*":free") + echo "✗ KILOCODE_SMOKE_MODEL is a FREE route ($KILOCODE_SMOKE_MODEL)." >&2 + echo " The live smoke must run on a PAID route with credits available, because" >&2 + echo " free-route failures reflect provider-side free-tier behaviour rather than" >&2 + echo " the image under test and are indistinguishable from a real regression." >&2 + echo " Default is kilocode/kilo-auto/balanced (what production instances use)." >&2 + echo " Override deliberately with ALLOW_FREE_SMOKE_MODEL=true if you know why." >&2 + if [ "${ALLOW_FREE_SMOKE_MODEL:-false}" != "true" ]; then + exit 1 + fi + echo " ALLOW_FREE_SMOKE_MODEL=true set — continuing on a free route." >&2 + ;; +esac + export KILOCODE_API_KEY export KILOCODE_DEFAULT_MODEL="$KILOCODE_SMOKE_MODEL" if [ -n "${KILOCODE_ORGANIZATION_ID:-}" ]; then @@ -151,15 +181,24 @@ if [ "$MODE" = "upgrade" ] && ! docker image inspect "$IMAGE_BEFORE" >/dev/null fi ROOTDIR="$(mktemp -d)" +# Every root this run allocates, so cleanup can remove them all at EXIT. Roots are +# never deleted mid-run: see the fresh-root leg below. +ROOTDIRS=("$ROOTDIR") CID="" PASS=0 FAIL=0 cleanup() { + local dir if [ -n "$CID" ]; then docker rm -f "$CID" >/dev/null 2>&1 || true fi - rm -rf "$ROOTDIR" + # The container writes /root as uid 0, so on native Linux Docker these trees can + # be root-owned and unremovable by a non-root host user. Cleanup must never be + # able to fail the run, hence `|| true` on every removal. + for dir in "${ROOTDIRS[@]}"; do + rm -rf "$dir" 2>/dev/null || true + done } trap cleanup EXIT @@ -176,11 +215,17 @@ check() { start_container() { local image="$1" + # Production parity: every real instance boots with AUTO_APPROVE_DEVICES=true — + # it is set unconditionally in services/kiloclaw/src/gateway/env.ts as a reserved + # system var. Omitting it here meant the smoke booted a configuration that does + # not exist in production: the controller's gateway-client device auto-approval + # was disabled, and openclaw.json never got gateway.controlUi.allowInsecureAuth. local -a docker_env=( -e OPENCLAW_GATEWAY_TOKEN="$TOKEN" -e KILOCODE_API_KEY -e KILOCODE_DEFAULT_MODEL -e REQUIRE_PROXY_TOKEN=true + -e AUTO_APPROVE_DEVICES=true ) if [ -n "${KILOCODE_ORGANIZATION_ID:-}" ]; then docker_env+=(-e KILOCODE_ORGANIZATION_ID) @@ -251,10 +296,48 @@ wait_for_gateway_serving() { sleep 1 done - echo "FAIL: $label gateway did not start serving calls" + echo "FAIL: $label gateway did not start serving calls (see wait_for_device_scopes note)" return 1 } +# Second warm-up gate, and the subtler one. Serving `status` is not the same as +# being able to run an AGENT turn: that needs the calling device to hold +# operator.write (or operator.admin). On a real Fly instance the device is granted +# operator.write within seconds. In local Docker the same image can sit for a +# while with only operator.pairing, and a turn landing in that window dies with +# `1008 pairing required: device is asking for more scopes than currently +# approved`, which reads as a product failure but is warm-up. +# +# The poll is opportunistic only, and usually CANNOT succeed: verified against a +# live instance, /root/.openclaw/devices does not exist before the first client +# connects, and paired.json is created BY that connection. So there is normally no +# signal to wait for, and the settle is what actually does the work. The poll is +# kept short for the case where a previous leg on this same persisted root already +# left an approved device behind. Never fatal: absence is the normal state, so +# failing on it would fail the gate on environment alone. +DEVICE_SCOPE_POLL_SECS="${DEVICE_SCOPE_POLL_SECS:-20}" +DEVICE_SCOPE_SETTLE_SECS="${DEVICE_SCOPE_SETTLE_SECS:-45}" + +wait_for_device_scopes() { + local label="$1" + local i + + echo "waiting for $label device to hold operator write scope ..." + for i in $(seq 1 "$DEVICE_SCOPE_POLL_SECS"); do + if docker exec "$CID" sh -c \ + 'grep -qE "operator\.(write|admin)" /root/.openclaw/devices/paired.json 2>/dev/null'; then + echo " device holds write scope after ${i}s" + return 0 + fi + sleep 1 + done + + echo " no pre-existing approved device (the normal case — the entry is created" + echo " by the connection itself); settling ${DEVICE_SCOPE_SETTLE_SECS}s before the turn" + sleep "$DEVICE_SCOPE_SETTLE_SECS" + return 0 +} + assert_configured_model() { local model model=$(docker exec -i "$CID" python3 - <<'PY' @@ -338,6 +421,11 @@ report_turn_error() { else echo " error: $ident (provider message suppressed — it can contain live credentials)" fi + if [[ "$ident" == *NO_CREDITS* ]]; then + echo " This is a BILLING state, not an image defect: the account backing this run" + echo " cannot pay for $KILOCODE_SMOKE_MODEL. Add credits, or set" + echo " KILOCODE_ORGANIZATION_ID so org credits are used, then re-run." + fi } assert_live_agent_turn() { @@ -407,11 +495,15 @@ ident = " ".join(str(err.get(k, "")) for k in ("type", "kind", "code") if err.ge reason = err.get("reason") if reason and str(err.get("type", "")) == "gateway_transport_error": ident = f"{ident} :: {reason}" +# A 402 is a billing state, not an image defect. Tag it so the run says so +# outright instead of leaving a credits problem looking like a regression. +if "402" in str(err.get("message", "")): + ident = f"{ident} :: NO_CREDITS" print(ident) ' <<< "$output" 2>/dev/null || echo "unparseable") if [[ "$kind" != *"gateway_transport_error"*"1006"* ]]; then - check "live Auto Free agent turn" "nonce returned" "command failed" + check "live agent turn" "nonce returned" "command failed" report_turn_error "$kind" return fi @@ -425,7 +517,7 @@ print(ident) done if [ -z "$output" ]; then - check "live Auto Free agent turn" "nonce returned" "command failed" + check "live agent turn" "nonce returned" "command failed" report_turn_error "$kind after 3 attempts" return fi @@ -443,9 +535,9 @@ if not any(nonce in text for text in texts): raise SystemExit("response did not contain nonce") print("nonce returned") ' "$nonce" <<< "$output" 2>&1); then - check "live Auto Free agent turn" "nonce returned" "$parsed" + check "live agent turn" "nonce returned" "$parsed" else - check "live Auto Free agent turn" "nonce returned" "unexpected response" + check "live agent turn" "nonce returned" "unexpected response" echo " details: $parsed" echo " Gateway output suppressed because provider responses can contain sensitive data." fi @@ -586,15 +678,56 @@ assert_github_gh_auth() { fi } +# Prove a leg is actually in the shape it claims BEFORE booting it. Without this, +# a silent failure of the fresh-root swap would turn the "new instance" leg into a +# second upgraded run that still passes every assertion — the leg would report +# green while covering nothing new. Checked host-side, before the container mounts +# the root. +# empty -> nothing persisted yet (a brand new instance) +# seeded -> a previous image already wrote state here (an upgrading instance) +assert_root_shape() { + local label="$1" expected="$2" + local entries + entries=$(find "$ROOTDIR" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' ') + + case "$expected" in + empty) + if [ "$entries" -eq 0 ]; then + check "$label root is genuinely fresh" "empty" "empty" + else + check "$label root is genuinely fresh" "empty" "$entries entries present" + fi + ;; + seeded) + if [ "$entries" -gt 0 ]; then + check "$label root carries baseline state" "seeded" "seeded" + else + check "$label root carries baseline state" "seeded" "empty" + fi + ;; + *) + # Without this branch an unrecognized shape (a typo, or a future label) would + # fall through asserting nothing, so the leg would pass while covering + # nothing — precisely the silent no-coverage this function exists to stop. + check "$label root shape" "$expected" "unknown shape argument" + ;; + esac +} + run_phase() { local label="$1" local image="$2" local expected_version="$3" # 1 = run the app config-write assertions (they MUTATE openclaw.json). Default 1. local mutate_config="${4:-1}" + # 1 = run the live agent turn on this leg. Default 1. + local live_turn="${5:-1}" + # empty|seeded|any — asserted before boot so the leg proves its own shape. + local root_shape="${6:-any}" echo echo "=== $label: $image ===" + [ "$root_shape" != "any" ] && assert_root_shape "$label" "$root_shape" start_container "$image" wait_for_ready "$label" wait_for_gateway_serving "$label" @@ -630,9 +763,18 @@ run_phase() { fi assert_exec_approvals_seeded "$CID" assert_github_gh_auth - echo - echo "--- live Auto Free agent turn ---" - assert_live_agent_turn + if [ "$live_turn" = "1" ]; then + echo + echo "--- live agent turn ($KILOCODE_SMOKE_MODEL) ---" + # Both warm-up gates must have passed before this: the gateway serving, and + # the calling device holding write scope. Without the second, the turn races + # device pairing and dies with a 1008 that looks like a product failure. + wait_for_device_scopes "$label" + assert_live_agent_turn + else + echo + echo "--- live agent turn: skipped for this leg ---" + fi stop_container } @@ -645,14 +787,40 @@ else fi if [ "$MODE" = "upgrade" ]; then - # Baseline: no config mutations, so its persisted root stays the pristine - # baseline-generated config the candidate then boots against. - run_phase "before-image" "$IMAGE_BEFORE" "$EXPECTED_VERSION_BEFORE" 0 - # Candidate: boots on the untouched baseline root, then exercises the config-write - # routes against the upgraded image. - run_phase "after-image persisted-root" "$IMAGE_AFTER" "$EXPECTED_VERSION_AFTER" 1 + # The gate asserts the CANDIDATE in the two shapes real instances actually take: + # an existing instance that upgraded onto its persisted root, and a brand new + # instance created after the release. Both legs run the live turn. + # + # The baseline leg only exists to GENERATE that persisted root, so it runs with + # no config mutations and NO live turn. A live turn there asserts the old version + # we are replacing, which proves nothing about the candidate, and it is the leg + # that produced repeated false failures (free-route provider rejections and + # device-pairing races) that were mistaken for candidate regressions. + run_phase "before-image (root seed only)" "$IMAGE_BEFORE" "$EXPECTED_VERSION_BEFORE" 0 0 empty + + # Shape 1 — existing user upgrading: candidate boots on the untouched + # baseline-generated root, then exercises the config-write routes. Asserts the + # root really does carry the baseline's state, or this is not an upgrade test. + run_phase "after-image persisted-root (upgraded instance)" \ + "$IMAGE_AFTER" "$EXPECTED_VERSION_AFTER" 1 1 seeded + + # Shape 2 — new signup: same candidate image on a brand new root, so nothing the + # baseline wrote can mask a first-boot problem. This is the shape a customer + # provisioned after the release gets, and nothing else in the gate covers it. + # Asserts the swap actually produced an empty root, or this silently duplicates + # the upgrade leg above. + # + # Allocate a second root rather than deleting the first. The baseline container + # wrote /root as uid 0, so on native Linux Docker `rm -rf` here would fail with + # EACCES for a non-root host user, and as a bare command under `set -e` that + # would abort the run after two legs had already passed and before the results + # summary. Both roots are tracked and removed at EXIT instead. + ROOTDIR="$(mktemp -d)" + ROOTDIRS+=("$ROOTDIR") + run_phase "after-image fresh-root (new instance)" \ + "$IMAGE_AFTER" "$EXPECTED_VERSION_AFTER" 1 1 empty else - run_phase "candidate-image" "$IMAGE_AFTER" "$EXPECTED_VERSION_AFTER" 1 + run_phase "candidate-image" "$IMAGE_AFTER" "$EXPECTED_VERSION_AFTER" 1 1 empty fi echo