[docs/agenteye] Add AgentEye product documentation

__tests__/scripts/translate-docs/mdx-translator.test.ts

@@ -4,8 +4,17 @@ import {
   rewriteInternalLinks,
   sanitizeJsxAttributes,
   stripStrayTrailingFence,
+  getEnglishMdxPages,
 } from "@/scripts/translate-docs/mdx-translator";
 
+describe("getEnglishMdxPages", () => {
+  it("excludes English-only AgentEye pages from automatic translation", () => {
+    const pages = getEnglishMdxPages();
+    expect(pages.length).toBeGreaterThan(0);
+    expect(pages.some((page) => page.includes("/agenteye/"))).toBe(false);
+  });
+});
+
 describe("rewriteInternalLinks", () => {
   it("rewrites MDX component href attributes with language prefix", () => {
     const input = `<Card title="Policies" href="/built-in-policies">`;

__tests__/scripts/translate-docs/mintlify-nav.test.ts

@@ -3,6 +3,7 @@ import { describe, it, expect } from "vitest";
 import {
   buildLanguageNav,
   generateLanguagesArray,
+  getNavigationPageReferences,
 } from "@/scripts/translate-docs/mintlify-nav";
 
 const sampleEnglishTabs = [
@@ -37,9 +38,7 @@ const sampleEnglishTabs = [
 describe("buildLanguageNav", () => {
   it("prefixes all page paths with the language code", () => {
     const nav = buildLanguageNav(sampleEnglishTabs, "es");
-    const allPages = nav.tabs.flatMap((t) =>
-      t.groups.flatMap((g) => g.pages),
-    );
+    const allPages = nav.tabs.flatMap((t) => t.groups.flatMap((g) => g.pages));
     for (const page of allPages) {
       expect(page).toMatch(/^es\//);
     }
@@ -104,11 +103,7 @@ describe("generateLanguagesArray", () => {
   });
 
   it("creates entries for each requested language", () => {
-    const langs = generateLanguagesArray(sampleEnglishTabs, [
-      "es",
-      "ja",
-      "zh",
-    ]);
+    const langs = generateLanguagesArray(sampleEnglishTabs, ["es", "ja", "zh"]);
     expect(langs).toHaveLength(4); // en + 3
     expect(langs.map((l) => l.language)).toEqual(["en", "es", "ja", "zh"]);
   });
@@ -135,3 +130,49 @@ describe("generateLanguagesArray", () => {
     }
   });
 });
+
+describe("getNavigationPageReferences", () => {
+  it("collects localized and English-only product pages", () => {
+    const references = getNavigationPageReferences({
+      products: [
+        {
+          product: "FailproofAI",
+          languages: [
+            { language: "en", tabs: sampleEnglishTabs },
+            {
+              language: "es",
+              tabs: [
+                {
+                  tab: "Documentación",
+                  groups: [{ group: "Inicio", pages: ["es/introduction"] }],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          product: "AgentEye",
+          tabs: [
+            {
+              tab: "Docs",
+              groups: [{ group: "Start", pages: ["agenteye/getting-started"] }],
+            },
+          ],
+        },
+      ],
+    });
+
+    expect(references).toContainEqual({
+      page: "introduction",
+      language: "en",
+    });
+    expect(references).toContainEqual({
+      page: "es/introduction",
+      language: "es",
+    });
+    expect(references).toContainEqual({
+      page: "agenteye/getting-started",
+      language: undefined,
+    });
+  });
+});

docs/agenteye/audits.mdx

@@ -0,0 +1,106 @@
+---
+title: "Audits — agentic improvement detection"
+description: "AgentEye Audits — agentic improvement detection documentation."
+---
+
+
+Audits are recurring jobs that mine your agent logs **across sessions** to find things worth improving. Where an alert watches one metric you already know about in near-real-time, an audit *investigates*: on a schedule you set, it runs a deterministic policy pass over the window, then turns an **AI reliability agent** loose on your sessions — the agent queries the data itself, reads suspicious transcripts, and (when it helps) runs small analysis scripts, then writes up **improvement recommendations** with the evidence behind each one.
+
+Use audits to answer "what should I fix or improve in my agents?" — and alerts to be paged the moment a specific threshold is crossed. Every improvement links to the exact sessions and queries behind it, and one click creates a prefilled alert to catch recurrences.
+
+The dashboard surface is **`/<org-slug>/audits`** (sidebar → *analyze* → *audits*), gated by the `audits:read` / `audits:write` permissions.
+
+---
+
+## How a run works
+
+Each run has two layers — a deterministic floor and an agentic investigation.
+
+### 1. The policy pass (deterministic)
+
+Before any model runs, the audit executes a small catalog of **SQL policy checks** over the window: bounded aggregate queries that flag known-bad patterns and report *how many* events / *which* sessions matched — never the matched text itself. The catalog includes:
+
+- **Secret / credential leakage** in event payloads — AWS access keys, `sk-…` API keys, PEM private keys, JWT / bearer tokens, and `KEY=…` credential assignments.
+- **Prompt-injection markers** — "ignore previous instructions", "reveal your system prompt", and similar.
+- **PII** — SSN-shaped numbers (heuristic).
+- **Tool-permission denials** and **runaway tool-call loops**.
+
+Policy hits are persisted as findings (kind `policy`) that **always surface** (they are never trimmed by the per-run cap), and they are handed to the AI agent as starting leads. Because this layer needs no model, an audit still produces its most important security signals even if the AI agent is unavailable.
+
+### 2. The agentic investigation (AI)
+
+The audit then runs an **autonomous reliability agent** (the same Claude Agent SDK service that powers the dashboard assistant, with an audit-specific prompt). Given the audit's **scope** (selected agents × environments) and **time window**, the agent:
+
+- runs read-only SQL queries against your analytics tables,
+- reads a handful of representative session transcripts,
+- optionally writes and runs short **Python scripts in a locked-down in-pod sandbox** (no network, no filesystem access, secrets scrubbed) for analysis SQL can't express — clustering errors, computing distributions, sweeping payloads it already fetched,
+- and records each well-evidenced **improvement** it finds.
+
+It works through several investigative lines — error clustering, drift vs. a baseline, goal failure in transcripts, tool misuse, quality/cost trade-offs, and coverage gaps — at the audit's **sensitivity** (low / medium / high). Every improvement **must cite evidence**: the session ids the agent actually inspected and/or the SQL it ran. The server validates that cited sessions exist and **discards any improvement with no surviving evidence**, so the agent investigates but never invents.
+
+Each improvement carries:
+
+- a **recommendation** (the concrete change to make — a prompt tweak, a tool-schema fix, a retry policy, a guardrail, more eval coverage),
+- an **expected impact** and an **effort** estimate (low / medium / high),
+- a **magnitude** — `big` (an operator should be paged), `medium` (belongs in the run report), or `small` (dashboard context),
+- a stable **fingerprint** (from the issue's category + scope, *not* this run's sessions) so the same issue is tracked run-over-run even as the evidence changes,
+- and, where a simple deterministic watcher could catch recurrence, a **suggested alert** you can create in one click.
+
+> **The AI layer is optional-but-recommended.** If no AI agent is configured for the audit pipeline, runs still execute, persist the policy findings, and honestly report "analysis unavailable" for the agentic layer rather than silently passing.
+
+### Failure modes
+
+Improvements classify into your org's durable **failure-mode catalog** (or propose a new mode). Modes give patterns a stable identity across runs and long-horizon recurrence tracking.
+
+## Triage lifecycle
+
+On a finding page (`/audits/<id>/findings/<finding-id>`):
+
+| Action | Effect |
+|---|---|
+| **acknowledge** | Keeps the finding visible but halves its priority. |
+| **resolve** | Marks it fixed. If the pattern genuinely returns later, it re-opens as **new** — so a regression is loud, not silently folded into history. |
+| **mute** / **dismiss** | Durable suppression: the pattern's fingerprint is remembered and never surfaces again, even across runs. Use mute for "known, accepted"; dismiss for "not useful". |
+| **reopen** | Clears suppression / resolution and ranks the pattern again. |
+
+Low-signal noise is controlled per audit with a per-run findings cap (`top_k`) on the agentic improvements. Policy findings bypass the cap (they are security-relevant and always shown). Anything cut by the cap is counted in the run's stats — nothing is silently dropped.
+
+## Scheduling
+
+- **Cadence** (`schedule_interval_secs`): hourly to weekly; **daily is the default**. Audits are deliberately coarser than alerts — an agentic investigation scans whole windows and runs for minutes.
+- **Window**: either a fixed rolling lookback (e.g. "each run scans the last 7 days") or **since-last-run** (the default) — each run picks up where the previous successful one ended, with a small overlap so boundary events are never missed.
+- The next run is scheduled a full interval after the previous one **completes**, so a slow run never stacks a second concurrent run of the same audit.
+- **Run now** on the audit page makes it due immediately.
+
+## Model selection
+
+When creating an audit you can pick which model the investigation uses, from the **list of models your operator has configured** for the agent service. With a single model configured, the picker shows it as a caption; with several, you choose. Leaving it unset uses the configured default.
+
+## Notifications
+
+When a run surfaces **new** findings, the audit notifies your organization's configured channels — the same `alerts.enabled_channels` gate and settings the alerts pipeline uses:
+
+- **Slack** — a summary of the significant (`big`) new items with a deep link.
+- **Email** — a designed **audit report** listing the new improvements (top severity, per-item recommendations, deep link), sent when the audit has an **email** channel attached and there is at least one new finding.
+
+Recurring-but-known findings do not re-notify.
+
+## Configuration reference
+
+Audit definitions are managed in the dashboard (`/audits/new`) or via the API. Per-audit settings include the schedule cadence and window, the scope (`{"environments": [...], "agent_ids": [...]}`), the sensitivity (`low` / `medium` / `high`), the notification channels, the per-run findings cap (`top_k`), and the model (via `llm_budget.model`). Operator-level server settings (timeouts, sandbox, the agent-service URL) are documented in [deployment.md](/agenteye/deployment).
+
+## API
+
+All endpoints are org-scoped and follow the standard bearer-key auth (see [api-keys.md](/agenteye/api-keys)).
+
+| Endpoint | Permission | Purpose |
+|---|---|---|
+| `GET /audits` · `POST /audits` | `audits:read` / `audits:write` | List / create audit definitions. |
+| `GET` / `PUT` / `DELETE /audits/:id` | read / write / write | Inspect, edit, delete an audit. |
+| `POST /audits/:id/run` | `audits:write` | Make the audit due immediately. |
+| `GET /audits/:id/runs` | `audits:read` | Run history (window, status, stats, finding counts). |
+| `GET /audits/findings` | `audits:read` | Org-wide findings, filterable by `audit_id`, `status`; sorted by priority. |
+| `GET /audits/findings/:fid` | `audits:read` | Full finding detail (recommendation, evidence, priority). |
+| `POST /audits/findings/:fid/status` | `audits:write` | Triage: `{"action": "ack" \| "mute" \| "dismiss" \| "resolve" \| "reopen" \| "assign"}`. |
+
+For "audit ran but found nothing", "the code sandbox is disabled", and "audit email not delivered", see [troubleshooting.md](/agenteye/troubleshooting#audits).

docs/agenteye/cli-recipes.mdx

@@ -0,0 +1,170 @@
+---
+title: "CLI recipes for agents"
+description: "AgentEye CLI recipes for agents documentation."
+---
+
+
+Pull session, event, and evaluation data (and trigger re-evaluations) straight from a script or coding agent, with clean JSON on stdout that pipes directly into `jq`. These recipes turn AgentEye's observability data into something a terminal user or an AI coding agent (Claude Code, Cursor) can query and automate, without clicking through the dashboard.
+
+The patterns below are copy-paste ready for the AgentEye CLI (`agenteye`). For installation, authentication, and the full option list see [CLI](/agenteye/cli); run `agenteye -h` or `agenteye <command> -h` for the built-in help.
+
+## Golden rules
+
+1. **Global options go *before* the command.** `agenteye --json sessions` is correct; `agenteye sessions --json` is not. The globals are `--json`, `--base-url`, `--org`, `--token`, `--insecure`/`--secure`, `--timeout`, `--quiet`, `--no-color`.
+2. **Pass `--json` whenever you parse output.** Data goes to **stdout** as JSON; human status and errors go to **stderr**, so stdout stays clean to pipe into `jq`.
+3. **Branch on the exit code**, not on stderr text: `0` ok · `2` bad arguments · `3` cannot reach the dashboard · `4` not logged in or expired · `5` missing permission.
+4. **Discover with `-h`.** Every command documents its filters, value formats, and JSON shape.
+
+## One-time setup
+
+```bash
+export AGENTEYE_DASHBOARD_URL=https://agenteye.example.com   # so you don't repeat --base-url
+agenteye login --email you@example.com                       # paste the emailed code; valid ~24h
+```
+
+## Confirm auth before doing work
+
+`whoami` never errors on a missing or expired session; it reports `logged_in:false` instead, so an agent can probe auth state safely. (It can still exit non-zero if no base URL is set or the dashboard is unreachable.)
+
+```bash
+if [ "$(agenteye --json whoami | jq -r .logged_in)" != "true" ]; then
+  echo "Not authenticated. Run: agenteye login" >&2; exit 1
+fi
+```
+
+## Find failing or low-scoring sessions
+
+```bash
+# sessions in the last 24h whose evaluation errored
+agenteye --json sessions --since 24h --status error | jq -r '.sessions[].session_id'
+
+# evaluations scoring <= 0.5 on helpfulness, for one agent
+agenteye --json evals --agent-id checkout-bot --score helpfulness:..0.5 \
+  | jq '.evaluations[] | {session_id, scores}'
+```
+
+Score filtering lives on **`evals`**, not `sessions`. `--score KEY:MIN..MAX` is repeatable and AND-combined; either bound is optional (`..0.5` means ≤ 0.5, `0.9..` means ≥ 0.9). You can pass up to 20 score filters per request; more returns HTTP 400. `sessions` shares the `--env`, `--status`, `--agent-id`, `--session-id`, and time-range filters with `evals`, but has no `--score`.
+
+## Read one session end-to-end
+
+There is no single `session show` command — combine the event trail with the session's evaluation:
+
+```bash
+# the session's latest evaluation (status + scores)
+agenteye --json evals --session-id run-001 | jq '.evaluations[0] | {status, scores}'
+
+# every event in the run (raise --limit for a full sweep)
+agenteye --json events --session-id run-001 --all --limit 1000 | jq '.events[] | {ts, event_type}'
+
+# just the tool calls in a session
+agenteye --json events --session-id run-001 --event-type tool_use,tool_result --all \
+  | jq '.events[].payload'
+```
+
+## Fetch everything (pagination)
+
+Results are newest-first and cursor-paginated.
+
+```bash
+# one shot: fetch up to 500 rows in 200-row pages
+agenteye --json events --session-id run-001 --limit 500 --all > events.json
+
+# manual paging: feed next_cursor back in
+page=$(agenteye --json events --limit 100)
+cursor=$(echo "$page" | jq -r '.next_cursor // empty')
+[ -n "$cursor" ] && agenteye --json events --limit 100 --cursor "$cursor"
+```
+
+## Slim the output with --fields
+
+Restrict the keys (in both the table and `--json`) to reduce what an agent must read.
+
+```bash
+agenteye --json sessions --since 7d --fields session_id,status,scores | jq -c '.sessions[]'
+agenteye --json events --session-id run-001 --fields ts,event_type --all
+```
+
+Unknown field names are rejected (exit `2`) with the valid list, a cheap way to discover field names.
+
+## Discover valid filter values
+
+```bash
+agenteye --json list envs | jq -r '.values[]'           # values for --env
+agenteye --json list tools | jq -r '.values[]'          # tool names; also agents, models, event_types, …
+agenteye --json list score_filters | jq -r '.values[]'  # valid KEY for --score KEY:MIN..MAX
+```
+
+## Pick your org (multi-tenant)
+
+If you belong to more than one org, choose the active tenant at login (it's saved):
+
+```bash
+agenteye login --org acme --email you@corp.com   # set the tenant in the same step as login
+agenteye --json orgs list | jq -r '.orgs[].org_slug'
+agenteye --org globex --json sessions --since 24h   # override for one command
+```
+
+A multi-org login without `--org` exits non-zero and prints the orgs to choose from.
+
+## Provision an API key for the SDK/collector
+
+```bash
+# the secret is printed ONCE — with --json it's the .key field
+key=$(agenteye --json keys create ci-bot --add events:read.add | jq -r '.key')
+agenteye keys regenerate ci-bot --yes    # rotate; agenteye keys disable ci-bot --yes to revoke
+```
+
+## Run a saved or ad-hoc query
+
+```bash
+agenteye --json query run --sql "select count(*) from analytics.events" | jq '.rows'
+agenteye --json query run errs --arg prod | jq '.rows'   # a saved query + a positional $1
+```
+
+## Triage an incident non-interactively
+
+```bash
+id=$(agenteye --json incidents list --state firing | jq -r '.incidents[0].id')
+agenteye incidents ack "$id"
+agenteye incidents assign "$id" --assignee you@corp.com
+agenteye incidents resolve "$id" --yes
+```
+
+> Mutations auto-skip their confirmation prompt under `--json` or when stdin isn't a TTY, so agents never hang; pass `--yes`/`-y` to skip it explicitly elsewhere.
+
+## Exit-code handling in a script
+
+```bash
+out=$(agenteye --json sessions --since 1h) || code=$?
+case "${code:-0}" in
+  0) echo "$out" | jq '.sessions | length' ;;
+  4) echo "Session expired - run 'agenteye login'." >&2 ;;
+  5) echo "Missing permission (ask an admin for evaluations:read)." >&2 ;;
+  3) echo "Dashboard unreachable - check the URL." >&2 ;;
+  *) echo "Unexpected error (exit ${code})." >&2 ;;
+esac
+```
+
+## JSON output shapes
+
+| Command | stdout JSON (with `--json`) |
+|---|---|
+| `whoami` | `{"logged_in": true, "id", "email", "is_instance_admin", "active_org", "permissions": [...], "memberships": [...]}` or `{"logged_in": false}` |
+| `orgs list` | `{"active_org", "orgs": [{"org_slug","org_name","permission_set","permissions"}]}` |
+| `events` | `{"events": [...], "next_cursor": <cursor or null>}` |
+| `evals` | `{"evaluations": [...], "next_cursor": <cursor or null>}` |
+| `sessions` | `{"sessions": [...], "next_cursor": <cursor or null>}` |
+| `errors` | `{"errors": [...], "next_cursor": <cursor or null>}` |
+| `list <kind>` | `{"kind", "values": [...]}` |
+| `keys list` / `keys create` | `{"keys": [...]}` / `{id, name, permissions, created_at, key}` (`key` shown once) |
+| `query run` | `{columns: [{name,type}], rows: [[...]], truncated, elapsed_ms}` |
+| `users list` / `settings list` | `{"users": [...]}` / `{"settings": [...]}` |
+| `alerts list` / `incidents list` | `{"alerts": [...]}` / `{"incidents": [...]}` |
+| create/update/delete (any) | the resource object, or `{"deleted": true, "id"}` for deletes |
+| failure (any, with `--json`) | `{"error": "...", "exit_code": <n>, "status"?: <http>, "hint"?: "..."}` on stdout |
+
+- Each **event** item (`events`): `id, session_id, agent_id, event_type, ts, payload, environment`.
+- Each **evaluation** item (`evals`): `id, session_id, agent_id, environment, status, scores, reasoning, summary, error, attempt_count, duration_ms, completed_at, created_at`.
+- Each **session** item (`sessions`): `session_id, agent_id, environment, status, scores, event_count, started_at, last_event_at, first_event_id, last_event_id, latest_evaluation`.
+
+Each command's `--fields` accepts exactly its own item's field names — the set differs between `sessions` and `evals`, so a name valid for one may be rejected by the other.