From dca50c6c8ce618be92de5c6f7799913999a62ba8 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 17 Jul 2026 11:37:03 -0700 Subject: [PATCH 1/3] refactor(e2e): remove bob_the_builder; drive remediation from a Grafana alert Delete tests/e2e/bob_the_builder.py and unwire its pytest_sessionfinish remediation call. bob was an opt-in (E2E_DEVIN_REMEDIATION=1) hook that, on a red run, collected the failing tests and called the Devin MCP devin_session_create tool through the proxy's own /mcp-rest gateway to open fix PRs. It was a circular dependency: the suite goes red most often when the proxy is unhealthy, which is exactly when the call back through /mcp-rest also failed, and it swallowed every exception so the failure was silent. It also put remediation orchestration inside the test runner and hand-rolled a sha256 node-id dedup that Grafana already does. Replace it with alerting-as-code under tests/e2e/grafana/alerting/: a Grafana rule on the E2E_RESULT failure signal the harness already ships to Loki, routed to a Slack contact point where Devin listens. Because the alert reads the already-shipped Loki signal instead of calling back through the proxy, it keeps working when the runner or proxy is degraded, and Grafana owns the grouping and repeat suppression. Resolves LIT-4551 --- tests/e2e/bob_the_builder.py | 247 ------------------ tests/e2e/conftest.py | 7 - tests/e2e/grafana/alerting/README.md | 88 +++++++ tests/e2e/grafana/alerting/alert_rules.yaml | 92 +++++++ .../e2e/grafana/alerting/contact_points.yaml | 31 +++ 5 files changed, 211 insertions(+), 254 deletions(-) delete mode 100644 tests/e2e/bob_the_builder.py create mode 100644 tests/e2e/grafana/alerting/README.md create mode 100644 tests/e2e/grafana/alerting/alert_rules.yaml create mode 100644 tests/e2e/grafana/alerting/contact_points.yaml diff --git a/tests/e2e/bob_the_builder.py b/tests/e2e/bob_the_builder.py deleted file mode 100644 index 18aff2edc98..00000000000 --- a/tests/e2e/bob_the_builder.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Bob the builder: on a red e2e run, ask Devin to fix the failing tests. - -Wired as a ``pytest_sessionfinish`` step (see ``conftest.py``). When the run went -red and remediation is enabled, it hands the failing tests plus their captured -tracebacks to Devin *through the LiteLLM proxy's own MCP gateway* -- the same -gateway + master key the suite already uses -- so Devin files a Linear ticket per -failure and opens fix PRs. Nothing new ships in the runner pod: the proxy already -registers the ``devin`` MCP server and holds ``DEVIN_API_KEY``, injecting it -upstream, so this process only needs the proxy key it always has. - -Opt-in via ``E2E_DEVIN_REMEDIATION=1`` so a normal local ``pytest tests/e2e`` run -never spawns a Devin session. ``DEVIN_DRY_RUN=1`` prints the prompt it would send -and makes no call. Everything is best-effort: any error here is logged and -swallowed so the run's exit status still reflects the tests, not remediation. -""" - -from __future__ import annotations - -import hashlib -import os -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Protocol, cast - -import pytest -from pydantic import BaseModel, ConfigDict - -from e2e_config import MASTER_KEY, PROXY_BASE_URL -from e2e_http import Success -from transport import HttpTransport - -REMEDIATION_ENV = "E2E_DEVIN_REMEDIATION" -_LIST_PATH = "/mcp-rest/tools/list" -_CALL_PATH = "/mcp-rest/tools/call" - - -@dataclass(frozen=True, slots=True) -class Failure: - """One failed test: its pytest node id and the captured failure text.""" - - nodeid: str - detail: str - - -@dataclass(frozen=True, slots=True) -class Config: - server: str - create_tool: str - linear_team: str - target_repo: str - target_ref: str - max_failures: int - max_detail_chars: int - tags: tuple[str, ...] - dry_run: bool - - -class _NoParams(BaseModel): - pass - - -class _McpToolInfo(BaseModel): - model_config = ConfigDict(extra="allow") - server_name: str | None = None - alias: str | None = None - - -class _McpTool(BaseModel): - model_config = ConfigDict(extra="allow") - name: str - mcp_info: _McpToolInfo | None = None - - -class _McpToolsList(BaseModel): - model_config = ConfigDict(extra="allow") - tools: tuple[_McpTool, ...] = () - - -class _DevinSessionArgs(BaseModel): - prompt: str - title: str - tags: list[str] - - -class _ToolCallBody(BaseModel): - name: str - arguments: _DevinSessionArgs - - -class _ToolCallResult(BaseModel): - model_config = ConfigDict(extra="allow") - - -class _Report(Protocol): - @property - def nodeid(self) -> str: ... - - @property - def longreprtext(self) -> str: ... - - -class _TerminalReporter(Protocol): - stats: Mapping[str, Sequence[_Report]] - - -def _env(name: str, default: str) -> str: - value = os.environ.get(name, "").strip() - return value or default - - -def load_config() -> Config: - raw_tags = _env("DEVIN_TAGS", "e2e,stage") - return Config( - server=_env("DEVIN_MCP_SERVER", "devin"), - create_tool=_env("DEVIN_SESSION_TOOL", "devin_session_create"), - linear_team=_env("DEVIN_LINEAR_TEAM", "LIT"), - target_repo=_env("DEVIN_TARGET_REPO", "BerriAI/litellm"), - target_ref=_env("DEVIN_TARGET_REF", "litellm_internal_staging"), - max_failures=int(_env("DEVIN_MAX_FAILURES", "50")), - max_detail_chars=int(_env("DEVIN_MAX_DETAIL_CHARS", "3000")), - tags=tuple(t.strip() for t in raw_tags.split(",") if t.strip()), - dry_run=_env("DEVIN_DRY_RUN", "0") == "1", - ) - - -def collect_failures(session: pytest.Session, max_detail_chars: int) -> tuple[Failure, ...]: - """Pull the failed and errored tests (with their tracebacks) off the run's - terminal reporter. Returns empty when nothing failed or the reporter is - absent (e.g. a skipped, proxy-less session).""" - plugin: object = session.config.pluginmanager.getplugin("terminalreporter") - if plugin is None: - return () - reporter = cast(_TerminalReporter, plugin) - reports = (*reporter.stats.get("failed", ()), *reporter.stats.get("error", ())) - return tuple( - Failure(nodeid=r.nodeid, detail=r.longreprtext.strip()[-max_detail_chars:]) for r in reports - ) - - -def dedup_tag(failures: tuple[Failure, ...]) -> str: - """Stable short tag identifying this exact set of failing tests, so repeated - nightly runs on the same failures reference one body of work.""" - joined = "\n".join(sorted(f.nodeid for f in failures)) - return "e2e-fail-" + hashlib.sha256(joined.encode()).hexdigest()[:12] - - -def _revision() -> str: - for candidate in (Path(__file__).parent / ".litellm-revision", Path("/app/e2e/.litellm-revision")): - try: - return candidate.read_text(encoding="utf-8").strip() - except OSError: - continue - return _env("E2E_REVISION", "unknown") - - -def build_prompt(cfg: Config, failures: tuple[Failure, ...], tag: str) -> str: - shown = failures[: cfg.max_failures] - header = ( - f"The LiteLLM end-to-end suite failed on the " - f"{_env('E2E_ENVIRONMENT', 'stage')} proxy. Source repo {cfg.target_repo} " - f"at revision {_revision()} (branch {cfg.target_ref}). {len(failures)} " - f"test(s) failed" - + (f"; the first {len(shown)} are shown" if len(shown) < len(failures) else "") - + ".\n\n" - ) - task = ( - "For each failing test below:\n" - f"1. Open a Linear ticket under the {cfg.linear_team} team describing the " - "failure (test id, the assertion/error, likely cause), unless an open " - "ticket for that same test already exists -- do not create duplicates.\n" - f"2. Fix it in {cfg.target_repo}, branching off {cfg.target_ref} and " - "following the repo's CONTRIBUTING and CLAUDE.md conventions (meaningful " - "regression coverage, conventional commits, run the suite locally), then " - "open a PR that references the Linear ticket.\n" - "3. Prefer one focused PR per failing test; if several share a root cause, " - "group them and say so.\n" - f"Before starting, search existing sessions/PRs tagged '{tag}' or " - "referencing these test ids and continue that work instead of restarting.\n\n" - "Failing tests and their captured output:\n" - ) - blocks = [f"### {i}. {f.nodeid}\n```\n{f.detail}\n```\n" for i, f in enumerate(shown, start=1)] - return header + task + "\n".join(blocks) - - -def _resolve_tool_name(transport: HttpTransport, cfg: Config) -> str | None: - """Find Devin's create-session tool on the gateway. The proxy prefixes tools - with the server alias, so match by suffix and (when present) the owning - server.""" - result = transport.get( - _LIST_PATH, headers=transport.master, params=_NoParams(), response_type=_McpToolsList - ) - if not isinstance(result, Success): - print(f"bob_the_builder: could not list gateway MCP tools: {result}") - return None - for tool in result.data.tools: - owner = tool.mcp_info.server_name or tool.mcp_info.alias if tool.mcp_info else None - if (owner is None or owner == cfg.server) and ( - tool.name == cfg.create_tool or tool.name.endswith(cfg.create_tool) - ): - return tool.name - print( - f"bob_the_builder: no '{cfg.create_tool}' tool for server '{cfg.server}' on the gateway; " - f"saw {[t.name for t in result.data.tools]}" - ) - return None - - -def remediate(session: pytest.Session) -> None: - """Entry point called from ``pytest_sessionfinish``. No-op unless remediation - is enabled and the run actually had failures.""" - if os.environ.get(REMEDIATION_ENV) != "1": - return - cfg = load_config() - failures = collect_failures(session, cfg.max_detail_chars) - if not failures: - return - - tag = dedup_tag(failures) - title = f"Fix {len(failures)} failing LiteLLM e2e test(s) [{tag}]" - prompt = build_prompt(cfg, failures, tag) - args = _DevinSessionArgs(prompt=prompt, title=title, tags=[*cfg.tags, tag]) - - if cfg.dry_run: - print("bob_the_builder: DRY RUN -- would create a Devin session:") - print(f" server : {cfg.server}\n tool : {cfg.create_tool}\n title : {title}") - print(f" tags : {args.tags}\n---- prompt ----\n{prompt}") - return - - try: - transport = HttpTransport(base_url=PROXY_BASE_URL, master_key=MASTER_KEY) - tool_name = _resolve_tool_name(transport, cfg) - if tool_name is None: - return - result = transport.post( - _CALL_PATH, - headers=transport.master, - json=_ToolCallBody(name=tool_name, arguments=args), - response_type=_ToolCallResult, - ) - if isinstance(result, Success): - print(f"bob_the_builder: created Devin session for {len(failures)} failure(s) [{tag}]") - print(result.data.model_dump_json()) - else: - print(f"bob_the_builder: Devin session call failed: {result}") - except Exception as exc: # noqa: BLE001 - remediation must never fail the run - print(f"bob_the_builder: remediation error (ignored): {exc}") diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 3aec104c861..ffb01942037 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -132,13 +132,6 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if spend_dir in sys.path: sys.path.remove(spend_dir) - try: - from bob_the_builder import remediate - - remediate(session) - except Exception as exc: # noqa: BLE001 - remediation is best-effort - print(f"devin remediation best-effort failed: {exc}") - @pytest.fixture def resources(client: GatewayProvider) -> Iterator[ResourceManager]: diff --git a/tests/e2e/grafana/alerting/README.md b/tests/e2e/grafana/alerting/README.md new file mode 100644 index 00000000000..095bd3f2098 --- /dev/null +++ b/tests/e2e/grafana/alerting/README.md @@ -0,0 +1,88 @@ +# Grafana: e2e failure alert -> Slack (Devin remediation) + +This directory holds the alerting-as-code that drives remediation off the e2e +suite going red. It replaces the old in-runner `bob_the_builder.py` +`pytest_sessionfinish` hook, which called Devin through the proxy's own +`/mcp-rest` gateway to open fix PRs. + +## Why this instead of an in-runner hook + +The old hook had three problems that this design removes. + +It was a circular dependency. The suite goes red most often when the proxy is +unhealthy, which is exactly when the hook's call back to +`PROXY_BASE_URL/mcp-rest` also failed. It swallowed every exception, so a +degraded proxy produced no remediation and no signal at all. This alert instead +evaluates the `E2E_RESULT` lines the harness already ships to Loki, so it fires +even when the runner or proxy is degraded. + +It put remediation orchestration inside the test runner. The runner's job is to +run tests and report results; routing a red result to whoever fixes it is +alerting infrastructure. Grafana Alerting is the standard place for that. + +It hand-rolled dedup with a sha256 over the failing node ids. Grafana already +groups and suppresses repeat notifications. The rule's `notification_settings` +group by `package` and re-notify only every `repeat_interval`, so one ongoing +failing package produces one Slack thread, not a new call per run. + +## The signal + +`tests/e2e/conftest.py` (`pytest_runtest_makereport`) prints one logfmt line per +finished test via `e2e_result_reporter.py`: + +``` +E2E_RESULT package=logging file=test_langfuse_e2e.py outcome=failed duration_ms=1500 node_id="logging/..." covers=cell.id +``` + +Loki ingests these under `{service_name="litellm-e2e"}`. See +`../status_history_panels.md` for the status-history dashboard that reads the +same signal. + +## What the rule does + +`alert_rules.yaml` defines one Grafana-managed rule, `litellm-e2e-suite-failure`, +evaluated every 1m over a 5m lookback against the Loki datasource: + +```logql +sum by (package) ( + count_over_time({service_name="litellm-e2e"} |= "E2E_RESULT" | logfmt | outcome=~"failed|error" [5m]) +) +``` + +Query `A` counts failed/errored lines per package over the window, `B` reduces to +the last value per series, and `C` thresholds `> 0`. A package with any failure +in the window fires an instance labelled with that `package`. The instance +annotations carry a ready-to-paste Loki drill-down query for the exact node ids. +The 1m interval sits well inside the 5m lookback, so each failing line is seen by +several consecutive evaluations and a boundary or late-ingested line is never +missed. + +`contact_points.yaml` defines the `devin-e2e-remediation` Slack contact point. +The rule routes straight to it through `notification_settings.receiver` (Grafana +simplified routing, v11+), so no change to the org's root notification policy +tree is needed, and none is provisioned here to avoid clobbering it. On older +Grafana, drop `notification_settings` from the rule and add a matching child +route (`service = litellm-e2e` -> `devin-e2e-remediation`) under the existing +root policy instead. Devin watches the target Slack channel and opens the tickets +and fix PRs that `bob_the_builder` used to open directly. + +## Applying it + +These are standard Grafana file-provisioning documents. Point Grafana at this +directory (or copy the two YAML files into the provisioning path) so both load: + +``` +provisioning/alerting/contact_points.yaml +provisioning/alerting/alert_rules.yaml +``` + +Two values are environment-supplied so no secret or environment-specific id +lands in git. Grafana expands `$__env{VAR}` in provisioning files at load time: + +- `SLACK_E2E_WEBHOOK_URL`: incoming-webhook URL for the channel Devin listens on +- `LOKI_DATASOURCE_UID`: uid of the Loki datasource that holds `E2E_RESULT` + +If you manage the Grafana Cloud stack (`berriai.grafana.net`) with Terraform +instead, the same three objects map to `grafana_contact_point` and +`grafana_rule_group`; keep this YAML as the reference for the query, grouping, +and routing. diff --git a/tests/e2e/grafana/alerting/alert_rules.yaml b/tests/e2e/grafana/alerting/alert_rules.yaml new file mode 100644 index 00000000000..802cb2e034d --- /dev/null +++ b/tests/e2e/grafana/alerting/alert_rules.yaml @@ -0,0 +1,92 @@ +# Grafana Alerting rule: fire when the e2e suite reports failed/errored tests. +# +# Grafana provisioning (apiVersion 1). This evaluates the E2E_RESULT signal the +# harness already ships to Loki (one logfmt line per finished test, emitted from +# pytest_runtest_makereport in tests/e2e/conftest.py via e2e_result_reporter.py), +# so it keeps working when the proxy or the runner is degraded -- exactly the +# case where an in-runner remediation call back through the proxy would fail. +# +# Grouping and repeat suppression on the rule's notification_settings replace the +# old hand-rolled sha256 node-id dedup: one Slack notification per failing +# package per window, re-sent only every repeat_interval while it stays red. +# +# Set LOKI_DATASOURCE_UID to your Loki datasource UID (Grafana expands $__env{} +# in provisioning files; substitute the literal UID if your Grafana predates it). +apiVersion: 1 + +groups: + - orgId: 1 + name: litellm-e2e + folder: LiteLLM E2E + # Evaluate every 1m over a 5m lookback so windows overlap 5x. A range that + # equals the evaluation interval tiles with no overlap, so a line ingested + # late or near a boundary (Loki ingestion lag, clock skew) can fall between + # two windows and never alert. The overlap closes that gap; grouping and + # repeat_interval on notification_settings keep it to one notification. + interval: 1m + rules: + - uid: litellm-e2e-suite-failure + title: LiteLLM e2e suite failure + condition: C + for: 0m + noDataState: OK + execErrState: Error + labels: + service: litellm-e2e + severity: ticket + annotations: + summary: 'e2e package "{{ $labels.package }}" has failing tests' + description: >- + One or more tests in the "{{ $labels.package }}" package under + tests/e2e/ failed or errored in the last evaluation window. + drilldown_query: >- + {service_name="litellm-e2e"} |= "E2E_RESULT" | logfmt | + package="{{ $labels.package }}" | outcome=~"failed|error" + notification_settings: + receiver: devin-e2e-remediation + group_by: + - alertname + - package + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: $__env{LOKI_DATASOURCE_UID} + model: + refId: A + datasource: + type: loki + uid: $__env{LOKI_DATASOURCE_UID} + editorMode: code + queryType: instant + expr: >- + sum by (package) (count_over_time({service_name="litellm-e2e"} + |= "E2E_RESULT" | logfmt | outcome=~"failed|error" [5m])) + - refId: B + datasourceUid: __expr__ + model: + refId: B + type: reduce + reducer: last + expression: A + datasource: + type: __expr__ + uid: __expr__ + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: + - 0 + datasource: + type: __expr__ + uid: __expr__ diff --git a/tests/e2e/grafana/alerting/contact_points.yaml b/tests/e2e/grafana/alerting/contact_points.yaml new file mode 100644 index 00000000000..04fd36a7d37 --- /dev/null +++ b/tests/e2e/grafana/alerting/contact_points.yaml @@ -0,0 +1,31 @@ +# Grafana Alerting contact point: Slack channel where Devin listens. +# +# Grafana provisioning (apiVersion 1). Import this alongside alert_rules.yaml. +# Provisioning here is additive per-name: it creates or updates the +# "devin-e2e-remediation" contact point without touching other receivers. +# +# The Slack webhook is a secret and must not live in git. Grafana expands +# $__env{VAR} in provisioning files at load time, so supply it through the +# environment (SLACK_E2E_WEBHOOK_URL). If your Grafana predates env expansion, +# substitute the literal incoming-webhook URL at apply time instead. +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: devin-e2e-remediation + receivers: + - uid: devin-e2e-slack + type: slack + # Only firing alerts (an actual red suite) reach the channel Devin acts + # on; resolved notifications are suppressed so Devin never opens work off + # a recovery message. + disableResolveMessage: true + settings: + url: $__env{SLACK_E2E_WEBHOOK_URL} + title: 'LiteLLM e2e failures ({{ len .Alerts.Firing }} firing)' + text: | + {{ len .Alerts.Firing }} LiteLLM e2e package(s) with failing tests: + {{ range .Alerts.Firing }} + - *{{ .Labels.package }}*: {{ .Annotations.summary }} + drill-down (Loki): {{ .Annotations.drilldown_query }} + {{ end }} From cbba2b0238616364c23b44c213913e477dad4323 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 17 Jul 2026 12:26:15 -0700 Subject: [PATCH 2/3] docs(e2e): note the alert selector must match the scrape label The rule's {service_name="litellm-e2e"} stream selector matches nothing if the scrape does not attach that label, so the alert would silently never fire; point at the pod-based fallback selector the status-history panels already use. --- tests/e2e/grafana/alerting/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/grafana/alerting/README.md b/tests/e2e/grafana/alerting/README.md index 095bd3f2098..de5760c1e42 100644 --- a/tests/e2e/grafana/alerting/README.md +++ b/tests/e2e/grafana/alerting/README.md @@ -57,6 +57,14 @@ The 1m interval sits well inside the 5m lookback, so each failing line is seen b several consecutive evaluations and a boundary or late-ingested line is never missed. +The `{service_name="litellm-e2e"}` stream selector has to match the label the +scrape actually attaches to the e2e runner's stdout, or the query matches nothing +and the rule silently never fires. If your scrape does not set `service_name`, +swap the selector for the pod-based fallback the status-history panels already +use (`{cluster="berrie-litellm-stage", pod=~"litellm-e2e-.+"}`, see +`../status_history_panels.md`) in both `alert_rules.yaml` and the drill-down +annotation. + `contact_points.yaml` defines the `devin-e2e-remediation` Slack contact point. The rule routes straight to it through `notification_settings.receiver` (Grafana simplified routing, v11+), so no change to the org's root notification policy From 82d878e20beed4575bb0058d9334e3a1f215b99f Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 17 Jul 2026 20:31:10 +0000 Subject: [PATCH 3/3] refactor(e2e): drop in-repo Grafana alerting provisioning Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/grafana/alerting/README.md | 96 ------------------- tests/e2e/grafana/alerting/alert_rules.yaml | 92 ------------------ .../e2e/grafana/alerting/contact_points.yaml | 31 ------ 3 files changed, 219 deletions(-) delete mode 100644 tests/e2e/grafana/alerting/README.md delete mode 100644 tests/e2e/grafana/alerting/alert_rules.yaml delete mode 100644 tests/e2e/grafana/alerting/contact_points.yaml diff --git a/tests/e2e/grafana/alerting/README.md b/tests/e2e/grafana/alerting/README.md deleted file mode 100644 index de5760c1e42..00000000000 --- a/tests/e2e/grafana/alerting/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Grafana: e2e failure alert -> Slack (Devin remediation) - -This directory holds the alerting-as-code that drives remediation off the e2e -suite going red. It replaces the old in-runner `bob_the_builder.py` -`pytest_sessionfinish` hook, which called Devin through the proxy's own -`/mcp-rest` gateway to open fix PRs. - -## Why this instead of an in-runner hook - -The old hook had three problems that this design removes. - -It was a circular dependency. The suite goes red most often when the proxy is -unhealthy, which is exactly when the hook's call back to -`PROXY_BASE_URL/mcp-rest` also failed. It swallowed every exception, so a -degraded proxy produced no remediation and no signal at all. This alert instead -evaluates the `E2E_RESULT` lines the harness already ships to Loki, so it fires -even when the runner or proxy is degraded. - -It put remediation orchestration inside the test runner. The runner's job is to -run tests and report results; routing a red result to whoever fixes it is -alerting infrastructure. Grafana Alerting is the standard place for that. - -It hand-rolled dedup with a sha256 over the failing node ids. Grafana already -groups and suppresses repeat notifications. The rule's `notification_settings` -group by `package` and re-notify only every `repeat_interval`, so one ongoing -failing package produces one Slack thread, not a new call per run. - -## The signal - -`tests/e2e/conftest.py` (`pytest_runtest_makereport`) prints one logfmt line per -finished test via `e2e_result_reporter.py`: - -``` -E2E_RESULT package=logging file=test_langfuse_e2e.py outcome=failed duration_ms=1500 node_id="logging/..." covers=cell.id -``` - -Loki ingests these under `{service_name="litellm-e2e"}`. See -`../status_history_panels.md` for the status-history dashboard that reads the -same signal. - -## What the rule does - -`alert_rules.yaml` defines one Grafana-managed rule, `litellm-e2e-suite-failure`, -evaluated every 1m over a 5m lookback against the Loki datasource: - -```logql -sum by (package) ( - count_over_time({service_name="litellm-e2e"} |= "E2E_RESULT" | logfmt | outcome=~"failed|error" [5m]) -) -``` - -Query `A` counts failed/errored lines per package over the window, `B` reduces to -the last value per series, and `C` thresholds `> 0`. A package with any failure -in the window fires an instance labelled with that `package`. The instance -annotations carry a ready-to-paste Loki drill-down query for the exact node ids. -The 1m interval sits well inside the 5m lookback, so each failing line is seen by -several consecutive evaluations and a boundary or late-ingested line is never -missed. - -The `{service_name="litellm-e2e"}` stream selector has to match the label the -scrape actually attaches to the e2e runner's stdout, or the query matches nothing -and the rule silently never fires. If your scrape does not set `service_name`, -swap the selector for the pod-based fallback the status-history panels already -use (`{cluster="berrie-litellm-stage", pod=~"litellm-e2e-.+"}`, see -`../status_history_panels.md`) in both `alert_rules.yaml` and the drill-down -annotation. - -`contact_points.yaml` defines the `devin-e2e-remediation` Slack contact point. -The rule routes straight to it through `notification_settings.receiver` (Grafana -simplified routing, v11+), so no change to the org's root notification policy -tree is needed, and none is provisioned here to avoid clobbering it. On older -Grafana, drop `notification_settings` from the rule and add a matching child -route (`service = litellm-e2e` -> `devin-e2e-remediation`) under the existing -root policy instead. Devin watches the target Slack channel and opens the tickets -and fix PRs that `bob_the_builder` used to open directly. - -## Applying it - -These are standard Grafana file-provisioning documents. Point Grafana at this -directory (or copy the two YAML files into the provisioning path) so both load: - -``` -provisioning/alerting/contact_points.yaml -provisioning/alerting/alert_rules.yaml -``` - -Two values are environment-supplied so no secret or environment-specific id -lands in git. Grafana expands `$__env{VAR}` in provisioning files at load time: - -- `SLACK_E2E_WEBHOOK_URL`: incoming-webhook URL for the channel Devin listens on -- `LOKI_DATASOURCE_UID`: uid of the Loki datasource that holds `E2E_RESULT` - -If you manage the Grafana Cloud stack (`berriai.grafana.net`) with Terraform -instead, the same three objects map to `grafana_contact_point` and -`grafana_rule_group`; keep this YAML as the reference for the query, grouping, -and routing. diff --git a/tests/e2e/grafana/alerting/alert_rules.yaml b/tests/e2e/grafana/alerting/alert_rules.yaml deleted file mode 100644 index 802cb2e034d..00000000000 --- a/tests/e2e/grafana/alerting/alert_rules.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# Grafana Alerting rule: fire when the e2e suite reports failed/errored tests. -# -# Grafana provisioning (apiVersion 1). This evaluates the E2E_RESULT signal the -# harness already ships to Loki (one logfmt line per finished test, emitted from -# pytest_runtest_makereport in tests/e2e/conftest.py via e2e_result_reporter.py), -# so it keeps working when the proxy or the runner is degraded -- exactly the -# case where an in-runner remediation call back through the proxy would fail. -# -# Grouping and repeat suppression on the rule's notification_settings replace the -# old hand-rolled sha256 node-id dedup: one Slack notification per failing -# package per window, re-sent only every repeat_interval while it stays red. -# -# Set LOKI_DATASOURCE_UID to your Loki datasource UID (Grafana expands $__env{} -# in provisioning files; substitute the literal UID if your Grafana predates it). -apiVersion: 1 - -groups: - - orgId: 1 - name: litellm-e2e - folder: LiteLLM E2E - # Evaluate every 1m over a 5m lookback so windows overlap 5x. A range that - # equals the evaluation interval tiles with no overlap, so a line ingested - # late or near a boundary (Loki ingestion lag, clock skew) can fall between - # two windows and never alert. The overlap closes that gap; grouping and - # repeat_interval on notification_settings keep it to one notification. - interval: 1m - rules: - - uid: litellm-e2e-suite-failure - title: LiteLLM e2e suite failure - condition: C - for: 0m - noDataState: OK - execErrState: Error - labels: - service: litellm-e2e - severity: ticket - annotations: - summary: 'e2e package "{{ $labels.package }}" has failing tests' - description: >- - One or more tests in the "{{ $labels.package }}" package under - tests/e2e/ failed or errored in the last evaluation window. - drilldown_query: >- - {service_name="litellm-e2e"} |= "E2E_RESULT" | logfmt | - package="{{ $labels.package }}" | outcome=~"failed|error" - notification_settings: - receiver: devin-e2e-remediation - group_by: - - alertname - - package - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - data: - - refId: A - relativeTimeRange: - from: 300 - to: 0 - datasourceUid: $__env{LOKI_DATASOURCE_UID} - model: - refId: A - datasource: - type: loki - uid: $__env{LOKI_DATASOURCE_UID} - editorMode: code - queryType: instant - expr: >- - sum by (package) (count_over_time({service_name="litellm-e2e"} - |= "E2E_RESULT" | logfmt | outcome=~"failed|error" [5m])) - - refId: B - datasourceUid: __expr__ - model: - refId: B - type: reduce - reducer: last - expression: A - datasource: - type: __expr__ - uid: __expr__ - - refId: C - datasourceUid: __expr__ - model: - refId: C - type: threshold - expression: B - conditions: - - evaluator: - type: gt - params: - - 0 - datasource: - type: __expr__ - uid: __expr__ diff --git a/tests/e2e/grafana/alerting/contact_points.yaml b/tests/e2e/grafana/alerting/contact_points.yaml deleted file mode 100644 index 04fd36a7d37..00000000000 --- a/tests/e2e/grafana/alerting/contact_points.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Grafana Alerting contact point: Slack channel where Devin listens. -# -# Grafana provisioning (apiVersion 1). Import this alongside alert_rules.yaml. -# Provisioning here is additive per-name: it creates or updates the -# "devin-e2e-remediation" contact point without touching other receivers. -# -# The Slack webhook is a secret and must not live in git. Grafana expands -# $__env{VAR} in provisioning files at load time, so supply it through the -# environment (SLACK_E2E_WEBHOOK_URL). If your Grafana predates env expansion, -# substitute the literal incoming-webhook URL at apply time instead. -apiVersion: 1 - -contactPoints: - - orgId: 1 - name: devin-e2e-remediation - receivers: - - uid: devin-e2e-slack - type: slack - # Only firing alerts (an actual red suite) reach the channel Devin acts - # on; resolved notifications are suppressed so Devin never opens work off - # a recovery message. - disableResolveMessage: true - settings: - url: $__env{SLACK_E2E_WEBHOOK_URL} - title: 'LiteLLM e2e failures ({{ len .Alerts.Firing }} firing)' - text: | - {{ len .Alerts.Firing }} LiteLLM e2e package(s) with failing tests: - {{ range .Alerts.Firing }} - - *{{ .Labels.package }}*: {{ .Annotations.summary }} - drill-down (Loki): {{ .Annotations.drilldown_query }} - {{ end }}