Add HypothesisWrite tool; attempt to turn the 271 infra-noise eval green

.github/workflows/eval-benchmarks.yaml

@@ -16,7 +16,7 @@ on:
         description: 'Comma-separated list of models to test'
         required: false
         # NOTE: Keep in sync with DEFAULT_BENCHMARK_MODELS env var
-        default: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex'
+        default: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex,opus-4.8,gpt-5.5'
       test_markers:
         description: 'Custom pytest markers (ONLY use if not using benchmark_type). Cannot be combined with benchmark_type.'
         required: false
@@ -25,6 +25,11 @@ on:
         description: 'Number of iterations per test (max 10)'
         required: false
         default: '5'
+      create_pr:
+        description: 'Open a PR with the benchmark results. Uncheck to only post a Slack notification (no PR).'
+        required: false
+        type: boolean
+        default: true
 
   # Run weekly on Sunday at 2 AM UTC
   schedule:
@@ -33,7 +38,7 @@ on:
 env:
   # Default models for benchmarks - single source of truth
   # NOTE: Keep workflow_dispatch default in sync with this for UI display
-  DEFAULT_BENCHMARK_MODELS: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex'
+  DEFAULT_BENCHMARK_MODELS: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex,opus-4.8,gpt-5.5'
 
 jobs:
   run-benchmarks:
@@ -119,6 +124,8 @@ jobs:
           MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
           CONFLUENCE_BASE_URL: ${{ secrets.CONFLUENCE_BASE_URL }}
           CONFLUENCE_API_KEY: ${{ secrets.CONFLUENCE_API_KEY }}
+          ELASTICSEARCH_URL: ${{ secrets.ELASTICSEARCH_URL }}
+          ELASTICSEARCH_API_KEY: ${{ secrets.ELASTICSEARCH_API_KEY }}
           MODEL_LIST_FILE_LOCATION: /tmp/model_list.yaml
           EXPERIMENT_ID: "ci-benchmark-${{ github.run_id }}"
         run: |
@@ -138,7 +145,7 @@ jobs:
 
       - name: Create PR with benchmark results
         id: create-pr
-        if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        if: always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_pr == 'true'))
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -194,10 +201,15 @@ jobs:
           SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
         run: |
           PR_URL="${{ steps.create-pr.outputs.pr_url }}"
+          if [ -n "$PR_URL" ]; then
+            TEXT="Weekly eval benchmarks finished! Here is the PR: ${PR_URL}"
+          else
+            TEXT="Eval benchmarks finished! No PR was created for this run."
+          fi
           curl -X POST https://slack.com/api/chat.postMessage \
             -H "Authorization: Bearer $SLACK_TOKEN" \
             -H "Content-Type: application/json" \
-            -d "{\"channel\": \"#backend\", \"text\": \"Weekly eval benchmarks finished! Here is the PR: ${PR_URL}\"}"
+            -d "{\"channel\": \"#backend\", \"text\": \"${TEXT}\"}"
 
       - name: Notify Slack on failure or cancellation
         if: steps.run-benchmarks.outcome != 'success'

.github/workflows/eval-master.yaml

@@ -1,16 +1,9 @@
 name: Run eval regression on master
 
 on:
-  # Run on every push to master so PRs can compare against the most recent
-  # known-good state of master, not just the weekly benchmark.
-  push:
-    branches: [master]
-    paths-ignore:
-      - 'docs/**'
-      - '**/*.md'
-      - 'mkdocs.yml'
-
-  # Allow manual triggering for one-off baselines / re-running after a fix.
+  # The master baseline no longer runs automatically on every push to master —
+  # that per-merge eval cost has been removed. Refresh the baseline on demand via
+  # workflow_dispatch (e.g. one-off baselines or re-running after a fix).
   workflow_dispatch:
     inputs:
       models:
@@ -69,6 +62,8 @@ jobs:
           MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
           CONFLUENCE_BASE_URL: ${{ secrets.CONFLUENCE_BASE_URL }}
           CONFLUENCE_API_KEY: ${{ secrets.CONFLUENCE_API_KEY }}
+          ELASTICSEARCH_URL: ${{ secrets.ELASTICSEARCH_URL }}
+          ELASTICSEARCH_API_KEY: ${{ secrets.ELASTICSEARCH_API_KEY }}
           MODEL_LIST_FILE_LOCATION: /tmp/model_list.yaml
           # This name must match MASTER_EXPERIMENT_PREFIX in braintrust_history.py
           EXPERIMENT_ID: "master-${{ github.run_id }}"

.github/workflows/eval-regression.yaml

@@ -1,9 +1,12 @@
 name: Run eval regression tests
 
 on:
+  # Evals no longer run automatically on every commit (this was the main CI eval
+  # cost driver). They run on-demand only: add an evals-* label to a PR (handled
+  # by the `labeled` event below), comment `/eval`, or trigger workflow_dispatch.
   pull_request:
     branches: ["*"]
-    types: [opened, synchronize, reopened, labeled]
+    types: [labeled]
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
@@ -452,6 +455,12 @@ jobs:
               prNumber = event === 'pull_request' ? context.payload.pull_request.number : null;
               triggeredBy = '';
 
+              // Evals do not run automatically on every commit anymore. For automatic
+              // (pull_request) triggers we skip by default and only run when the PR has
+              // at least one evals-* label opting it in (evals-tag-*, evals-id-*, or
+              // evals-model-*). /eval comments and workflow_dispatch are unaffected.
+              skipEval = true;
+
               // Parse PR labels for evals-tag-*, evals-id-*, and evals-model-* to augment the regression run
               if (event === 'pull_request' && context.payload.pull_request.labels) {
                 const labelSafe = /^[a-zA-Z0-9_\-]+$/;
@@ -480,22 +489,27 @@ jobs:
                   // Only IDs specified: broaden markers to all LLM tests, filter by ID
                   markers = '';
                   filter = idLabels.join(' or ');
+                  skipEval = false;
                 } else if (tagLabels.length > 0 && idLabels.length === 0) {
                   // Only tags specified: add to regression markers (unless skip-default)
                   markers = skipDefault ? tagLabels.join(' or ') : 'regression or ' + tagLabels.join(' or ');
+                  skipEval = false;
                 } else if (tagLabels.length > 0 && idLabels.length > 0) {
                   // Both specified: use tag markers + ID filter
                   markers = skipDefault ? tagLabels.join(' or ') : 'regression or ' + tagLabels.join(' or ');
                   filter = idLabels.join(' or ');
-                } else if (skipDefault) {
-                  // Only skip-default, no other evals-* labels: skip eval run entirely
-                  skipEval = true;
+                  skipEval = false;
                 }
-                // else: no evals-* labels, keep defaults (markers = 'regression')
+                // else: no evals-tag-*/evals-id-* labels — stays skipped (skipEval=true)
+                // unless an evals-model-* label opts in below. evals-skip-default alone
+                // is now a no-op since skipping is the default.
 
-                // Override model if evals-model-* label is present
+                // Override model if evals-model-* label is present. A model label on
+                // its own means "run the default regression suite on this model", so it
+                // opts the PR in (markers falls through to the 'regression' default below).
                 if (modelLabels.length > 0) {
                   model = modelLabels.join(',');
+                  skipEval = false;
                 }
               }
 

CLAUDE.md

@@ -217,6 +217,7 @@ For the complete eval CLI reference (flags, env vars, model comparison, debuggin
 - Type hints required (mypy configuration in pyproject.toml)
 - Pre-commit hooks enforce quality checks in CI
 - **ALWAYS place Python imports at the top of the file**, not inside functions or methods
+- **Use `tenacity` for retries**, not hand-rolled retry loops — the `@retry(...)` decorator normally, or the `Retrying(...)` iterator form when the attempt budget/params must come from per-instance or dynamic values. Only hand-roll a retry when there's a real reason to, and document why.
 - **NEVER run `pre-commit`, `ruff`, or `mypy` unless the user explicitly asks you to**. These tools are triggered by commit hooks which are not installed on all machines, and running them causes widespread formatting/type changes to files unrelated to your task. Only lint/format files you are actively editing, and only if asked.
 
 **Documentation Examples**:
@@ -855,7 +856,7 @@ When writing documentation in the `docs/` directory:
 
 ## Robusta Platform/API URLs in Docs
 
-Whenever you reference `api.robusta.dev` or `platform.robusta.dev` in a docs page, use the `robusta-region` custom fence so readers can pick US/EU/AP. **Never hardcode a single region** — the Robusta platform is hosted in multiple regions and a bare `api.robusta.dev` link silently breaks for EU/AP users.
+Whenever you reference `api.robusta.dev`, `platform.robusta.dev`, or `sp.robusta.dev` (the Supabase host) in a docs page, use the `robusta-region` custom fence so readers can pick US/EU/AP. **Never hardcode a single region** — the Robusta platform is hosted in multiple regions and a bare `api.robusta.dev` link silently breaks for EU/AP users.
 
 The fence (defined in `docs/custom_fences.py`, registered in `mkdocs.yml`) takes a US URL as input and emits a three-tab picker with the domain rewritten per region. Pick the input shape that matches your context:
 
@@ -886,6 +887,6 @@ The fence (defined in `docs/custom_fences.py`, registered in `mkdocs.yml`) takes
     ```
     ````
 
-Author the URL once using the US domain (`api.robusta.dev` / `platform.robusta.dev`); the fence handles the `.eu` / `.ap` rewrites. If you add a new region or rename one, update `ROBUSTA_REGIONS` in `docs/custom_fences.py` — that single change propagates to every page.
+Author the URL once using the US domain (`api.robusta.dev` / `platform.robusta.dev` / `sp.robusta.dev`); the fence handles the `.eu` / `.ap` rewrites. The set of rewritten hosts lives in `ROBUSTA_DOMAIN_RE` in `docs/custom_fences.py` — add a host there if you need another subdomain covered. If you add a new region or rename one, update `ROBUSTA_REGIONS` in the same file — that single change propagates to every page.
 
-Existing usages (greppable starting points): `docs/ai-providers/robusta-ai.md`, `docs/installation/ui-installation.md`, `docs/reference/environment-variables.md`, `docs/data-sources/builtin-toolsets/coralogix-logs.md`, `docs/data-sources/builtin-toolsets/kubernetes-mcp.md`.
+Existing usages (greppable starting points): `docs/ai-providers/robusta-ai.md`, `docs/installation/ui-installation.md`, `docs/reference/environment-variables.md`, `docs/reference/troubleshooting.md`, `docs/data-sources/builtin-toolsets/coralogix-logs.md`, `docs/data-sources/builtin-toolsets/kubernetes-mcp.md`.

docs/custom_fences.py

@@ -4,21 +4,23 @@
 Fences available:
 - yaml-toolset-config: Creates 3 tabs (Holmes CLI, Holmes Helm Chart, Robusta Helm Chart) for toolset configurations
 - yaml-helm-values: Creates 2 tabs (Holmes Helm Chart, Robusta Helm Chart) for Helm-only configurations like permissions
-- robusta-region: Creates 3 tabs (US, EU, AP) for any text containing api.robusta.dev or platform.robusta.dev. Plain
-  URLs render as code blocks; markdown links `[text](url)` render as clickable links.
+- robusta-region: Creates 3 tabs (US, EU, AP) for any text containing api.robusta.dev, platform.robusta.dev, or
+  sp.robusta.dev. Plain URLs render as code blocks; markdown links `[text](url)` render as clickable links.
 """
 
 import html
 import re
 import uuid
 
+import yaml  # type: ignore
+
 ROBUSTA_REGIONS = (("US", ""), ("EU", "eu"), ("AP", "ap"))
-ROBUSTA_DOMAIN_RE = re.compile(r"\b(api|platform)\.robusta\.dev\b")
+ROBUSTA_DOMAIN_RE = re.compile(r"\b(api|platform|sp)\.robusta\.dev\b")
 MARKDOWN_LINK_RE = re.compile(r"^\[([^\]]+)\]\(([^)\s]+)\)(\{[^}]*\})?$")
 
 
 def _rewrite_robusta_domain(text: str, region_infix: str) -> str:
-    """Rewrite api.robusta.dev / platform.robusta.dev to the regional variant."""
+    """Rewrite api/platform/sp .robusta.dev to the regional variant."""
     if not region_infix:
         return text
     return ROBUSTA_DOMAIN_RE.sub(rf"\1.{region_infix}.robusta.dev", text)
@@ -138,8 +140,8 @@ def helm_tabs_fence_format(source, language, css_class, options, md, **kwargs):
 
 def robusta_region_fence_format(source, language, css_class, options, md, **kwargs):
     """
-    Render the source as three tabs (US, EU, AP), rewriting `api.robusta.dev`
-    and `platform.robusta.dev` to the regional subdomain in each tab.
+    Render the source as three tabs (US, EU, AP), rewriting `api.robusta.dev`,
+    `platform.robusta.dev` and `sp.robusta.dev` to the regional subdomain in each tab.
 
     Auto-detects two input shapes:
 
@@ -215,3 +217,84 @@ def robusta_region_fence_format(source, language, css_class, options, md, **kwar
         f'<div class="tabbed-content">\n{blocks_html}</div>\n'
         "</div>"
     )
+
+
+# Central page that documents how multi-instance toolsets work. Linked from every
+# rendered ``multi-instance`` block so each toolset page doesn't repeat the prose.
+MULTI_INSTANCE_DOC_URL = "/data-sources/multi-instance-toolsets/"
+
+
+def _reindent(text: str, spaces: int) -> str:
+    """Dedent ``text`` to its common leading whitespace, then indent every
+    non-empty line by ``spaces``. Used to nest a flat config example under
+    ``instances:`` at the correct YAML depth."""
+    lines = text.strip("\n").split("\n")
+    nonempty = [ln for ln in lines if ln.strip()]
+    base = min((len(ln) - len(ln.lstrip()) for ln in nonempty), default=0)
+    pad = " " * spaces
+    return "\n".join(pad + ln[base:] if ln.strip() else "" for ln in lines)
+
+
+def multi_instance_fence_format(source, language, css_class, options, md, **kwargs):
+    """Render the standard "Multiple Instances" section for a toolset.
+
+    The fence body is YAML with three keys:
+
+        ```multi-instance
+        toolset: grafana/dashboards   # the toolset key used in config examples
+        name: Grafana                 # human-readable name (optional; derived from toolset)
+        config: |                     # a single-instance config example for this toolset
+          api_url: <your grafana url>
+          api_key: <your api key>
+        ```
+
+    It emits a note admonition that:
+    - explains the toolset can connect to several instances via ``instances:``;
+    - shows the supplied config example nested under ``instances:`` (two entries);
+    - notes the auto-injected ``instance`` parameter and ``<toolset>_list_instances``
+      tool that appear when more than one instance is configured;
+    - links to the central Multiple Instances page for the full behaviour.
+
+    The same component renders identically for every toolset, so each page imports
+    it in one fenced block instead of repeating the prose.
+    """
+    spec = yaml.safe_load(source) or {}
+    toolset = str(spec.get("toolset", "")).strip()
+    name = str(spec.get("name") or toolset or "this").strip()
+    config = str(spec.get("config", "")).strip()
+    if not toolset or not config:
+        raise ValueError(
+            "multi-instance fence requires 'toolset' and 'config' keys in its YAML body"
+        )
+
+    # The wrapper names the discovery tool by replacing '/' with '_' in the toolset name.
+    list_tool = spec.get("list_tool") or (toolset.replace("/", "_") + "_list_instances")
+
+    fields = _reindent(config, 10)
+    yaml_example = (
+        "toolsets:\n"
+        f"  {toolset}:\n"
+        "    enabled: true\n"
+        "    config:\n"
+        "      instances:\n"
+        f"        - name: prod\n{fields}\n"
+        f"        - name: staging\n{fields}\n"
+    )
+
+    name_e = html.escape(name)
+    list_tool_e = html.escape(str(list_tool))
+    return (
+        f"<p>The {name_e} toolset can connect to more than one {name_e} instance. "
+        "List each one under <code>instances:</code> with a unique <code>name</code>. "
+        "Any config field set outside <code>instances:</code> becomes a default that "
+        "every instance inherits, so shared settings only need to be written once.</p>\n"
+        f'<pre><code class="language-yaml">{html.escape(yaml_example)}</code></pre>\n'
+        "<p>When more than one instance is configured, HolmesGPT automatically adds an "
+        f"<code>instance</code> parameter to every {name_e} tool (so it can pick which "
+        f"instance to query) and a <code>{list_tool_e}</code> tool to list the configured "
+        "instances. With a single instance — including the flat config without "
+        "<code>instances:</code> — the tools are unchanged and fully backwards "
+        "compatible.</p>\n"
+        f'<p>See <a href="{MULTI_INSTANCE_DOC_URL}">Multiple Instances</a> for the full '
+        "behaviour, including global defaults and health reporting.</p>"
+    )

docs/data-sources/.nav.yml

@@ -2,6 +2,7 @@ nav:
   - index.md
   - Recommended Setup: recommended-setup.md
   - Built-in Toolsets: builtin-toolsets
+  - Multiple Instances: multi-instance-toolsets.md
   - HTTP Connectors: api-toolsets.md
   - Database Connectors: database-connectors.md
   - Custom Toolsets: custom-toolsets.md

docs/data-sources/builtin-toolsets/azure-sql.md

@@ -190,6 +190,19 @@ By enabling this toolset, HolmesGPT can analyze Azure SQL Database performance,
 
     --8<-- "snippets/helm_upgrade_command.md"
 
+## Multiple Instances
+
+```multi-instance
+toolset: azure/sql
+name: Azure SQL
+config: |
+  database:
+    subscription_id: "your-subscription-id"
+    resource_group: "your-resource-group"
+    server_name: "your-azure-sql-server-name"
+    database_name: "your-azure-sql-database-name"
+```
+
 ## Roles / Access controls
 
 The service principal requires these roles:

docs/data-sources/builtin-toolsets/confluence.md

@@ -314,6 +314,17 @@ HolmesGPT authenticates to a self-hosted Confluence Data Center (or Server) inst
 
     --8<-- "snippets/helm_upgrade_command.md"
 
+## Multiple Instances
+
+```multi-instance
+toolset: confluence
+name: Confluence
+config: |
+  api_url: "https://yourcompany.atlassian.net"
+  user: "your-email@example.com"
+  api_key: "your-api-token"
+```
+
 ## Configuration Reference
 
 `subtype` is set at the toolset level (sibling of `enabled:` and `config:`); the rest of the fields below go inside `config:`.

docs/data-sources/builtin-toolsets/coralogix-logs.md

@@ -122,6 +122,16 @@ Configure both the Coralogix DataPrime toolset (for logs/traces) and the Prometh
 
 **Note**: Both toolsets use the same API key. Helm-tab users only need to create one Kubernetes secret — the env var feeds both the `coralogix` toolset's `api_key` field and the Prometheus toolset's `Authorization` header.
 
+## Multiple Instances
+
+```multi-instance
+toolset: coralogix
+name: Coralogix
+config: |
+  api_key: "<your Coralogix API key>"
+  domain: "eu2.coralogix.com"
+```
+
 ## Recommended: Customize Coralogix Instructions
 
 By specifying details about your Coralogix metrics, logs, and traces, you can significantly speed up and improve investigations. This allows Holmes to work with your environment directly, rather than spending time discovering labels, mappings, and metric names on its own.

docs/data-sources/builtin-toolsets/datadog.md

@@ -186,6 +186,17 @@ holmes ask "list Datadog monitors"
 
 That's it! You're now connected to Datadog with all toolsets enabled.
 
+## Multiple Instances
+
+```multi-instance
+toolset: datadog/logs
+name: Datadog
+config: |
+  api_key: "{{ env.DATADOG_API_KEY }}"
+  app_key: "{{ env.DATADOG_APP_KEY }}"
+  api_url: https://api.datadoghq.com
+```
+
 ## Available Toolsets
 
 HolmesGPT provides four specialized Datadog toolsets: