From 33ae3d88f506542fca117026d92ad171c2ca3042 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 09:46:59 +0000 Subject: [PATCH 1/5] Fix Loki toolset docs for Grafana Cloud and trailing slash bug Add dedicated Grafana Cloud section to Loki docs with step-by-step instructions for service account setup, datasource UID discovery, and common mistake warnings. Fix trailing slash bug in get_base_url() that produced double-slash URLs (e.g. https://x.grafana.net//api/...) causing 404 errors. Also add direct Grafana Cloud Loki access method and correct the capabilities table tool name. https://claude.ai/code/session_01CcT3RF7eByFp1W1mtdbXyK Signed-off-by: Claude --- .../builtin-toolsets/grafanaloki.md | 63 ++++++++++++++++--- holmes/plugins/toolsets/grafana/common.py | 5 +- .../plugins/toolsets/grafana/test_grafana.py | 2 +- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/docs/data-sources/builtin-toolsets/grafanaloki.md b/docs/data-sources/builtin-toolsets/grafanaloki.md index 0383467ca6..268d375f90 100644 --- a/docs/data-sources/builtin-toolsets/grafanaloki.md +++ b/docs/data-sources/builtin-toolsets/grafanaloki.md @@ -35,26 +35,60 @@ kubectl port-forward svc/grafana 3000:80 curl -s -u admin:admin http://localhost:3000/api/datasources | jq '.[] | select(.type == "loki") | .uid' ``` -### Configuration (Grafana Proxy) +**Configuration (Grafana Proxy):** ```yaml-toolset-config toolsets: grafana/loki: enabled: true config: - api_key: - api_url: https://xxxxxxx.grafana.net # Your Grafana cloud account URL + api_key: + api_url: http://grafana.monitoring.svc:3000 grafana_datasource_uid: kubernetes/logs: enabled: false # HolmesGPT's default logging mechanism MUST be disabled ``` -## Direct Connection +### Option 2: Grafana Cloud -The toolset can directly connect to a Loki instance without proxying through a Grafana instance. This is done by not setting the `grafana_datasource_uid` field. Not setting this field makes HolmesGPT assume that it is directly connecting to Loki. +**1. Create a service account token in Grafana Cloud:** -### Configuration (Direct Connection) + - Navigate to "Administration → Service accounts" + - Create a new service account with "Viewer" role + - Generate a service account token (starts with `glsa_`) + +**2. Find your Loki datasource UID:** + +```bash +curl -H "Authorization: Bearer YOUR_GLSA_TOKEN" \ + "https://YOUR-INSTANCE.grafana.net/api/datasources" | \ + jq '.[] | select(.type=="loki") | {name, uid}' +``` + +**3. Configure HolmesGPT:** + +```yaml-toolset-config +toolsets: + grafana/loki: + enabled: true + config: + api_key: + api_url: https://YOUR-INSTANCE.grafana.net + grafana_datasource_uid: + + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled +``` + +!!! warning "Common mistakes" + - **Do not include a trailing slash** in `api_url` (use `https://mystack.grafana.net`, not `https://mystack.grafana.net/`) + - **Use your Grafana instance URL** (`https://YOUR-INSTANCE.grafana.net`), not the Loki endpoint URL (`https://logs-prod-xxx.grafana.net`) + - **Verify the datasource UID** using the `curl` command above — a wrong UID is the most common cause of 404 errors + +### Option 3: Direct Connection + +The toolset can directly connect to a Loki instance without proxying through a Grafana instance. This is done by not setting the `grafana_datasource_uid` field. ```yaml-toolset-config toolsets: @@ -69,6 +103,21 @@ toolsets: enabled: false # HolmesGPT's default logging mechanism MUST be disabled ``` +For **Grafana Cloud Loki direct access** (without proxying through Grafana), find your Loki URL and user ID in Grafana Cloud under "My Account → Loki": + +```yaml-toolset-config +toolsets: + grafana/loki: + enabled: true + config: + api_url: https://logs-prod-XXX.grafana.net + additional_headers: + Authorization: "Basic " + + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled +``` + ## Advanced Configuration ### SSL Verification @@ -101,4 +150,4 @@ toolsets: | Tool Name | Description | |-----------|-------------| -| fetch_pod_logs | Fetches pod logs from Loki | +| grafana_loki_query | Run LogQL queries against Loki | diff --git a/holmes/plugins/toolsets/grafana/common.py b/holmes/plugins/toolsets/grafana/common.py index 3c7735a85e..23a3ba41ad 100644 --- a/holmes/plugins/toolsets/grafana/common.py +++ b/holmes/plugins/toolsets/grafana/common.py @@ -68,10 +68,11 @@ def build_headers(api_key: Optional[str], additional_headers: Optional[Dict[str, def get_base_url(config: GrafanaConfig) -> str: + api_url = config.api_url.rstrip("/") if config.grafana_datasource_uid: - return f"{config.api_url}/api/datasources/proxy/uid/{config.grafana_datasource_uid}" + return f"{api_url}/api/datasources/proxy/uid/{config.grafana_datasource_uid}" else: - return config.api_url + return api_url class GrafanaTempoLabelsConfig(ToolsetConfig): diff --git a/tests/plugins/toolsets/grafana/test_grafana.py b/tests/plugins/toolsets/grafana/test_grafana.py index 385f37e7f0..4c99a2ac74 100644 --- a/tests/plugins/toolsets/grafana/test_grafana.py +++ b/tests/plugins/toolsets/grafana/test_grafana.py @@ -64,7 +64,7 @@ def test_loki_datasource_toolset_error_health_check(): toolset.check_prerequisites() assert ( - "Unable to connect to Loki.\nFailed to query Loki logs: 404 Client Error: Not Found for url: http://localhost:3000//api/datasources/proxy/uid/wrong-uid/loki/api/v1/query_range" + "Unable to connect to Loki.\nFailed to query Loki logs: 404 Client Error: Not Found for url: http://localhost:3000/api/datasources/proxy/uid/wrong-uid/loki/api/v1/query_range" in toolset.error ) assert toolset.status == ToolsetStatusEnum.FAILED From 7ce37627b75d86b837404ae360c96b0e108284d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 09:49:37 +0000 Subject: [PATCH 2/5] Remove trailing slash warning from Loki docs since code handles it https://claude.ai/code/session_01CcT3RF7eByFp1W1mtdbXyK Signed-off-by: Claude --- docs/data-sources/builtin-toolsets/grafanaloki.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/data-sources/builtin-toolsets/grafanaloki.md b/docs/data-sources/builtin-toolsets/grafanaloki.md index 268d375f90..016b86dd19 100644 --- a/docs/data-sources/builtin-toolsets/grafanaloki.md +++ b/docs/data-sources/builtin-toolsets/grafanaloki.md @@ -81,8 +81,7 @@ toolsets: enabled: false # HolmesGPT's default logging mechanism MUST be disabled ``` -!!! warning "Common mistakes" - - **Do not include a trailing slash** in `api_url` (use `https://mystack.grafana.net`, not `https://mystack.grafana.net/`) +!!! warning "Getting 404 errors?" - **Use your Grafana instance URL** (`https://YOUR-INSTANCE.grafana.net`), not the Loki endpoint URL (`https://logs-prod-xxx.grafana.net`) - **Verify the datasource UID** using the `curl` command above — a wrong UID is the most common cause of 404 errors From 7ddeeabe815f2c53e10a2b28013102c1acdc9c49 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 09:56:43 +0000 Subject: [PATCH 3/5] Restructure Loki docs: merge confusing 3 options into 2 clear ones The old Options 1/2/3 were confusing because "Grafana Cloud" is just Grafana. Restructured into two real options: - Option 1: Through Grafana (self-hosted or cloud, via tabs) - Option 2: Direct to Loki (self-hosted or cloud, via tabs) https://claude.ai/code/session_01CcT3RF7eByFp1W1mtdbXyK Signed-off-by: Claude --- .../builtin-toolsets/grafanaloki.md | 139 +++++++++--------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/docs/data-sources/builtin-toolsets/grafanaloki.md b/docs/data-sources/builtin-toolsets/grafanaloki.md index 016b86dd19..853ca6ca35 100644 --- a/docs/data-sources/builtin-toolsets/grafanaloki.md +++ b/docs/data-sources/builtin-toolsets/grafanaloki.md @@ -17,105 +17,108 @@ Connect HolmesGPT to Loki for log analysis through Grafana or direct API access. ## Configuration -Choose one of the following methods: - ### Option 1: Through Grafana (Recommended) +HolmesGPT queries Loki through Grafana's datasource proxy. This works with any Grafana instance — self-hosted or Grafana Cloud. + **Required:** - [Grafana service account token](https://grafana.com/docs/grafana/latest/administration/service-accounts/) with Viewer role - Loki datasource UID from Grafana **Find your Loki datasource UID:** -```bash -# Port forward to Grafana -kubectl port-forward svc/grafana 3000:80 -# Get Loki datasource UID -curl -s -u admin:admin http://localhost:3000/api/datasources | jq '.[] | select(.type == "loki") | .uid' -``` +=== "Self-Hosted Grafana" -**Configuration (Grafana Proxy):** + ```bash + # Port forward to Grafana + kubectl port-forward svc/grafana 3000:80 -```yaml-toolset-config -toolsets: - grafana/loki: - enabled: true - config: - api_key: - api_url: http://grafana.monitoring.svc:3000 - grafana_datasource_uid: + # Get Loki datasource UID + curl -s -u admin:admin http://localhost:3000/api/datasources | jq '.[] | select(.type == "loki") | .uid' + ``` - kubernetes/logs: - enabled: false # HolmesGPT's default logging mechanism MUST be disabled -``` +=== "Grafana Cloud" -### Option 2: Grafana Cloud + ```bash + curl -H "Authorization: Bearer YOUR_GLSA_TOKEN" \ + "https://YOUR-INSTANCE.grafana.net/api/datasources" | \ + jq '.[] | select(.type=="loki") | {name, uid}' + ``` -**1. Create a service account token in Grafana Cloud:** + To create a service account token: navigate to "Administration → Service accounts", create a new service account with "Viewer" role, and generate a token (starts with `glsa_`). - - Navigate to "Administration → Service accounts" - - Create a new service account with "Viewer" role - - Generate a service account token (starts with `glsa_`) +**Configure HolmesGPT:** -**2. Find your Loki datasource UID:** +=== "Self-Hosted Grafana" -```bash -curl -H "Authorization: Bearer YOUR_GLSA_TOKEN" \ - "https://YOUR-INSTANCE.grafana.net/api/datasources" | \ - jq '.[] | select(.type=="loki") | {name, uid}' -``` + ```yaml-toolset-config + toolsets: + grafana/loki: + enabled: true + config: + api_key: + api_url: http://grafana.monitoring.svc:3000 + grafana_datasource_uid: -**3. Configure HolmesGPT:** + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled + ``` -```yaml-toolset-config -toolsets: - grafana/loki: - enabled: true - config: - api_key: - api_url: https://YOUR-INSTANCE.grafana.net - grafana_datasource_uid: +=== "Grafana Cloud" - kubernetes/logs: - enabled: false # HolmesGPT's default logging mechanism MUST be disabled -``` + ```yaml-toolset-config + toolsets: + grafana/loki: + enabled: true + config: + api_key: + api_url: https://YOUR-INSTANCE.grafana.net + grafana_datasource_uid: + + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled + ``` !!! warning "Getting 404 errors?" - **Use your Grafana instance URL** (`https://YOUR-INSTANCE.grafana.net`), not the Loki endpoint URL (`https://logs-prod-xxx.grafana.net`) - **Verify the datasource UID** using the `curl` command above — a wrong UID is the most common cause of 404 errors -### Option 3: Direct Connection +### Option 2: Direct to Loki -The toolset can directly connect to a Loki instance without proxying through a Grafana instance. This is done by not setting the `grafana_datasource_uid` field. +Connect directly to the Loki API without going through Grafana. Use this if you don't have Grafana or prefer not to proxy through it. -```yaml-toolset-config -toolsets: - grafana/loki: - enabled: true - config: - api_url: http://loki.logging - additional_headers: - X-Scope-OrgID: "" # Set the X-Scope-OrgID if loki multitenancy is enabled +=== "Self-Hosted Loki" - kubernetes/logs: - enabled: false # HolmesGPT's default logging mechanism MUST be disabled -``` + ```yaml-toolset-config + toolsets: + grafana/loki: + enabled: true + config: + api_url: http://loki.logging + additional_headers: + X-Scope-OrgID: "" # Set if Loki multitenancy is enabled -For **Grafana Cloud Loki direct access** (without proxying through Grafana), find your Loki URL and user ID in Grafana Cloud under "My Account → Loki": + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled + ``` -```yaml-toolset-config -toolsets: - grafana/loki: - enabled: true - config: - api_url: https://logs-prod-XXX.grafana.net - additional_headers: - Authorization: "Basic " +=== "Grafana Cloud Loki" - kubernetes/logs: - enabled: false # HolmesGPT's default logging mechanism MUST be disabled -``` + Find your Loki URL and user ID in Grafana Cloud under "My Account → Loki": + + ```yaml-toolset-config + toolsets: + grafana/loki: + enabled: true + config: + api_url: https://logs-prod-XXX.grafana.net + additional_headers: + Authorization: "Basic " + + kubernetes/logs: + enabled: false # HolmesGPT's default logging mechanism MUST be disabled + ``` ## Advanced Configuration From 17fa0eec127a3d5a572323e13c42e31c790909f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 10:18:03 +0000 Subject: [PATCH 4/5] Simplify Grafana Cloud Loki direct config to just api_url + api_key The code automatically adds Bearer auth from api_key, so no need for manual additional_headers with Basic auth. https://claude.ai/code/session_01CcT3RF7eByFp1W1mtdbXyK Signed-off-by: Claude --- docs/data-sources/builtin-toolsets/grafanaloki.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/data-sources/builtin-toolsets/grafanaloki.md b/docs/data-sources/builtin-toolsets/grafanaloki.md index 853ca6ca35..5b0eb62bce 100644 --- a/docs/data-sources/builtin-toolsets/grafanaloki.md +++ b/docs/data-sources/builtin-toolsets/grafanaloki.md @@ -105,7 +105,7 @@ Connect directly to the Loki API without going through Grafana. Use this if you === "Grafana Cloud Loki" - Find your Loki URL and user ID in Grafana Cloud under "My Account → Loki": + Find your Loki URL in Grafana Cloud under "My Account → Loki": ```yaml-toolset-config toolsets: @@ -113,8 +113,7 @@ Connect directly to the Loki API without going through Grafana. Use this if you enabled: true config: api_url: https://logs-prod-XXX.grafana.net - additional_headers: - Authorization: "Basic " + api_key: kubernetes/logs: enabled: false # HolmesGPT's default logging mechanism MUST be disabled From 96716a723afd64d05b608029e0531d5a26344650 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Feb 2026 19:37:04 +0000 Subject: [PATCH 5/5] Fix token instructions order and remove Capabilities section - Move Grafana Cloud service account creation instructions before the curl command that references the token - Remove Capabilities table per docs guidelines https://claude.ai/code/session_01CcT3RF7eByFp1W1mtdbXyK Signed-off-by: Claude --- docs/data-sources/builtin-toolsets/grafanaloki.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/data-sources/builtin-toolsets/grafanaloki.md b/docs/data-sources/builtin-toolsets/grafanaloki.md index 5b0eb62bce..912154d1f5 100644 --- a/docs/data-sources/builtin-toolsets/grafanaloki.md +++ b/docs/data-sources/builtin-toolsets/grafanaloki.md @@ -40,14 +40,14 @@ HolmesGPT queries Loki through Grafana's datasource proxy. This works with any G === "Grafana Cloud" + Create a [service account token](https://grafana.com/docs/grafana-cloud/account-management/authentication-and-permissions/service-accounts/): navigate to "Administration → Service accounts", create a new service account with "Viewer" role, and generate a token (starts with `glsa_`). + ```bash curl -H "Authorization: Bearer YOUR_GLSA_TOKEN" \ "https://YOUR-INSTANCE.grafana.net/api/datasources" | \ jq '.[] | select(.type=="loki") | {name, uid}' ``` - To create a service account token: navigate to "Administration → Service accounts", create a new service account with "Viewer" role, and generate a token (starts with `glsa_`). - **Configure HolmesGPT:** === "Self-Hosted Grafana" @@ -146,9 +146,3 @@ toolsets: api_url: http://loki.internal:3100 # Internal URL for API calls external_url: https://loki.example.com # URL for links in results ``` - -## Capabilities - -| Tool Name | Description | -|-----------|-------------| -| grafana_loki_query | Run LogQL queries against Loki |