[ROB-3883] Loki returns raw response on parse failure

holmes/plugins/toolsets/grafana/loki_api.py

@@ -65,10 +65,19 @@ def _make_request():
 
     try:
         response = _make_request()
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Failed to query Loki logs: {str(e)}")
+
+    try:
         result = response.json()
         if "data" in result and "result" in result["data"]:
             return parse_loki_response(result["data"]["result"])
         return []
-
-    except requests.exceptions.RequestException as e:
-        raise Exception(f"Failed to query Loki logs: {str(e)}")
+    except Exception as e:
+        raw = response.text
+        raise Exception(
+            f"Failed to process Loki response: {e}\n"
+            f"--- raw response ({len(raw)} chars, content-type={response.headers.get('Content-Type', 'unknown')}) ---\n"
+            f"{raw}\n"
+            f"--- end raw response ---"
+        )

tests/plugins/toolsets/grafana/test_grafana.py

@@ -1,18 +1,24 @@
 import pytest
+import responses
 
 from holmes.core.tools import ToolsetStatusEnum
 from holmes.plugins.toolsets.grafana.loki.toolset_grafana_loki import (
     GrafanaLokiToolset,
 )
+from holmes.plugins.toolsets.grafana.loki_api import execute_loki_query
 from holmes.plugins.toolsets.grafana.toolset_grafana import GrafanaToolset
 from tests.plugins.toolsets.grafana.conftest import check_service_running
 
-# Skip all tests in this module if Grafana and loki are not running. use loki/docker-compose.yaml
-skip_reason = check_service_running("Grafana", 3000)
-if skip_reason:
-    pytestmark = pytest.mark.skip(reason=skip_reason)
+# Skip integration tests that require Grafana/Loki running on localhost:3000.
+# Use loki/docker-compose.yaml to bring them up.
+_grafana_skip_reason = check_service_running("Grafana", 3000)
+needs_grafana = pytest.mark.skipif(
+    _grafana_skip_reason is not None,
+    reason=_grafana_skip_reason or "",
+)
 
 
+@needs_grafana
 def test_grafana_toolset_direct_health_check():
     toolset = GrafanaToolset()
     toolset.config = {"api_url": "http://localhost:3000/"}
@@ -22,6 +28,7 @@ def test_grafana_toolset_direct_health_check():
     assert toolset.status == ToolsetStatusEnum.ENABLED
 
 
+@needs_grafana
 def test_grafana_toolset_error_health_check():
     toolset = GrafanaToolset()
     toolset.config = {"api_url": "http://localhost:2000/"}
@@ -34,6 +41,7 @@ def test_grafana_toolset_error_health_check():
     assert toolset.status == ToolsetStatusEnum.FAILED
 
 
+@needs_grafana
 def test_loki_toolset_direct_health_check():
     toolset = GrafanaLokiToolset()
     toolset.config = {"api_url": "http://localhost:3100/"}
@@ -43,6 +51,7 @@ def test_loki_toolset_direct_health_check():
     assert toolset.status == ToolsetStatusEnum.ENABLED
 
 
+@needs_grafana
 def test_loki_datasource_toolset_health_check():
     toolset = GrafanaLokiToolset()
     toolset.config = {
@@ -55,6 +64,7 @@ def test_loki_datasource_toolset_health_check():
     assert toolset.status == ToolsetStatusEnum.ENABLED
 
 
+@needs_grafana
 def test_loki_datasource_toolset_error_health_check():
     toolset = GrafanaLokiToolset()
     toolset.config = {
@@ -68,3 +78,37 @@ def test_loki_datasource_toolset_error_health_check():
         in toolset.error
     )
     assert toolset.status == ToolsetStatusEnum.FAILED
+
+
+def test_execute_loki_query_includes_raw_body_on_malformed_json():
+    """When Loki returns malformed JSON, the raised exception should include
+    both the JSON parser error and the raw response body so an operator (and
+    the LLM) can see what actually came back over the wire."""
+    base_url = "http://loki.example.com"
+    raw_body = '{"data": {"result": [oops not json'
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            responses.GET,
+            f"{base_url}/loki/api/v1/query_range",
+            body=raw_body,
+            status=200,
+            content_type="application/json",
+        )
+
+        with pytest.raises(Exception) as exc_info:
+            execute_loki_query(
+                base_url=base_url,
+                api_key=None,
+                headers=None,
+                query='{job="foo"}',
+                start=0,
+                end=1,
+                limit=10,
+            )
+
+    message = str(exc_info.value)
+    assert "Failed to process Loki response" in message
+    assert raw_body in message
+    assert "content-type=application/json" in message
+    assert f"{len(raw_body)} chars" in message