Holmes workload health with json response

holmes/core/tool_calling_llm.py

@@ -7,7 +7,7 @@
 import os
 from typing import Dict, List, Optional
 from holmes.plugins.prompts import load_prompt
-
+from litellm import get_supported_openai_params
 import litellm
 import jinja2
 from openai import BadRequestError
@@ -58,6 +58,9 @@ def __init__(
         self.api_key = api_key
         self.base_url = None
 
+        params = get_supported_openai_params(model=self.model)
+        self.support_response_format: bool = params and "response_format" in params
+
         if ROBUSTA_AI:
             self.base_url = ROBUSTA_API_ENDPOINT
             self.model = f"openai/{self.model}"
@@ -85,7 +88,7 @@ def check_llm(self, model, api_key):
         #if not litellm.supports_function_calling(model=model):
         #    raise Exception(f"model {model} does not support function calling. You must use HolmesGPT with a model that supports function calling.")
 
-    def call(self, system_prompt, user_prompt, post_process_prompt: Optional[str] = None) -> LLMResult:
+    def call(self, system_prompt, user_prompt, post_process_prompt: Optional[str] = None, response_format: dict = None) -> LLMResult:
         messages = [
             {
                 "role": "system",
@@ -98,6 +101,8 @@ def call(self, system_prompt, user_prompt, post_process_prompt: Optional[str] =
         ]
         tool_calls = []
         tools = self.tool_executor.get_all_tools_openai_format()
+
+        response_format = response_format if self.support_response_format else None
         for i in range(self.max_steps):
             logging.debug(f"running iteration {i}")
             # on the last step we don't allow tools - we want to force a reply, not a request to run another tool
@@ -112,7 +117,9 @@ def call(self, system_prompt, user_prompt, post_process_prompt: Optional[str] =
                     tools=tools,
                     tool_choice=tool_choice,
                     base_url=self.base_url,
-                    temperature=0.00000001
+                    temperature=0.00000001,
+                    response_format=response_format
+
                 )
                 logging.debug(f"got response {full_response}")
             # catch a known error that occurs with Azure and replace the error message with something more obvious to the user

holmes/plugins/prompts/kubernetes_workload_ask.jinja2

@@ -0,0 +1,65 @@
+You are a tool-calling AI assist provided with common devops and IT tools that you can use to troubleshoot problems or answer questions.
+Whenever possible you MUST first use tools to investigate then answer the question.
+Do not say 'based on the tool output' or explicitly refer to tools at all.
+If you output an answer and then realize you need to call more tools or there are possible next steps, you may do so by calling tools at that point in time.
+
+If the user provides you with extra instructions in a triple single quotes section, ALWAYS perform their instructions and then perform your investigation.
+
+In general:
+* when it can provide extra information, first run as many tools as you need to gather more information, then respond. 
+* if possible, do so repeatedly with different tool calls each time to gather more information.
+* do not stop investigating until you are at the final root cause you are able to find. 
+* use the "five whys" methodology to find the root cause.
+* for example, if you found a problem in microservice A that is due to an error in microservice B, look at microservice B too and find the error in that.
+* if you cannot find the resource/application that the user referred to, assume they made a typo or included/excluded characters like - and.
+* in this case, try to find substrings or search for the correct spellings
+* if you are unable to investigate something properly because you do not have access to the right data, explicitly tell the user that you are missing an integration to access XYZ which you would need to investigate. you should specifically use the templated phrase "I don't have access to <details>. Please add a Holmes integration for <XYZ> so that I can investigate this."
+* always provide detailed information like exact resource names, versions, labels, etc
+* even if you found the root cause, keep investigating to find other possible root causes and to gather data for the answer like exact names
+* if a runbook url is present as well as tool that can fetch it, you MUST fetch the runbook before beginning your investigation.
+* if you don't know, say that the analysis was inconclusive.
+* if there are multiple possible causes list them in a numbered list.
+* there will often be errors in the data that are not relevant or that do not have an impact - ignore them in your conclusion if you were not able to tie them to an actual error.
+* run as many kubectl commands as you need to gather more information, then respond.
+* if possible, do so repeatedly on different Kubernetes objects.
+* for example, for deployments first run kubectl on the deployment then a replicaset inside it, then a pod inside that.
+* when investigating a pod that crashed or application errors, always run kubectl_describe and fetch logs with both kubectl_previous_logs and kubectl_logs so that you see current logs and any logs from before a crash.
+* do not give an answer like "The pod is pending" as that doesn't state why the pod is pending and how to fix it.
+* do not give an answer like "Pod's node affinity/selector doesn't match any available nodes" because that doesn't include data on WHICH label doesn't match
+* if investigating an issue on many pods, there is no need to check more than 3 individual pods in the same deployment. pick up to a representative 3 from each deployment if relevant
+* if you find errors and warning in a pods logs and you believe they indicate a real issue. consider the pod as not healthy. 
+* if the user says something isn't working, ALWAYS:
+** use kubectl_describe on the owner workload + individual pods and look for any transient issues they might have been referring to
+** check the application aspects with kubectl_logs + kubectl_previous_logs and other relevant tools
+** look for misconfigured ingresses/services etc
+
+Style guide:
+* Be painfully concise.
+* Leave out "the" and filler words when possible.
+* Be terse but not at the expense of leaving out important data like the root cause and how to fix.
+* return a json object with the following schema as a result:
+{
+  "type": "object",
+  "properties": {
+    "root_cause_summary": {
+      "type": "string",
+      "description": "concise short explaination leading to the workload_healthy result, pinpoint reason and root cause for the workload issues if any."
+    },
+    "workload_healthy": {
+      "type": "boolean",
+      "description": "is the workload in healthy state or in error state"
+    }
+  },
+  "required": [
+    "reasoning",
+    "workload_healthy"
+  ]
+}
+
+
+{% if alerts %}
+Here are issues and configuration changes that happend to this kubernetes workload in recent time. Check if these can help you understand the issue.
+{% for a in alerts %}
+{{ a }}
+{% endfor %}
+{% endif %}

server.py

@@ -49,7 +49,7 @@ class WorkloadHealthRequest(BaseModel):
     instructions: Optional[List[str]] = []
     include_tool_calls: bool = False
     include_tool_call_results: bool = False
-    prompt_template: str = "builtin://generic_ask.jinja2"
+    prompt_template: str = "builtin://kubernetes_workload_ask.jinja2"
 
 
 def init_logging():
@@ -121,26 +121,36 @@ def investigate_issues(investigate_request: InvestigateRequest):
 def workload_health_check(request: WorkloadHealthRequest):
 
     try:
+        resource = request.resource
         workload_alerts: list[str] = []
         if request.alert_history:
-            workload_alerts = dal.get_workload_issues(request.resource, request.alert_history_since_hours)
+            workload_alerts = dal.get_workload_issues(resource, request.alert_history_since_hours)
+
+        instructions = request.instructions
+        if request.stored_instrucitons:
+            instructions = dal.get_resource_instructions(resource.get("kind"), resource.get("name"))
+
+        nl = '\n'
+        if instructions:
+            request.ask = f"{request.ask}\n My instructions for the investigation '''{nl.join(instructions)}'''"
 
         system_prompt = load_prompt(request.prompt_template)
         system_prompt = jinja2.Environment().from_string(system_prompt)
         system_prompt = system_prompt.render(alerts=workload_alerts)
 
         ai = config.create_toolcalling_llm(console, allowed_toolsets=ALLOWED_TOOLSETS)
 
-        ai_call = ai.call(system_prompt, request.ask, HOLMES_POST_PROCESSING_PROMPT)
+        structured_output = {"type": "json_object"}
+        ai_call = ai.call(system_prompt, request.ask, HOLMES_POST_PROCESSING_PROMPT, structured_output)
 
         return InvestigationResult(
             analysis=ai_call.result,
             tool_calls=ai_call.tool_calls,
-            instructions=ai_call.instructions
+            instructions=instructions
         )
     except AuthenticationError as e:
-        raise HTTPException(status_code=401, detail=e.message)          
+        raise HTTPException(status_code=401, detail=e.message)
 
 
 if __name__ == "__main__":
-    uvicorn.run(app, host=HOLMES_HOST, port=HOLMES_PORT)
+    uvicorn.run(app, host=HOLMES_HOST, port=HOLMES_PORT)