Remove post-processing support

docs/data-sources/custom-toolsets.md

@@ -359,7 +359,7 @@ toolsets:
           - name: param_name
             description: "Parameter description"
         additionalInstructions: |
-          Instructions for post-processing the command output
+          Additional guidance for interpreting the command output
 ```
 
 ### Tool Configuration

docs/reference/environment-variables.md

@@ -82,9 +82,6 @@ export HOLMES_LOG_LEVEL="DEBUG"
 ### HOLMES_CACHE_DIR
 Directory for caching HolmesGPT data and temporary files.
 
-### HOLMES_POST_PROCESSING_PROMPT
-Custom prompt template for post-processing LLM responses.
-
 ## Data Source Configuration
 
 ### Prometheus

docs/reference/helm-configuration.md

@@ -184,10 +184,6 @@ additionalVolumeMounts: []
 # OpenShift compatibility mode
 openshift: false
 
-# Post-processing configuration
-enablePostProcessing: false
-postProcessingPrompt: "builtin://generic_post_processing.jinja2"
-
 # Account creation
 enableAccountsCreate: true
 

helm/holmes/templates/holmes.yaml

@@ -73,10 +73,6 @@ spec:
           - name: CERTIFICATE
             value: {{ .Values.certificate }}
           {{- end }}
-          {{ if .Values.enablePostProcessing -}}
-          - name: HOLMES_POST_PROCESSING_PROMPT
-            value: {{ .Values.postProcessingPrompt }}
-          {{- end }}
           {{ if .Values.openshift }}
           - name: IS_OPENSHIFT
             value: "True"

helm/holmes/values.yaml

@@ -41,8 +41,6 @@ crdPermissions:
   velero: true
   externalSecrets: true
 
-enablePostProcessing: false
-postProcessingPrompt: "builtin://generic_post_processing.jinja2"
 openshift: false
 
 affinity: {}

holmes/common/env_vars.py

@@ -35,7 +35,6 @@ def load_bool(env_var, default: Optional[bool]) -> Optional[bool]:
 STORE_API_KEY = os.environ.get("STORE_API_KEY", "")
 STORE_EMAIL = os.environ.get("STORE_EMAIL", "")
 STORE_PASSWORD = os.environ.get("STORE_PASSWORD", "")
-HOLMES_POST_PROCESSING_PROMPT = os.environ.get("HOLMES_POST_PROCESSING_PROMPT", "")
 ROBUSTA_AI = load_bool("ROBUSTA_AI", None)
 LOAD_ALL_ROBUSTA_MODELS = load_bool("LOAD_ALL_ROBUSTA_MODELS", True)
 ROBUSTA_API_ENDPOINT = os.environ.get("ROBUSTA_API_ENDPOINT", "https://api.robusta.dev")

holmes/core/investigation.py

@@ -1,8 +1,6 @@
 import logging
 from typing import Optional
 
-
-from holmes.common.env_vars import HOLMES_POST_PROCESSING_PROMPT
 from holmes.config import Config
 from holmes.core.investigation_structured_output import process_response_into_sections
 from holmes.core.issue import Issue
@@ -57,7 +55,6 @@ def investigate_issues(
     investigation = ai.investigate(
         issue,
         prompt=investigate_request.prompt_template,
-        post_processing_prompt=HOLMES_POST_PROCESSING_PROMPT,
         global_instructions=global_instructions,
         sections=investigate_request.sections,
         trace_span=trace_span,

holmes/core/tool_calling_llm.py

@@ -53,7 +53,7 @@
 from holmes.utils.global_instructions import (
     Instructions,
 )
-from holmes.utils.tags import format_tags_in_string, parse_messages_tags
+from holmes.utils.tags import parse_messages_tags
 from holmes.core.tools_utils.tool_executor import ToolExecutor
 from holmes.core.tracing import DummySpan
 from holmes.utils.colors import AI_COLOR
@@ -269,7 +269,6 @@ def prompt_call(
         self,
         system_prompt: str,
         user_prompt: str,
-        post_process_prompt: Optional[str] = None,
         response_format: Optional[Union[dict, Type[BaseModel]]] = None,
         sections: Optional[InputSectionsDataType] = None,
         trace_span=DummySpan(),
@@ -280,8 +279,7 @@ def prompt_call(
         ]
         return self.call(
             messages,
-            post_process_prompt,
-            response_format,
+            response_format=response_format,
             user_prompt=user_prompt,
             sections=sections,
             trace_span=trace_span,
@@ -290,19 +288,15 @@ def prompt_call(
     def messages_call(
         self,
         messages: List[Dict[str, str]],
-        post_process_prompt: Optional[str] = None,
         response_format: Optional[Union[dict, Type[BaseModel]]] = None,
         trace_span=DummySpan(),
     ) -> LLMResult:
-        return self.call(
-            messages, post_process_prompt, response_format, trace_span=trace_span
-        )
+        return self.call(messages, response_format=response_format, trace_span=trace_span)
 
     @sentry_sdk.trace
     def call(  # type: ignore
         self,
         messages: List[Dict[str, str]],
-        post_process_prompt: Optional[str] = None,
         response_format: Optional[Union[dict, Type[BaseModel]]] = None,
         user_prompt: Optional[str] = None,
         sections: Optional[InputSectionsDataType] = None,
@@ -403,40 +397,15 @@ def call(  # type: ignore
                 )
 
             if not tools_to_call:
-                # For chatty models post process and summarize the result
-                # this only works for calls where user prompt is explicitly passed through
-                if post_process_prompt and user_prompt:
-                    logging.info("Running post processing on investigation.")
-                    raw_response = text_response
-                    post_processed_response, post_processing_cost = (
-                        self._post_processing_call(
-                            prompt=user_prompt,
-                            investigation=raw_response,
-                            user_prompt=post_process_prompt,
-                        )
-                    )
-                    costs.total_cost += post_processing_cost
-
-                    tokens = self.llm.count_tokens(messages=messages, tools=tools)
-
-                    add_token_count_to_metadata(
-                        tokens=tokens,
-                        full_llm_response=full_response,
-                        max_context_size=limit_result.max_context_size,
-                        maximum_output_token=limit_result.maximum_output_token,
-                        metadata=metadata,
-                    )
+                tokens = self.llm.count_tokens(messages=messages, tools=tools)
 
-                    return LLMResult(
-                        result=post_processed_response,
-                        unprocessed_result=raw_response,
-                        tool_calls=all_tool_calls,
-                        num_llm_calls=i,
-                        prompt=json.dumps(messages, indent=2),
-                        messages=messages,
-                        **costs.model_dump(),  # Include all cost fields
-                        metadata=metadata,
-                    )
+                add_token_count_to_metadata(
+                    tokens=tokens,
+                    full_llm_response=full_response,
+                    max_context_size=limit_result.max_context_size,
+                    maximum_output_token=limit_result.maximum_output_token,
+                    metadata=metadata,
+                )
 
                 return LLMResult(
                     result=text_response,
@@ -749,54 +718,6 @@ def _handle_tool_call_approval(
 
         return tool_call_result
 
-    @staticmethod
-    def __load_post_processing_user_prompt(
-        input_prompt, investigation, user_prompt: Optional[str] = None
-    ) -> str:
-        if not user_prompt:
-            user_prompt = "builtin://generic_post_processing.jinja2"
-        return load_and_render_prompt(
-            user_prompt, {"investigation": investigation, "prompt": input_prompt}
-        )
-
-    def _post_processing_call(
-        self,
-        prompt,
-        investigation,
-        user_prompt: Optional[str] = None,
-        system_prompt: str = "You are an AI assistant summarizing Kubernetes issues.",
-    ) -> tuple[Optional[str], float]:
-        try:
-            user_prompt = ToolCallingLLM.__load_post_processing_user_prompt(
-                prompt, investigation, user_prompt
-            )
-
-            logging.debug(f'Post processing prompt:\n"""\n{user_prompt}\n"""')
-            messages = [
-                {
-                    "role": "system",
-                    "content": system_prompt,
-                },
-                {
-                    "role": "user",
-                    "content": format_tags_in_string(user_prompt),
-                },
-            ]
-            full_response = self.llm.completion(messages=messages, temperature=0)
-            logging.debug(f"Post processing response {full_response}")
-
-            # Extract and log cost information for post-processing
-            post_processing_cost = _extract_cost_from_response(full_response)
-            if post_processing_cost > 0:
-                cost_logger.debug(
-                    f"Post-processing LLM cost: ${post_processing_cost:.6f}"
-                )
-
-            return full_response.choices[0].message.content, post_processing_cost  # type: ignore
-        except Exception:
-            logging.exception("Failed to run post processing", exc_info=True)
-            return investigation, 0.0
-
     def call_stream(
         self,
         system_prompt: str = "",
@@ -1080,7 +1001,6 @@ def investigate(
         prompt: str,
         console: Optional[Console] = None,
         global_instructions: Optional[Instructions] = None,
-        post_processing_prompt: Optional[str] = None,
         sections: Optional[InputSectionsDataType] = None,
         trace_span=DummySpan(),
         runbooks: Optional[RunbookCatalog] = None,
@@ -1154,7 +1074,6 @@ def investigate(
         res = self.prompt_call(
             system_prompt,
             user_prompt,
-            post_processing_prompt,
             response_format=response_format,
             sections=sections,
             trace_span=trace_span,

holmes/interactive.py

@@ -991,7 +991,6 @@ def run_interactive_loop(
     console: Console,
     initial_user_input: Optional[str],
     include_files: Optional[List[Path]],
-    post_processing_prompt: Optional[str],
     show_tool_output: bool,
     tracer=None,
     runbooks=None,
@@ -1268,7 +1267,6 @@ def get_bottom_toolbar():
                 )
                 response = ai.call(
                     messages,
-                    post_processing_prompt,
                     trace_span=trace_span,
                     tool_number_offset=len(all_tool_calls_history),
                 )

holmes/main.py

@@ -137,13 +137,6 @@
     envvar="HOLMES_JSON_OUTPUT_FILE",
 )
 
-opt_post_processing_prompt: Optional[str] = typer.Option(
-    None,
-    "--post-processing-prompt",
-    help="Adds a prompt for post processing. (Preferable for chatty ai models)",
-    envvar="HOLMES_POST_PROCESSING_PROMPT",
-)
-
 opt_documents: Optional[str] = typer.Option(
     None,
     "--documents",
@@ -201,7 +194,6 @@ def ask(
     ),
     json_output_file: Optional[str] = opt_json_output_file,
     echo_request: bool = opt_echo_request,
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
     interactive: bool = typer.Option(
         True,
         "--interactive/--no-interactive",
@@ -299,7 +291,6 @@ def ask(
             console,
             prompt,
             include_file,
-            post_processing_prompt,
             show_tool_output,
             tracer,
             config.get_runbook_catalog(),
@@ -321,7 +312,7 @@ def ask(
         f'holmes ask "{prompt}"', span_type=SpanType.TASK
     ) as trace_span:
         trace_span.log(input=prompt, metadata={"type": "user_question"})
-        response = ai.call(messages, post_processing_prompt, trace_span=trace_span)
+        response = ai.call(messages, trace_span=trace_span)
         trace_span.log(
             output=response.result,
         )
@@ -393,7 +384,6 @@ def alertmanager(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_investigation.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
 ):
     """
     Investigate a Prometheus/Alertmanager alert
@@ -449,7 +439,6 @@ def alertmanager(
             issue=issue,
             prompt=system_prompt,  # type: ignore
             console=console,
-            post_processing_prompt=post_processing_prompt,
         )
         results.append({"issue": issue.model_dump(), "result": result.model_dump()})
         handle_result(result, console, destination, config, issue, False, True)  # type: ignore
@@ -526,7 +515,6 @@ def jira(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_investigation.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
 ):
     """
     Investigate a Jira ticket
@@ -565,7 +553,6 @@ def jira(
             issue=issue,
             prompt=system_prompt,  # type: ignore
             console=console,
-            post_processing_prompt=post_processing_prompt,
         )
 
         console.print(Rule())
@@ -618,7 +605,6 @@ def ticket(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_ticket.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
     model: Optional[str] = opt_model,
 ):
     """
@@ -670,7 +656,7 @@ def ticket(
     )
 
     ticket_user_prompt = generate_user_prompt(prompt, context={})
-    result = ai.prompt_call(system_prompt, ticket_user_prompt, post_processing_prompt)
+    result = ai.prompt_call(system_prompt, ticket_user_prompt)
 
     console.print(Rule())
     console.print(
@@ -717,7 +703,6 @@ def github(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_investigation.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
 ):
     """
     Investigate a GitHub issue
@@ -756,7 +741,6 @@ def github(
             issue=issue,
             prompt=system_prompt,  # type: ignore
             console=console,
-            post_processing_prompt=post_processing_prompt,
         )
 
         console.print(Rule())
@@ -802,7 +786,6 @@ def pagerduty(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_investigation.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
 ):
     """
     Investigate a PagerDuty incident
@@ -841,7 +824,6 @@ def pagerduty(
             issue=issue,
             prompt=system_prompt,  # type: ignore
             console=console,
-            post_processing_prompt=post_processing_prompt,
         )
 
         console.print(Rule())
@@ -886,7 +868,6 @@ def opsgenie(
     system_prompt: Optional[str] = typer.Option(
         "builtin://generic_investigation.jinja2", help=system_prompt_help
     ),
-    post_processing_prompt: Optional[str] = opt_post_processing_prompt,
     documents: Optional[str] = opt_documents,
 ):
     """
@@ -923,7 +904,6 @@ def opsgenie(
             issue=issue,
             prompt=system_prompt,  # type: ignore
             console=console,
-            post_processing_prompt=post_processing_prompt,
         )
 
         console.print(Rule())