diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index cab284986f5d..cd51b853b42b 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -1094,6 +1094,12 @@ def run_conversation( request_messages = api_kwargs.get("input") if not isinstance(request_messages, list): request_messages = api_messages + # Anthropic Messages API moves system out of messages; + # pass it explicitly for observability plugins (Langfuse). + system_prompt_for_hooks = api_kwargs.get("system") + if system_prompt_for_hooks is None and isinstance(request_messages, list): + if request_messages and request_messages[0].get("role") == "system": + system_prompt_for_hooks = request_messages[0].get("content") # Shallow-copy the outer list so plugins that retain the # reference for async snapshotting don't observe later # mutations of api_messages. The inner dicts are not @@ -1113,6 +1119,7 @@ def run_conversation( api_mode=agent.api_mode, api_call_count=api_call_count, request_messages=list(request_messages) if isinstance(request_messages, list) else [], + system_prompt=system_prompt_for_hooks, message_count=len(api_messages), tool_count=len(agent.tools or []), approx_input_tokens=approx_tokens, diff --git a/plugins/observability/langfuse/README.md b/plugins/observability/langfuse/README.md index 97f4757e5a84..6c4bd4c97c4f 100644 --- a/plugins/observability/langfuse/README.md +++ b/plugins/observability/langfuse/README.md @@ -5,16 +5,20 @@ you explicitly enable it. ## Enable +Pick one: + ```bash +# Interactive: walks you through credentials + SDK install + enable +hermes tools # → Langfuse Observability + +# Manual pip install langfuse hermes plugins enable observability/langfuse ``` -Or check the box in the interactive `hermes plugins` UI. - ## Required credentials -Set these in `~/.hermes/.env`: +Set these in `~/.hermes/.env` (or via `hermes tools`): ```bash HERMES_LANGFUSE_PUBLIC_KEY=pk-lf-... @@ -32,6 +36,10 @@ hermes plugins list # observability/langfuse should show "enable hermes chat -q "hello" # then check Langfuse for a "Hermes turn" trace ``` +Generation observations include the Hermes system prompt when the provider +uses a separate `system` param (Anthropic Messages API). Open an **LLM call** +child span to inspect `role: system` (truncated via `HERMES_LANGFUSE_MAX_CHARS`). + ## Optional tuning ```bash diff --git a/plugins/observability/langfuse/__init__.py b/plugins/observability/langfuse/__init__.py index a99a8eb92791..caea1b2449b9 100644 --- a/plugins/observability/langfuse/__init__.py +++ b/plugins/observability/langfuse/__init__.py @@ -421,6 +421,53 @@ def _coerce_request_messages( return [{"role": "user", "content": user_message}] +def _serialize_system_prompt(system_prompt: Any) -> Optional[dict[str, Any]]: + """Normalize Anthropic ``system`` param or OpenAI-style system content for Langfuse.""" + if system_prompt is None: + return None + if isinstance(system_prompt, str): + text = system_prompt.strip() + if not text: + return None + return {"role": "system", "content": _safe_value(text)} + if isinstance(system_prompt, list): + parts: list[str] = [] + for block in system_prompt: + if isinstance(block, dict) and block.get("type") == "text": + piece = block.get("text", "") + if isinstance(piece, str) and piece: + parts.append(piece) + elif isinstance(block, str) and block: + parts.append(block) + if not parts: + return None + return {"role": "system", "content": _safe_value("\n\n".join(parts))} + return None + + +def _messages_for_langfuse_input( + *, + request_messages: Any = None, + messages: Any = None, + conversation_history: Any = None, + user_message: Any = None, + system_prompt: Any = None, +) -> list[dict[str, Any]]: + """Build generation input: include Anthropic ``system`` when split out of ``messages``.""" + raw = _coerce_request_messages( + request_messages=request_messages, + messages=messages, + conversation_history=conversation_history, + user_message=user_message, + ) + if raw and raw[0].get("role") == "system": + return _serialize_messages(raw) + system_msg = _serialize_system_prompt(system_prompt) + if system_msg is None: + return _serialize_messages(raw) + return [system_msg, *_serialize_messages(raw)] + + def _serialize_messages(messages: Any) -> list[dict[str, Any]]: if not isinstance(messages, list): return [] @@ -746,6 +793,7 @@ def on_pre_llm_request( max_tokens: Any = None, conversation_history: Any = None, user_message: Any = None, + system_prompt: Any = None, **_: Any, ) -> None: client = _get_langfuse() @@ -758,6 +806,16 @@ def on_pre_llm_request( conversation_history=conversation_history, user_message=user_message, ) + langfuse_input = _messages_for_langfuse_input( + request_messages=request_messages, + messages=messages, + conversation_history=conversation_history, + user_message=user_message, + system_prompt=system_prompt, + ) + system_chars = 0 + if langfuse_input and langfuse_input[0].get("role") == "system": + system_chars = len(str(langfuse_input[0].get("content") or "")) task_key = _trace_key(task_id, session_id) req_key = _request_key(api_call_count) @@ -781,18 +839,23 @@ def on_pre_llm_request( previous = state.generations.pop(req_key, None) if previous is not None: _end_observation(previous) + gen_metadata = { + "provider": provider, + "platform": platform, + "api_mode": api_mode, + "base_url": base_url, + "message_count": message_count, + "approx_input_tokens": approx_input_tokens, + } + if system_chars: + gen_metadata["system_prompt_chars"] = system_chars state.generations[req_key] = _start_child_observation( state, client=client, name=f"LLM call {api_call_count}", as_type="generation", - input_value=_serialize_messages(input_messages), - metadata={ - "provider": provider, - "platform": platform, - "api_mode": api_mode, - "base_url": base_url, - }, + input_value=langfuse_input, + metadata=gen_metadata, model=model, model_parameters={"api_mode": api_mode, "provider": provider}, ) diff --git a/tests/plugins/test_langfuse_plugin.py b/tests/plugins/test_langfuse_plugin.py index 313d2e94a72f..3d9580f54577 100644 --- a/tests/plugins/test_langfuse_plugin.py +++ b/tests/plugins/test_langfuse_plugin.py @@ -467,6 +467,33 @@ def test_prefers_request_messages_then_messages_then_history_then_user_message(s ) == [{"role": "user", "content": "h"}] assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}] + def test_messages_for_langfuse_includes_anthropic_system_param(self): + sys.modules.pop("plugins.observability.langfuse", None) + mod = importlib.import_module("plugins.observability.langfuse") + + out = mod._messages_for_langfuse_input( + request_messages=[{"role": "user", "content": "hi"}], + system_prompt="You are Hermes.", + ) + assert out[0]["role"] == "system" + assert out[0]["content"] == "You are Hermes." + assert out[1]["role"] == "user" + + def test_messages_for_langfuse_skips_duplicate_system(self): + sys.modules.pop("plugins.observability.langfuse", None) + mod = importlib.import_module("plugins.observability.langfuse") + + out = mod._messages_for_langfuse_input( + request_messages=[ + {"role": "system", "content": "already here"}, + {"role": "user", "content": "hi"}, + ], + system_prompt="ignored when messages include system", + ) + assert out[0]["role"] == "system" + assert out[0]["content"] == "already here" + assert out[1]["role"] == "user" + class TestToolCallOutputBackfill: def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):