From bbb9e9cf4c04b2a617c4ed9bd3ab4625c9243399 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Mon, 31 Aug 2026 15:20:11 +0100 Subject: [PATCH 1/6] feat(agentic_rag): add config-driven MLflow GenAI evaluation - Config-driven scorer setup via evaluation/eval_config.yaml (7 core + 3 RAG scorers, 10 total) - evaluation/scorers.py for custom scorer definitions - evaluation/run_eval.py: sends golden queries to agent, reads traces, evaluates with dynamically resolved scorers - Content-based poll matching for data readiness - httpx.RequestError catch-all for invalid URLs - Empty-trace guard before evaluate() - Configurable EVAL_REQUEST_TIMEOUT and EVAL_POLL_INTERVAL - evaluation/eval_data.yaml: placeholder golden queries with expectations - RETRIEVER span logging in tools.py for RAG scorer compatibility - make eval target - EVAL_JUDGE_*, EVAL_ENABLE_PII, EVAL_REQUEST_TIMEOUT, EVAL_POLL_INTERVAL, AGENT_URL env vars in .env.example - README evaluation section with scorer docs and enablement link Co-Authored-By: Claude Opus 4.6 (1M context) --- .../templates/agentic_rag/.env.example | 13 + .../langgraph/templates/agentic_rag/Makefile | 5 +- .../langgraph/templates/agentic_rag/README.md | 156 +++++++++- .../agentic_rag/evaluation/__init__.py | 0 .../agentic_rag/evaluation/eval_config.yaml | 60 ++++ .../agentic_rag/evaluation/eval_data.yaml | 35 +++ .../agentic_rag/evaluation/run_eval.py | 288 ++++++++++++++++++ .../agentic_rag/evaluation/scorers.py | 17 ++ .../templates/agentic_rag/pyproject.toml | 9 +- .../agentic_rag/src/agentic_rag/tools.py | 21 ++ 10 files changed, 601 insertions(+), 3 deletions(-) create mode 100644 agents/langgraph/templates/agentic_rag/evaluation/__init__.py create mode 100644 agents/langgraph/templates/agentic_rag/evaluation/eval_config.yaml create mode 100644 agents/langgraph/templates/agentic_rag/evaluation/eval_data.yaml create mode 100644 agents/langgraph/templates/agentic_rag/evaluation/run_eval.py create mode 100644 agents/langgraph/templates/agentic_rag/evaluation/scorers.py diff --git a/agents/langgraph/templates/agentic_rag/.env.example b/agents/langgraph/templates/agentic_rag/.env.example index e6715362..58b2a6cc 100644 --- a/agents/langgraph/templates/agentic_rag/.env.example +++ b/agents/langgraph/templates/agentic_rag/.env.example @@ -32,5 +32,18 @@ PORT=8000 # MLFLOW_WORKSPACE=default # MLFLOW_TRACKING_AUTH= # Use Kubernetes service account for authentication (if running inside the cluster) +# Optional — Evaluation (only needed for `make eval`) +# The judge model scores agent responses. Can be the same as the agent's model or a different one. +# EVAL_JUDGE_MODEL=openai:/gpt-4o-mini +# EVAL_JUDGE_API_KEY= +# EVAL_JUDGE_BASE_URL=https://api.openai.com/v1 +# EVAL_PII_MODEL=openai:/gpt-4.1-mini +# EVAL_ENABLE_PII=false +# EVAL_REQUEST_TIMEOUT=60 +# EVAL_TRACE_TIMEOUT=60 +# EVAL_POLL_INTERVAL=4 +DEEPEVAL_TELEMETRY_OPT_OUT=YES +# AGENT_URL=http://localhost:8000 + # Optional — openshell sandbox # OPENSHELL_SANDBOX_NAME=rag-sandbox diff --git a/agents/langgraph/templates/agentic_rag/Makefile b/agents/langgraph/templates/agentic_rag/Makefile index a6c2063d..85729f96 100644 --- a/agents/langgraph/templates/agentic_rag/Makefile +++ b/agents/langgraph/templates/agentic_rag/Makefile @@ -7,7 +7,7 @@ CONTAINER_CLI := $(shell command -v podman 2>/dev/null || command -v docker 2>/ MODEL ?= llama3.1:8b OLLAMA_EMBEDDING_MODEL ?= embeddinggemma:latest -.PHONY: init re-init env ollama ogx-server run-app run-app-fresh run-cli load-docs build push build-openshift deploy undeploy test test-integration dry-run setup-gateway teardown-gateway build-openshell create-sandbox wait-sandbox setup-egress load-docs-sandbox start-agent deploy-openshell undeploy-openshell playground-sandbox help +.PHONY: init re-init env ollama ogx-server run-app run-app-fresh run-cli load-docs build push build-openshift deploy undeploy test test-integration eval dry-run setup-gateway teardown-gateway build-openshell create-sandbox wait-sandbox setup-egress load-docs-sandbox start-agent deploy-openshell undeploy-openshell playground-sandbox help help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-14s %s\n", $$1, $$2}' @@ -442,6 +442,9 @@ playground-sandbox: ## Start sandbox playground web UI (auto-fetches agent URL a echo "" && \ python playground-sandbox/app.py +eval: ## Run MLflow GenAI evaluation + @uv run --extra eval python evaluation/run_eval.py + test: ## Run unit tests uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS) diff --git a/agents/langgraph/templates/agentic_rag/README.md b/agents/langgraph/templates/agentic_rag/README.md index 65887252..9dfb4e51 100644 --- a/agents/langgraph/templates/agentic_rag/README.md +++ b/agents/langgraph/templates/agentic_rag/README.md @@ -48,7 +48,7 @@ make env ### Tracing (optional) -Tracing is optional. If MLflow tracing is required, enable it by uncommenting and setting the following environment variables in the `.env` file. +Tracing is optional. If MLflow tracing is required, enable it by uncommenting and setting the following environment variables in the `.env` file. For how tracing works under the hood (span structure, autolog behavior, verification steps), see [tracing.md](../../../../tracing.md). #### Tracing with a local MLflow server @@ -448,6 +448,160 @@ python -c "from agent_auth.middleware import SATokenAuthMiddleware; print('OK')" make test ``` +## Evaluation + +This template includes a config-driven MLflow GenAI evaluation setup. The eval script reads existing traces from MLflow and scores them using scorers defined in `evaluation/eval_config.yaml`. + +### Eval Directory Structure + +```text +evaluation/ +├── eval_config.yaml # Scorer configuration (core, use-case, guidelines) +├── eval_data.yaml # Golden queries with expected results +├── scorers.py # Custom scorer definitions (add your own here) +└── run_eval.py # Evaluation logic (no need to edit) +``` + +### Prerequisites + +- Agent must be running with tracing enabled (`MLFLOW_TRACKING_URI` set in `.env`) +- Complete the Setup steps below + +### Setup + +**Set environment variables** in `.env`: + +```ini +# Required for eval +EVAL_JUDGE_MODEL=:/ +EVAL_JUDGE_API_KEY= +EVAL_JUDGE_BASE_URL= # only needed for OpenAI-compatible endpoints +``` + +The judge model format is `:/`. The eval script automatically sets the correct API key env var based on the provider. Examples: + +- **OpenAI:** `EVAL_JUDGE_MODEL=openai:/gpt-4o-mini`, `EVAL_JUDGE_BASE_URL=https://api.openai.com/v1` +- **vLLM / RHOAI:** `EVAL_JUDGE_MODEL=openai:/`, `EVAL_JUDGE_BASE_URL=https:///v1` +- **Anthropic:** `EVAL_JUDGE_MODEL=anthropic:/claude-sonnet-4-20250514` (no `BASE_URL` needed) + +### Running Evaluation + +1. Add your golden queries and expectations to `evaluation/eval_data.yaml` +2. Customize the guidelines in `evaluation/eval_config.yaml` for your domain +3. Start your agent with tracing enabled (`make run-app`) if it's not already running +4. Run: + + ```bash + make eval + ``` + + This will: + - Send each golden query from `eval_data.yaml` to the running agent + - Wait for traces to be recorded in MLflow + - Attach expectations (e.g., `expected_facts`) to matching traces + - Score all traces with the configured scorers + + By default, the eval script connects to the agent at `http://localhost:8000`. To target a deployed agent, set `AGENT_URL` in `.env`: + + ```ini + AGENT_URL=https:// + ``` + +### Scorer Configuration + +Scorers are configured in `evaluation/eval_config.yaml`, organized into three sections. Scorers in both **core** and **use-case** sections can come from any of three sources: + +| Source | `module` value | Example | +|--------|---------------|---------| +| MLflow built-in | `mlflow.genai.scorers` | Safety, Correctness | +| Third-party (any MLflow-integrated framework) | `mlflow.genai.scorers.` | DeepEval PIILeakage, Phoenix Hallucination | +| Custom (defined in `scorers.py`) | `scorers` | Your own domain-specific scorers | + +#### Core Scorers + +Cover universal failure modes. Pre-configured for all agents — remove only with a specific reason. + +| Scorer | What it checks | +|--------|---------------| +| **Safety** | Responses are free of harmful or toxic content | +| **RelevanceToQuery** | Responses directly address the user's question | +| **Completeness** | All parts of the user's request are addressed | +| **Correctness** | Responses match expected facts (requires `expected_facts` in eval data) | +| **ToolCallCorrectness** | Agent selects appropriate tools with correct arguments | +| **ToolCallEfficiency** | No redundant or duplicate tool calls | +| **PIILeakage** *(opt-in)* | Responses don't leak personally identifiable information | + +> **PIILeakage** is disabled by default. Enable it by setting `EVAL_ENABLE_PII=true` in `.env`. It uses DeepEval and defaults to `openai:/gpt-4.1-mini` (override with `EVAL_PII_MODEL`). DeepEval has a known JSON parsing issue with `gpt-4o` — use `gpt-4.1-mini` or `gpt-4o-mini`. + +#### Use-case Scorers (RAG) + +This template includes three RAG-specific scorers pre-configured under `use_case`. These evaluate retrieval quality and require `RETRIEVER`-type spans in traces (handled automatically by the retriever tool). + +| Scorer | What it checks | +|--------|---------------| +| **RetrievalGroundedness** | The final answer is grounded in the retrieved documents | +| **RetrievalRelevance** | Each retrieved document is relevant to the query (per-document) | +| **RetrievalSufficiency** | The retrieved documents provide enough information to answer | + +> **RetrievalRelevance** is a per-document scorer — it produces one assessment per retrieved chunk. Queries that don't trigger retrieval (e.g., greetings) will show errors for RAG scorers, which is expected. + +You can add more use-case scorers from MLflow, third-party frameworks, or custom definitions in `scorers.py`: + +```yaml +use_case: + # Third-party scorer (any MLflow-integrated framework) + - name: Faithfulness + module: mlflow.genai.scorers.deepeval + + # Custom scorer (defined in scorers.py) + - name: my_domain_scorer + module: scorers +``` + +#### Guidelines + +A list of plain-text rules evaluated by the `Guidelines` scorer. Customize these for your agent: + +```yaml +guidelines: + - "The response should be helpful and informative" + - "The response should not fabricate information" + - "Answers to knowledge questions must be grounded in retrieved documents" +``` + +### Adding a Custom Scorer + +1. Define your scorer in `evaluation/scorers.py`: + + ```python + from mlflow.genai.scorers import scorer + + @scorer + def my_domain_scorer(request, response, expected_facts=None): + """Check domain-specific quality criteria.""" + # Your scoring logic here + return {"score": 1.0, "rationale": "Meets criteria"} + ``` + +2. Add it to `evaluation/eval_config.yaml` under `core` or `use_case`: + + ```yaml + use_case: + - name: my_domain_scorer + module: scorers + ``` + +3. Run `make eval` — your scorer will be picked up automatically. + +### Viewing Results + +Results are logged to MLflow. View them in the MLflow UI under the Traces tab. + +### Advanced Evaluation + +For failure mode deep-dives, custom scorer patterns, and end-to-end evaluation, see the +[Agentic Evaluation Enablement Material](https://github.com/red-hat-data-services/red-hat-ai-examples/tree/main/examples/agentic-evaluation). + ## API Endpoints ### POST /chat/completions diff --git a/agents/langgraph/templates/agentic_rag/evaluation/__init__.py b/agents/langgraph/templates/agentic_rag/evaluation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/agents/langgraph/templates/agentic_rag/evaluation/eval_config.yaml b/agents/langgraph/templates/agentic_rag/evaluation/eval_config.yaml new file mode 100644 index 00000000..9e8db30a --- /dev/null +++ b/agents/langgraph/templates/agentic_rag/evaluation/eval_config.yaml @@ -0,0 +1,60 @@ +# Evaluation scorer configuration. +# +# Each scorer entry requires: +# name: Class or function name to import +# module: Python module to import from +# +# Optional: +# enabled_by_env: Only load this scorer if the specified env var is set to "true" +# model_env: Read the judge model from this env var instead of EVAL_JUDGE_MODEL +# +# Sources: +# MLflow scorers: module: mlflow.genai.scorers +# Third-party: module: mlflow.genai.scorers. (any MLflow-integrated framework, e.g., deepeval, ragas, phoenix etc.) +# Custom (scorers.py): module: scorers (custom built scorers) + +scorers: + # Core scorers — cover universal failure modes, apply to all agents. + # These come pre-configured. Remove only if you have a specific reason. + core: + - name: Safety + module: mlflow.genai.scorers + + - name: RelevanceToQuery + module: mlflow.genai.scorers + + - name: Completeness + module: mlflow.genai.scorers + + - name: Correctness + module: mlflow.genai.scorers + + - name: ToolCallCorrectness + module: mlflow.genai.scorers + + - name: ToolCallEfficiency + module: mlflow.genai.scorers + + - name: PIILeakage + module: mlflow.genai.scorers.deepeval + enabled_by_env: EVAL_ENABLE_PII + model_env: EVAL_PII_MODEL + + # Use-case specific scorers — RAG retrieval quality. + # These require RETRIEVER-type spans in traces (see tools.py). + use_case: + - name: RetrievalGroundedness + module: mlflow.genai.scorers + + - name: RetrievalRelevance + module: mlflow.genai.scorers + + - name: RetrievalSufficiency + module: mlflow.genai.scorers + + # Domain-specific rules for the Guidelines scorer. + # Customize these for your agent's expected behavior. + guidelines: + - "The response should be helpful and informative" + - "The response should not fabricate information" + - "Answers to knowledge questions must be grounded in retrieved documents" diff --git a/agents/langgraph/templates/agentic_rag/evaluation/eval_data.yaml b/agents/langgraph/templates/agentic_rag/evaluation/eval_data.yaml new file mode 100644 index 00000000..61614299 --- /dev/null +++ b/agents/langgraph/templates/agentic_rag/evaluation/eval_data.yaml @@ -0,0 +1,35 @@ +# Golden queries for MLflow GenAI evaluation. +# +# Add your test queries below. Each query is sent to the running agent, +# and the resulting trace is scored by the configured scorers. +# +# Fields: +# inputs: Dict with the user query (required) +# expectations: Dict with ground truth (required for Correctness scorer) +# expected_facts: List of facts the response should contain +# expected_response: Reference response text +# +# Trace-based scorers (ToolCallCorrectness, ToolCallEfficiency) +# extract data from MLflow traces automatically — no expected tools needed here. + +queries: + # Example: query that requires a tool call + # - inputs: + # question: "Your test question here" + # expectations: + # expected_facts: + # - "Expected fact 1" + # - "Expected fact 2" + + # Example: query that should NOT trigger a tool call + # - inputs: + # question: "A simple greeting or general question" + # expectations: + # expected_response: "A friendly response without using any tools" + + # Example: edge case + # - inputs: + # question: "A query that tests error handling or boundaries" + # expectations: + # expected_facts: + # - "Expected behavior description" diff --git a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py new file mode 100644 index 00000000..68be987c --- /dev/null +++ b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py @@ -0,0 +1,288 @@ +"""MLflow GenAI evaluation script. + +Reads existing traces from MLflow and evaluates them using scorers +configured in eval_config.yaml. + +Usage: + make eval +""" + +import importlib +import inspect +import json +import os +import time +from pathlib import Path + +import httpx +import mlflow +import yaml +from dotenv import load_dotenv +from mlflow.genai.scorers import Guidelines + + +def load_eval_config(path: str | None = None) -> dict: + """Load scorer configuration from YAML.""" + if path is None: + path = str(Path(__file__).parent / "eval_config.yaml") + with open(path) as f: + return yaml.safe_load(f) or {} + + +def load_eval_data(path: str | None = None) -> list[dict]: + """Load golden queries with expectations from YAML.""" + if path is None: + path = str(Path(__file__).parent / "eval_data.yaml") + with open(path) as f: + data = yaml.safe_load(f) or {} + return data.get("queries") or [] + + +def generate_traces(eval_data: list[dict], agent_url: str) -> None: + """Send golden queries to the running agent to generate traces.""" + request_timeout = int(os.getenv("EVAL_REQUEST_TIMEOUT", "60")) + print(f"Sending {len(eval_data)} golden queries to {agent_url}...") + for query in eval_data: + question = query["inputs"]["question"] + try: + response = httpx.post( + f"{agent_url}/chat/completions", + json={ + "messages": [{"role": "user", "content": question}], + "stream": False, + }, + timeout=request_timeout, + ) + response.raise_for_status() + except httpx.ConnectError: + print( + f"\nERROR: Could not connect to agent at {agent_url}." + "\nIs the agent running? Start it with: make run-app" + ) + raise SystemExit(1) + except httpx.TimeoutException: + print( + f"\nERROR: Agent at {agent_url} did not respond within {request_timeout}s." + "\nThe agent may be overloaded or the model endpoint may be slow." + ) + raise SystemExit(1) + except httpx.HTTPStatusError as e: + print(f"\nERROR: Agent returned HTTP {e.response.status_code}.") + print("Check the agent logs for details.") + raise SystemExit(1) + except httpx.RequestError as e: + print(f"\nERROR: Request to {agent_url} failed: {e}") + raise SystemExit(1) + print(f" -> {question}") + print(f"Generated {len(eval_data)} traces.\n") + + +def _extract_question(trace): + """Extract the user question from a trace's request data.""" + try: + request = json.loads(trace.data.request) + return request["messages"][0]["content"] + except (json.JSONDecodeError, KeyError, IndexError, TypeError): + return None + + +def attach_expectations(traces, eval_data): + """Match traces to golden queries by content and log expectations. + + Each golden query's question is matched against the question extracted + from trace.data.request using exact string comparison. This is immune + to interleaved traces from concurrent agent traffic. + + Returns the set of trace IDs that matched a golden query. + """ + matched_ids = set() + + for query in eval_data: + question = query["inputs"]["question"] + for trace in traces: + if trace.info.trace_id in matched_ids: + continue + if _extract_question(trace) == question: + expectations = query.get("expectations", {}) + for name, value in expectations.items(): + mlflow.log_expectation( + trace_id=trace.info.trace_id, + name=name, + value=value, + ) + matched_ids.add(trace.info.trace_id) + break + + print( + f"Matched {len(matched_ids)}/{len(traces)} traces to golden queries with expectations." + ) + return matched_ids + + +def resolve_scorer(entry: dict, judge_model: str): + """Resolve a scorer entry from config into a scorer instance.""" + env_flag = entry.get("enabled_by_env") + if env_flag and os.getenv(env_flag, "").lower() != "true": + return None + + model = judge_model + if entry.get("model_env"): + model = os.getenv(entry["model_env"], model) + + module = importlib.import_module(entry["module"]) + cls_or_obj = getattr(module, entry["name"]) + + if inspect.isclass(cls_or_obj): + return cls_or_obj(model=model) + return cls_or_obj + + +def get_scorers(config: dict, judge_model: str) -> list: + """Build the scorer list from eval_config.yaml.""" + scorers = [] + scorer_config = config.get("scorers", {}) + + for entry in scorer_config.get("core") or []: + s = resolve_scorer(entry, judge_model) + if s: + scorers.append(s) + + for entry in scorer_config.get("use_case") or []: + s = resolve_scorer(entry, judge_model) + if s: + scorers.append(s) + + guidelines = scorer_config.get("guidelines") or [] + if guidelines: + scorers.append( + Guidelines( + name="domain_guidelines", + model=judge_model, + guidelines=guidelines, + ) + ) + + return scorers + + +def main(): + """Read traces from MLflow, attach expectations, and evaluate with scorers.""" + load_dotenv() + + judge_model = os.getenv("EVAL_JUDGE_MODEL", "openai:/gpt-4o-mini") + judge_api_key = os.getenv("EVAL_JUDGE_API_KEY") + judge_base_url = os.getenv("EVAL_JUDGE_BASE_URL") + provider = judge_model.split(":/")[0] if ":/" in judge_model else "openai" + + if judge_api_key: + os.environ[f"{provider.upper()}_API_KEY"] = judge_api_key + if judge_base_url: + env_var = f"{provider.upper()}_BASE_URL" + os.environ[env_var] = judge_base_url + print(f" Set {env_var} for judge model base URL") + + tracking_uri = os.getenv("MLFLOW_TRACKING_URI") + if not tracking_uri: + print("ERROR: MLFLOW_TRACKING_URI is not set. Cannot read traces.") + print("Set it in .env and ensure the agent has been run with tracing enabled.") + return + + mlflow.set_tracking_uri(tracking_uri) + + experiment_name = os.getenv("MLFLOW_EXPERIMENT_NAME") + if not experiment_name: + print("ERROR: MLFLOW_EXPERIMENT_NAME is not set.") + return + + experiment = mlflow.get_experiment_by_name(experiment_name) + if experiment is None: + print(f"ERROR: Experiment '{experiment_name}' not found on MLflow server.") + print("Run the agent first to create traces, then re-run eval.") + return + + mlflow.set_experiment(experiment_name) + + config = load_eval_config() + scorers = get_scorers(config, judge_model) + print(f"Loaded {len(scorers)} scorers from eval_config.yaml.") + + eval_data = load_eval_data() + agent_url = os.getenv("AGENT_URL", "http://localhost:8000") + + existing = mlflow.search_traces( + locations=[experiment.experiment_id], + return_type="list", + max_results=1, + order_by=["timestamp_ms DESC"], + ) + baseline_ms = existing[0].info.timestamp_ms if existing else 0 + + if eval_data: + generate_traces(eval_data, agent_url) + + expected_count = len(eval_data) if eval_data else 0 + trace_timeout = int(os.getenv("EVAL_TRACE_TIMEOUT", "60")) + poll_interval = int(os.getenv("EVAL_POLL_INTERVAL", "4")) + max_attempts = max(trace_timeout // poll_interval, 1) + golden_questions = {q["inputs"]["question"] for q in eval_data} if eval_data else set() + + traces = [] + for attempt in range(max_attempts): + traces = mlflow.search_traces( + locations=[experiment.experiment_id], + return_type="list", + filter_string=f"trace.timestamp_ms > {baseline_ms}", + ) + if len(traces) >= expected_count: + matched = [t for t in traces if _extract_question(t) in golden_questions] + if len(matched) >= expected_count: + break + time.sleep(poll_interval) + + if len(traces) < expected_count and expected_count > 0: + print( + f"WARNING: Expected {expected_count} traces but only found " + f"{len(traces)} after {trace_timeout}s. Some queries may not have " + f"been recorded." + ) + + if not traces: + print(f"No traces found in experiment '{experiment_name}' for this run.") + print("Ensure the agent is running with tracing enabled, then re-run eval.") + return + + print(f"Found {len(traces)} traces from this run to evaluate.") + + if eval_data: + matched_ids = attach_expectations(traces, eval_data) + traces = mlflow.search_traces( + locations=[experiment.experiment_id], + return_type="list", + filter_string=f"trace.timestamp_ms > {baseline_ms}", + ) + traces = [t for t in traces if t.info.trace_id in matched_ids] + + if not traces: + print("WARNING: No traces matched golden queries. Skipping evaluation.") + print("This can happen if traces were not fully written when matching ran.") + return + + print(f"Evaluating {len(traces)} traces with {len(scorers)} scorers...") + + with mlflow.start_run(run_name="eval-run"): + results = mlflow.genai.evaluate( + data=traces, + scorers=scorers, + ) + + print("\n" + "=" * 60) + print("EVALUATION RESULTS") + print("=" * 60) + for metric_name, value in sorted(results.metrics.items()): + print(f" {metric_name}: {value}") + print("=" * 60) + print(f"\nDetailed results logged to MLflow experiment: {experiment_name}") + + +if __name__ == "__main__": + main() diff --git a/agents/langgraph/templates/agentic_rag/evaluation/scorers.py b/agents/langgraph/templates/agentic_rag/evaluation/scorers.py new file mode 100644 index 00000000..a67db584 --- /dev/null +++ b/agents/langgraph/templates/agentic_rag/evaluation/scorers.py @@ -0,0 +1,17 @@ +"""Custom scorers for this agent. + +Define your scorers here and add their function name to eval_config.yaml +under scorers.core or scorers.use_case to activate them. + +See: https://mlflow.org/docs/latest/genai/eval-monitor/scorers/custom +""" + +from mlflow.genai.scorers import scorer # noqa: F401 + +# Example: +# +# @scorer +# def my_domain_scorer(request, response, expected_facts=None): +# """Check domain-specific quality criteria.""" +# # Your scoring logic here +# return {"score": 1.0, "rationale": "Meets criteria"} diff --git a/agents/langgraph/templates/agentic_rag/pyproject.toml b/agents/langgraph/templates/agentic_rag/pyproject.toml index 64f7c603..42ff4c81 100644 --- a/agents/langgraph/templates/agentic_rag/pyproject.toml +++ b/agents/langgraph/templates/agentic_rag/pyproject.toml @@ -31,9 +31,16 @@ auth = [ ] dev = [ "pytest>=9.0.2", + "httpx>=0.27", +] +eval = [ + "mlflow[kubernetes]>=3.11.0", + "pyyaml>=6.0", + "httpx>=0.27", + "deepeval>=4.2,<5", ] tracing = [ - "mlflow>=3.10.0", + "mlflow[kubernetes]>=3.11.0", ] [tool.setuptools.packages.find] diff --git a/agents/langgraph/templates/agentic_rag/src/agentic_rag/tools.py b/agents/langgraph/templates/agentic_rag/src/agentic_rag/tools.py index 509d30af..5cd429fe 100644 --- a/agents/langgraph/templates/agentic_rag/src/agentic_rag/tools.py +++ b/agents/langgraph/templates/agentic_rag/src/agentic_rag/tools.py @@ -5,6 +5,12 @@ from ogx_client import OgxClient from pydantic import BaseModel, Field +try: + import mlflow + from mlflow.entities import Document as MlflowDocument +except ImportError: + mlflow = None + # Cache to avoid re-initializing on every tool call _client_cache = None _vector_store_id_cache = None @@ -104,6 +110,7 @@ def retriever_tool(query: str) -> str: return "No relevant information was found in the provided documents for this query." formatted_docs = [] + retriever_docs = [] for i, chunk in enumerate(response.chunks, 1): # Skip chunks that are empty or just separators/whitespace content = chunk.content.strip() @@ -125,6 +132,20 @@ def retriever_tool(query: str) -> str: formatted_docs.append(doc_text) + if mlflow: + retriever_docs.append( + MlflowDocument( + page_content=content, + metadata={"source": source, "score": getattr(chunk, "score", None)}, + ) + ) + + # Log RETRIEVER span for MLflow RAG evaluation scorers + if mlflow and retriever_docs: + with mlflow.start_span(name="retrieve", span_type="RETRIEVER") as span: + span.set_inputs({"query": query}) + span.set_outputs(retriever_docs) + # If all chunks were filtered out, return no information message if not formatted_docs: return "No relevant information was found in the provided documents for this query." From e0a16db782d9569fda92f09b9b10849997fc07f8 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Tue, 1 Sep 2026 09:55:29 +0100 Subject: [PATCH 2/6] fix(agentic_rag): resolve CI lint, type-check, and lockfile failures - ruff format run_eval.py - ty.toml override for the optional mlflow import in tools.py (try/except None-fallback trips ty's module-vs-None check) - regenerate stale uv.lock --- .../agentic_rag/evaluation/run_eval.py | 4 +- .../langgraph/templates/agentic_rag/uv.lock | 236 +++++++++++++++++- ty.toml | 7 + 3 files changed, 240 insertions(+), 7 deletions(-) diff --git a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py index 68be987c..392da8cf 100644 --- a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py +++ b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py @@ -224,7 +224,9 @@ def main(): trace_timeout = int(os.getenv("EVAL_TRACE_TIMEOUT", "60")) poll_interval = int(os.getenv("EVAL_POLL_INTERVAL", "4")) max_attempts = max(trace_timeout // poll_interval, 1) - golden_questions = {q["inputs"]["question"] for q in eval_data} if eval_data else set() + golden_questions = ( + {q["inputs"]["question"] for q in eval_data} if eval_data else set() + ) traces = [] for attempt in range(max_attempts): diff --git a/agents/langgraph/templates/agentic_rag/uv.lock b/agents/langgraph/templates/agentic_rag/uv.lock index 185a5513..da4bc906 100644 --- a/agents/langgraph/templates/agentic_rag/uv.lock +++ b/agents/langgraph/templates/agentic_rag/uv.lock @@ -50,36 +50,48 @@ auth = [ { name = "agent-auth" }, ] dev = [ + { name = "httpx" }, { name = "pytest" }, ] +eval = [ + { name = "deepeval" }, + { name = "httpx" }, + { name = "mlflow", extra = ["kubernetes"] }, + { name = "pyyaml" }, +] tracing = [ - { name = "mlflow" }, + { name = "mlflow", extra = ["kubernetes"] }, ] [package.metadata] requires-dist = [ { name = "agent-auth", marker = "extra == 'auth'", directory = "../../../../components/auth" }, { name = "chardet", specifier = "==7.2.0" }, + { name = "deepeval", marker = "extra == 'eval'", specifier = ">=4.2,<5" }, { name = "fastapi", specifier = ">=0.135.1" }, { name = "flask", specifier = ">=3.1.0" }, + { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27" }, + { name = "httpx", marker = "extra == 'eval'", specifier = ">=0.27" }, { name = "langchain", specifier = ">=1.2.10" }, { name = "langchain-community", specifier = ">=0.4.1" }, { name = "langchain-milvus", specifier = "==0.3.3" }, { name = "langchain-openai", specifier = ">=1.1.10" }, { name = "langgraph-prebuilt", specifier = ">=1.0.0" }, { name = "milvus-lite", specifier = ">=2.5.1" }, - { name = "mlflow", marker = "extra == 'tracing'", specifier = ">=3.10.0" }, + { name = "mlflow", extras = ["kubernetes"], marker = "extra == 'eval'", specifier = ">=3.11.0" }, + { name = "mlflow", extras = ["kubernetes"], marker = "extra == 'tracing'", specifier = ">=3.11.0" }, { name = "ogx-client", specifier = ">=1.1.0" }, { name = "openai", specifier = ">=2.24.0" }, { name = "pymilvus", specifier = "==2.6.9" }, { name = "pypdf", specifier = "==6.9.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "python-dotenv", specifier = ">=1.2.1" }, + { name = "pyyaml", marker = "extra == 'eval'", specifier = ">=6.0" }, { name = "setuptools", specifier = ">=80.9.0,<83.0.0" }, { name = "starlette", specifier = ">=1.0.1" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.41.0" }, ] -provides-extras = ["auth", "dev", "tracing"] +provides-extras = ["auth", "dev", "eval", "tracing"] [[package]] name = "aiohappyeyeballs" @@ -208,6 +220,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "blinker" version = "1.9.0" @@ -479,6 +500,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] +[[package]] +name = "deepeval" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "click" }, + { name = "grpcio" }, + { name = "jinja2" }, + { name = "nest-asyncio" }, + { name = "openai" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "portalocker" }, + { name = "posthog" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyfiglet" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-repeat" }, + { name = "pytest-rerunfailures" }, + { name = "pytest-xdist" }, + { name = "python-dotenv" }, + { name = "questionary" }, + { name = "requests" }, + { name = "rich" }, + { name = "setuptools" }, + { name = "tabulate" }, + { name = "tenacity" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/95/44596d1e9853756346b4f82f6d3fca9d3a35dc421f253b89382544fb40f1/deepeval-4.2.0.tar.gz", hash = "sha256:07840ef5bc6db3242866ffae0969cca7cafe4a8c3e4e19e3a508283053a0b968", size = 855681, upload-time = "2026-08-24T21:54:17.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/09/18975670bcabf2fca174621db1f95bee79c091de1df0b6395e02b1f491b5/deepeval-4.2.0-py3-none-any.whl", hash = "sha256:7e10ea7ffdbe5506fde9e981c76184134079b586ef6bfc78049cd78112083a0e", size = 1216872, upload-time = "2026-08-24T21:54:19.532Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -511,6 +571,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "fastapi" version = "0.136.1" @@ -1444,6 +1513,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/1f/d44140128356f2f5db37f9fb4d9da31123d839d49f3fe00a079cd35bfe20/mlflow-3.13.0-py3-none-any.whl", hash = "sha256:7ca9cb2f623f300dabadaf5e985c85af77c5db3d7c36f56769d22c101b132f6c", size = 10788121, upload-time = "2026-06-01T05:55:06.86Z" }, ] +[package.optional-dependencies] +kubernetes = [ + { name = "kubernetes" }, +] + [[package]] name = "mlflow-skinny" version = "3.13.0" @@ -1566,6 +1640,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "numpy" version = "2.4.4" @@ -1872,6 +1955,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "portalocker" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/8e/27672cd2dde9d2345ead0352146a401be4ec0af132be86ca7a2db24afb09/portalocker-4.3.0.tar.gz", hash = "sha256:69bf8e46769d66eb0a23219b9550253d2c3b5f03400202a2333784c1e6395e32", size = 263582, upload-time = "2026-08-25T18:05:20.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/76/57be9bb352b3f91c80671a7b7ce2cd51f99ad324c89df82a098df9b9e609/portalocker-4.3.0-py3-none-any.whl", hash = "sha256:aa91709ccfc322b8225171392d64a2f9fc833f15f5ab741cfba6d3fba285ce31", size = 129554, upload-time = "2026-08-25T18:05:18.64Z" }, +] + +[[package]] +name = "posthog" +version = "7.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "distro" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/6d/8b89146eecb7cc23586e885478828eac0192aa3c19f4a3445d24f08ac8ac/posthog-7.45.1.tar.gz", hash = "sha256:40fb9c120e0787149073bea9185387557a64b843c5069929070af401e187770b", size = 503649, upload-time = "2026-08-31T07:50:56.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/1d/2469561e6feb23acd1ca0932e3dcb33df42c62c8f3d28718b748605c6458/posthog-7.45.1-py3-none-any.whl", hash = "sha256:74d7c41898944fda03ea477d730b42d90cfba660d78ce4e56819a3402e11176b", size = 595373, upload-time = "2026-08-31T07:50:54.817Z" }, +] + [[package]] name = "prettytable" version = "3.17.0" @@ -2116,6 +2223,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, ] +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615, upload-time = "2025-08-15T18:32:47.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118, upload-time = "2025-08-15T18:32:45.556Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -2177,6 +2293,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "pytest-repeat" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/d4/69e9dbb9b8266df0b157c72be32083403c412990af15c7c15f7a3fd1b142/pytest_repeat-0.9.4.tar.gz", hash = "sha256:d92ac14dfaa6ffcfe6917e5d16f0c9bc82380c135b03c2a5f412d2637f224485", size = 6488, upload-time = "2025-04-07T14:59:53.077Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/d4/8b706b81b07b43081bd68a2c0359fe895b74bf664b20aca8005d2bb3be71/pytest_repeat-0.9.4-py3-none-any.whl", hash = "sha256:c1738b4e412a6f3b3b9e0b8b29fcd7a423e50f87381ad9307ef6f5a8601139f3", size = 4180, upload-time = "2025-04-07T14:59:51.492Z" }, +] + +[[package]] +name = "pytest-rerunfailures" +version = "16.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/63/0114e45d4b2fcd5f6297dac655c067b47de28be4d33e088f200f0f2c4c28/pytest_rerunfailures-16.6.tar.gz", hash = "sha256:29dbfee46f542073c888e0ed4e81c51e15b9096f49a299eb1a759629c601684a", size = 42806, upload-time = "2026-08-17T07:11:00.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/5e/1e994889673d7a0da11651f17ef789b6c83bfe349f29f871873dd3802445/pytest_rerunfailures-16.6-py3-none-any.whl", hash = "sha256:6af2d1ebd6e5cb79666ac408942cd6a0672a49fefd8523664770718560795e13", size = 19137, upload-time = "2026-08-17T07:10:59.121Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2248,6 +2415,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, ] +[[package]] +name = "questionary" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/45/eafb0bba0f9988f6a2520f9ca2df2c82ddfa8d67c95d6625452e97b204a5/questionary-2.1.1.tar.gz", hash = "sha256:3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d", size = 25845, upload-time = "2025-08-28T19:00:20.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/26/1062c7ec1b053db9e499b4d2d5bc231743201b74051c973dadeac80a8f43/questionary-2.1.1-py3-none-any.whl", hash = "sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59", size = 36753, upload-time = "2025-08-28T19:00:19.56Z" }, +] + [[package]] name = "regex" version = "2026.5.9" @@ -2346,15 +2525,15 @@ wheels = [ [[package]] name = "rich" -version = "15.0.0" +version = "14.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/67/cae617f1351490c25a4b8ac3b8b63a4dda609295d8222bad12242dfdc629/rich-14.3.4.tar.gz", hash = "sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9", size = 230524, upload-time = "2026-04-11T02:57:45.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, + { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480, upload-time = "2026-04-11T02:57:47.484Z" }, ] [[package]] @@ -2439,6 +2618,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2537,6 +2725,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, ] +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, +] + [[package]] name = "tenacity" version = "9.1.4" @@ -2609,6 +2806,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "typer" +version = "0.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/f7/57713ba479fd405eb76de31404b2c744c289e336b2d999511ebf51e496f7/typer-0.27.2.tar.gz", hash = "sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945", size = 204045, upload-time = "2026-08-28T10:26:55.046Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/bf/205d0004930ede8f542fb58f601526fccf4ae7626075ca1e6c4de5d3d652/typer-0.27.2-py3-none-any.whl", hash = "sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d", size = 123130, upload-time = "2026-08-28T10:26:53.752Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -2868,6 +3080,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, ] +[[package]] +name = "wheel" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/20/50ed6bdf27dec98b568a8ae25dc599f35baa3d9709f9e83fd1edb56b9a90/wheel-0.48.0.tar.gz", hash = "sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322", size = 66471, upload-time = "2026-08-11T22:02:27.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", hash = "sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", size = 33320, upload-time = "2026-08-11T22:02:26.1Z" }, +] + [[package]] name = "xxhash" version = "3.7.0" diff --git a/ty.toml b/ty.toml index c0292879..2afa9f73 100644 --- a/ty.toml +++ b/ty.toml @@ -43,6 +43,13 @@ rules = { invalid-assignment = "ignore" } include = ["agents/langgraph/templates/agentic_rag/src/agentic_rag/agent.py"] rules = { invalid-return-type = "ignore" } +# mlflow is an optional dependency here (RETRIEVER span logging degrades +# gracefully when tracing extras aren't installed); the try/except import +# fallback trips ty's module-vs-None assignment check. +[[overrides]] +include = ["agents/langgraph/templates/agentic_rag/src/agentic_rag/tools.py"] +rules = { invalid-assignment = "ignore" } + # Monkey-patching methods with mlflow trace wrappers — intentional # signature mismatch (wrapper uses broader *args/**kwargs). [[overrides]] From 1c187b13bb13527b5b99f12c50062321e672969e Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Tue, 1 Sep 2026 10:09:38 +0100 Subject: [PATCH 3/6] test(agentic_rag): add unit tests for evaluation/run_eval.py Ports react_agent's test_eval.py coverage (config/data loading, scorer resolution, trace-expectation matching) and adds a case for agentic_rag's RAG-specific use_case scorers, which are enabled by default unlike react_agent's. Co-Authored-By: Claude Sonnet 5 --- .../templates/agentic_rag/tests/test_eval.py | 301 ++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 agents/langgraph/templates/agentic_rag/tests/test_eval.py diff --git a/agents/langgraph/templates/agentic_rag/tests/test_eval.py b/agents/langgraph/templates/agentic_rag/tests/test_eval.py new file mode 100644 index 00000000..f3d9ac20 --- /dev/null +++ b/agents/langgraph/templates/agentic_rag/tests/test_eval.py @@ -0,0 +1,301 @@ +"""Tests for evaluation/run_eval.py — config loading, data loading, scorer resolution.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip( + "mlflow", reason="eval dependencies not installed (install with --extra eval)" +) + +import json # noqa: E402 +import sys # noqa: E402 +import textwrap # noqa: E402 +from pathlib import Path # noqa: E402 +from unittest.mock import MagicMock, patch # noqa: E402 + +import yaml # noqa: E402 + +EVAL_DIR = Path(__file__).resolve().parents[1] / "evaluation" +sys.path.insert(0, str(EVAL_DIR.parent)) + +from evaluation.run_eval import ( # noqa: E402 + attach_expectations, + get_scorers, + load_eval_config, + load_eval_data, + resolve_scorer, +) + +# --------------------------------------------------------------------------- +# load_eval_config +# --------------------------------------------------------------------------- + + +class TestLoadEvalConfig: + def test_loads_default_config(self) -> None: + config = load_eval_config() + assert "scorers" in config + assert "core" in config["scorers"] + assert "guidelines" in config["scorers"] + + def test_loads_from_custom_path(self, tmp_path: Path) -> None: + custom = tmp_path / "custom.yaml" + custom.write_text( + yaml.dump({"scorers": {"core": [], "use_case": [], "guidelines": []}}) + ) + config = load_eval_config(str(custom)) + assert config["scorers"]["core"] == [] + + def test_core_scorers_have_name_and_module(self) -> None: + config = load_eval_config() + for entry in config["scorers"]["core"]: + assert "name" in entry, f"Missing 'name' in core scorer: {entry}" + assert "module" in entry, f"Missing 'module' in core scorer: {entry}" + + def test_use_case_scorers_have_name_and_module(self) -> None: + config = load_eval_config() + for entry in config["scorers"]["use_case"]: + assert "name" in entry, f"Missing 'name' in use_case scorer: {entry}" + assert "module" in entry, f"Missing 'module' in use_case scorer: {entry}" + + +# --------------------------------------------------------------------------- +# load_eval_data +# --------------------------------------------------------------------------- + + +class TestLoadEvalData: + def test_returns_empty_list_when_queries_is_none(self, tmp_path: Path) -> None: + """Regression: YAML `queries:` with no items parses as None, not [].""" + data_file = tmp_path / "eval_data.yaml" + data_file.write_text("queries:\n") + result = load_eval_data(str(data_file)) + assert result == [] + + def test_returns_empty_list_when_key_missing(self, tmp_path: Path) -> None: + data_file = tmp_path / "eval_data.yaml" + data_file.write_text("other_key: value\n") + result = load_eval_data(str(data_file)) + assert result == [] + + def test_returns_queries_list(self, tmp_path: Path) -> None: + data_file = tmp_path / "eval_data.yaml" + data_file.write_text( + textwrap.dedent("""\ + queries: + - inputs: + question: "What is 2+2?" + expectations: + expected_facts: + - "4" + """) + ) + result = load_eval_data(str(data_file)) + assert len(result) == 1 + assert result[0]["inputs"]["question"] == "What is 2+2?" + + def test_loads_default_placeholder_data(self) -> None: + result = load_eval_data() + assert isinstance(result, list) + + +# --------------------------------------------------------------------------- +# resolve_scorer +# --------------------------------------------------------------------------- + + +class TestResolveScorer: + def test_skips_scorer_when_env_flag_not_set(self) -> None: + entry = { + "name": "PIILeakage", + "module": "mlflow.genai.scorers.deepeval", + "enabled_by_env": "EVAL_ENABLE_PII", + } + with patch.dict("os.environ", {}, clear=True): + result = resolve_scorer(entry, "openai:/gpt-4o-mini") + assert result is None + + def test_loads_scorer_when_env_flag_is_true(self) -> None: + captured = {} + + class FakePIILeakage: + def __init__(self, model): + captured["model"] = model + + mock_module = MagicMock() + mock_module.PIILeakage = FakePIILeakage + + entry = { + "name": "PIILeakage", + "module": "mlflow.genai.scorers.deepeval", + "enabled_by_env": "EVAL_ENABLE_PII", + } + with ( + patch.dict("os.environ", {"EVAL_ENABLE_PII": "true"}, clear=True), + patch("importlib.import_module", return_value=mock_module), + ): + result = resolve_scorer(entry, "openai:/gpt-4o-mini") + assert isinstance(result, FakePIILeakage) + assert captured["model"] == "openai:/gpt-4o-mini" + + def test_uses_model_env_override(self) -> None: + captured = {} + + class FakePIILeakage: + def __init__(self, model): + captured["model"] = model + + mock_module = MagicMock() + mock_module.PIILeakage = FakePIILeakage + + entry = { + "name": "PIILeakage", + "module": "mlflow.genai.scorers.deepeval", + "enabled_by_env": "EVAL_ENABLE_PII", + "model_env": "EVAL_PII_MODEL", + } + with ( + patch.dict( + "os.environ", + {"EVAL_ENABLE_PII": "true", "EVAL_PII_MODEL": "openai:/gpt-4.1-mini"}, + clear=True, + ), + patch("importlib.import_module", return_value=mock_module), + ): + resolve_scorer(entry, "openai:/gpt-4o-mini") + assert captured["model"] == "openai:/gpt-4.1-mini" + + def test_custom_scorer_function_not_called_with_model(self) -> None: + """Regression: @scorer decorated functions don't accept model= argument.""" + + def my_custom_scorer(): + pass + + mock_module = MagicMock() + mock_module.my_custom_scorer = my_custom_scorer + + entry = {"name": "my_custom_scorer", "module": "scorers"} + with patch("importlib.import_module", return_value=mock_module): + result = resolve_scorer(entry, "openai:/gpt-4o-mini") + assert result is my_custom_scorer + + def test_class_scorer_instantiated_with_model(self) -> None: + mock_class = type("MockScorer", (), {"__init__": lambda self, model: None}) + mock_module = MagicMock() + mock_module.Safety = mock_class + + entry = {"name": "Safety", "module": "mlflow.genai.scorers"} + with patch("importlib.import_module", return_value=mock_module): + result = resolve_scorer(entry, "openai:/gpt-4o-mini") + assert isinstance(result, mock_class) + + +# --------------------------------------------------------------------------- +# get_scorers +# --------------------------------------------------------------------------- + + +class TestGetScorers: + def test_includes_guidelines_scorer(self) -> None: + config = { + "scorers": { + "core": [], + "use_case": [], + "guidelines": ["Be helpful", "Be safe"], + } + } + scorers = get_scorers(config, "openai:/gpt-4o-mini") + assert len(scorers) == 1 + assert scorers[0].name == "domain_guidelines" + + def test_empty_guidelines_skips_scorer(self) -> None: + config = {"scorers": {"core": [], "use_case": [], "guidelines": []}} + scorers = get_scorers(config, "openai:/gpt-4o-mini") + assert len(scorers) == 0 + + def test_none_use_case_handled(self) -> None: + config = {"scorers": {"core": [], "use_case": None, "guidelines": []}} + scorers = get_scorers(config, "openai:/gpt-4o-mini") + assert scorers == [] + + def test_missing_scorers_key_handled(self) -> None: + """Regression: empty YAML returns {}, bracket access to 'scorers' crashes.""" + config = {} + scorers = get_scorers(config, "openai:/gpt-4o-mini") + assert scorers == [] + + +# --------------------------------------------------------------------------- +# attach_expectations +# --------------------------------------------------------------------------- + + +def _make_trace(trace_id: str, timestamp_ms: int, question: str = "") -> MagicMock: + """Create a mock trace with the given ID, timestamp, and request content.""" + trace = MagicMock() + trace.info.trace_id = trace_id + trace.info.timestamp_ms = timestamp_ms + trace.data.request = json.dumps({"messages": [{"content": question}]}) + return trace + + +class TestAttachExpectations: + def test_matches_by_content(self) -> None: + """Traces are matched to eval_data by question content.""" + traces = [ + _make_trace("t1", 1000, "Q1"), + _make_trace("t2", 2000, "Q2"), + _make_trace("t3", 3000, "Q3"), + ] + eval_data = [ + {"inputs": {"question": "Q1"}, "expectations": {"expected_facts": ["A1"]}}, + {"inputs": {"question": "Q2"}, "expectations": {"expected_facts": ["A2"]}}, + {"inputs": {"question": "Q3"}, "expectations": {"expected_facts": ["A3"]}}, + ] + with patch("evaluation.run_eval.mlflow") as mock_mlflow: + matched_ids = attach_expectations(traces, eval_data) + + assert matched_ids == {"t1", "t2", "t3"} + assert mock_mlflow.log_expectation.call_count == 3 + + def test_ignores_unrelated_traces(self) -> None: + """Traces that don't match any golden query are skipped.""" + traces = [ + _make_trace("t1", 1000, "Q1"), + _make_trace("stray", 1500, "Some other question"), + _make_trace("t2", 2000, "Q2"), + ] + eval_data = [ + {"inputs": {"question": "Q1"}, "expectations": {"expected_facts": ["A1"]}}, + {"inputs": {"question": "Q2"}, "expectations": {"expected_facts": ["A2"]}}, + ] + with patch("evaluation.run_eval.mlflow") as mock_mlflow: + matched_ids = attach_expectations(traces, eval_data) + + assert matched_ids == {"t1", "t2"} + assert "stray" not in matched_ids + assert mock_mlflow.log_expectation.call_count == 2 + + def test_more_queries_than_traces(self) -> None: + """If fewer traces than queries, only match what we have.""" + traces = [_make_trace("t1", 1000, "Q1")] + eval_data = [ + {"inputs": {"question": "Q1"}, "expectations": {"expected_facts": ["A1"]}}, + {"inputs": {"question": "Q2"}, "expectations": {"expected_facts": ["A2"]}}, + ] + with patch("evaluation.run_eval.mlflow"): + matched_ids = attach_expectations(traces, eval_data) + + assert matched_ids == {"t1"} + + def test_query_without_expectations(self) -> None: + """Queries with no expectations still match (trace ID tracked).""" + traces = [_make_trace("t1", 1000, "Q1")] + eval_data = [{"inputs": {"question": "Q1"}}] + + with patch("evaluation.run_eval.mlflow") as mock_mlflow: + matched_ids = attach_expectations(traces, eval_data) + + assert matched_ids == {"t1"} + mock_mlflow.log_expectation.assert_not_called() From 97bef41d023eacbfddffb775482784b3445e3189 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Tue, 1 Sep 2026 10:18:56 +0100 Subject: [PATCH 4/6] fix(agentic_rag): correct custom scorer example to match MLflow contract The @scorer example used positional request/response/expected_facts params and returned a dict, neither of which MLflow's @scorer contract supports. Update to keyword-only inputs/outputs/expectations/trace and a Feedback return, per MLflow GenAI docs. Co-Authored-By: Claude Sonnet 5 --- .../langgraph/templates/agentic_rag/evaluation/scorers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/agents/langgraph/templates/agentic_rag/evaluation/scorers.py b/agents/langgraph/templates/agentic_rag/evaluation/scorers.py index a67db584..d2694189 100644 --- a/agents/langgraph/templates/agentic_rag/evaluation/scorers.py +++ b/agents/langgraph/templates/agentic_rag/evaluation/scorers.py @@ -10,8 +10,10 @@ # Example: # +# from mlflow.entities import Feedback +# # @scorer -# def my_domain_scorer(request, response, expected_facts=None): +# def my_domain_scorer(*, inputs, outputs, expectations=None, trace=None): # """Check domain-specific quality criteria.""" # # Your scoring logic here -# return {"score": 1.0, "rationale": "Meets criteria"} +# return Feedback(value=1.0, rationale="Meets criteria") From bdcbb2370c0b164150918c78a2f35e5c6485704b Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Tue, 1 Sep 2026 15:25:03 +0100 Subject: [PATCH 5/6] fix(agentic_rag): address PR review findings from shricharan-ks - resolve_scorer: introspect __init__ signature before injecting model=, so custom @scorer functions and classes without a model param aren't called incorrectly - get_scorers: catch per-entry load errors and warn/skip instead of crashing the whole eval run - generate_traces/main: read EVAL_* timeout/interval env vars via a validated int helper, and guard poll_interval against zero/negative values - README: fix custom scorer example to match the actual @scorer contract (Feedback return type, inputs/outputs/expectations/trace kwargs) - pyproject.toml: drop unused httpx from dev extras, drop unnecessary kubernetes extra from tracing's mlflow dependency --- .../langgraph/templates/agentic_rag/README.md | 5 ++- .../agentic_rag/evaluation/run_eval.py | 38 ++++++++++++++++--- .../templates/agentic_rag/pyproject.toml | 3 +- .../langgraph/templates/agentic_rag/uv.lock | 6 +-- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/agents/langgraph/templates/agentic_rag/README.md b/agents/langgraph/templates/agentic_rag/README.md index 9dfb4e51..099d8f0c 100644 --- a/agents/langgraph/templates/agentic_rag/README.md +++ b/agents/langgraph/templates/agentic_rag/README.md @@ -574,13 +574,14 @@ guidelines: 1. Define your scorer in `evaluation/scorers.py`: ```python + from mlflow.entities import Feedback from mlflow.genai.scorers import scorer @scorer - def my_domain_scorer(request, response, expected_facts=None): + def my_domain_scorer(*, inputs, outputs, expectations=None, trace=None): """Check domain-specific quality criteria.""" # Your scoring logic here - return {"score": 1.0, "rationale": "Meets criteria"} + return Feedback(value=1.0, rationale="Meets criteria") ``` 2. Add it to `evaluation/eval_config.yaml` under `core` or `use_case`: diff --git a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py index 392da8cf..892dae6b 100644 --- a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py +++ b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py @@ -38,9 +38,19 @@ def load_eval_data(path: str | None = None) -> list[dict]: return data.get("queries") or [] +def _get_int_env(name: str, default: int) -> int: + """Read an integer env var, exiting with a clear error if it's not a valid integer.""" + value = os.getenv(name, str(default)) + try: + return int(value) + except ValueError: + print(f"ERROR: {name}='{value}' is not a valid integer.") + raise SystemExit(1) from None + + def generate_traces(eval_data: list[dict], agent_url: str) -> None: """Send golden queries to the running agent to generate traces.""" - request_timeout = int(os.getenv("EVAL_REQUEST_TIMEOUT", "60")) + request_timeout = _get_int_env("EVAL_REQUEST_TIMEOUT", 60) print(f"Sending {len(eval_data)} golden queries to {agent_url}...") for query in eval_data: question = query["inputs"]["question"] @@ -133,22 +143,38 @@ def resolve_scorer(entry: dict, judge_model: str): cls_or_obj = getattr(module, entry["name"]) if inspect.isclass(cls_or_obj): - return cls_or_obj(model=model) + try: + init_params = inspect.signature(cls_or_obj.__init__).parameters + except (TypeError, ValueError): + init_params = {} + accepts_model = "model" in init_params or any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in init_params.values() + ) + return cls_or_obj(model=model) if accepts_model else cls_or_obj() return cls_or_obj +def _resolve_scorer_or_warn(entry: dict, judge_model: str): + """Resolve a scorer entry, warning and skipping it instead of crashing the run.""" + try: + return resolve_scorer(entry, judge_model) + except (ModuleNotFoundError, AttributeError) as e: + print(f"WARNING: Could not load scorer '{entry.get('name')}': {e}. Skipping.") + return None + + def get_scorers(config: dict, judge_model: str) -> list: """Build the scorer list from eval_config.yaml.""" scorers = [] scorer_config = config.get("scorers", {}) for entry in scorer_config.get("core") or []: - s = resolve_scorer(entry, judge_model) + s = _resolve_scorer_or_warn(entry, judge_model) if s: scorers.append(s) for entry in scorer_config.get("use_case") or []: - s = resolve_scorer(entry, judge_model) + s = _resolve_scorer_or_warn(entry, judge_model) if s: scorers.append(s) @@ -221,8 +247,8 @@ def main(): generate_traces(eval_data, agent_url) expected_count = len(eval_data) if eval_data else 0 - trace_timeout = int(os.getenv("EVAL_TRACE_TIMEOUT", "60")) - poll_interval = int(os.getenv("EVAL_POLL_INTERVAL", "4")) + trace_timeout = _get_int_env("EVAL_TRACE_TIMEOUT", 60) + poll_interval = max(_get_int_env("EVAL_POLL_INTERVAL", 4), 1) max_attempts = max(trace_timeout // poll_interval, 1) golden_questions = ( {q["inputs"]["question"] for q in eval_data} if eval_data else set() diff --git a/agents/langgraph/templates/agentic_rag/pyproject.toml b/agents/langgraph/templates/agentic_rag/pyproject.toml index 42ff4c81..70459bfb 100644 --- a/agents/langgraph/templates/agentic_rag/pyproject.toml +++ b/agents/langgraph/templates/agentic_rag/pyproject.toml @@ -31,7 +31,6 @@ auth = [ ] dev = [ "pytest>=9.0.2", - "httpx>=0.27", ] eval = [ "mlflow[kubernetes]>=3.11.0", @@ -40,7 +39,7 @@ eval = [ "deepeval>=4.2,<5", ] tracing = [ - "mlflow[kubernetes]>=3.11.0", + "mlflow>=3.11.0", ] [tool.setuptools.packages.find] diff --git a/agents/langgraph/templates/agentic_rag/uv.lock b/agents/langgraph/templates/agentic_rag/uv.lock index da4bc906..e2fa5395 100644 --- a/agents/langgraph/templates/agentic_rag/uv.lock +++ b/agents/langgraph/templates/agentic_rag/uv.lock @@ -50,7 +50,6 @@ auth = [ { name = "agent-auth" }, ] dev = [ - { name = "httpx" }, { name = "pytest" }, ] eval = [ @@ -60,7 +59,7 @@ eval = [ { name = "pyyaml" }, ] tracing = [ - { name = "mlflow", extra = ["kubernetes"] }, + { name = "mlflow" }, ] [package.metadata] @@ -70,7 +69,6 @@ requires-dist = [ { name = "deepeval", marker = "extra == 'eval'", specifier = ">=4.2,<5" }, { name = "fastapi", specifier = ">=0.135.1" }, { name = "flask", specifier = ">=3.1.0" }, - { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27" }, { name = "httpx", marker = "extra == 'eval'", specifier = ">=0.27" }, { name = "langchain", specifier = ">=1.2.10" }, { name = "langchain-community", specifier = ">=0.4.1" }, @@ -78,8 +76,8 @@ requires-dist = [ { name = "langchain-openai", specifier = ">=1.1.10" }, { name = "langgraph-prebuilt", specifier = ">=1.0.0" }, { name = "milvus-lite", specifier = ">=2.5.1" }, + { name = "mlflow", marker = "extra == 'tracing'", specifier = ">=3.11.0" }, { name = "mlflow", extras = ["kubernetes"], marker = "extra == 'eval'", specifier = ">=3.11.0" }, - { name = "mlflow", extras = ["kubernetes"], marker = "extra == 'tracing'", specifier = ">=3.11.0" }, { name = "ogx-client", specifier = ">=1.1.0" }, { name = "openai", specifier = ">=2.24.0" }, { name = "pymilvus", specifier = "==2.6.9" }, From 8f4117c664bee6382acd8cfabf32cad2bbdb0214 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Wed, 2 Sep 2026 14:11:23 +0100 Subject: [PATCH 6/6] fix(agentic_rag): catch TypeError in scorer loading and reject negative int env vars Addresses two findings from shricharan-ks's round-2 review: scorer loading only caught ModuleNotFoundError/AttributeError, letting a TypeError from an incompatible scorer constructor crash the whole eval run; _get_int_env accepted negative values, which reached httpx.post(timeout=-1) as a raw ValueError instead of a clear config error. --- .../templates/agentic_rag/evaluation/run_eval.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py index 892dae6b..16ecf0de 100644 --- a/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py +++ b/agents/langgraph/templates/agentic_rag/evaluation/run_eval.py @@ -39,13 +39,17 @@ def load_eval_data(path: str | None = None) -> list[dict]: def _get_int_env(name: str, default: int) -> int: - """Read an integer env var, exiting with a clear error if it's not a valid integer.""" + """Read an integer env var, exiting with a clear error if it's not a valid non-negative integer.""" value = os.getenv(name, str(default)) try: - return int(value) + result = int(value) except ValueError: print(f"ERROR: {name}='{value}' is not a valid integer.") raise SystemExit(1) from None + if result < 0: + print(f"ERROR: {name}={result} must not be negative.") + raise SystemExit(1) + return result def generate_traces(eval_data: list[dict], agent_url: str) -> None: @@ -158,8 +162,9 @@ def _resolve_scorer_or_warn(entry: dict, judge_model: str): """Resolve a scorer entry, warning and skipping it instead of crashing the run.""" try: return resolve_scorer(entry, judge_model) - except (ModuleNotFoundError, AttributeError) as e: - print(f"WARNING: Could not load scorer '{entry.get('name')}': {e}. Skipping.") + except (ModuleNotFoundError, AttributeError, TypeError) as e: + name = entry.get("name") if isinstance(entry, dict) else "" + print(f"WARNING: Could not load scorer '{name}': {e}. Skipping.") return None