feat(agentic_rag): add config-driven MLflow GenAI evaluation

agents/langgraph/templates/agentic_rag/.env.example

@@ -32,5 +32,18 @@ PORT=8000
 # MLFLOW_WORKSPACE=default
 # MLFLOW_TRACKING_AUTH= # Use Kubernetes service account for authentication (if running inside the cluster)
 
+# Optional — Evaluation (only needed for `make eval`)
+# The judge model scores agent responses. Can be the same as the agent's model or a different one.
+# EVAL_JUDGE_MODEL=openai:/gpt-4o-mini
+# EVAL_JUDGE_API_KEY=<your-api-key>
+# EVAL_JUDGE_BASE_URL=https://api.openai.com/v1
+# EVAL_PII_MODEL=openai:/gpt-4.1-mini
+# EVAL_ENABLE_PII=false
+# EVAL_REQUEST_TIMEOUT=60
+# EVAL_TRACE_TIMEOUT=60
+# EVAL_POLL_INTERVAL=4
+DEEPEVAL_TELEMETRY_OPT_OUT=YES
+# AGENT_URL=http://localhost:8000
+
 # Optional — openshell sandbox
 # OPENSHELL_SANDBOX_NAME=rag-sandbox

agents/langgraph/templates/agentic_rag/Makefile

@@ -7,7 +7,7 @@ CONTAINER_CLI  := $(shell command -v podman 2>/dev/null || command -v docker 2>/
 MODEL          ?= llama3.1:8b
 OLLAMA_EMBEDDING_MODEL ?= embeddinggemma:latest
 
-.PHONY: init re-init env ollama ogx-server run-app run-app-fresh run-cli load-docs build push build-openshift deploy undeploy test test-integration dry-run setup-gateway teardown-gateway build-openshell create-sandbox wait-sandbox setup-egress load-docs-sandbox start-agent deploy-openshell undeploy-openshell playground-sandbox help
+.PHONY: init re-init env ollama ogx-server run-app run-app-fresh run-cli load-docs build push build-openshift deploy undeploy test test-integration eval dry-run setup-gateway teardown-gateway build-openshell create-sandbox wait-sandbox setup-egress load-docs-sandbox start-agent deploy-openshell undeploy-openshell playground-sandbox help
 
 help: ## Show this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  %-14s %s\n", $$1, $$2}'
@@ -442,6 +442,9 @@ playground-sandbox: ## Start sandbox playground web UI (auto-fetches agent URL a
 	  echo "" && \
 	  python playground-sandbox/app.py
 
+eval: ## Run MLflow GenAI evaluation
+	@uv run --extra eval python evaluation/run_eval.py
+
 test: ## Run unit tests
 	uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)
 

agents/langgraph/templates/agentic_rag/README.md

@@ -48,7 +48,7 @@ make env
 
 ### Tracing (optional)
 
-Tracing is optional. If MLflow tracing is required, enable it by uncommenting and setting the following environment variables in the `.env` file.
+Tracing is optional. If MLflow tracing is required, enable it by uncommenting and setting the following environment variables in the `.env` file. For how tracing works under the hood (span structure, autolog behavior, verification steps), see [tracing.md](../../../../tracing.md).
 
 #### Tracing with a local MLflow server
 
@@ -448,6 +448,161 @@ python -c "from agent_auth.middleware import SATokenAuthMiddleware; print('OK')"
 make test
 ```
 
+## Evaluation
+
+This template includes a config-driven MLflow GenAI evaluation setup. The eval script reads existing traces from MLflow and scores them using scorers defined in `evaluation/eval_config.yaml`.
+
+### Eval Directory Structure
+
+```text
+evaluation/
+├── eval_config.yaml   # Scorer configuration (core, use-case, guidelines)
+├── eval_data.yaml     # Golden queries with expected results
+├── scorers.py         # Custom scorer definitions (add your own here)
+└── run_eval.py        # Evaluation logic (no need to edit)
+```
+
+### Prerequisites
+
+- Agent must be running with tracing enabled (`MLFLOW_TRACKING_URI` set in `.env`)
+- Complete the Setup steps below
+
+### Setup
+
+**Set environment variables** in `.env`:
+
+```ini
+# Required for eval
+EVAL_JUDGE_MODEL=<provider>:/<model>
+EVAL_JUDGE_API_KEY=<api-key>
+EVAL_JUDGE_BASE_URL=<endpoint-url>  # only needed for OpenAI-compatible endpoints
+```
+
+The judge model format is `<provider>:/<model>`. The eval script automatically sets the correct API key env var based on the provider. Examples:
+
+- **OpenAI:** `EVAL_JUDGE_MODEL=openai:/gpt-4o-mini`, `EVAL_JUDGE_BASE_URL=https://api.openai.com/v1`
+- **vLLM / RHOAI:** `EVAL_JUDGE_MODEL=openai:/<served-model-name>`, `EVAL_JUDGE_BASE_URL=https://<your-vllm-route>/v1`
+- **Anthropic:** `EVAL_JUDGE_MODEL=anthropic:/claude-sonnet-4-20250514` (no `BASE_URL` needed)
+
+### Running Evaluation
+
+1. Add your golden queries and expectations to `evaluation/eval_data.yaml`
+2. Customize the guidelines in `evaluation/eval_config.yaml` for your domain
+3. Start your agent with tracing enabled (`make run-app`) if it's not already running
+4. Run:
+
+   ```bash
+   make eval
+   ```
+
+   This will:
+   - Send each golden query from `eval_data.yaml` to the running agent
+   - Wait for traces to be recorded in MLflow
+   - Attach expectations (e.g., `expected_facts`) to matching traces
+   - Score all traces with the configured scorers
+
+   By default, the eval script connects to the agent at `http://localhost:8000`. To target a deployed agent, set `AGENT_URL` in `.env`:
+
+   ```ini
+   AGENT_URL=https://<your-agent-route>
+   ```
+
+### Scorer Configuration
+
+Scorers are configured in `evaluation/eval_config.yaml`, organized into three sections. Scorers in both **core** and **use-case** sections can come from any of three sources:
+
+| Source | `module` value | Example |
+|--------|---------------|---------|
+| MLflow built-in | `mlflow.genai.scorers` | Safety, Correctness |
+| Third-party (any MLflow-integrated framework) | `mlflow.genai.scorers.<framework>` | DeepEval PIILeakage, Phoenix Hallucination |
+| Custom (defined in `scorers.py`) | `scorers` | Your own domain-specific scorers |
+
+#### Core Scorers
+
+Cover universal failure modes. Pre-configured for all agents — remove only with a specific reason.
+
+| Scorer | What it checks |
+|--------|---------------|
+| **Safety** | Responses are free of harmful or toxic content |
+| **RelevanceToQuery** | Responses directly address the user's question |
+| **Completeness** | All parts of the user's request are addressed |
+| **Correctness** | Responses match expected facts (requires `expected_facts` in eval data) |
+| **ToolCallCorrectness** | Agent selects appropriate tools with correct arguments |
+| **ToolCallEfficiency** | No redundant or duplicate tool calls |
+| **PIILeakage** *(opt-in)* | Responses don't leak personally identifiable information |
+
+> **PIILeakage** is disabled by default. Enable it by setting `EVAL_ENABLE_PII=true` in `.env`. It uses DeepEval and defaults to `openai:/gpt-4.1-mini` (override with `EVAL_PII_MODEL`). DeepEval has a known JSON parsing issue with `gpt-4o` — use `gpt-4.1-mini` or `gpt-4o-mini`.
+
+#### Use-case Scorers (RAG)
+
+This template includes three RAG-specific scorers pre-configured under `use_case`. These evaluate retrieval quality and require `RETRIEVER`-type spans in traces (handled automatically by the retriever tool).
+
+| Scorer | What it checks |
+|--------|---------------|
+| **RetrievalGroundedness** | The final answer is grounded in the retrieved documents |
+| **RetrievalRelevance** | Each retrieved document is relevant to the query (per-document) |
+| **RetrievalSufficiency** | The retrieved documents provide enough information to answer |
+
+> **RetrievalRelevance** is a per-document scorer — it produces one assessment per retrieved chunk. Queries that don't trigger retrieval (e.g., greetings) will show errors for RAG scorers, which is expected.
+
+You can add more use-case scorers from MLflow, third-party frameworks, or custom definitions in `scorers.py`:
+
+```yaml
+use_case:
+  # Third-party scorer (any MLflow-integrated framework)
+  - name: Faithfulness
+    module: mlflow.genai.scorers.deepeval
+
+  # Custom scorer (defined in scorers.py)
+  - name: my_domain_scorer
+    module: scorers
+```
+
+#### Guidelines
+
+A list of plain-text rules evaluated by the `Guidelines` scorer. Customize these for your agent:
+
+```yaml
+guidelines:
+  - "The response should be helpful and informative"
+  - "The response should not fabricate information"
+  - "Answers to knowledge questions must be grounded in retrieved documents"
+```
+
+### Adding a Custom Scorer
+
+1. Define your scorer in `evaluation/scorers.py`:
+
+   ```python
+   from mlflow.entities import Feedback
+   from mlflow.genai.scorers import scorer
+
+   @scorer
+   def my_domain_scorer(*, inputs, outputs, expectations=None, trace=None):
+       """Check domain-specific quality criteria."""
+       # Your scoring logic here
+       return Feedback(value=1.0, rationale="Meets criteria")
+   ```
+
+2. Add it to `evaluation/eval_config.yaml` under `core` or `use_case`:
+
+   ```yaml
+   use_case:
+     - name: my_domain_scorer
+       module: scorers
+   ```
+
+3. Run `make eval` — your scorer will be picked up automatically.
+
+### Viewing Results
+
+Results are logged to MLflow. View them in the MLflow UI under the Traces tab.
+
+### Advanced Evaluation
+
+For failure mode deep-dives, custom scorer patterns, and end-to-end evaluation, see the
+[Agentic Evaluation Enablement Material](https://github.com/red-hat-data-services/red-hat-ai-examples/tree/main/examples/agentic-evaluation).
+
 ## API Endpoints
 
 ### POST /chat/completions

agents/langgraph/templates/agentic_rag/evaluation/eval_config.yaml

@@ -0,0 +1,60 @@
+# Evaluation scorer configuration.
+#
+# Each scorer entry requires:
+#   name:   Class or function name to import
+#   module: Python module to import from
+#
+# Optional:
+#   enabled_by_env: Only load this scorer if the specified env var is set to "true"
+#   model_env:      Read the judge model from this env var instead of EVAL_JUDGE_MODEL
+#
+# Sources:
+#   MLflow scorers:     module: mlflow.genai.scorers
+#   Third-party:        module: mlflow.genai.scorers.<framework> (any MLflow-integrated framework, e.g., deepeval, ragas, phoenix etc.)
+#   Custom (scorers.py): module: scorers (custom built scorers)
+
+scorers:
+  # Core scorers — cover universal failure modes, apply to all agents.
+  # These come pre-configured. Remove only if you have a specific reason.
+  core:
+    - name: Safety
+      module: mlflow.genai.scorers
+
+    - name: RelevanceToQuery
+      module: mlflow.genai.scorers
+
+    - name: Completeness
+      module: mlflow.genai.scorers
+
+    - name: Correctness
+      module: mlflow.genai.scorers
+
+    - name: ToolCallCorrectness
+      module: mlflow.genai.scorers
+
+    - name: ToolCallEfficiency
+      module: mlflow.genai.scorers
+
+    - name: PIILeakage
+      module: mlflow.genai.scorers.deepeval
+      enabled_by_env: EVAL_ENABLE_PII
+      model_env: EVAL_PII_MODEL
+
+  # Use-case specific scorers — RAG retrieval quality.
+  # These require RETRIEVER-type spans in traces (see tools.py).
+  use_case:
+    - name: RetrievalGroundedness
+      module: mlflow.genai.scorers
+
+    - name: RetrievalRelevance
+      module: mlflow.genai.scorers
+
+    - name: RetrievalSufficiency
+      module: mlflow.genai.scorers
+
+  # Domain-specific rules for the Guidelines scorer.
+  # Customize these for your agent's expected behavior.
+  guidelines:
+    - "The response should be helpful and informative"
+    - "The response should not fabricate information"
+    - "Answers to knowledge questions must be grounded in retrieved documents"

agents/langgraph/templates/agentic_rag/evaluation/eval_data.yaml

@@ -0,0 +1,35 @@
+# Golden queries for MLflow GenAI evaluation.
+#
+# Add your test queries below. Each query is sent to the running agent,
+# and the resulting trace is scored by the configured scorers.
+#
+# Fields:
+#   inputs:       Dict with the user query (required)
+#   expectations: Dict with ground truth (required for Correctness scorer)
+#     expected_facts:    List of facts the response should contain
+#     expected_response: Reference response text
+#
+# Trace-based scorers (ToolCallCorrectness, ToolCallEfficiency)
+# extract data from MLflow traces automatically — no expected tools needed here.
+
+queries:
+  # Example: query that requires a tool call
+  # - inputs:
+  #     question: "Your test question here"
+  #   expectations:
+  #     expected_facts:
+  #       - "Expected fact 1"
+  #       - "Expected fact 2"
+
+  # Example: query that should NOT trigger a tool call
+  # - inputs:
+  #     question: "A simple greeting or general question"
+  #   expectations:
+  #     expected_response: "A friendly response without using any tools"
+
+  # Example: edge case
+  # - inputs:
+  #     question: "A query that tests error handling or boundaries"
+  #   expectations:
+  #     expected_facts:
+  #       - "Expected behavior description"