fix linting issues

Author: asamal4

lsc_eval/README.md

@@ -30,7 +30,7 @@ cd lsc_eval && pdm install
 export OPENAI_API_KEY="your-key"
 
 # Run evaluation (Create your own data)
-python runner.py --system-config config/system.yaml --eval-data config/evaluation_data.yaml
+python -m runner --system-config config/system.yaml --eval-data config/evaluation_data.yaml
 ```
 
 ## 📊 Supported Metrics

lsc_eval/runner.py

@@ -48,7 +48,9 @@ def run_evaluation(
         data_validator = DataValidator()
         evaluation_data = data_validator.load_evaluation_data(evaluation_data_path)
 
-        print(f"✅ System config: {system_config.llm_provider}/{system_config.llm_model}")
+        print(
+            f"✅ System config: {system_config.llm_provider}/{system_config.llm_model}"
+        )
         print(f"✅ Evaluation data: {len(evaluation_data)} conversation groups")
 
         # Step 2: Initialize evaluation engine (core controller)
@@ -67,7 +69,9 @@ def run_evaluation(
             system_config=system_config,
         )
 
-        output_handler.generate_reports(results, include_graphs=system_config.include_graphs)
+        output_handler.generate_reports(
+            results, include_graphs=system_config.include_graphs
+        )
 
         print("\n🎉 Evaluation Complete!")
         print(f"📊 {len(results)} evaluations completed")
@@ -81,7 +85,9 @@ def run_evaluation(
         )
 
         if summary["ERROR"] > 0:
-            print(f"⚠️ {summary['ERROR']} evaluations had errors - check detailed report")
+            print(
+                f"⚠️ {summary['ERROR']} evaluations had errors - check detailed report"
+            )
 
         return {
             "TOTAL": summary["TOTAL"],

lsc_eval/src/lsc_eval/core/__init__.py

@@ -1,6 +1,11 @@
 """Core functionality - Configuration, data validation, and models."""
 
-from .config_loader import ConfigLoader, SystemConfig, setup_environment_variables, validate_metrics
+from .config_loader import (
+    ConfigLoader,
+    SystemConfig,
+    setup_environment_variables,
+    validate_metrics,
+)
 from .data_validator import DataValidator
 from .models import EvaluationData, EvaluationResult, LLMConfig, TurnData
 

lsc_eval/src/lsc_eval/core/config_loader.py

@@ -51,14 +51,18 @@ def setup_logging(logging_config: Dict[str, Any]) -> logging.Logger:
     """Configure logging for application and packages."""
     # Get logging settings with new structure
     source_level = getattr(logging, logging_config.get("source_level", "INFO").upper())
-    package_level = getattr(logging, logging_config.get("package_level", "WARNING").upper())
+    package_level = getattr(
+        logging, logging_config.get("package_level", "WARNING").upper()
+    )
     log_format = logging_config.get(
         "format", "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     )
 
     # Configure root logger for our application
     logging.basicConfig(
-        level=source_level, format=log_format, force=True  # Override any existing configuration
+        level=source_level,
+        format=log_format,
+        force=True,  # Override any existing configuration
     )
 
     # Set logging levels for packages using system configuration
@@ -88,7 +92,9 @@ def setup_logging(logging_config: Dict[str, Any]) -> logging.Logger:
             override_level_obj = getattr(logging, override_level.upper())
             logging.getLogger(package_name).setLevel(override_level_obj)
         except AttributeError:
-            print(f"Warning: Invalid log level '{override_level}' for package '{package_name}'")
+            print(
+                f"Warning: Invalid log level '{override_level}' for package '{package_name}'"
+            )
 
     # Get logger for our application
     logger = logging.getLogger("lsc_eval")
@@ -123,7 +129,9 @@ def populate_metric_mappings(metrics_metadata: Dict[str, Any]) -> None:
             CONVERSATION_LEVEL_METRICS.add(metric_name)
 
 
-def validate_metrics(turn_metrics: List[str], conversation_metrics: List[str]) -> List[str]:
+def validate_metrics(
+    turn_metrics: List[str], conversation_metrics: List[str]
+) -> List[str]:
     """Validate that provided metrics are recognized."""
     errors = []
 
@@ -154,7 +162,9 @@ class SystemConfig(BaseModel):
     # Logging Configuration
     logging_source_level: str = Field(default="INFO")
     logging_package_level: str = Field(default="WARNING")
-    logging_format: str = Field(default="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    logging_format: str = Field(
+        default="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
     logging_show_timestamps: bool = Field(default=True)
     logging_package_overrides: Dict[str, str] = Field(default_factory=dict)
 
@@ -172,8 +182,12 @@ class SystemConfig(BaseModel):
     visualization_dpi: int = Field(default=300)
 
     # Default metrics metadata from system config
-    default_turn_metrics_metadata: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
-    default_conversation_metrics_metadata: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
+    default_turn_metrics_metadata: Dict[str, Dict[str, Any]] = Field(
+        default_factory=dict
+    )
+    default_conversation_metrics_metadata: Dict[str, Dict[str, Any]] = Field(
+        default_factory=dict
+    )
 
 
 class ConfigLoader:
@@ -232,7 +246,9 @@ def load_system_config(self, config_path: str) -> SystemConfig:
             visualization_dpi=visualization_config.get("dpi", 300),
             # Default metrics metadata from system config
             default_turn_metrics_metadata=metrics_metadata.get("turn_level", {}),
-            default_conversation_metrics_metadata=metrics_metadata.get("conversation_level", {}),
+            default_conversation_metrics_metadata=metrics_metadata.get(
+                "conversation_level", {}
+            ),
         )
 
         self.logger.debug(

lsc_eval/src/lsc_eval/core/models.py

@@ -56,7 +56,9 @@ class EvaluationData(BaseModel):
 
     # Metric-specific configuration (threshold, weights, etc.)
     turn_metrics_metadata: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
-    conversation_metrics_metadata: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
+    conversation_metrics_metadata: Dict[str, Dict[str, Any]] = Field(
+        default_factory=dict
+    )
 
     # Conversation turns
     turns: List[TurnData]
@@ -83,7 +85,9 @@ def validate_metrics(cls, v: List[str]) -> List[str]:
         """Validate metrics are properly formatted."""
         for metric in v:
             if not metric or ":" not in metric:
-                raise ValueError(f'Metric "{metric}" must be in format "framework:metric_name"')
+                raise ValueError(
+                    f'Metric "{metric}" must be in format "framework:metric_name"'
+                )
         return v
 
     def validate_metric_requirements(self) -> List[str]:
@@ -114,7 +118,10 @@ def validate_metric_requirements(self) -> List[str]:
                         f"TurnData {turn_data.turn_id}: Metric '{metric}' requires contexts"
                     )
 
-                if metric in expected_response_required_metrics and not turn_data.expected_response:
+                if (
+                    metric in expected_response_required_metrics
+                    and not turn_data.expected_response
+                ):
                     errors.append(
                         f"TurnData {turn_data.turn_id}: "
                         f"Metric '{metric}' requires expected_response"
@@ -168,10 +175,16 @@ class LLMConfig(BaseModel):
     model_name: str = Field(..., description="The model name to use")
     api_base: Optional[str] = Field(None, description="Custom API base URL")
     api_key: Optional[str] = Field(None, description="API key for the model")
-    temperature: float = Field(0.0, ge=0.0, le=2.0, description="Temperature for sampling")
-    max_tokens: Optional[int] = Field(None, ge=1, description="Maximum tokens to generate")
+    temperature: float = Field(
+        0.0, ge=0.0, le=2.0, description="Temperature for sampling"
+    )
+    max_tokens: Optional[int] = Field(
+        None, ge=1, description="Maximum tokens to generate"
+    )
     timeout: Optional[int] = Field(None, ge=1, description="Request timeout in seconds")
-    num_retries: int = Field(3, ge=0, description="Number of retries for failed requests")
+    num_retries: int = Field(
+        3, ge=0, description="Number of retries for failed requests"
+    )
 
     @field_validator("model_name")
     @classmethod

lsc_eval/src/lsc_eval/evaluation_engine.py

@@ -7,7 +7,13 @@
 import time
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from .core import ConfigLoader, DataValidator, EvaluationData, EvaluationResult, TurnData
+from .core import (
+    ConfigLoader,
+    DataValidator,
+    EvaluationData,
+    EvaluationResult,
+    TurnData,
+)
 from .llm_managers.llm_manager import LLMManager
 from .metrics.custom_metrics import CustomMetrics
 from .metrics.deepeval_metrics import DeepEvalMetrics
@@ -29,7 +35,11 @@ def __init__(self, conv_data: EvaluationData, metric_identifier: str):
 
     @classmethod
     def for_turn(
-        cls, conv_data: EvaluationData, metric_identifier: str, turn_idx: int, turn_data: TurnData
+        cls,
+        conv_data: EvaluationData,
+        metric_identifier: str,
+        turn_idx: int,
+        turn_data: TurnData,
     ) -> "EvaluationRequest":
         """Create request for turn-level evaluation."""
         request = cls(conv_data, metric_identifier)
@@ -74,7 +84,9 @@ def __init__(self, llm_manager: LLMManager):
         self.custom_metrics = CustomMetrics(llm_manager)
 
         # Metric routing map
-        self.handlers: Dict[str, Union[RagasMetrics, DeepEvalMetrics, CustomMetrics]] = {
+        self.handlers: Dict[
+            str, Union[RagasMetrics, DeepEvalMetrics, CustomMetrics]
+        ] = {
             "ragas": self.ragas_metrics,
             "deepeval": self.deepeval_metrics,
             "custom": self.custom_metrics,
@@ -131,7 +143,9 @@ def validate_data(self, evaluation_data: List[EvaluationData]) -> bool:
         """Validate evaluation data using data validator."""
         return self.data_validator.validate_evaluation_data(evaluation_data)
 
-    def run_evaluation(self, evaluation_data: List[EvaluationData]) -> List[EvaluationResult]:
+    def run_evaluation(
+        self, evaluation_data: List[EvaluationData]
+    ) -> List[EvaluationResult]:
         """
         Run complete evaluation pipeline.
 
@@ -177,10 +191,14 @@ def _process_conversation(self, conv_data: EvaluationData) -> None:
             print(f"🗣️ Conversation-level metrics: {conv_data.conversation_metrics}")
             self._evaluate_conversation(conv_data)
 
-    def _evaluate_turn(self, conv_data: EvaluationData, turn_idx: int, turn_data: TurnData) -> None:
+    def _evaluate_turn(
+        self, conv_data: EvaluationData, turn_idx: int, turn_data: TurnData
+    ) -> None:
         """Evaluate single turn with specified turn metrics."""
         for metric_identifier in conv_data.turn_metrics:
-            request = EvaluationRequest.for_turn(conv_data, metric_identifier, turn_idx, turn_data)
+            request = EvaluationRequest.for_turn(
+                conv_data, metric_identifier, turn_idx, turn_data
+            )
             result = self._evaluate_metric(request)
             if result:
                 self.results.append(result)
@@ -193,7 +211,9 @@ def _evaluate_conversation(self, conv_data: EvaluationData) -> None:
             if result:
                 self.results.append(result)
 
-    def _evaluate_metric(self, request: EvaluationRequest) -> Optional[EvaluationResult]:
+    def _evaluate_metric(
+        self, request: EvaluationRequest
+    ) -> Optional[EvaluationResult]:
         """
         Evaluate single metric using context.
 
@@ -213,7 +233,9 @@ def _evaluate_metric(self, request: EvaluationRequest) -> Optional[EvaluationRes
             print(f"    {request.metric_identifier} (threshold: {threshold})")
 
             # Route to metrics manager
-            score, reason = self.metrics_manager.evaluate_metric(framework, metric_name, request)
+            score, reason = self.metrics_manager.evaluate_metric(
+                framework, metric_name, request
+            )
 
             # Determine result status
             if score is None:
@@ -222,7 +244,9 @@ def _evaluate_metric(self, request: EvaluationRequest) -> Optional[EvaluationRes
             else:
                 result_status = self._determine_status(score, threshold)
                 status_emoji = (
-                    "✅" if result_status == "PASS" else "❌" if result_status == "FAIL" else "⚠️"
+                    "✅"
+                    if result_status == "PASS"
+                    else "❌" if result_status == "FAIL" else "⚠️"
                 )
                 print(f"      {status_emoji} {result_status}: {score:.3f}")
 
@@ -266,7 +290,9 @@ def _get_effective_threshold(
         """Get effective threshold for metric (conversation-specific or system default)."""
         # Check conversation-specific metadata first
         if is_conversation:
-            metadata = conv_data.conversation_metrics_metadata.get(metric_identifier, {})
+            metadata = conv_data.conversation_metrics_metadata.get(
+                metric_identifier, {}
+            )
         else:
             metadata = conv_data.turn_metrics_metadata.get(metric_identifier, {})
 
@@ -279,9 +305,9 @@ def _get_effective_threshold(
             return None
 
         if is_conversation:
-            default_metadata = (system_config.default_conversation_metrics_metadata or {}).get(
-                metric_identifier, {}
-            )
+            default_metadata = (
+                system_config.default_conversation_metrics_metadata or {}
+            ).get(metric_identifier, {})
         else:
             default_metadata = (system_config.default_turn_metrics_metadata or {}).get(
                 metric_identifier, {}

lsc_eval/src/lsc_eval/llm_managers/llm_manager.py

@@ -47,7 +47,9 @@ def __init__(self, config: LLMConfig):
         """Initialize with validated environment and constructed model name."""
         self.config = config
         self.model_name = self._construct_model_name_and_validate()
-        print(f"✅ LLM Manager: {self.config.provider}/{self.config.model} -> {self.model_name}")
+        print(
+            f"✅ LLM Manager: {self.config.provider}/{self.config.model} -> {self.model_name}"
+        )
 
     def _construct_model_name_and_validate(self) -> str:
         """Construct model name for LiteLLM and validate required environment variables."""
@@ -73,7 +75,9 @@ def _construct_model_name_and_validate(self) -> str:
     def _validate_openai_env(self) -> None:
         """Validate OpenAI environment variables."""
         if not os.environ.get("OPENAI_API_KEY"):
-            raise LLMError("OPENAI_API_KEY environment variable is required for OpenAI provider")
+            raise LLMError(
+                "OPENAI_API_KEY environment variable is required for OpenAI provider"
+            )
 
     def _validate_azure_env(self) -> None:
         """Validate Azure OpenAI environment variables."""
@@ -85,7 +89,9 @@ def _validate_watsonx_env(self) -> None:
         """Validate Watsonx environment variables."""
         required = ["WATSONX_API_KEY", "WATSONX_API_BASE", "WATSONX_PROJECT_ID"]
         if not all(os.environ.get(var) for var in required):
-            raise LLMError(f"Watsonx provider requires environment variables: {required}")
+            raise LLMError(
+                f"Watsonx provider requires environment variables: {required}"
+            )
 
     def _validate_anthropic_env(self) -> None:
         """Validate Anthropic environment variables."""

lsc_eval/src/lsc_eval/llm_managers/ragas_llm.py

@@ -29,7 +29,11 @@ def generate_text(  # pylint: disable=too-many-arguments,too-many-positional-arg
         prompt_text = str(prompt)
 
         # Use temperature from params unless explicitly overridden
-        temp = temperature if temperature != 1e-08 else self.litellm_params.get("temperature", 0.0)
+        temp = (
+            temperature
+            if temperature != 1e-08
+            else self.litellm_params.get("temperature", 0.0)
+        )
 
         try:
             response = litellm.completion(
@@ -68,7 +72,9 @@ async def agenerate_text(  # pylint: disable=too-many-arguments,too-many-positio
     ) -> LLMResult:
         """Async generate."""
         temp = temperature if temperature is not None else 1e-08
-        return self.generate_text(prompt, n=n, temperature=temp, stop=stop, callbacks=callbacks)
+        return self.generate_text(
+            prompt, n=n, temperature=temp, stop=stop, callbacks=callbacks
+        )
 
     def is_finished(self, response: LLMResult) -> bool:
         """Check if response is complete."""