diff --git a/holmes/plugins/toolsets/__init__.py b/holmes/plugins/toolsets/__init__.py index e05aaa5753..6932c82edf 100644 --- a/holmes/plugins/toolsets/__init__.py +++ b/holmes/plugins/toolsets/__init__.py @@ -20,7 +20,12 @@ from holmes.plugins.toolsets.datadog.toolset_datadog_metrics import ( DatadogMetricsToolset, ) -from holmes.plugins.toolsets.datadog.toolset_datadog_traces import DatadogTracesToolset +from holmes.plugins.toolsets.datadog.toolset_datadog_traces import ( + DatadogTracesToolset, +) +from holmes.plugins.toolsets.datadog.toolset_datadog_rds import ( + DatadogRDSToolset, +) from holmes.plugins.toolsets.git import GitToolset from holmes.plugins.toolsets.grafana.toolset_grafana import GrafanaToolset from holmes.plugins.toolsets.grafana.toolset_grafana_loki import GrafanaLokiToolset @@ -75,6 +80,7 @@ def load_python_toolsets(dal: Optional[SupabaseDal]) -> List[Toolset]: DatadogLogsToolset(), DatadogMetricsToolset(), DatadogTracesToolset(), + DatadogRDSToolset(), PrometheusToolset(), OpenSearchLogsToolset(), OpenSearchTracesToolset(), diff --git a/holmes/plugins/toolsets/datadog/datadog_rds_instructions.jinja2 b/holmes/plugins/toolsets/datadog/datadog_rds_instructions.jinja2 new file mode 100644 index 0000000000..91e21d42ea --- /dev/null +++ b/holmes/plugins/toolsets/datadog/datadog_rds_instructions.jinja2 @@ -0,0 +1,82 @@ +## Datadog RDS Performance Analysis Instructions + +You have access to tools for analyzing RDS database performance and identifying problematic instances using Datadog metrics. + +### Available Tools: + +1. **datadog_rds_performance_report** - Generate comprehensive performance report for a specific RDS instance + - Analyzes latency, resource utilization, and storage metrics + - Identifies performance issues and bottlenecks + - Provides actionable recommendations + - Returns formatted report with executive summary + +2. **datadog_rds_top_worst_performing** - Get summary of worst performing RDS instances + - Analyzes all RDS instances in the environment + - Ranks by latency, CPU, or composite performance score + - Shows top N worst performers with their key metrics + - Helps prioritize optimization efforts + +### Usage Guidelines: + +**For investigating a specific RDS instance:** +``` +Use datadog_rds_performance_report with: +- db_instance_identifier: "instance-name" +- start_time: "-3600" (last hour) +``` + +**For finding problematic instances across the fleet:** +``` +Use datadog_rds_top_worst_performing with: +- top_n: 10 (show top 10 worst) +- sort_by: "latency" (or "cpu", "composite") +- start_time: "-3600" +``` + +### Key Performance Thresholds: + +The tools automatically flag issues based on these thresholds: +- **Latency**: >10ms average (warning), >50ms peak (critical) +- **CPU**: >70% average (warning), >90% peak (critical) +- **Memory**: <100MB freeable memory (warning) +- **Burst Balance**: <30% (warning, indicates I/O constraints) +- **Disk Queue Depth**: >5 average (indicates I/O bottleneck) + +### Common Scenarios: + +1. **Application experiencing slow database queries:** + - Generate performance report for the specific RDS instance + - Look for latency spikes and resource constraints + - Follow recommendations for optimization + +2. **Proactive performance monitoring:** + - Use top worst performing to identify problem instances + - Generate detailed reports for the worst performers + - Plan capacity upgrades based on findings + +3. **Capacity planning:** + - Analyze resource utilization trends + - Identify instances approaching limits + - Plan upgrades before performance degradation + +### Interpreting Results: + +**Performance Report Sections:** +- **Executive Summary**: High-level assessment and severity +- **Metrics Tables**: Statistical analysis of each metric +- **Issues**: Specific problems detected with thresholds exceeded +- **Recommendations**: Prioritized actions to resolve issues + +**Top Worst Performing Report:** +- **Rankings**: Instances sorted by selected metric +- **Key Metrics**: Latency, CPU, burst balance for each instance +- **Summary**: Overall patterns across the fleet + +### Example Responses: + +When asked about database performance issues: +1. First use `datadog_rds_top_worst_performing` to identify problem instances +2. Then use `datadog_rds_performance_report` on the worst performers +3. Summarize findings and provide prioritized recommendations + +Always consider the time range - recent data (last hour) for current issues, longer ranges (last 24 hours) for trends. diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_rds.py b/holmes/plugins/toolsets/datadog/toolset_datadog_rds.py new file mode 100644 index 0000000000..9826a5618b --- /dev/null +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_rds.py @@ -0,0 +1,735 @@ +import json +import logging +import os +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple + + +from holmes.core.tools import ( + CallablePrerequisite, + StructuredToolResult, + Tool, + ToolParameter, + ToolResultStatus, + Toolset, + ToolsetTag, +) +from holmes.plugins.toolsets.consts import ( + TOOLSET_CONFIG_MISSING_ERROR, + STANDARD_END_DATETIME_TOOL_PARAM_DESCRIPTION, +) +from holmes.plugins.toolsets.datadog.datadog_api import ( + DatadogBaseConfig, + DataDogRequestError, + execute_datadog_http_request, + get_headers, +) +from holmes.plugins.toolsets.utils import ( + get_param_or_raise, + process_timestamps_to_int, + standard_start_datetime_tool_param_description, +) + +DEFAULT_TIME_SPAN_SECONDS = 3600 +DEFAULT_TOP_INSTANCES = 10 + +# Metric definitions +LATENCY_METRICS = [ + ("aws.rds.read_latency", "Read Latency", "ms"), + ("aws.rds.write_latency", "Write Latency", "ms"), + ("aws.rds.commit_latency", "Commit Latency", "ms"), + ("aws.rds.disk_queue_depth", "Disk Queue Depth", ""), +] + +RESOURCE_METRICS = [ + ("aws.rds.cpuutilization", "CPU Utilization", "%"), + ("aws.rds.database_connections", "Database Connections", "connections"), + ("aws.rds.freeable_memory", "Freeable Memory", "bytes"), + ("aws.rds.swap_usage", "Swap Usage", "bytes"), +] + +STORAGE_METRICS = [ + ("aws.rds.read_iops", "Read IOPS", "iops"), + ("aws.rds.write_iops", "Write IOPS", "iops"), + ("aws.rds.burst_balance", "Burst Balance", "%"), + ("aws.rds.free_storage_space", "Free Storage Space", "bytes"), +] + + +class DatadogRDSConfig(DatadogBaseConfig): + default_time_span_seconds: int = DEFAULT_TIME_SPAN_SECONDS + default_top_instances: int = DEFAULT_TOP_INSTANCES + + +class BaseDatadogRDSTool(Tool): + toolset: "DatadogRDSToolset" + + +class GenerateRDSPerformanceReport(BaseDatadogRDSTool): + def __init__(self, toolset: "DatadogRDSToolset"): + super().__init__( + name="datadog_rds_performance_report", + description="Generate a comprehensive performance report for a specific RDS instance including latency, resource utilization, and storage metrics with analysis", + parameters={ + "db_instance_identifier": ToolParameter( + description="The RDS database instance identifier", + type="string", + required=True, + ), + "start_time": ToolParameter( + description=standard_start_datetime_tool_param_description( + DEFAULT_TIME_SPAN_SECONDS + ), + type="string", + required=False, + ), + "end_time": ToolParameter( + description=STANDARD_END_DATETIME_TOOL_PARAM_DESCRIPTION, + type="string", + required=False, + ), + }, + toolset=toolset, + ) + + def _invoke(self, params: Any) -> StructuredToolResult: + if not self.toolset.dd_config: + return StructuredToolResult( + status=ToolResultStatus.ERROR, + error=TOOLSET_CONFIG_MISSING_ERROR, + params=params, + ) + + try: + db_instance = get_param_or_raise(params, "db_instance_identifier") + start_time, end_time = process_timestamps_to_int( + start=params.get("start_time"), + end=params.get("end_time"), + default_time_span_seconds=self.toolset.dd_config.default_time_span_seconds, + ) + + report: dict[str, Any] = { + "instance_id": db_instance, + "report_time": datetime.now(timezone.utc).isoformat(), + "time_range": { + "start": datetime.fromtimestamp( + start_time, tz=timezone.utc + ).isoformat(), + "end": datetime.fromtimestamp( + end_time, tz=timezone.utc + ).isoformat(), + }, + "sections": {}, + "issues": [], + "executive_summary": "", + } + + # Collect all metrics + all_metrics = [] + for metric_group, group_name in [ + (LATENCY_METRICS, "latency"), + (RESOURCE_METRICS, "resources"), + (STORAGE_METRICS, "storage"), + ]: + section_data = self._collect_metrics( + db_instance, metric_group, start_time, end_time + ) + if section_data: + report["sections"][group_name] = section_data + all_metrics.extend(section_data.get("metrics", {}).items()) + + # Analyze metrics and generate insights + self._analyze_metrics(report, all_metrics) + + # Generate executive summary + report["executive_summary"] = self._generate_executive_summary(report) + + # Format the report as readable text + formatted_report = self._format_report(report) + + return StructuredToolResult( + status=ToolResultStatus.SUCCESS, + data=formatted_report, + params=params, + ) + + except Exception as e: + logging.error(f"Error generating RDS performance report: {str(e)}") + return StructuredToolResult( + status=ToolResultStatus.ERROR, + error=f"Failed to generate RDS performance report: {str(e)}", + params=params, + ) + + def _collect_metrics( + self, + db_instance: str, + metric_list: List[Tuple[str, str, str]], + start_time: int, + end_time: int, + ) -> Dict[str, Any]: + """Collect metrics for a specific group""" + if not self.toolset.dd_config: + raise Exception(TOOLSET_CONFIG_MISSING_ERROR) + + metrics = {} + + for metric_name, display_name, unit in metric_list: + query = f"{metric_name}{{dbinstanceidentifier:{db_instance}}}" + + try: + url = f"{self.toolset.dd_config.site_api_url}/api/v1/query" + headers = get_headers(self.toolset.dd_config) + payload = { + "query": query, + "from": start_time, + "to": end_time, + } + + response = execute_datadog_http_request( + url=url, + headers=headers, + payload_or_params=payload, + timeout=self.toolset.dd_config.request_timeout, + method="GET", + ) + + if response and "series" in response and response["series"]: + series = response["series"][0] + points = series.get("pointlist", []) + + if points: + values = [p[1] for p in points if p[1] is not None] + if values: + metrics[display_name] = { + "unit": unit + or series.get("unit", [{"short_name": ""}])[0].get( + "short_name", "" + ), + "avg": round(sum(values) / len(values), 2), + "max": round(max(values), 2), + "min": round(min(values), 2), + "latest": round(values[-1], 2), + "data_points": len(values), + } + except DataDogRequestError: + continue + + return {"metrics": metrics} if metrics else {} + + def _analyze_metrics(self, report: Dict, all_metrics: List[Tuple[str, Dict]]): + """Analyze metrics and generate issues""" + for metric_name, data in all_metrics: + # Latency analysis + if "Latency" in metric_name and metric_name != "Commit Latency": + if data["avg"] > 10: + report["issues"].append( + f"{metric_name} averaging {data['avg']}ms (above 10ms threshold)" + ) + if data["max"] > 50: + report["issues"].append(f"{metric_name} peaked at {data['max']}ms") + + # Disk queue depth + elif metric_name == "Disk Queue Depth": + if data["avg"] > 5: + report["issues"].append( + f"High disk queue depth (avg: {data['avg']})" + ) + + # CPU utilization + elif metric_name == "CPU Utilization": + if data["avg"] > 70: + report["issues"].append( + f"High CPU utilization (avg: {data['avg']}%)" + ) + if data["max"] > 90: + report["issues"].append( + f"CPU saturation detected (max: {data['max']}%)" + ) + + # Memory + elif metric_name == "Freeable Memory": + if data["min"] < 100 * 1024 * 1024: # 100MB + report["issues"].append( + f"Low memory availability (min: {data['min'] / 1024 / 1024:.1f}MB)" + ) + + # Swap usage + elif metric_name == "Swap Usage": + if data["avg"] > 0: + report["issues"].append( + "Swap usage detected, indicating memory pressure" + ) + + # Burst balance + elif metric_name == "Burst Balance": + if data["min"] < 30: + report["issues"].append( + f"Low burst balance detected (min: {data['min']}%)" + ) + + # IOPS + elif "IOPS" in metric_name: + if data["max"] > 3000: + report["issues"].append( + f"High {metric_name} (max: {data['max']} IOPS)" + ) + + def _generate_executive_summary(self, report: Dict) -> str: + """Generate executive summary""" + issue_count = len(report["issues"]) + + if issue_count == 0: + return "Database is operating within normal parameters. No significant issues detected." + elif issue_count <= 2: + severity = "Low" + elif issue_count <= 5: + severity = "Medium" + else: + severity = "High" + + summary = f"Performance diagnosis: {severity} severity - {issue_count} issues detected.\n\n" + + # Add key findings + if any("latency" in issue.lower() for issue in report["issues"]): + summary += "• Latency issues affecting database response times\n" + if any("cpu" in issue.lower() for issue in report["issues"]): + summary += "• CPU resource constraints detected\n" + if any( + "memory" in issue.lower() or "swap" in issue.lower() + for issue in report["issues"] + ): + summary += "• Memory pressure affecting performance\n" + if any( + "burst" in issue.lower() or "iops" in issue.lower() + for issue in report["issues"] + ): + summary += "• Storage I/O bottlenecks identified\n" + + return summary + + def _format_report(self, report: Dict) -> str: + """Format the report as readable text""" + lines = [] + lines.append(f"RDS Performance Report - {report['instance_id']}") + lines.append("=" * 70) + lines.append(f"Generated: {report['report_time']}") + lines.append( + f"Time Range: {report['time_range']['start']} to {report['time_range']['end']}" + ) + lines.append("") + + # Executive Summary + lines.append("EXECUTIVE SUMMARY") + lines.append("-" * 40) + lines.append(report["executive_summary"]) + lines.append("") + + # Metrics sections + for section_name, section_data in report["sections"].items(): + lines.append(f"{section_name.upper()} METRICS") + lines.append("-" * 40) + + if section_data.get("metrics"): + lines.append( + f"{'Metric':<25} {'Avg':>10} {'Max':>10} {'Min':>10} {'Latest':>10} {'Unit':>8}" + ) + lines.append("-" * 80) + + for metric_name, data in section_data["metrics"].items(): + lines.append( + f"{metric_name:<25} {data['avg']:>10.2f} {data['max']:>10.2f} " + f"{data['min']:>10.2f} {data['latest']:>10.2f} {data['unit']:>8}" + ) + lines.append("") + + # Issues + if report["issues"]: + lines.append(f"ISSUES DETECTED ({len(report['issues'])})") + lines.append("-" * 40) + for i, issue in enumerate(report["issues"], 1): + lines.append(f"{i}. {issue}") + lines.append("") + + return "\n".join(lines) + + def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: + db_instance = params.get("db_instance_identifier", "unknown") + return f"Generating performance report for RDS instance: {db_instance}" + + +class GetTopWorstPerformingRDSInstances(BaseDatadogRDSTool): + def __init__(self, toolset: "DatadogRDSToolset"): + super().__init__( + name="datadog_rds_top_worst_performing", + description="Get a summarized report of the top worst performing RDS instances based on latency, CPU utilization, and error rates", + parameters={ + "top_n": ToolParameter( + description=f"Number of worst performing instances to return (default: {DEFAULT_TOP_INSTANCES})", + type="number", + required=False, + ), + "start_time": ToolParameter( + description=standard_start_datetime_tool_param_description( + DEFAULT_TIME_SPAN_SECONDS + ), + type="string", + required=False, + ), + "end_time": ToolParameter( + description=STANDARD_END_DATETIME_TOOL_PARAM_DESCRIPTION, + type="string", + required=False, + ), + "sort_by": ToolParameter( + description="Metric to sort by: 'latency' (default), 'cpu', 'errors', or 'composite'", + type="string", + required=False, + ), + }, + toolset=toolset, + ) + + def _invoke(self, params: Any) -> StructuredToolResult: + if not self.toolset.dd_config: + return StructuredToolResult( + status=ToolResultStatus.ERROR, + error=TOOLSET_CONFIG_MISSING_ERROR, + params=params, + ) + + try: + top_n = params.get("top_n", self.toolset.dd_config.default_top_instances) + sort_by = params.get("sort_by", "latency").lower() + start_time, end_time = process_timestamps_to_int( + start=params.get("start_time"), + end=params.get("end_time"), + default_time_span_seconds=self.toolset.dd_config.default_time_span_seconds, + ) + + # Get all RDS instances + instances = self._get_all_rds_instances(start_time, end_time) + + if not instances: + return StructuredToolResult( + status=ToolResultStatus.NO_DATA, + data="No RDS instances found with metrics in the specified time range", + params=params, + ) + + # Collect performance data for each instance + instance_performance = [] + for instance_id in instances[:50]: # Limit to 50 instances to avoid timeout + perf_data = self._get_instance_performance_summary( + instance_id, start_time, end_time + ) + if perf_data: + instance_performance.append(perf_data) + + # Sort by the specified metric + instance_performance = self._sort_instances(instance_performance, sort_by) + + # Get top N worst performers + worst_performers = instance_performance[:top_n] + + # Format the report + report = self._format_summary_report(worst_performers, sort_by) + + report += f"\n\nTotal instances analyzed: {len(instance_performance)}" + report += f"\n\nInstances:\n{json.dumps(worst_performers, indent=2)}" + + return StructuredToolResult( + status=ToolResultStatus.SUCCESS, + data=report, + params=params, + ) + + except Exception as e: + logging.error(f"Error getting top worst performing RDS instances: {str(e)}") + return StructuredToolResult( + status=ToolResultStatus.ERROR, + error=f"Failed to get top worst performing RDS instances: {str(e)}", + params=params, + ) + + def _get_all_rds_instances(self, start_time: int, end_time: int) -> List[str]: + """Get list of all RDS instances with metrics""" + if not self.toolset.dd_config: + raise Exception(TOOLSET_CONFIG_MISSING_ERROR) + try: + # Query for any RDS metric grouped by instance + query = "avg:aws.rds.cpuutilization{*} by {dbinstanceidentifier}" + + url = f"{self.toolset.dd_config.site_api_url}/api/v1/query" + headers = get_headers(self.toolset.dd_config) + payload = { + "query": query, + "from": start_time, + "to": end_time, + } + + response = execute_datadog_http_request( + url=url, + headers=headers, + payload_or_params=payload, + timeout=self.toolset.dd_config.request_timeout, + method="GET", + ) + + instances = [] + if response and "series" in response: + for series in response["series"]: + # Extract instance ID from tags + scope = series.get("scope", "") + if "dbinstanceidentifier:" in scope: + instance_id = scope.split("dbinstanceidentifier:")[1].split( + "," + )[0] + instances.append(instance_id) + + return list(set(instances)) # Remove duplicates + + except Exception as e: + logging.error(f"Error getting RDS instances: {str(e)}") + return [] + + def _get_instance_performance_summary( + self, instance_id: str, start_time: int, end_time: int + ) -> Optional[Dict]: + """Get performance summary for a single instance""" + + if not self.toolset.dd_config: + raise Exception(TOOLSET_CONFIG_MISSING_ERROR) + + summary: dict[str, Any] = { + "instance_id": instance_id, + "metrics": {}, + "score": 0, # Composite score for sorting + "issues": [], + } + + # Key metrics to collect + metrics_to_collect = [ + ("aws.rds.read_latency", "read_latency", 1.0), # weight for composite score + ("aws.rds.write_latency", "write_latency", 1.0), + ("aws.rds.cpuutilization", "cpu_utilization", 0.5), + ("aws.rds.database_connections", "connections", 0.2), + ("aws.rds.burst_balance", "burst_balance", 0.8), + ] + + for metric_name, key, weight in metrics_to_collect: + query = f"avg:{metric_name}{{dbinstanceidentifier:{instance_id}}}" + + try: + url = f"{self.toolset.dd_config.site_api_url}/api/v1/query" + headers = get_headers(self.toolset.dd_config) + payload = { + "query": query, + "from": start_time, + "to": end_time, + } + + response = execute_datadog_http_request( + url=url, + headers=headers, + payload_or_params=payload, + timeout=self.toolset.dd_config.request_timeout, + method="GET", + ) + + if response and "series" in response and response["series"]: + series = response["series"][0] + points = series.get("pointlist", []) + + if points: + values = [p[1] for p in points if p[1] is not None] + if values: + avg_value = sum(values) / len(values) + max_value = max(values) + + summary["metrics"][key] = { + "avg": round(avg_value, 2), + "max": round(max_value, 2), + } + + # Calculate contribution to composite score + if key in ["read_latency", "write_latency"]: + # Higher latency = worse performance + score_contrib = avg_value * weight + if avg_value > 10: + summary["issues"].append( + f"High {key.replace('_', ' ')}: {avg_value:.1f}ms" + ) + elif key == "cpu_utilization": + # Higher CPU = worse performance + score_contrib = avg_value * weight + if avg_value > 70: + summary["issues"].append( + f"High CPU: {avg_value:.1f}%" + ) + elif key == "burst_balance": + # Lower burst balance = worse performance + score_contrib = (100 - avg_value) * weight + if avg_value < 30: + summary["issues"].append( + f"Low burst balance: {avg_value:.1f}%" + ) + else: + score_contrib = 0 + + summary["score"] += score_contrib + + except Exception: + continue + + return summary if summary["metrics"] else None + + def _sort_instances(self, instances: List[Dict], sort_by: str) -> List[Dict]: + """Sort instances by specified metric""" + if sort_by == "latency": + # Sort by average of read and write latency + def latency_key(inst): + read_lat = inst["metrics"].get("read_latency", {}).get("avg", 0) + write_lat = inst["metrics"].get("write_latency", {}).get("avg", 0) + return (read_lat + write_lat) / 2 + + return sorted(instances, key=latency_key, reverse=True) + + elif sort_by == "cpu": + return sorted( + instances, + key=lambda x: x["metrics"].get("cpu_utilization", {}).get("avg", 0), + reverse=True, + ) + + elif sort_by == "composite": + return sorted(instances, key=lambda x: x["score"], reverse=True) + + else: # Default to latency + return self._sort_instances(instances, "latency") + + def _format_summary_report(self, instances: List[Dict], sort_by: str) -> str: + """Format the summary report""" + lines = [] + lines.append("Top Worst Performing RDS Instances") + lines.append("=" * 70) + lines.append(f"Sorted by: {sort_by}") + lines.append(f"Instances shown: {len(instances)}") + lines.append("") + + for rank, inst in enumerate(instances, 1): + lines.append(f"{rank}. {inst['instance_id']}") + lines.append("-" * 40) + + # Show key metrics + metrics = inst["metrics"] + if "read_latency" in metrics: + lines.append( + f" Read Latency: {metrics['read_latency']['avg']:.1f}ms avg, {metrics['read_latency']['max']:.1f}ms max" + ) + if "write_latency" in metrics: + lines.append( + f" Write Latency: {metrics['write_latency']['avg']:.1f}ms avg, {metrics['write_latency']['max']:.1f}ms max" + ) + if "cpu_utilization" in metrics: + lines.append( + f" CPU Usage: {metrics['cpu_utilization']['avg']:.1f}% avg, {metrics['cpu_utilization']['max']:.1f}% max" + ) + if "burst_balance" in metrics: + lines.append( + f" Burst Balance: {metrics['burst_balance']['avg']:.1f}% avg" + ) + + # Show issues + if inst["issues"]: + lines.append(" Issues:") + for issue in inst["issues"]: + lines.append(f" • {issue}") + + lines.append("") + + return "\n".join(lines) + + def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: + top_n = params.get("top_n", DEFAULT_TOP_INSTANCES) + sort_by = params.get("sort_by", "latency") + return f"Getting top {top_n} worst performing RDS instances sorted by {sort_by}" + + +class DatadogRDSToolset(Toolset): + dd_config: Optional[DatadogRDSConfig] = None + + def __init__(self): + super().__init__( + name="datadog/rds", + description="Analyze RDS database performance and identify worst performers using Datadog metrics", + tags=[ToolsetTag.CORE], + tools=[ + GenerateRDSPerformanceReport(toolset=self), + GetTopWorstPerformingRDSInstances(toolset=self), + ], + ) + + def prerequisites_check(self, config: Dict[str, Any]) -> CallablePrerequisite: + def check_datadog_connectivity(config_dict: Dict[str, Any]) -> Tuple[bool, str]: + """Check Datadog API connectivity and permissions""" + try: + # Validate config + self.dd_config = DatadogRDSConfig(**config_dict) + + # Test API connectivity + url = f"{self.dd_config.site_api_url}/api/v1/validate" + headers = get_headers(self.dd_config) + + response = execute_datadog_http_request( + url=url, + headers=headers, + payload_or_params={}, + timeout=self.dd_config.request_timeout, + method="GET", + ) + + if response and response.get("valid", False): + # Test metrics API access + metrics_url = f"{self.dd_config.site_api_url}/api/v1/metrics" + execute_datadog_http_request( + url=metrics_url, + headers=headers, + payload_or_params={"from": 0}, + timeout=self.dd_config.request_timeout, + method="GET", + ) + return True, "" + else: + return False, "Invalid Datadog API credentials" + + except DataDogRequestError as e: + if e.status_code == 403: + return False, "Invalid Datadog API keys or insufficient permissions" + else: + return False, f"Datadog API error: {str(e)}" + except Exception as e: + return False, f"Failed to initialize Datadog RDS toolset: {str(e)}" + + return CallablePrerequisite(callable=check_datadog_connectivity) + + def post_init(self, config: dict): + """Load LLM instructions after initialization""" + self._reload_instructions() + + def _reload_instructions(self): + """Load RDS analysis specific instructions""" + template_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "datadog_rds_instructions.jinja2") + ) + self._load_llm_instructions(jinja_template=f"file://{template_file_path}") + + def get_example_config(self) -> Dict[str, Any]: + """Get example configuration for this toolset.""" + return { + "dd_api_key": "your-datadog-api-key", + "dd_app_key": "your-datadog-application-key", + "site_api_url": "https://api.datadoghq.com", + "default_time_span_seconds": 3600, + "default_top_instances": 10, + } diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py b/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py index 6fff0fdab5..b602c8a75a 100644 --- a/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py @@ -1,3 +1,5 @@ +"""Datadog Traces toolset for HolmesGPT.""" + import json import logging import os diff --git a/tests/plugins/toolsets/datadog/rds/test_datadog_rds_integration.py b/tests/plugins/toolsets/datadog/rds/test_datadog_rds_integration.py new file mode 100644 index 0000000000..97c85c1feb --- /dev/null +++ b/tests/plugins/toolsets/datadog/rds/test_datadog_rds_integration.py @@ -0,0 +1,633 @@ +""" +Integration tests for Datadog RDS toolset. + +These tests use mocked responses to test the toolset functionality +without requiring actual Datadog API access. +""" + +import json +from unittest.mock import patch +import pytest +from datetime import datetime, timezone + +from holmes.plugins.toolsets.datadog.toolset_datadog_rds import ( + DatadogRDSToolset, + DatadogRDSConfig, +) +from holmes.plugins.toolsets.datadog.datadog_api import DataDogRequestError +from holmes.core.tools import ToolResultStatus + + +@pytest.fixture +def mock_config(): + """Mock Datadog RDS configuration.""" + return { + "dd_api_key": "test_api_key", + "dd_app_key": "test_app_key", + "site_api_url": "https://api.datadoghq.com", + "request_timeout": 30, + } + + +@pytest.fixture +def datadog_rds_toolset(mock_config): + """Create Datadog RDS toolset with mocked prerequisites.""" + toolset = DatadogRDSToolset() + + # Directly set the config without going through prerequisites + toolset.dd_config = DatadogRDSConfig(**mock_config) + toolset.post_init(mock_config) + + return toolset + + +def create_mock_metric_response( + metric_name, values, unit="ms", instance_id="test-instance" +): + """Helper to create mock Datadog metric response.""" + points = [ + [ + int(datetime.now(timezone.utc).timestamp() - (len(values) - i - 1) * 60) + * 1000, + val, + ] + for i, val in enumerate(values) + ] + + return { + "series": [ + { + "metric": metric_name, + "pointlist": points, + "unit": [{"short_name": unit}], + "scope": f"dbinstanceidentifier:{instance_id}", + } + ] + } + + +def create_mock_instances_response(instance_ids): + """Helper to create mock response for instance list query.""" + series = [] + for instance_id in instance_ids: + series.append( + { + "metric": "aws.rds.cpuutilization", + "scope": f"dbinstanceidentifier:{instance_id},engine:postgres", + "pointlist": [[datetime.now(timezone.utc).timestamp() * 1000, 50.0]], + } + ) + return {"series": series} + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_generate_performance_report_success(mock_execute, datadog_rds_toolset): + """Test successful performance report generation.""" + # Mock responses for different metric groups + mock_execute.side_effect = [ + # Latency metrics + create_mock_metric_response("aws.rds.read_latency", [5.2, 6.1, 5.8, 7.2, 6.5]), + create_mock_metric_response( + "aws.rds.write_latency", [8.1, 9.2, 8.5, 12.1, 10.3] + ), + {"series": []}, # Commit latency - no data + create_mock_metric_response( + "aws.rds.disk_queue_depth", [2.1, 3.2, 2.8, 4.5, 3.9], "" + ), + # Resource metrics + create_mock_metric_response( + "aws.rds.cpuutilization", [45.2, 52.1, 48.5, 62.1, 55.3], "%" + ), + create_mock_metric_response( + "aws.rds.database_connections", [120, 135, 128, 145, 140], "connections" + ), + create_mock_metric_response( + "aws.rds.freeable_memory", + [ + 2048 * 1024 * 1024, + 1856 * 1024 * 1024, + 1984 * 1024 * 1024, + 1728 * 1024 * 1024, + 1920 * 1024 * 1024, + ], + "bytes", + ), + create_mock_metric_response("aws.rds.swap_usage", [0, 0, 0, 0, 0], "bytes"), + # Storage metrics + create_mock_metric_response( + "aws.rds.read_iops", [1500, 2200, 1800, 2500, 2000], "iops" + ), + create_mock_metric_response( + "aws.rds.write_iops", [800, 1200, 950, 1300, 1100], "iops" + ), + create_mock_metric_response("aws.rds.burst_balance", [85, 80, 75, 70, 65], "%"), + create_mock_metric_response( + "aws.rds.free_storage_space", + [50 * 1024**3, 49 * 1024**3, 48 * 1024**3, 47 * 1024**3, 46 * 1024**3], + "bytes", + ), + ] + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_performance_report" + ) + params = { + "db_instance_identifier": "test-instance", + "start_time": "-3600", + } + + result = tool._invoke(params) + + assert result.status == ToolResultStatus.SUCCESS + assert isinstance(result.data, str) + + # Check report content + report = result.data + assert "RDS Performance Report - test-instance" in report + assert "EXECUTIVE SUMMARY" in report + assert "LATENCY METRICS" in report + assert "RESOURCES METRICS" in report + assert "STORAGE METRICS" in report + assert ( + "Database is operating within normal parameters. No significant issues detected." + in report + ) + + # Verify no issues section when everything is normal + assert "ISSUES DETECTED" not in report + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_generate_performance_report_with_issues(mock_execute, datadog_rds_toolset): + """Test performance report with metrics triggering issues.""" + # Mock responses with problematic values + mock_execute.side_effect = [ + # Latency metrics - high values + create_mock_metric_response( + "aws.rds.read_latency", [15.2, 18.1, 55.8, 17.2, 16.5] + ), + create_mock_metric_response( + "aws.rds.write_latency", [22.1, 65.2, 28.5, 32.1, 25.3] + ), + {"series": []}, + create_mock_metric_response( + "aws.rds.disk_queue_depth", [8.1, 12.2, 9.8, 15.5, 10.9], "" + ), + # Resource metrics - high CPU, low memory + create_mock_metric_response( + "aws.rds.cpuutilization", [75.2, 82.1, 78.5, 92.1, 85.3], "%" + ), + create_mock_metric_response( + "aws.rds.database_connections", [120, 135, 128, 145, 140], "connections" + ), + create_mock_metric_response( + "aws.rds.freeable_memory", + [ + 80 * 1024 * 1024, + 60 * 1024 * 1024, + 70 * 1024 * 1024, + 50 * 1024 * 1024, + 65 * 1024 * 1024, + ], + "bytes", + ), + create_mock_metric_response( + "aws.rds.swap_usage", + [1024 * 1024, 2048 * 1024, 1536 * 1024, 3072 * 1024, 2560 * 1024], + "bytes", + ), + # Storage metrics - low burst balance + create_mock_metric_response( + "aws.rds.read_iops", [3500, 4200, 3800, 4500, 4000], "iops" + ), + create_mock_metric_response( + "aws.rds.write_iops", [1500, 1800, 1650, 2000, 1750], "iops" + ), + create_mock_metric_response("aws.rds.burst_balance", [25, 20, 15, 10, 5], "%"), + create_mock_metric_response( + "aws.rds.free_storage_space", + [20 * 1024**3, 19 * 1024**3, 18 * 1024**3, 17 * 1024**3, 16 * 1024**3], + "bytes", + ), + ] + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_performance_report" + ) + result = tool._invoke( + {"db_instance_identifier": "test-instance", "start_time": "-3600"} + ) + + assert result.status == ToolResultStatus.SUCCESS + + report = result.data + assert "ISSUES DETECTED" in report + assert "High severity" in report + + # Check specific issues are mentioned + assert "latency" in report.lower() + assert "cpu" in report.lower() + assert "memory" in report.lower() + assert "swap" in report.lower() + assert "burst balance" in report.lower() + assert "disk queue" in report.lower() + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_get_top_worst_performing_instances(mock_execute, datadog_rds_toolset): + """Test getting top worst performing instances.""" + # Define metrics for each instance + instance_metrics = { + "instance-1": { + "aws.rds.read_latency": 25.0, + "aws.rds.write_latency": 35.0, + "aws.rds.cpuutilization": 85.0, + "aws.rds.database_connections": 200, + "aws.rds.burst_balance": 20.0, + }, + "instance-2": { + "aws.rds.read_latency": 15.0, + "aws.rds.write_latency": 20.0, + "aws.rds.cpuutilization": 60.0, + "aws.rds.database_connections": 150, + "aws.rds.burst_balance": 50.0, + }, + "instance-3": { + "aws.rds.read_latency": 5.0, + "aws.rds.write_latency": 8.0, + "aws.rds.cpuutilization": 30.0, + "aws.rds.database_connections": 80, + "aws.rds.burst_balance": 90.0, + }, + } + + def mock_response(url, **kwargs): + # First call is instance discovery + if not hasattr(mock_response, "call_count"): + mock_response.call_count = 0 + mock_response.call_count += 1 + + if mock_response.call_count == 1: + return create_mock_instances_response( + ["instance-1", "instance-2", "instance-3"] + ) + + # Extract query from payload + query = kwargs.get("payload_or_params", {}).get("query", "") + + # Parse metric and instance from query + for instance_id, metrics in instance_metrics.items(): + if f"dbinstanceidentifier:{instance_id}" in query: + for metric_name, value in metrics.items(): + if metric_name in query: + unit = ( + "%" + if "cpu" in metric_name + else "connections" + if "connections" in metric_name + else "" + ) + return create_mock_metric_response( + metric_name, [value], unit, instance_id + ) + + return {"series": []} + + mock_execute.side_effect = mock_response + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_top_worst_performing" + ) + params = { + "top_n": 2, + "start_time": "-3600", + "sort_by": "latency", + } + + result = tool._invoke(params) + + assert result.status == ToolResultStatus.SUCCESS + assert isinstance(result.data, str) + + # Check report format + report = result.data + assert "Top Worst Performing RDS Instances" in report + assert "Sorted by: latency" in report + assert "Total instances analyzed: 3" in report + + # Check that instances are included in the report + assert "instance-1" in report # Highest latency + assert "instance-2" in report # Second highest + + # Check JSON data is included + assert "Instances:" in report + assert json.loads(report.split("Instances:\n")[1]) # Should be valid JSON + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_top_worst_performing_sort_by_cpu(mock_execute, datadog_rds_toolset): + """Test sorting instances by CPU utilization.""" + # Define metrics for each instance + instance_metrics = { + "instance-1": { + "aws.rds.read_latency": 5.0, + "aws.rds.write_latency": 8.0, + "aws.rds.cpuutilization": 95.0, # High CPU + "aws.rds.database_connections": 100, + "aws.rds.burst_balance": 80.0, + }, + "instance-2": { + "aws.rds.read_latency": 25.0, + "aws.rds.write_latency": 30.0, + "aws.rds.cpuutilization": 40.0, # Low CPU + "aws.rds.database_connections": 80, + "aws.rds.burst_balance": 70.0, + }, + } + + def mock_response(url, **kwargs): + if not hasattr(mock_response, "call_count"): + mock_response.call_count = 0 + mock_response.call_count += 1 + + if mock_response.call_count == 1: + return create_mock_instances_response(["instance-1", "instance-2"]) + + query = kwargs.get("payload_or_params", {}).get("query", "") + + for instance_id, metrics in instance_metrics.items(): + if f"dbinstanceidentifier:{instance_id}" in query: + for metric_name, value in metrics.items(): + if metric_name in query: + unit = ( + "%" + if "cpu" in metric_name + else "connections" + if "connections" in metric_name + else "" + ) + return create_mock_metric_response( + metric_name, [value], unit, instance_id + ) + + return {"series": []} + + mock_execute.side_effect = mock_response + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_top_worst_performing" + ) + result = tool._invoke({"top_n": 10, "start_time": "-3600", "sort_by": "cpu"}) + + assert result.status == ToolResultStatus.SUCCESS + report = result.data + + # When sorted by CPU, instance-1 (95%) should appear before instance-2 (40%) + assert "Sorted by: cpu" in report + # Check that instance-1 appears in the ranking + lines = report.split("\n") + instance_1_line = None + instance_2_line = None + for i, line in enumerate(lines): + if "1. instance-1" in line: + instance_1_line = i + elif "2. instance-2" in line: + instance_2_line = i + + # instance-1 should be ranked higher (appear first) + if instance_1_line is not None and instance_2_line is not None: + assert instance_1_line < instance_2_line + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_no_instances_found(mock_execute, datadog_rds_toolset): + """Test handling when no instances are found.""" + # Return empty series + mock_execute.return_value = {"series": []} + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_top_worst_performing" + ) + result = tool._invoke({"top_n": 5, "start_time": "-3600"}) + + assert result.status == ToolResultStatus.NO_DATA + assert "No RDS instances found" in result.data + + +@patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" +) +def test_api_error_handling(mock_execute, datadog_rds_toolset): + """Test handling of API errors.""" + # Simulate API error for all requests + mock_execute.side_effect = DataDogRequestError( + payload={"query": "test"}, + status_code=403, + response_text="Forbidden", + response_headers={}, + ) + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_performance_report" + ) + result = tool._invoke( + {"db_instance_identifier": "test-instance", "start_time": "-3600"} + ) + + # The tool should succeed but with no metrics collected due to API errors + assert result.status == ToolResultStatus.SUCCESS + report = result.data + assert ( + "Database is operating within normal parameters. No significant issues detected." + in report + ) + + # When API errors occur, no metric sections are included in the report + assert "LATENCY METRICS" not in report + assert "RESOURCES METRICS" not in report + assert "STORAGE METRICS" not in report + + +def test_missing_required_parameter(datadog_rds_toolset): + """Test handling of missing required parameters.""" + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_performance_report" + ) + + # Missing db_instance_identifier + result = tool._invoke({"start_time": "-3600"}) + + assert result.status == ToolResultStatus.ERROR + assert "db_instance_identifier" in result.error + + +def test_prerequisites_check_with_valid_config(): + """Test prerequisites check with valid configuration.""" + toolset = DatadogRDSToolset() + + valid_config = { + "dd_api_key": "test_api_key", + "dd_app_key": "test_app_key", + "site_api_url": "https://api.datadoghq.com", + } + + with patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" + ) as mock_execute: + # Mock successful validation + mock_execute.return_value = {"valid": True} + + prereq = toolset.prerequisites_check(valid_config) + success, message = prereq.callable(valid_config) + + assert success is True + assert message == "" + + +def test_prerequisites_check_with_invalid_credentials(): + """Test prerequisites check with invalid credentials.""" + toolset = DatadogRDSToolset() + + config = { + "dd_api_key": "invalid_key", + "dd_app_key": "invalid_app_key", + "site_api_url": "https://api.datadoghq.com", + } + + with patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" + ) as mock_execute: + # Mock 403 error + mock_execute.side_effect = DataDogRequestError( + payload={}, status_code=403, response_text="Forbidden", response_headers={} + ) + + prereq = toolset.prerequisites_check(config) + success, message = prereq.callable(config) + + assert success is False + assert "Invalid Datadog API keys" in message + + +def test_get_example_config(): + """Test example configuration generation.""" + toolset = DatadogRDSToolset() + example_config = toolset.get_example_config() + + assert "dd_api_key" in example_config + assert "dd_app_key" in example_config + assert "site_api_url" in example_config + assert "default_time_span_seconds" in example_config + assert "default_top_instances" in example_config + + +def test_performance_report_formatting(datadog_rds_toolset): + """Test that performance report is properly formatted.""" + with patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" + ) as mock_execute: + # Mock minimal metrics + mock_execute.side_effect = [ + create_mock_metric_response("aws.rds.read_latency", [5.0]), + {"series": []}, # write_latency - no data + {"series": []}, # commit_latency - no data + {"series": []}, # disk_queue_depth - no data + create_mock_metric_response("aws.rds.cpuutilization", [50.0], "%"), + {"series": []}, # database_connections - no data + {"series": []}, # freeable_memory - no data + {"series": []}, # swap_usage - no data + {"series": []}, # read_iops - no data + {"series": []}, # write_iops - no data + {"series": []}, # burst_balance - no data + {"series": []}, # free_storage_space - no data + ] + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_performance_report" + ) + result = tool._invoke( + {"db_instance_identifier": "test-db", "start_time": "-300"} + ) + + assert result.status == ToolResultStatus.SUCCESS + report = result.data + + # Check report structure + assert "RDS Performance Report - test-db" in report + assert "Generated:" in report + assert "Time Range:" in report + assert "EXECUTIVE SUMMARY" in report + + # Check metrics sections exist even with partial data + assert "LATENCY METRICS" in report + assert "Read Latency" in report + assert "5.00" in report # The metric value + + assert "RESOURCES METRICS" in report + assert "CPU Utilization" in report + assert "50.00" in report # The metric value + + +def test_top_worst_performing_no_metrics(datadog_rds_toolset): + """Test worst performing when instances have no metrics.""" + with patch( + "holmes.plugins.toolsets.datadog.toolset_datadog_rds.execute_datadog_http_request" + ) as mock_execute: + + def mock_response(url, **kwargs): + if not hasattr(mock_response, "call_count"): + mock_response.call_count = 0 + mock_response.call_count += 1 + + if mock_response.call_count == 1: + # Return instances + return create_mock_instances_response(["instance-1", "instance-2"]) + + # Return no metrics for any query + return {"series": []} + + mock_execute.side_effect = mock_response + + tool = next( + t + for t in datadog_rds_toolset.tools + if t.name == "datadog_rds_top_worst_performing" + ) + result = tool._invoke({"top_n": 5, "start_time": "-3600"}) + + assert result.status == ToolResultStatus.SUCCESS + report = result.data + + # Should show 0 instances analyzed (no metrics found) + assert "Total instances analyzed: 0" in report + assert "Instances shown: 0" in report + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/plugins/toolsets/datadog/rds/test_datadog_rds_live.py b/tests/plugins/toolsets/datadog/rds/test_datadog_rds_live.py new file mode 100644 index 0000000000..3beb52c584 --- /dev/null +++ b/tests/plugins/toolsets/datadog/rds/test_datadog_rds_live.py @@ -0,0 +1,99 @@ +""" +Live tests for Datadog RDS toolset. + +These tests require valid Datadog API credentials and access to RDS instances. +Set the following environment variables: +- DD_API_KEY: Datadog API key +- DD_APP_KEY: Datadog Application key +- DD_SITE_API_URL: Datadog site API URL (e.g., https://api.datadoghq.com) +- DD_TEST_RDS_INSTANCE: RDS instance identifier to test with +""" + +import os +import pytest + +from holmes.plugins.toolsets.datadog.toolset_datadog_rds import DatadogRDSToolset +from holmes.core.tools import ToolResultStatus + + +@pytest.fixture +def datadog_rds_config(): + """Get Datadog RDS configuration from environment variables.""" + api_key = os.getenv("DD_API_KEY") + app_key = os.getenv("DD_APP_KEY") + site_url = os.getenv("DD_SITE_URL", "https://api.datadoghq.com") + + if not api_key or not app_key: + pytest.skip("Datadog API credentials not found in environment variables") + + return { + "dd_api_key": api_key, + "dd_app_key": app_key, + "site_api_url": site_url, + "request_timeout": 30, + } + + +@pytest.fixture +def test_rds_instance(): + """Get test RDS instance identifier from environment.""" + instance = os.getenv("DD_TEST_RDS_INSTANCE", "demo-rds-postgres-prod") + return instance + + +@pytest.fixture +def datadog_rds_toolset(datadog_rds_config): + """Create and initialize Datadog RDS toolset.""" + toolset = DatadogRDSToolset() + prereq = toolset.prerequisites_check(datadog_rds_config) + success, message = prereq.callable(datadog_rds_config) + + if not success: + pytest.skip(f"Prerequisites check failed: {message}") + + toolset.post_init(datadog_rds_config) + return toolset + + +def test_generate_performance_report(datadog_rds_toolset, test_rds_instance): + """Test generating RDS performance report.""" + # Get tool by name + tools = {tool.name: tool for tool in datadog_rds_toolset.tools} + tool = tools.get("datadog_rds_performance_report") + assert tool is not None, "datadog_rds_performance_report tool not found" + + # Test with recent data (last hour) + params = { + "db_instance_identifier": test_rds_instance, + "start_time": "-3600", # 1 hour ago + } + + result = tool._invoke(params) + + assert result.status == ToolResultStatus.SUCCESS + assert test_rds_instance in result.data + + +def test_get_top_worst_performing_instances(datadog_rds_toolset, test_rds_instance): + """Test getting top worst performing RDS instances.""" + # Get tool by name + tools = {tool.name: tool for tool in datadog_rds_toolset.tools} + tool = tools.get("datadog_rds_top_worst_performing") + assert tool is not None, "datadog_rds_top_worst_performing tool not found" + + params = { + "top_n": 5, + "start_time": "-3600", + "sort_by": "latency", + } + + result = tool._invoke(params) + + assert result.status == ToolResultStatus.SUCCESS + + assert test_rds_instance in result.data + + +if __name__ == "__main__": + # Run tests with pytest + pytest.main([__file__, "-v"])