diff --git a/conftest.py b/conftest.py index d1beded0e8..dce404fa30 100644 --- a/conftest.py +++ b/conftest.py @@ -260,7 +260,10 @@ def responses(): rsps.add_passthru("https://api.ap1.datadoghq.com") rsps.add_passthru("https://app.datadoghq.com") rsps.add_passthru("https://app.datadoghq.eu") - rsps.add_passthru("https://ng-api-http.eu2.coralogix.com") + # Allow all Coralogix API calls (query and ingestion endpoints, all regions) + rsps.add_passthru(re.compile(r"https://.*\.coralogix\.com")) + rsps.add_passthru(re.compile(r"https://.*\.coralogix\.us")) + rsps.add_passthru(re.compile(r"https://.*\.coralogix\.in")) # Allow Elasticsearch/OpenSearch Cloud API calls (various hosting regions) rsps.add_passthru(re.compile(r"https://.*\.cloud\.es\.io")) # Elastic Cloud diff --git a/docs/data-sources/builtin-toolsets/coralogix-logs.md b/docs/data-sources/builtin-toolsets/coralogix-logs.md index 2ede89ef79..6fcb3c1af1 100644 --- a/docs/data-sources/builtin-toolsets/coralogix-logs.md +++ b/docs/data-sources/builtin-toolsets/coralogix-logs.md @@ -33,7 +33,7 @@ toolsets: ``` -**Note**: Both toolsets use the same API key. The DataPrime toolset supports fields (`CoralogixConfig`): `api_key`, `domain`, `team_hostname`, optional `labels`. +**Note**: Both toolsets use the same API key. The DataPrime toolset supports fields (`CoralogixConfig`): `api_key`, `domain`, `team_hostname`. ## Recommended: Customize Coralogix Instructions diff --git a/holmes/plugins/toolsets/coralogix/utils.py b/holmes/plugins/toolsets/coralogix/utils.py index 8ce72c62fb..f3b73ba8ff 100644 --- a/holmes/plugins/toolsets/coralogix/utils.py +++ b/holmes/plugins/toolsets/coralogix/utils.py @@ -1,34 +1,15 @@ import json import logging from datetime import datetime -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel -class FlattenedLog(NamedTuple): - timestamp: str - log_message: str - - -class CoralogixQueryResult(BaseModel): - logs: List[FlattenedLog] - http_status: Optional[int] - error: Optional[str] - - -class CoralogixLabelsConfig(BaseModel): - pod: str = "resource.attributes.k8s.pod.name" - namespace: str = "resource.attributes.k8s.namespace.name" - log_message: str = "logRecord.body" - timestamp: str = "logRecord.attributes.time" - - class CoralogixConfig(BaseModel): team_hostname: str domain: str api_key: str - labels: CoralogixLabelsConfig = CoralogixLabelsConfig() def parse_json_lines(raw_text) -> List[Dict[str, Any]]: @@ -73,72 +54,3 @@ def normalize_datetime(date_str: Optional[str]) -> str: return date_str -def extract_field(data_obj: dict[str, Any], field: str): - """returns a nested field from a dict - e.g. extract_field({"parent": {"child": "value"}}, "parent.child") => value - """ - current_object: Any = data_obj - fields = field.split(".") - - for field in fields: - if not current_object: - return None - if isinstance(current_object, dict): - current_object = current_object.get(field) - else: - return None - - return current_object - - -def flatten_structured_log_entries( - log_entries: List[Dict[str, Any]], - labels_config: CoralogixLabelsConfig, -) -> List[FlattenedLog]: - flattened_logs = [] - for log_entry in log_entries: - try: - userData = json.loads(log_entry.get("userData", "{}")) - log_message = extract_field(userData, labels_config.log_message) - timestamp = extract_field(userData, labels_config.timestamp) - if not log_message or not timestamp: - log_message = json.dumps(userData) - else: - flattened_logs.append( - FlattenedLog(timestamp=timestamp, log_message=log_message) - ) # Store as tuple for sorting - - except json.JSONDecodeError: - logging.error(f"Failed to decode userData JSON: {json.dumps(log_entry)}") - return flattened_logs - - -def stringify_flattened_logs(log_entries: List[FlattenedLog]) -> str: - formatted_logs = [] - for entry in log_entries: - formatted_logs.append(entry.log_message) - - return "\n".join(formatted_logs) if formatted_logs else "No logs found." - - -def parse_json_objects( - json_objects: List[Dict[str, Any]], labels_config: CoralogixLabelsConfig -) -> List[FlattenedLog]: - """Extracts timestamp and log values from parsed JSON objects, sorted in ascending order (oldest first).""" - logs: List[FlattenedLog] = [] - - for data in json_objects: - if isinstance(data, dict) and "result" in data and "results" in data["result"]: - logs += flatten_structured_log_entries( - log_entries=data["result"]["results"], labels_config=labels_config - ) - elif isinstance(data, dict) and data.get("warning"): - logging.info( - f"Received the following warning when fetching coralogix logs: {data}" - ) - else: - logging.debug(f"Unrecognised partial response from coralogix logs: {data}") - - logs.sort(key=lambda x: x[0]) - - return logs diff --git a/tests/llm/fixtures/shared/coralogix/coralogix_app.py b/tests/llm/fixtures/shared/coralogix/coralogix_app.py deleted file mode 100644 index a67de9cb0d..0000000000 --- a/tests/llm/fixtures/shared/coralogix/coralogix_app.py +++ /dev/null @@ -1,182 +0,0 @@ -import logging -import os -import random -import time - -from flask import Flask, jsonify, request - -# OpenTelemetry imports -from opentelemetry._logs import set_logger_provider -from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter -from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.metrics import set_meter_provider -from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import ( - Status, - StatusCode, - get_tracer, - set_tracer_provider, -) - -LOGS_ENABLED = os.environ.get("CORALOGIX_LOGS_ENABLED", "false").lower() == "true" -TRACES_ENABLED = os.environ.get("CORALOGIX_TRACES_ENABLED", "false").lower() == "true" -METRICS_ENABLED = os.environ.get("CORALOGIX_METRICS_ENABLED", "false").lower() == "true" -SERVICE_NAME = os.environ.get("SERVICE_NAME", "payment-service") -SERVICE_LABEL = os.environ.get("SERVICE_LABEL", "payment") - -# Configure OpenTelemetry resource -resource = Resource.create( - { - "service.name": SERVICE_NAME, - "k8s.namespace.name": os.environ.get("K8S_NAMESPACE", "app-173"), - "k8s.pod.name": os.environ.get("K8S_POD_NAME", "unknown"), - } -) - -# Send telemetry to local OTLP collector; it will forward to Coralogix. -K8S_NODE_IP = os.environ.get("K8S_NODE_IP", "192.168.13.204") -otlp_endpoint = f"http://{K8S_NODE_IP}:4318/v1/" - -if TRACES_ENABLED: - trace_exporter = OTLPSpanExporter(endpoint=f"{otlp_endpoint}traces") - tracer_provider = TracerProvider(resource=resource) - tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) - set_tracer_provider(tracer_provider) - -if LOGS_ENABLED: - log_exporter = OTLPLogExporter(endpoint=f"{otlp_endpoint}logs") - logger_provider = LoggerProvider(resource=resource) - logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter)) - set_logger_provider(logger_provider) - -if METRICS_ENABLED: - metric_exporter = OTLPMetricExporter(endpoint=f"{otlp_endpoint}metrics") - reader = PeriodicExportingMetricReader(metric_exporter) - meter_provider = MeterProvider(resource=resource, metric_readers=[reader]) - set_meter_provider(meter_provider) -else: - meter_provider = None - -app = Flask(__name__) -tracer = get_tracer(__name__) - -if METRICS_ENABLED: - meter = meter_provider.get_meter(SERVICE_NAME) - payment_success_counter = meter.create_counter("payment_success_total") - payment_failure_counter = meter.create_counter("payment_failure_total") - payment_duration_hist = meter.create_histogram("payment_duration_seconds", unit="s") -else: - payment_success_counter = None - payment_failure_counter = None - payment_duration_hist = None - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(levelname)s: %(message)s", -) -logger = logging.getLogger(__name__) - -if LOGS_ENABLED: - log_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) - logger.addHandler(log_handler) - - -@app.route("/health") -def health(): - return "OK" - - -@app.route("/payment", methods=["POST"]) -def payment(): - start_time = time.perf_counter() - with tracer.start_as_current_span("process_payment") as span: - data = request.json or {} - user_id = data.get("user_id", "guest") - amount = data.get("amount", 0) - - span.set_attribute("payment.user_id", user_id) - span.set_attribute("payment.amount", float(amount)) - - logger.info( - f"Processing {SERVICE_LABEL} request for user {user_id}, amount: ${amount}" - ) - - # Simulate database query that sometimes times out - with tracer.start_as_current_span("database_query") as db_span: - db_span.set_attribute("db.system", "postgresql") - db_span.set_attribute("db.operation", "SELECT") - - query_time = random.uniform(0.1, 0.3) - if random.random() < 0.3: # 30% chance of slow query - query_time = random.uniform(2.5, 5.0) - logger.warning(f"Slow database query detected: {query_time:.2f}s") - db_span.set_attribute("db.query.duration", f"{query_time:.2f}s") - db_span.set_attribute("db.slow_query", True) - - time.sleep(query_time) - - # Simulate connection pool exhaustion errors - if random.random() < 0.2: # 20% chance of error - error_msg = ( - "Database connection timeout after 30s - MaxConnectionsReached" - ) - logger.error(error_msg) - db_span.set_status(Status(StatusCode.ERROR, error_msg)) - db_span.set_attribute("error", True) - db_span.set_attribute("error.type", "connection_timeout") - span.set_status(Status(StatusCode.ERROR, error_msg)) - span.set_attribute("error", True) - - if METRICS_ENABLED and payment_failure_counter: - duration = time.perf_counter() - start_time - payment_failure_counter.add( - 1, attributes={"payment.user_id": user_id} - ) - payment_duration_hist.record( - duration, - attributes={ - "payment.user_id": user_id, - "payment.status": "failed", - }, - ) - - return jsonify({"error": "Payment processing failed"}), 500 - - response = { - "payment_id": f"pay-{random.randint(1000, 9999)}", - "user_id": user_id, - "amount": amount, - "status": "completed", - } - - span.set_attribute("payment.payment_id", response["payment_id"]) - span.set_attribute("payment.status", "completed") - logger.info( - f"{SERVICE_LABEL.title()} completed successfully: {response['payment_id']}" - ) - - if METRICS_ENABLED and payment_success_counter: - duration = time.perf_counter() - start_time - payment_success_counter.add(1, attributes={"payment.user_id": user_id}) - payment_duration_hist.record( - duration, - attributes={ - "payment.user_id": user_id, - "payment.status": "success", - }, - ) - - return jsonify(response) - - -if __name__ == "__main__": - logger.info("Starting payment service on port 8080") - app.run(host="0.0.0.0", port=8080) diff --git a/tests/llm/fixtures/shared/coralogix/coralogix_test_utils.sh b/tests/llm/fixtures/shared/coralogix/coralogix_test_utils.sh new file mode 100644 index 0000000000..f2f74347ae --- /dev/null +++ b/tests/llm/fixtures/shared/coralogix/coralogix_test_utils.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Shared utilities for Coralogix eval tests +# Source this file at the start of before_test scripts: +# source ../../shared/coralogix_test_utils.sh + +# Validate Coralogix environment variables +cx_validate_env() { + local missing=() + + if [ -z "$CORALOGIX_API_KEY" ]; then + missing+=("CORALOGIX_API_KEY") + fi + + if [ -z "$CORALOGIX_DOMAIN" ]; then + missing+=("CORALOGIX_DOMAIN") + fi + + if [ -z "$CORALOGIX_TEAM_HOSTNAME" ]; then + missing+=("CORALOGIX_TEAM_HOSTNAME") + fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing required environment variables: ${missing[*]}" + exit 1 + fi + + echo "✅ Coralogix environment validated" +} + +# Get the ingestion endpoint for sending logs +# Usage: INGRESS_URL=$(cx_ingress_url) +cx_ingress_url() { + echo "https://ingress.${CORALOGIX_DOMAIN}" +} + +# Get the DataPrime query endpoint +# Usage: QUERY_URL=$(cx_query_url) +cx_query_url() { + echo "https://ng-api-http.${CORALOGIX_DOMAIN}/api/v1/dataprime/query" +} + +# Send logs to Coralogix via REST API +# Usage: cx_send_logs "app-name" "subsystem-name" '[{"timestamp":..., "severity":1, "text":"..."}]' +cx_send_logs() { + local app_name="$1" + local subsystem_name="$2" + local log_entries="$3" + + local ingress_url=$(cx_ingress_url) + local payload=$(cat <&1) + local exit_code=$? + + if [ $exit_code -ne 0 ]; then + echo "❌ Failed to send logs (curl exit code: $exit_code)" + echo "Response: $response" + return 1 + fi + + echo "✅ Logs sent successfully to $app_name/$subsystem_name" + return 0 +} + +# Query Coralogix using DataPrime and return results +# Usage: RESULT=$(cx_query "source logs | lucene 'error' | limit 10") +cx_query() { + local query="$1" + local start_date="${2:-$(date -u -d '1 hour ago' '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u -v-1H '+%Y-%m-%dT%H:%M:%SZ')}" + local end_date="${3:-$(date -u '+%Y-%m-%dT%H:%M:%SZ')}" + + local query_url=$(cx_query_url) + + curl -sf -X POST "$query_url" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"$query\", \"metadata\": {\"syntax\": \"QUERY_SYNTAX_DATAPRIME\", \"startDate\": \"$start_date\", \"endDate\": \"$end_date\"}}" +} + +# Wait for logs to be queryable in Coralogix +# Usage: cx_wait_for_logs "search-term" [max_attempts] [sleep_interval] +cx_wait_for_logs() { + local search_term="$1" + local max_attempts="${2:-60}" + local sleep_interval="${3:-5}" + + echo "⏳ Waiting for logs containing '$search_term' to be queryable..." + + for i in $(seq 1 $max_attempts); do + local result=$(cx_query "source logs | lucene '$search_term' | limit 1") + + if echo "$result" | grep -q "$search_term"; then + echo "✅ Logs are queryable after $((i * sleep_interval)) seconds" + return 0 + fi + + echo " Attempt $i/$max_attempts: Logs not yet available..." + sleep $sleep_interval + done + + echo "❌ Timeout waiting for logs after $((max_attempts * sleep_interval)) seconds" + return 1 +} + +# Generate a unique verification code for anti-hallucination testing +# Usage: VERIFY_CODE=$(cx_generate_verify_code) +cx_generate_verify_code() { + local code=$(cat /dev/urandom | tr -dc 'A-Z0-9' | fold -w 8 | head -n 1) + echo "HOLMES-CX-${code}" +} + +# Get current timestamp in Coralogix format (milliseconds since epoch) +# Usage: TIMESTAMP=$(cx_timestamp) +cx_timestamp() { + # Returns milliseconds since epoch + echo $(($(date +%s) * 1000)) +} + +# Get timestamp for N minutes ago in Coralogix format +# Usage: TIMESTAMP=$(cx_timestamp_minutes_ago 5) +cx_timestamp_minutes_ago() { + local minutes="$1" + local seconds=$((minutes * 60)) + echo $((($(date +%s) - seconds) * 1000)) +} + +# Combined setup: validate env +cx_setup() { + cx_validate_env +} diff --git a/tests/llm/fixtures/shared/coralogix/send_metrics.py b/tests/llm/fixtures/shared/coralogix/send_metrics.py new file mode 100644 index 0000000000..2e44de4103 --- /dev/null +++ b/tests/llm/fixtures/shared/coralogix/send_metrics.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +""" +Send test metrics to Coralogix via OTLP gRPC. + +Usage: + python send_metrics.py --domain eu2.coralogix.com --api-key \ + --app-name holmes-eval-175 --subsystem metrics-test \ + --metric-prefix eval175 + +Environment variables (alternative to CLI args): + CORALOGIX_DOMAIN, CORALOGIX_API_KEY +""" + +import argparse +import os +import random +import sys +import time + + +def send_metrics(domain: str, api_key: str, app_name: str, subsystem: str, + metric_prefix: str) -> bool: + """Send test metrics to Coralogix via OTLP gRPC.""" + try: + from opentelemetry import metrics + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + except ImportError: + print("ERROR: OpenTelemetry packages not installed. Run:") + print(" pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc") + return False + + # Configure resource with Coralogix-specific attributes + resource = Resource.create({ + "service.name": app_name, + "cx.application.name": app_name, + "cx.subsystem.name": subsystem, + }) + + # Configure OTLP exporter for Coralogix + endpoint = f"ingress.{domain}:443" + exporter = OTLPMetricExporter( + endpoint=endpoint, + headers={ + "Authorization": f"Bearer {api_key}", + "CX-Application-Name": app_name, + "CX-Subsystem-Name": subsystem, + }, + ) + + # Use a short export interval for faster test setup + reader = PeriodicExportingMetricReader(exporter, export_interval_millis=1000) + provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(provider) + + meter = metrics.get_meter(__name__) + + print(f"Sending metrics to {endpoint}...") + + # Create metrics with the specified prefix + request_counter = meter.create_counter( + f"{metric_prefix}_http_requests_total", + description="Total HTTP requests", + unit="1", + ) + + error_counter = meter.create_counter( + f"{metric_prefix}_http_errors_total", + description="Total HTTP errors", + unit="1", + ) + + latency_histogram = meter.create_histogram( + f"{metric_prefix}_request_latency_seconds", + description="Request latency in seconds", + unit="s", + ) + + # Generate some metric data points + endpoints = ["/api/checkout", "/api/cart", "/api/products", "/api/users"] + status_codes = ["200", "201", "400", "500", "503"] + + print("Generating metric data points...") + for _ in range(50): + endpoint = random.choice(endpoints) + status = random.choice(status_codes) + latency = random.uniform(0.01, 2.0) + + attributes = {"endpoint": endpoint, "status_code": status, "app": app_name} + + request_counter.add(1, attributes) + latency_histogram.record(latency, attributes) + + if status in ["500", "503"]: + error_counter.add(1, attributes) + + time.sleep(0.02) + + # Force flush to ensure metrics are sent + print("Flushing metrics...") + provider.force_flush() + + # Give some time for final export + time.sleep(3) + provider.shutdown() + + print(f"✅ Metrics sent successfully with prefix={metric_prefix}") + return True + + +def main(): + parser = argparse.ArgumentParser(description="Send test metrics to Coralogix") + parser.add_argument("--domain", default=os.environ.get("CORALOGIX_DOMAIN"), + help="Coralogix domain (e.g., eu2.coralogix.com)") + parser.add_argument("--api-key", default=os.environ.get("CORALOGIX_API_KEY"), + help="Coralogix API key") + parser.add_argument("--app-name", required=True, help="Application name") + parser.add_argument("--subsystem", required=True, help="Subsystem name") + parser.add_argument("--metric-prefix", required=True, help="Prefix for metric names") + + args = parser.parse_args() + + if not args.domain: + print("ERROR: --domain or CORALOGIX_DOMAIN required") + sys.exit(1) + if not args.api_key: + print("ERROR: --api-key or CORALOGIX_API_KEY required") + sys.exit(1) + + success = send_metrics( + domain=args.domain, + api_key=args.api_key, + app_name=args.app_name, + subsystem=args.subsystem, + metric_prefix=args.metric_prefix, + ) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/llm/fixtures/shared/coralogix/send_traces.py b/tests/llm/fixtures/shared/coralogix/send_traces.py new file mode 100644 index 0000000000..174395f90d --- /dev/null +++ b/tests/llm/fixtures/shared/coralogix/send_traces.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Send test traces to Coralogix via OTLP gRPC. + +Usage: + python send_traces.py --domain eu2.coralogix.com --api-key \ + --app-name holmes-eval-174 --subsystem traces-test \ + --trace-id TRACE-ABC123 --error-code ERR-5847 + +Environment variables (alternative to CLI args): + CORALOGIX_DOMAIN, CORALOGIX_API_KEY +""" + +import argparse +import os +import sys +import time + + +def send_traces(domain: str, api_key: str, app_name: str, subsystem: str, + trace_id: str, error_code: str) -> bool: + """Send test traces to Coralogix via OTLP gRPC.""" + try: + from opentelemetry import trace + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.trace import Status, StatusCode + except ImportError: + print("ERROR: OpenTelemetry packages not installed. Run:") + print(" pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc") + return False + + # Configure resource with Coralogix-specific attributes + resource = Resource.create({ + "service.name": app_name, + "cx.application.name": app_name, + "cx.subsystem.name": subsystem, + }) + + # Configure OTLP exporter for Coralogix + endpoint = f"ingress.{domain}:443" + exporter = OTLPSpanExporter( + endpoint=endpoint, + headers={ + "Authorization": f"Bearer {api_key}", + "CX-Application-Name": app_name, + "CX-Subsystem-Name": subsystem, + }, + ) + + # Set up tracer provider + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + + tracer = trace.get_tracer(__name__) + + print(f"Sending traces to {endpoint}...") + + # Create a realistic trace with multiple spans + with tracer.start_as_current_span("http_request") as root_span: + root_span.set_attribute("http.method", "POST") + root_span.set_attribute("http.url", "/api/checkout") + root_span.set_attribute("trace.id.custom", trace_id) + + # Simulate database query + with tracer.start_as_current_span("db_query") as db_span: + db_span.set_attribute("db.system", "postgresql") + db_span.set_attribute("db.operation", "SELECT") + db_span.set_attribute("db.statement", "SELECT * FROM orders WHERE id = ?") + time.sleep(0.05) + + # Simulate external API call that fails + with tracer.start_as_current_span("external_api_call") as api_span: + api_span.set_attribute("http.url", "https://payment-gateway.example.com/charge") + api_span.set_attribute("error.code", error_code) + api_span.set_attribute("error.message", f"Payment gateway timeout - {error_code}") + api_span.set_status(Status(StatusCode.ERROR, f"Payment failed: {error_code}")) + time.sleep(0.02) + + # Mark root span as error due to child failure + root_span.set_status(Status(StatusCode.ERROR, "Request failed")) + root_span.set_attribute("error", True) + + # Force flush to ensure spans are sent + provider.force_flush() + provider.shutdown() + + print(f"✅ Traces sent successfully with trace_id={trace_id}, error_code={error_code}") + return True + + +def main(): + parser = argparse.ArgumentParser(description="Send test traces to Coralogix") + parser.add_argument("--domain", default=os.environ.get("CORALOGIX_DOMAIN"), + help="Coralogix domain (e.g., eu2.coralogix.com)") + parser.add_argument("--api-key", default=os.environ.get("CORALOGIX_API_KEY"), + help="Coralogix API key") + parser.add_argument("--app-name", required=True, help="Application name") + parser.add_argument("--subsystem", required=True, help="Subsystem name") + parser.add_argument("--trace-id", required=True, help="Custom trace ID for verification") + parser.add_argument("--error-code", required=True, help="Error code to inject") + + args = parser.parse_args() + + if not args.domain: + print("ERROR: --domain or CORALOGIX_DOMAIN required") + sys.exit(1) + if not args.api_key: + print("ERROR: --api-key or CORALOGIX_API_KEY required") + sys.exit(1) + + success = send_traces( + domain=args.domain, + api_key=args.api_key, + app_name=args.app_name, + subsystem=args.subsystem, + trace_id=args.trace_id, + error_code=args.error_code, + ) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/llm/fixtures/shared/coralogix/traffic_generator.py b/tests/llm/fixtures/shared/coralogix/traffic_generator.py deleted file mode 100644 index 241e467918..0000000000 --- a/tests/llm/fixtures/shared/coralogix/traffic_generator.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import random -import time - -import requests - -TARGET_URL = os.environ.get("TARGET_URL", "http://payment:8080/payment") -SERVICE_LABEL = os.environ.get("SERVICE_LABEL", "payment") -DURATION_SECONDS = float(os.environ.get("DURATION_SECONDS", "120")) -DELAY_MIN_SECONDS = float(os.environ.get("DELAY_MIN_SECONDS", "0.5")) -DELAY_MAX_SECONDS = float(os.environ.get("DELAY_MAX_SECONDS", "2.0")) - - -def send_request(): - user_id = f"user-{random.randint(1000, 9999)}" - amount = round(random.uniform(10.0, 500.0), 2) - payload = {"user_id": user_id, "amount": amount} - try: - resp = requests.post(TARGET_URL, json=payload, timeout=10) - if resp.status_code == 200: - print(f"✓ {SERVICE_LABEL} request ok: {payload}") - else: - print(f"✗ {SERVICE_LABEL} request failed: {resp.status_code} - {payload}") - except Exception as exc: # noqa: BLE001 - print(f"✗ Request error: {exc}") - - -def main(): - print(f"Starting traffic generator for {SERVICE_LABEL}...") - print(f"Target: {TARGET_URL}") - end_time = time.time() + DURATION_SECONDS - count = 0 - while time.time() < end_time: - send_request() - count += 1 - time.sleep(random.uniform(DELAY_MIN_SECONDS, DELAY_MAX_SECONDS)) - print(f"\nTraffic generation complete. Sent {count} requests.") - - -if __name__ == "__main__": - main() diff --git a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/payment-service.yaml b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/payment-service.yaml deleted file mode 100644 index d34334ce1a..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/payment-service.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: payment -spec: - replicas: 1 - selector: - matchLabels: - app: payment - template: - metadata: - labels: - app: payment - spec: - containers: - - name: payment - image: python:3.9-slim - command: - - sh - - -c - - | - pip install flask opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-instrumentation --quiet - python /app/app.py - volumeMounts: - - name: app - mountPath: /app - ports: - - containerPort: 8080 - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: SERVICE_NAME - value: "payment-service" - - name: SERVICE_LABEL - value: "payment" - - name: CORALOGIX_LOGS_ENABLED - value: "true" - - name: CORALOGIX_TRACES_ENABLED - value: "true" - - name: CORALOGIX_METRICS_ENABLED - value: "false" - - name: K8S_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: K8S_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: CORALOGIX_DOMAIN - value: "${CORALOGIX_DOMAIN}" - - name: CORALOGIX_API_KEY - value: "${CORALOGIX_API_KEY}" - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "500m" - volumes: - - name: app - secret: - secretName: payment-app ---- -apiVersion: v1 -kind: Service -metadata: - name: payment -spec: - selector: - app: payment - ports: - - port: 8080 - targetPort: 8080 diff --git a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/test_case.yaml index d89fc097c6..8e121fe3b5 100644 --- a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/test_case.yaml @@ -1,134 +1,119 @@ -user_prompt: - - "The payment service in namespace app-173 is experiencing issues." +# Test: Coralogix Log Search with DataPrime +# Tests that Holmes can search logs using Coralogix DataPrime query language +# +# This is a fast cloud-only test that writes logs directly to Coralogix via REST API, +# no Kubernetes cluster required. +# +# Requirements: +# - CORALOGIX_API_KEY: API key with DataQuerying and Send Data permissions +# - CORALOGIX_DOMAIN: e.g., "eu2.coralogix.com" +# - CORALOGIX_TEAM_HOSTNAME: Your team name +# +# Anti-hallucination: The test injects 3 specific error codes (ERR-XXXX format). +# The LLM cannot know these codes without actually querying Coralogix. + +user_prompt: "Search the Coralogix logs for the 'holmes-eval-173' application. Find ERROR level messages and report what error codes (format ERR-XXXX) appear in the logs." expected_output: - - The answer must identify that database connection timeouts are occurring - - The answer must mention Database connection pool exhausted, or MaxConnectionsReached + - "Must identify that there are ERROR level logs" + - "Must find and list the 3 specific error codes: ERR-7291, ERR-4058, and ERR-9463" + - "The response should mention the services or error messages associated with these codes" tags: - - kubernetes - - chain-of-causation - - logs - coralogix + - logs + - medium -setup_timeout: 900 # 15 minutes to handle pod scheduling delays +setup_timeout: 300 before_test: | - # Validate required Coralogix environment variables - [ -n "${CORALOGIX_API_KEY:-}" ] && [ -n "${CORALOGIX_DOMAIN:-}" ] && [ -n "${CORALOGIX_TEAM_HOSTNAME:-}" ] || { for v in CORALOGIX_API_KEY CORALOGIX_DOMAIN CORALOGIX_TEAM_HOSTNAME; do [ -n "${!v:-}" ] || echo "Missing env var: $v"; done; exit 1; } + source ../../shared/coralogix/coralogix_test_utils.sh + cx_setup + set -e - echo "🚀 Setting up test 163 - Creating namespace app-173" - kubectl create namespace app-173 || true - echo "✅ Namespace app-173 created successfully!" + echo "🚀 Setting up Coralogix logs test 173" - echo "🔐 Creating payment-app secret from shared coralogix_app.py" - kubectl create secret generic payment-app \ - --from-file=app.py=../../shared/coralogix/coralogix_app.py \ - -n app-173 --dry-run=client -o yaml | kubectl apply -f - + APP_NAME="holmes-eval-173" + SUBSYSTEM="payment-api" - echo "🔐 Creating traffic-generator secret from shared traffic_generator.py" - kubectl create secret generic coralogix-generator-app \ - --from-file=traffic_generator.py=../../shared/coralogix/traffic_generator.py \ - -n app-173 --dry-run=client -o yaml | kubectl apply -f - + # Define the specific error codes for anti-hallucination testing + ERROR_CODE_1="ERR-7291" + ERROR_CODE_2="ERR-4058" + ERROR_CODE_3="ERR-9463" - echo "💳 Deploying payment service" - kubectl apply -f payment-service.yaml -n app-173 + # Get timestamps (Coralogix expects milliseconds since epoch) + NOW=$(date +%s) + TS1=$(( (NOW - 300) * 1000 )) + TS2=$(( (NOW - 240) * 1000 )) + TS3=$(( (NOW - 180) * 1000 )) + TS4=$(( (NOW - 120) * 1000 )) + TS5=$(( (NOW - 60) * 1000 )) + TS6=$(( NOW * 1000 )) - echo "⏳ Waiting for payment pod to exist" - for i in {1..60}; do - if kubectl get pod -l app=payment -n app-173 2>/dev/null | grep -q payment; then - echo "✅ Payment pod exists" - break - fi - echo "Waiting for payment pod to be created... ($i/60)" - sleep 2 - done + echo "⏳ Sending logs to Coralogix..." - echo "⏳ Waiting for payment pod to be ready (timeout 780s)" - PAYMENT_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=payment -n app-173 --timeout=5s 2>/dev/null; then - echo "✅ Payment pod is ready!" - PAYMENT_POD_READY=true - break - else - echo "⏳ Attempt $i/156: Payment pod not ready yet, waiting 5s..." - sleep 5 - fi - done + # Build log entries JSON (avoiding heredoc for YAML compatibility) + LOG_ENTRIES="[{\"timestamp\": $TS1, \"severity\": 3, \"text\": \"Service started successfully - payment-api initialized\"}, {\"timestamp\": $TS2, \"severity\": 5, \"text\": \"Database connection timeout - $ERROR_CODE_1: Failed to connect to primary database after 30s\"}, {\"timestamp\": $TS3, \"severity\": 4, \"text\": \"Retrying database connection attempt 1\"}, {\"timestamp\": $TS4, \"severity\": 5, \"text\": \"Payment validation failed - $ERROR_CODE_2: Invalid card number format for user U-12345\"}, {\"timestamp\": $TS5, \"severity\": 3, \"text\": \"Processing order batch complete - 150 orders processed\"}, {\"timestamp\": $TS6, \"severity\": 5, \"text\": \"Stock sync failed - $ERROR_CODE_3: External inventory API returned 503\"}]" + + # Send logs via REST API + INGRESS_URL="https://ingress.${CORALOGIX_DOMAIN}" - if [ "$PAYMENT_POD_READY" = false ]; then - echo "❌ Payment pod failed to become ready after 780 seconds" - kubectl get pods -n app-173 -l app=payment + RESPONSE=$(curl -sf -X POST "${INGRESS_URL}/logs/v1/singles" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"applicationName\": \"$APP_NAME\", \"subsystemName\": \"$SUBSYSTEM\", \"logEntries\": $LOG_ENTRIES}" 2>&1) + + if [ $? -ne 0 ]; then + echo "❌ Failed to send logs: $RESPONSE" exit 1 fi - echo "🔍 Checking payment deployment status" - kubectl get pods -n app-173 -l app=payment + echo "✅ Logs sent to $APP_NAME/$SUBSYSTEM" - echo "🚦 Deploying coralogix traffic generator" - kubectl apply -f traffic-generator.yaml -n app-173 + # Wait for logs to be queryable (Coralogix has some ingestion delay) + echo "⏳ Waiting for logs to be queryable in Coralogix..." + QUERY_URL="https://ng-api-http.${CORALOGIX_DOMAIN}/api/v1/dataprime/query" - echo "⏳ Waiting for traffic-generator pod to exist" + LOGS_READY=false for i in {1..60}; do - if kubectl get pod -l app=traffic-generator -n app-173 2>/dev/null | grep -q traffic-generator; then - echo "✅ Traffic-generator pod exists" + RESULT=$(curl -sf -X POST "$QUERY_URL" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"source logs | lucene 'holmes-eval-173 AND $ERROR_CODE_1' | limit 1\", \"metadata\": {\"syntax\": \"QUERY_SYNTAX_DATAPRIME\"}}" 2>/dev/null) + + if echo "$RESULT" | grep -q "$ERROR_CODE_1"; then + echo "✅ Logs are queryable after $((i * 5)) seconds" + LOGS_READY=true break fi - echo "Waiting for traffic-generator pod to be created... ($i/60)" - sleep 2 - done - echo "⏳ Waiting for traffic-generator pod to be ready (timeout 780s)" - TRAFFIC_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=traffic-generator -n app-173 --timeout=5s 2>/dev/null; then - echo "✅ Traffic-generator pod is ready!" - TRAFFIC_POD_READY=true - break - else - echo "⏳ Attempt $i/156: Traffic-generator pod not ready yet, waiting 5s..." - sleep 5 - fi + echo " Attempt $i/60: Logs not yet indexed, waiting 5s..." + sleep 5 done - if [ "$TRAFFIC_POD_READY" = false ]; then - echo "❌ Traffic-generator pod failed to become ready after 780 seconds" - kubectl get pods -n app-173 -l app=traffic-generator + if [ "$LOGS_READY" = false ]; then + echo "❌ Timeout waiting for logs to be queryable" exit 1 fi - echo "🔍 Checking all pods status" - kubectl get pods -n app-173 - - echo "⏰ Waiting for traffic generator to produce requests and errors..." - TRAFFIC_READY=false - for i in {1..90}; do - PAYMENT_LOGS=$(kubectl logs -n app-173 -l app=payment --tail=100 2>/dev/null | grep -c "Processing payment request" || echo "0") - ERROR_LOGS=$(kubectl logs -n app-173 -l app=payment --tail=100 2>/dev/null | grep -c "ERROR\|WARN" || echo "0") - - if [ "$PAYMENT_LOGS" -gt "10" ] && [ "$ERROR_LOGS" -gt "0" ]; then - echo "✅ Found required logs after $i seconds:" - echo " - Payment processing logs: $PAYMENT_LOGS" - echo " - Error/Warning logs: $ERROR_LOGS" - TRAFFIC_READY=true - sleep 10 - break + # Verify all 3 error codes are queryable + echo "🔍 Verifying all error codes are queryable..." + for CODE in $ERROR_CODE_1 $ERROR_CODE_2 $ERROR_CODE_3; do + VERIFY=$(curl -sf -X POST "$QUERY_URL" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"source logs | lucene 'holmes-eval-173 AND $CODE' | limit 1\", \"metadata\": {\"syntax\": \"QUERY_SYNTAX_DATAPRIME\"}}" 2>/dev/null) + + if ! echo "$VERIFY" | grep -q "$CODE"; then + echo "❌ Could not find error code $CODE" + exit 1 fi - - echo "⏳ Attempt $i/90: PAYMENT=$PAYMENT_LOGS, ERRORS=$ERROR_LOGS" - sleep 1 + echo " ✓ Found $CODE" done - if [ "$TRAFFIC_READY" = false ]; then - echo "❌ Failed to generate required traffic patterns" - exit 1 - fi - - # Delete traffic generator so the ai won't cheat - kubectl delete -f traffic-generator.yaml -n app-173 - echo "✅ Test setup complete!" after_test: | - kubectl delete namespace app-173 || true + # No cleanup needed - logs will naturally age out of Coralogix + # The application name is unique to this test so it won't interfere with others + echo "✅ Cleanup complete (logs will age out naturally)" diff --git a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/toolsets.yaml index 3034f44d45..aaf96bb8cf 100644 --- a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/toolsets.yaml @@ -1,5 +1,5 @@ # Coralogix toolset configuration for test 173 -# Enables Coralogix logs toolset to query logs and traces +# Cloud-only test - no Kubernetes required toolsets: coralogix: @@ -8,7 +8,3 @@ toolsets: api_key: "{{env.CORALOGIX_API_KEY}}" domain: "{{env.CORALOGIX_DOMAIN}}" team_hostname: "{{env.CORALOGIX_TEAM_HOSTNAME}}" - kubernetes/logs: - enabled: false # Disable default Kubernetes logging to use Coralogix - kubernetes/core: - enabled: true diff --git a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/traffic-generator.yaml b/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/traffic-generator.yaml deleted file mode 100644 index ade7cddcc6..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/173_coralogix_logs/traffic-generator.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Secret is created in test_case.yaml setup from shared traffic_generator.py -apiVersion: apps/v1 -kind: Deployment -metadata: - name: traffic-generator -spec: - replicas: 1 - selector: - matchLabels: - app: traffic-generator - template: - metadata: - labels: - app: traffic-generator - spec: - containers: - - name: traffic-generator - image: python:3.9-slim - command: - - sh - - -c - - | - pip install requests -q - python /app/traffic_generator.py - volumeMounts: - - name: app - mountPath: /app - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: TARGET_URL - value: "http://payment:8080/payment" - - name: SERVICE_LABEL - value: "payment" - volumes: - - name: app - secret: - secretName: coralogix-generator-app diff --git a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/test_case.yaml new file mode 100644 index 0000000000..2ea7b1316d --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/test_case.yaml @@ -0,0 +1,88 @@ +# Test: Coralogix Trace Search with DataPrime +# Tests that Holmes can search traces using Coralogix DataPrime query language +# +# This is a fast cloud-only test that sends traces via OTLP gRPC to Coralogix, +# no Kubernetes cluster required. +# +# Requirements: +# - CORALOGIX_API_KEY: API key with DataQuerying and Send Data permissions +# - CORALOGIX_DOMAIN: e.g., "eu2.coralogix.com" +# - CORALOGIX_TEAM_HOSTNAME: Your team name +# - OpenTelemetry Python packages (installed via poetry) +# +# Anti-hallucination: The test injects a specific error code (ERR-XXXX format) +# and trace ID that the LLM can only find by actually querying Coralogix. + +user_prompt: "Search the Coralogix traces for the 'holmes-eval-174' application. Look for failed spans and report what error codes you find. Also identify which operation failed." + +expected_output: + - "Must identify that there are error/failed spans" + - "Must find the error code ERR-5847" + - "Must mention that the external_api_call or payment gateway operation failed" + +tags: + - coralogix + - traces + - medium + +setup_timeout: 300 + +before_test: | + source ../../shared/coralogix/coralogix_test_utils.sh + cx_setup + set -e + + echo "🚀 Setting up Coralogix traces test 174" + + APP_NAME="holmes-eval-174" + SUBSYSTEM="checkout-service" + TRACE_ID="TRACE-174-$(date +%s)" + ERROR_CODE="ERR-5847" + + echo "⏳ Sending traces to Coralogix via OTLP..." + + # Send traces using Python OTLP exporter + python3 ../../shared/coralogix/send_traces.py \ + --app-name "$APP_NAME" \ + --subsystem "$SUBSYSTEM" \ + --trace-id "$TRACE_ID" \ + --error-code "$ERROR_CODE" + + if [ $? -ne 0 ]; then + echo "❌ Failed to send traces" + exit 1 + fi + + echo "✅ Traces sent to $APP_NAME/$SUBSYSTEM" + + # Wait for traces to be queryable + echo "⏳ Waiting for traces to be queryable in Coralogix..." + QUERY_URL="https://ng-api-http.${CORALOGIX_DOMAIN}/api/v1/dataprime/query" + + TRACES_READY=false + for i in {1..60}; do + RESULT=$(curl -sf -X POST "$QUERY_URL" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"source spans | lucene 'holmes-eval-174 AND $ERROR_CODE' | limit 1\", \"metadata\": {\"syntax\": \"QUERY_SYNTAX_DATAPRIME\"}}" 2>/dev/null) + + if echo "$RESULT" | grep -q "$ERROR_CODE"; then + echo "✅ Traces are queryable after $((i * 5)) seconds" + TRACES_READY=true + break + fi + + echo " Attempt $i/60: Traces not yet indexed, waiting 5s..." + sleep 5 + done + + if [ "$TRACES_READY" = false ]; then + echo "❌ Timeout waiting for traces to be queryable" + exit 1 + fi + + echo "✅ Test setup complete!" + +after_test: | + # No cleanup needed - traces will naturally age out of Coralogix + echo "✅ Cleanup complete (traces will age out naturally)" diff --git a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/toolsets.yaml similarity index 60% rename from tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/toolsets.yaml index 54c1357a41..b397bfe60d 100644 --- a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces/toolsets.yaml @@ -1,4 +1,6 @@ -# Coralogix traces/logs for ad service scenario +# Coralogix toolset configuration for test 174 +# Cloud-only test - no Kubernetes required + toolsets: coralogix: enabled: true @@ -6,7 +8,3 @@ toolsets: api_key: "{{env.CORALOGIX_API_KEY}}" domain: "{{env.CORALOGIX_DOMAIN}}" team_hostname: "{{env.CORALOGIX_TEAM_HOSTNAME}}" - kubernetes/logs: - enabled: false - kubernetes/core: - enabled: true diff --git a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-service.yaml b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-service.yaml deleted file mode 100644 index 2ac65ea3a5..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-service.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Secret is created in test_case.yaml setup from ../coralogix_app.py -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ad-service -spec: - replicas: 1 - selector: - matchLabels: - app: ad-service - template: - metadata: - labels: - app: ad-service - spec: - containers: - - name: ad-service - image: python:3.9-slim - command: - - sh - - -c - - | - pip install flask opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-instrumentation --quiet - python /app/app.py - volumeMounts: - - name: app - mountPath: /app - ports: - - containerPort: 8080 - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: CORALOGIX_LOGS_ENABLED - value: "false" - - name: CORALOGIX_TRACES_ENABLED - value: "true" - - name: CORALOGIX_METRICS_ENABLED - value: "false" - - name: SERVICE_NAME - value: "ad-service" - - name: SERVICE_LABEL - value: "ad" - - name: K8S_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: K8S_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "500m" - volumes: - - name: app - secret: - secretName: ad-service-app ---- -apiVersion: v1 -kind: Service -metadata: - name: ad-service -spec: - selector: - app: ad-service - ports: - - port: 8080 - targetPort: 8080 diff --git a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-traffic-generator.yaml b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-traffic-generator.yaml deleted file mode 100644 index 99fbdcaaeb..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/ad-traffic-generator.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ad-generator -spec: - replicas: 1 - selector: - matchLabels: - app: ad-generator - template: - metadata: - labels: - app: ad-generator - spec: - containers: - - name: ad-generator - image: python:3.9-slim - command: - - sh - - -c - - | - pip install requests -q - python /app/traffic_generator.py - volumeMounts: - - name: app - mountPath: /app - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: TARGET_URL - value: "http://ad-service:8080/payment" - - name: SERVICE_LABEL - value: "ad" - volumes: - - name: app - secret: - secretName: ad-generator-app diff --git a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/test_case.yaml deleted file mode 100644 index d100575e3c..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/174_coralogix_traces_ad/test_case.yaml +++ /dev/null @@ -1,136 +0,0 @@ -user_prompt: - - "The ad-serving service in namespace app-174 is showing slow requests." - -expected_output: - - The answer must identify a long-duration trace involving a database query - - The answer must mention database slowness/connection issues causing the latency - -tags: - - kubernetes - - traces - - coralogix - -setup_timeout: 900 - -before_test: | - # Validate required Coralogix environment variables - [ -n "${CORALOGIX_API_KEY:-}" ] && [ -n "${CORALOGIX_DOMAIN:-}" ] && [ -n "${CORALOGIX_TEAM_HOSTNAME:-}" ] || { for v in CORALOGIX_API_KEY CORALOGIX_DOMAIN CORALOGIX_TEAM_HOSTNAME; do [ -n "${!v:-}" ] || echo "Missing env var: $v"; done; exit 1; } - - echo "🚀 Setting up test 174 - Creating namespace app-174" - kubectl create namespace app-174 || true - echo "✅ Namespace app-174 created successfully!" - - echo "🔐 Creating ad-service secret from shared coralogix_app.py" - kubectl create secret generic ad-service-app \ - --from-file=app.py=../../shared/coralogix/coralogix_app.py \ - -n app-174 --dry-run=client -o yaml | kubectl apply -f - - - echo "🔐 Creating ad traffic-generator secret from shared traffic_generator.py" - kubectl create secret generic ad-generator-app \ - --from-file=traffic_generator.py=../../shared/coralogix/traffic_generator.py \ - -n app-174 --dry-run=client -o yaml | kubectl apply -f - - - echo "📦 Deploying ad service" - kubectl apply -f ad-service.yaml -n app-174 - - echo "⏳ Waiting for ad-service pod to exist" - for i in {1..60}; do - if kubectl get pod -l app=ad-service -n app-174 2>/dev/null | grep -q ad-service; then - echo "✅ ad-service pod exists" - break - fi - echo "Waiting for ad-service pod to be created... ($i/60)" - sleep 2 - done - - echo "⏳ Waiting for ad-service pod to be ready (timeout 780s)" - AD_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=ad-service -n app-174 --timeout=5s 2>/dev/null; then - echo "✅ ad-service pod is ready!" - AD_POD_READY=true - break - else - echo "⏳ Attempt $i/156: ad-service pod not ready yet, waiting 5s..." - sleep 5 - fi - done - - if [ "$AD_POD_READY" = false ]; then - echo "❌ ad-service pod failed to become ready after 780 seconds" - kubectl get pods -n app-174 -l app=ad-service - exit 1 - fi - - echo "🔍 Checking ad-service deployment status" - kubectl get pods -n app-174 -l app=ad-service - - echo "🚦 Deploying ad traffic generator" - kubectl apply -f ad-traffic-generator.yaml -n app-174 - - echo "⏳ Waiting for ad-generator pod to exist" - for i in {1..60}; do - if kubectl get pod -l app=ad-generator -n app-174 2>/dev/null | grep -q ad-generator; then - echo "✅ ad-generator pod exists" - break - fi - echo "Waiting for ad-generator pod to be created... ($i/60)" - sleep 2 - done - - echo "⏳ Waiting for ad-generator pod to be ready (timeout 780s)" - TRAFFIC_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=ad-generator -n app-174 --timeout=5s 2>/dev/null; then - echo "✅ ad-generator pod is ready!" - TRAFFIC_POD_READY=true - break - else - echo "⏳ Attempt $i/156: ad-generator pod not ready yet, waiting 5s..." - sleep 5 - fi - done - - if [ "$TRAFFIC_POD_READY" = false ]; then - echo "❌ ad-generator pod failed to become ready after 780 seconds" - kubectl get pods -n app-174 -l app=ad-generator - exit 1 - fi - - echo "🔍 Checking all pods status" - kubectl get pods -n app-174 - - echo "⏰ Waiting for ad traffic to produce traces..." - TRAFFIC_READY=false - - echo "⏳ Waiting for at least 10 traces in Coralogix (poll every 5s)" - TRACES_READY=false - for i in {1..60}; do - TRACE_COUNT=$(curl -s -X POST "https://ng-api-http.${CORALOGIX_DOMAIN}/api/v1/dataprime/query" \ - -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ - -H "Content-Type: application/json" \ - -d '{"query": "source spans | lucene '\''app-174 AND ad-service'\'' | limit 50"}' \ - | grep -o 'traceID' | wc -l) - - if [ "$TRACE_COUNT" -ge 10 ]; then - echo "✅ Found $TRACE_COUNT traces in Coralogix" - TRACES_READY=true - break - fi - - echo "⏳ Attempt $i/60: traces found=$TRACE_COUNT" - sleep 5 - done - - if [ "$TRACES_READY" = false ]; then - echo "❌ Did not see 10 traces in Coralogix within timeout" - exit 1 - fi - - # Delete ad-generator so the ai won't cheat - kubectl delete -f ad-traffic-generator.yaml -n app-174 - - echo "✅ Test setup complete!" - -after_test: | - kubectl delete namespace app-174 || true diff --git a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/test_case.yaml new file mode 100644 index 0000000000..c5482ff5e9 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/test_case.yaml @@ -0,0 +1,87 @@ +# Test: Coralogix Metrics Query via Prometheus API +# Tests that Holmes can query metrics from Coralogix using the Prometheus API +# +# This is a fast cloud-only test that sends metrics via OTLP gRPC to Coralogix, +# then queries them via the Prometheus-compatible API. No Kubernetes required. +# +# Requirements: +# - CORALOGIX_API_KEY: API key with DataQuerying and Send Data permissions +# - CORALOGIX_DOMAIN: e.g., "eu2.coralogix.com" +# - CORALOGIX_TEAM_HOSTNAME: Your team name +# - OpenTelemetry Python packages (installed via poetry) +# +# Anti-hallucination: The test creates metrics with a unique prefix (eval175_) +# that the LLM can only find by actually querying Coralogix. + +user_prompt: "Query the Coralogix metrics for the 'holmes-eval-175' application. Look for HTTP request metrics with prefix 'eval175_' and tell me: what metrics are available, how many total requests were made, and what endpoints have the most errors." + +expected_output: + - "Must identify metrics with the eval175_ prefix" + - "Must mention eval175_http_requests_total, eval175_http_errors_total, or eval175_request_latency_seconds" + - "Must provide information about request counts or error rates by endpoint" + +tags: + - coralogix + - metrics + - prometheus + - medium + +setup_timeout: 300 + +before_test: | + source ../../shared/coralogix/coralogix_test_utils.sh + cx_setup + set -e + + echo "🚀 Setting up Coralogix metrics test 175" + + APP_NAME="holmes-eval-175" + SUBSYSTEM="api-gateway" + METRIC_PREFIX="eval175" + + echo "⏳ Sending metrics to Coralogix via OTLP..." + + # Send metrics using Python OTLP exporter + python3 ../../shared/coralogix/send_metrics.py \ + --app-name "$APP_NAME" \ + --subsystem "$SUBSYSTEM" \ + --metric-prefix "$METRIC_PREFIX" + + if [ $? -ne 0 ]; then + echo "❌ Failed to send metrics" + exit 1 + fi + + echo "✅ Metrics sent to $APP_NAME/$SUBSYSTEM" + + # Wait for metrics to be queryable via Prometheus API + echo "⏳ Waiting for metrics to be queryable in Coralogix..." + PROMETHEUS_URL="https://ng-api-http.${CORALOGIX_DOMAIN}/metrics" + + METRICS_READY=false + for i in {1..60}; do + # Query for our specific metric + RESULT=$(curl -sf -G "$PROMETHEUS_URL/api/v1/query" \ + -H "Authorization: Bearer ${CORALOGIX_API_KEY}" \ + --data-urlencode "query=${METRIC_PREFIX}_http_requests_total" 2>/dev/null) + + if echo "$RESULT" | grep -q "${METRIC_PREFIX}_http_requests_total"; then + echo "✅ Metrics are queryable after $((i * 5)) seconds" + METRICS_READY=true + break + fi + + echo " Attempt $i/60: Metrics not yet available, waiting 5s..." + sleep 5 + done + + if [ "$METRICS_READY" = false ]; then + echo "❌ Timeout waiting for metrics to be queryable" + exit 1 + fi + + echo "✅ Test setup complete!" + +after_test: | + # No cleanup needed - metrics will naturally age out of Coralogix + echo "✅ Cleanup complete (metrics will age out naturally)" diff --git a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/toolsets.yaml similarity index 57% rename from tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/toolsets.yaml index 6cacd00971..f77f0c4ad0 100644 --- a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics/toolsets.yaml @@ -1,4 +1,7 @@ -# Coralogix metrics + logs for frontend latency scenario +# Coralogix metrics toolset configuration for test 175 +# Uses the Prometheus-compatible API for querying metrics +# Cloud-only test - no Kubernetes required + toolsets: prometheus/metrics: enabled: true @@ -6,7 +9,3 @@ toolsets: prometheus_url: "https://ng-api-http.{{env.CORALOGIX_DOMAIN}}/metrics" headers: Authorization: "Bearer {{env.CORALOGIX_API_KEY}}" - kubernetes/logs: - enabled: false - kubernetes/core: - enabled: true diff --git a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-service.yaml b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-service.yaml deleted file mode 100644 index 38d560f94e..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-service.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Secret is created in test_case.yaml setup from ../coralogix_app.py -apiVersion: apps/v1 -kind: Deployment -metadata: - name: frontend-service -spec: - replicas: 1 - selector: - matchLabels: - app: frontend-service - template: - metadata: - labels: - app: frontend-service - spec: - containers: - - name: frontend-service - image: python:3.9-slim - command: - - sh - - -c - - | - pip install flask opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-instrumentation --quiet - python /app/app.py - volumeMounts: - - name: app - mountPath: /app - ports: - - containerPort: 8080 - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: CORALOGIX_LOGS_ENABLED - value: "false" - - name: CORALOGIX_TRACES_ENABLED - value: "false" - - name: CORALOGIX_METRICS_ENABLED - value: "true" - - name: SERVICE_NAME - value: "frontend-service" - - name: SERVICE_LABEL - value: "frontend" - - name: K8S_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: K8S_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "500m" - volumes: - - name: app - secret: - secretName: frontend-service-app ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend-service -spec: - selector: - app: frontend-service - ports: - - port: 8080 - targetPort: 8080 diff --git a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-traffic-generator.yaml b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-traffic-generator.yaml deleted file mode 100644 index 2479e827de..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/frontend-traffic-generator.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: frontend-generator -spec: - replicas: 1 - selector: - matchLabels: - app: frontend-generator - template: - metadata: - labels: - app: frontend-generator - spec: - containers: - - name: frontend-generator - image: python:3.9-slim - command: - - sh - - -c - - | - pip install requests -q - python /app/traffic_generator.py - volumeMounts: - - name: app - mountPath: /app - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: TARGET_URL - value: "http://frontend-service:8080/payment" - - name: SERVICE_LABEL - value: "frontend" - volumes: - - name: app - secret: - secretName: frontend-generator-app diff --git a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/test_case.yaml deleted file mode 100644 index b3b3dd6cf4..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/175_coralogix_metrics_frontend/test_case.yaml +++ /dev/null @@ -1,132 +0,0 @@ -user_prompt: - - "Frontend in namespace app-175 is reporting high latency according to the metrics dashboard. Look at the custom payments metrics to find the failure rate and average duration. Share what metrics you found." - -expected_output: - - The answer must identify elevated latency in metrics for the frontend service - - "The answer must reference custom payments metrics: payment_duration_seconds, payment_failure_total, payment_success_total (e.g., failure rate and average duration)" - -tags: - - kubernetes - - metrics - - logs - - coralogix - - prometheus - -setup_timeout: 900 - -before_test: | - # Validate required Coralogix environment variables - [ -n "${CORALOGIX_API_KEY:-}" ] && [ -n "${CORALOGIX_DOMAIN:-}" ] && [ -n "${CORALOGIX_TEAM_HOSTNAME:-}" ] || { for v in CORALOGIX_API_KEY CORALOGIX_DOMAIN CORALOGIX_TEAM_HOSTNAME; do [ -n "${!v:-}" ] || echo "Missing env var: $v"; done; exit 1; } - - echo "🚀 Setting up test 175 - Creating namespace app-175" - kubectl create namespace app-175 || true - echo "✅ Namespace app-175 created successfully!" - - echo "🔐 Creating frontend-service secret from shared coralogix_app.py" - kubectl create secret generic frontend-service-app \ - --from-file=app.py=../../shared/coralogix/coralogix_app.py \ - -n app-175 --dry-run=client -o yaml | kubectl apply -f - - - echo "🔐 Creating frontend traffic-generator secret from shared traffic_generator.py" - kubectl create secret generic frontend-generator-app \ - --from-file=traffic_generator.py=../../shared/coralogix/traffic_generator.py \ - -n app-175 --dry-run=client -o yaml | kubectl apply -f - - - echo "📦 Deploying frontend service" - kubectl apply -f frontend-service.yaml -n app-175 - - echo "⏳ Waiting for frontend-service pod to exist" - for i in {1..60}; do - if kubectl get pod -l app=frontend-service -n app-175 2>/dev/null | grep -q frontend-service; then - echo "✅ frontend-service pod exists" - break - fi - echo "Waiting for frontend-service pod to be created... ($i/60)" - sleep 2 - done - - echo "⏳ Waiting for frontend-service pod to be ready (timeout 780s)" - FRONTEND_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=frontend-service -n app-175 --timeout=5s 2>/dev/null; then - echo "✅ frontend-service pod is ready!" - FRONTEND_POD_READY=true - break - else - echo "⏳ Attempt $i/156: frontend-service pod not ready yet, waiting 5s..." - sleep 5 - fi - done - - if [ "$FRONTEND_POD_READY" = false ]; then - echo "❌ frontend-service pod failed to become ready after 780 seconds" - kubectl get pods -n app-175 -l app=frontend-service - exit 1 - fi - - echo "🔍 Checking frontend-service deployment status" - kubectl get pods -n app-175 -l app=frontend-service - - echo "🚦 Deploying frontend traffic generator" - kubectl apply -f frontend-traffic-generator.yaml -n app-175 - - echo "⏳ Waiting for frontend-generator pod to exist" - for i in {1..60}; do - if kubectl get pod -l app=frontend-generator -n app-175 2>/dev/null | grep -q frontend-generator; then - echo "✅ frontend-generator pod exists" - break - fi - echo "Waiting for frontend-generator pod to be created... ($i/60)" - sleep 2 - done - - echo "⏳ Waiting for frontend-generator pod to be ready (timeout 780s)" - TRAFFIC_POD_READY=false - for i in {1..156}; do - if kubectl wait --for=condition=ready pod -l app=frontend-generator -n app-175 --timeout=5s 2>/dev/null; then - echo "✅ frontend-generator pod is ready!" - TRAFFIC_POD_READY=true - break - else - echo "⏳ Attempt $i/156: frontend-generator pod not ready yet, waiting 5s..." - sleep 5 - fi - done - - if [ "$TRAFFIC_POD_READY" = false ]; then - echo "❌ frontend-generator pod failed to become ready after 780 seconds" - kubectl get pods -n app-175 -l app=frontend-generator - exit 1 - fi - - echo "🔍 Checking all pods status" - kubectl get pods -n app-175 - - echo "⏰ Waiting for frontend traffic to produce logs..." - TRAFFIC_READY=false - for i in {1..90}; do - FRONTEND_LOGS=$(kubectl logs -n app-175 -l app=frontend-service --tail=100 2>/dev/null | grep -c "Processing frontend request" || echo "0") - ERROR_LOGS=$(kubectl logs -n app-175 -l app=frontend-service --tail=100 2>/dev/null | grep -c "ERROR\\|WARN" || echo "0") - - if [ "$FRONTEND_LOGS" -gt "10" ] && [ "$ERROR_LOGS" -gt "0" ]; then - echo "✅ Found required logs after $i seconds:" - echo " - Frontend processing logs: $FRONTEND_LOGS" - echo " - Error/Warning logs: $ERROR_LOGS" - TRAFFIC_READY=true - sleep 10 - break - fi - - echo "⏳ Attempt $i/90: FRONTEND_LOGS=$FRONTEND_LOGS, ERRORS=$ERROR_LOGS" - sleep 1 - done - - if [ "$TRAFFIC_READY" = false ]; then - echo "❌ Failed to generate required traffic patterns" - exit 1 - fi - - echo "✅ Test setup complete!" - -after_test: | - kubectl delete namespace app-175 || true diff --git a/tests/plugins/toolsets/coralogix/test_coralogix.py b/tests/plugins/toolsets/coralogix/test_coralogix.py index 9d7eac4f1c..b6a3f6af22 100644 --- a/tests/plugins/toolsets/coralogix/test_coralogix.py +++ b/tests/plugins/toolsets/coralogix/test_coralogix.py @@ -14,26 +14,16 @@ ) from holmes.plugins.toolsets.coralogix.utils import ( CoralogixConfig, - extract_field, normalize_datetime, ) @pytest.fixture def coralogix_config(): - from holmes.plugins.toolsets.coralogix.utils import CoralogixLabelsConfig - - labels_config = CoralogixLabelsConfig( - pod="kubernetes.pod_name", - namespace="kubernetes.namespace_name", - log_message="log", - timestamp="time", - ) return CoralogixConfig( api_key="dummy_api_key", team_hostname="my-team", domain="eu2.coralogix.com", - labels=labels_config, ) @@ -58,18 +48,6 @@ def test_normalize_datetime(input_date, expected_output): assert normalize_datetime(input_date) == expected_output -@pytest.mark.parametrize( - "data_obj, field, expected", - [ - ({"key": "value"}, "key", "value"), - ({"parent": {"child": "deep_value"}}, "parent.child", "deep_value"), - ({}, "key", None), - ], -) -def test_extract_field(data_obj, field, expected): - assert extract_field(data_obj, field) == expected - - class TestExecuteDataPrimeQuery: """Tests for execute_dataprime_query function."""