diff --git a/control-plane/config/samples/loads.yaml b/control-plane/config/samples/loads.yaml index de33733..e5b44be 100644 --- a/control-plane/config/samples/loads.yaml +++ b/control-plane/config/samples/loads.yaml @@ -8,15 +8,13 @@ spec: name: lazy-koala-simulation namespace: default # requestCount: 1 - replicas: 50 + replicas: 25 # timeout: 15m betweenDelay: 1s requests: - | - {"designation":"service_4","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_6","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_7","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_1","probability":77,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_2","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_9","probability":90,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_7","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_4","probability":52,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":76,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_6","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_4","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} + {"designation":"service_10","probability":98,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_4","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_6","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_7","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_10","probability":66,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_7","probability":69,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_4","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_5","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_1","probability":81,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_6","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_4","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_6","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_9","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_7","probability":86,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_9","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":[{"designation":"service_3","probability":52,"faults":{"before":[{"type":"latency","args":{"delay":299}}],"after":[{"type":"latency","args":{"delay":281}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} - | - {"designation":"service_4","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_3","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":66,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_6","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":72,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_3","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_5","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":68,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_6","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_4","probability":50,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_5","probability":82,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_4","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_9","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} + {"designation":"service_8","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_3","probability":69,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_1","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_2","probability":96,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_1","probability":96,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_3","probability":100,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_7","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_10","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_7","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_6","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_10","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_3","probability":66,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_1","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_7","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_2","probability":67,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":[{"designation":"service_1","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":118}}],"after":[{"type":"latency","args":{"delay":183}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} - | - {"designation":"service_9","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_1","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_8","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_6","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_8","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":98,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_5","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_9","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_3","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":63,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_5","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_1","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} - - | - {"designation":"service_9","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_5","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_4","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_7","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_3","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_2","probability":74,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_8","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_4","probability":90,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_1","probability":69,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_5","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_8","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_1","probability":85,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} \ No newline at end of file + {"designation":"service_4","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_9","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_2","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_10","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_3","probability":83,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_10","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_1","probability":67,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_10","probability":96,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_4","probability":83,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_6","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_5","probability":77,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_6","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_10","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_8","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_6","probability":63,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":[{"designation":"service_8","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":194}}],"after":[{"type":"latency","args":{"delay":314}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} diff --git a/gazer/deployment.yaml b/gazer/deployment.yaml index a55053c..3735595 100644 --- a/gazer/deployment.yaml +++ b/gazer/deployment.yaml @@ -227,6 +227,13 @@ rules: - get - list - watch + - apiGroups: ["apps"] + resources: + - deployments + verbs: + - get + - list + - watch - apiGroups: - extensions resources: diff --git a/inspector/deployment.yaml b/inspector/deployment.yaml index 03e29d7..ae81c6d 100644 --- a/inspector/deployment.yaml +++ b/inspector/deployment.yaml @@ -19,7 +19,7 @@ spec: command: ["inspector"] # args: ["-c", "while true; do echo hello; sleep 10;done"] imagePullPolicy: Always - image: ghcr.io/mrsupiri/lazy-koala/inspector:commit-342ac6a8 + image: ghcr.io/mrsupiri/lazy-koala/inspector:latest ports: - containerPort: 8090 name: http diff --git a/scripts/data-exporter.py b/scripts/data-exporter.py new file mode 100644 index 0000000..e72f5a9 --- /dev/null +++ b/scripts/data-exporter.py @@ -0,0 +1,74 @@ +from flask import Flask, request, jsonify +from datetime import datetime +import pymongo +import os + +app = Flask(__name__) + + +myclient = pymongo.MongoClient(os.getenv("MONGODB_URI")) +mydb = myclient["metrics"] + + +@app.route('/save', methods=['GET', 'POST']) +def add_message(): + content = request.json + mycol = mydb[content['service']] + x = mycol.insert_one({"data": content['data'], "time": datetime.now()}) + return jsonify({"success": True}) + +if __name__ == '__main__': + app.run(host= '0.0.0.0',debug=True) + + + + + + + +# import grequests +# import urllib.parse +# import numpy as np + +# services = [ +# "service-1-18f76028", +# "service-10-18f76028", +# "service-2-18f76028", +# "service-3-18f76028", +# "service-4-18f76028", +# "service-5-18f76028", +# "service-6-18f76028", +# "service-7-18f76028", +# "service-8-18f76028", +# "service-9-18f76028", +# ] + +# samples = ["1m", "5m", "15m"] + +# metrics = [ +# 'rate(requests_sent_total{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'sum by (serviceName) (rate(requests_received_total{serviceName="SERVICE_NAME"}[SAMPLE]))', +# 'rate(request_duration_seconds_sum{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'avg_over_time(cpu_seconds{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'avg_over_time(memory_usage_bytes{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'rate(acknowledged_bytes_sum{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'rate(transmitted_bytes_sum{serviceName="SERVICE_NAME"}[SAMPLE])', +# 'avg_over_time(backlog{level="1",serviceName="SERVICE_NAME"}[SAMPLE])', +# 'sum by (serviceName) (avg_over_time(backlog{level!="1",serviceName="SERVICE_NAME"}[SAMPLE]))', +# ] + +# requests = [] + +# def chunks(l, n): +# n = max(1, n) +# return (l[i:i+n] for i in range(0, len(l), n)) + +# for service in services: +# for sample in samples: +# for metric in metrics: +# query = metric.replace("SERVICE_NAME", service).replace("SAMPLE", sample) +# url = "http://127.0.0.1:9090/api/v1/query?query="+urllib.parse.quote_plus(query) +# requests.append(grequests.get(url)) +# respon = grequests.map(requests) +# f +# break; diff --git a/scripts/loadTemplete.py b/scripts/loadTemplete.py index 31f5215..7e54a7f 100644 --- a/scripts/loadTemplete.py +++ b/scripts/loadTemplete.py @@ -1,7 +1,7 @@ import json import random -services = list(range(1,10)) +services = list(range(1,11)) random.shuffle(services) diff --git a/sherlock/deployment.yaml b/sherlock/deployment.yaml index ed27c21..b3992b5 100644 --- a/sherlock/deployment.yaml +++ b/sherlock/deployment.yaml @@ -32,6 +32,15 @@ spec: volumeMounts: - name: sherlock-config mountPath: /app/config + - name: data-exporter + imagePullPolicy: Always + image: asia.gcr.io/iconicto/data-exporter:latest + ports: + - containerPort: 5000 + name: http + env: + - name: MONGODB_URI + value: "http://localhost:8501/v1/models" - image: google/cloud-sdk name: model-poller imagePullPolicy: IfNotPresent diff --git a/sherlock/src/inference.rs b/sherlock/src/inference.rs index b8b05cd..0961e54 100644 --- a/sherlock/src/inference.rs +++ b/sherlock/src/inference.rs @@ -64,10 +64,27 @@ async fn query_model(service: &str, input: [[[f64; 1]; 9]; 10]) -> Result Result<(), Box> { + let query = json!({ + "service": service, + "data": input, + }); + + let client = reqwest::Client::new(); + let _res = client.post("http://localhost:5000/save") + .json::(&query) + .send() + .await?; + + Ok(()) +} + async fn calculate_anomaly_score(service: &str, args: &InferenceData) -> Result<(), Box> { println!("Calculate anomaly score for {} using {}", service, &args.model_name); let input = build_telemetry_matrix(&service).await?; - let score = query_model(&args.model_name, input).await?; + save(&service, input).await?; + // let score = query_model(&args.model_name, input).await?; + let score = 0.4; ANOMLAY_GAUGE.with_label_values(&[service, &args.namespace]).set(score); println!("Anomaly score for {}: {}", service, score); Ok(()) diff --git a/sherlock/src/query.rs b/sherlock/src/query.rs index 744319b..be52fe4 100644 --- a/sherlock/src/query.rs +++ b/sherlock/src/query.rs @@ -18,6 +18,12 @@ lazy_static! { r#"avg_over_time(backlog{level="1",serviceName="SERVICE_NAME"}[SAMPLE])"#, r#"sum by (serviceName) (avg_over_time(backlog{level!="1",serviceName="SERVICE_NAME"}[SAMPLE]))"# ]; + + static ref SAMPLES: [&'static str; 3] = [ + "1m", + "5m", + "15m" + ]; } @@ -31,8 +37,8 @@ async fn query_prometheus(query: &str, time: DateTime) -> Result Result<[[[f64; 1]; 9]; 10], Box> { - let mut data: [[[f64; 1]; 9]; 10] = [[[0.0; 1]; 9]; 10]; +pub async fn build_telemetry_matrix(service: &str) -> Result<[[[f64; 3]; 9]; 10], Box> { + let mut data: [[[f64; 3]; 9]; 10] = [[[0.0; 3]; 9]; 10]; let time_steps: [DateTime; 10] = [ Local::now(), @@ -49,10 +55,12 @@ pub async fn build_telemetry_matrix(service: &str) -> Result<[[[f64; 1]; 9]; 10] for (x, time_step) in time_steps.iter().enumerate() { for (y, metric) in METRICS.iter().enumerate() { - let query = &metric.replace("SERVICE_NAME", service).replace("SAMPLE", "1m"); - match query_prometheus(query, *time_step).await { - Ok(value) => data[x][y][0] = value, - Err(e) => return Err(e), + for (z, sample) in SAMPLES.iter().enumerate(){ + let query = &metric.replace("SERVICE_NAME", service).replace("SAMPLE", sample); + match query_prometheus(query, *time_step).await { + Ok(value) => data[x][y][z] = value, + Err(e) => return Err(e), + } } } }