From 8f1a791ace4c7012178f487fb6726e84c80bcb7a Mon Sep 17 00:00:00 2001 From: Isala Piyarisi Date: Thu, 7 Apr 2022 15:42:25 +0530 Subject: [PATCH] fix(control-plane): Models not registering --- .../samples/lazykoala_v1alpha1_inspector.yaml | 108 +++++++++++++++++- control-plane/config/samples/loads.yaml | 8 +- control-plane/config/samples/simulation.yaml | 16 +-- .../controllers/inspector_controller.go | 6 +- sherlock/src/inference.rs | 1 + sherlock/src/query.rs | 2 +- 6 files changed, 124 insertions(+), 17 deletions(-) diff --git a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml index ad64c34..975b8d2 100644 --- a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml +++ b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml @@ -1,10 +1,110 @@ apiVersion: lazykoala.isala.me/v1alpha1 kind: Inspector metadata: - name: sample-service + name: service-1-7a942121 namespace: default spec: - deploymentRef: sample-service - serviceRef: sample-service - modelName: sample_model + deploymentRef: service-1-7a942121 + serviceRef: service-1-7a942121 + modelName: service-1-7a942121 namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-2-7a942121 + namespace: default +spec: + deploymentRef: service-2-7a942121 + serviceRef: service-2-7a942121 + modelName: service-2-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-3-7a942121 + namespace: default +spec: + deploymentRef: service-3-7a942121 + serviceRef: service-3-7a942121 + modelName: service-3-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-4-7a942121 + namespace: default +spec: + deploymentRef: service-4-7a942121 + serviceRef: service-4-7a942121 + modelName: service-4-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-5-7a942121 + namespace: default +spec: + deploymentRef: service-5-7a942121 + serviceRef: service-5-7a942121 + modelName: service-5-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-6-7a942121 + namespace: default +spec: + deploymentRef: service-6-7a942121 + serviceRef: service-6-7a942121 + modelName: service-6-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-7-7a942121 + namespace: default +spec: + deploymentRef: service-7-7a942121 + serviceRef: service-7-7a942121 + modelName: service-7-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-8-7a942121 + namespace: default +spec: + deploymentRef: service-8-7a942121 + serviceRef: service-8-7a942121 + modelName: service-8-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-9-7a942121 + namespace: default +spec: + deploymentRef: service-9-7a942121 + serviceRef: service-9-7a942121 + modelName: service-9-7a942121 + namespace: default +--- +apiVersion: lazykoala.isala.me/v1alpha1 +kind: Inspector +metadata: + name: service-10-7a942121 + namespace: default +spec: + deploymentRef: service-10-7a942121 + serviceRef: service-10-7a942121 + modelName: service-10-7a942121 + namespace: default +--- \ No newline at end of file diff --git a/control-plane/config/samples/loads.yaml b/control-plane/config/samples/loads.yaml index 6f13c67..f292a75 100644 --- a/control-plane/config/samples/loads.yaml +++ b/control-plane/config/samples/loads.yaml @@ -8,7 +8,7 @@ spec: name: lazy-koala-simulation namespace: default # requestCount: 1 - replicas: 25 + replicas: 50 # timeout: 15m betweenDelay: 1s requests: @@ -23,4 +23,8 @@ spec: - | {"designation":"service_2","probability":99,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_10","probability":67,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_2","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_9","probability":50,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_1","probability":76,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_7","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_2","probability":67,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_4","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_7","probability":98,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_5","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_1","probability":62,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_8","probability":57,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_5","probability":85,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_9","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_8","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":[{"designation":"service_6","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":322}}],"after":[{"type":"latency","args":{"delay":254}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} - | - {"designation":"service_3","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_5","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_1","probability":91,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_7","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_4","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_8","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_8","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_9","probability":91,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":58,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_5","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} \ No newline at end of file + {"designation":"service_3","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_5","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_1","probability":91,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_7","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_4","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_8","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_2","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_8","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_9","probability":91,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_10","probability":58,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":[{"designation":"service_5","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":375}}],"after":[{"type":"latency","args":{"delay":358}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} + - | + {"designation":"service_1","probability":62,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_6","probability":77,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_2","probability":89,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_3","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_7","probability":70,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_3","probability":90,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_8","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_4","probability":83,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_3","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_6","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_1","probability":96,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_4","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_5","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_3","probability":85,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_4","probability":57,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":[{"designation":"service_3","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":275}}],"after":[{"type":"latency","args":{"delay":119}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} + - | + {"designation":"service_3","probability":63,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_7","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_4","probability":58,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_2","probability":68,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_5","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_2","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_3","probability":55,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_8","probability":82,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_3","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_10","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_8","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_4","probability":96,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_7","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_5","probability":81,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_1","probability":68,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":[{"designation":"service_4","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":139}}],"after":[{"type":"latency","args":{"delay":223}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]} \ No newline at end of file diff --git a/control-plane/config/samples/simulation.yaml b/control-plane/config/samples/simulation.yaml index 308e631..9b20954 100644 --- a/control-plane/config/samples/simulation.yaml +++ b/control-plane/config/samples/simulation.yaml @@ -13,23 +13,23 @@ spec: language: go framework: gorilla service_3: - language: go - framework: gorilla + language: node + framework: express service_4: language: go framework: gorilla service_5: - language: node - framework: express + language: python + framework: flask service_6: - language: node - framework: express + language: go + framework: gorilla service_7: language: node framework: express service_8: - language: python - framework: flask + language: node + framework: express service_9: language: python framework: flask diff --git a/control-plane/controllers/inspector_controller.go b/control-plane/controllers/inspector_controller.go index 6ca8adf..15a0a5e 100644 --- a/control-plane/controllers/inspector_controller.go +++ b/control-plane/controllers/inspector_controller.go @@ -301,14 +301,16 @@ func (r *InspectorReconciler) configureSherlock(ctx context.Context, inspector * ModelName: inspector.Spec.ModelName, Namespace: inspector.Spec.Namespace, } - modelsList[inspector.Spec.ModelName] = true } else { if _, ok := sherlockServiceList[inspector.ObjectMeta.Name]; ok { delete(sherlockServiceList, inspector.ObjectMeta.Name) - delete(modelsList, inspector.Spec.ModelName) } } + for _, service := range sherlockServiceList { + modelsList[service.ModelName] = true + } + // Generate the Servings Config servingsConfig, err := createServingsConfig(modelsList) if err != nil { diff --git a/sherlock/src/inference.rs b/sherlock/src/inference.rs index 6d3fc5d..a8ad35b 100644 --- a/sherlock/src/inference.rs +++ b/sherlock/src/inference.rs @@ -58,6 +58,7 @@ async fn query_model(service: &str, input: [[[f64; 3]; 9]; 10]) -> Result().await?.predictions.into_iter().flatten().flatten().flatten().collect::>(); let mse: f64 = MeanAbsoluteError{}.get_score(&input.flat().flat().to_vec(), &predictions); diff --git a/sherlock/src/query.rs b/sherlock/src/query.rs index 18076f2..8b971da 100644 --- a/sherlock/src/query.rs +++ b/sherlock/src/query.rs @@ -1,4 +1,4 @@ -use std::{env::var, f64::{INFINITY, NEG_INFINITY}, cmp}; +use std::{env::var, f64::{INFINITY, NEG_INFINITY}}; use prometheus_http_query::{Client, RangeVector}; use chrono::{Duration, Local, DateTime}; use lazy_static::lazy_static;