diff --git a/control-plane/api/v1alpha1/inspector_types.go b/control-plane/api/v1alpha1/inspector_types.go index 82c87e6..de59003 100644 --- a/control-plane/api/v1alpha1/inspector_types.go +++ b/control-plane/api/v1alpha1/inspector_types.go @@ -39,7 +39,6 @@ type InspectorSpec struct { DeploymentRef string `json:"deploymentRef"` ServiceRef string `json:"serviceRef"` Namespace string `json:"namespace"` - ModelURI string `json:"modelURI"` } type Status string diff --git a/control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml b/control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml index be6ae14..263fb32 100644 --- a/control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml +++ b/control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml @@ -55,15 +55,12 @@ spec: description: Foo is an example field of Inspector. Edit inspector_types.go to remove/update type: string - modelURI: - type: string namespace: type: string serviceRef: type: string required: - deploymentRef - - modelURI - namespace - serviceRef type: object diff --git a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml index 98006a4..3c537a1 100644 --- a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml +++ b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml @@ -4,10 +4,9 @@ metadata: name: service-1 namespace: default spec: - deploymentRef: service-1-0e762959 - serviceRef: service-1-0e762959 + deploymentRef: service-1-05588114 + serviceRef: service-1-05588114 namespace: default - modelURI: path/to/checkpoint.ckpt --- apiVersion: lazykoala.isala.me/v1alpha1 kind: Inspector @@ -15,10 +14,9 @@ metadata: name: service-2 namespace: default spec: - deploymentRef: service-2-0e762959 - serviceRef: service-2-0e762959 + deploymentRef: service-2-05588114 + serviceRef: service-2-05588114 namespace: default - modelURI: path/to/checkpoint.ckpt --- apiVersion: lazykoala.isala.me/v1alpha1 kind: Inspector @@ -26,10 +24,9 @@ metadata: name: service-3 namespace: default spec: - deploymentRef: service-3-0e762959 - serviceRef: service-3-0e762959 + deploymentRef: service-3-05588114 + serviceRef: service-3-05588114 namespace: default - modelURI: path/to/checkpoint.ckpt --- apiVersion: lazykoala.isala.me/v1alpha1 kind: Inspector @@ -37,9 +34,8 @@ metadata: name: service-4 namespace: default spec: - deploymentRef: service-4-0e762959 - serviceRef: service-4-0e762959 + deploymentRef: service-4-05588114 + serviceRef: service-4-05588114 namespace: default - modelURI: path/to/checkpoint.ckpt --- diff --git a/control-plane/controllers/inspector_controller.go b/control-plane/controllers/inspector_controller.go index 9e0c2ce..6ea0c38 100644 --- a/control-plane/controllers/inspector_controller.go +++ b/control-plane/controllers/inspector_controller.go @@ -17,12 +17,14 @@ limitations under the License. package controllers import ( + "bytes" "context" "fmt" + appsv1 "k8s.io/api/apps/v1" + "text/template" "time" "gopkg.in/yaml.v3" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -89,6 +91,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if err := r.Update(ctx, &inspector); err != nil { return ctrl.Result{}, err } + return ctrl.Result{Requeue: true}, nil } } else { // The object is being deleted @@ -100,6 +103,10 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } + if err := r.configureSherlock(ctx, &inspector, false); err != nil { + return ctrl.Result{}, err + } + // remove our finalizer from the list and update it. controllerutil.RemoveFinalizer(&inspector, finalizerName) if err := r.Update(ctx, &inspector); err != nil { @@ -110,13 +117,80 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Stop reconciliation as the item is being deleted return ctrl.Result{}, nil } + + scrapePoints, err := r.configureGazer(ctx, &inspector) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.configureSherlock(ctx, &inspector, true); err != nil { + return ctrl.Result{}, err + } + + // Update local status + var MonitoredIPs []string + for k := range scrapePoints { + MonitoredIPs = append(MonitoredIPs, k) + } + inspector.Status.MonitoredIPs = MonitoredIPs + inspector.Status.Status = lazykoalav1alpha1.Running + + if err := r.Status().Update(ctx, &inspector); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Minute}, nil +} + +func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error { + // --------------------- START OF GAZER CONFIG --------------------- // + // Get the Gazer config file + var configMap v1.ConfigMap + if err := r.Get(context.Background(), types.NamespacedName{ + Namespace: "lazy-koala", + Name: "gazer-config", + }, &configMap); err != nil { + return err + } + + // Phase the config.yaml + configData := make(map[string]ScrapePoint) + if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil { + return err + } + + // Remove all the existing scrape points created from this Inspector + for _, ip := range inspector.Status.MonitoredIPs { + if _, ok := configData[ip]; ok { + delete(configData, ip) + } + } + + // Encode the config.yaml + encodedConfig, err := yaml.Marshal(&configData) + if err != nil { + return err + } + + // Patch the config file + configMap.Data["config.yaml"] = string(encodedConfig) + if err := r.Update(context.Background(), &configMap); err != nil { + return err + } + + return nil +} + +func (r *InspectorReconciler) configureGazer(ctx context.Context, inspector *lazykoalav1alpha1.Inspector) (map[string]ScrapePoint, error) { + logger := log.FromContext(ctx) + // Get the intended deployment var deploymentRef appsv1.Deployment if err := r.Get(ctx, types.NamespacedName{ Namespace: inspector.Spec.Namespace, Name: inspector.Spec.DeploymentRef, }, &deploymentRef); err != nil { - return ctrl.Result{}, err + return nil, err } scrapePoints := make(map[string]ScrapePoint) @@ -127,7 +201,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( var podList v1.PodList if err := r.List(ctx, &podList, &selector); client.IgnoreNotFound(err) != nil { logger.Error(err, fmt.Sprintf("failed to pods for deployment %s", deploymentRef.ObjectMeta.Name)) - return ctrl.Result{}, err + return nil, err } // Create Scrape point for each pod @@ -147,7 +221,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Namespace: inspector.Spec.Namespace, Name: inspector.Spec.ServiceRef, }, &serviceRef); err != nil { - return ctrl.Result{}, err + return nil, err } scrapePoints[serviceRef.Spec.ClusterIP] = ScrapePoint{ @@ -159,91 +233,89 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Get the Gazer config file - var configMap v1.ConfigMap + var gazerConfigMap v1.ConfigMap if err := r.Get(ctx, types.NamespacedName{ Namespace: "lazy-koala", Name: "gazer-config", - }, &configMap); err != nil { - return ctrl.Result{}, err + }, &gazerConfigMap); err != nil { + return nil, err } // Phase the config.yaml - configData := make(map[string]ScrapePoint) - if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil { - return ctrl.Result{}, err + gazerData := make(map[string]ScrapePoint) + if err := yaml.Unmarshal([]byte(gazerConfigMap.Data["config.yaml"]), &gazerData); err != nil { + return nil, err } // Remove all the existing scrape points created from this Inspector for _, ip := range inspector.Status.MonitoredIPs { - if _, ok := configData[ip]; ok { - delete(configData, ip) + if _, ok := gazerData[ip]; ok { + delete(gazerData, ip) } } // Add the new scrape points for k, v := range scrapePoints { - configData[k] = v + gazerData[k] = v } // Encode the config.yaml - encodedConfig, err := yaml.Marshal(&configData) + encodedConfig, err := yaml.Marshal(&gazerData) if err != nil { - return ctrl.Result{}, err + return nil, err } // Patch the config file - configMap.Data["config.yaml"] = string(encodedConfig) - if err := r.Update(ctx, &configMap); err != nil { - return ctrl.Result{}, err + gazerConfigMap.Data["config.yaml"] = string(encodedConfig) + if err := r.Update(ctx, &gazerConfigMap); err != nil { + return nil, err } - // Update local status - var MonitoredIPs []string - for k := range scrapePoints { - MonitoredIPs = append(MonitoredIPs, k) - } - inspector.Status.MonitoredIPs = MonitoredIPs - inspector.Status.Status = lazykoalav1alpha1.Running - - if err := r.Status().Update(ctx, &inspector); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: time.Minute}, nil + return scrapePoints, nil } -func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error { - // Get the Gazer config file - var configMap v1.ConfigMap - if err := r.Get(context.Background(), types.NamespacedName{ +func (r *InspectorReconciler) configureSherlock(ctx context.Context, inspector *lazykoalav1alpha1.Inspector, append bool) error { + // Get the Sherlock config file + var sherlockConfigMap v1.ConfigMap + if err := r.Get(ctx, types.NamespacedName{ Namespace: "lazy-koala", - Name: "gazer-config", - }, &configMap); err != nil { + Name: "sherlock-config", + }, &sherlockConfigMap); err != nil { return err } - // Phase the config.yaml - configData := make(map[string]ScrapePoint) - if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil { + // Phase the services.yaml + var sherlockServiceList []string + if err := yaml.Unmarshal([]byte(sherlockConfigMap.Data["services.yaml"]), &sherlockServiceList); err != nil { return err } - // Remove all the existing scrape points created from this Inspector - for _, ip := range inspector.Status.MonitoredIPs { - if _, ok := configData[ip]; ok { - delete(configData, ip) - } + if append { + sherlockServiceList = AppendIfMissing(sherlockServiceList, inspector.Spec.DeploymentRef) + } else { + sherlockServiceList = RemoveIfExists(sherlockServiceList, inspector.Spec.DeploymentRef) } - // Encode the config.yaml - encodedConfig, err := yaml.Marshal(&configData) + // Generate the Servings Config + servingsConfig, err := createServingsConfig(sherlockServiceList) if err != nil { return err } - // Patch the config file - configMap.Data["config.yaml"] = string(encodedConfig) - if err := r.Update(context.Background(), &configMap); err != nil { + if err := r.Update(ctx, &sherlockConfigMap); err != nil { + return err + } + + // Encode the services.yaml + encodedSherlockConfig, err := yaml.Marshal(&sherlockServiceList) + if err != nil { + return err + } + + sherlockConfigMap.Data["services.yaml"] = string(encodedSherlockConfig) + sherlockConfigMap.Data["models.config"] = servingsConfig + + if err := r.Update(ctx, &sherlockConfigMap); err != nil { return err } return nil @@ -265,3 +337,44 @@ func eventFilter() predicate.Predicate { }, } } + +func AppendIfMissing(slice []string, item string) []string { + for _, ele := range slice { + if ele == item { + return slice + } + } + return append(slice, item) +} + +func RemoveIfExists(slice []string, item string) []string { + for i, other := range slice { + if other == item { + return append(slice[:i], slice[i+1:]...) + } + } + return slice +} + +func createServingsConfig(service []string) (string, error) { + tmpl := template.New("config") + + tmpl, err := tmpl.Parse(`model_config_list { + {{range .}} + config { + name: '{{.}}' + base_path: '/models/{{.}}/' + model_platform: 'tensorflow' + } + {{end}} +}`) + if err != nil { + return "", err + } + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, service) + if err != nil { + return "", err + } + return fmt.Sprintf("%+v", buf), nil +} diff --git a/control-plane/go.mod b/control-plane/go.mod index c9da798..a38e6fa 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -61,8 +61,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/api v0.23.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + k8s.io/api v0.23.0 k8s.io/apiextensions-apiserver v0.23.0 // indirect k8s.io/component-base v0.23.0 // indirect k8s.io/klog/v2 v2.30.0 // indirect diff --git a/sherlock/.gitignore b/sherlock/.gitignore index abe5ee3..2aca851 100644 --- a/sherlock/.gitignore +++ b/sherlock/.gitignore @@ -15,8 +15,12 @@ Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk -# MSVC Windows builds of rustc generate these, which store debugging information +# MSVC Windows builds of rustc generate these, which store debugging informationl *.pdb +# End of https://www.toptal.com/developers/gitignore/api/rust + models/* !models/.gitkeep -# End of https://www.toptal.com/developers/gitignore/api/rust \ No newline at end of file + +config/services.yaml +!config/.gitkeep \ No newline at end of file diff --git a/sherlock/config/.gitkeep b/sherlock/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sherlock/deployment.yaml b/sherlock/deployment.yaml index 3352240..ede0dd8 100644 --- a/sherlock/deployment.yaml +++ b/sherlock/deployment.yaml @@ -27,7 +27,7 @@ spec: - name: POOL_DURATION value: "1" volumeMounts: - - name: model-config + - name: sherlock-config mountPath: /app/config - image: google/cloud-sdk name: model-poller @@ -59,7 +59,7 @@ spec: - name: MODEL_NAME value: "sherlock" volumeMounts: - - name: model-config + - name: sherlock-config mountPath: /config - name: model-files mountPath: /models @@ -77,19 +77,9 @@ metadata: namespace: lazy-koala data: models.config: | - model_config_list { - config { - name: 'service_1' - base_path: '/models/service_1/' - model_platform: 'tensorflow' - } - config { - name: 'service_2' - base_path: '/models/service_2/' - model_platform: 'tensorflow' - } - } + services.yaml: | + # --- ## https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity # apiVersion: v1 diff --git a/sherlock/src/main.rs b/sherlock/src/main.rs index 8da83c2..c016de8 100644 --- a/sherlock/src/main.rs +++ b/sherlock/src/main.rs @@ -11,8 +11,8 @@ use std::collections::HashMap; use serde_yaml; lazy_static! { - static ref END_POINT: String = var("END_POINT").unwrap(); - static ref POOL_DURATION: String = var("POOL_DURATION").unwrap(); + static ref END_POINT: String = var("END_POINT").unwrap_or("http://localhost:8501/v1/models".to_string()); + static ref POOL_DURATION: String = var("POOL_DURATION").unwrap_or("60".to_string()); static ref ANOMLAY_GAUGE: GaugeVec = register_gauge_vec!( opts!( "anomaly_score", @@ -66,7 +66,7 @@ fn poll_anomaly_scores(delay: u64) { } fn read_config() -> Result, Box> { - let f = std::fs::File::open("config/config.yaml")?; + let f = std::fs::File::open("config/services.yaml")?; let services: Vec = serde_yaml::from_reader(f)?; Ok(services) }