Skip to content

Commit

Permalink
feat(control-plane): Added support for sherlock module
Browse files Browse the repository at this point in the history
  • Loading branch information
@isala404
isala404 committed Mar 12, 2022
1 parent 432a9df commit c341c76
Show file tree
Hide file tree
Showing 9 changed files with 186 additions and 87 deletions.
1 change: 0 additions & 1 deletion control-plane/api/v1alpha1/inspector_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ type InspectorSpec struct {
DeploymentRef string `json:"deploymentRef"`
ServiceRef string `json:"serviceRef"`
Namespace string `json:"namespace"`
ModelURI string `json:"modelURI"`
}

type Status string
Expand Down
3 changes: 0 additions & 3 deletions control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,12 @@ spec:
description: Foo is an example field of Inspector. Edit inspector_types.go
to remove/update
type: string
modelURI:
type: string
namespace:
type: string
serviceRef:
type: string
required:
- deploymentRef
- modelURI
- namespace
- serviceRef
type: object
Expand Down
20 changes: 8 additions & 12 deletions control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,42 +4,38 @@ metadata:
name: service-1
namespace: default
spec:
deploymentRef: service-1-0e762959
serviceRef: service-1-0e762959
deploymentRef: service-1-05588114
serviceRef: service-1-05588114
namespace: default
modelURI: path/to/checkpoint.ckpt
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-2
namespace: default
spec:
deploymentRef: service-2-0e762959
serviceRef: service-2-0e762959
deploymentRef: service-2-05588114
serviceRef: service-2-05588114
namespace: default
modelURI: path/to/checkpoint.ckpt
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-3
namespace: default
spec:
deploymentRef: service-3-0e762959
serviceRef: service-3-0e762959
deploymentRef: service-3-05588114
serviceRef: service-3-05588114
namespace: default
modelURI: path/to/checkpoint.ckpt
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-4
namespace: default
spec:
deploymentRef: service-4-0e762959
serviceRef: service-4-0e762959
deploymentRef: service-4-05588114
serviceRef: service-4-05588114
namespace: default
modelURI: path/to/checkpoint.ckpt
---

213 changes: 163 additions & 50 deletions control-plane/controllers/inspector_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@ limitations under the License.
package controllers

import (
"bytes"
"context"
"fmt"
appsv1 "k8s.io/api/apps/v1"
"text/template"
"time"

"gopkg.in/yaml.v3"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -89,6 +91,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if err := r.Update(ctx, &inspector); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}
} else {
// The object is being deleted
Expand All @@ -100,6 +103,10 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{}, err
}

if err := r.configureSherlock(ctx, &inspector, false); err != nil {
return ctrl.Result{}, err
}

// remove our finalizer from the list and update it.
controllerutil.RemoveFinalizer(&inspector, finalizerName)
if err := r.Update(ctx, &inspector); err != nil {
Expand All @@ -110,13 +117,80 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// Stop reconciliation as the item is being deleted
return ctrl.Result{}, nil
}

scrapePoints, err := r.configureGazer(ctx, &inspector)
if err != nil {
return ctrl.Result{}, err
}

if err := r.configureSherlock(ctx, &inspector, true); err != nil {
return ctrl.Result{}, err
}

// Update local status
var MonitoredIPs []string
for k := range scrapePoints {
MonitoredIPs = append(MonitoredIPs, k)
}
inspector.Status.MonitoredIPs = MonitoredIPs
inspector.Status.Status = lazykoalav1alpha1.Running

if err := r.Status().Update(ctx, &inspector); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{RequeueAfter: time.Minute}, nil
}

func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error {
// --------------------- START OF GAZER CONFIG --------------------- //
// Get the Gazer config file
var configMap v1.ConfigMap
if err := r.Get(context.Background(), types.NamespacedName{
Namespace: "lazy-koala",
Name: "gazer-config",
}, &configMap); err != nil {
return err
}

// Phase the config.yaml
configData := make(map[string]ScrapePoint)
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
return err
}

// Remove all the existing scrape points created from this Inspector
for _, ip := range inspector.Status.MonitoredIPs {
if _, ok := configData[ip]; ok {
delete(configData, ip)
}
}

// Encode the config.yaml
encodedConfig, err := yaml.Marshal(&configData)
if err != nil {
return err
}

// Patch the config file
configMap.Data["config.yaml"] = string(encodedConfig)
if err := r.Update(context.Background(), &configMap); err != nil {
return err
}

return nil
}

func (r *InspectorReconciler) configureGazer(ctx context.Context, inspector *lazykoalav1alpha1.Inspector) (map[string]ScrapePoint, error) {
logger := log.FromContext(ctx)

// Get the intended deployment
var deploymentRef appsv1.Deployment
if err := r.Get(ctx, types.NamespacedName{
Namespace: inspector.Spec.Namespace,
Name: inspector.Spec.DeploymentRef,
}, &deploymentRef); err != nil {
return ctrl.Result{}, err
return nil, err
}

scrapePoints := make(map[string]ScrapePoint)
Expand All @@ -127,7 +201,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
var podList v1.PodList
if err := r.List(ctx, &podList, &selector); client.IgnoreNotFound(err) != nil {
logger.Error(err, fmt.Sprintf("failed to pods for deployment %s", deploymentRef.ObjectMeta.Name))
return ctrl.Result{}, err
return nil, err
}

// Create Scrape point for each pod
Expand All @@ -147,7 +221,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
Namespace: inspector.Spec.Namespace,
Name: inspector.Spec.ServiceRef,
}, &serviceRef); err != nil {
return ctrl.Result{}, err
return nil, err
}

scrapePoints[serviceRef.Spec.ClusterIP] = ScrapePoint{
Expand All @@ -159,91 +233,89 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}

// Get the Gazer config file
var configMap v1.ConfigMap
var gazerConfigMap v1.ConfigMap
if err := r.Get(ctx, types.NamespacedName{
Namespace: "lazy-koala",
Name: "gazer-config",
}, &configMap); err != nil {
return ctrl.Result{}, err
}, &gazerConfigMap); err != nil {
return nil, err
}

// Phase the config.yaml
configData := make(map[string]ScrapePoint)
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
return ctrl.Result{}, err
gazerData := make(map[string]ScrapePoint)
if err := yaml.Unmarshal([]byte(gazerConfigMap.Data["config.yaml"]), &gazerData); err != nil {
return nil, err
}

// Remove all the existing scrape points created from this Inspector
for _, ip := range inspector.Status.MonitoredIPs {
if _, ok := configData[ip]; ok {
delete(configData, ip)
if _, ok := gazerData[ip]; ok {
delete(gazerData, ip)
}
}

// Add the new scrape points
for k, v := range scrapePoints {
configData[k] = v
gazerData[k] = v
}

// Encode the config.yaml
encodedConfig, err := yaml.Marshal(&configData)
encodedConfig, err := yaml.Marshal(&gazerData)
if err != nil {
return ctrl.Result{}, err
return nil, err
}

// Patch the config file
configMap.Data["config.yaml"] = string(encodedConfig)
if err := r.Update(ctx, &configMap); err != nil {
return ctrl.Result{}, err
gazerConfigMap.Data["config.yaml"] = string(encodedConfig)
if err := r.Update(ctx, &gazerConfigMap); err != nil {
return nil, err
}

// Update local status
var MonitoredIPs []string
for k := range scrapePoints {
MonitoredIPs = append(MonitoredIPs, k)
}
inspector.Status.MonitoredIPs = MonitoredIPs
inspector.Status.Status = lazykoalav1alpha1.Running

if err := r.Status().Update(ctx, &inspector); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{RequeueAfter: time.Minute}, nil
return scrapePoints, nil
}

func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error {
// Get the Gazer config file
var configMap v1.ConfigMap
if err := r.Get(context.Background(), types.NamespacedName{
func (r *InspectorReconciler) configureSherlock(ctx context.Context, inspector *lazykoalav1alpha1.Inspector, append bool) error {
// Get the Sherlock config file
var sherlockConfigMap v1.ConfigMap
if err := r.Get(ctx, types.NamespacedName{
Namespace: "lazy-koala",
Name: "gazer-config",
}, &configMap); err != nil {
Name: "sherlock-config",
}, &sherlockConfigMap); err != nil {
return err
}

// Phase the config.yaml
configData := make(map[string]ScrapePoint)
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
// Phase the services.yaml
var sherlockServiceList []string
if err := yaml.Unmarshal([]byte(sherlockConfigMap.Data["services.yaml"]), &sherlockServiceList); err != nil {
return err
}

// Remove all the existing scrape points created from this Inspector
for _, ip := range inspector.Status.MonitoredIPs {
if _, ok := configData[ip]; ok {
delete(configData, ip)
}
if append {
sherlockServiceList = AppendIfMissing(sherlockServiceList, inspector.Spec.DeploymentRef)
} else {
sherlockServiceList = RemoveIfExists(sherlockServiceList, inspector.Spec.DeploymentRef)
}

// Encode the config.yaml
encodedConfig, err := yaml.Marshal(&configData)
// Generate the Servings Config
servingsConfig, err := createServingsConfig(sherlockServiceList)
if err != nil {
return err
}

// Patch the config file
configMap.Data["config.yaml"] = string(encodedConfig)
if err := r.Update(context.Background(), &configMap); err != nil {
if err := r.Update(ctx, &sherlockConfigMap); err != nil {
return err
}

// Encode the services.yaml
encodedSherlockConfig, err := yaml.Marshal(&sherlockServiceList)
if err != nil {
return err
}

sherlockConfigMap.Data["services.yaml"] = string(encodedSherlockConfig)
sherlockConfigMap.Data["models.config"] = servingsConfig

if err := r.Update(ctx, &sherlockConfigMap); err != nil {
return err
}
return nil
Expand All @@ -265,3 +337,44 @@ func eventFilter() predicate.Predicate {
},
}
}

func AppendIfMissing(slice []string, item string) []string {
for _, ele := range slice {
if ele == item {
return slice
}
}
return append(slice, item)
}

func RemoveIfExists(slice []string, item string) []string {
for i, other := range slice {
if other == item {
return append(slice[:i], slice[i+1:]...)
}
}
return slice
}

func createServingsConfig(service []string) (string, error) {
tmpl := template.New("config")

tmpl, err := tmpl.Parse(`model_config_list {
{{range .}}
config {
name: '{{.}}'
base_path: '/models/{{.}}/'
model_platform: 'tensorflow'
}
{{end}}
}`)
if err != nil {
return "", err
}
buf := new(bytes.Buffer)
err = tmpl.Execute(buf, service)
if err != nil {
return "", err
}
return fmt.Sprintf("%+v", buf), nil
}
Loading

0 comments on commit c341c76

Please sign in to comment.