From 55fb9faeca46f6348cb65c43e668ee8d4e2e9273 Mon Sep 17 00:00:00 2001 From: Isala Piyarisi Date: Mon, 14 Feb 2022 17:43:21 +0530 Subject: [PATCH] feat(control-plane): Implemented the core functionality --- .github/workflows/build-controller.yaml | 28 +++ control-plane/Makefile | 2 +- control-plane/api/v1alpha1/inspector_types.go | 15 +- .../api/v1alpha1/zz_generated.deepcopy.go | 30 ++- ...zykoala.lazykoala.isala.me_inspectors.yaml | 63 ++++-- control-plane/config/rbac/role.yaml | 27 +++ control-plane/config/samples/helper.yaml | 12 + .../samples/lazykoala_v1alpha1_inspector.yaml | 6 +- .../controllers/inspector_controller.go | 211 +++++++++++++++++- 9 files changed, 358 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/build-controller.yaml create mode 100644 control-plane/config/samples/helper.yaml diff --git a/.github/workflows/build-controller.yaml b/.github/workflows/build-controller.yaml new file mode 100644 index 0000000..8897472 --- /dev/null +++ b/.github/workflows/build-controller.yaml @@ -0,0 +1,28 @@ +name: controller + +on: + push: + branches: + - main + +env: + IMAGE_NAME: ghcr.io/mrsupiri/lazy-koala/controller + DOCKER_BUILDKIT: 1 + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: control-plane + steps: + - uses: actions/checkout@v2 + - uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: docker build -t $IMAGE_NAME:latest . + - run: docker tag $IMAGE_NAME:latest $IMAGE_NAME:commit-${GITHUB_SHA:0:8} + - run: docker push $IMAGE_NAME --all-tags diff --git a/control-plane/Makefile b/control-plane/Makefile index 5592f2c..b76bec7 100644 --- a/control-plane/Makefile +++ b/control-plane/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= ghcr.io/mrsupiri/lazy-koala/controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.23 diff --git a/control-plane/api/v1alpha1/inspector_types.go b/control-plane/api/v1alpha1/inspector_types.go index bd0d46b..82c87e6 100644 --- a/control-plane/api/v1alpha1/inspector_types.go +++ b/control-plane/api/v1alpha1/inspector_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -35,8 +36,10 @@ type InspectorSpec struct { // Important: Run "make" to regenerate code after modifying this file // Foo is an example field of Inspector. Edit inspector_types.go to remove/update - Service DeploymentReference `json:"service"` - ModelURI string `json:"modelURI"` + DeploymentRef string `json:"deploymentRef"` + ServiceRef string `json:"serviceRef"` + Namespace string `json:"namespace"` + ModelURI string `json:"modelURI"` } type Status string @@ -51,13 +54,19 @@ const ( type InspectorStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file - Inspector DeploymentReference `json:"inspector"` + MonitoredIPs []string `json:"monitoredIPs"` + PodsSelector client.MatchingLabels `json:"podsSelector"` // +kubebuilder:validation:Enum=Creating;Running;Error Status Status `json:"status"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status +//+kubebuilder:printcolumn:JSONPath=".spec.namespace",name="Namespace",type="string" +//+kubebuilder:printcolumn:JSONPath=".spec.deploymentRef",name="Target Deployment",type="string" +//+kubebuilder:printcolumn:JSONPath=".spec.serviceRef",name="Target ClusterIP",type="string" +//+kubebuilder:printcolumn:JSONPath=".spec.modelURI",name="Model URI",type="string" +//+kubebuilder:printcolumn:JSONPath=".status.status",name="Status",type="string" // Inspector is the Schema for the inspectors API type Inspector struct { diff --git a/control-plane/api/v1alpha1/zz_generated.deepcopy.go b/control-plane/api/v1alpha1/zz_generated.deepcopy.go index 7f1decc..52e8e68 100644 --- a/control-plane/api/v1alpha1/zz_generated.deepcopy.go +++ b/control-plane/api/v1alpha1/zz_generated.deepcopy.go @@ -23,15 +23,31 @@ package v1alpha1 import ( runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentReference) DeepCopyInto(out *DeploymentReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentReference. +func (in *DeploymentReference) DeepCopy() *DeploymentReference { + if in == nil { + return nil + } + out := new(DeploymentReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Inspector) DeepCopyInto(out *Inspector) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Inspector. @@ -102,6 +118,18 @@ func (in *InspectorSpec) DeepCopy() *InspectorSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InspectorStatus) DeepCopyInto(out *InspectorStatus) { *out = *in + if in.MonitoredIPs != nil { + in, out := &in.MonitoredIPs, &out.MonitoredIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PodsSelector != nil { + in, out := &in.PodsSelector, &out.PodsSelector + *out = make(client.MatchingLabels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InspectorStatus. diff --git a/control-plane/config/crd/bases/lazykoala.lazykoala.isala.me_inspectors.yaml b/control-plane/config/crd/bases/lazykoala.lazykoala.isala.me_inspectors.yaml index 15d238b..6b807e4 100644 --- a/control-plane/config/crd/bases/lazykoala.lazykoala.isala.me_inspectors.yaml +++ b/control-plane/config/crd/bases/lazykoala.lazykoala.isala.me_inspectors.yaml @@ -15,7 +15,23 @@ spec: singular: inspector scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.namespace + name: Namespace + type: string + - jsonPath: .spec.deploymentRef + name: Target Deployment + type: string + - jsonPath: .spec.serviceRef + name: Target ClusterIP + type: string + - jsonPath: .spec.modelURI + name: Model URI + type: string + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: description: Inspector is the Schema for the inspectors API @@ -35,39 +51,37 @@ spec: spec: description: InspectorSpec defines the desired state of Inspector properties: - modelURI: - type: string - service: + deploymentRef: description: Foo is an example field of Inspector. Edit inspector_types.go to remove/update - properties: - name: - type: string - namespace: - type: string - required: - - name - - namespace - type: object + type: string + modelURI: + type: string + namespace: + type: string + serviceRef: + type: string required: + - deploymentRef - modelURI - - service + - namespace + - serviceRef type: object status: description: InspectorStatus defines the observed state of Inspector properties: - inspector: + monitoredIPs: description: 'INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "make" to regenerate code after modifying this file' - properties: - name: - type: string - namespace: - type: string - required: - - name - - namespace + items: + type: string + type: array + podsSelector: + additionalProperties: + type: string + description: MatchingLabels filters the list/delete operation on the + given set of labels. type: object status: enum: @@ -76,7 +90,8 @@ spec: - Error type: string required: - - inspector + - monitoredIPs + - podsSelector - status type: object type: object diff --git a/control-plane/config/rbac/role.yaml b/control-plane/config/rbac/role.yaml index b34709b..248ba18 100644 --- a/control-plane/config/rbac/role.yaml +++ b/control-plane/config/rbac/role.yaml @@ -5,6 +5,33 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + verbs: + - get + - list + - watch - apiGroups: - lazykoala.lazykoala.isala.me resources: diff --git a/control-plane/config/samples/helper.yaml b/control-plane/config/samples/helper.yaml new file mode 100644 index 0000000..987114c --- /dev/null +++ b/control-plane/config/samples/helper.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: lazy-koala +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gazer-config + namespace: lazy-koala +data: + config.yaml: | \ No newline at end of file diff --git a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml index f5360d4..e2adeba 100644 --- a/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml +++ b/control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml @@ -3,8 +3,8 @@ kind: Inspector metadata: name: inspector-sample spec: - service: - name: order-service - namespace: default + deploymentRef: service-2-5e7b1ab0 + serviceRef: service-2-5e7b1ab0 + namespace: default modelURI: path/to/checkpoint.ckpt diff --git a/control-plane/controllers/inspector_controller.go b/control-plane/controllers/inspector_controller.go index 31e36b7..5bdaba9 100644 --- a/control-plane/controllers/inspector_controller.go +++ b/control-plane/controllers/inspector_controller.go @@ -18,11 +18,19 @@ package controllers import ( "context" - + "fmt" + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "time" lazykoalav1alpha1 "github.com/MrSupiri/LazyKoala/api/v1alpha1" ) @@ -33,9 +41,20 @@ type InspectorReconciler struct { Scheme *runtime.Scheme } +type ScrapePoint struct { + Name string `yaml:"name"` + ServiceName string `yaml:"serviceName"` + Namespace string `yaml:"namespace"` + Node *string `yaml:"node"` + IsService bool `yaml:"isService"` +} + //+kubebuilder:rbac:groups=lazykoala.lazykoala.isala.me,resources=inspectors,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=lazykoala.lazykoala.isala.me,resources=inspectors/status,verbs=get;update;patch //+kubebuilder:rbac:groups=lazykoala.lazykoala.isala.me,resources=inspectors/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=pods;services;namespaces,verbs=get;watch;list +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;watch;list;update;patch +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -47,16 +66,200 @@ type InspectorReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.0/pkg/reconcile func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + logger := log.FromContext(ctx) + + logger.Info("Reconciling") + + var inspector lazykoalav1alpha1.Inspector + if err := r.Get(ctx, req.NamespacedName, &inspector); err != nil { + return ctrl.Result{Requeue: false}, client.IgnoreNotFound(err) + } + + // name of our custom finalizer + finalizerName := "inspector.lazykoala.isala.me/finalizer" + // examine DeletionTimestamp to determine if object is under deletion + if inspector.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !controllerutil.ContainsFinalizer(&inspector, finalizerName) { + controllerutil.AddFinalizer(&inspector, finalizerName) + if err := r.Update(ctx, &inspector); err != nil { + return ctrl.Result{}, err + } + } + } else { + // The object is being deleted + if controllerutil.ContainsFinalizer(&inspector, finalizerName) { + // our finalizer is present, so lets handle any external dependency + if err := r.removeMonitoredIPs(&inspector); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return ctrl.Result{}, err + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(&inspector, finalizerName) + if err := r.Update(ctx, &inspector); err != nil { + return ctrl.Result{}, err + } + } + + // Stop reconciliation as the item is being deleted + return ctrl.Result{}, nil + } + // Get the intended deployment + var deploymentRef appsv1.Deployment + if err := r.Get(ctx, types.NamespacedName{ + Namespace: inspector.Spec.Namespace, + Name: inspector.Spec.DeploymentRef, + }, &deploymentRef); err != nil { + return ctrl.Result{}, err + } + + scrapePoints := make(map[string]ScrapePoint) + + // Get Pods for that deployment + selector := client.MatchingLabels(deploymentRef.Spec.Selector.MatchLabels) + inspector.Status.PodsSelector = selector + var podList v1.PodList + if err := r.List(ctx, &podList, &selector); client.IgnoreNotFound(err) != nil { + logger.Error(err, fmt.Sprintf("failed to pods for deployment %s", deploymentRef.ObjectMeta.Name)) + return ctrl.Result{}, err + } + + // Create Scrape point for each pod + for _, pod := range podList.Items { + scrapePoints[pod.Status.PodIP] = ScrapePoint{ + Name: pod.ObjectMeta.Name, + ServiceName: inspector.Spec.DeploymentRef, + Namespace: inspector.Spec.Namespace, + Node: &pod.Spec.NodeName, + IsService: false, + } + } + + // Create Scrape point for Cluster DNS for the deployment + var serviceRef v1.Service + if err := r.Get(ctx, types.NamespacedName{ + Namespace: inspector.Spec.Namespace, + Name: inspector.Spec.ServiceRef, + }, &serviceRef); err != nil { + return ctrl.Result{}, err + } + + scrapePoints[serviceRef.Spec.ClusterIP] = ScrapePoint{ + Name: serviceRef.ObjectMeta.Name, + ServiceName: inspector.Spec.DeploymentRef, + Namespace: inspector.Spec.Namespace, + Node: nil, + IsService: true, + } + + // Get the Gazer config file + var configMap v1.ConfigMap + if err := r.Get(ctx, types.NamespacedName{ + Namespace: "lazy-koala", + Name: "gazer-config", + }, &configMap); err != nil { + return ctrl.Result{}, err + } - // TODO(user): your logic here + // Phase the config.yaml + configData := make(map[string]ScrapePoint) + if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil { + return ctrl.Result{}, err + } - return ctrl.Result{}, nil + // Remove all the existing scrape points created from this Inspector + for _, ip := range inspector.Status.MonitoredIPs { + if _, ok := configData[ip]; ok { + delete(configData, ip) + } + } + + // Add the new scrape points + for k, v := range scrapePoints { + configData[k] = v + } + + // Encode the config.yaml + encodedConfig, err := yaml.Marshal(&configData) + if err != nil { + return ctrl.Result{}, err + } + + // Patch the config file + configMap.Data["config.yaml"] = string(encodedConfig) + if err := r.Update(ctx, &configMap); err != nil { + return ctrl.Result{}, err + } + + // Update local status + var MonitoredIPs []string + for k := range scrapePoints { + MonitoredIPs = append(MonitoredIPs, k) + } + inspector.Status.MonitoredIPs = MonitoredIPs + inspector.Status.Status = lazykoalav1alpha1.Running + + if err := r.Status().Update(ctx, &inspector); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Minute}, nil +} + +func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error { + // Get the Gazer config file + var configMap v1.ConfigMap + if err := r.Get(context.Background(), types.NamespacedName{ + Namespace: "lazy-koala", + Name: "gazer-config", + }, &configMap); err != nil { + return err + } + + // Phase the config.yaml + configData := make(map[string]ScrapePoint) + if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil { + return err + } + + // Remove all the existing scrape points created from this Inspector + for _, ip := range inspector.Status.MonitoredIPs { + if _, ok := configData[ip]; ok { + delete(configData, ip) + } + } + + // Encode the config.yaml + encodedConfig, err := yaml.Marshal(&configData) + if err != nil { + return err + } + + // Patch the config file + configMap.Data["config.yaml"] = string(encodedConfig) + if err := r.Update(context.Background(), &configMap); err != nil { + return err + } + return nil } // SetupWithManager sets up the controller with the Manager. func (r *InspectorReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&lazykoalav1alpha1.Inspector{}). + WithEventFilter(eventFilter()). Complete(r) } + +func eventFilter() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Ignore updates to CDR status in which case metadata.Generation does not change + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + } +}