From 899906be7846e20edcfae51508f0c757fbd20cc7 Mon Sep 17 00:00:00 2001 From: Amir Blum Date: Fri, 10 Jan 2025 20:30:42 +0200 Subject: [PATCH] feat: create effective-config configmap (#2176) The "odigos-config" configmap in odigos is currently treated as "user" object, containing raw configuration as the user supplied it. It can come from few places: - cli (via install) - manual edits of the cm - gitops (forcing the content) - ui (not at the moment, but in the future) This PR introduces a new "effective-config" configmap that is reconciled in schedualer. Having a reconciliation step will allow us to: - consolidate duplicate code in cli and helm that does defaulting, maintaining the default values and merging in one place and update regardless of the source of these values - ability to apply profiles as odigos-config changes - removing any config handling specific logic from consumers (like defaulting, applying profiles, merging different options), o they will get the digested values ready to use - good visibility into the reconciled values that effectively used by various odigos components after they are processed --- cli/cmd/resources/README.md | 3 +- cli/cmd/resources/instrumentor.go | 9 +- cli/cmd/resources/odiglet.go | 2 +- cli/cmd/resources/scheduler.go | 11 ++- common/consts/consts.go | 1 + frontend/services/namespaces.go | 2 +- helm/odigos/templates/instrumentor/role.yaml | 2 +- helm/odigos/templates/odiglet/role.yaml | 2 +- helm/odigos/templates/scheduler/role.yaml | 10 +- .../controllers/startlangdetection/manager.go | 2 +- instrumentor/main.go | 2 +- k8sutils/pkg/predicate/objectname.go | 4 + k8sutils/pkg/utils/config_util.go | 2 +- odiglet/pkg/kube/manager.go | 4 +- .../clustercollectorsgroup/manager.go | 2 +- .../nodecollectorsgroup/manager.go | 2 +- scheduler/controllers/odigosconfig/manager.go | 24 +++++ .../odigosconfig/odigosconfig_controller.go | 96 +++++++++++++++++++ scheduler/go.mod | 2 +- scheduler/main.go | 12 ++- 20 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 scheduler/controllers/odigosconfig/manager.go create mode 100644 scheduler/controllers/odigosconfig/odigosconfig_controller.go diff --git a/cli/cmd/resources/README.md b/cli/cmd/resources/README.md index 8e6bdbc755..86ba82877f 100644 --- a/cli/cmd/resources/README.md +++ b/cli/cmd/resources/README.md @@ -46,7 +46,8 @@ In this doc, we'll keep track of the permissions requested across different reso | Odiglet | "" | configmaps | get, list, watch | Reads `odigos_config` for ignored containers. | | Instrumentor | "" | configmaps | get, list, watch | Accesses `odigos-config` for instrumentation configuration. | | Instrumentor | odigos.io | collectorsgroups | get, list, watch | Monitors collectors and their statuses. | -| Scheduler | "" | configmaps | get, list, watch | Reads configuration details from `odigos-config`. | +| Scheduler | "" | configmaps | get, list, watch | react and reconcile `odigos-config` changes to effective config. | +| Scheduler | "" | configmaps | get, list, watch, patch | apply effective config after reconciling (defaulting and profile applying) and react to it | | Scheduler | odigos.io | collectorsgroups | get, list, create, patch, update, watch, delete | Manages `collectorsgroups`. | | Scheduler | odigos.io | destinations | get, list, watch | Tracks destinations for scheduling behavior. | | Autoscaler | "" | configmaps, services | get, list, watch, create, patch, update, delete, deletecollection | Manages collector configurations and services. | diff --git a/cli/cmd/resources/instrumentor.go b/cli/cmd/resources/instrumentor.go index 0140a7905e..9f05259157 100644 --- a/cli/cmd/resources/instrumentor.go +++ b/cli/cmd/resources/instrumentor.go @@ -9,8 +9,9 @@ import ( "github.com/odigos-io/odigos/cli/pkg/crypto" "github.com/odigos-io/odigos/cli/pkg/kube" "github.com/odigos-io/odigos/common" + "github.com/odigos-io/odigos/common/consts" - "github.com/odigos-io/odigos/k8sutils/pkg/consts" + k8sutilsconsts "github.com/odigos-io/odigos/k8sutils/pkg/consts" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -88,7 +89,7 @@ func NewInstrumentorRole(ns string) *rbacv1.Role { { APIGroups: []string{""}, Resources: []string{"configmaps"}, - ResourceNames: []string{"odigos-config"}, + ResourceNames: []string{consts.OdigosEffectiveConfigName}, Verbs: []string{"get", "list", "watch"}, }, { @@ -289,7 +290,7 @@ func NewMutatingWebhookConfiguration(ns string, caBundle []byte) *admissionregis TimeoutSeconds: intPtr(10), ObjectSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - consts.OdigosInjectInstrumentationLabel: "true", + k8sutilsconsts.OdigosInjectInstrumentationLabel: "true", }, }, AdmissionReviewVersions: []string{ @@ -411,7 +412,7 @@ func NewInstrumentorDeployment(ns string, version string, telemetryEnabled bool, { ConfigMapRef: &corev1.ConfigMapEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: consts.OdigosDeploymentConfigMapName, + Name: k8sutilsconsts.OdigosDeploymentConfigMapName, }, }, }, diff --git a/cli/cmd/resources/odiglet.go b/cli/cmd/resources/odiglet.go index fdfdba5c47..3034699a1c 100644 --- a/cli/cmd/resources/odiglet.go +++ b/cli/cmd/resources/odiglet.go @@ -71,7 +71,7 @@ func NewOdigletRole(ns string) *rbacv1.Role { { // Needed to read the odigos_config for ignored containers APIGroups: []string{""}, Resources: []string{"configmaps"}, - ResourceNames: []string{consts.OdigosConfigurationName}, + ResourceNames: []string{consts.OdigosEffectiveConfigName}, Verbs: []string{"get", "list", "watch"}, }, }, diff --git a/cli/cmd/resources/scheduler.go b/cli/cmd/resources/scheduler.go index bfb338be9a..3861b47316 100644 --- a/cli/cmd/resources/scheduler.go +++ b/cli/cmd/resources/scheduler.go @@ -78,11 +78,16 @@ func NewSchedulerRole(ns string) *rbacv1.Role { Namespace: ns, }, Rules: []rbacv1.PolicyRule{ - { // Needed to extract the configmap of odigos-config + { // Needed to react and reconcile odigos-config changes to effective config + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + { // Needed to apply effective config after reconciling (defaulting and profile applying) and react to it APIGroups: []string{""}, Resources: []string{"configmaps"}, - ResourceNames: []string{consts.OdigosConfigurationName}, - Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{consts.OdigosEffectiveConfigName}, + Verbs: []string{"patch"}, }, { // Needed because the scheduler is managing the collectorsgroups APIGroups: []string{"odigos.io"}, diff --git a/common/consts/consts.go b/common/consts/consts.go index 65f60dd62f..62c6a61521 100644 --- a/common/consts/consts.go +++ b/common/consts/consts.go @@ -8,6 +8,7 @@ const ( CurrentNamespaceEnvVar = "CURRENT_NS" DefaultOdigosNamespace = "odigos-system" OdigosConfigurationName = "odigos-config" + OdigosEffectiveConfigName = "effective-config" OdigosConfigurationFileName = "config.yaml" OTLPPort = 4317 OTLPHttpPort = 4318 diff --git a/frontend/services/namespaces.go b/frontend/services/namespaces.go index d7be276abe..6093b449dc 100644 --- a/frontend/services/namespaces.go +++ b/frontend/services/namespaces.go @@ -86,7 +86,7 @@ func getRelevantNameSpaces(ctx context.Context, odigosns string) ([]v1.Namespace g, ctx := errgroup.WithContext(ctx) g.Go(func() error { var err error - configMap, err := kube.DefaultClient.CoreV1().ConfigMaps(odigosns).Get(ctx, consts.OdigosConfigurationName, metav1.GetOptions{}) + configMap, err := kube.DefaultClient.CoreV1().ConfigMaps(odigosns).Get(ctx, consts.OdigosEffectiveConfigName, metav1.GetOptions{}) if err != nil { return err } diff --git a/helm/odigos/templates/instrumentor/role.yaml b/helm/odigos/templates/instrumentor/role.yaml index c3df9c2fb0..bbfbd5ec60 100644 --- a/helm/odigos/templates/instrumentor/role.yaml +++ b/helm/odigos/templates/instrumentor/role.yaml @@ -7,7 +7,7 @@ rules: - apiGroups: - '' resourceNames: - - odigos-config + - effective-config resources: - configmaps verbs: diff --git a/helm/odigos/templates/odiglet/role.yaml b/helm/odigos/templates/odiglet/role.yaml index fec67cf6f0..6d7f3b10fa 100644 --- a/helm/odigos/templates/odiglet/role.yaml +++ b/helm/odigos/templates/odiglet/role.yaml @@ -16,7 +16,7 @@ rules: - apiGroups: - '' resourceNames: - - odigos-config + - effective-config resources: - configmaps verbs: diff --git a/helm/odigos/templates/scheduler/role.yaml b/helm/odigos/templates/scheduler/role.yaml index 85bef445dd..6210ae7718 100644 --- a/helm/odigos/templates/scheduler/role.yaml +++ b/helm/odigos/templates/scheduler/role.yaml @@ -6,14 +6,20 @@ metadata: rules: - apiGroups: - '' - resourceNames: - - odigos-config resources: - configmaps verbs: - get - list - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - effective-config + verbs: + - patch - apiGroups: - odigos.io resources: diff --git a/instrumentor/controllers/startlangdetection/manager.go b/instrumentor/controllers/startlangdetection/manager.go index 2be2646cdd..027a43bcbb 100644 --- a/instrumentor/controllers/startlangdetection/manager.go +++ b/instrumentor/controllers/startlangdetection/manager.go @@ -65,7 +65,7 @@ func SetupWithManager(mgr ctrl.Manager) error { ControllerManagedBy(mgr). Named("startlangdetection-configmaps"). For(&corev1.ConfigMap{}). - WithEventFilter(predicate.And(odigospredicate.OdigosConfigMapPredicate, odigospredicate.ConfigMapDataChangedPredicate{})). + WithEventFilter(predicate.And(odigospredicate.OdigosEffectiveConfigMapPredicate, odigospredicate.ConfigMapDataChangedPredicate{})). Complete(&OdigosConfigReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/instrumentor/main.go b/instrumentor/main.go index b979f7bb01..890a3d3f75 100644 --- a/instrumentor/main.go +++ b/instrumentor/main.go @@ -102,7 +102,7 @@ func main() { odigosNs := env.GetCurrentNamespace() nsSelector := client.InNamespace(odigosNs).AsSelector() - odigosConfigNameSelector := fields.OneTermEqualSelector("metadata.name", consts.OdigosConfigurationName) + odigosConfigNameSelector := fields.OneTermEqualSelector("metadata.name", consts.OdigosEffectiveConfigName) odigosConfigSelector := fields.AndSelectors(nsSelector, odigosConfigNameSelector) mgrOptions := ctrl.Options{ diff --git a/k8sutils/pkg/predicate/objectname.go b/k8sutils/pkg/predicate/objectname.go index 2c28bdaa93..20c7657839 100644 --- a/k8sutils/pkg/predicate/objectname.go +++ b/k8sutils/pkg/predicate/objectname.go @@ -55,6 +55,10 @@ var OdigosConfigMapPredicate = ObjectNamePredicate{ AllowedObjectName: consts.OdigosConfigurationName, } +var OdigosEffectiveConfigMapPredicate = ObjectNamePredicate{ + AllowedObjectName: consts.OdigosEffectiveConfigName, +} + // use this event filter to reconcile only collectors group events for node collectors group objects // this is useful if you reconcile only depends on changes from the node collectors group and should not react to cluster collectors group changes // example usage: diff --git a/k8sutils/pkg/utils/config_util.go b/k8sutils/pkg/utils/config_util.go index 3845f60ec4..aa196241c5 100644 --- a/k8sutils/pkg/utils/config_util.go +++ b/k8sutils/pkg/utils/config_util.go @@ -16,7 +16,7 @@ func GetCurrentOdigosConfig(ctx context.Context, k8sClient client.Client) (commo var configMap v1.ConfigMap var odigosConfig common.OdigosConfiguration odigosSystemNamespaceName := env.GetCurrentNamespace() - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: odigosSystemNamespaceName, Name: consts.OdigosConfigurationName}, &configMap); err != nil { + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: odigosSystemNamespaceName, Name: consts.OdigosEffectiveConfigName}, &configMap); err != nil { return odigosConfig, err } if err := yaml.Unmarshal([]byte(configMap.Data[consts.OdigosConfigurationFileName]), &odigosConfig); err != nil { diff --git a/odiglet/pkg/kube/manager.go b/odiglet/pkg/kube/manager.go index 8264fc6c96..79be059eaa 100644 --- a/odiglet/pkg/kube/manager.go +++ b/odiglet/pkg/kube/manager.go @@ -49,8 +49,8 @@ func CreateManager() (ctrl.Manager, error) { odigosNs := env.Current.Namespace nsSelector := client.InNamespace(odigosNs).AsSelector() - nameSelector := fields.OneTermEqualSelector("metadata.name", consts.OdigosConfigurationName) - odigosConfigSelector := fields.AndSelectors(nsSelector, nameSelector) + odigosConfigNameSelector := fields.OneTermEqualSelector("metadata.name", consts.OdigosEffectiveConfigName) + odigosConfigSelector := fields.AndSelectors(nsSelector, odigosConfigNameSelector) currentNodeSelector := fields.OneTermEqualSelector("spec.nodeName", env.Current.NodeName) return manager.New(config.GetConfigOrDie(), manager.Options{ diff --git a/scheduler/controllers/clustercollectorsgroup/manager.go b/scheduler/controllers/clustercollectorsgroup/manager.go index a0d398ff0a..0ca9662354 100644 --- a/scheduler/controllers/clustercollectorsgroup/manager.go +++ b/scheduler/controllers/clustercollectorsgroup/manager.go @@ -24,7 +24,7 @@ func SetupWithManager(mgr ctrl.Manager) error { err = ctrl.NewControllerManagedBy(mgr). For(&corev1.ConfigMap{}). Named("clustercollectorgroup-odigosconfig"). - WithEventFilter(&odigospredicates.OdigosConfigMapPredicate). + WithEventFilter(&odigospredicates.OdigosEffectiveConfigMapPredicate). Complete(&odigosConfigController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/scheduler/controllers/nodecollectorsgroup/manager.go b/scheduler/controllers/nodecollectorsgroup/manager.go index de9653d312..2ef537be71 100644 --- a/scheduler/controllers/nodecollectorsgroup/manager.go +++ b/scheduler/controllers/nodecollectorsgroup/manager.go @@ -25,7 +25,7 @@ func SetupWithManager(mgr ctrl.Manager) error { err = ctrl.NewControllerManagedBy(mgr). For(&corev1.ConfigMap{}). Named("nodecollectorgroup-odigosconfig"). - WithEventFilter(&odigospredicates.OdigosConfigMapPredicate). + WithEventFilter(&odigospredicates.OdigosEffectiveConfigMapPredicate). Complete(&odigosConfigController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/scheduler/controllers/odigosconfig/manager.go b/scheduler/controllers/odigosconfig/manager.go new file mode 100644 index 0000000000..1976d76d56 --- /dev/null +++ b/scheduler/controllers/odigosconfig/manager.go @@ -0,0 +1,24 @@ +package odigosconfig + +import ( + odigospredicates "github.com/odigos-io/odigos/k8sutils/pkg/predicate" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +func SetupWithManager(mgr ctrl.Manager) error { + + err := ctrl.NewControllerManagedBy(mgr). + For(&corev1.ConfigMap{}). + Named("odigosconfig-odigosconfig"). + WithEventFilter(&odigospredicates.OdigosConfigMapPredicate). + Complete(&odigosConfigController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }) + if err != nil { + return err + } + + return nil +} diff --git a/scheduler/controllers/odigosconfig/odigosconfig_controller.go b/scheduler/controllers/odigosconfig/odigosconfig_controller.go new file mode 100644 index 0000000000..69d1507249 --- /dev/null +++ b/scheduler/controllers/odigosconfig/odigosconfig_controller.go @@ -0,0 +1,96 @@ +package odigosconfig + +import ( + "context" + + "github.com/odigos-io/odigos/common" + "github.com/odigos-io/odigos/common/consts" + "github.com/odigos-io/odigos/k8sutils/pkg/env" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +type odigosConfigController struct { + client.Client + Scheme *runtime.Scheme +} + +func (r *odigosConfigController) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { + + odigosConfig, err := r.getOdigosConfigUserObject(ctx) + if err != nil { + return ctrl.Result{}, err + } + + err = r.persistEffectiveConfig(ctx, odigosConfig) + if err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *odigosConfigController) getOdigosConfigUserObject(ctx context.Context) (*common.OdigosConfiguration, error) { + var configMap corev1.ConfigMap + var odigosConfig common.OdigosConfiguration + odigosNs := env.GetCurrentNamespace() + + // read current content in odigos-config, which is the content supplied by the user. + // this is the baseline for reconciling, without defaults and profiles applied. + err := r.Client.Get(ctx, types.NamespacedName{Namespace: odigosNs, Name: consts.OdigosConfigurationName}, &configMap) + if err != nil { + return nil, err + } + err = yaml.Unmarshal([]byte(configMap.Data[consts.OdigosConfigurationFileName]), &odigosConfig) + if err != nil { + return nil, err + } + + return &odigosConfig, nil +} + +func (r *odigosConfigController) persistEffectiveConfig(ctx context.Context, effectiveConfig *common.OdigosConfiguration) error { + odigosNs := env.GetCurrentNamespace() + + // apply patch the OdigosEffectiveConfigName configmap with the effective configuration + // this is the configuration after applying defaults and profiles. + + effectiveConfigYamlText, err := yaml.Marshal(effectiveConfig) + if err != nil { + return err + } + + effectiveConfigMap := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: odigosNs, + Name: consts.OdigosEffectiveConfigName, + }, + Data: map[string]string{ + consts.OdigosConfigurationFileName: string(effectiveConfigYamlText), + }, + } + + objApplyBytes, err := yaml.Marshal(effectiveConfigMap) + if err != nil { + return err + } + + err = r.Client.Patch(ctx, &effectiveConfigMap, client.RawPatch(types.ApplyYAMLPatchType, objApplyBytes), client.ForceOwnership, client.FieldOwner("scheduler-odigosconfig")) + if err != nil { + return err + } + + logger := ctrl.LoggerFrom(ctx) + logger.Info("Successfully persisted effective configuration") + + return nil +} diff --git a/scheduler/go.mod b/scheduler/go.mod index 4eec73768a..eaba0939fd 100644 --- a/scheduler/go.mod +++ b/scheduler/go.mod @@ -14,6 +14,7 @@ require ( k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -87,7 +88,6 @@ require ( k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) replace ( diff --git a/scheduler/main.go b/scheduler/main.go index 666668e144..7eec35ebfe 100644 --- a/scheduler/main.go +++ b/scheduler/main.go @@ -31,10 +31,8 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1" - "github.com/odigos-io/odigos/common/consts" "github.com/odigos-io/odigos/k8sutils/pkg/env" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -44,6 +42,7 @@ import ( "github.com/odigos-io/odigos/scheduler/controllers/clustercollectorsgroup" "github.com/odigos-io/odigos/scheduler/controllers/nodecollectorsgroup" + "github.com/odigos-io/odigos/scheduler/controllers/odigosconfig" //+kubebuilder:scaffold:imports ) @@ -80,8 +79,6 @@ func main() { odigosNs := env.GetCurrentNamespace() nsSelector := client.InNamespace(odigosNs).AsSelector() - nameSelector := fields.OneTermEqualSelector("metadata.name", consts.OdigosConfigurationName) - odigosConfigSelector := fields.AndSelectors(nsSelector, nameSelector) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, @@ -92,7 +89,7 @@ func main() { DefaultTransform: cache.TransformStripManagedFields(), ByObject: map[client.Object]cache.ByObject{ &corev1.ConfigMap{}: { - Field: odigosConfigSelector, + Field: nsSelector, }, &odigosv1.CollectorsGroup{}: { Field: nsSelector, @@ -121,6 +118,11 @@ func main() { setupLog.Error(err, "unable to create controllers for node collectors group") os.Exit(1) } + err = odigosconfig.SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controllers for odigos config") + os.Exit(1) + } if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check")