diff --git a/pkg/karmadactl/addons/addons.go b/pkg/karmadactl/addons/addons.go new file mode 100644 index 000000000000..67d719b15aaf --- /dev/null +++ b/pkg/karmadactl/addons/addons.go @@ -0,0 +1,38 @@ +package addons + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/karmada-io/karmada/pkg/karmadactl/addons/install" +) + +var ( + addonsExamples = templates.Examples(` + # Enable or disable Karmada addons to the karmada-host cluster + %[1]s addons enable karmada-search + `) +) + +func init() { + install.Install() +} + +// NewCommandAddons enable or disable Karmada addons on karmada-host cluster +func NewCommandAddons(parentCommand string) *cobra.Command { + cmd := &cobra.Command{ + Use: "addons", + Short: "Enable or disable a Karmada addon", + Long: "Enable or disable a Karmada addon", + Example: fmt.Sprintf(addonsExamples, parentCommand), + } + + addonsParentCommand := fmt.Sprintf("%s %s", parentCommand, "addons") + cmd.AddCommand(NewCmdAddonsList(addonsParentCommand)) + cmd.AddCommand(NewCmdAddonsEnable(addonsParentCommand)) + cmd.AddCommand(NewCmdAddonsDisable(addonsParentCommand)) + + return cmd +} diff --git a/pkg/karmadactl/addons/descheduler/descheduler.go b/pkg/karmadactl/addons/descheduler/descheduler.go new file mode 100644 index 000000000000..6def2e8c252a --- /dev/null +++ b/pkg/karmadactl/addons/descheduler/descheduler.go @@ -0,0 +1,83 @@ +package descheduler + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +var karmadaDeschedulerLabels = map[string]string{"app": addoninit.DeschedulerResourceName} + +// AddonDescheduler describe the descheduler addon command process +var AddonDescheduler = &addoninit.Addon{ + Name: addoninit.DeschedulerResourceName, + Status: status, + Enable: enableDescheduler, + Disable: disableDescheduler, +} + +var status = func(opts *addoninit.CommandAddonsListOption) (string, error) { + deployment, err := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace).Get(context.TODO(), addoninit.DeschedulerResourceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + if deployment.Status.Replicas != deployment.Status.ReadyReplicas || + deployment.Status.Replicas != deployment.Status.AvailableReplicas { + return addoninit.AddonUnhealthyStatus, nil + } + + return addoninit.AddonEnabledStatus, nil +} + +var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error { + // install karmada descheduler deployment on host cluster + karmadaDeschedulerDeploymentBytes, err := addonutils.ParseTemplate(karmadaDeschedulerDeployment, DeploymentReplace{ + Namespace: opts.Namespace, + Replicas: &opts.KarmadaDeschedulerReplicas, + Image: opts.KarmadaDeschedulerImage, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada descheduler deployment template :%v", err) + } + + karmadaDeschedulerDeployment := &appsv1.Deployment{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaDeschedulerDeploymentBytes, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("decode descheduler deployment error: %v", err) + } + + if err := addonutils.CreateOrUpdateDeployment(opts.KubeClientSet, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("create karmada descheduler deployment error: %v", err) + } + + if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaDeschedulerLabels), opts.WaitPodReadyTimeout); err != nil { + return fmt.Errorf("wait karmada descheduler pod timeout: %v", err) + } + + klog.Infof("Install karmada descheduler deployment on host cluster successfully") + return nil +} + +var disableDescheduler = func(opts *addoninit.CommandAddonsDisableOption) error { + // uninstall karmada descheduler deployment on host cluster + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + if err := deployClient.Delete(context.TODO(), addoninit.DeschedulerResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + + klog.Infof("Uninstall karmada descheduler deployment on host cluster successfully") + return nil +} diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go new file mode 100644 index 000000000000..b64c47fd8f49 --- /dev/null +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -0,0 +1,59 @@ +package descheduler + +const karmadaDeschedulerDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-descheduler + namespace: {{ .Namespace }} + labels: + app: karmada-descheduler +spec: + selector: + matchLabels: + app: karmada-descheduler + replicas: {{ .Replicas }} + template: + metadata: + labels: + app: karmada-descheduler + spec: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - name: karmada-descheduler + image: {{ .Image }} + imagePullPolicy: IfNotPresent + command: + - /bin/karmada-descheduler + - --kubeconfig=/etc/kubeconfig + - --bind-address=0.0.0.0 + - --leader-elect-resource-namespace={{ .Namespace }} + - --v=4 + livenessProbe: + httpGet: + path: /healthz + port: 10358 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: kubeconfig + subPath: kubeconfig + mountPath: /etc/kubeconfig + volumes: + - name: kubeconfig + secret: + secretName: kubeconfig +` + +// DeploymentReplace is a struct to help to concrete +// the karamda-descheduler deployment bytes with the deployment template +type DeploymentReplace struct { + Namespace string + Replicas *int32 + Image string +} diff --git a/pkg/karmadactl/addons/disable.go b/pkg/karmadactl/addons/disable.go new file mode 100644 index 000000000000..ac9a175e377f --- /dev/null +++ b/pkg/karmadactl/addons/disable.go @@ -0,0 +1,61 @@ +package addons + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" +) + +var ( + disableExample = templates.Examples(` + # Disable Karmada all addons except karmada-scheduler-estimator on Kubernetes cluster + %[1]s disable all + + # Disable Karmada search on Kubernetes cluster + %[1]s disable karmada-search + + # Disable Karmada search and descheduler on Kubernetes cluster + %[1]s disable karmada-search karmada-descheduler + + # Disable karmada search and scheduler-estimator of member1 cluster to the kubernetes cluster + %[1]s disable karmada-search karmada-scheduler-estimator --cluster member1 + + # Specify the host cluster kubeconfig + %[1]s disable Karmada-search --kubeconfig /root/.kube/config + + # Specify the Karmada control plane kubeconfig + %[1]s disable karmada-search --karmada-kubeconfig /etc/karmada/karmada-apiserver.config + + # Sepcify the namespace where Karmada components are installed + %[1]s disable karmada-search --namespace karmada-system + `) +) + +// NewCmdAddonsDisable disable Karmada addons on Kubernetes +func NewCmdAddonsDisable(parentCommand string) *cobra.Command { + opts := addoninit.CommandAddonsDisableOption{} + cmd := &cobra.Command{ + Use: "disable", + Short: "Disable karmada addons from Kubernetes", + Long: "Disable Karmada addons from Kubernetes", + Example: fmt.Sprintf(disableExample, parentCommand), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if err := opts.Complete(); err != nil { + return err + } + if err := opts.Validate(args); err != nil { + return err + } + if err := opts.Run(args); err != nil { + return err + } + return nil + }, + } + opts.GlobalCommandOptions.AddFlags(cmd.PersistentFlags()) + return cmd +} diff --git a/pkg/karmadactl/addons/enable.go b/pkg/karmadactl/addons/enable.go new file mode 100644 index 000000000000..219cd3a70dd8 --- /dev/null +++ b/pkg/karmadactl/addons/enable.go @@ -0,0 +1,84 @@ +package addons + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + "k8s.io/kubectl/pkg/util/templates" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + "github.com/karmada-io/karmada/pkg/version" +) + +var ( + enableExample = templates.Examples(` + # Enable Karmada all addons except karmada-scheduler-estimator to Kubernetes cluster + %[1]s enable all + + # Enable Karmada search to the Kubernetes cluster + %[1]s enable karmada-search + + # Enable karmada search and descheduler to the kubernetes cluster + %[1]s enable karmada-search karmada-descheduler + + # Enable karmada search and scheduler-estimator of cluster member1 to the kubernetes cluster + %[1]s enable karmada-search karmada-scheduler-estimator -C member1 --member-kubeconfig /etc/karmada/member.config --member-context member1 + + # Specify the host cluster kubeconfig + %[1]s enable karmada-search --kubeconfig /root/.kube/config + + # Specify the Karmada control plane kubeconfig + %[1]s enable karmada-search --karmada-kubeconfig /etc/karmada/karmada-apiserver.config + + # Specify the karmada-search image + %[1]s enable karmada-search --karmada-search-image swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-scheduler-estimator:latest + + # Sepcify the namespace where Karmada components are installed + %[1]s enable karmada-search --namespace karmada-system + `) +) + +// NewCmdAddonsEnable enable Karmada addons on Kubernetes +func NewCmdAddonsEnable(parentCommand string) *cobra.Command { + opts := addoninit.CommandAddonsEnableOption{} + cmd := &cobra.Command{ + Use: "enable", + Short: "Enable Karmada addons from Kubernetes", + Long: "Enable Karmada addons from Kubernetes", + Example: fmt.Sprintf(enableExample, parentCommand), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if err := opts.Complete(); err != nil { + return err + } + if err := opts.Validate(args); err != nil { + return err + } + if err := opts.Run(args); err != nil { + return err + } + return nil + }, + } + + releaseVer, err := version.ParseGitVersion(version.Get().GitVersion) + if err != nil { + klog.Infof("No default release version found. build version: %s", version.Get().String()) + releaseVer = &version.ReleaseVersion{} // initialize to avoid panic + } + + flags := cmd.PersistentFlags() + opts.GlobalCommandOptions.AddFlags(flags) + flags.IntVar(&opts.WaitPodReadyTimeout, "pod-timeout", 30, "Wait pod ready timeout.") + flags.IntVar(&opts.WaitAPIServiceReadyTimeout, "apiservice-timeout", 30, "Wait apiservice ready timeout.") + flags.StringVar(&opts.KarmadaSearchImage, "karmada-search-image", fmt.Sprintf("swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-search:%s", releaseVer.PatchRelease()), "karmada search image") + flags.Int32Var(&opts.KarmadaSearchReplicas, "karmada-search-replicas", 1, "Karmada search replica set") + flags.StringVar(&opts.KarmadaDeschedulerImage, "karmada-descheduler-image", fmt.Sprintf("swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-descheduler:%s", releaseVer.PatchRelease()), "karmada descheduler image") + flags.Int32Var(&opts.KarmadaDeschedulerReplicas, "karmada-descheduler-replicas", 1, "Karmada descheduler replica set") + flags.StringVar(&opts.KarmadaSchedulerEstimatorImage, "karmada-scheduler-estimator-image", fmt.Sprintf("swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-scheduler-estimator:%s", releaseVer.PatchRelease()), "karmada scheduler-estimator image") + flags.Int32Var(&opts.KarmadaEstimatorReplicas, "karmada-estimator-replicas", 1, "Karmada scheduler estimator replica set") + flags.StringVar(&opts.MemberKubeConfig, "member-kubeconfig", "", "Member cluster's kubeconfig which to deploy scheduler estimator") + flags.StringVar(&opts.MemberContext, "member-context", "", "Member cluster's context which to deploy scheduler estimator") + return cmd +} diff --git a/pkg/karmadactl/addons/estimator/estimator.go b/pkg/karmadactl/addons/estimator/estimator.go new file mode 100644 index 000000000000..e5f6cad7f2a6 --- /dev/null +++ b/pkg/karmadactl/addons/estimator/estimator.go @@ -0,0 +1,166 @@ +package estimator + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" + "github.com/karmada-io/karmada/pkg/util/names" +) + +// AddonEstimator describe the estimator addon command process +var AddonEstimator = &addoninit.Addon{ + Name: addoninit.EstimatorResourceName, + Status: status, + Enable: enableEstimator, + Disable: disableEstimator, +} + +var status = func(opts *addoninit.CommandAddonsListOption) (string, error) { + if len(opts.Cluster) == 0 { + return addoninit.AddonUnknownStatus, nil + } + + esName := names.GenerateEstimatorDeploymentName(opts.Cluster) + deployment, err := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace).Get(context.TODO(), esName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + if deployment.Status.Replicas != deployment.Status.ReadyReplicas || + deployment.Status.Replicas != deployment.Status.AvailableReplicas { + return addoninit.AddonUnhealthyStatus, nil + } + + return addoninit.AddonEnabledStatus, nil +} + +var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error { + if len(opts.Cluster) == 0 { + klog.Warning("Cluster is not specified in CommandAddonsEnableOption, estimator installation will skip.") + return nil + } + + pathOptions := &clientcmd.PathOptions{ + LoadingRules: &clientcmd.ClientConfigLoadingRules{ + ExplicitPath: opts.MemberKubeConfig, + }, + } + config, err := pathOptions.GetStartingConfig() + if err != nil { + return err + } + config.CurrentContext = opts.MemberContext + configBytes, err := clientcmd.Write(*config) + if err != nil { + return fmt.Errorf("failure while serializing admin kubeConfig. %v", err) + } + + secretName := fmt.Sprintf("%s-kubeconfig", opts.Cluster) + secret := secretFromSpec(secretName, opts.Namespace, corev1.SecretTypeOpaque, map[string]string{secretName: string(configBytes)}) + if err := addonutils.CreateOrUpdateSecret(opts.KubeClientSet, secret); err != nil { + return fmt.Errorf("create or update scheduler estimator secret error: %v", err) + } + + // init estimator service + karmadaEstimatorServiceBytes, err := addonutils.ParseTemplate(karmadaEstimatorService, ServiceReplace{ + Namespace: opts.Namespace, + MemberClusterName: opts.Cluster, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada scheduler estimator service template :%v", err) + } + + karmadaEstimatorService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaEstimatorServiceBytes, karmadaEstimatorService); err != nil { + return fmt.Errorf("decode karmada scheduler estimator service error: %v", err) + } + if err := addonutils.CreateService(opts.KubeClientSet, karmadaEstimatorService); err != nil { + return fmt.Errorf("create or update scheduler estimator service error: %v", err) + } + + // init estimator deployment + karmadaEstimatorDeploymentBytes, err := addonutils.ParseTemplate(karmadaEstimatorDeployment, DeploymentReplace{ + Namespace: opts.Namespace, + Replicas: &opts.KarmadaEstimatorReplicas, + Image: opts.KarmadaSchedulerEstimatorImage, + MemberClusterName: opts.Cluster, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada scheduler estimator deployment template :%v", err) + } + + karmadaEstimatorDeployment := &appsv1.Deployment{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaEstimatorDeploymentBytes, karmadaEstimatorDeployment); err != nil { + return fmt.Errorf("decode karmada scheduler estimator deployment error: %v", err) + } + if err := addonutils.CreateOrUpdateDeployment(opts.KubeClientSet, karmadaEstimatorDeployment); err != nil { + return fmt.Errorf("create or update scheduler estimator deployment error: %v", err) + } + + karmadaEstimatorLabels := map[string]string{"cluster": opts.Cluster} + if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaEstimatorLabels), opts.WaitPodReadyTimeout); err != nil { + klog.Warning(err) + } + klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster) + + return nil +} + +var disableEstimator = func(opts *addoninit.CommandAddonsDisableOption) error { + if len(opts.Cluster) == 0 { + klog.Warning("Cluster is not specified in CommandAddonsDisableOption, estimator uninstallation will skip.") + return nil + } + + //delete deployment + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + if err := deployClient.Delete(context.TODO(), fmt.Sprintf("%s-%s", addoninit.EstimatorResourceName, opts.Cluster), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Exitln(err) + } + + // delete service + serviceClient := opts.KubeClientSet.CoreV1().Services(opts.Namespace) + if err := serviceClient.Delete(context.TODO(), fmt.Sprintf("%s-%s", addoninit.EstimatorResourceName, opts.Cluster), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Exitln(err) + } + + // delete secret + secretClient := opts.KubeClientSet.CoreV1().Secrets(opts.Namespace) + if err := secretClient.Delete(context.TODO(), fmt.Sprintf("%s-kubeconfig", opts.Cluster), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Exitln(err) + } + + klog.Infof("Karmada scheduler estimator of member cluster %s is removed successfully.", opts.Cluster) + return nil +} + +func secretFromSpec(name string, namespace string, secretType corev1.SecretType, data map[string]string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + //Immutable: immutable, + Type: secretType, + StringData: data, + } +} diff --git a/pkg/karmadactl/addons/estimator/manifests.go b/pkg/karmadactl/addons/estimator/manifests.go new file mode 100644 index 000000000000..ff79cc92064b --- /dev/null +++ b/pkg/karmadactl/addons/estimator/manifests.go @@ -0,0 +1,86 @@ +package estimator + +const ( + karmadaEstimatorDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-scheduler-estimator-{{ .MemberClusterName}} + namespace: {{ .Namespace }} + labels: + cluster: {{ .MemberClusterName}} +spec: + selector: + matchLabels: + app: karmada-scheduler-estimator-{{ .MemberClusterName}} + replicas: {{ .Replicas }} + template: + metadata: + labels: + app: karmada-scheduler-estimator-{{ .MemberClusterName}} + cluster: {{ .MemberClusterName}} + spec: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - name: karmada-scheduler-estimator + image: {{ .Image }} + imagePullPolicy: IfNotPresent + command: + - /bin/karmada-scheduler-estimator + - --kubeconfig=/etc/{{ .MemberClusterName}}-kubeconfig + - --cluster-name={{ .MemberClusterName}} + livenessProbe: + httpGet: + path: /healthz + port: 10351 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: member-kubeconfig + subPath: {{ .MemberClusterName}}-kubeconfig + mountPath: /etc/{{ .MemberClusterName}}-kubeconfig + volumes: + - name: member-kubeconfig + secret: + secretName: {{ .MemberClusterName}}-kubeconfig + +` + + karmadaEstimatorService = ` +apiVersion: v1 +kind: Service +metadata: + name: karmada-scheduler-estimator-{{ .MemberClusterName}} + namespace: {{ .Namespace }} + labels: + cluster: {{ .MemberClusterName}} +spec: + selector: + app: karmada-scheduler-estimator-{{ .MemberClusterName}} + ports: + - protocol: TCP + port: 10352 + targetPort: 10352 +` +) + +// DeploymentReplace is a struct to help to concrete +// the karamda-estimator deployment bytes with the deployment template +type DeploymentReplace struct { + Namespace string + Replicas *int32 + Image string + MemberClusterName string +} + +// ServiceReplace is a struct to help to concrete +// the karamda-estimator Service bytes with the Service template +type ServiceReplace struct { + Namespace string + MemberClusterName string +} diff --git a/pkg/karmadactl/addons/init/addon.go b/pkg/karmadactl/addons/init/addon.go new file mode 100644 index 000000000000..7bcd5c9d17d0 --- /dev/null +++ b/pkg/karmadactl/addons/init/addon.go @@ -0,0 +1,43 @@ +package init + +const ( + // AddonDisabledStatus describe a karmada addon is not installed + AddonDisabledStatus = "disabled" + + // AddonEnabledStatus describe a karmada addon is installed + AddonEnabledStatus = "enabled" + + // AddonUnhealthyStatus describe a karmada addon is unhealthy + AddonUnhealthyStatus = "unhealthy" + + // AddonUnknownStatus describe a karmada addon is unknown + AddonUnknownStatus = "unknown" +) + +const ( + // DeschedulerResourceName define Descheduler Addon and component installed name + DeschedulerResourceName = "karmada-descheduler" + + // EstimatorResourceName define Estimator Addon and component installed name + EstimatorResourceName = "karmada-scheduler-estimator" + + // SearchResourceName define Search Addon and component installed name + SearchResourceName = "karmada-search" +) + +// Addons hosts the optional components that support by karmada +var Addons = map[string]*Addon{} + +// Addon describe how to enable or disable an optional component that support by karmada +type Addon struct { + Name string + + // Status return current addon install status + Status func(opts *CommandAddonsListOption) (string, error) + + // Enable install current addon in host cluster and Karmada control plane + Enable func(opts *CommandAddonsEnableOption) error + + // Disable uninstall current addon in host cluster and Karmada control plane + Disable func(opts *CommandAddonsDisableOption) error +} diff --git a/pkg/karmadactl/addons/init/disable_option.go b/pkg/karmadactl/addons/init/disable_option.go new file mode 100644 index 000000000000..9dcc5b69ec2c --- /dev/null +++ b/pkg/karmadactl/addons/init/disable_option.go @@ -0,0 +1,76 @@ +package init + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" + + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +// CommandAddonsDisableOption options for addons list. +type CommandAddonsDisableOption struct { + GlobalCommandOptions + + KarmadaKubeClientSet *kubernetes.Clientset + + WaitPodReadyTimeout int +} + +// Complete the conditions required to be able to run disable. +func (o *CommandAddonsDisableOption) Complete() error { + err := o.GlobalCommandOptions.Complete() + if err != nil { + return err + } + + o.KarmadaKubeClientSet, err = utils.NewClientSet(o.KarmadaRestConfig) + if err != nil { + return err + } + + return nil +} + +// Validate Check that there are enough conditions to run the disable. +func (o *CommandAddonsDisableOption) Validate(args []string) error { + err := validAddonNames(args) + if err != nil { + return err + } + + if slices.Contains(args, EstimatorResourceName) && o.Cluster == "" { + return fmt.Errorf("member cluster and config is needed when disable karmada-scheduler-estimator,use `--cluster=member --member-kubeconfig /root/.kube/config --member-context member1` to disable karmada-scheduler-estimator") + } + return nil +} + +// Run start disable Karmada addons +func (o *CommandAddonsDisableOption) Run(args []string) error { + var disableAddons = map[string]*Addon{} + + // collect disabled addons + for _, item := range args { + if item == "all" { + disableAddons = Addons + break + } + if addon := Addons[item]; addon != nil { + disableAddons[item] = addon + } + } + + // disable addons + for name, addon := range disableAddons { + klog.Infof("Start to disable addon %s", name) + if err := addon.Disable(o); err != nil { + klog.Errorf("Disable addon %s failed", name) + return err + } + klog.Infof("Successfully disable addon %s", name) + } + + return nil +} diff --git a/pkg/karmadactl/addons/init/enable_option.go b/pkg/karmadactl/addons/init/enable_option.go new file mode 100644 index 000000000000..88766c1f8d89 --- /dev/null +++ b/pkg/karmadactl/addons/init/enable_option.go @@ -0,0 +1,147 @@ +package init + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" + + cmdinit "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +// CommandAddonsEnableOption options for addons list. +type CommandAddonsEnableOption struct { + GlobalCommandOptions + + KarmadaSearchImage string + + KarmadaSearchReplicas int32 + + KarmadaDeschedulerImage string + + KarmadaDeschedulerReplicas int32 + + KarmadaSchedulerEstimatorImage string + + KarmadaEstimatorReplicas int32 + + KarmadaKubeClientSet *kubernetes.Clientset + + WaitPodReadyTimeout int + + WaitAPIServiceReadyTimeout int + + MemberKubeConfig string + + MemberContext string +} + +// Complete the conditions required to be able to run enable. +func (o *CommandAddonsEnableOption) Complete() error { + err := o.GlobalCommandOptions.Complete() + if err != nil { + return err + } + + o.KarmadaKubeClientSet, err = utils.NewClientSet(o.KarmadaRestConfig) + if err != nil { + return err + } + + return nil +} + +// Validate Check that there are enough conditions to run addon enable. +func (o *CommandAddonsEnableOption) Validate(args []string) error { + err := validAddonNames(args) + if err != nil { + return err + } + + secretClient := o.KubeClientSet.CoreV1().Secrets(o.Namespace) + _, err = secretClient.Get(context.TODO(), cmdinit.KubeConfigSecretAndMountName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("secrets `kubeconfig` is not found in namespace %s, please execute karmadactl init to deploy karmada first", o.Namespace) + } + } + + if o.Cluster == "" { + if slices.Contains(args, EstimatorResourceName) { + return fmt.Errorf("member cluster is needed when enable karmada-scheduler-estimator,use `--cluster=member --member-kubeconfig /root/.kube/config --member-context member1` to enable karmada-scheduler-estimator") + } + } else { + if !slices.Contains(args, EstimatorResourceName) && !slices.Contains(args, "all") { + return fmt.Errorf("cluster is needed only when enable karmada-scheduler-estimator or enable all") + } + if o.MemberKubeConfig == "" { + return fmt.Errorf("member config is needed when enable karmada-scheduler-estimator,use `--cluster=member --member-kubeconfig /root/.kube/member.config --member-context member1` to enable karmada-scheduler-estimator") + } + + // Check member kubeconfig and context is valid + memberConfig, err := utils.RestConfig(o.MemberContext, o.MemberKubeConfig) + if err != nil { + return fmt.Errorf("failed to get member cluster config. error: %v", err) + } + memberKubeClient := kubernetes.NewForConfigOrDie(memberConfig) + _, err = memberKubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to get nodes from cluster %s with member-kubeconfig and member-context. error: %v, Please check the Role or ClusterRole of the serviceAccount in your member-kubeconfig", o.Cluster, err) + } + _, err = memberKubeClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to get pods from cluster %s with member-kubeconfig and member-context. error: %v, Please check the Role or ClusterRole of the serviceAccount in your member-kubeconfig", o.Cluster, err) + } + } + return nil +} + +// Run start enable Karmada addons +func (o *CommandAddonsEnableOption) Run(args []string) error { + var enableAddons = map[string]*Addon{} + + // collect enabled addons + for _, item := range args { + if item == "all" { + enableAddons = Addons + break + } + if addon := Addons[item]; addon != nil { + enableAddons[item] = addon + } + } + + // enable addons + for name, addon := range enableAddons { + klog.Infof("Start to enable addon %s", name) + if err := addon.Enable(o); err != nil { + klog.Errorf("Install addon %s failed", name) + return err + } + klog.Infof("Successfully enable addon %s", name) + } + + return nil +} + +// validAddonNames valid whether addon names is supported now +func validAddonNames(addonNames []string) error { + if len(addonNames) == 0 { + return fmt.Errorf("addonNames must be not be null") + } + for _, addonName := range addonNames { + if addonName == "all" { + continue + } + _, ok := Addons[addonName] + if !ok { + return fmt.Errorf("addon %s is not be supported now", addonName) + } + } + return nil +} diff --git a/pkg/karmadactl/addons/init/global.go b/pkg/karmadactl/addons/init/global.go new file mode 100644 index 000000000000..fdab7682b224 --- /dev/null +++ b/pkg/karmadactl/addons/init/global.go @@ -0,0 +1,81 @@ +package init + +import ( + "os" + "path/filepath" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/homedir" + aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +// GlobalCommandOptions holds the configuration shared by the all sub-commands of `karmadactl`. +type GlobalCommandOptions struct { + // KubeConfig holds host cluster KUBECONFIG file path. + KubeConfig string + Context string + + // KubeConfig holds karmada control plane KUBECONFIG file path. + KarmadaConfig string + KarmadaContext string + + // Namespace holds the namespace where Karmada components intalled + Namespace string + + // Cluster holds the name of member cluster to enable or disable scheduler estimator + Cluster string + + KubeClientSet *kubernetes.Clientset + + KarmadaRestConfig *rest.Config + + KarmadaAggregatorClientSet *aggregator.Clientset +} + +// AddFlags adds flags to the specified FlagSet. +func (o *GlobalCommandOptions) AddFlags(flags *pflag.FlagSet) { + flags.StringVarP(&o.Namespace, "namespace", "n", "karmada-system", "namespace where Karmada components are installed.") + flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Path to the host cluster kubeconfig file.") + flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context to use.") + flags.StringVar(&o.KarmadaConfig, "karmada-kubeconfig", "/etc/karmada/karmada-apiserver.config", "Path to the karmada control plane kubeconfig file.") + flags.StringVar(&o.KarmadaContext, "karmada-context", "", "The name of the karmada control plane kubeconfig context to use.") + flags.StringVarP(&o.Cluster, "cluster", "C", "", "The name of member cluster to disable scheduler estimator") +} + +// Complete the conditions required to be able to run list. +func (o *GlobalCommandOptions) Complete() error { + if o.KubeConfig == "" { + env := os.Getenv("KUBECONFIG") + if env != "" { + o.KubeConfig = env + } else { + o.KubeConfig = filepath.Join(homedir.HomeDir(), ".kube", "config") + } + } + + restConfig, err := utils.RestConfig(o.Context, o.KubeConfig) + if err != nil { + return err + } + + o.KubeClientSet, err = utils.NewClientSet(restConfig) + if err != nil { + return err + } + + o.KarmadaRestConfig, err = utils.RestConfig(o.KarmadaContext, o.KarmadaConfig) + if err != nil { + return err + } + + o.KarmadaAggregatorClientSet, err = utils.NewAPIRegistrationClient(o.KarmadaRestConfig) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/karmadactl/addons/init/list_option.go b/pkg/karmadactl/addons/init/list_option.go new file mode 100644 index 000000000000..c9c906eae413 --- /dev/null +++ b/pkg/karmadactl/addons/init/list_option.go @@ -0,0 +1,55 @@ +package init + +import ( + "os" + "sort" + + "github.com/olekukonko/tablewriter" +) + +// CommandAddonsListOption options for addons list. +type CommandAddonsListOption struct { + GlobalCommandOptions +} + +// Complete the conditions required to be able to run list. +func (o *CommandAddonsListOption) Complete() error { + return o.GlobalCommandOptions.Complete() +} + +// Run start list Karmada addons +func (o *CommandAddonsListOption) Run() error { + addonNames := make([]string, 0, len(Addons)) + for addonName := range Addons { + addonNames = append(addonNames, addonName) + } + sort.Strings(addonNames) + + // Init tableWriter + table := tablewriter.NewWriter(os.Stdout) + table.SetAutoFormatHeaders(true) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetCenterSeparator("|") + + // Create table header + tHeader := []string{"Addon Name", "Status"} + table.SetHeader(tHeader) + + // Create table data + var tData [][]string + var temp []string + for _, addonName := range addonNames { + tStatus, err := Addons[addonName].Status(o) + if err != nil { + return err + } + temp = []string{addonName, tStatus} + tData = append(tData, temp) + } + + table.AppendBulk(tData) + + table.Render() + + return nil +} diff --git a/pkg/karmadactl/addons/install/install.go b/pkg/karmadactl/addons/install/install.go new file mode 100644 index 000000000000..5462e4a21985 --- /dev/null +++ b/pkg/karmadactl/addons/install/install.go @@ -0,0 +1,15 @@ +package install + +import ( + "github.com/karmada-io/karmada/pkg/karmadactl/addons/descheduler" + "github.com/karmada-io/karmada/pkg/karmadactl/addons/estimator" + addonsinit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + "github.com/karmada-io/karmada/pkg/karmadactl/addons/search" +) + +// Install intall the karmada addons process in Addons +func Install() { + addonsinit.Addons["karmada-search"] = search.AddonSearch + addonsinit.Addons["karmada-descheduler"] = descheduler.AddonDescheduler + addonsinit.Addons["karmada-scheduler-estimator"] = estimator.AddonEstimator +} diff --git a/pkg/karmadactl/addons/list.go b/pkg/karmadactl/addons/list.go new file mode 100644 index 000000000000..0ff5acef7195 --- /dev/null +++ b/pkg/karmadactl/addons/list.go @@ -0,0 +1,53 @@ +package addons + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" +) + +var ( + listExample = templates.Examples(` + # List Karmada all addons installed in Kubernetes cluster + %[1]s list + + # List Karmada all addons included scheduler estimator of member1 installed in Kubernetes cluster + %[1]s list -C member1 + + # Specify the host cluster kubeconfig + %[1]s list --kubeconfig /root/.kube/config + + # Specify the karmada control plane kubeconfig + %[1]s list --karmada-kubeconfig /etc/karmada/karmada-apiserver.config + + # Sepcify the namespace where Karmada components are installed + %[1]s list --namespace karmada-system + `) +) + +// NewCmdAddonsList list Karmada addons on Kubernetes +func NewCmdAddonsList(parentCommand string) *cobra.Command { + opts := addoninit.CommandAddonsListOption{} + cmd := &cobra.Command{ + Use: "list", + Short: "List karmada addons from Kubernetes", + Long: "List Karmada addons from Kubernetes", + Example: fmt.Sprintf(listExample, parentCommand), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if err := opts.Complete(); err != nil { + return err + } + if err := opts.Run(); err != nil { + return err + } + return nil + }, + } + + opts.GlobalCommandOptions.AddFlags(cmd.Flags()) + return cmd +} diff --git a/pkg/karmadactl/addons/search/manifests.go b/pkg/karmadactl/addons/search/manifests.go new file mode 100644 index 000000000000..fcd16b94484b --- /dev/null +++ b/pkg/karmadactl/addons/search/manifests.go @@ -0,0 +1,148 @@ +package search + +const ( + karmadaSearchDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-search + namespace: {{ .Namespace }} + labels: + app: karmada-search + apiserver: "true" +spec: + selector: + matchLabels: + app: karmada-search + apiserver: "true" + replicas: {{ .Replicas }} + template: + metadata: + labels: + app: karmada-search + apiserver: "true" + spec: + automountServiceAccountToken: false + containers: + - name: karmada-search + image: {{ .Image }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: k8s-certs + mountPath: /etc/kubernetes/pki + readOnly: true + - name: kubeconfig + subPath: kubeconfig + mountPath: /etc/kubeconfig + command: + - /bin/karmada-search + - --kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --etcd-servers={{ .ETCDSevers }} + - --etcd-cafile=/etc/kubernetes/pki/ca.crt + - --etcd-certfile=/etc/kubernetes/pki/etcd-client.crt + - --etcd-keyfile=/etc/kubernetes/pki/etcd-client.key + - --tls-cert-file=/etc/kubernetes/pki/karmada.crt + - --tls-private-key-file=/etc/kubernetes/pki/karmada.key + - --audit-log-path=- + - --feature-gates=APIPriorityAndFairness=false + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + livenessProbe: + httpGet: + path: /livez + port: 443 + scheme: HTTPS + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + resources: + requests: + cpu: 100m + volumes: + - name: k8s-certs + secret: + secretName: karmada-cert + - name: kubeconfig + secret: + secretName: kubeconfig +` + + karmadaSearchService = ` +apiVersion: v1 +kind: Service +metadata: + name: karmada-search + namespace: {{ .Namespace }} + labels: + app: karmada-search + apiserver: "true" +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + app: karmada-search +` + + karmadaSearchAAAPIService = ` +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: {{ .Name }} + labels: + app: karmada-search + apiserver: "true" +spec: + insecureSkipTLSVerify: true + group: search.karmada.io + groupPriorityMinimum: 2000 + service: + name: karmada-search + namespace: {{ .Namespace }} + version: v1alpha1 + versionPriority: 10 +` + + karmadaSearchAAService = ` +apiVersion: v1 +kind: Service +metadata: + name: karmada-search + namespace: {{ .Namespace }} +spec: + type: ExternalName + externalName: karmada-search.{{ .Namespace }}.svc.cluster.local +` +) + +// DeploymentReplace is a struct to help to concrete +// the karamda-search deployment bytes with the deployment template +type DeploymentReplace struct { + Namespace string + Replicas *int32 + Image string + ETCDSevers string +} + +// ServiceReplace is a struct to help to concrete +// the karamda-search Service bytes with the Service template +type ServiceReplace struct { + Namespace string +} + +// AAApiServiceReplace is a struct to help to concrete +// the karamda-search ApiService bytes with the AAApiService template +type AAApiServiceReplace struct { + Name string + Namespace string +} + +// AAServiceReplace is a struct to help to concrete +// the karamda-search AA Service bytes with the AAService template +type AAServiceReplace struct { + Namespace string +} diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go new file mode 100644 index 000000000000..21ce6f38465b --- /dev/null +++ b/pkg/karmadactl/addons/search/search.go @@ -0,0 +1,233 @@ +package search + +import ( + "context" + "fmt" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + apiregistrationv1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +const ( + // aaAPIServiceName define apiservice name install on karmada control plane + aaAPIServiceName = "v1alpha1.search.karmada.io" + + // etcdStatefulSetAndServiceName define etcd statefulSet and serviceName installed by init command + etcdStatefulSetAndServiceName = "etcd" + + // etcdContainerClientPort define etcd pod installed by init command + etcdContainerClientPort = 2379 +) + +var ( + karmadaSearchLabels = map[string]string{"app": addoninit.SearchResourceName, "apiserver": "true"} +) + +// AddonSearch describe the search addon command process +var AddonSearch = &addoninit.Addon{ + Name: addoninit.SearchResourceName, + Status: status, + Enable: enableSearch, + Disable: disableSearch, +} + +var status = func(opts *addoninit.CommandAddonsListOption) (string, error) { + // check karmada-search deployment status on host cluster + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + deployment, err := deployClient.Get(context.TODO(), addoninit.SearchResourceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + if deployment.Status.Replicas != deployment.Status.ReadyReplicas || + deployment.Status.Replicas != deployment.Status.AvailableReplicas { + return addoninit.AddonUnhealthyStatus, nil + } + + // check karmada-search apiservice is available on karmada control plane + apiService, err := opts.KarmadaAggregatorClientSet.ApiregistrationV1().APIServices().Get(context.TODO(), aaAPIServiceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + + if !apiregistrationv1helper.IsAPIServiceConditionTrue(apiService, apiregistrationv1.Available) { + return addoninit.AddonUnhealthyStatus, nil + } + + return addoninit.AddonEnabledStatus, nil +} + +var enableSearch = func(opts *addoninit.CommandAddonsEnableOption) error { + + if err := installComponentsOnHostCluster(opts); err != nil { + return err + } + + if err := installComponentsOnKarmadaControlPlane(opts); err != nil { + return err + } + + return nil +} + +var disableSearch = func(opts *addoninit.CommandAddonsDisableOption) error { + // delete karmada search service on host cluster + serviceClient := opts.KubeClientSet.CoreV1().Services(opts.Namespace) + if err := serviceClient.Delete(context.TODO(), addoninit.SearchResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada search service on host cluster successfully") + + // delete karmada search deployment on host cluster + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + if err := deployClient.Delete(context.TODO(), addoninit.SearchResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada search deployment on host cluster successfully") + + // delete karmada search aa service on karmada control plane + karmadaServiceClient := opts.KarmadaKubeClientSet.CoreV1().Services(opts.Namespace) + if err := karmadaServiceClient.Delete(context.TODO(), addoninit.SearchResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada search AA service on karmada control plane successfully") + + // delete karmada search aa apiservice on karmada control plane + if err := opts.KarmadaAggregatorClientSet.ApiregistrationV1().APIServices().Delete(context.TODO(), aaAPIServiceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + + klog.Infof("Uninstall karmada search AA apiservice on karmada control plane successfully") + return nil +} + +func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) error { + // install karmada search service on host cluster + karmadaSearchServiceBytes, err := addonutils.ParseTemplate(karmadaSearchService, ServiceReplace{ + Namespace: opts.Namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada search service template :%v", err) + } + + karmadaSearchService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaSearchServiceBytes, karmadaSearchService); err != nil { + return fmt.Errorf("decode karmada search service error: %v", err) + } + + if err := addonutils.CreateService(opts.KubeClientSet, karmadaSearchService); err != nil { + return fmt.Errorf("create karmada search service error: %v", err) + } + + etcdServers, err := etcdServers(opts) + if err != nil { + return err + } + + klog.Infof("Install karmada search service on host cluster successfully") + + // install karmada search deployment on host clusters + karmadaSearchDeploymentBytes, err := addonutils.ParseTemplate(karmadaSearchDeployment, DeploymentReplace{ + Namespace: opts.Namespace, + Replicas: &opts.KarmadaSearchReplicas, + ETCDSevers: etcdServers, + Image: opts.KarmadaSearchImage, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada search deployment template :%v", err) + } + + karmadaSearchDeployment := &appsv1.Deployment{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaSearchDeploymentBytes, karmadaSearchDeployment); err != nil { + return fmt.Errorf("decode karmada search deployment error: %v", err) + } + if err := addonutils.CreateOrUpdateDeployment(opts.KubeClientSet, karmadaSearchDeployment); err != nil { + return fmt.Errorf("create karmada search deployment error: %v", err) + } + + if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaSearchLabels), opts.WaitPodReadyTimeout); err != nil { + return fmt.Errorf("wait karmada search pod status ready timeout: %v", err) + } + + klog.Infof("Install karmada search deployment on host cluster successfully") + return nil +} + +func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableOption) error { + // install karmada search AA service on karmada control plane + aaServiceBytes, err := addonutils.ParseTemplate(karmadaSearchAAService, AAServiceReplace{ + Namespace: opts.Namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada search AA service template :%v", err) + } + + aaService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaServiceBytes, aaService); err != nil { + return fmt.Errorf("decode karmada search AA service error: %v", err) + } + if err := addonutils.CreateService(opts.KarmadaKubeClientSet, aaService); err != nil { + return fmt.Errorf("create karmada search AA service error: %v", err) + } + + // install karmada search apiservice on karmada control plane + aaAPIServiceBytes, err := addonutils.ParseTemplate(karmadaSearchAAAPIService, AAApiServiceReplace{ + Name: aaAPIServiceName, + Namespace: opts.Namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada search AA apiservice template :%v", err) + } + + aaAPIService := &apiregistrationv1.APIService{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaAPIServiceBytes, aaAPIService); err != nil { + return fmt.Errorf("decode karmada search AA apiservice error: %v", err) + } + + if err = addonutils.CreateOrUpdateAPIService(opts.KarmadaAggregatorClientSet, aaAPIService); err != nil { + return fmt.Errorf("craete karmada search AA apiservice error: %v", err) + } + + if err := initkarmada.WaitAPIServiceReady(opts.KarmadaAggregatorClientSet, aaAPIServiceName, time.Duration(opts.WaitAPIServiceReadyTimeout)*time.Second); err != nil { + return err + } + + klog.Infof("Install karmada search api server on karmada control plane successfully") + return nil +} + +func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) { + sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(context.TODO(), "etcd", metav1.GetOptions{}) + if err != nil { + return "", err + } + + ectdReplicas := *sts.Spec.Replicas + ectdServers := "" + + for v := int32(0); v < ectdReplicas; v++ { + ectdServers += fmt.Sprintf("https://%s-%v.%s.%s.svc.cluster.local:%v", etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, opts.Namespace, etcdContainerClientPort) + "," + } + + return strings.TrimRight(ectdServers, ","), nil +} diff --git a/pkg/karmadactl/addons/utils/idempotency.go b/pkg/karmadactl/addons/utils/idempotency.go new file mode 100644 index 000000000000..18fe51ce1934 --- /dev/null +++ b/pkg/karmadactl/addons/utils/idempotency.go @@ -0,0 +1,76 @@ +package utils + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" +) + +// CreateService creates a Service if the target resource doesn't exist. If the resource exists already, return directly +func CreateService(KubeClientSet *kubernetes.Clientset, service *corev1.Service) error { + if _, err := KubeClientSet.CoreV1().Services(service.ObjectMeta.Namespace).Create(context.TODO(), service, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create service: %v", err) + } + + klog.Warningf("Service %s is existed, creation process will skip", service.ObjectMeta.Name) + } + return nil +} + +// CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateOrUpdateSecret(KubeClientSet *kubernetes.Clientset, secret *corev1.Secret) error { + if _, err := KubeClientSet.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create service: %v", err) + } + + if _, err := KubeClientSet.CoreV1().Secrets(secret.ObjectMeta.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("unable to update deployment: %v", err) + } + } + return nil +} + +// CreateOrUpdateDeployment creates a Deployment if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateOrUpdateDeployment(client kubernetes.Interface, deploy *appsv1.Deployment) error { + if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Create(context.TODO(), deploy, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create deployment: %v", err) + } + + if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("unable to update deployment: %v", err) + } + } + return nil +} + +// CreateOrUpdateAPIService creates a ApiService if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateOrUpdateAPIService(apiRegistrationClient *aggregator.Clientset, apiservice *apiregistrationv1.APIService) error { + if _, err := apiRegistrationClient.ApiregistrationV1().APIServices().Create(context.TODO(), apiservice, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create apiService: %v", err) + } + + existAPIService, err := apiRegistrationClient.ApiregistrationV1().APIServices().Get(context.TODO(), apiservice.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + apiservice.ObjectMeta.ResourceVersion = existAPIService.ObjectMeta.ResourceVersion + + if _, err := apiRegistrationClient.ApiregistrationV1().APIServices().Update(context.TODO(), apiservice, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("unable to update apiService: %v", err) + } + } + return nil +} diff --git a/pkg/karmadactl/addons/utils/template.go b/pkg/karmadactl/addons/utils/template.go new file mode 100644 index 000000000000..9c3e951cb5e2 --- /dev/null +++ b/pkg/karmadactl/addons/utils/template.go @@ -0,0 +1,21 @@ +package utils + +import ( + "bytes" + "fmt" + "text/template" +) + +// ParseTemplate validates and parses passed as argument template +func ParseTemplate(strTmpl string, obj interface{}) ([]byte, error) { + var buf bytes.Buffer + tmpl, err := template.New("template").Parse(strTmpl) + if err != nil { + return nil, fmt.Errorf("error when parsing template: %v", err) + } + err = tmpl.Execute(&buf, obj) + if err != nil { + return nil, fmt.Errorf("error when executing template: %v", err) + } + return buf.Bytes(), nil +} diff --git a/pkg/karmadactl/cmdinit/karmada/check.go b/pkg/karmadactl/cmdinit/karmada/check.go index 4616ee10e618..5b7507f82b98 100644 --- a/pkg/karmadactl/cmdinit/karmada/check.go +++ b/pkg/karmadactl/cmdinit/karmada/check.go @@ -12,7 +12,8 @@ import ( aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" ) -func waitAPIServiceReady(c *aggregator.Clientset, name string, timeout time.Duration) error { +// WaitAPIServiceReady wait the api service condition true +func WaitAPIServiceReady(c *aggregator.Clientset, name string, timeout time.Duration) error { if err := wait.PollImmediate(time.Second, timeout, func() (done bool, err error) { apiService, e := c.ApiregistrationV1().APIServices().Get(context.TODO(), name, metav1.GetOptions{}) if e != nil { diff --git a/pkg/karmadactl/cmdinit/karmada/deploy.go b/pkg/karmadactl/cmdinit/karmada/deploy.go index 813b06bc803c..d515f5a48282 100644 --- a/pkg/karmadactl/cmdinit/karmada/deploy.go +++ b/pkg/karmadactl/cmdinit/karmada/deploy.go @@ -240,7 +240,7 @@ func initAPIService(clientSet *kubernetes.Clientset, restConfig *rest.Config, sy if _, err := apiRegistrationClient.ApiregistrationV1().APIServices().Create(context.TODO(), aaAPIService, metav1.CreateOptions{}); err != nil { return err } - if err := waitAPIServiceReady(apiRegistrationClient, aaAPIServiceObjName, 120*time.Second); err != nil { + if err := WaitAPIServiceReady(apiRegistrationClient, aaAPIServiceObjName, 120*time.Second); err != nil { return err } return nil diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index 1116a8fd9b7b..e7b2420f36f9 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -266,7 +266,7 @@ func (i *CommandInitOption) createCertsSecrets() error { return fmt.Errorf("failure while serializing admin kubeConfig. %v", err) } - kubeConfigSecret := i.SecretFromSpec(kubeConfigSecretAndMountName, corev1.SecretTypeOpaque, map[string]string{kubeConfigSecretAndMountName: string(configBytes)}) + kubeConfigSecret := i.SecretFromSpec(KubeConfigSecretAndMountName, corev1.SecretTypeOpaque, map[string]string{KubeConfigSecretAndMountName: string(configBytes)}) if err = i.CreateSecret(kubeConfigSecret); err != nil { return err } diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 774446f9b6ee..4bf931cfaf01 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -14,10 +14,12 @@ import ( ) const ( - deploymentAPIVersion = "apps/v1" - deploymentKind = "Deployment" - portName = "server" - kubeConfigSecretAndMountName = "kubeconfig" + deploymentAPIVersion = "apps/v1" + deploymentKind = "Deployment" + portName = "server" + + // KubeConfigSecretAndMountName is the secret and volume mount name of karmada kubeconfig + KubeConfigSecretAndMountName = "kubeconfig" karmadaCertsName = "karmada-cert" karmadaCertsVolumeMountPath = "/etc/kubernetes/pki" kubeConfigContainerMountPath = "/etc/kubeconfig" @@ -286,10 +288,10 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, VolumeMounts: []corev1.VolumeMount{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, ReadOnly: true, MountPath: kubeConfigContainerMountPath, - SubPath: kubeConfigSecretAndMountName, + SubPath: KubeConfigSecretAndMountName, }, { Name: karmadaCertsName, @@ -301,10 +303,10 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, Volumes: []corev1.Volume{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kubeConfigSecretAndMountName, + SecretName: KubeConfigSecretAndMountName, }, }, }, @@ -412,20 +414,20 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment LivenessProbe: livenessProbe, VolumeMounts: []corev1.VolumeMount{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, ReadOnly: true, MountPath: kubeConfigContainerMountPath, - SubPath: kubeConfigSecretAndMountName, + SubPath: KubeConfigSecretAndMountName, }, }, }, }, Volumes: []corev1.Volume{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kubeConfigSecretAndMountName, + SecretName: KubeConfigSecretAndMountName, }, }, }, @@ -532,20 +534,20 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep }, VolumeMounts: []corev1.VolumeMount{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, ReadOnly: true, MountPath: kubeConfigContainerMountPath, - SubPath: kubeConfigSecretAndMountName, + SubPath: KubeConfigSecretAndMountName, }, }, }, }, Volumes: []corev1.Volume{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kubeConfigSecretAndMountName, + SecretName: KubeConfigSecretAndMountName, }, }, }, @@ -646,10 +648,10 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, VolumeMounts: []corev1.VolumeMount{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, ReadOnly: true, MountPath: kubeConfigContainerMountPath, - SubPath: kubeConfigSecretAndMountName, + SubPath: KubeConfigSecretAndMountName, }, { Name: webhookCertsName, @@ -662,10 +664,10 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, Volumes: []corev1.Volume{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kubeConfigSecretAndMountName, + SecretName: KubeConfigSecretAndMountName, }, }, }, @@ -777,10 +779,10 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D }, VolumeMounts: []corev1.VolumeMount{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, ReadOnly: true, MountPath: kubeConfigContainerMountPath, - SubPath: kubeConfigSecretAndMountName, + SubPath: KubeConfigSecretAndMountName, }, { Name: karmadaCertsName, @@ -798,10 +800,10 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D }, Volumes: []corev1.Volume{ { - Name: kubeConfigSecretAndMountName, + Name: KubeConfigSecretAndMountName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kubeConfigSecretAndMountName, + SecretName: KubeConfigSecretAndMountName, }, }, }, diff --git a/pkg/karmadactl/karmadactl.go b/pkg/karmadactl/karmadactl.go index cf4c7497a555..aae21d4a2b19 100644 --- a/pkg/karmadactl/karmadactl.go +++ b/pkg/karmadactl/karmadactl.go @@ -11,6 +11,7 @@ import ( "k8s.io/klog/v2" "k8s.io/kubectl/pkg/util/templates" + "github.com/karmada-io/karmada/pkg/karmadactl/addons" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit" "github.com/karmada-io/karmada/pkg/version/sharedcommand" ) @@ -60,6 +61,7 @@ func NewKarmadaCtlCommand(cmdUse, parentCommand string) *cobra.Command { Commands: []*cobra.Command{ cmdinit.NewCmdInit(parentCommand), NewCmdDeInit(parentCommand), + addons.NewCommandAddons(parentCommand), NewCmdJoin(karmadaConfig, parentCommand), NewCmdUnjoin(karmadaConfig, parentCommand), }, diff --git a/pkg/util/names/names.go b/pkg/util/names/names.go index 69927f193a16..8b07832dc900 100644 --- a/pkg/util/names/names.go +++ b/pkg/util/names/names.go @@ -109,6 +109,11 @@ func GenerateEstimatorServiceName(clusterName string) string { return fmt.Sprintf("%s-%s", estimatorServicePrefix, clusterName) } +// GenerateEstimatorDeploymentName generates the gRPC scheduler estimator deployment name which belongs to a cluster. +func GenerateEstimatorDeploymentName(clusterName string) string { + return fmt.Sprintf("%s-%s", estimatorServicePrefix, clusterName) +} + // IsReservedNamespace return whether it is a reserved namespace func IsReservedNamespace(namespace string) bool { return namespace == NamespaceKarmadaSystem ||