From e38cf3b8dbfc8e05a692b4ff9b0b3e85092c3eda Mon Sep 17 00:00:00 2001 From: zhujian Date: Thu, 2 Dec 2021 07:28:46 +0000 Subject: [PATCH 1/2] support run work agent outside of managed cluster add a flag spoke-kubeconfig to specify the kubeconfig of the apoke/managed cluster Signed-off-by: zhujian --- Makefile | 6 +++-- deploy/webhook/kustomization.yaml | 2 +- pkg/spoke/spokeagent.go | 42 +++++++++++++++++++++---------- test/integration/work_test.go | 1 + 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 8f18b74a..dbfb08ef 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ clean: cluster-ip: $(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG) - CLUSTER_IP?=$(shell $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) get svc kubernetes -n default -o jsonpath="{.spec.clusterIP}") + $(eval CLUSTER_IP?=$(shell $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) get svc kubernetes -n default -o jsonpath="{.spec.clusterIP}")) hub-kubeconfig-secret: cluster-ip $(KUBECTL) config use-context $(SPOKE_KUBECONFIG_CONTEXT) --kubeconfig $(SPOKE_KUBECONFIG) @@ -48,8 +48,10 @@ hub-kubeconfig-secret: cluster-ip $(KUBECTL) delete secret hub-kubeconfig-secret -n open-cluster-management-agent --ignore-not-found --kubeconfig $(SPOKE_KUBECONFIG) $(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG) $(KUBECTL) config view --flatten --minify --kubeconfig $(HUB_KUBECONFIG) > hub-kubeconfig -ifeq ($(HUB_KUBECONFIG), $(SPOKE_KUBECONFIG)) && ($(HUB_KUBECONFIG_CONTEXT), $(SPOKE_KUBECONFIG_CONTEXT)) +ifeq ($(HUB_KUBECONFIG), $(SPOKE_KUBECONFIG)) +ifeq ($(HUB_KUBECONFIG_CONTEXT), $(SPOKE_KUBECONFIG_CONTEXT)) $(KUBECTL) config set clusters.$(HUB_KUBECONFIG_CONTEXT).server https://$(CLUSTER_IP) --kubeconfig hub-kubeconfig +endif endif $(KUBECTL) config use-context $(SPOKE_KUBECONFIG_CONTEXT) --kubeconfig $(SPOKE_KUBECONFIG) $(KUBECTL) create secret generic hub-kubeconfig-secret --from-file=kubeconfig=hub-kubeconfig -n open-cluster-management-agent --kubeconfig $(SPOKE_KUBECONFIG) diff --git a/deploy/webhook/kustomization.yaml b/deploy/webhook/kustomization.yaml index a967ceac..24691009 100644 --- a/deploy/webhook/kustomization.yaml +++ b/deploy/webhook/kustomization.yaml @@ -1,6 +1,6 @@ # Adds namespace to all resources. -namespace: open-cluster-management +namespace: open-cluster-management-hub # Value of this field is prepended to the # names of all resources, e.g. a deployment named diff --git a/pkg/spoke/spokeagent.go b/pkg/spoke/spokeagent.go index aaeda26e..4a2eb5f2 100644 --- a/pkg/spoke/spokeagent.go +++ b/pkg/spoke/spokeagent.go @@ -2,6 +2,7 @@ package spoke import ( "context" + "fmt" "time" "open-cluster-management.io/work/pkg/helper" @@ -16,6 +17,7 @@ import ( apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" workclientset "open-cluster-management.io/api/client/work/clientset/versioned" @@ -24,10 +26,12 @@ import ( // WorkloadAgentOptions defines the flags for workload agent type WorkloadAgentOptions struct { - HubKubeconfigFile string - SpokeClusterName string - QPS float32 - Burst int + HubKubeconfigFile string + SpokeKubeconfigFile string + SpokeKubeconfig *rest.Config + SpokeClusterName string + QPS float32 + Burst int } // NewWorkloadAgentOptions returns the flags with default value set @@ -43,6 +47,7 @@ func (o *WorkloadAgentOptions) AddFlags(cmd *cobra.Command) { flags := cmd.Flags() // This command only supports reading from config flags.StringVar(&o.HubKubeconfigFile, "hub-kubeconfig", o.HubKubeconfigFile, "Location of kubeconfig file to connect to hub cluster.") + flags.StringVar(&o.SpokeKubeconfigFile, "spoke-kubeconfig", o.SpokeKubeconfigFile, "Location of kubeconfig file to connect to spoke cluster.") flags.StringVar(&o.SpokeClusterName, "spoke-cluster-name", o.SpokeClusterName, "Name of spoke cluster.") flags.Float32Var(&o.QPS, "spoke-kube-api-qps", o.QPS, "QPS to use while talking with apiserver on spoke cluster.") flags.IntVar(&o.Burst, "spoke-kube-api-burst", o.Burst, "Burst to use while talking with apiserver on spoke cluster.") @@ -64,28 +69,39 @@ func (o *WorkloadAgentOptions) RunWorkloadAgent(ctx context.Context, controllerC // Only watch the cluster namespace on hub workInformerFactory := workinformers.NewSharedInformerFactoryWithOptions(hubWorkClient, 5*time.Minute, workinformers.WithNamespace(o.SpokeClusterName)) - // Build dynamic client and informer for spoke cluster - spokeRestConfig := controllerContext.KubeConfig - spokeRestConfig.QPS = o.QPS - spokeRestConfig.Burst = o.Burst - spokeDynamicClient, err := dynamic.NewForConfig(spokeRestConfig) + // load spoke client config and create spoke clients, + // the work agent may running not in the spoke/managed cluster. + if o.SpokeKubeconfig == nil { + if o.SpokeKubeconfigFile != "" { + o.SpokeKubeconfig, err = clientcmd.BuildConfigFromFlags("" /* leave masterurl as empty */, o.SpokeKubeconfigFile) + if err != nil { + return fmt.Errorf("unable to load spoke kubeconfig from file %q: %w", o.SpokeKubeconfigFile, err) + } + } else { + o.SpokeKubeconfig = controllerContext.KubeConfig + } + } + + o.SpokeKubeconfig.QPS = o.QPS + o.SpokeKubeconfig.Burst = o.Burst + spokeDynamicClient, err := dynamic.NewForConfig(o.SpokeKubeconfig) if err != nil { return err } - spokeKubeClient, err := kubernetes.NewForConfig(spokeRestConfig) + spokeKubeClient, err := kubernetes.NewForConfig(o.SpokeKubeconfig) if err != nil { return err } - spokeAPIExtensionClient, err := apiextensionsclient.NewForConfig(spokeRestConfig) + spokeAPIExtensionClient, err := apiextensionsclient.NewForConfig(o.SpokeKubeconfig) if err != nil { return err } - spokeWorkClient, err := workclientset.NewForConfig(spokeRestConfig) + spokeWorkClient, err := workclientset.NewForConfig(o.SpokeKubeconfig) if err != nil { return err } spokeWorkInformerFactory := workinformers.NewSharedInformerFactory(spokeWorkClient, 5*time.Minute) - restMapper, err := apiutil.NewDynamicRESTMapper(spokeRestConfig, apiutil.WithLazyDiscovery) + restMapper, err := apiutil.NewDynamicRESTMapper(o.SpokeKubeconfig, apiutil.WithLazyDiscovery) if err != nil { return err } diff --git a/test/integration/work_test.go b/test/integration/work_test.go index 86400e47..dd1347c6 100644 --- a/test/integration/work_test.go +++ b/test/integration/work_test.go @@ -44,6 +44,7 @@ var _ = ginkgo.Describe("ManifestWork", func() { ginkgo.BeforeEach(func() { o = spoke.NewWorkloadAgentOptions() o.HubKubeconfigFile = hubKubeconfigFileName + o.SpokeKubeconfig = spokeRestConfig o.SpokeClusterName = utilrand.String(5) ns := &corev1.Namespace{} From 68c815c8b9bfaf787e97aab92abab0f0de521216 Mon Sep 17 00:00:00 2001 From: zhujian Date: Thu, 2 Dec 2021 14:37:21 +0000 Subject: [PATCH 2/2] delete field SpokeKubeconfig Signed-off-by: zhujian --- pkg/spoke/spokeagent.go | 45 ++++++++++++++++++++--------------- test/integration/work_test.go | 1 - 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pkg/spoke/spokeagent.go b/pkg/spoke/spokeagent.go index 4a2eb5f2..886e240d 100644 --- a/pkg/spoke/spokeagent.go +++ b/pkg/spoke/spokeagent.go @@ -28,7 +28,6 @@ import ( type WorkloadAgentOptions struct { HubKubeconfigFile string SpokeKubeconfigFile string - SpokeKubeconfig *rest.Config SpokeClusterName string QPS float32 Burst int @@ -47,7 +46,8 @@ func (o *WorkloadAgentOptions) AddFlags(cmd *cobra.Command) { flags := cmd.Flags() // This command only supports reading from config flags.StringVar(&o.HubKubeconfigFile, "hub-kubeconfig", o.HubKubeconfigFile, "Location of kubeconfig file to connect to hub cluster.") - flags.StringVar(&o.SpokeKubeconfigFile, "spoke-kubeconfig", o.SpokeKubeconfigFile, "Location of kubeconfig file to connect to spoke cluster.") + flags.StringVar(&o.SpokeKubeconfigFile, "spoke-kubeconfig", o.SpokeKubeconfigFile, + "Location of kubeconfig file to connect to spoke cluster. If this is not set, will use '--kubeconfig' to build client to connect to the managed cluster.") flags.StringVar(&o.SpokeClusterName, "spoke-cluster-name", o.SpokeClusterName, "Name of spoke cluster.") flags.Float32Var(&o.QPS, "spoke-kube-api-qps", o.QPS, "QPS to use while talking with apiserver on spoke cluster.") flags.IntVar(&o.Burst, "spoke-kube-api-burst", o.Burst, "Burst to use while talking with apiserver on spoke cluster.") @@ -70,38 +70,32 @@ func (o *WorkloadAgentOptions) RunWorkloadAgent(ctx context.Context, controllerC workInformerFactory := workinformers.NewSharedInformerFactoryWithOptions(hubWorkClient, 5*time.Minute, workinformers.WithNamespace(o.SpokeClusterName)) // load spoke client config and create spoke clients, - // the work agent may running not in the spoke/managed cluster. - if o.SpokeKubeconfig == nil { - if o.SpokeKubeconfigFile != "" { - o.SpokeKubeconfig, err = clientcmd.BuildConfigFromFlags("" /* leave masterurl as empty */, o.SpokeKubeconfigFile) - if err != nil { - return fmt.Errorf("unable to load spoke kubeconfig from file %q: %w", o.SpokeKubeconfigFile, err) - } - } else { - o.SpokeKubeconfig = controllerContext.KubeConfig - } + // the work agent may not running in the spoke/managed cluster. + spokeRestConfig, err := o.spokeKubeConfig(controllerContext) + if err != nil { + return err } - o.SpokeKubeconfig.QPS = o.QPS - o.SpokeKubeconfig.Burst = o.Burst - spokeDynamicClient, err := dynamic.NewForConfig(o.SpokeKubeconfig) + spokeRestConfig.QPS = o.QPS + spokeRestConfig.Burst = o.Burst + spokeDynamicClient, err := dynamic.NewForConfig(spokeRestConfig) if err != nil { return err } - spokeKubeClient, err := kubernetes.NewForConfig(o.SpokeKubeconfig) + spokeKubeClient, err := kubernetes.NewForConfig(spokeRestConfig) if err != nil { return err } - spokeAPIExtensionClient, err := apiextensionsclient.NewForConfig(o.SpokeKubeconfig) + spokeAPIExtensionClient, err := apiextensionsclient.NewForConfig(spokeRestConfig) if err != nil { return err } - spokeWorkClient, err := workclientset.NewForConfig(o.SpokeKubeconfig) + spokeWorkClient, err := workclientset.NewForConfig(spokeRestConfig) if err != nil { return err } spokeWorkInformerFactory := workinformers.NewSharedInformerFactory(spokeWorkClient, 5*time.Minute) - restMapper, err := apiutil.NewDynamicRESTMapper(o.SpokeKubeconfig, apiutil.WithLazyDiscovery) + restMapper, err := apiutil.NewDynamicRESTMapper(spokeRestConfig, apiutil.WithLazyDiscovery) if err != nil { return err } @@ -170,3 +164,16 @@ func (o *WorkloadAgentOptions) RunWorkloadAgent(ctx context.Context, controllerC <-ctx.Done() return nil } + +// spokeKubeConfig builds kubeconfig for the spoke/managed cluster +func (o *WorkloadAgentOptions) spokeKubeConfig(controllerContext *controllercmd.ControllerContext) (*rest.Config, error) { + if o.SpokeKubeconfigFile == "" { + return controllerContext.KubeConfig, nil + } + + spokeRestConfig, err := clientcmd.BuildConfigFromFlags("" /* leave masterurl as empty */, o.SpokeKubeconfigFile) + if err != nil { + return nil, fmt.Errorf("unable to load spoke kubeconfig from file %q: %w", o.SpokeKubeconfigFile, err) + } + return spokeRestConfig, nil +} diff --git a/test/integration/work_test.go b/test/integration/work_test.go index dd1347c6..86400e47 100644 --- a/test/integration/work_test.go +++ b/test/integration/work_test.go @@ -44,7 +44,6 @@ var _ = ginkgo.Describe("ManifestWork", func() { ginkgo.BeforeEach(func() { o = spoke.NewWorkloadAgentOptions() o.HubKubeconfigFile = hubKubeconfigFileName - o.SpokeKubeconfig = spokeRestConfig o.SpokeClusterName = utilrand.String(5) ns := &corev1.Namespace{}