From 091e87eb4f0954e5a105238b41db5eaf16c06388 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Fri, 6 Oct 2017 01:33:33 +0700 Subject: [PATCH 1/6] Added PolicyConfigMap and PolicyConfigMapNamespace to KubeSchedulerConfig --- pkg/apis/kops/componentconfig.go | 4 ++++ pkg/apis/kops/v1alpha1/componentconfig.go | 4 ++++ pkg/apis/kops/v1alpha2/componentconfig.go | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index ecbe2f56593bb..d42807f6076dd 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -330,6 +330,10 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` + // Name of configmap to use for scheduler policy + PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` + // Namespace of configmap + PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index a616a55fc0c93..de5438c24d76a 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -330,6 +330,10 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` + // Name of configmap to use for scheduler policy + PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` + // Namespace of configmap + PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 4777462d02c37..1e0c59f1a6fa5 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -331,6 +331,10 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` + // Name of configmap to use for scheduler policy + PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` + // Namespace of configmap + PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } // LeaderElectionConfiguration defines the configuration of leader election From 0f2dde994c8394fb39503be151ebc19b2cb8a778 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Fri, 6 Oct 2017 01:33:54 +0700 Subject: [PATCH 2/6] Regenerated API --- pkg/apis/kops/v1alpha1/zz_generated.conversion.go | 4 ++++ pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 482c3113bce33..4c464d15a36b6 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2003,6 +2003,8 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku } else { out.LeaderElection = nil } + out.PolicyConfigMap = in.PolicyConfigMap + out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace return nil } @@ -2024,6 +2026,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko } else { out.LeaderElection = nil } + out.PolicyConfigMap = in.PolicyConfigMap + out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index c58c9f384f8ad..4c1046fa01b28 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2265,6 +2265,8 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku } else { out.LeaderElection = nil } + out.PolicyConfigMap = in.PolicyConfigMap + out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace return nil } @@ -2286,6 +2288,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko } else { out.LeaderElection = nil } + out.PolicyConfigMap = in.PolicyConfigMap + out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace return nil } From 78023ba0dfd668b2084a7c85775d3a2d76239c73 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Fri, 6 Oct 2017 23:56:55 +0700 Subject: [PATCH 3/6] Updated PolicyConfigMap comment --- pkg/apis/kops/componentconfig.go | 4 ++-- pkg/apis/kops/v1alpha1/componentconfig.go | 4 ++-- pkg/apis/kops/v1alpha2/componentconfig.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index d42807f6076dd..c4baab07ac28b 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -330,9 +330,9 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // Name of configmap to use for scheduler policy + // PolicyConfigMap is the name of configmap to use for scheduler policy PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // Namespace of configmap + // PolicyConfigMapNamespace is the namespace containing the configmap PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index de5438c24d76a..6882ba357f751 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -330,9 +330,9 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // Name of configmap to use for scheduler policy + // PolicyConfigMap is the name of configmap to use for scheduler policy PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // Namespace of configmap + // PolicyConfigMapNamespace is the namespace containing the configmap PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 1e0c59f1a6fa5..d8701b56d6612 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -331,9 +331,9 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // Name of configmap to use for scheduler policy + // PolicyConfigMap is the name of configmap to use for scheduler policy PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // Namespace of configmap + // PolicyConfigMapNamespace is the namespace containing the configmap PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` } From be8aa9fecc7735e8f8032984fafaab37967bcf73 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Sat, 7 Oct 2017 00:02:49 +0700 Subject: [PATCH 4/6] Added docs on policyConfigMap --- docs/cluster_spec.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index d75bd2fe242d6..f64e4c7613f8a 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -194,6 +194,21 @@ NOTE: Where the corresponding configuration value can be empty, fields can be se Will result in the flag `--resolv-conf=` being built. +### kubeScheduler + +This block contains configurations for `kube-scheduler`. See https://kubernetes.io/docs/admin/kube-scheduler/ + + ```yaml + spec: + kubeScheduler: + policyConfigMap: scheduler-policy + policyConfigMapNamespace: default +``` + +Will resulting to running kube-scheduler with the arguments `--policy-configmap=scheduler-policy --policy-configmap-namespace=default`. + +Note that as of Kubernetes 1.8.0 kube-scheduler does not reload its configuration from configmap automatically. You will need to ssh into the master instance and restart the Docker container manually. + #### Feature Gates ```yaml From c00f5ea9a7ea789be14e3862e0d6224981850091 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Sat, 7 Oct 2017 00:28:16 +0700 Subject: [PATCH 5/6] Added error when trying to use kube-scheduler policyConfigMap before Kube 1.7.x --- pkg/model/components/kubescheduler.go | 6 + pkg/model/components/kubescheduler_test.go | 135 +++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 pkg/model/components/kubescheduler_test.go diff --git a/pkg/model/components/kubescheduler.go b/pkg/model/components/kubescheduler.go index 09b22bda1270f..85de3667cbe16 100644 --- a/pkg/model/components/kubescheduler.go +++ b/pkg/model/components/kubescheduler.go @@ -17,6 +17,8 @@ limitations under the License. package components import ( + "fmt" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/loader" @@ -37,6 +39,10 @@ func (b *KubeSchedulerOptionsBuilder) BuildOptions(o interface{}) error { config := clusterSpec.KubeScheduler + if config.PolicyConfigMap != "" && b.IsKubernetesLT("v1.7.0") { + return fmt.Errorf("policyConfigMap is only supported in Kubernetes 1.7.0 or later") + } + if config.LogLevel == 0 { // TODO: No way to set to 0? config.LogLevel = 2 diff --git a/pkg/model/components/kubescheduler_test.go b/pkg/model/components/kubescheduler_test.go new file mode 100644 index 0000000000000..be950af64cab5 --- /dev/null +++ b/pkg/model/components/kubescheduler_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "testing" + + api "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/util" + "k8s.io/kops/pkg/assets" +) + +func buildSchedulerConfigMapCluster() *api.Cluster { + return &api.Cluster{ + Spec: api.ClusterSpec{ + CloudProvider: "aws", + KubernetesVersion: "v1.4.0", + KubeScheduler: &api.KubeSchedulerConfig{ + PolicyConfigMap: "scheduler-config", + }, + }, + } +} + +func Test_Build_Scheduler_Without_PolicyConfigMap(t *testing.T) { + versions := []string{"v1.6.0", "v1.6.4", "v1.7.0", "v1.7.4"} + b := assets.NewAssetBuilder(nil) + + for _, v := range versions { + + c := buildCluster() + + version, err := util.ParseKubernetesVersion(v) + + if err != nil { + t.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", v, err) + } + + ks := &KubeSchedulerOptionsBuilder{ + &OptionsContext{ + AssetBuilder: b, + KubernetesVersion: *version, + }, + } + + spec := c.Spec + + spec.KubernetesVersion = v + err = ks.BuildOptions(&spec) + + if err != nil { + t.Fatalf("unexpected error from BuildOptions: %v", err) + } + } + +} +func Test_Build_Scheduler_PolicyConfigMap_Unsupported_Version(t *testing.T) { + versions := []string{"v1.6.0", "v1.6.4"} + b := assets.NewAssetBuilder(nil) + + for _, v := range versions { + + c := buildSchedulerConfigMapCluster() + + version, err := util.ParseKubernetesVersion(v) + + if err != nil { + t.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", v, err) + } + + ks := &KubeSchedulerOptionsBuilder{ + &OptionsContext{ + AssetBuilder: b, + KubernetesVersion: *version, + }, + } + + spec := c.Spec + + spec.KubernetesVersion = v + err = ks.BuildOptions(&spec) + + if err == nil { + t.Fatalf("error is expected, but none are returned") + } + } + +} + +func Test_Build_Scheduler_PolicyConfigMap_Supported_Version(t *testing.T) { + versions := []string{"v1.7.0", "v1.7.4", "v1.8.0"} + b := assets.NewAssetBuilder(nil) + + for _, v := range versions { + + c := buildSchedulerConfigMapCluster() + + version, err := util.ParseKubernetesVersion(v) + + if err != nil { + t.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", v, err) + } + + ks := &KubeSchedulerOptionsBuilder{ + &OptionsContext{ + AssetBuilder: b, + KubernetesVersion: *version, + }, + } + + spec := c.Spec + + spec.KubernetesVersion = v + err = ks.BuildOptions(&spec) + + if err != nil { + t.Fatalf("unexpected error from BuildOptions %s: %v", v, err) + } + } + +} From a06fbbac79089a527c32ec99bd7a2ba986aa8e17 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Tue, 10 Oct 2017 09:33:48 +0700 Subject: [PATCH 6/6] Added documentation that PolicyConfigMap should not be used during cluster creation --- docs/cluster_spec.md | 2 +- pkg/model/components/BUILD.bazel | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index f64e4c7613f8a..e4be51ae95735 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -207,7 +207,7 @@ This block contains configurations for `kube-scheduler`. See https://kubernetes Will resulting to running kube-scheduler with the arguments `--policy-configmap=scheduler-policy --policy-configmap-namespace=default`. -Note that as of Kubernetes 1.8.0 kube-scheduler does not reload its configuration from configmap automatically. You will need to ssh into the master instance and restart the Docker container manually. +Note that as of Kubernetes 1.8.0 kube-scheduler does not reload its configuration from configmap automatically. You will need to ssh into the master instance and restart the Docker container manually. Also, this option is not supported during cluster creation, only during updates. #### Feature Gates diff --git a/pkg/model/components/BUILD.bazel b/pkg/model/components/BUILD.bazel index 26386b1f1eec8..924d125399ac1 100644 --- a/pkg/model/components/BUILD.bazel +++ b/pkg/model/components/BUILD.bazel @@ -36,6 +36,7 @@ go_test( srcs = [ "kubecontrollermanager_test.go", "kubelet_test.go", + "kubescheduler_test.go", ], library = ":go_default_library", deps = [