From cbc32605647d63128736d74fb88ea08e892e29ec Mon Sep 17 00:00:00 2001 From: Cesar Wong Date: Thu, 12 Mar 2020 14:30:25 -0400 Subject: [PATCH 1/3] Exclude Kubernetes control plane rules when running on IBM Cloud --- pkg/client/client.go | 4 +++ pkg/manifests/config.go | 12 ++++++- pkg/manifests/manifests.go | 20 +++++++++++ pkg/manifests/manifests_test.go | 59 +++++++++++++++++++++++++++++++++ pkg/operator/operator.go | 7 ++++ 5 files changed, 101 insertions(+), 1 deletion(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index af4c13dfd0..7cd1ef0744 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -237,6 +237,10 @@ func (c *Client) GetProxy(name string) (*configv1.Proxy, error) { return c.oscclient.ConfigV1().Proxies().Get(name, metav1.GetOptions{}) } +func (c *Client) GetInfrastructure(name string) (*configv1.Infrastructure, error) { + return c.oscclient.ConfigV1().Infrastructures().Get(name, metav1.GetOptions{}) +} + func (c *Client) GetConfigmap(namespace, name string) (*v1.ConfigMap, error) { return c.kclient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) } diff --git a/pkg/manifests/config.go b/pkg/manifests/config.go index c5ad91932d..27aba1a877 100644 --- a/pkg/manifests/config.go +++ b/pkg/manifests/config.go @@ -27,7 +27,8 @@ import ( ) type Config struct { - Images *Images `json:"-"` + Images *Images `json:"-"` + Platform configv1.PlatformType `json:"-"` PrometheusOperatorConfig *PrometheusOperatorConfig `json:"prometheusOperator"` PrometheusOperatorUserWorkloadConfig *PrometheusOperatorConfig `json:"prometheusOperatorUserWorkload"` @@ -308,6 +309,15 @@ func (c *Config) LoadProxy(load func() (*configv1.Proxy, error)) error { return nil } +func (c *Config) LoadPlatform(load func() (*configv1.Infrastructure, error)) error { + i, err := load() + if err != nil { + return fmt.Errorf("error loading platform: %v", err) + } + c.Platform = i.Status.Platform + return nil +} + func NewConfigFromString(content string) (*Config, error) { if content == "" { return NewDefaultConfig(), nil diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index d5b73469e5..9e1d98fd84 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -30,6 +30,7 @@ import ( "strings" monv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" + configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" securityv1 "github.com/openshift/api/security/v1" "github.com/pkg/errors" @@ -196,6 +197,10 @@ var ( AuthProxyRedirectURLFlag = "-redirect-url=" ) +const ( + IBMCloudPlatformType configv1.PlatformType = "IBMCloud" +) + func MustAssetReader(asset string) io.Reader { return bytes.NewReader(MustAsset(asset)) } @@ -776,6 +781,21 @@ func (f *Factory) PrometheusK8sRules() (*monv1.PrometheusRule, error) { r.Spec.Groups = groups } + if f.config.Platform == IBMCloudPlatformType { + groups := []monv1.RuleGroup{} + for _, g := range r.Spec.Groups { + switch g.Name { + case "kubernetes-system-apiserver", + "kubernetes-system-controller-manager", + "kubernetes-system-scheduler": + // skip + default: + groups = append(groups, g) + } + } + r.Spec.Groups = groups + } + return r, nil } diff --git a/pkg/manifests/manifests_test.go b/pkg/manifests/manifests_test.go index dcc093aed3..896614a004 100644 --- a/pkg/manifests/manifests_test.go +++ b/pkg/manifests/manifests_test.go @@ -19,6 +19,8 @@ import ( "strings" "testing" + configv1 "github.com/openshift/api/config/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -996,6 +998,63 @@ func TestOpenShiftStateMetrics(t *testing.T) { } } +func TestPrometheusK8sControlPlaneRulesFiltered(t *testing.T) { + tests := []struct { + name string + config *Config + verify func(bool, bool, bool) + }{ + { + name: "default config", + config: func() *Config { + c := NewDefaultConfig() + c.Platform = configv1.AWSPlatformType + return c + }(), + verify: func(api, cm, sched bool) { + if !api || !cm || !sched { + t.Fatal("did not get all expected kubernetes control plane rules") + } + }, + }, + { + name: "hosted control plane", + config: func() *Config { + c := NewDefaultConfig() + c.Platform = IBMCloudPlatformType + return c + }(), + verify: func(api, cm, sched bool) { + if api || cm || sched { + t.Fatalf("kubernetes control plane rules found, none expected") + } + }, + }, + } + + for _, tc := range tests { + f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", tc.config) + r, err := f.PrometheusK8sRules() + if err != nil { + t.Fatal(err) + } + apiServerRulesFound := false + controllerManagerRulesFound := false + schedulerRulesFound := false + for _, g := range r.Spec.Groups { + switch g.Name { + case "kubernetes-system-apiserver": + apiServerRulesFound = true + case "kubernetes-system-controller-manager": + controllerManagerRulesFound = true + case "kubernetes-system-scheduler": + schedulerRulesFound = true + } + } + tc.verify(apiServerRulesFound, controllerManagerRulesFound, schedulerRulesFound) + } +} + func TestPrometheusEtcdRulesFiltered(t *testing.T) { enabled := false c := NewDefaultConfig() diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 64e28bb5c1..cae6efd97c 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -403,6 +403,13 @@ func (o *Operator) Config(key string) *manifests.Config { klog.Warningf("Could not load proxy configuration from API. This is expected and message can be ignored when proxy configuration doesn't exist. Proceeding without it: %v", err) } + err = c.LoadPlatform(func() (*configv1.Infrastructure, error) { + return o.client.GetInfrastructure("cluster") + }) + if err != nil { + klog.Warningf("Could not load platform from infrastructure resource: %v. This may result in alerts that are not appropriate for the platform.", err) + } + cm, err := o.client.GetConfigmap("openshift-config", "etcd-metric-serving-ca") if err != nil { klog.Warningf("Error loading etcd CA certificates for Prometheus. Proceeding with etcd disabled. Error: %v", err) From ef8b1b33c9f73935e68ede298ca2fec1f8191023 Mon Sep 17 00:00:00 2001 From: Cesar Wong Date: Thu, 19 Mar 2020 13:06:02 -0400 Subject: [PATCH 2/3] Update generated files --- jsonnet/jsonnetfile.json | 32 +++++----- jsonnet/jsonnetfile.lock.json | 116 ++++++++++++++++------------------ 2 files changed, 69 insertions(+), 79 deletions(-) diff --git a/jsonnet/jsonnetfile.json b/jsonnet/jsonnetfile.json index b5b9909fcf..86b632f3e5 100644 --- a/jsonnet/jsonnetfile.json +++ b/jsonnet/jsonnetfile.json @@ -1,7 +1,7 @@ { + "version": 1, "dependencies": [ { - "name": "kube-prometheus", "source": { "git": { "remote": "https://github.com/coreos/kube-prometheus", @@ -11,17 +11,15 @@ "version": "release-0.3" }, { - "name": "kube-thanos", "source": { "git": { - "remote": "https://github.com/thanos-io/kube-thanos", - "subdir": "jsonnet/kube-thanos" + "remote": "https://github.com/coreos/prometheus-operator", + "subdir": "jsonnet/prometheus-operator" } }, - "version": "release-0.1" + "version": "release-0.34" }, { - "name": "kubernetes-mixin", "source": { "git": { "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", @@ -31,34 +29,34 @@ "version": "release-0.2" }, { - "name": "openshift-state-metrics", "source": { "git": { "remote": "https://github.com/openshift/openshift-state-metrics", "subdir": "jsonnet" } }, - "version": "release-4.3" + "version": "release-4.3", + "name": "openshift-state-metrics" }, { - "name": "prometheus-operator", "source": { "git": { - "remote": "https://github.com/coreos/prometheus-operator", - "subdir": "jsonnet/prometheus-operator" + "remote": "https://github.com/openshift/telemeter", + "subdir": "jsonnet/telemeter" } }, - "version": "release-0.34" + "version": "release-4.3", + "name": "telemeter-client" }, { - "name": "telemeter-client", "source": { "git": { - "remote": "https://github.com/openshift/telemeter", - "subdir": "jsonnet/telemeter" + "remote": "https://github.com/thanos-io/kube-thanos", + "subdir": "jsonnet/kube-thanos" } }, - "version": "release-4.3" + "version": "release-0.1" } - ] + ], + "legacyImports": true } diff --git a/jsonnet/jsonnetfile.lock.json b/jsonnet/jsonnetfile.lock.json index 6d98f3f260..e8f1d1a4c6 100644 --- a/jsonnet/jsonnetfile.lock.json +++ b/jsonnet/jsonnetfile.lock.json @@ -1,7 +1,17 @@ { + "version": 1, "dependencies": [ { - "name": "etcd-mixin", + "source": { + "git": { + "remote": "https://github.com/brancz/kubernetes-grafana", + "subdir": "grafana" + } + }, + "version": "539a90dbf63c812ad0194d8078dd776868a11c81", + "sum": "b8faWX1qqLGyN67sA36oRqYZ5HX+tHBRMPtrWRqIysE=" + }, + { "source": { "git": { "remote": "https://github.com/coreos/etcd", @@ -12,29 +22,26 @@ "sum": "Ko3qhNfC2vN/houLh6C0Ryacjv70gl0DVPGU/PQ4OD0=" }, { - "name": "grafana", "source": { "git": { - "remote": "https://github.com/brancz/kubernetes-grafana", - "subdir": "grafana" + "remote": "https://github.com/coreos/kube-prometheus", + "subdir": "jsonnet/kube-prometheus" } }, - "version": "539a90dbf63c812ad0194d8078dd776868a11c81", - "sum": "b8faWX1qqLGyN67sA36oRqYZ5HX+tHBRMPtrWRqIysE=" + "version": "ed71719c8e3573db358c3c6e76c3c65ca1fad719", + "sum": "qaE0WINXp2AT83A7Qi+3sy6mhHy3QEK6Ae02a0ejsEU=" }, { - "name": "grafana-builder", "source": { "git": { - "remote": "https://github.com/grafana/jsonnet-libs", - "subdir": "grafana-builder" + "remote": "https://github.com/coreos/prometheus-operator", + "subdir": "jsonnet/prometheus-operator" } }, - "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4", - "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE=" + "version": "1aa773ddbbbd9b05405e9785e0190b975b1faadc", + "sum": "PlruMWn3QMjgLQVzn0DqlD1G8ijkew9icNNW4cpwRVA=" }, { - "name": "grafonnet", "source": { "git": { "remote": "https://github.com/grafana/grafonnet-lib", @@ -45,40 +52,27 @@ "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y=" }, { - "name": "ksonnet", "source": { "git": { - "remote": "https://github.com/ksonnet/ksonnet-lib", - "subdir": "" - } - }, - "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f", - "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=" - }, - { - "name": "kube-prometheus", - "source": { - "git": { - "remote": "https://github.com/coreos/kube-prometheus", - "subdir": "jsonnet/kube-prometheus" + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" } }, - "version": "ed71719c8e3573db358c3c6e76c3c65ca1fad719", - "sum": "qaE0WINXp2AT83A7Qi+3sy6mhHy3QEK6Ae02a0ejsEU=" + "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4", + "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE=" }, { - "name": "kube-thanos", "source": { "git": { - "remote": "https://github.com/thanos-io/kube-thanos", - "subdir": "jsonnet/kube-thanos" + "remote": "https://github.com/ksonnet/ksonnet-lib", + "subdir": "" } }, - "version": "05d490681404e9dae6e42284b6a2776493a9355c", - "sum": "rTPcq23U6UNXoN2Fpxt5JzIYTnylMuG794lr9Pt4xEc=" + "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f", + "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=", + "name": "ksonnet" }, { - "name": "kubernetes-mixin", "source": { "git": { "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", @@ -89,18 +83,16 @@ "sum": "aD1RSVL8mCfpiZc+koGJFz4/u/IPUV7hJlIXKCCDyiI=" }, { - "name": "node-mixin", "source": { "git": { - "remote": "https://github.com/prometheus/node_exporter", - "subdir": "docs/node-mixin" + "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", + "subdir": "lib/promgrafonnet" } }, - "version": "eac3e30f7f7f564c2bd110c7bb97390711e45e32", - "sum": "7vEamDTP9AApeiF4Zu9ZyXzDIs3rYHzwf9k7g8X+wsg=" + "version": "92309e9c7a7637c38a12c0964e62a7aeccaf49ae", + "sum": "VhgBM39yv0f4bKv8VfGg4FXkg573evGDRalip9ypKbc=" }, { - "name": "openshift-state-metrics", "source": { "git": { "remote": "https://github.com/openshift/openshift-state-metrics", @@ -108,51 +100,51 @@ } }, "version": "c01d2de651071389d2621c46e934fd9cb2bf4b8d", - "sum": "CHoXXlCYfTLaqKSaydhDr8+wn5KEceZD0aJuvh2BgQQ=" + "sum": "CHoXXlCYfTLaqKSaydhDr8+wn5KEceZD0aJuvh2BgQQ=", + "name": "openshift-state-metrics" }, { - "name": "prometheus", "source": { "git": { - "remote": "https://github.com/prometheus/prometheus", - "subdir": "documentation/prometheus-mixin" + "remote": "https://github.com/openshift/telemeter", + "subdir": "jsonnet/telemeter" } }, - "version": "a3369086782879d7ed8393168cb580b8e4682bc1", - "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw=" + "version": "227ae59d226d17eee26a60621bc884f4e8e83254", + "sum": "9avi9p5cB8ESzgjhw13WkCDxrablWodHUUTAasHJ8WA=", + "name": "telemeter-client" }, { - "name": "prometheus-operator", "source": { "git": { - "remote": "https://github.com/coreos/prometheus-operator", - "subdir": "jsonnet/prometheus-operator" + "remote": "https://github.com/prometheus/node_exporter", + "subdir": "docs/node-mixin" } }, - "version": "1aa773ddbbbd9b05405e9785e0190b975b1faadc", - "sum": "PlruMWn3QMjgLQVzn0DqlD1G8ijkew9icNNW4cpwRVA=" + "version": "eac3e30f7f7f564c2bd110c7bb97390711e45e32", + "sum": "7vEamDTP9AApeiF4Zu9ZyXzDIs3rYHzwf9k7g8X+wsg=" }, { - "name": "promgrafonnet", "source": { "git": { - "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", - "subdir": "lib/promgrafonnet" + "remote": "https://github.com/prometheus/prometheus", + "subdir": "documentation/prometheus-mixin" } }, - "version": "92309e9c7a7637c38a12c0964e62a7aeccaf49ae", - "sum": "VhgBM39yv0f4bKv8VfGg4FXkg573evGDRalip9ypKbc=" + "version": "a3369086782879d7ed8393168cb580b8e4682bc1", + "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw=", + "name": "prometheus" }, { - "name": "telemeter-client", "source": { "git": { - "remote": "https://github.com/openshift/telemeter", - "subdir": "jsonnet/telemeter" + "remote": "https://github.com/thanos-io/kube-thanos", + "subdir": "jsonnet/kube-thanos" } }, - "version": "227ae59d226d17eee26a60621bc884f4e8e83254", - "sum": "9avi9p5cB8ESzgjhw13WkCDxrablWodHUUTAasHJ8WA=" + "version": "05d490681404e9dae6e42284b6a2776493a9355c", + "sum": "rTPcq23U6UNXoN2Fpxt5JzIYTnylMuG794lr9Pt4xEc=" } - ] + ], + "legacyImports": false } From b4f2d960db57d3a8c29228a94a6e5dca52c14c23 Mon Sep 17 00:00:00 2001 From: Cesar Wong Date: Fri, 20 Mar 2020 05:43:18 -0400 Subject: [PATCH 3/3] Add infrastructures.config.openshift.io to cluster-monitoring-operator role --- hack/cluster-monitoring-operator-role.yaml.in | 3 +++ manifests/0000_50_cluster_monitoring_operator_02-role.yaml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/hack/cluster-monitoring-operator-role.yaml.in b/hack/cluster-monitoring-operator-role.yaml.in index fd3bdb1cd6..087ac56a61 100644 --- a/hack/cluster-monitoring-operator-role.yaml.in +++ b/hack/cluster-monitoring-operator-role.yaml.in @@ -24,6 +24,9 @@ rules: - apiGroups: ["config.openshift.io"] resources: ["clusterversions"] verbs: ["get"] +- apiGroups: ["config.openshift.io"] + resources: ["infrastructures"] + verbs: ["get"] - apiGroups: ["config.openshift.io"] resources: ["proxies"] verbs: ["get"] diff --git a/manifests/0000_50_cluster_monitoring_operator_02-role.yaml b/manifests/0000_50_cluster_monitoring_operator_02-role.yaml index fb38875868..358a3d293e 100644 --- a/manifests/0000_50_cluster_monitoring_operator_02-role.yaml +++ b/manifests/0000_50_cluster_monitoring_operator_02-role.yaml @@ -106,6 +106,12 @@ rules: - clusterversions verbs: - get +- apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get - apiGroups: - config.openshift.io resources: